diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a09db2ea4d1bd1fc1c09f6fca45e8e4054953535 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..e14ce68b0a559601dc936eba5e7c4d1c3d8ffc84 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Zhipu AI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index d42172315a9dc4a74d0de30332897f217e3cd7f3..f51a4ab1e599a51df35be17cce360773892ac31f 100644 --- a/README.md +++ b/README.md @@ -1,79 +1,92 @@ --- -tags: -- auto-round -license: other -license_name: mit +language: +- en +- zh library_name: transformers -base_model: -- zai-org/GLM-5.2 +license: mit +pipeline_tag: text-generation --- -## Model Details -This model is a MXFP4 and MXFP8 mixed model of [zai-org/GLM-5.2](https://huggingface.co/zai-org/GLM-5.2) generated by [intel/auto-round](https://github.com/intel/auto-round) with RTN mode. Please follow the license of the original model. - - - -## vllm Infernece Example - -~~~bash -vllm serve INCModel2/GLM-5.2-MXFP4-Mixed-LLMC \ - --kv-cache-dtype fp8_e4m3 \ - --tensor-parallel-size 4 \ - --tool-call-parser glm47 \ - --enable-auto-tool-choice \ - --reasoning-parser glm45 \ - --served-model-name glm \ - --max-model-len 4096 -~~~ - -~~~bash -curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d ' { - "model": "glm", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Write code to fine-tune an LLM."} - ], - "temperature": 1, - "max_tokens": 2048 - } ' - -~~~ - - -## Generate the Model - -RTN version - -~~~bash -auto-round zai-org/GLM-5.2 --model_free --format llm_compressor --scheme MXFP8 --output_dir /workspace/models/zai-org/GLM-5.2-MXFP4-MXFP8 --ignore_layers indexer.weights_proj --layer_config "{mlp.experts:{scheme:MXFP4}}" -~~~ - - -## Ethical Considerations and Limitations - -The model can produce factually incorrect output, and should not be relied on to produce factually accurate information. Because of the limitations of the pretrained model and the finetuning datasets, it is possible that this model could generate lewd, biased or otherwise offensive outputs. - -Therefore, before deploying any applications of the model, developers should perform safety testing. - -## Caveats and Recommendations - -Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. - -Here are a couple of useful links to learn more about Intel's AI software: - -- [Intel Neural Compressor](https://github.com/intel/neural-compressor) - -## Disclaimer - -The license on this model does not constitute legal advice. We are not responsible for the actions of third parties who use this model. Please consult an attorney before using this model for commercial purposes. - -## Cite - -@article{cheng2023optimize, title={Optimize weight rounding via signed gradient descent for the quantization of llms}, author={Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao and Liu, Yi}, journal={arXiv preprint arXiv:2309.05516}, year={2023} } - -[arxiv](https://arxiv.org/abs/2309.05516) [github](https://github.com/intel/auto-round) \ No newline at end of file +# GLM-5.2 + +
+ +
+

+ 👋 Join our WeChat or Discord community. +
+ 📖 Check out the GLM-5.2 blog and GLM-5 Technical report. +
+ 📍 Use GLM-5.2 API services on Z.ai API Platform. +
+ 🔜 Try GLM-5.2 here. +

+ +

+ [Paper] + [GitHub] +

+ +## Introduction + +We're introducing GLM-5.2, our latest flagship model for long-horizon tasks. It marks a substantial leap in long-horizon task capability over its predecessor GLM-5.1 and, for the first time, delivers that capability on a **solid 1M-token context**. GLM-5.2's new capabilities include: +- **Solid 1M Context:** A solid 1M-token context that stably sustains long-horizon work +- **Advanced Coding with Flexible Effort**: Stronger coding capabilities with multiple thinking effort levels to balance performance and latency +- **Improved Architecture**: We propose [IndexShare](https://arxiv.org/abs/2603.12201), which reuses the same indexer across every four sparse attention layers, reducing per-token FLOPs by 2.9× at a 1M context length. We also improve GLM-5.2’s MTP layer for speculative decoding, increasing the acceptance length by up to 20% +- **Pure Open**: An MIT open-source license — no regional limits, technical access without borders + +![bench_52](https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_52.png) + +## Benchmark + +|Benchmark|GLM-5.2|GLM-5.1|Qwen3.7-Max|MiniMax M3|DeepSeek-V4-Pro|Claude Opus 4.8|GPT-5.5|Gemini 3.1 Pro| +|:---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +|Reasoning||||||||||| +|HLE|40.5|31|41.4|37|37.7|49.8*|41.4*|45| +|HLE (w/ Tools)|54.7|52.3|53.5|-|48.2|57.9*|52.2*|51.4*| +|CritPt|20.9|4.6|13.4|3.7|12.9|20.9|27.1|17.7| +|AIME 2026|99.2|95.3|97|-|94.6|95.7|98.3|98.2| +|HMMT Nov. 2025|94.4|94|95|84.4|94.4|96.5|96.5|94.8| +|HMMT Feb. 2026|92.5|82.6|97.1|84.4|95.2|96.7|96.7|87.3| +|IMOAnswerBench|91.0|83.8|90|-|89.8|83.5|-|81| +|GPQA-Diamond|91.2|86.2|90|93|90.1|93.6|93.6|94.3| +|Coding||||||||||| +|SWE-bench Pro|62.1|58.4|60.6|59|55.4|69.2|58.6|54.2| +|NL2Repo|48.9|42.7|47.2|42.1|35.5|69.7|50.7|33.4| +|DeepSWE|46.2|18|18|20|8|58|70|10| +|ProgramBench|63.7|50.9|-|-|47.8|71.9|70.8|39.5| +|Terminal Bench 2.1 (Terminus-2)|81.0|63.5|75|65|64| 85|84|74| +|Terminal Bench 2.1 (Best Reported Harness)|82.7|69|-|-|-|78.9|83.4|70.7| +|FrontierSWE (Dominance)|74.4|30.5|-|-|29.0|75.1|72.6|39.6| +|PostTrainBench|34.3|20.1|-|-|-|37.2|28.4|21.6| +|SWE-Marathon|13.0|1.0|-|-|-|26.0|12.0|4.0| +|Agentic||||||||||| +|MCP-Atlas (Public Set)|76.8|71.8|76.4|74.2|73.6|77.8|75.3|69.2| +|Tool-Decathlon|48.2|40.7|-|-|52.8|59.9|55.6|48.8| + +## Serve GLM-5.2 Locally + +GLM-5.2 supports deployment with the following frameworks. Feel free to try them out: + +- [SGLang](https://github.com/sgl-project/sglang) (v0.5.13.post1+) — see [cookbook](https://cookbook.sglang.io/autoregressive/GLM/GLM-5.2) +- [vLLM](https://github.com/vllm-project/vllm) (v0.23.0+) — see [recipes](https://recipes.vllm.ai/zai-org/GLM-5.2) +- [Transformers](https://github.com/huggingface/transformers) (v0.5.12+) — see [transformers docs](https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/glm_moe_dsa.md) +- [KTransformers](https://github.com/kvcache-ai/ktransformers) (v0.5.12+) — see [tutorial](https://github.com/kvcache-ai/ktransformers/blob/main/doc/en/kt-kernel/GLM-5.2-Tutorial.md) +- [Unsloth](https://github.com/unslothai/unsloth) (v0.1.47-beta+) — see [guide](https://unsloth.ai/docs/models/glm-5.2) +- For deployment on the `Ascend NPU` platform, inference frameworks such as vLLM-Ascend, xLLM and SGLang are supported — see [here](github.com/zai-org/GLM-5/blob/main/example/ascend.md). + +## Citation + +If you find GLM-5.2 useful in your research, please cite our technical report: + +```bibtex +@misc{glm5team2026glm5vibecodingagentic, + title={GLM-5: from Vibe Coding to Agentic Engineering}, + author={GLM-5-Team and : and Aohan Zeng and Xin Lv and Zhenyu Hou and Zhengxiao Du and Qinkai Zheng and Bin Chen and Da Yin and Chendi Ge and Chenghua Huang and Chengxing Xie and Chenzheng Zhu and Congfeng Yin and Cunxiang Wang and Gengzheng Pan and Hao Zeng and Haoke Zhang and Haoran Wang and Huilong Chen and Jiajie Zhang and Jian Jiao and Jiaqi Guo and Jingsen Wang and Jingzhao Du and Jinzhu Wu and Kedong Wang and Lei Li and Lin Fan and Lucen Zhong and Mingdao Liu and Mingming Zhao and Pengfan Du and Qian Dong and Rui Lu and Shuang-Li and Shulin Cao and Song Liu and Ting Jiang and Xiaodong Chen and Xiaohan Zhang and Xuancheng Huang and Xuezhen Dong and Yabo Xu and Yao Wei and Yifan An and Yilin Niu and Yitong Zhu and Yuanhao Wen and Yukuo Cen and Yushi Bai and Zhongpei Qiao and Zihan Wang and Zikang Wang and Zilin Zhu and Ziqiang Liu and Zixuan Li and Bojie Wang and Bosi Wen and Can Huang and Changpeng Cai and Chao Yu and Chen Li and Chengwei Hu and Chenhui Zhang and Dan Zhang and Daoyan Lin and Dayong Yang and Di Wang and Ding Ai and Erle Zhu and Fangzhou Yi and Feiyu Chen and Guohong Wen and Hailong Sun and Haisha Zhao and Haiyi Hu and Hanchen Zhang and Hanrui Liu and Hanyu Zhang and Hao Peng and Hao Tai and Haobo Zhang and He Liu and Hongwei Wang and Hongxi Yan and Hongyu Ge and Huan Liu and Huanpeng Chu and Jia'ni Zhao and Jiachen Wang and Jiajing Zhao and Jiamin Ren and Jiapeng Wang and Jiaxin Zhang and Jiayi Gui and Jiayue Zhao and Jijie Li and Jing An and Jing Li and Jingwei Yuan and Jinhua Du and Jinxin Liu and Junkai Zhi and Junwen Duan and Kaiyue Zhou and Kangjian Wei and Ke Wang and Keyun Luo and Laiqiang Zhang and Leigang Sha and Liang Xu and Lindong Wu and Lintao Ding and Lu Chen and Minghao Li and Nianyi Lin and Pan Ta and Qiang Zou and Rongjun Song and Ruiqi Yang and Shangqing Tu and Shangtong Yang and Shaoxiang Wu and Shengyan Zhang and Shijie Li and Shuang Li and Shuyi Fan and Wei Qin and Wei Tian and Weining Zhang and Wenbo Yu and Wenjie Liang and Xiang Kuang and Xiangmeng Cheng and Xiangyang Li and Xiaoquan Yan and Xiaowei Hu and Xiaoying Ling and Xing Fan and Xingye Xia and Xinyuan Zhang and Xinze Zhang and Xirui Pan and Xu Zou and Xunkai Zhang and Yadi Liu and Yandong Wu and Yanfu Li and Yidong Wang and Yifan Zhu and Yijun Tan and Yilin Zhou and Yiming Pan and Ying Zhang and Yinpei Su and Yipeng Geng and Yong Yan and Yonglin Tan and Yuean Bi and Yuhan Shen and Yuhao Yang and Yujiang Li and Yunan Liu and Yunqing Wang and Yuntao Li and Yurong Wu and Yutao Zhang and Yuxi Duan and Yuxuan Zhang and Zezhen Liu and Zhengtao Jiang and Zhenhe Yan and Zheyu Zhang and Zhixiang Wei and Zhuo Chen and Zhuoer Feng and Zijun Yao and Ziwei Chai and Ziyuan Wang and Zuzhou Zhang and Bin Xu and Minlie Huang and Hongning Wang and Juanzi Li and Yuxiao Dong and Jie Tang}, + year={2026}, + eprint={2602.15763}, + archivePrefix={arXiv}, + primaryClass={cs.LG}, + url={https://arxiv.org/abs/2602.15763}, +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..672034e3a1196d8b9fc0116ddc037cb554227f89 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,119 @@ +[gMASK] +{%- set effective_reasoning_effort = 'high' if reasoning_effort is defined and reasoning_effort == 'high' else 'max' -%} +{%- if (enable_thinking is not defined or enable_thinking) and effective_reasoning_effort is not none -%}<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}{%- endif -%} +{%- if tools -%} +{%- macro tool_to_json(tool) -%} + {%- set ns_tool = namespace(first=true) -%} + {{ '{' -}} + {%- for k, v in tool.items() -%} + {%- if k != 'defer_loading' and k != 'strict' -%} + {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%} + {%- set ns_tool.first = false -%} + "{{ k }}": {{ v | tojson(ensure_ascii=False) }} + {%- endif -%} + {%- endfor -%} + {{- '}' -}} +{%- endmacro -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} +{%- endif -%} +{% if tool.defer_loading is not defined or not tool.defer_loading %} +{{ tool_to_json(tool) }} +{% endif %} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- elif item is mapping and item.type in ['image', 'image_url', 'video', 'video_url', 'audio', 'audio_url', 'input_audio'] -%} + {%- set media_type = item.type | replace('_url', '') | replace('input_', '') -%} + {{- "You are unable to process this " ~ media_type ~ " because you don't have multi-modal input ability. Try different methods." }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1) -%} +{%- for m in messages %} + {%- if m.role == 'user' %} + {%- set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{%- for m in messages -%} +{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- elif '' in content %} + {%- set reasoning_content = content.split('')[0].split('')[-1] %} + {%- set content = content.split('')[-1] %} +{%- endif %} +{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%} +{{ '' + reasoning_content + ''}} +{%- else -%} +{{ '' }} +{%- endif -%} +{%- if content.strip() -%} +{{ content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{- '' + tc.name -}} +{% set _args = tc.arguments %}{% for k, v in _args.items() %}{{ k }}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{% endfor %}{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' -}} +{%- endif %} +{%- if m.content is string -%} + {{- '' + m.content + '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%} + {{- '\n' -}} + {% for tr in m.content %} + {%- for tool in tools -%} + {%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} + {%- endif -%} + {%- if tool.name == tr.name -%} + {{- tool_to_json(tool) + '\n' -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0 is mapping and m.content.0.output is defined -%} + {%- for tr in m.content -%} + {{- '' + tr.output + '' -}} + {%- endfor -%} +{%- else -%} + {{- '' + visible_text(m.content) + '' -}} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|>{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d06cd81b081681efb78e834503ea8f89f9660f00 --- /dev/null +++ b/config.json @@ -0,0 +1,58782 @@ +{ + "architectures": [ + "GlmMoeDsaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "ep_size": 1, + "first_k_dense_replace": 3, + "head_dim": 192, + "hidden_act": "silu", + "hidden_size": 6144, + "index_head_dim": 128, + "index_n_heads": 32, + "index_share_for_mtp_iteration": true, + "index_skip_topk_offset": 3, + "index_topk": 2048, + "index_topk_freq": 4, + "index_topk_pattern": null, + "indexer_rope_interleave": true, + "indexer_types": [ + "full", + "full", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared" + ], + "initializer_range": 0.02, + "intermediate_size": 12288, + "kv_lora_rank": 512, + "max_position_embeddings": 1048576, + "mlp_layer_types": [ + "dense", + "dense", + "dense", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse" + ], + "model_type": "glm_moe_dsa", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 1, + "n_routed_experts": 256, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts_per_tok": 8, + "num_hidden_layers": 78, + "num_key_value_heads": 64, + "num_nextn_predict_layers": 1, + "pad_token_id": 154820, + "pretraining_tp": 1, + "q_lora_rank": 2048, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-05, + "rope_interleave": true, + "rope_parameters": { + "rope_theta": 8000000, + "rope_type": "default" + }, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": false, + "topk_group": 1, + "topk_method": "noaux_tc", + "transformers_version": "5.12.0", + "use_cache": true, + "v_head_dim": 256, + "vocab_size": 154880, + "quantization_config": { + "config_groups": { + "group_0": { + "targets": [ + "RoutedExperts", + "model.layers.10.mlp.experts.0.down_proj", + "model.layers.10.mlp.experts.0.gate_proj", + "model.layers.10.mlp.experts.0.up_proj", + "model.layers.10.mlp.experts.1.down_proj", + "model.layers.10.mlp.experts.1.gate_proj", + "model.layers.10.mlp.experts.1.up_proj", + "model.layers.10.mlp.experts.10.down_proj", + "model.layers.10.mlp.experts.10.gate_proj", + "model.layers.10.mlp.experts.10.up_proj", + "model.layers.10.mlp.experts.100.down_proj", + "model.layers.10.mlp.experts.100.gate_proj", + "model.layers.10.mlp.experts.100.up_proj", + "model.layers.10.mlp.experts.101.down_proj", + "model.layers.10.mlp.experts.101.gate_proj", + "model.layers.10.mlp.experts.101.up_proj", + "model.layers.10.mlp.experts.102.down_proj", + "model.layers.10.mlp.experts.102.gate_proj", + "model.layers.10.mlp.experts.102.up_proj", + "model.layers.10.mlp.experts.103.down_proj", + "model.layers.10.mlp.experts.103.gate_proj", + "model.layers.10.mlp.experts.103.up_proj", + "model.layers.10.mlp.experts.104.down_proj", + "model.layers.10.mlp.experts.104.gate_proj", + "model.layers.10.mlp.experts.104.up_proj", + "model.layers.10.mlp.experts.105.down_proj", + "model.layers.10.mlp.experts.105.gate_proj", + "model.layers.10.mlp.experts.105.up_proj", + "model.layers.10.mlp.experts.106.down_proj", + "model.layers.10.mlp.experts.106.gate_proj", + "model.layers.10.mlp.experts.106.up_proj", + "model.layers.10.mlp.experts.107.down_proj", + "model.layers.10.mlp.experts.107.gate_proj", + "model.layers.10.mlp.experts.107.up_proj", + "model.layers.10.mlp.experts.108.down_proj", + "model.layers.10.mlp.experts.108.gate_proj", + "model.layers.10.mlp.experts.108.up_proj", + "model.layers.10.mlp.experts.109.down_proj", + "model.layers.10.mlp.experts.109.gate_proj", + "model.layers.10.mlp.experts.109.up_proj", + "model.layers.10.mlp.experts.11.down_proj", + "model.layers.10.mlp.experts.11.gate_proj", + "model.layers.10.mlp.experts.11.up_proj", + "model.layers.10.mlp.experts.110.down_proj", + "model.layers.10.mlp.experts.110.gate_proj", + "model.layers.10.mlp.experts.110.up_proj", + "model.layers.10.mlp.experts.111.down_proj", + "model.layers.10.mlp.experts.111.gate_proj", + "model.layers.10.mlp.experts.111.up_proj", + "model.layers.10.mlp.experts.112.down_proj", + "model.layers.10.mlp.experts.112.gate_proj", + "model.layers.10.mlp.experts.112.up_proj", + "model.layers.10.mlp.experts.113.down_proj", + "model.layers.10.mlp.experts.113.gate_proj", + "model.layers.10.mlp.experts.113.up_proj", + "model.layers.10.mlp.experts.114.down_proj", + "model.layers.10.mlp.experts.114.gate_proj", + "model.layers.10.mlp.experts.114.up_proj", + "model.layers.10.mlp.experts.115.down_proj", + "model.layers.10.mlp.experts.115.gate_proj", + "model.layers.10.mlp.experts.115.up_proj", + "model.layers.10.mlp.experts.116.down_proj", + "model.layers.10.mlp.experts.116.gate_proj", + "model.layers.10.mlp.experts.116.up_proj", + "model.layers.10.mlp.experts.117.down_proj", + "model.layers.10.mlp.experts.117.gate_proj", + "model.layers.10.mlp.experts.117.up_proj", + "model.layers.10.mlp.experts.118.down_proj", + "model.layers.10.mlp.experts.118.gate_proj", + "model.layers.10.mlp.experts.118.up_proj", + "model.layers.10.mlp.experts.119.down_proj", + "model.layers.10.mlp.experts.119.gate_proj", + "model.layers.10.mlp.experts.119.up_proj", + "model.layers.10.mlp.experts.12.down_proj", + "model.layers.10.mlp.experts.12.gate_proj", + "model.layers.10.mlp.experts.12.up_proj", + "model.layers.10.mlp.experts.120.down_proj", + "model.layers.10.mlp.experts.120.gate_proj", + "model.layers.10.mlp.experts.120.up_proj", + "model.layers.10.mlp.experts.121.down_proj", + "model.layers.10.mlp.experts.121.gate_proj", + "model.layers.10.mlp.experts.121.up_proj", + "model.layers.10.mlp.experts.122.down_proj", + "model.layers.10.mlp.experts.122.gate_proj", + "model.layers.10.mlp.experts.122.up_proj", + "model.layers.10.mlp.experts.123.down_proj", + "model.layers.10.mlp.experts.123.gate_proj", + "model.layers.10.mlp.experts.123.up_proj", + "model.layers.10.mlp.experts.124.down_proj", + "model.layers.10.mlp.experts.124.gate_proj", + "model.layers.10.mlp.experts.124.up_proj", + "model.layers.10.mlp.experts.125.down_proj", + "model.layers.10.mlp.experts.125.gate_proj", + "model.layers.10.mlp.experts.125.up_proj", + "model.layers.10.mlp.experts.126.down_proj", + "model.layers.10.mlp.experts.126.gate_proj", + "model.layers.10.mlp.experts.126.up_proj", + "model.layers.10.mlp.experts.127.down_proj", + "model.layers.10.mlp.experts.127.gate_proj", + "model.layers.10.mlp.experts.127.up_proj", + "model.layers.10.mlp.experts.128.down_proj", + "model.layers.10.mlp.experts.128.gate_proj", + "model.layers.10.mlp.experts.128.up_proj", + "model.layers.10.mlp.experts.129.down_proj", + "model.layers.10.mlp.experts.129.gate_proj", + "model.layers.10.mlp.experts.129.up_proj", + "model.layers.10.mlp.experts.13.down_proj", + "model.layers.10.mlp.experts.13.gate_proj", + "model.layers.10.mlp.experts.13.up_proj", + "model.layers.10.mlp.experts.130.down_proj", + "model.layers.10.mlp.experts.130.gate_proj", + "model.layers.10.mlp.experts.130.up_proj", + "model.layers.10.mlp.experts.131.down_proj", + "model.layers.10.mlp.experts.131.gate_proj", + "model.layers.10.mlp.experts.131.up_proj", + "model.layers.10.mlp.experts.132.down_proj", + "model.layers.10.mlp.experts.132.gate_proj", + "model.layers.10.mlp.experts.132.up_proj", + "model.layers.10.mlp.experts.133.down_proj", + "model.layers.10.mlp.experts.133.gate_proj", + "model.layers.10.mlp.experts.133.up_proj", + "model.layers.10.mlp.experts.134.down_proj", + "model.layers.10.mlp.experts.134.gate_proj", + "model.layers.10.mlp.experts.134.up_proj", + "model.layers.10.mlp.experts.135.down_proj", + "model.layers.10.mlp.experts.135.gate_proj", + "model.layers.10.mlp.experts.135.up_proj", + "model.layers.10.mlp.experts.136.down_proj", + "model.layers.10.mlp.experts.136.gate_proj", + "model.layers.10.mlp.experts.136.up_proj", + "model.layers.10.mlp.experts.137.down_proj", + "model.layers.10.mlp.experts.137.gate_proj", + "model.layers.10.mlp.experts.137.up_proj", + "model.layers.10.mlp.experts.138.down_proj", + "model.layers.10.mlp.experts.138.gate_proj", + "model.layers.10.mlp.experts.138.up_proj", + "model.layers.10.mlp.experts.139.down_proj", + "model.layers.10.mlp.experts.139.gate_proj", + "model.layers.10.mlp.experts.139.up_proj", + "model.layers.10.mlp.experts.14.down_proj", + "model.layers.10.mlp.experts.14.gate_proj", + "model.layers.10.mlp.experts.14.up_proj", + "model.layers.10.mlp.experts.140.down_proj", + "model.layers.10.mlp.experts.140.gate_proj", + "model.layers.10.mlp.experts.140.up_proj", + "model.layers.10.mlp.experts.141.down_proj", + "model.layers.10.mlp.experts.141.gate_proj", + "model.layers.10.mlp.experts.141.up_proj", + "model.layers.10.mlp.experts.142.down_proj", + "model.layers.10.mlp.experts.142.gate_proj", + "model.layers.10.mlp.experts.142.up_proj", + "model.layers.10.mlp.experts.143.down_proj", + "model.layers.10.mlp.experts.143.gate_proj", + "model.layers.10.mlp.experts.143.up_proj", + "model.layers.10.mlp.experts.144.down_proj", + "model.layers.10.mlp.experts.144.gate_proj", + "model.layers.10.mlp.experts.144.up_proj", + "model.layers.10.mlp.experts.145.down_proj", + "model.layers.10.mlp.experts.145.gate_proj", + "model.layers.10.mlp.experts.145.up_proj", + "model.layers.10.mlp.experts.146.down_proj", + "model.layers.10.mlp.experts.146.gate_proj", + "model.layers.10.mlp.experts.146.up_proj", + "model.layers.10.mlp.experts.147.down_proj", + "model.layers.10.mlp.experts.147.gate_proj", + "model.layers.10.mlp.experts.147.up_proj", + "model.layers.10.mlp.experts.148.down_proj", + "model.layers.10.mlp.experts.148.gate_proj", + "model.layers.10.mlp.experts.148.up_proj", + "model.layers.10.mlp.experts.149.down_proj", + "model.layers.10.mlp.experts.149.gate_proj", + "model.layers.10.mlp.experts.149.up_proj", + "model.layers.10.mlp.experts.15.down_proj", + "model.layers.10.mlp.experts.15.gate_proj", + "model.layers.10.mlp.experts.15.up_proj", + "model.layers.10.mlp.experts.150.down_proj", + "model.layers.10.mlp.experts.150.gate_proj", + "model.layers.10.mlp.experts.150.up_proj", + "model.layers.10.mlp.experts.151.down_proj", + "model.layers.10.mlp.experts.151.gate_proj", + "model.layers.10.mlp.experts.151.up_proj", + "model.layers.10.mlp.experts.152.down_proj", + "model.layers.10.mlp.experts.152.gate_proj", + "model.layers.10.mlp.experts.152.up_proj", + "model.layers.10.mlp.experts.153.down_proj", + "model.layers.10.mlp.experts.153.gate_proj", + "model.layers.10.mlp.experts.153.up_proj", + "model.layers.10.mlp.experts.154.down_proj", + "model.layers.10.mlp.experts.154.gate_proj", + "model.layers.10.mlp.experts.154.up_proj", + "model.layers.10.mlp.experts.155.down_proj", + "model.layers.10.mlp.experts.155.gate_proj", + "model.layers.10.mlp.experts.155.up_proj", + "model.layers.10.mlp.experts.156.down_proj", + "model.layers.10.mlp.experts.156.gate_proj", + "model.layers.10.mlp.experts.156.up_proj", + "model.layers.10.mlp.experts.157.down_proj", + "model.layers.10.mlp.experts.157.gate_proj", + "model.layers.10.mlp.experts.157.up_proj", + "model.layers.10.mlp.experts.158.down_proj", + "model.layers.10.mlp.experts.158.gate_proj", + "model.layers.10.mlp.experts.158.up_proj", + "model.layers.10.mlp.experts.159.down_proj", + "model.layers.10.mlp.experts.159.gate_proj", + "model.layers.10.mlp.experts.159.up_proj", + "model.layers.10.mlp.experts.16.down_proj", + "model.layers.10.mlp.experts.16.gate_proj", + "model.layers.10.mlp.experts.16.up_proj", + "model.layers.10.mlp.experts.160.down_proj", + "model.layers.10.mlp.experts.160.gate_proj", + "model.layers.10.mlp.experts.160.up_proj", + "model.layers.10.mlp.experts.161.down_proj", + "model.layers.10.mlp.experts.161.gate_proj", + "model.layers.10.mlp.experts.161.up_proj", + "model.layers.10.mlp.experts.162.down_proj", + "model.layers.10.mlp.experts.162.gate_proj", + "model.layers.10.mlp.experts.162.up_proj", + "model.layers.10.mlp.experts.163.down_proj", + "model.layers.10.mlp.experts.163.gate_proj", + "model.layers.10.mlp.experts.163.up_proj", + "model.layers.10.mlp.experts.164.down_proj", + "model.layers.10.mlp.experts.164.gate_proj", + "model.layers.10.mlp.experts.164.up_proj", + "model.layers.10.mlp.experts.165.down_proj", + "model.layers.10.mlp.experts.165.gate_proj", + "model.layers.10.mlp.experts.165.up_proj", + "model.layers.10.mlp.experts.166.down_proj", + "model.layers.10.mlp.experts.166.gate_proj", + "model.layers.10.mlp.experts.166.up_proj", + "model.layers.10.mlp.experts.167.down_proj", + "model.layers.10.mlp.experts.167.gate_proj", + "model.layers.10.mlp.experts.167.up_proj", + "model.layers.10.mlp.experts.168.down_proj", + "model.layers.10.mlp.experts.168.gate_proj", + "model.layers.10.mlp.experts.168.up_proj", + "model.layers.10.mlp.experts.169.down_proj", + "model.layers.10.mlp.experts.169.gate_proj", + "model.layers.10.mlp.experts.169.up_proj", + "model.layers.10.mlp.experts.17.down_proj", + "model.layers.10.mlp.experts.17.gate_proj", + "model.layers.10.mlp.experts.17.up_proj", + "model.layers.10.mlp.experts.170.down_proj", + "model.layers.10.mlp.experts.170.gate_proj", + "model.layers.10.mlp.experts.170.up_proj", + "model.layers.10.mlp.experts.171.down_proj", + "model.layers.10.mlp.experts.171.gate_proj", + "model.layers.10.mlp.experts.171.up_proj", + "model.layers.10.mlp.experts.172.down_proj", + "model.layers.10.mlp.experts.172.gate_proj", + "model.layers.10.mlp.experts.172.up_proj", + "model.layers.10.mlp.experts.173.down_proj", + "model.layers.10.mlp.experts.173.gate_proj", + "model.layers.10.mlp.experts.173.up_proj", + "model.layers.10.mlp.experts.174.down_proj", + "model.layers.10.mlp.experts.174.gate_proj", + "model.layers.10.mlp.experts.174.up_proj", + "model.layers.10.mlp.experts.175.down_proj", + "model.layers.10.mlp.experts.175.gate_proj", + "model.layers.10.mlp.experts.175.up_proj", + "model.layers.10.mlp.experts.176.down_proj", + "model.layers.10.mlp.experts.176.gate_proj", + "model.layers.10.mlp.experts.176.up_proj", + "model.layers.10.mlp.experts.177.down_proj", + "model.layers.10.mlp.experts.177.gate_proj", + "model.layers.10.mlp.experts.177.up_proj", + "model.layers.10.mlp.experts.178.down_proj", + "model.layers.10.mlp.experts.178.gate_proj", + "model.layers.10.mlp.experts.178.up_proj", + "model.layers.10.mlp.experts.179.down_proj", + "model.layers.10.mlp.experts.179.gate_proj", + "model.layers.10.mlp.experts.179.up_proj", + "model.layers.10.mlp.experts.18.down_proj", + "model.layers.10.mlp.experts.18.gate_proj", + "model.layers.10.mlp.experts.18.up_proj", + "model.layers.10.mlp.experts.180.down_proj", + "model.layers.10.mlp.experts.180.gate_proj", + "model.layers.10.mlp.experts.180.up_proj", + "model.layers.10.mlp.experts.181.down_proj", + "model.layers.10.mlp.experts.181.gate_proj", + "model.layers.10.mlp.experts.181.up_proj", + "model.layers.10.mlp.experts.182.down_proj", + "model.layers.10.mlp.experts.182.gate_proj", + "model.layers.10.mlp.experts.182.up_proj", + "model.layers.10.mlp.experts.183.down_proj", + "model.layers.10.mlp.experts.183.gate_proj", + "model.layers.10.mlp.experts.183.up_proj", + "model.layers.10.mlp.experts.184.down_proj", + "model.layers.10.mlp.experts.184.gate_proj", + "model.layers.10.mlp.experts.184.up_proj", + "model.layers.10.mlp.experts.185.down_proj", + "model.layers.10.mlp.experts.185.gate_proj", + "model.layers.10.mlp.experts.185.up_proj", + "model.layers.10.mlp.experts.186.down_proj", + "model.layers.10.mlp.experts.186.gate_proj", + "model.layers.10.mlp.experts.186.up_proj", + "model.layers.10.mlp.experts.187.down_proj", + "model.layers.10.mlp.experts.187.gate_proj", + "model.layers.10.mlp.experts.187.up_proj", + "model.layers.10.mlp.experts.188.down_proj", + "model.layers.10.mlp.experts.188.gate_proj", + "model.layers.10.mlp.experts.188.up_proj", + "model.layers.10.mlp.experts.189.down_proj", + "model.layers.10.mlp.experts.189.gate_proj", + "model.layers.10.mlp.experts.189.up_proj", + "model.layers.10.mlp.experts.19.down_proj", + "model.layers.10.mlp.experts.19.gate_proj", + "model.layers.10.mlp.experts.19.up_proj", + "model.layers.10.mlp.experts.190.down_proj", + "model.layers.10.mlp.experts.190.gate_proj", + "model.layers.10.mlp.experts.190.up_proj", + "model.layers.10.mlp.experts.191.down_proj", + "model.layers.10.mlp.experts.191.gate_proj", + "model.layers.10.mlp.experts.191.up_proj", + "model.layers.10.mlp.experts.192.down_proj", + "model.layers.10.mlp.experts.192.gate_proj", + "model.layers.10.mlp.experts.192.up_proj", + "model.layers.10.mlp.experts.193.down_proj", + "model.layers.10.mlp.experts.193.gate_proj", + "model.layers.10.mlp.experts.193.up_proj", + "model.layers.10.mlp.experts.194.down_proj", + "model.layers.10.mlp.experts.194.gate_proj", + "model.layers.10.mlp.experts.194.up_proj", + "model.layers.10.mlp.experts.195.down_proj", + "model.layers.10.mlp.experts.195.gate_proj", + "model.layers.10.mlp.experts.195.up_proj", + "model.layers.10.mlp.experts.196.down_proj", + "model.layers.10.mlp.experts.196.gate_proj", + "model.layers.10.mlp.experts.196.up_proj", + "model.layers.10.mlp.experts.197.down_proj", + "model.layers.10.mlp.experts.197.gate_proj", + "model.layers.10.mlp.experts.197.up_proj", + "model.layers.10.mlp.experts.198.down_proj", + "model.layers.10.mlp.experts.198.gate_proj", + "model.layers.10.mlp.experts.198.up_proj", + "model.layers.10.mlp.experts.199.down_proj", + "model.layers.10.mlp.experts.199.gate_proj", + "model.layers.10.mlp.experts.199.up_proj", + "model.layers.10.mlp.experts.2.down_proj", + "model.layers.10.mlp.experts.2.gate_proj", + "model.layers.10.mlp.experts.2.up_proj", + "model.layers.10.mlp.experts.20.down_proj", + "model.layers.10.mlp.experts.20.gate_proj", + "model.layers.10.mlp.experts.20.up_proj", + "model.layers.10.mlp.experts.200.down_proj", + "model.layers.10.mlp.experts.200.gate_proj", + "model.layers.10.mlp.experts.200.up_proj", + "model.layers.10.mlp.experts.201.down_proj", + "model.layers.10.mlp.experts.201.gate_proj", + "model.layers.10.mlp.experts.201.up_proj", + "model.layers.10.mlp.experts.202.down_proj", + "model.layers.10.mlp.experts.202.gate_proj", + "model.layers.10.mlp.experts.202.up_proj", + "model.layers.10.mlp.experts.203.down_proj", + "model.layers.10.mlp.experts.203.gate_proj", + "model.layers.10.mlp.experts.203.up_proj", + "model.layers.10.mlp.experts.204.down_proj", + "model.layers.10.mlp.experts.204.gate_proj", + "model.layers.10.mlp.experts.204.up_proj", + "model.layers.10.mlp.experts.205.down_proj", + "model.layers.10.mlp.experts.205.gate_proj", + "model.layers.10.mlp.experts.205.up_proj", + "model.layers.10.mlp.experts.206.down_proj", + "model.layers.10.mlp.experts.206.gate_proj", + "model.layers.10.mlp.experts.206.up_proj", + "model.layers.10.mlp.experts.207.down_proj", + "model.layers.10.mlp.experts.207.gate_proj", + "model.layers.10.mlp.experts.207.up_proj", + "model.layers.10.mlp.experts.208.down_proj", + "model.layers.10.mlp.experts.208.gate_proj", + "model.layers.10.mlp.experts.208.up_proj", + "model.layers.10.mlp.experts.209.down_proj", + "model.layers.10.mlp.experts.209.gate_proj", + "model.layers.10.mlp.experts.209.up_proj", + "model.layers.10.mlp.experts.21.down_proj", + "model.layers.10.mlp.experts.21.gate_proj", + "model.layers.10.mlp.experts.21.up_proj", + "model.layers.10.mlp.experts.210.down_proj", + "model.layers.10.mlp.experts.210.gate_proj", + "model.layers.10.mlp.experts.210.up_proj", + "model.layers.10.mlp.experts.211.down_proj", + "model.layers.10.mlp.experts.211.gate_proj", + "model.layers.10.mlp.experts.211.up_proj", + "model.layers.10.mlp.experts.212.down_proj", + "model.layers.10.mlp.experts.212.gate_proj", + "model.layers.10.mlp.experts.212.up_proj", + "model.layers.10.mlp.experts.213.down_proj", + "model.layers.10.mlp.experts.213.gate_proj", + "model.layers.10.mlp.experts.213.up_proj", + "model.layers.10.mlp.experts.214.down_proj", + "model.layers.10.mlp.experts.214.gate_proj", + "model.layers.10.mlp.experts.214.up_proj", + "model.layers.10.mlp.experts.215.down_proj", + "model.layers.10.mlp.experts.215.gate_proj", + "model.layers.10.mlp.experts.215.up_proj", + "model.layers.10.mlp.experts.216.down_proj", + "model.layers.10.mlp.experts.216.gate_proj", + "model.layers.10.mlp.experts.216.up_proj", + "model.layers.10.mlp.experts.217.down_proj", + "model.layers.10.mlp.experts.217.gate_proj", + "model.layers.10.mlp.experts.217.up_proj", + "model.layers.10.mlp.experts.218.down_proj", + "model.layers.10.mlp.experts.218.gate_proj", + "model.layers.10.mlp.experts.218.up_proj", + "model.layers.10.mlp.experts.219.down_proj", + "model.layers.10.mlp.experts.219.gate_proj", + "model.layers.10.mlp.experts.219.up_proj", + "model.layers.10.mlp.experts.22.down_proj", + "model.layers.10.mlp.experts.22.gate_proj", + "model.layers.10.mlp.experts.22.up_proj", + "model.layers.10.mlp.experts.220.down_proj", + "model.layers.10.mlp.experts.220.gate_proj", + "model.layers.10.mlp.experts.220.up_proj", + "model.layers.10.mlp.experts.221.down_proj", + "model.layers.10.mlp.experts.221.gate_proj", + "model.layers.10.mlp.experts.221.up_proj", + "model.layers.10.mlp.experts.222.down_proj", + "model.layers.10.mlp.experts.222.gate_proj", + "model.layers.10.mlp.experts.222.up_proj", + "model.layers.10.mlp.experts.223.down_proj", + "model.layers.10.mlp.experts.223.gate_proj", + "model.layers.10.mlp.experts.223.up_proj", + "model.layers.10.mlp.experts.224.down_proj", + "model.layers.10.mlp.experts.224.gate_proj", + "model.layers.10.mlp.experts.224.up_proj", + "model.layers.10.mlp.experts.225.down_proj", + "model.layers.10.mlp.experts.225.gate_proj", + "model.layers.10.mlp.experts.225.up_proj", + "model.layers.10.mlp.experts.226.down_proj", + "model.layers.10.mlp.experts.226.gate_proj", + "model.layers.10.mlp.experts.226.up_proj", + "model.layers.10.mlp.experts.227.down_proj", + "model.layers.10.mlp.experts.227.gate_proj", + "model.layers.10.mlp.experts.227.up_proj", + "model.layers.10.mlp.experts.228.down_proj", + "model.layers.10.mlp.experts.228.gate_proj", + "model.layers.10.mlp.experts.228.up_proj", + "model.layers.10.mlp.experts.229.down_proj", + "model.layers.10.mlp.experts.229.gate_proj", + "model.layers.10.mlp.experts.229.up_proj", + "model.layers.10.mlp.experts.23.down_proj", + "model.layers.10.mlp.experts.23.gate_proj", + "model.layers.10.mlp.experts.23.up_proj", + "model.layers.10.mlp.experts.230.down_proj", + "model.layers.10.mlp.experts.230.gate_proj", + "model.layers.10.mlp.experts.230.up_proj", + "model.layers.10.mlp.experts.231.down_proj", + "model.layers.10.mlp.experts.231.gate_proj", + "model.layers.10.mlp.experts.231.up_proj", + "model.layers.10.mlp.experts.232.down_proj", + "model.layers.10.mlp.experts.232.gate_proj", + "model.layers.10.mlp.experts.232.up_proj", + "model.layers.10.mlp.experts.233.down_proj", + "model.layers.10.mlp.experts.233.gate_proj", + "model.layers.10.mlp.experts.233.up_proj", + "model.layers.10.mlp.experts.234.down_proj", + "model.layers.10.mlp.experts.234.gate_proj", + "model.layers.10.mlp.experts.234.up_proj", + "model.layers.10.mlp.experts.235.down_proj", + "model.layers.10.mlp.experts.235.gate_proj", + "model.layers.10.mlp.experts.235.up_proj", + "model.layers.10.mlp.experts.236.down_proj", + "model.layers.10.mlp.experts.236.gate_proj", + "model.layers.10.mlp.experts.236.up_proj", + "model.layers.10.mlp.experts.237.down_proj", + "model.layers.10.mlp.experts.237.gate_proj", + "model.layers.10.mlp.experts.237.up_proj", + "model.layers.10.mlp.experts.238.down_proj", + "model.layers.10.mlp.experts.238.gate_proj", + "model.layers.10.mlp.experts.238.up_proj", + "model.layers.10.mlp.experts.239.down_proj", + "model.layers.10.mlp.experts.239.gate_proj", + "model.layers.10.mlp.experts.239.up_proj", + "model.layers.10.mlp.experts.24.down_proj", + "model.layers.10.mlp.experts.24.gate_proj", + "model.layers.10.mlp.experts.24.up_proj", + "model.layers.10.mlp.experts.240.down_proj", + "model.layers.10.mlp.experts.240.gate_proj", + "model.layers.10.mlp.experts.240.up_proj", + "model.layers.10.mlp.experts.241.down_proj", + "model.layers.10.mlp.experts.241.gate_proj", + "model.layers.10.mlp.experts.241.up_proj", + "model.layers.10.mlp.experts.242.down_proj", + "model.layers.10.mlp.experts.242.gate_proj", + "model.layers.10.mlp.experts.242.up_proj", + "model.layers.10.mlp.experts.243.down_proj", + "model.layers.10.mlp.experts.243.gate_proj", + "model.layers.10.mlp.experts.243.up_proj", + "model.layers.10.mlp.experts.244.down_proj", + "model.layers.10.mlp.experts.244.gate_proj", + "model.layers.10.mlp.experts.244.up_proj", + "model.layers.10.mlp.experts.245.down_proj", + "model.layers.10.mlp.experts.245.gate_proj", + "model.layers.10.mlp.experts.245.up_proj", + "model.layers.10.mlp.experts.246.down_proj", + "model.layers.10.mlp.experts.246.gate_proj", + "model.layers.10.mlp.experts.246.up_proj", + "model.layers.10.mlp.experts.247.down_proj", + "model.layers.10.mlp.experts.247.gate_proj", + "model.layers.10.mlp.experts.247.up_proj", + "model.layers.10.mlp.experts.248.down_proj", + "model.layers.10.mlp.experts.248.gate_proj", + "model.layers.10.mlp.experts.248.up_proj", + "model.layers.10.mlp.experts.249.down_proj", + "model.layers.10.mlp.experts.249.gate_proj", + "model.layers.10.mlp.experts.249.up_proj", + "model.layers.10.mlp.experts.25.down_proj", + "model.layers.10.mlp.experts.25.gate_proj", + "model.layers.10.mlp.experts.25.up_proj", + "model.layers.10.mlp.experts.250.down_proj", + "model.layers.10.mlp.experts.250.gate_proj", + "model.layers.10.mlp.experts.250.up_proj", + "model.layers.10.mlp.experts.251.down_proj", + "model.layers.10.mlp.experts.251.gate_proj", + "model.layers.10.mlp.experts.251.up_proj", + "model.layers.10.mlp.experts.252.down_proj", + "model.layers.10.mlp.experts.252.gate_proj", + "model.layers.10.mlp.experts.252.up_proj", + "model.layers.10.mlp.experts.253.down_proj", + "model.layers.10.mlp.experts.253.gate_proj", + "model.layers.10.mlp.experts.253.up_proj", + "model.layers.10.mlp.experts.254.down_proj", + "model.layers.10.mlp.experts.254.gate_proj", + "model.layers.10.mlp.experts.254.up_proj", + "model.layers.10.mlp.experts.255.down_proj", + "model.layers.10.mlp.experts.255.gate_proj", + "model.layers.10.mlp.experts.255.up_proj", + "model.layers.10.mlp.experts.26.down_proj", + "model.layers.10.mlp.experts.26.gate_proj", + "model.layers.10.mlp.experts.26.up_proj", + "model.layers.10.mlp.experts.27.down_proj", + "model.layers.10.mlp.experts.27.gate_proj", + "model.layers.10.mlp.experts.27.up_proj", + "model.layers.10.mlp.experts.28.down_proj", + "model.layers.10.mlp.experts.28.gate_proj", + "model.layers.10.mlp.experts.28.up_proj", + "model.layers.10.mlp.experts.29.down_proj", + "model.layers.10.mlp.experts.29.gate_proj", + "model.layers.10.mlp.experts.29.up_proj", + "model.layers.10.mlp.experts.3.down_proj", + "model.layers.10.mlp.experts.3.gate_proj", + "model.layers.10.mlp.experts.3.up_proj", + "model.layers.10.mlp.experts.30.down_proj", + "model.layers.10.mlp.experts.30.gate_proj", + "model.layers.10.mlp.experts.30.up_proj", + "model.layers.10.mlp.experts.31.down_proj", + "model.layers.10.mlp.experts.31.gate_proj", + "model.layers.10.mlp.experts.31.up_proj", + "model.layers.10.mlp.experts.32.down_proj", + "model.layers.10.mlp.experts.32.gate_proj", + "model.layers.10.mlp.experts.32.up_proj", + "model.layers.10.mlp.experts.33.down_proj", + "model.layers.10.mlp.experts.33.gate_proj", + "model.layers.10.mlp.experts.33.up_proj", + "model.layers.10.mlp.experts.34.down_proj", + "model.layers.10.mlp.experts.34.gate_proj", + "model.layers.10.mlp.experts.34.up_proj", + "model.layers.10.mlp.experts.35.down_proj", + "model.layers.10.mlp.experts.35.gate_proj", + "model.layers.10.mlp.experts.35.up_proj", + "model.layers.10.mlp.experts.36.down_proj", + "model.layers.10.mlp.experts.36.gate_proj", + "model.layers.10.mlp.experts.36.up_proj", + "model.layers.10.mlp.experts.37.down_proj", + "model.layers.10.mlp.experts.37.gate_proj", + "model.layers.10.mlp.experts.37.up_proj", + "model.layers.10.mlp.experts.38.down_proj", + "model.layers.10.mlp.experts.38.gate_proj", + "model.layers.10.mlp.experts.38.up_proj", + "model.layers.10.mlp.experts.39.down_proj", + "model.layers.10.mlp.experts.39.gate_proj", + "model.layers.10.mlp.experts.39.up_proj", + "model.layers.10.mlp.experts.4.down_proj", + "model.layers.10.mlp.experts.4.gate_proj", + "model.layers.10.mlp.experts.4.up_proj", + "model.layers.10.mlp.experts.40.down_proj", + "model.layers.10.mlp.experts.40.gate_proj", + "model.layers.10.mlp.experts.40.up_proj", + "model.layers.10.mlp.experts.41.down_proj", + "model.layers.10.mlp.experts.41.gate_proj", + "model.layers.10.mlp.experts.41.up_proj", + "model.layers.10.mlp.experts.42.down_proj", + "model.layers.10.mlp.experts.42.gate_proj", + "model.layers.10.mlp.experts.42.up_proj", + "model.layers.10.mlp.experts.43.down_proj", + "model.layers.10.mlp.experts.43.gate_proj", + "model.layers.10.mlp.experts.43.up_proj", + "model.layers.10.mlp.experts.44.down_proj", + "model.layers.10.mlp.experts.44.gate_proj", + "model.layers.10.mlp.experts.44.up_proj", + "model.layers.10.mlp.experts.45.down_proj", + "model.layers.10.mlp.experts.45.gate_proj", + "model.layers.10.mlp.experts.45.up_proj", + "model.layers.10.mlp.experts.46.down_proj", + "model.layers.10.mlp.experts.46.gate_proj", + "model.layers.10.mlp.experts.46.up_proj", + "model.layers.10.mlp.experts.47.down_proj", + "model.layers.10.mlp.experts.47.gate_proj", + "model.layers.10.mlp.experts.47.up_proj", + "model.layers.10.mlp.experts.48.down_proj", + "model.layers.10.mlp.experts.48.gate_proj", + "model.layers.10.mlp.experts.48.up_proj", + "model.layers.10.mlp.experts.49.down_proj", + "model.layers.10.mlp.experts.49.gate_proj", + "model.layers.10.mlp.experts.49.up_proj", + "model.layers.10.mlp.experts.5.down_proj", + "model.layers.10.mlp.experts.5.gate_proj", + "model.layers.10.mlp.experts.5.up_proj", + "model.layers.10.mlp.experts.50.down_proj", + "model.layers.10.mlp.experts.50.gate_proj", + "model.layers.10.mlp.experts.50.up_proj", + "model.layers.10.mlp.experts.51.down_proj", + "model.layers.10.mlp.experts.51.gate_proj", + "model.layers.10.mlp.experts.51.up_proj", + "model.layers.10.mlp.experts.52.down_proj", + "model.layers.10.mlp.experts.52.gate_proj", + "model.layers.10.mlp.experts.52.up_proj", + "model.layers.10.mlp.experts.53.down_proj", + "model.layers.10.mlp.experts.53.gate_proj", + "model.layers.10.mlp.experts.53.up_proj", + "model.layers.10.mlp.experts.54.down_proj", + "model.layers.10.mlp.experts.54.gate_proj", + "model.layers.10.mlp.experts.54.up_proj", + "model.layers.10.mlp.experts.55.down_proj", + "model.layers.10.mlp.experts.55.gate_proj", + "model.layers.10.mlp.experts.55.up_proj", + "model.layers.10.mlp.experts.56.down_proj", + "model.layers.10.mlp.experts.56.gate_proj", + "model.layers.10.mlp.experts.56.up_proj", + "model.layers.10.mlp.experts.57.down_proj", + "model.layers.10.mlp.experts.57.gate_proj", + "model.layers.10.mlp.experts.57.up_proj", + "model.layers.10.mlp.experts.58.down_proj", + "model.layers.10.mlp.experts.58.gate_proj", + "model.layers.10.mlp.experts.58.up_proj", + "model.layers.10.mlp.experts.59.down_proj", + "model.layers.10.mlp.experts.59.gate_proj", + "model.layers.10.mlp.experts.59.up_proj", + "model.layers.10.mlp.experts.6.down_proj", + "model.layers.10.mlp.experts.6.gate_proj", + "model.layers.10.mlp.experts.6.up_proj", + "model.layers.10.mlp.experts.60.down_proj", + "model.layers.10.mlp.experts.60.gate_proj", + "model.layers.10.mlp.experts.60.up_proj", + "model.layers.10.mlp.experts.61.down_proj", + "model.layers.10.mlp.experts.61.gate_proj", + "model.layers.10.mlp.experts.61.up_proj", + "model.layers.10.mlp.experts.62.down_proj", + "model.layers.10.mlp.experts.62.gate_proj", + "model.layers.10.mlp.experts.62.up_proj", + "model.layers.10.mlp.experts.63.down_proj", + "model.layers.10.mlp.experts.63.gate_proj", + "model.layers.10.mlp.experts.63.up_proj", + "model.layers.10.mlp.experts.64.down_proj", + "model.layers.10.mlp.experts.64.gate_proj", + "model.layers.10.mlp.experts.64.up_proj", + "model.layers.10.mlp.experts.65.down_proj", + "model.layers.10.mlp.experts.65.gate_proj", + "model.layers.10.mlp.experts.65.up_proj", + "model.layers.10.mlp.experts.66.down_proj", + "model.layers.10.mlp.experts.66.gate_proj", + "model.layers.10.mlp.experts.66.up_proj", + "model.layers.10.mlp.experts.67.down_proj", + "model.layers.10.mlp.experts.67.gate_proj", + "model.layers.10.mlp.experts.67.up_proj", + "model.layers.10.mlp.experts.68.down_proj", + "model.layers.10.mlp.experts.68.gate_proj", + "model.layers.10.mlp.experts.68.up_proj", + "model.layers.10.mlp.experts.69.down_proj", + "model.layers.10.mlp.experts.69.gate_proj", + "model.layers.10.mlp.experts.69.up_proj", + "model.layers.10.mlp.experts.7.down_proj", + "model.layers.10.mlp.experts.7.gate_proj", + "model.layers.10.mlp.experts.7.up_proj", + "model.layers.10.mlp.experts.70.down_proj", + "model.layers.10.mlp.experts.70.gate_proj", + "model.layers.10.mlp.experts.70.up_proj", + "model.layers.10.mlp.experts.71.down_proj", + "model.layers.10.mlp.experts.71.gate_proj", + "model.layers.10.mlp.experts.71.up_proj", + "model.layers.10.mlp.experts.72.down_proj", + "model.layers.10.mlp.experts.72.gate_proj", + "model.layers.10.mlp.experts.72.up_proj", + "model.layers.10.mlp.experts.73.down_proj", + "model.layers.10.mlp.experts.73.gate_proj", + "model.layers.10.mlp.experts.73.up_proj", + "model.layers.10.mlp.experts.74.down_proj", + "model.layers.10.mlp.experts.74.gate_proj", + "model.layers.10.mlp.experts.74.up_proj", + "model.layers.10.mlp.experts.75.down_proj", + "model.layers.10.mlp.experts.75.gate_proj", + "model.layers.10.mlp.experts.75.up_proj", + "model.layers.10.mlp.experts.76.down_proj", + "model.layers.10.mlp.experts.76.gate_proj", + "model.layers.10.mlp.experts.76.up_proj", + "model.layers.10.mlp.experts.77.down_proj", + "model.layers.10.mlp.experts.77.gate_proj", + "model.layers.10.mlp.experts.77.up_proj", + "model.layers.10.mlp.experts.78.down_proj", + "model.layers.10.mlp.experts.78.gate_proj", + "model.layers.10.mlp.experts.78.up_proj", + "model.layers.10.mlp.experts.79.down_proj", + "model.layers.10.mlp.experts.79.gate_proj", + "model.layers.10.mlp.experts.79.up_proj", + "model.layers.10.mlp.experts.8.down_proj", + "model.layers.10.mlp.experts.8.gate_proj", + "model.layers.10.mlp.experts.8.up_proj", + "model.layers.10.mlp.experts.80.down_proj", + "model.layers.10.mlp.experts.80.gate_proj", + "model.layers.10.mlp.experts.80.up_proj", + "model.layers.10.mlp.experts.81.down_proj", + "model.layers.10.mlp.experts.81.gate_proj", + "model.layers.10.mlp.experts.81.up_proj", + "model.layers.10.mlp.experts.82.down_proj", + "model.layers.10.mlp.experts.82.gate_proj", + "model.layers.10.mlp.experts.82.up_proj", + "model.layers.10.mlp.experts.83.down_proj", + "model.layers.10.mlp.experts.83.gate_proj", + "model.layers.10.mlp.experts.83.up_proj", + "model.layers.10.mlp.experts.84.down_proj", + "model.layers.10.mlp.experts.84.gate_proj", + "model.layers.10.mlp.experts.84.up_proj", + "model.layers.10.mlp.experts.85.down_proj", + "model.layers.10.mlp.experts.85.gate_proj", + "model.layers.10.mlp.experts.85.up_proj", + "model.layers.10.mlp.experts.86.down_proj", + "model.layers.10.mlp.experts.86.gate_proj", + "model.layers.10.mlp.experts.86.up_proj", + "model.layers.10.mlp.experts.87.down_proj", + "model.layers.10.mlp.experts.87.gate_proj", + "model.layers.10.mlp.experts.87.up_proj", + "model.layers.10.mlp.experts.88.down_proj", + "model.layers.10.mlp.experts.88.gate_proj", + "model.layers.10.mlp.experts.88.up_proj", + "model.layers.10.mlp.experts.89.down_proj", + "model.layers.10.mlp.experts.89.gate_proj", + "model.layers.10.mlp.experts.89.up_proj", + "model.layers.10.mlp.experts.9.down_proj", + "model.layers.10.mlp.experts.9.gate_proj", + "model.layers.10.mlp.experts.9.up_proj", + "model.layers.10.mlp.experts.90.down_proj", + "model.layers.10.mlp.experts.90.gate_proj", + "model.layers.10.mlp.experts.90.up_proj", + "model.layers.10.mlp.experts.91.down_proj", + "model.layers.10.mlp.experts.91.gate_proj", + "model.layers.10.mlp.experts.91.up_proj", + "model.layers.10.mlp.experts.92.down_proj", + "model.layers.10.mlp.experts.92.gate_proj", + "model.layers.10.mlp.experts.92.up_proj", + "model.layers.10.mlp.experts.93.down_proj", + "model.layers.10.mlp.experts.93.gate_proj", + "model.layers.10.mlp.experts.93.up_proj", + "model.layers.10.mlp.experts.94.down_proj", + "model.layers.10.mlp.experts.94.gate_proj", + "model.layers.10.mlp.experts.94.up_proj", + "model.layers.10.mlp.experts.95.down_proj", + "model.layers.10.mlp.experts.95.gate_proj", + "model.layers.10.mlp.experts.95.up_proj", + "model.layers.10.mlp.experts.96.down_proj", + "model.layers.10.mlp.experts.96.gate_proj", + "model.layers.10.mlp.experts.96.up_proj", + "model.layers.10.mlp.experts.97.down_proj", + "model.layers.10.mlp.experts.97.gate_proj", + "model.layers.10.mlp.experts.97.up_proj", + "model.layers.10.mlp.experts.98.down_proj", + "model.layers.10.mlp.experts.98.gate_proj", + "model.layers.10.mlp.experts.98.up_proj", + "model.layers.10.mlp.experts.99.down_proj", + "model.layers.10.mlp.experts.99.gate_proj", + "model.layers.10.mlp.experts.99.up_proj", + "model.layers.11.mlp.experts.0.down_proj", + "model.layers.11.mlp.experts.0.gate_proj", + "model.layers.11.mlp.experts.0.up_proj", + "model.layers.11.mlp.experts.1.down_proj", + "model.layers.11.mlp.experts.1.gate_proj", + "model.layers.11.mlp.experts.1.up_proj", + "model.layers.11.mlp.experts.10.down_proj", + "model.layers.11.mlp.experts.10.gate_proj", + "model.layers.11.mlp.experts.10.up_proj", + "model.layers.11.mlp.experts.100.down_proj", + "model.layers.11.mlp.experts.100.gate_proj", + "model.layers.11.mlp.experts.100.up_proj", + "model.layers.11.mlp.experts.101.down_proj", + "model.layers.11.mlp.experts.101.gate_proj", + "model.layers.11.mlp.experts.101.up_proj", + "model.layers.11.mlp.experts.102.down_proj", + "model.layers.11.mlp.experts.102.gate_proj", + "model.layers.11.mlp.experts.102.up_proj", + "model.layers.11.mlp.experts.103.down_proj", + "model.layers.11.mlp.experts.103.gate_proj", + "model.layers.11.mlp.experts.103.up_proj", + "model.layers.11.mlp.experts.104.down_proj", + "model.layers.11.mlp.experts.104.gate_proj", + "model.layers.11.mlp.experts.104.up_proj", + "model.layers.11.mlp.experts.105.down_proj", + "model.layers.11.mlp.experts.105.gate_proj", + "model.layers.11.mlp.experts.105.up_proj", + "model.layers.11.mlp.experts.106.down_proj", + "model.layers.11.mlp.experts.106.gate_proj", + "model.layers.11.mlp.experts.106.up_proj", + "model.layers.11.mlp.experts.107.down_proj", + "model.layers.11.mlp.experts.107.gate_proj", + "model.layers.11.mlp.experts.107.up_proj", + "model.layers.11.mlp.experts.108.down_proj", + "model.layers.11.mlp.experts.108.gate_proj", + "model.layers.11.mlp.experts.108.up_proj", + "model.layers.11.mlp.experts.109.down_proj", + "model.layers.11.mlp.experts.109.gate_proj", + "model.layers.11.mlp.experts.109.up_proj", + "model.layers.11.mlp.experts.11.down_proj", + "model.layers.11.mlp.experts.11.gate_proj", + "model.layers.11.mlp.experts.11.up_proj", + "model.layers.11.mlp.experts.110.down_proj", + "model.layers.11.mlp.experts.110.gate_proj", + "model.layers.11.mlp.experts.110.up_proj", + "model.layers.11.mlp.experts.111.down_proj", + "model.layers.11.mlp.experts.111.gate_proj", + "model.layers.11.mlp.experts.111.up_proj", + "model.layers.11.mlp.experts.112.down_proj", + "model.layers.11.mlp.experts.112.gate_proj", + "model.layers.11.mlp.experts.112.up_proj", + "model.layers.11.mlp.experts.113.down_proj", + "model.layers.11.mlp.experts.113.gate_proj", + "model.layers.11.mlp.experts.113.up_proj", + "model.layers.11.mlp.experts.114.down_proj", + "model.layers.11.mlp.experts.114.gate_proj", + "model.layers.11.mlp.experts.114.up_proj", + "model.layers.11.mlp.experts.115.down_proj", + "model.layers.11.mlp.experts.115.gate_proj", + "model.layers.11.mlp.experts.115.up_proj", + "model.layers.11.mlp.experts.116.down_proj", + "model.layers.11.mlp.experts.116.gate_proj", + "model.layers.11.mlp.experts.116.up_proj", + "model.layers.11.mlp.experts.117.down_proj", + "model.layers.11.mlp.experts.117.gate_proj", + "model.layers.11.mlp.experts.117.up_proj", + "model.layers.11.mlp.experts.118.down_proj", + "model.layers.11.mlp.experts.118.gate_proj", + "model.layers.11.mlp.experts.118.up_proj", + "model.layers.11.mlp.experts.119.down_proj", + "model.layers.11.mlp.experts.119.gate_proj", + "model.layers.11.mlp.experts.119.up_proj", + "model.layers.11.mlp.experts.12.down_proj", + "model.layers.11.mlp.experts.12.gate_proj", + "model.layers.11.mlp.experts.12.up_proj", + "model.layers.11.mlp.experts.120.down_proj", + "model.layers.11.mlp.experts.120.gate_proj", + "model.layers.11.mlp.experts.120.up_proj", + "model.layers.11.mlp.experts.121.down_proj", + "model.layers.11.mlp.experts.121.gate_proj", + "model.layers.11.mlp.experts.121.up_proj", + "model.layers.11.mlp.experts.122.down_proj", + "model.layers.11.mlp.experts.122.gate_proj", + "model.layers.11.mlp.experts.122.up_proj", + "model.layers.11.mlp.experts.123.down_proj", + "model.layers.11.mlp.experts.123.gate_proj", + "model.layers.11.mlp.experts.123.up_proj", + "model.layers.11.mlp.experts.124.down_proj", + "model.layers.11.mlp.experts.124.gate_proj", + "model.layers.11.mlp.experts.124.up_proj", + "model.layers.11.mlp.experts.125.down_proj", + "model.layers.11.mlp.experts.125.gate_proj", + "model.layers.11.mlp.experts.125.up_proj", + "model.layers.11.mlp.experts.126.down_proj", + "model.layers.11.mlp.experts.126.gate_proj", + "model.layers.11.mlp.experts.126.up_proj", + "model.layers.11.mlp.experts.127.down_proj", + "model.layers.11.mlp.experts.127.gate_proj", + "model.layers.11.mlp.experts.127.up_proj", + "model.layers.11.mlp.experts.128.down_proj", + "model.layers.11.mlp.experts.128.gate_proj", + "model.layers.11.mlp.experts.128.up_proj", + "model.layers.11.mlp.experts.129.down_proj", + "model.layers.11.mlp.experts.129.gate_proj", + "model.layers.11.mlp.experts.129.up_proj", + "model.layers.11.mlp.experts.13.down_proj", + "model.layers.11.mlp.experts.13.gate_proj", + "model.layers.11.mlp.experts.13.up_proj", + "model.layers.11.mlp.experts.130.down_proj", + "model.layers.11.mlp.experts.130.gate_proj", + "model.layers.11.mlp.experts.130.up_proj", + "model.layers.11.mlp.experts.131.down_proj", + "model.layers.11.mlp.experts.131.gate_proj", + "model.layers.11.mlp.experts.131.up_proj", + "model.layers.11.mlp.experts.132.down_proj", + "model.layers.11.mlp.experts.132.gate_proj", + "model.layers.11.mlp.experts.132.up_proj", + "model.layers.11.mlp.experts.133.down_proj", + "model.layers.11.mlp.experts.133.gate_proj", + "model.layers.11.mlp.experts.133.up_proj", + "model.layers.11.mlp.experts.134.down_proj", + "model.layers.11.mlp.experts.134.gate_proj", + "model.layers.11.mlp.experts.134.up_proj", + "model.layers.11.mlp.experts.135.down_proj", + "model.layers.11.mlp.experts.135.gate_proj", + "model.layers.11.mlp.experts.135.up_proj", + "model.layers.11.mlp.experts.136.down_proj", + "model.layers.11.mlp.experts.136.gate_proj", + "model.layers.11.mlp.experts.136.up_proj", + "model.layers.11.mlp.experts.137.down_proj", + "model.layers.11.mlp.experts.137.gate_proj", + "model.layers.11.mlp.experts.137.up_proj", + "model.layers.11.mlp.experts.138.down_proj", + "model.layers.11.mlp.experts.138.gate_proj", + "model.layers.11.mlp.experts.138.up_proj", + "model.layers.11.mlp.experts.139.down_proj", + "model.layers.11.mlp.experts.139.gate_proj", + "model.layers.11.mlp.experts.139.up_proj", + "model.layers.11.mlp.experts.14.down_proj", + "model.layers.11.mlp.experts.14.gate_proj", + "model.layers.11.mlp.experts.14.up_proj", + "model.layers.11.mlp.experts.140.down_proj", + "model.layers.11.mlp.experts.140.gate_proj", + "model.layers.11.mlp.experts.140.up_proj", + "model.layers.11.mlp.experts.141.down_proj", + "model.layers.11.mlp.experts.141.gate_proj", + "model.layers.11.mlp.experts.141.up_proj", + "model.layers.11.mlp.experts.142.down_proj", + "model.layers.11.mlp.experts.142.gate_proj", + "model.layers.11.mlp.experts.142.up_proj", + "model.layers.11.mlp.experts.143.down_proj", + "model.layers.11.mlp.experts.143.gate_proj", + "model.layers.11.mlp.experts.143.up_proj", + "model.layers.11.mlp.experts.144.down_proj", + "model.layers.11.mlp.experts.144.gate_proj", + "model.layers.11.mlp.experts.144.up_proj", + "model.layers.11.mlp.experts.145.down_proj", + "model.layers.11.mlp.experts.145.gate_proj", + "model.layers.11.mlp.experts.145.up_proj", + "model.layers.11.mlp.experts.146.down_proj", + "model.layers.11.mlp.experts.146.gate_proj", + "model.layers.11.mlp.experts.146.up_proj", + "model.layers.11.mlp.experts.147.down_proj", + "model.layers.11.mlp.experts.147.gate_proj", + "model.layers.11.mlp.experts.147.up_proj", + "model.layers.11.mlp.experts.148.down_proj", + "model.layers.11.mlp.experts.148.gate_proj", + "model.layers.11.mlp.experts.148.up_proj", + "model.layers.11.mlp.experts.149.down_proj", + "model.layers.11.mlp.experts.149.gate_proj", + "model.layers.11.mlp.experts.149.up_proj", + "model.layers.11.mlp.experts.15.down_proj", + "model.layers.11.mlp.experts.15.gate_proj", + "model.layers.11.mlp.experts.15.up_proj", + "model.layers.11.mlp.experts.150.down_proj", + "model.layers.11.mlp.experts.150.gate_proj", + "model.layers.11.mlp.experts.150.up_proj", + "model.layers.11.mlp.experts.151.down_proj", + "model.layers.11.mlp.experts.151.gate_proj", + "model.layers.11.mlp.experts.151.up_proj", + "model.layers.11.mlp.experts.152.down_proj", + "model.layers.11.mlp.experts.152.gate_proj", + "model.layers.11.mlp.experts.152.up_proj", + "model.layers.11.mlp.experts.153.down_proj", + "model.layers.11.mlp.experts.153.gate_proj", + "model.layers.11.mlp.experts.153.up_proj", + "model.layers.11.mlp.experts.154.down_proj", + "model.layers.11.mlp.experts.154.gate_proj", + "model.layers.11.mlp.experts.154.up_proj", + "model.layers.11.mlp.experts.155.down_proj", + "model.layers.11.mlp.experts.155.gate_proj", + "model.layers.11.mlp.experts.155.up_proj", + "model.layers.11.mlp.experts.156.down_proj", + "model.layers.11.mlp.experts.156.gate_proj", + "model.layers.11.mlp.experts.156.up_proj", + "model.layers.11.mlp.experts.157.down_proj", + "model.layers.11.mlp.experts.157.gate_proj", + "model.layers.11.mlp.experts.157.up_proj", + "model.layers.11.mlp.experts.158.down_proj", + "model.layers.11.mlp.experts.158.gate_proj", + "model.layers.11.mlp.experts.158.up_proj", + "model.layers.11.mlp.experts.159.down_proj", + "model.layers.11.mlp.experts.159.gate_proj", + "model.layers.11.mlp.experts.159.up_proj", + "model.layers.11.mlp.experts.16.down_proj", + "model.layers.11.mlp.experts.16.gate_proj", + "model.layers.11.mlp.experts.16.up_proj", + "model.layers.11.mlp.experts.160.down_proj", + "model.layers.11.mlp.experts.160.gate_proj", + "model.layers.11.mlp.experts.160.up_proj", + "model.layers.11.mlp.experts.161.down_proj", + "model.layers.11.mlp.experts.161.gate_proj", + "model.layers.11.mlp.experts.161.up_proj", + "model.layers.11.mlp.experts.162.down_proj", + "model.layers.11.mlp.experts.162.gate_proj", + "model.layers.11.mlp.experts.162.up_proj", + "model.layers.11.mlp.experts.163.down_proj", + "model.layers.11.mlp.experts.163.gate_proj", + "model.layers.11.mlp.experts.163.up_proj", + "model.layers.11.mlp.experts.164.down_proj", + "model.layers.11.mlp.experts.164.gate_proj", + "model.layers.11.mlp.experts.164.up_proj", + "model.layers.11.mlp.experts.165.down_proj", + "model.layers.11.mlp.experts.165.gate_proj", + "model.layers.11.mlp.experts.165.up_proj", + "model.layers.11.mlp.experts.166.down_proj", + "model.layers.11.mlp.experts.166.gate_proj", + "model.layers.11.mlp.experts.166.up_proj", + "model.layers.11.mlp.experts.167.down_proj", + "model.layers.11.mlp.experts.167.gate_proj", + "model.layers.11.mlp.experts.167.up_proj", + "model.layers.11.mlp.experts.168.down_proj", + "model.layers.11.mlp.experts.168.gate_proj", + "model.layers.11.mlp.experts.168.up_proj", + "model.layers.11.mlp.experts.169.down_proj", + "model.layers.11.mlp.experts.169.gate_proj", + "model.layers.11.mlp.experts.169.up_proj", + "model.layers.11.mlp.experts.17.down_proj", + "model.layers.11.mlp.experts.17.gate_proj", + "model.layers.11.mlp.experts.17.up_proj", + "model.layers.11.mlp.experts.170.down_proj", + "model.layers.11.mlp.experts.170.gate_proj", + "model.layers.11.mlp.experts.170.up_proj", + "model.layers.11.mlp.experts.171.down_proj", + "model.layers.11.mlp.experts.171.gate_proj", + "model.layers.11.mlp.experts.171.up_proj", + "model.layers.11.mlp.experts.172.down_proj", + "model.layers.11.mlp.experts.172.gate_proj", + "model.layers.11.mlp.experts.172.up_proj", + "model.layers.11.mlp.experts.173.down_proj", + "model.layers.11.mlp.experts.173.gate_proj", + "model.layers.11.mlp.experts.173.up_proj", + "model.layers.11.mlp.experts.174.down_proj", + "model.layers.11.mlp.experts.174.gate_proj", + "model.layers.11.mlp.experts.174.up_proj", + "model.layers.11.mlp.experts.175.down_proj", + "model.layers.11.mlp.experts.175.gate_proj", + "model.layers.11.mlp.experts.175.up_proj", + "model.layers.11.mlp.experts.176.down_proj", + "model.layers.11.mlp.experts.176.gate_proj", + "model.layers.11.mlp.experts.176.up_proj", + "model.layers.11.mlp.experts.177.down_proj", + "model.layers.11.mlp.experts.177.gate_proj", + "model.layers.11.mlp.experts.177.up_proj", + "model.layers.11.mlp.experts.178.down_proj", + "model.layers.11.mlp.experts.178.gate_proj", + "model.layers.11.mlp.experts.178.up_proj", + "model.layers.11.mlp.experts.179.down_proj", + "model.layers.11.mlp.experts.179.gate_proj", + "model.layers.11.mlp.experts.179.up_proj", + "model.layers.11.mlp.experts.18.down_proj", + "model.layers.11.mlp.experts.18.gate_proj", + "model.layers.11.mlp.experts.18.up_proj", + "model.layers.11.mlp.experts.180.down_proj", + "model.layers.11.mlp.experts.180.gate_proj", + "model.layers.11.mlp.experts.180.up_proj", + "model.layers.11.mlp.experts.181.down_proj", + "model.layers.11.mlp.experts.181.gate_proj", + "model.layers.11.mlp.experts.181.up_proj", + "model.layers.11.mlp.experts.182.down_proj", + "model.layers.11.mlp.experts.182.gate_proj", + "model.layers.11.mlp.experts.182.up_proj", + "model.layers.11.mlp.experts.183.down_proj", + "model.layers.11.mlp.experts.183.gate_proj", + "model.layers.11.mlp.experts.183.up_proj", + "model.layers.11.mlp.experts.184.down_proj", + "model.layers.11.mlp.experts.184.gate_proj", + "model.layers.11.mlp.experts.184.up_proj", + "model.layers.11.mlp.experts.185.down_proj", + "model.layers.11.mlp.experts.185.gate_proj", + "model.layers.11.mlp.experts.185.up_proj", + "model.layers.11.mlp.experts.186.down_proj", + "model.layers.11.mlp.experts.186.gate_proj", + "model.layers.11.mlp.experts.186.up_proj", + "model.layers.11.mlp.experts.187.down_proj", + "model.layers.11.mlp.experts.187.gate_proj", + "model.layers.11.mlp.experts.187.up_proj", + "model.layers.11.mlp.experts.188.down_proj", + "model.layers.11.mlp.experts.188.gate_proj", + "model.layers.11.mlp.experts.188.up_proj", + "model.layers.11.mlp.experts.189.down_proj", + "model.layers.11.mlp.experts.189.gate_proj", + "model.layers.11.mlp.experts.189.up_proj", + "model.layers.11.mlp.experts.19.down_proj", + "model.layers.11.mlp.experts.19.gate_proj", + "model.layers.11.mlp.experts.19.up_proj", + "model.layers.11.mlp.experts.190.down_proj", + "model.layers.11.mlp.experts.190.gate_proj", + "model.layers.11.mlp.experts.190.up_proj", + "model.layers.11.mlp.experts.191.down_proj", + "model.layers.11.mlp.experts.191.gate_proj", + "model.layers.11.mlp.experts.191.up_proj", + "model.layers.11.mlp.experts.192.down_proj", + "model.layers.11.mlp.experts.192.gate_proj", + "model.layers.11.mlp.experts.192.up_proj", + "model.layers.11.mlp.experts.193.down_proj", + "model.layers.11.mlp.experts.193.gate_proj", + "model.layers.11.mlp.experts.193.up_proj", + "model.layers.11.mlp.experts.194.down_proj", + "model.layers.11.mlp.experts.194.gate_proj", + "model.layers.11.mlp.experts.194.up_proj", + "model.layers.11.mlp.experts.195.down_proj", + "model.layers.11.mlp.experts.195.gate_proj", + "model.layers.11.mlp.experts.195.up_proj", + "model.layers.11.mlp.experts.196.down_proj", + "model.layers.11.mlp.experts.196.gate_proj", + "model.layers.11.mlp.experts.196.up_proj", + "model.layers.11.mlp.experts.197.down_proj", + "model.layers.11.mlp.experts.197.gate_proj", + "model.layers.11.mlp.experts.197.up_proj", + "model.layers.11.mlp.experts.198.down_proj", + "model.layers.11.mlp.experts.198.gate_proj", + "model.layers.11.mlp.experts.198.up_proj", + "model.layers.11.mlp.experts.199.down_proj", + "model.layers.11.mlp.experts.199.gate_proj", + "model.layers.11.mlp.experts.199.up_proj", + "model.layers.11.mlp.experts.2.down_proj", + "model.layers.11.mlp.experts.2.gate_proj", + "model.layers.11.mlp.experts.2.up_proj", + "model.layers.11.mlp.experts.20.down_proj", + "model.layers.11.mlp.experts.20.gate_proj", + "model.layers.11.mlp.experts.20.up_proj", + "model.layers.11.mlp.experts.200.down_proj", + "model.layers.11.mlp.experts.200.gate_proj", + "model.layers.11.mlp.experts.200.up_proj", + "model.layers.11.mlp.experts.201.down_proj", + "model.layers.11.mlp.experts.201.gate_proj", + "model.layers.11.mlp.experts.201.up_proj", + "model.layers.11.mlp.experts.202.down_proj", + "model.layers.11.mlp.experts.202.gate_proj", + "model.layers.11.mlp.experts.202.up_proj", + "model.layers.11.mlp.experts.203.down_proj", + "model.layers.11.mlp.experts.203.gate_proj", + "model.layers.11.mlp.experts.203.up_proj", + "model.layers.11.mlp.experts.204.down_proj", + "model.layers.11.mlp.experts.204.gate_proj", + "model.layers.11.mlp.experts.204.up_proj", + "model.layers.11.mlp.experts.205.down_proj", + "model.layers.11.mlp.experts.205.gate_proj", + "model.layers.11.mlp.experts.205.up_proj", + "model.layers.11.mlp.experts.206.down_proj", + "model.layers.11.mlp.experts.206.gate_proj", + "model.layers.11.mlp.experts.206.up_proj", + "model.layers.11.mlp.experts.207.down_proj", + "model.layers.11.mlp.experts.207.gate_proj", + "model.layers.11.mlp.experts.207.up_proj", + "model.layers.11.mlp.experts.208.down_proj", + "model.layers.11.mlp.experts.208.gate_proj", + "model.layers.11.mlp.experts.208.up_proj", + "model.layers.11.mlp.experts.209.down_proj", + "model.layers.11.mlp.experts.209.gate_proj", + "model.layers.11.mlp.experts.209.up_proj", + "model.layers.11.mlp.experts.21.down_proj", + "model.layers.11.mlp.experts.21.gate_proj", + "model.layers.11.mlp.experts.21.up_proj", + "model.layers.11.mlp.experts.210.down_proj", + "model.layers.11.mlp.experts.210.gate_proj", + "model.layers.11.mlp.experts.210.up_proj", + "model.layers.11.mlp.experts.211.down_proj", + "model.layers.11.mlp.experts.211.gate_proj", + "model.layers.11.mlp.experts.211.up_proj", + "model.layers.11.mlp.experts.212.down_proj", + "model.layers.11.mlp.experts.212.gate_proj", + "model.layers.11.mlp.experts.212.up_proj", + "model.layers.11.mlp.experts.213.down_proj", + "model.layers.11.mlp.experts.213.gate_proj", + "model.layers.11.mlp.experts.213.up_proj", + "model.layers.11.mlp.experts.214.down_proj", + "model.layers.11.mlp.experts.214.gate_proj", + "model.layers.11.mlp.experts.214.up_proj", + "model.layers.11.mlp.experts.215.down_proj", + "model.layers.11.mlp.experts.215.gate_proj", + "model.layers.11.mlp.experts.215.up_proj", + "model.layers.11.mlp.experts.216.down_proj", + "model.layers.11.mlp.experts.216.gate_proj", + "model.layers.11.mlp.experts.216.up_proj", + "model.layers.11.mlp.experts.217.down_proj", + "model.layers.11.mlp.experts.217.gate_proj", + "model.layers.11.mlp.experts.217.up_proj", + "model.layers.11.mlp.experts.218.down_proj", + "model.layers.11.mlp.experts.218.gate_proj", + "model.layers.11.mlp.experts.218.up_proj", + "model.layers.11.mlp.experts.219.down_proj", + "model.layers.11.mlp.experts.219.gate_proj", + "model.layers.11.mlp.experts.219.up_proj", + "model.layers.11.mlp.experts.22.down_proj", + "model.layers.11.mlp.experts.22.gate_proj", + "model.layers.11.mlp.experts.22.up_proj", + "model.layers.11.mlp.experts.220.down_proj", + "model.layers.11.mlp.experts.220.gate_proj", + "model.layers.11.mlp.experts.220.up_proj", + "model.layers.11.mlp.experts.221.down_proj", + "model.layers.11.mlp.experts.221.gate_proj", + "model.layers.11.mlp.experts.221.up_proj", + "model.layers.11.mlp.experts.222.down_proj", + "model.layers.11.mlp.experts.222.gate_proj", + "model.layers.11.mlp.experts.222.up_proj", + "model.layers.11.mlp.experts.223.down_proj", + "model.layers.11.mlp.experts.223.gate_proj", + "model.layers.11.mlp.experts.223.up_proj", + "model.layers.11.mlp.experts.224.down_proj", + "model.layers.11.mlp.experts.224.gate_proj", + "model.layers.11.mlp.experts.224.up_proj", + "model.layers.11.mlp.experts.225.down_proj", + "model.layers.11.mlp.experts.225.gate_proj", + "model.layers.11.mlp.experts.225.up_proj", + "model.layers.11.mlp.experts.226.down_proj", + "model.layers.11.mlp.experts.226.gate_proj", + "model.layers.11.mlp.experts.226.up_proj", + "model.layers.11.mlp.experts.227.down_proj", + "model.layers.11.mlp.experts.227.gate_proj", + "model.layers.11.mlp.experts.227.up_proj", + "model.layers.11.mlp.experts.228.down_proj", + "model.layers.11.mlp.experts.228.gate_proj", + "model.layers.11.mlp.experts.228.up_proj", + "model.layers.11.mlp.experts.229.down_proj", + "model.layers.11.mlp.experts.229.gate_proj", + "model.layers.11.mlp.experts.229.up_proj", + "model.layers.11.mlp.experts.23.down_proj", + "model.layers.11.mlp.experts.23.gate_proj", + "model.layers.11.mlp.experts.23.up_proj", + "model.layers.11.mlp.experts.230.down_proj", + "model.layers.11.mlp.experts.230.gate_proj", + "model.layers.11.mlp.experts.230.up_proj", + "model.layers.11.mlp.experts.231.down_proj", + "model.layers.11.mlp.experts.231.gate_proj", + "model.layers.11.mlp.experts.231.up_proj", + "model.layers.11.mlp.experts.232.down_proj", + "model.layers.11.mlp.experts.232.gate_proj", + "model.layers.11.mlp.experts.232.up_proj", + "model.layers.11.mlp.experts.233.down_proj", + "model.layers.11.mlp.experts.233.gate_proj", + "model.layers.11.mlp.experts.233.up_proj", + "model.layers.11.mlp.experts.234.down_proj", + "model.layers.11.mlp.experts.234.gate_proj", + "model.layers.11.mlp.experts.234.up_proj", + "model.layers.11.mlp.experts.235.down_proj", + "model.layers.11.mlp.experts.235.gate_proj", + "model.layers.11.mlp.experts.235.up_proj", + "model.layers.11.mlp.experts.236.down_proj", + "model.layers.11.mlp.experts.236.gate_proj", + "model.layers.11.mlp.experts.236.up_proj", + "model.layers.11.mlp.experts.237.down_proj", + "model.layers.11.mlp.experts.237.gate_proj", + "model.layers.11.mlp.experts.237.up_proj", + "model.layers.11.mlp.experts.238.down_proj", + "model.layers.11.mlp.experts.238.gate_proj", + "model.layers.11.mlp.experts.238.up_proj", + "model.layers.11.mlp.experts.239.down_proj", + "model.layers.11.mlp.experts.239.gate_proj", + "model.layers.11.mlp.experts.239.up_proj", + "model.layers.11.mlp.experts.24.down_proj", + "model.layers.11.mlp.experts.24.gate_proj", + "model.layers.11.mlp.experts.24.up_proj", + "model.layers.11.mlp.experts.240.down_proj", + "model.layers.11.mlp.experts.240.gate_proj", + "model.layers.11.mlp.experts.240.up_proj", + "model.layers.11.mlp.experts.241.down_proj", + "model.layers.11.mlp.experts.241.gate_proj", + "model.layers.11.mlp.experts.241.up_proj", + "model.layers.11.mlp.experts.242.down_proj", + "model.layers.11.mlp.experts.242.gate_proj", + "model.layers.11.mlp.experts.242.up_proj", + "model.layers.11.mlp.experts.243.down_proj", + "model.layers.11.mlp.experts.243.gate_proj", + "model.layers.11.mlp.experts.243.up_proj", + "model.layers.11.mlp.experts.244.down_proj", + "model.layers.11.mlp.experts.244.gate_proj", + "model.layers.11.mlp.experts.244.up_proj", + "model.layers.11.mlp.experts.245.down_proj", + "model.layers.11.mlp.experts.245.gate_proj", + "model.layers.11.mlp.experts.245.up_proj", + "model.layers.11.mlp.experts.246.down_proj", + "model.layers.11.mlp.experts.246.gate_proj", + "model.layers.11.mlp.experts.246.up_proj", + "model.layers.11.mlp.experts.247.down_proj", + "model.layers.11.mlp.experts.247.gate_proj", + "model.layers.11.mlp.experts.247.up_proj", + "model.layers.11.mlp.experts.248.down_proj", + "model.layers.11.mlp.experts.248.gate_proj", + "model.layers.11.mlp.experts.248.up_proj", + "model.layers.11.mlp.experts.249.down_proj", + "model.layers.11.mlp.experts.249.gate_proj", + "model.layers.11.mlp.experts.249.up_proj", + "model.layers.11.mlp.experts.25.down_proj", + "model.layers.11.mlp.experts.25.gate_proj", + "model.layers.11.mlp.experts.25.up_proj", + "model.layers.11.mlp.experts.250.down_proj", + "model.layers.11.mlp.experts.250.gate_proj", + "model.layers.11.mlp.experts.250.up_proj", + "model.layers.11.mlp.experts.251.down_proj", + "model.layers.11.mlp.experts.251.gate_proj", + "model.layers.11.mlp.experts.251.up_proj", + "model.layers.11.mlp.experts.252.down_proj", + "model.layers.11.mlp.experts.252.gate_proj", + "model.layers.11.mlp.experts.252.up_proj", + "model.layers.11.mlp.experts.253.down_proj", + "model.layers.11.mlp.experts.253.gate_proj", + "model.layers.11.mlp.experts.253.up_proj", + "model.layers.11.mlp.experts.254.down_proj", + "model.layers.11.mlp.experts.254.gate_proj", + "model.layers.11.mlp.experts.254.up_proj", + "model.layers.11.mlp.experts.255.down_proj", + "model.layers.11.mlp.experts.255.gate_proj", + "model.layers.11.mlp.experts.255.up_proj", + "model.layers.11.mlp.experts.26.down_proj", + "model.layers.11.mlp.experts.26.gate_proj", + "model.layers.11.mlp.experts.26.up_proj", + "model.layers.11.mlp.experts.27.down_proj", + "model.layers.11.mlp.experts.27.gate_proj", + "model.layers.11.mlp.experts.27.up_proj", + "model.layers.11.mlp.experts.28.down_proj", + "model.layers.11.mlp.experts.28.gate_proj", + "model.layers.11.mlp.experts.28.up_proj", + "model.layers.11.mlp.experts.29.down_proj", + "model.layers.11.mlp.experts.29.gate_proj", + "model.layers.11.mlp.experts.29.up_proj", + "model.layers.11.mlp.experts.3.down_proj", + "model.layers.11.mlp.experts.3.gate_proj", + "model.layers.11.mlp.experts.3.up_proj", + "model.layers.11.mlp.experts.30.down_proj", + "model.layers.11.mlp.experts.30.gate_proj", + "model.layers.11.mlp.experts.30.up_proj", + "model.layers.11.mlp.experts.31.down_proj", + "model.layers.11.mlp.experts.31.gate_proj", + "model.layers.11.mlp.experts.31.up_proj", + "model.layers.11.mlp.experts.32.down_proj", + "model.layers.11.mlp.experts.32.gate_proj", + "model.layers.11.mlp.experts.32.up_proj", + "model.layers.11.mlp.experts.33.down_proj", + "model.layers.11.mlp.experts.33.gate_proj", + "model.layers.11.mlp.experts.33.up_proj", + "model.layers.11.mlp.experts.34.down_proj", + "model.layers.11.mlp.experts.34.gate_proj", + "model.layers.11.mlp.experts.34.up_proj", + "model.layers.11.mlp.experts.35.down_proj", + "model.layers.11.mlp.experts.35.gate_proj", + "model.layers.11.mlp.experts.35.up_proj", + "model.layers.11.mlp.experts.36.down_proj", + "model.layers.11.mlp.experts.36.gate_proj", + "model.layers.11.mlp.experts.36.up_proj", + "model.layers.11.mlp.experts.37.down_proj", + "model.layers.11.mlp.experts.37.gate_proj", + "model.layers.11.mlp.experts.37.up_proj", + "model.layers.11.mlp.experts.38.down_proj", + "model.layers.11.mlp.experts.38.gate_proj", + "model.layers.11.mlp.experts.38.up_proj", + "model.layers.11.mlp.experts.39.down_proj", + "model.layers.11.mlp.experts.39.gate_proj", + "model.layers.11.mlp.experts.39.up_proj", + "model.layers.11.mlp.experts.4.down_proj", + "model.layers.11.mlp.experts.4.gate_proj", + "model.layers.11.mlp.experts.4.up_proj", + "model.layers.11.mlp.experts.40.down_proj", + "model.layers.11.mlp.experts.40.gate_proj", + "model.layers.11.mlp.experts.40.up_proj", + "model.layers.11.mlp.experts.41.down_proj", + "model.layers.11.mlp.experts.41.gate_proj", + "model.layers.11.mlp.experts.41.up_proj", + "model.layers.11.mlp.experts.42.down_proj", + "model.layers.11.mlp.experts.42.gate_proj", + "model.layers.11.mlp.experts.42.up_proj", + "model.layers.11.mlp.experts.43.down_proj", + "model.layers.11.mlp.experts.43.gate_proj", + "model.layers.11.mlp.experts.43.up_proj", + "model.layers.11.mlp.experts.44.down_proj", + "model.layers.11.mlp.experts.44.gate_proj", + "model.layers.11.mlp.experts.44.up_proj", + "model.layers.11.mlp.experts.45.down_proj", + "model.layers.11.mlp.experts.45.gate_proj", + "model.layers.11.mlp.experts.45.up_proj", + "model.layers.11.mlp.experts.46.down_proj", + "model.layers.11.mlp.experts.46.gate_proj", + "model.layers.11.mlp.experts.46.up_proj", + "model.layers.11.mlp.experts.47.down_proj", + "model.layers.11.mlp.experts.47.gate_proj", + "model.layers.11.mlp.experts.47.up_proj", + "model.layers.11.mlp.experts.48.down_proj", + "model.layers.11.mlp.experts.48.gate_proj", + "model.layers.11.mlp.experts.48.up_proj", + "model.layers.11.mlp.experts.49.down_proj", + "model.layers.11.mlp.experts.49.gate_proj", + "model.layers.11.mlp.experts.49.up_proj", + "model.layers.11.mlp.experts.5.down_proj", + "model.layers.11.mlp.experts.5.gate_proj", + "model.layers.11.mlp.experts.5.up_proj", + "model.layers.11.mlp.experts.50.down_proj", + "model.layers.11.mlp.experts.50.gate_proj", + "model.layers.11.mlp.experts.50.up_proj", + "model.layers.11.mlp.experts.51.down_proj", + "model.layers.11.mlp.experts.51.gate_proj", + "model.layers.11.mlp.experts.51.up_proj", + "model.layers.11.mlp.experts.52.down_proj", + "model.layers.11.mlp.experts.52.gate_proj", + "model.layers.11.mlp.experts.52.up_proj", + "model.layers.11.mlp.experts.53.down_proj", + "model.layers.11.mlp.experts.53.gate_proj", + "model.layers.11.mlp.experts.53.up_proj", + "model.layers.11.mlp.experts.54.down_proj", + "model.layers.11.mlp.experts.54.gate_proj", + "model.layers.11.mlp.experts.54.up_proj", + "model.layers.11.mlp.experts.55.down_proj", + "model.layers.11.mlp.experts.55.gate_proj", + "model.layers.11.mlp.experts.55.up_proj", + "model.layers.11.mlp.experts.56.down_proj", + "model.layers.11.mlp.experts.56.gate_proj", + "model.layers.11.mlp.experts.56.up_proj", + "model.layers.11.mlp.experts.57.down_proj", + "model.layers.11.mlp.experts.57.gate_proj", + "model.layers.11.mlp.experts.57.up_proj", + "model.layers.11.mlp.experts.58.down_proj", + "model.layers.11.mlp.experts.58.gate_proj", + "model.layers.11.mlp.experts.58.up_proj", + "model.layers.11.mlp.experts.59.down_proj", + "model.layers.11.mlp.experts.59.gate_proj", + "model.layers.11.mlp.experts.59.up_proj", + "model.layers.11.mlp.experts.6.down_proj", + "model.layers.11.mlp.experts.6.gate_proj", + "model.layers.11.mlp.experts.6.up_proj", + "model.layers.11.mlp.experts.60.down_proj", + "model.layers.11.mlp.experts.60.gate_proj", + "model.layers.11.mlp.experts.60.up_proj", + "model.layers.11.mlp.experts.61.down_proj", + "model.layers.11.mlp.experts.61.gate_proj", + "model.layers.11.mlp.experts.61.up_proj", + "model.layers.11.mlp.experts.62.down_proj", + "model.layers.11.mlp.experts.62.gate_proj", + "model.layers.11.mlp.experts.62.up_proj", + "model.layers.11.mlp.experts.63.down_proj", + "model.layers.11.mlp.experts.63.gate_proj", + "model.layers.11.mlp.experts.63.up_proj", + "model.layers.11.mlp.experts.64.down_proj", + "model.layers.11.mlp.experts.64.gate_proj", + "model.layers.11.mlp.experts.64.up_proj", + "model.layers.11.mlp.experts.65.down_proj", + "model.layers.11.mlp.experts.65.gate_proj", + "model.layers.11.mlp.experts.65.up_proj", + "model.layers.11.mlp.experts.66.down_proj", + "model.layers.11.mlp.experts.66.gate_proj", + "model.layers.11.mlp.experts.66.up_proj", + "model.layers.11.mlp.experts.67.down_proj", + "model.layers.11.mlp.experts.67.gate_proj", + "model.layers.11.mlp.experts.67.up_proj", + "model.layers.11.mlp.experts.68.down_proj", + "model.layers.11.mlp.experts.68.gate_proj", + "model.layers.11.mlp.experts.68.up_proj", + "model.layers.11.mlp.experts.69.down_proj", + "model.layers.11.mlp.experts.69.gate_proj", + "model.layers.11.mlp.experts.69.up_proj", + "model.layers.11.mlp.experts.7.down_proj", + "model.layers.11.mlp.experts.7.gate_proj", + "model.layers.11.mlp.experts.7.up_proj", + "model.layers.11.mlp.experts.70.down_proj", + "model.layers.11.mlp.experts.70.gate_proj", + "model.layers.11.mlp.experts.70.up_proj", + "model.layers.11.mlp.experts.71.down_proj", + "model.layers.11.mlp.experts.71.gate_proj", + "model.layers.11.mlp.experts.71.up_proj", + "model.layers.11.mlp.experts.72.down_proj", + "model.layers.11.mlp.experts.72.gate_proj", + "model.layers.11.mlp.experts.72.up_proj", + "model.layers.11.mlp.experts.73.down_proj", + "model.layers.11.mlp.experts.73.gate_proj", + "model.layers.11.mlp.experts.73.up_proj", + "model.layers.11.mlp.experts.74.down_proj", + "model.layers.11.mlp.experts.74.gate_proj", + "model.layers.11.mlp.experts.74.up_proj", + "model.layers.11.mlp.experts.75.down_proj", + "model.layers.11.mlp.experts.75.gate_proj", + "model.layers.11.mlp.experts.75.up_proj", + "model.layers.11.mlp.experts.76.down_proj", + "model.layers.11.mlp.experts.76.gate_proj", + "model.layers.11.mlp.experts.76.up_proj", + "model.layers.11.mlp.experts.77.down_proj", + "model.layers.11.mlp.experts.77.gate_proj", + "model.layers.11.mlp.experts.77.up_proj", + "model.layers.11.mlp.experts.78.down_proj", + "model.layers.11.mlp.experts.78.gate_proj", + "model.layers.11.mlp.experts.78.up_proj", + "model.layers.11.mlp.experts.79.down_proj", + "model.layers.11.mlp.experts.79.gate_proj", + "model.layers.11.mlp.experts.79.up_proj", + "model.layers.11.mlp.experts.8.down_proj", + "model.layers.11.mlp.experts.8.gate_proj", + "model.layers.11.mlp.experts.8.up_proj", + "model.layers.11.mlp.experts.80.down_proj", + "model.layers.11.mlp.experts.80.gate_proj", + "model.layers.11.mlp.experts.80.up_proj", + "model.layers.11.mlp.experts.81.down_proj", + "model.layers.11.mlp.experts.81.gate_proj", + "model.layers.11.mlp.experts.81.up_proj", + "model.layers.11.mlp.experts.82.down_proj", + "model.layers.11.mlp.experts.82.gate_proj", + "model.layers.11.mlp.experts.82.up_proj", + "model.layers.11.mlp.experts.83.down_proj", + "model.layers.11.mlp.experts.83.gate_proj", + "model.layers.11.mlp.experts.83.up_proj", + "model.layers.11.mlp.experts.84.down_proj", + "model.layers.11.mlp.experts.84.gate_proj", + "model.layers.11.mlp.experts.84.up_proj", + "model.layers.11.mlp.experts.85.down_proj", + "model.layers.11.mlp.experts.85.gate_proj", + "model.layers.11.mlp.experts.85.up_proj", + "model.layers.11.mlp.experts.86.down_proj", + "model.layers.11.mlp.experts.86.gate_proj", + "model.layers.11.mlp.experts.86.up_proj", + "model.layers.11.mlp.experts.87.down_proj", + "model.layers.11.mlp.experts.87.gate_proj", + "model.layers.11.mlp.experts.87.up_proj", + "model.layers.11.mlp.experts.88.down_proj", + "model.layers.11.mlp.experts.88.gate_proj", + "model.layers.11.mlp.experts.88.up_proj", + "model.layers.11.mlp.experts.89.down_proj", + "model.layers.11.mlp.experts.89.gate_proj", + "model.layers.11.mlp.experts.89.up_proj", + "model.layers.11.mlp.experts.9.down_proj", + "model.layers.11.mlp.experts.9.gate_proj", + "model.layers.11.mlp.experts.9.up_proj", + "model.layers.11.mlp.experts.90.down_proj", + "model.layers.11.mlp.experts.90.gate_proj", + "model.layers.11.mlp.experts.90.up_proj", + "model.layers.11.mlp.experts.91.down_proj", + "model.layers.11.mlp.experts.91.gate_proj", + "model.layers.11.mlp.experts.91.up_proj", + "model.layers.11.mlp.experts.92.down_proj", + "model.layers.11.mlp.experts.92.gate_proj", + "model.layers.11.mlp.experts.92.up_proj", + "model.layers.11.mlp.experts.93.down_proj", + "model.layers.11.mlp.experts.93.gate_proj", + "model.layers.11.mlp.experts.93.up_proj", + "model.layers.11.mlp.experts.94.down_proj", + "model.layers.11.mlp.experts.94.gate_proj", + "model.layers.11.mlp.experts.94.up_proj", + "model.layers.11.mlp.experts.95.down_proj", + "model.layers.11.mlp.experts.95.gate_proj", + "model.layers.11.mlp.experts.95.up_proj", + "model.layers.11.mlp.experts.96.down_proj", + "model.layers.11.mlp.experts.96.gate_proj", + "model.layers.11.mlp.experts.96.up_proj", + "model.layers.11.mlp.experts.97.down_proj", + "model.layers.11.mlp.experts.97.gate_proj", + "model.layers.11.mlp.experts.97.up_proj", + "model.layers.11.mlp.experts.98.down_proj", + "model.layers.11.mlp.experts.98.gate_proj", + "model.layers.11.mlp.experts.98.up_proj", + "model.layers.11.mlp.experts.99.down_proj", + "model.layers.11.mlp.experts.99.gate_proj", + "model.layers.11.mlp.experts.99.up_proj", + "model.layers.12.mlp.experts.0.down_proj", + "model.layers.12.mlp.experts.0.gate_proj", + "model.layers.12.mlp.experts.0.up_proj", + "model.layers.12.mlp.experts.1.down_proj", + "model.layers.12.mlp.experts.1.gate_proj", + "model.layers.12.mlp.experts.1.up_proj", + "model.layers.12.mlp.experts.10.down_proj", + "model.layers.12.mlp.experts.10.gate_proj", + "model.layers.12.mlp.experts.10.up_proj", + "model.layers.12.mlp.experts.100.down_proj", + "model.layers.12.mlp.experts.100.gate_proj", + "model.layers.12.mlp.experts.100.up_proj", + "model.layers.12.mlp.experts.101.down_proj", + "model.layers.12.mlp.experts.101.gate_proj", + "model.layers.12.mlp.experts.101.up_proj", + "model.layers.12.mlp.experts.102.down_proj", + "model.layers.12.mlp.experts.102.gate_proj", + "model.layers.12.mlp.experts.102.up_proj", + "model.layers.12.mlp.experts.103.down_proj", + "model.layers.12.mlp.experts.103.gate_proj", + "model.layers.12.mlp.experts.103.up_proj", + "model.layers.12.mlp.experts.104.down_proj", + "model.layers.12.mlp.experts.104.gate_proj", + "model.layers.12.mlp.experts.104.up_proj", + "model.layers.12.mlp.experts.105.down_proj", + "model.layers.12.mlp.experts.105.gate_proj", + "model.layers.12.mlp.experts.105.up_proj", + "model.layers.12.mlp.experts.106.down_proj", + "model.layers.12.mlp.experts.106.gate_proj", + "model.layers.12.mlp.experts.106.up_proj", + "model.layers.12.mlp.experts.107.down_proj", + "model.layers.12.mlp.experts.107.gate_proj", + "model.layers.12.mlp.experts.107.up_proj", + "model.layers.12.mlp.experts.108.down_proj", + "model.layers.12.mlp.experts.108.gate_proj", + "model.layers.12.mlp.experts.108.up_proj", + "model.layers.12.mlp.experts.109.down_proj", + "model.layers.12.mlp.experts.109.gate_proj", + "model.layers.12.mlp.experts.109.up_proj", + "model.layers.12.mlp.experts.11.down_proj", + "model.layers.12.mlp.experts.11.gate_proj", + "model.layers.12.mlp.experts.11.up_proj", + "model.layers.12.mlp.experts.110.down_proj", + "model.layers.12.mlp.experts.110.gate_proj", + "model.layers.12.mlp.experts.110.up_proj", + "model.layers.12.mlp.experts.111.down_proj", + "model.layers.12.mlp.experts.111.gate_proj", + "model.layers.12.mlp.experts.111.up_proj", + "model.layers.12.mlp.experts.112.down_proj", + "model.layers.12.mlp.experts.112.gate_proj", + "model.layers.12.mlp.experts.112.up_proj", + "model.layers.12.mlp.experts.113.down_proj", + "model.layers.12.mlp.experts.113.gate_proj", + "model.layers.12.mlp.experts.113.up_proj", + "model.layers.12.mlp.experts.114.down_proj", + "model.layers.12.mlp.experts.114.gate_proj", + "model.layers.12.mlp.experts.114.up_proj", + "model.layers.12.mlp.experts.115.down_proj", + "model.layers.12.mlp.experts.115.gate_proj", + "model.layers.12.mlp.experts.115.up_proj", + "model.layers.12.mlp.experts.116.down_proj", + "model.layers.12.mlp.experts.116.gate_proj", + "model.layers.12.mlp.experts.116.up_proj", + "model.layers.12.mlp.experts.117.down_proj", + "model.layers.12.mlp.experts.117.gate_proj", + "model.layers.12.mlp.experts.117.up_proj", + "model.layers.12.mlp.experts.118.down_proj", + "model.layers.12.mlp.experts.118.gate_proj", + "model.layers.12.mlp.experts.118.up_proj", + "model.layers.12.mlp.experts.119.down_proj", + "model.layers.12.mlp.experts.119.gate_proj", + "model.layers.12.mlp.experts.119.up_proj", + "model.layers.12.mlp.experts.12.down_proj", + "model.layers.12.mlp.experts.12.gate_proj", + "model.layers.12.mlp.experts.12.up_proj", + "model.layers.12.mlp.experts.120.down_proj", + "model.layers.12.mlp.experts.120.gate_proj", + "model.layers.12.mlp.experts.120.up_proj", + "model.layers.12.mlp.experts.121.down_proj", + "model.layers.12.mlp.experts.121.gate_proj", + "model.layers.12.mlp.experts.121.up_proj", + "model.layers.12.mlp.experts.122.down_proj", + "model.layers.12.mlp.experts.122.gate_proj", + "model.layers.12.mlp.experts.122.up_proj", + "model.layers.12.mlp.experts.123.down_proj", + "model.layers.12.mlp.experts.123.gate_proj", + "model.layers.12.mlp.experts.123.up_proj", + "model.layers.12.mlp.experts.124.down_proj", + "model.layers.12.mlp.experts.124.gate_proj", + "model.layers.12.mlp.experts.124.up_proj", + "model.layers.12.mlp.experts.125.down_proj", + "model.layers.12.mlp.experts.125.gate_proj", + "model.layers.12.mlp.experts.125.up_proj", + "model.layers.12.mlp.experts.126.down_proj", + "model.layers.12.mlp.experts.126.gate_proj", + "model.layers.12.mlp.experts.126.up_proj", + "model.layers.12.mlp.experts.127.down_proj", + "model.layers.12.mlp.experts.127.gate_proj", + "model.layers.12.mlp.experts.127.up_proj", + "model.layers.12.mlp.experts.128.down_proj", + "model.layers.12.mlp.experts.128.gate_proj", + "model.layers.12.mlp.experts.128.up_proj", + "model.layers.12.mlp.experts.129.down_proj", + "model.layers.12.mlp.experts.129.gate_proj", + "model.layers.12.mlp.experts.129.up_proj", + "model.layers.12.mlp.experts.13.down_proj", + "model.layers.12.mlp.experts.13.gate_proj", + "model.layers.12.mlp.experts.13.up_proj", + "model.layers.12.mlp.experts.130.down_proj", + "model.layers.12.mlp.experts.130.gate_proj", + "model.layers.12.mlp.experts.130.up_proj", + "model.layers.12.mlp.experts.131.down_proj", + "model.layers.12.mlp.experts.131.gate_proj", + "model.layers.12.mlp.experts.131.up_proj", + "model.layers.12.mlp.experts.132.down_proj", + "model.layers.12.mlp.experts.132.gate_proj", + "model.layers.12.mlp.experts.132.up_proj", + "model.layers.12.mlp.experts.133.down_proj", + "model.layers.12.mlp.experts.133.gate_proj", + "model.layers.12.mlp.experts.133.up_proj", + "model.layers.12.mlp.experts.134.down_proj", + "model.layers.12.mlp.experts.134.gate_proj", + "model.layers.12.mlp.experts.134.up_proj", + "model.layers.12.mlp.experts.135.down_proj", + "model.layers.12.mlp.experts.135.gate_proj", + "model.layers.12.mlp.experts.135.up_proj", + "model.layers.12.mlp.experts.136.down_proj", + "model.layers.12.mlp.experts.136.gate_proj", + "model.layers.12.mlp.experts.136.up_proj", + "model.layers.12.mlp.experts.137.down_proj", + "model.layers.12.mlp.experts.137.gate_proj", + "model.layers.12.mlp.experts.137.up_proj", + "model.layers.12.mlp.experts.138.down_proj", + "model.layers.12.mlp.experts.138.gate_proj", + "model.layers.12.mlp.experts.138.up_proj", + "model.layers.12.mlp.experts.139.down_proj", + "model.layers.12.mlp.experts.139.gate_proj", + "model.layers.12.mlp.experts.139.up_proj", + "model.layers.12.mlp.experts.14.down_proj", + "model.layers.12.mlp.experts.14.gate_proj", + "model.layers.12.mlp.experts.14.up_proj", + "model.layers.12.mlp.experts.140.down_proj", + "model.layers.12.mlp.experts.140.gate_proj", + "model.layers.12.mlp.experts.140.up_proj", + "model.layers.12.mlp.experts.141.down_proj", + "model.layers.12.mlp.experts.141.gate_proj", + "model.layers.12.mlp.experts.141.up_proj", + "model.layers.12.mlp.experts.142.down_proj", + "model.layers.12.mlp.experts.142.gate_proj", + "model.layers.12.mlp.experts.142.up_proj", + "model.layers.12.mlp.experts.143.down_proj", + "model.layers.12.mlp.experts.143.gate_proj", + "model.layers.12.mlp.experts.143.up_proj", + "model.layers.12.mlp.experts.144.down_proj", + "model.layers.12.mlp.experts.144.gate_proj", + "model.layers.12.mlp.experts.144.up_proj", + "model.layers.12.mlp.experts.145.down_proj", + "model.layers.12.mlp.experts.145.gate_proj", + "model.layers.12.mlp.experts.145.up_proj", + "model.layers.12.mlp.experts.146.down_proj", + "model.layers.12.mlp.experts.146.gate_proj", + "model.layers.12.mlp.experts.146.up_proj", + "model.layers.12.mlp.experts.147.down_proj", + "model.layers.12.mlp.experts.147.gate_proj", + "model.layers.12.mlp.experts.147.up_proj", + "model.layers.12.mlp.experts.148.down_proj", + "model.layers.12.mlp.experts.148.gate_proj", + "model.layers.12.mlp.experts.148.up_proj", + "model.layers.12.mlp.experts.149.down_proj", + "model.layers.12.mlp.experts.149.gate_proj", + "model.layers.12.mlp.experts.149.up_proj", + "model.layers.12.mlp.experts.15.down_proj", + "model.layers.12.mlp.experts.15.gate_proj", + "model.layers.12.mlp.experts.15.up_proj", + "model.layers.12.mlp.experts.150.down_proj", + "model.layers.12.mlp.experts.150.gate_proj", + "model.layers.12.mlp.experts.150.up_proj", + "model.layers.12.mlp.experts.151.down_proj", + "model.layers.12.mlp.experts.151.gate_proj", + "model.layers.12.mlp.experts.151.up_proj", + "model.layers.12.mlp.experts.152.down_proj", + "model.layers.12.mlp.experts.152.gate_proj", + "model.layers.12.mlp.experts.152.up_proj", + "model.layers.12.mlp.experts.153.down_proj", + "model.layers.12.mlp.experts.153.gate_proj", + "model.layers.12.mlp.experts.153.up_proj", + "model.layers.12.mlp.experts.154.down_proj", + "model.layers.12.mlp.experts.154.gate_proj", + "model.layers.12.mlp.experts.154.up_proj", + "model.layers.12.mlp.experts.155.down_proj", + "model.layers.12.mlp.experts.155.gate_proj", + "model.layers.12.mlp.experts.155.up_proj", + "model.layers.12.mlp.experts.156.down_proj", + "model.layers.12.mlp.experts.156.gate_proj", + "model.layers.12.mlp.experts.156.up_proj", + "model.layers.12.mlp.experts.157.down_proj", + "model.layers.12.mlp.experts.157.gate_proj", + "model.layers.12.mlp.experts.157.up_proj", + "model.layers.12.mlp.experts.158.down_proj", + "model.layers.12.mlp.experts.158.gate_proj", + "model.layers.12.mlp.experts.158.up_proj", + "model.layers.12.mlp.experts.159.down_proj", + "model.layers.12.mlp.experts.159.gate_proj", + "model.layers.12.mlp.experts.159.up_proj", + "model.layers.12.mlp.experts.16.down_proj", + "model.layers.12.mlp.experts.16.gate_proj", + "model.layers.12.mlp.experts.16.up_proj", + "model.layers.12.mlp.experts.160.down_proj", + "model.layers.12.mlp.experts.160.gate_proj", + "model.layers.12.mlp.experts.160.up_proj", + "model.layers.12.mlp.experts.161.down_proj", + "model.layers.12.mlp.experts.161.gate_proj", + "model.layers.12.mlp.experts.161.up_proj", + "model.layers.12.mlp.experts.162.down_proj", + "model.layers.12.mlp.experts.162.gate_proj", + "model.layers.12.mlp.experts.162.up_proj", + "model.layers.12.mlp.experts.163.down_proj", + "model.layers.12.mlp.experts.163.gate_proj", + "model.layers.12.mlp.experts.163.up_proj", + "model.layers.12.mlp.experts.164.down_proj", + "model.layers.12.mlp.experts.164.gate_proj", + "model.layers.12.mlp.experts.164.up_proj", + "model.layers.12.mlp.experts.165.down_proj", + "model.layers.12.mlp.experts.165.gate_proj", + "model.layers.12.mlp.experts.165.up_proj", + "model.layers.12.mlp.experts.166.down_proj", + "model.layers.12.mlp.experts.166.gate_proj", + "model.layers.12.mlp.experts.166.up_proj", + "model.layers.12.mlp.experts.167.down_proj", + "model.layers.12.mlp.experts.167.gate_proj", + "model.layers.12.mlp.experts.167.up_proj", + "model.layers.12.mlp.experts.168.down_proj", + "model.layers.12.mlp.experts.168.gate_proj", + "model.layers.12.mlp.experts.168.up_proj", + "model.layers.12.mlp.experts.169.down_proj", + "model.layers.12.mlp.experts.169.gate_proj", + "model.layers.12.mlp.experts.169.up_proj", + "model.layers.12.mlp.experts.17.down_proj", + "model.layers.12.mlp.experts.17.gate_proj", + "model.layers.12.mlp.experts.17.up_proj", + "model.layers.12.mlp.experts.170.down_proj", + "model.layers.12.mlp.experts.170.gate_proj", + "model.layers.12.mlp.experts.170.up_proj", + "model.layers.12.mlp.experts.171.down_proj", + "model.layers.12.mlp.experts.171.gate_proj", + "model.layers.12.mlp.experts.171.up_proj", + "model.layers.12.mlp.experts.172.down_proj", + "model.layers.12.mlp.experts.172.gate_proj", + "model.layers.12.mlp.experts.172.up_proj", + "model.layers.12.mlp.experts.173.down_proj", + "model.layers.12.mlp.experts.173.gate_proj", + "model.layers.12.mlp.experts.173.up_proj", + "model.layers.12.mlp.experts.174.down_proj", + "model.layers.12.mlp.experts.174.gate_proj", + "model.layers.12.mlp.experts.174.up_proj", + "model.layers.12.mlp.experts.175.down_proj", + "model.layers.12.mlp.experts.175.gate_proj", + "model.layers.12.mlp.experts.175.up_proj", + "model.layers.12.mlp.experts.176.down_proj", + "model.layers.12.mlp.experts.176.gate_proj", + "model.layers.12.mlp.experts.176.up_proj", + "model.layers.12.mlp.experts.177.down_proj", + "model.layers.12.mlp.experts.177.gate_proj", + "model.layers.12.mlp.experts.177.up_proj", + "model.layers.12.mlp.experts.178.down_proj", + "model.layers.12.mlp.experts.178.gate_proj", + "model.layers.12.mlp.experts.178.up_proj", + "model.layers.12.mlp.experts.179.down_proj", + "model.layers.12.mlp.experts.179.gate_proj", + "model.layers.12.mlp.experts.179.up_proj", + "model.layers.12.mlp.experts.18.down_proj", + "model.layers.12.mlp.experts.18.gate_proj", + "model.layers.12.mlp.experts.18.up_proj", + "model.layers.12.mlp.experts.180.down_proj", + "model.layers.12.mlp.experts.180.gate_proj", + "model.layers.12.mlp.experts.180.up_proj", + "model.layers.12.mlp.experts.181.down_proj", + "model.layers.12.mlp.experts.181.gate_proj", + "model.layers.12.mlp.experts.181.up_proj", + "model.layers.12.mlp.experts.182.down_proj", + "model.layers.12.mlp.experts.182.gate_proj", + "model.layers.12.mlp.experts.182.up_proj", + "model.layers.12.mlp.experts.183.down_proj", + "model.layers.12.mlp.experts.183.gate_proj", + "model.layers.12.mlp.experts.183.up_proj", + "model.layers.12.mlp.experts.184.down_proj", + "model.layers.12.mlp.experts.184.gate_proj", + "model.layers.12.mlp.experts.184.up_proj", + "model.layers.12.mlp.experts.185.down_proj", + "model.layers.12.mlp.experts.185.gate_proj", + "model.layers.12.mlp.experts.185.up_proj", + "model.layers.12.mlp.experts.186.down_proj", + "model.layers.12.mlp.experts.186.gate_proj", + "model.layers.12.mlp.experts.186.up_proj", + "model.layers.12.mlp.experts.187.down_proj", + "model.layers.12.mlp.experts.187.gate_proj", + "model.layers.12.mlp.experts.187.up_proj", + "model.layers.12.mlp.experts.188.down_proj", + "model.layers.12.mlp.experts.188.gate_proj", + "model.layers.12.mlp.experts.188.up_proj", + "model.layers.12.mlp.experts.189.down_proj", + "model.layers.12.mlp.experts.189.gate_proj", + "model.layers.12.mlp.experts.189.up_proj", + "model.layers.12.mlp.experts.19.down_proj", + "model.layers.12.mlp.experts.19.gate_proj", + "model.layers.12.mlp.experts.19.up_proj", + "model.layers.12.mlp.experts.190.down_proj", + "model.layers.12.mlp.experts.190.gate_proj", + "model.layers.12.mlp.experts.190.up_proj", + "model.layers.12.mlp.experts.191.down_proj", + "model.layers.12.mlp.experts.191.gate_proj", + "model.layers.12.mlp.experts.191.up_proj", + "model.layers.12.mlp.experts.192.down_proj", + "model.layers.12.mlp.experts.192.gate_proj", + "model.layers.12.mlp.experts.192.up_proj", + "model.layers.12.mlp.experts.193.down_proj", + "model.layers.12.mlp.experts.193.gate_proj", + "model.layers.12.mlp.experts.193.up_proj", + "model.layers.12.mlp.experts.194.down_proj", + "model.layers.12.mlp.experts.194.gate_proj", + "model.layers.12.mlp.experts.194.up_proj", + "model.layers.12.mlp.experts.195.down_proj", + "model.layers.12.mlp.experts.195.gate_proj", + "model.layers.12.mlp.experts.195.up_proj", + "model.layers.12.mlp.experts.196.down_proj", + "model.layers.12.mlp.experts.196.gate_proj", + "model.layers.12.mlp.experts.196.up_proj", + "model.layers.12.mlp.experts.197.down_proj", + "model.layers.12.mlp.experts.197.gate_proj", + "model.layers.12.mlp.experts.197.up_proj", + "model.layers.12.mlp.experts.198.down_proj", + "model.layers.12.mlp.experts.198.gate_proj", + "model.layers.12.mlp.experts.198.up_proj", + "model.layers.12.mlp.experts.199.down_proj", + "model.layers.12.mlp.experts.199.gate_proj", + "model.layers.12.mlp.experts.199.up_proj", + "model.layers.12.mlp.experts.2.down_proj", + "model.layers.12.mlp.experts.2.gate_proj", + "model.layers.12.mlp.experts.2.up_proj", + "model.layers.12.mlp.experts.20.down_proj", + "model.layers.12.mlp.experts.20.gate_proj", + "model.layers.12.mlp.experts.20.up_proj", + "model.layers.12.mlp.experts.200.down_proj", + "model.layers.12.mlp.experts.200.gate_proj", + "model.layers.12.mlp.experts.200.up_proj", + "model.layers.12.mlp.experts.201.down_proj", + "model.layers.12.mlp.experts.201.gate_proj", + "model.layers.12.mlp.experts.201.up_proj", + "model.layers.12.mlp.experts.202.down_proj", + "model.layers.12.mlp.experts.202.gate_proj", + "model.layers.12.mlp.experts.202.up_proj", + "model.layers.12.mlp.experts.203.down_proj", + "model.layers.12.mlp.experts.203.gate_proj", + "model.layers.12.mlp.experts.203.up_proj", + "model.layers.12.mlp.experts.204.down_proj", + "model.layers.12.mlp.experts.204.gate_proj", + "model.layers.12.mlp.experts.204.up_proj", + "model.layers.12.mlp.experts.205.down_proj", + "model.layers.12.mlp.experts.205.gate_proj", + "model.layers.12.mlp.experts.205.up_proj", + "model.layers.12.mlp.experts.206.down_proj", + "model.layers.12.mlp.experts.206.gate_proj", + "model.layers.12.mlp.experts.206.up_proj", + "model.layers.12.mlp.experts.207.down_proj", + "model.layers.12.mlp.experts.207.gate_proj", + "model.layers.12.mlp.experts.207.up_proj", + "model.layers.12.mlp.experts.208.down_proj", + "model.layers.12.mlp.experts.208.gate_proj", + "model.layers.12.mlp.experts.208.up_proj", + "model.layers.12.mlp.experts.209.down_proj", + "model.layers.12.mlp.experts.209.gate_proj", + "model.layers.12.mlp.experts.209.up_proj", + "model.layers.12.mlp.experts.21.down_proj", + "model.layers.12.mlp.experts.21.gate_proj", + "model.layers.12.mlp.experts.21.up_proj", + "model.layers.12.mlp.experts.210.down_proj", + "model.layers.12.mlp.experts.210.gate_proj", + "model.layers.12.mlp.experts.210.up_proj", + "model.layers.12.mlp.experts.211.down_proj", + "model.layers.12.mlp.experts.211.gate_proj", + "model.layers.12.mlp.experts.211.up_proj", + "model.layers.12.mlp.experts.212.down_proj", + "model.layers.12.mlp.experts.212.gate_proj", + "model.layers.12.mlp.experts.212.up_proj", + "model.layers.12.mlp.experts.213.down_proj", + "model.layers.12.mlp.experts.213.gate_proj", + "model.layers.12.mlp.experts.213.up_proj", + "model.layers.12.mlp.experts.214.down_proj", + "model.layers.12.mlp.experts.214.gate_proj", + "model.layers.12.mlp.experts.214.up_proj", + "model.layers.12.mlp.experts.215.down_proj", + "model.layers.12.mlp.experts.215.gate_proj", + "model.layers.12.mlp.experts.215.up_proj", + "model.layers.12.mlp.experts.216.down_proj", + "model.layers.12.mlp.experts.216.gate_proj", + "model.layers.12.mlp.experts.216.up_proj", + "model.layers.12.mlp.experts.217.down_proj", + "model.layers.12.mlp.experts.217.gate_proj", + "model.layers.12.mlp.experts.217.up_proj", + "model.layers.12.mlp.experts.218.down_proj", + "model.layers.12.mlp.experts.218.gate_proj", + "model.layers.12.mlp.experts.218.up_proj", + "model.layers.12.mlp.experts.219.down_proj", + "model.layers.12.mlp.experts.219.gate_proj", + "model.layers.12.mlp.experts.219.up_proj", + "model.layers.12.mlp.experts.22.down_proj", + "model.layers.12.mlp.experts.22.gate_proj", + "model.layers.12.mlp.experts.22.up_proj", + "model.layers.12.mlp.experts.220.down_proj", + "model.layers.12.mlp.experts.220.gate_proj", + "model.layers.12.mlp.experts.220.up_proj", + "model.layers.12.mlp.experts.221.down_proj", + "model.layers.12.mlp.experts.221.gate_proj", + "model.layers.12.mlp.experts.221.up_proj", + "model.layers.12.mlp.experts.222.down_proj", + "model.layers.12.mlp.experts.222.gate_proj", + "model.layers.12.mlp.experts.222.up_proj", + "model.layers.12.mlp.experts.223.down_proj", + "model.layers.12.mlp.experts.223.gate_proj", + "model.layers.12.mlp.experts.223.up_proj", + "model.layers.12.mlp.experts.224.down_proj", + "model.layers.12.mlp.experts.224.gate_proj", + "model.layers.12.mlp.experts.224.up_proj", + "model.layers.12.mlp.experts.225.down_proj", + "model.layers.12.mlp.experts.225.gate_proj", + "model.layers.12.mlp.experts.225.up_proj", + "model.layers.12.mlp.experts.226.down_proj", + "model.layers.12.mlp.experts.226.gate_proj", + "model.layers.12.mlp.experts.226.up_proj", + "model.layers.12.mlp.experts.227.down_proj", + "model.layers.12.mlp.experts.227.gate_proj", + "model.layers.12.mlp.experts.227.up_proj", + "model.layers.12.mlp.experts.228.down_proj", + "model.layers.12.mlp.experts.228.gate_proj", + "model.layers.12.mlp.experts.228.up_proj", + "model.layers.12.mlp.experts.229.down_proj", + "model.layers.12.mlp.experts.229.gate_proj", + "model.layers.12.mlp.experts.229.up_proj", + "model.layers.12.mlp.experts.23.down_proj", + "model.layers.12.mlp.experts.23.gate_proj", + "model.layers.12.mlp.experts.23.up_proj", + "model.layers.12.mlp.experts.230.down_proj", + "model.layers.12.mlp.experts.230.gate_proj", + "model.layers.12.mlp.experts.230.up_proj", + "model.layers.12.mlp.experts.231.down_proj", + "model.layers.12.mlp.experts.231.gate_proj", + "model.layers.12.mlp.experts.231.up_proj", + "model.layers.12.mlp.experts.232.down_proj", + "model.layers.12.mlp.experts.232.gate_proj", + "model.layers.12.mlp.experts.232.up_proj", + "model.layers.12.mlp.experts.233.down_proj", + "model.layers.12.mlp.experts.233.gate_proj", + "model.layers.12.mlp.experts.233.up_proj", + "model.layers.12.mlp.experts.234.down_proj", + "model.layers.12.mlp.experts.234.gate_proj", + "model.layers.12.mlp.experts.234.up_proj", + "model.layers.12.mlp.experts.235.down_proj", + "model.layers.12.mlp.experts.235.gate_proj", + "model.layers.12.mlp.experts.235.up_proj", + "model.layers.12.mlp.experts.236.down_proj", + "model.layers.12.mlp.experts.236.gate_proj", + "model.layers.12.mlp.experts.236.up_proj", + "model.layers.12.mlp.experts.237.down_proj", + "model.layers.12.mlp.experts.237.gate_proj", + "model.layers.12.mlp.experts.237.up_proj", + "model.layers.12.mlp.experts.238.down_proj", + "model.layers.12.mlp.experts.238.gate_proj", + "model.layers.12.mlp.experts.238.up_proj", + "model.layers.12.mlp.experts.239.down_proj", + "model.layers.12.mlp.experts.239.gate_proj", + "model.layers.12.mlp.experts.239.up_proj", + "model.layers.12.mlp.experts.24.down_proj", + "model.layers.12.mlp.experts.24.gate_proj", + "model.layers.12.mlp.experts.24.up_proj", + "model.layers.12.mlp.experts.240.down_proj", + "model.layers.12.mlp.experts.240.gate_proj", + "model.layers.12.mlp.experts.240.up_proj", + "model.layers.12.mlp.experts.241.down_proj", + "model.layers.12.mlp.experts.241.gate_proj", + "model.layers.12.mlp.experts.241.up_proj", + "model.layers.12.mlp.experts.242.down_proj", + "model.layers.12.mlp.experts.242.gate_proj", + "model.layers.12.mlp.experts.242.up_proj", + "model.layers.12.mlp.experts.243.down_proj", + "model.layers.12.mlp.experts.243.gate_proj", + "model.layers.12.mlp.experts.243.up_proj", + "model.layers.12.mlp.experts.244.down_proj", + "model.layers.12.mlp.experts.244.gate_proj", + "model.layers.12.mlp.experts.244.up_proj", + "model.layers.12.mlp.experts.245.down_proj", + "model.layers.12.mlp.experts.245.gate_proj", + "model.layers.12.mlp.experts.245.up_proj", + "model.layers.12.mlp.experts.246.down_proj", + "model.layers.12.mlp.experts.246.gate_proj", + "model.layers.12.mlp.experts.246.up_proj", + "model.layers.12.mlp.experts.247.down_proj", + "model.layers.12.mlp.experts.247.gate_proj", + "model.layers.12.mlp.experts.247.up_proj", + "model.layers.12.mlp.experts.248.down_proj", + "model.layers.12.mlp.experts.248.gate_proj", + "model.layers.12.mlp.experts.248.up_proj", + "model.layers.12.mlp.experts.249.down_proj", + "model.layers.12.mlp.experts.249.gate_proj", + "model.layers.12.mlp.experts.249.up_proj", + "model.layers.12.mlp.experts.25.down_proj", + "model.layers.12.mlp.experts.25.gate_proj", + "model.layers.12.mlp.experts.25.up_proj", + "model.layers.12.mlp.experts.250.down_proj", + "model.layers.12.mlp.experts.250.gate_proj", + "model.layers.12.mlp.experts.250.up_proj", + "model.layers.12.mlp.experts.251.down_proj", + "model.layers.12.mlp.experts.251.gate_proj", + "model.layers.12.mlp.experts.251.up_proj", + "model.layers.12.mlp.experts.252.down_proj", + "model.layers.12.mlp.experts.252.gate_proj", + "model.layers.12.mlp.experts.252.up_proj", + "model.layers.12.mlp.experts.253.down_proj", + "model.layers.12.mlp.experts.253.gate_proj", + "model.layers.12.mlp.experts.253.up_proj", + "model.layers.12.mlp.experts.254.down_proj", + "model.layers.12.mlp.experts.254.gate_proj", + "model.layers.12.mlp.experts.254.up_proj", + "model.layers.12.mlp.experts.255.down_proj", + "model.layers.12.mlp.experts.255.gate_proj", + "model.layers.12.mlp.experts.255.up_proj", + "model.layers.12.mlp.experts.26.down_proj", + "model.layers.12.mlp.experts.26.gate_proj", + "model.layers.12.mlp.experts.26.up_proj", + "model.layers.12.mlp.experts.27.down_proj", + "model.layers.12.mlp.experts.27.gate_proj", + "model.layers.12.mlp.experts.27.up_proj", + "model.layers.12.mlp.experts.28.down_proj", + "model.layers.12.mlp.experts.28.gate_proj", + "model.layers.12.mlp.experts.28.up_proj", + "model.layers.12.mlp.experts.29.down_proj", + "model.layers.12.mlp.experts.29.gate_proj", + "model.layers.12.mlp.experts.29.up_proj", + "model.layers.12.mlp.experts.3.down_proj", + "model.layers.12.mlp.experts.3.gate_proj", + "model.layers.12.mlp.experts.3.up_proj", + "model.layers.12.mlp.experts.30.down_proj", + "model.layers.12.mlp.experts.30.gate_proj", + "model.layers.12.mlp.experts.30.up_proj", + "model.layers.12.mlp.experts.31.down_proj", + "model.layers.12.mlp.experts.31.gate_proj", + "model.layers.12.mlp.experts.31.up_proj", + "model.layers.12.mlp.experts.32.down_proj", + "model.layers.12.mlp.experts.32.gate_proj", + "model.layers.12.mlp.experts.32.up_proj", + "model.layers.12.mlp.experts.33.down_proj", + "model.layers.12.mlp.experts.33.gate_proj", + "model.layers.12.mlp.experts.33.up_proj", + "model.layers.12.mlp.experts.34.down_proj", + "model.layers.12.mlp.experts.34.gate_proj", + "model.layers.12.mlp.experts.34.up_proj", + "model.layers.12.mlp.experts.35.down_proj", + "model.layers.12.mlp.experts.35.gate_proj", + "model.layers.12.mlp.experts.35.up_proj", + "model.layers.12.mlp.experts.36.down_proj", + "model.layers.12.mlp.experts.36.gate_proj", + "model.layers.12.mlp.experts.36.up_proj", + "model.layers.12.mlp.experts.37.down_proj", + "model.layers.12.mlp.experts.37.gate_proj", + "model.layers.12.mlp.experts.37.up_proj", + "model.layers.12.mlp.experts.38.down_proj", + "model.layers.12.mlp.experts.38.gate_proj", + "model.layers.12.mlp.experts.38.up_proj", + "model.layers.12.mlp.experts.39.down_proj", + "model.layers.12.mlp.experts.39.gate_proj", + "model.layers.12.mlp.experts.39.up_proj", + "model.layers.12.mlp.experts.4.down_proj", + "model.layers.12.mlp.experts.4.gate_proj", + "model.layers.12.mlp.experts.4.up_proj", + "model.layers.12.mlp.experts.40.down_proj", + "model.layers.12.mlp.experts.40.gate_proj", + "model.layers.12.mlp.experts.40.up_proj", + "model.layers.12.mlp.experts.41.down_proj", + "model.layers.12.mlp.experts.41.gate_proj", + "model.layers.12.mlp.experts.41.up_proj", + "model.layers.12.mlp.experts.42.down_proj", + "model.layers.12.mlp.experts.42.gate_proj", + "model.layers.12.mlp.experts.42.up_proj", + "model.layers.12.mlp.experts.43.down_proj", + "model.layers.12.mlp.experts.43.gate_proj", + "model.layers.12.mlp.experts.43.up_proj", + "model.layers.12.mlp.experts.44.down_proj", + "model.layers.12.mlp.experts.44.gate_proj", + "model.layers.12.mlp.experts.44.up_proj", + "model.layers.12.mlp.experts.45.down_proj", + "model.layers.12.mlp.experts.45.gate_proj", + "model.layers.12.mlp.experts.45.up_proj", + "model.layers.12.mlp.experts.46.down_proj", + "model.layers.12.mlp.experts.46.gate_proj", + "model.layers.12.mlp.experts.46.up_proj", + "model.layers.12.mlp.experts.47.down_proj", + "model.layers.12.mlp.experts.47.gate_proj", + "model.layers.12.mlp.experts.47.up_proj", + "model.layers.12.mlp.experts.48.down_proj", + "model.layers.12.mlp.experts.48.gate_proj", + "model.layers.12.mlp.experts.48.up_proj", + "model.layers.12.mlp.experts.49.down_proj", + "model.layers.12.mlp.experts.49.gate_proj", + "model.layers.12.mlp.experts.49.up_proj", + "model.layers.12.mlp.experts.5.down_proj", + "model.layers.12.mlp.experts.5.gate_proj", + "model.layers.12.mlp.experts.5.up_proj", + "model.layers.12.mlp.experts.50.down_proj", + "model.layers.12.mlp.experts.50.gate_proj", + "model.layers.12.mlp.experts.50.up_proj", + "model.layers.12.mlp.experts.51.down_proj", + "model.layers.12.mlp.experts.51.gate_proj", + "model.layers.12.mlp.experts.51.up_proj", + "model.layers.12.mlp.experts.52.down_proj", + "model.layers.12.mlp.experts.52.gate_proj", + "model.layers.12.mlp.experts.52.up_proj", + "model.layers.12.mlp.experts.53.down_proj", + "model.layers.12.mlp.experts.53.gate_proj", + "model.layers.12.mlp.experts.53.up_proj", + "model.layers.12.mlp.experts.54.down_proj", + "model.layers.12.mlp.experts.54.gate_proj", + "model.layers.12.mlp.experts.54.up_proj", + "model.layers.12.mlp.experts.55.down_proj", + "model.layers.12.mlp.experts.55.gate_proj", + "model.layers.12.mlp.experts.55.up_proj", + "model.layers.12.mlp.experts.56.down_proj", + "model.layers.12.mlp.experts.56.gate_proj", + "model.layers.12.mlp.experts.56.up_proj", + "model.layers.12.mlp.experts.57.down_proj", + "model.layers.12.mlp.experts.57.gate_proj", + "model.layers.12.mlp.experts.57.up_proj", + "model.layers.12.mlp.experts.58.down_proj", + "model.layers.12.mlp.experts.58.gate_proj", + "model.layers.12.mlp.experts.58.up_proj", + "model.layers.12.mlp.experts.59.down_proj", + "model.layers.12.mlp.experts.59.gate_proj", + "model.layers.12.mlp.experts.59.up_proj", + "model.layers.12.mlp.experts.6.down_proj", + "model.layers.12.mlp.experts.6.gate_proj", + "model.layers.12.mlp.experts.6.up_proj", + "model.layers.12.mlp.experts.60.down_proj", + "model.layers.12.mlp.experts.60.gate_proj", + "model.layers.12.mlp.experts.60.up_proj", + "model.layers.12.mlp.experts.61.down_proj", + "model.layers.12.mlp.experts.61.gate_proj", + "model.layers.12.mlp.experts.61.up_proj", + "model.layers.12.mlp.experts.62.down_proj", + "model.layers.12.mlp.experts.62.gate_proj", + "model.layers.12.mlp.experts.62.up_proj", + "model.layers.12.mlp.experts.63.down_proj", + "model.layers.12.mlp.experts.63.gate_proj", + "model.layers.12.mlp.experts.63.up_proj", + "model.layers.12.mlp.experts.64.down_proj", + "model.layers.12.mlp.experts.64.gate_proj", + "model.layers.12.mlp.experts.64.up_proj", + "model.layers.12.mlp.experts.65.down_proj", + "model.layers.12.mlp.experts.65.gate_proj", + "model.layers.12.mlp.experts.65.up_proj", + "model.layers.12.mlp.experts.66.down_proj", + "model.layers.12.mlp.experts.66.gate_proj", + "model.layers.12.mlp.experts.66.up_proj", + "model.layers.12.mlp.experts.67.down_proj", + "model.layers.12.mlp.experts.67.gate_proj", + "model.layers.12.mlp.experts.67.up_proj", + "model.layers.12.mlp.experts.68.down_proj", + "model.layers.12.mlp.experts.68.gate_proj", + "model.layers.12.mlp.experts.68.up_proj", + "model.layers.12.mlp.experts.69.down_proj", + "model.layers.12.mlp.experts.69.gate_proj", + "model.layers.12.mlp.experts.69.up_proj", + "model.layers.12.mlp.experts.7.down_proj", + "model.layers.12.mlp.experts.7.gate_proj", + "model.layers.12.mlp.experts.7.up_proj", + "model.layers.12.mlp.experts.70.down_proj", + "model.layers.12.mlp.experts.70.gate_proj", + "model.layers.12.mlp.experts.70.up_proj", + "model.layers.12.mlp.experts.71.down_proj", + "model.layers.12.mlp.experts.71.gate_proj", + "model.layers.12.mlp.experts.71.up_proj", + "model.layers.12.mlp.experts.72.down_proj", + "model.layers.12.mlp.experts.72.gate_proj", + "model.layers.12.mlp.experts.72.up_proj", + "model.layers.12.mlp.experts.73.down_proj", + "model.layers.12.mlp.experts.73.gate_proj", + "model.layers.12.mlp.experts.73.up_proj", + "model.layers.12.mlp.experts.74.down_proj", + "model.layers.12.mlp.experts.74.gate_proj", + "model.layers.12.mlp.experts.74.up_proj", + "model.layers.12.mlp.experts.75.down_proj", + "model.layers.12.mlp.experts.75.gate_proj", + "model.layers.12.mlp.experts.75.up_proj", + "model.layers.12.mlp.experts.76.down_proj", + "model.layers.12.mlp.experts.76.gate_proj", + "model.layers.12.mlp.experts.76.up_proj", + "model.layers.12.mlp.experts.77.down_proj", + "model.layers.12.mlp.experts.77.gate_proj", + "model.layers.12.mlp.experts.77.up_proj", + "model.layers.12.mlp.experts.78.down_proj", + "model.layers.12.mlp.experts.78.gate_proj", + "model.layers.12.mlp.experts.78.up_proj", + "model.layers.12.mlp.experts.79.down_proj", + "model.layers.12.mlp.experts.79.gate_proj", + "model.layers.12.mlp.experts.79.up_proj", + "model.layers.12.mlp.experts.8.down_proj", + "model.layers.12.mlp.experts.8.gate_proj", + "model.layers.12.mlp.experts.8.up_proj", + "model.layers.12.mlp.experts.80.down_proj", + "model.layers.12.mlp.experts.80.gate_proj", + "model.layers.12.mlp.experts.80.up_proj", + "model.layers.12.mlp.experts.81.down_proj", + "model.layers.12.mlp.experts.81.gate_proj", + "model.layers.12.mlp.experts.81.up_proj", + "model.layers.12.mlp.experts.82.down_proj", + "model.layers.12.mlp.experts.82.gate_proj", + "model.layers.12.mlp.experts.82.up_proj", + "model.layers.12.mlp.experts.83.down_proj", + "model.layers.12.mlp.experts.83.gate_proj", + "model.layers.12.mlp.experts.83.up_proj", + "model.layers.12.mlp.experts.84.down_proj", + "model.layers.12.mlp.experts.84.gate_proj", + "model.layers.12.mlp.experts.84.up_proj", + "model.layers.12.mlp.experts.85.down_proj", + "model.layers.12.mlp.experts.85.gate_proj", + "model.layers.12.mlp.experts.85.up_proj", + "model.layers.12.mlp.experts.86.down_proj", + "model.layers.12.mlp.experts.86.gate_proj", + "model.layers.12.mlp.experts.86.up_proj", + "model.layers.12.mlp.experts.87.down_proj", + "model.layers.12.mlp.experts.87.gate_proj", + "model.layers.12.mlp.experts.87.up_proj", + "model.layers.12.mlp.experts.88.down_proj", + "model.layers.12.mlp.experts.88.gate_proj", + "model.layers.12.mlp.experts.88.up_proj", + "model.layers.12.mlp.experts.89.down_proj", + "model.layers.12.mlp.experts.89.gate_proj", + "model.layers.12.mlp.experts.89.up_proj", + "model.layers.12.mlp.experts.9.down_proj", + "model.layers.12.mlp.experts.9.gate_proj", + "model.layers.12.mlp.experts.9.up_proj", + "model.layers.12.mlp.experts.90.down_proj", + "model.layers.12.mlp.experts.90.gate_proj", + "model.layers.12.mlp.experts.90.up_proj", + "model.layers.12.mlp.experts.91.down_proj", + "model.layers.12.mlp.experts.91.gate_proj", + "model.layers.12.mlp.experts.91.up_proj", + "model.layers.12.mlp.experts.92.down_proj", + "model.layers.12.mlp.experts.92.gate_proj", + "model.layers.12.mlp.experts.92.up_proj", + "model.layers.12.mlp.experts.93.down_proj", + "model.layers.12.mlp.experts.93.gate_proj", + "model.layers.12.mlp.experts.93.up_proj", + "model.layers.12.mlp.experts.94.down_proj", + "model.layers.12.mlp.experts.94.gate_proj", + "model.layers.12.mlp.experts.94.up_proj", + "model.layers.12.mlp.experts.95.down_proj", + "model.layers.12.mlp.experts.95.gate_proj", + "model.layers.12.mlp.experts.95.up_proj", + "model.layers.12.mlp.experts.96.down_proj", + "model.layers.12.mlp.experts.96.gate_proj", + "model.layers.12.mlp.experts.96.up_proj", + "model.layers.12.mlp.experts.97.down_proj", + "model.layers.12.mlp.experts.97.gate_proj", + "model.layers.12.mlp.experts.97.up_proj", + "model.layers.12.mlp.experts.98.down_proj", + "model.layers.12.mlp.experts.98.gate_proj", + "model.layers.12.mlp.experts.98.up_proj", + "model.layers.12.mlp.experts.99.down_proj", + "model.layers.12.mlp.experts.99.gate_proj", + "model.layers.12.mlp.experts.99.up_proj", + "model.layers.13.mlp.experts.0.down_proj", + "model.layers.13.mlp.experts.0.gate_proj", + "model.layers.13.mlp.experts.0.up_proj", + "model.layers.13.mlp.experts.1.down_proj", + "model.layers.13.mlp.experts.1.gate_proj", + "model.layers.13.mlp.experts.1.up_proj", + "model.layers.13.mlp.experts.10.down_proj", + "model.layers.13.mlp.experts.10.gate_proj", + "model.layers.13.mlp.experts.10.up_proj", + "model.layers.13.mlp.experts.100.down_proj", + "model.layers.13.mlp.experts.100.gate_proj", + "model.layers.13.mlp.experts.100.up_proj", + "model.layers.13.mlp.experts.101.down_proj", + "model.layers.13.mlp.experts.101.gate_proj", + "model.layers.13.mlp.experts.101.up_proj", + "model.layers.13.mlp.experts.102.down_proj", + "model.layers.13.mlp.experts.102.gate_proj", + "model.layers.13.mlp.experts.102.up_proj", + "model.layers.13.mlp.experts.103.down_proj", + "model.layers.13.mlp.experts.103.gate_proj", + "model.layers.13.mlp.experts.103.up_proj", + "model.layers.13.mlp.experts.104.down_proj", + "model.layers.13.mlp.experts.104.gate_proj", + "model.layers.13.mlp.experts.104.up_proj", + "model.layers.13.mlp.experts.105.down_proj", + "model.layers.13.mlp.experts.105.gate_proj", + "model.layers.13.mlp.experts.105.up_proj", + "model.layers.13.mlp.experts.106.down_proj", + "model.layers.13.mlp.experts.106.gate_proj", + "model.layers.13.mlp.experts.106.up_proj", + "model.layers.13.mlp.experts.107.down_proj", + "model.layers.13.mlp.experts.107.gate_proj", + "model.layers.13.mlp.experts.107.up_proj", + "model.layers.13.mlp.experts.108.down_proj", + "model.layers.13.mlp.experts.108.gate_proj", + "model.layers.13.mlp.experts.108.up_proj", + "model.layers.13.mlp.experts.109.down_proj", + "model.layers.13.mlp.experts.109.gate_proj", + "model.layers.13.mlp.experts.109.up_proj", + "model.layers.13.mlp.experts.11.down_proj", + "model.layers.13.mlp.experts.11.gate_proj", + "model.layers.13.mlp.experts.11.up_proj", + "model.layers.13.mlp.experts.110.down_proj", + "model.layers.13.mlp.experts.110.gate_proj", + "model.layers.13.mlp.experts.110.up_proj", + "model.layers.13.mlp.experts.111.down_proj", + "model.layers.13.mlp.experts.111.gate_proj", + "model.layers.13.mlp.experts.111.up_proj", + "model.layers.13.mlp.experts.112.down_proj", + "model.layers.13.mlp.experts.112.gate_proj", + "model.layers.13.mlp.experts.112.up_proj", + "model.layers.13.mlp.experts.113.down_proj", + "model.layers.13.mlp.experts.113.gate_proj", + "model.layers.13.mlp.experts.113.up_proj", + "model.layers.13.mlp.experts.114.down_proj", + "model.layers.13.mlp.experts.114.gate_proj", + "model.layers.13.mlp.experts.114.up_proj", + "model.layers.13.mlp.experts.115.down_proj", + "model.layers.13.mlp.experts.115.gate_proj", + "model.layers.13.mlp.experts.115.up_proj", + "model.layers.13.mlp.experts.116.down_proj", + "model.layers.13.mlp.experts.116.gate_proj", + "model.layers.13.mlp.experts.116.up_proj", + "model.layers.13.mlp.experts.117.down_proj", + "model.layers.13.mlp.experts.117.gate_proj", + "model.layers.13.mlp.experts.117.up_proj", + "model.layers.13.mlp.experts.118.down_proj", + "model.layers.13.mlp.experts.118.gate_proj", + "model.layers.13.mlp.experts.118.up_proj", + "model.layers.13.mlp.experts.119.down_proj", + "model.layers.13.mlp.experts.119.gate_proj", + "model.layers.13.mlp.experts.119.up_proj", + "model.layers.13.mlp.experts.12.down_proj", + "model.layers.13.mlp.experts.12.gate_proj", + "model.layers.13.mlp.experts.12.up_proj", + "model.layers.13.mlp.experts.120.down_proj", + "model.layers.13.mlp.experts.120.gate_proj", + "model.layers.13.mlp.experts.120.up_proj", + "model.layers.13.mlp.experts.121.down_proj", + "model.layers.13.mlp.experts.121.gate_proj", + "model.layers.13.mlp.experts.121.up_proj", + "model.layers.13.mlp.experts.122.down_proj", + "model.layers.13.mlp.experts.122.gate_proj", + "model.layers.13.mlp.experts.122.up_proj", + "model.layers.13.mlp.experts.123.down_proj", + "model.layers.13.mlp.experts.123.gate_proj", + "model.layers.13.mlp.experts.123.up_proj", + "model.layers.13.mlp.experts.124.down_proj", + "model.layers.13.mlp.experts.124.gate_proj", + "model.layers.13.mlp.experts.124.up_proj", + "model.layers.13.mlp.experts.125.down_proj", + "model.layers.13.mlp.experts.125.gate_proj", + "model.layers.13.mlp.experts.125.up_proj", + "model.layers.13.mlp.experts.126.down_proj", + "model.layers.13.mlp.experts.126.gate_proj", + "model.layers.13.mlp.experts.126.up_proj", + "model.layers.13.mlp.experts.127.down_proj", + "model.layers.13.mlp.experts.127.gate_proj", + "model.layers.13.mlp.experts.127.up_proj", + "model.layers.13.mlp.experts.128.down_proj", + "model.layers.13.mlp.experts.128.gate_proj", + "model.layers.13.mlp.experts.128.up_proj", + "model.layers.13.mlp.experts.129.down_proj", + "model.layers.13.mlp.experts.129.gate_proj", + "model.layers.13.mlp.experts.129.up_proj", + "model.layers.13.mlp.experts.13.down_proj", + "model.layers.13.mlp.experts.13.gate_proj", + "model.layers.13.mlp.experts.13.up_proj", + "model.layers.13.mlp.experts.130.down_proj", + "model.layers.13.mlp.experts.130.gate_proj", + "model.layers.13.mlp.experts.130.up_proj", + "model.layers.13.mlp.experts.131.down_proj", + "model.layers.13.mlp.experts.131.gate_proj", + "model.layers.13.mlp.experts.131.up_proj", + "model.layers.13.mlp.experts.132.down_proj", + "model.layers.13.mlp.experts.132.gate_proj", + "model.layers.13.mlp.experts.132.up_proj", + "model.layers.13.mlp.experts.133.down_proj", + "model.layers.13.mlp.experts.133.gate_proj", + "model.layers.13.mlp.experts.133.up_proj", + "model.layers.13.mlp.experts.134.down_proj", + "model.layers.13.mlp.experts.134.gate_proj", + "model.layers.13.mlp.experts.134.up_proj", + "model.layers.13.mlp.experts.135.down_proj", + "model.layers.13.mlp.experts.135.gate_proj", + "model.layers.13.mlp.experts.135.up_proj", + "model.layers.13.mlp.experts.136.down_proj", + "model.layers.13.mlp.experts.136.gate_proj", + "model.layers.13.mlp.experts.136.up_proj", + "model.layers.13.mlp.experts.137.down_proj", + "model.layers.13.mlp.experts.137.gate_proj", + "model.layers.13.mlp.experts.137.up_proj", + "model.layers.13.mlp.experts.138.down_proj", + "model.layers.13.mlp.experts.138.gate_proj", + "model.layers.13.mlp.experts.138.up_proj", + "model.layers.13.mlp.experts.139.down_proj", + "model.layers.13.mlp.experts.139.gate_proj", + "model.layers.13.mlp.experts.139.up_proj", + "model.layers.13.mlp.experts.14.down_proj", + "model.layers.13.mlp.experts.14.gate_proj", + "model.layers.13.mlp.experts.14.up_proj", + "model.layers.13.mlp.experts.140.down_proj", + "model.layers.13.mlp.experts.140.gate_proj", + "model.layers.13.mlp.experts.140.up_proj", + "model.layers.13.mlp.experts.141.down_proj", + "model.layers.13.mlp.experts.141.gate_proj", + "model.layers.13.mlp.experts.141.up_proj", + "model.layers.13.mlp.experts.142.down_proj", + "model.layers.13.mlp.experts.142.gate_proj", + "model.layers.13.mlp.experts.142.up_proj", + "model.layers.13.mlp.experts.143.down_proj", + "model.layers.13.mlp.experts.143.gate_proj", + "model.layers.13.mlp.experts.143.up_proj", + "model.layers.13.mlp.experts.144.down_proj", + "model.layers.13.mlp.experts.144.gate_proj", + "model.layers.13.mlp.experts.144.up_proj", + "model.layers.13.mlp.experts.145.down_proj", + "model.layers.13.mlp.experts.145.gate_proj", + "model.layers.13.mlp.experts.145.up_proj", + "model.layers.13.mlp.experts.146.down_proj", + "model.layers.13.mlp.experts.146.gate_proj", + "model.layers.13.mlp.experts.146.up_proj", + "model.layers.13.mlp.experts.147.down_proj", + "model.layers.13.mlp.experts.147.gate_proj", + "model.layers.13.mlp.experts.147.up_proj", + "model.layers.13.mlp.experts.148.down_proj", + "model.layers.13.mlp.experts.148.gate_proj", + "model.layers.13.mlp.experts.148.up_proj", + "model.layers.13.mlp.experts.149.down_proj", + "model.layers.13.mlp.experts.149.gate_proj", + "model.layers.13.mlp.experts.149.up_proj", + "model.layers.13.mlp.experts.15.down_proj", + "model.layers.13.mlp.experts.15.gate_proj", + "model.layers.13.mlp.experts.15.up_proj", + "model.layers.13.mlp.experts.150.down_proj", + "model.layers.13.mlp.experts.150.gate_proj", + "model.layers.13.mlp.experts.150.up_proj", + "model.layers.13.mlp.experts.151.down_proj", + "model.layers.13.mlp.experts.151.gate_proj", + "model.layers.13.mlp.experts.151.up_proj", + "model.layers.13.mlp.experts.152.down_proj", + "model.layers.13.mlp.experts.152.gate_proj", + "model.layers.13.mlp.experts.152.up_proj", + "model.layers.13.mlp.experts.153.down_proj", + "model.layers.13.mlp.experts.153.gate_proj", + "model.layers.13.mlp.experts.153.up_proj", + "model.layers.13.mlp.experts.154.down_proj", + "model.layers.13.mlp.experts.154.gate_proj", + "model.layers.13.mlp.experts.154.up_proj", + "model.layers.13.mlp.experts.155.down_proj", + "model.layers.13.mlp.experts.155.gate_proj", + "model.layers.13.mlp.experts.155.up_proj", + "model.layers.13.mlp.experts.156.down_proj", + "model.layers.13.mlp.experts.156.gate_proj", + "model.layers.13.mlp.experts.156.up_proj", + "model.layers.13.mlp.experts.157.down_proj", + "model.layers.13.mlp.experts.157.gate_proj", + "model.layers.13.mlp.experts.157.up_proj", + "model.layers.13.mlp.experts.158.down_proj", + "model.layers.13.mlp.experts.158.gate_proj", + "model.layers.13.mlp.experts.158.up_proj", + "model.layers.13.mlp.experts.159.down_proj", + "model.layers.13.mlp.experts.159.gate_proj", + "model.layers.13.mlp.experts.159.up_proj", + "model.layers.13.mlp.experts.16.down_proj", + "model.layers.13.mlp.experts.16.gate_proj", + "model.layers.13.mlp.experts.16.up_proj", + "model.layers.13.mlp.experts.160.down_proj", + "model.layers.13.mlp.experts.160.gate_proj", + "model.layers.13.mlp.experts.160.up_proj", + "model.layers.13.mlp.experts.161.down_proj", + "model.layers.13.mlp.experts.161.gate_proj", + "model.layers.13.mlp.experts.161.up_proj", + "model.layers.13.mlp.experts.162.down_proj", + "model.layers.13.mlp.experts.162.gate_proj", + "model.layers.13.mlp.experts.162.up_proj", + "model.layers.13.mlp.experts.163.down_proj", + "model.layers.13.mlp.experts.163.gate_proj", + "model.layers.13.mlp.experts.163.up_proj", + "model.layers.13.mlp.experts.164.down_proj", + "model.layers.13.mlp.experts.164.gate_proj", + "model.layers.13.mlp.experts.164.up_proj", + "model.layers.13.mlp.experts.165.down_proj", + "model.layers.13.mlp.experts.165.gate_proj", + "model.layers.13.mlp.experts.165.up_proj", + "model.layers.13.mlp.experts.166.down_proj", + "model.layers.13.mlp.experts.166.gate_proj", + "model.layers.13.mlp.experts.166.up_proj", + "model.layers.13.mlp.experts.167.down_proj", + "model.layers.13.mlp.experts.167.gate_proj", + "model.layers.13.mlp.experts.167.up_proj", + "model.layers.13.mlp.experts.168.down_proj", + "model.layers.13.mlp.experts.168.gate_proj", + "model.layers.13.mlp.experts.168.up_proj", + "model.layers.13.mlp.experts.169.down_proj", + "model.layers.13.mlp.experts.169.gate_proj", + "model.layers.13.mlp.experts.169.up_proj", + "model.layers.13.mlp.experts.17.down_proj", + "model.layers.13.mlp.experts.17.gate_proj", + "model.layers.13.mlp.experts.17.up_proj", + "model.layers.13.mlp.experts.170.down_proj", + "model.layers.13.mlp.experts.170.gate_proj", + "model.layers.13.mlp.experts.170.up_proj", + "model.layers.13.mlp.experts.171.down_proj", + "model.layers.13.mlp.experts.171.gate_proj", + "model.layers.13.mlp.experts.171.up_proj", + "model.layers.13.mlp.experts.172.down_proj", + "model.layers.13.mlp.experts.172.gate_proj", + "model.layers.13.mlp.experts.172.up_proj", + "model.layers.13.mlp.experts.173.down_proj", + "model.layers.13.mlp.experts.173.gate_proj", + "model.layers.13.mlp.experts.173.up_proj", + "model.layers.13.mlp.experts.174.down_proj", + "model.layers.13.mlp.experts.174.gate_proj", + "model.layers.13.mlp.experts.174.up_proj", + "model.layers.13.mlp.experts.175.down_proj", + "model.layers.13.mlp.experts.175.gate_proj", + "model.layers.13.mlp.experts.175.up_proj", + "model.layers.13.mlp.experts.176.down_proj", + "model.layers.13.mlp.experts.176.gate_proj", + "model.layers.13.mlp.experts.176.up_proj", + "model.layers.13.mlp.experts.177.down_proj", + "model.layers.13.mlp.experts.177.gate_proj", + "model.layers.13.mlp.experts.177.up_proj", + "model.layers.13.mlp.experts.178.down_proj", + "model.layers.13.mlp.experts.178.gate_proj", + "model.layers.13.mlp.experts.178.up_proj", + "model.layers.13.mlp.experts.179.down_proj", + "model.layers.13.mlp.experts.179.gate_proj", + "model.layers.13.mlp.experts.179.up_proj", + "model.layers.13.mlp.experts.18.down_proj", + "model.layers.13.mlp.experts.18.gate_proj", + "model.layers.13.mlp.experts.18.up_proj", + "model.layers.13.mlp.experts.180.down_proj", + "model.layers.13.mlp.experts.180.gate_proj", + "model.layers.13.mlp.experts.180.up_proj", + "model.layers.13.mlp.experts.181.down_proj", + "model.layers.13.mlp.experts.181.gate_proj", + "model.layers.13.mlp.experts.181.up_proj", + "model.layers.13.mlp.experts.182.down_proj", + "model.layers.13.mlp.experts.182.gate_proj", + "model.layers.13.mlp.experts.182.up_proj", + "model.layers.13.mlp.experts.183.down_proj", + "model.layers.13.mlp.experts.183.gate_proj", + "model.layers.13.mlp.experts.183.up_proj", + "model.layers.13.mlp.experts.184.down_proj", + "model.layers.13.mlp.experts.184.gate_proj", + "model.layers.13.mlp.experts.184.up_proj", + "model.layers.13.mlp.experts.185.down_proj", + "model.layers.13.mlp.experts.185.gate_proj", + "model.layers.13.mlp.experts.185.up_proj", + "model.layers.13.mlp.experts.186.down_proj", + "model.layers.13.mlp.experts.186.gate_proj", + "model.layers.13.mlp.experts.186.up_proj", + "model.layers.13.mlp.experts.187.down_proj", + "model.layers.13.mlp.experts.187.gate_proj", + "model.layers.13.mlp.experts.187.up_proj", + "model.layers.13.mlp.experts.188.down_proj", + "model.layers.13.mlp.experts.188.gate_proj", + "model.layers.13.mlp.experts.188.up_proj", + "model.layers.13.mlp.experts.189.down_proj", + "model.layers.13.mlp.experts.189.gate_proj", + "model.layers.13.mlp.experts.189.up_proj", + "model.layers.13.mlp.experts.19.down_proj", + "model.layers.13.mlp.experts.19.gate_proj", + "model.layers.13.mlp.experts.19.up_proj", + "model.layers.13.mlp.experts.190.down_proj", + "model.layers.13.mlp.experts.190.gate_proj", + "model.layers.13.mlp.experts.190.up_proj", + "model.layers.13.mlp.experts.191.down_proj", + "model.layers.13.mlp.experts.191.gate_proj", + "model.layers.13.mlp.experts.191.up_proj", + "model.layers.13.mlp.experts.192.down_proj", + "model.layers.13.mlp.experts.192.gate_proj", + "model.layers.13.mlp.experts.192.up_proj", + "model.layers.13.mlp.experts.193.down_proj", + "model.layers.13.mlp.experts.193.gate_proj", + "model.layers.13.mlp.experts.193.up_proj", + "model.layers.13.mlp.experts.194.down_proj", + "model.layers.13.mlp.experts.194.gate_proj", + "model.layers.13.mlp.experts.194.up_proj", + "model.layers.13.mlp.experts.195.down_proj", + "model.layers.13.mlp.experts.195.gate_proj", + "model.layers.13.mlp.experts.195.up_proj", + "model.layers.13.mlp.experts.196.down_proj", + "model.layers.13.mlp.experts.196.gate_proj", + "model.layers.13.mlp.experts.196.up_proj", + "model.layers.13.mlp.experts.197.down_proj", + "model.layers.13.mlp.experts.197.gate_proj", + "model.layers.13.mlp.experts.197.up_proj", + "model.layers.13.mlp.experts.198.down_proj", + "model.layers.13.mlp.experts.198.gate_proj", + "model.layers.13.mlp.experts.198.up_proj", + "model.layers.13.mlp.experts.199.down_proj", + "model.layers.13.mlp.experts.199.gate_proj", + "model.layers.13.mlp.experts.199.up_proj", + "model.layers.13.mlp.experts.2.down_proj", + "model.layers.13.mlp.experts.2.gate_proj", + "model.layers.13.mlp.experts.2.up_proj", + "model.layers.13.mlp.experts.20.down_proj", + "model.layers.13.mlp.experts.20.gate_proj", + "model.layers.13.mlp.experts.20.up_proj", + "model.layers.13.mlp.experts.200.down_proj", + "model.layers.13.mlp.experts.200.gate_proj", + "model.layers.13.mlp.experts.200.up_proj", + "model.layers.13.mlp.experts.201.down_proj", + "model.layers.13.mlp.experts.201.gate_proj", + "model.layers.13.mlp.experts.201.up_proj", + "model.layers.13.mlp.experts.202.down_proj", + "model.layers.13.mlp.experts.202.gate_proj", + "model.layers.13.mlp.experts.202.up_proj", + "model.layers.13.mlp.experts.203.down_proj", + "model.layers.13.mlp.experts.203.gate_proj", + "model.layers.13.mlp.experts.203.up_proj", + "model.layers.13.mlp.experts.204.down_proj", + "model.layers.13.mlp.experts.204.gate_proj", + "model.layers.13.mlp.experts.204.up_proj", + "model.layers.13.mlp.experts.205.down_proj", + "model.layers.13.mlp.experts.205.gate_proj", + "model.layers.13.mlp.experts.205.up_proj", + "model.layers.13.mlp.experts.206.down_proj", + "model.layers.13.mlp.experts.206.gate_proj", + "model.layers.13.mlp.experts.206.up_proj", + "model.layers.13.mlp.experts.207.down_proj", + "model.layers.13.mlp.experts.207.gate_proj", + "model.layers.13.mlp.experts.207.up_proj", + "model.layers.13.mlp.experts.208.down_proj", + "model.layers.13.mlp.experts.208.gate_proj", + "model.layers.13.mlp.experts.208.up_proj", + "model.layers.13.mlp.experts.209.down_proj", + "model.layers.13.mlp.experts.209.gate_proj", + "model.layers.13.mlp.experts.209.up_proj", + "model.layers.13.mlp.experts.21.down_proj", + "model.layers.13.mlp.experts.21.gate_proj", + "model.layers.13.mlp.experts.21.up_proj", + "model.layers.13.mlp.experts.210.down_proj", + "model.layers.13.mlp.experts.210.gate_proj", + "model.layers.13.mlp.experts.210.up_proj", + "model.layers.13.mlp.experts.211.down_proj", + "model.layers.13.mlp.experts.211.gate_proj", + "model.layers.13.mlp.experts.211.up_proj", + "model.layers.13.mlp.experts.212.down_proj", + "model.layers.13.mlp.experts.212.gate_proj", + "model.layers.13.mlp.experts.212.up_proj", + "model.layers.13.mlp.experts.213.down_proj", + "model.layers.13.mlp.experts.213.gate_proj", + "model.layers.13.mlp.experts.213.up_proj", + "model.layers.13.mlp.experts.214.down_proj", + "model.layers.13.mlp.experts.214.gate_proj", + "model.layers.13.mlp.experts.214.up_proj", + "model.layers.13.mlp.experts.215.down_proj", + "model.layers.13.mlp.experts.215.gate_proj", + "model.layers.13.mlp.experts.215.up_proj", + "model.layers.13.mlp.experts.216.down_proj", + "model.layers.13.mlp.experts.216.gate_proj", + "model.layers.13.mlp.experts.216.up_proj", + "model.layers.13.mlp.experts.217.down_proj", + "model.layers.13.mlp.experts.217.gate_proj", + "model.layers.13.mlp.experts.217.up_proj", + "model.layers.13.mlp.experts.218.down_proj", + "model.layers.13.mlp.experts.218.gate_proj", + "model.layers.13.mlp.experts.218.up_proj", + "model.layers.13.mlp.experts.219.down_proj", + "model.layers.13.mlp.experts.219.gate_proj", + "model.layers.13.mlp.experts.219.up_proj", + "model.layers.13.mlp.experts.22.down_proj", + "model.layers.13.mlp.experts.22.gate_proj", + "model.layers.13.mlp.experts.22.up_proj", + "model.layers.13.mlp.experts.220.down_proj", + "model.layers.13.mlp.experts.220.gate_proj", + "model.layers.13.mlp.experts.220.up_proj", + "model.layers.13.mlp.experts.221.down_proj", + "model.layers.13.mlp.experts.221.gate_proj", + "model.layers.13.mlp.experts.221.up_proj", + "model.layers.13.mlp.experts.222.down_proj", + "model.layers.13.mlp.experts.222.gate_proj", + "model.layers.13.mlp.experts.222.up_proj", + "model.layers.13.mlp.experts.223.down_proj", + "model.layers.13.mlp.experts.223.gate_proj", + "model.layers.13.mlp.experts.223.up_proj", + "model.layers.13.mlp.experts.224.down_proj", + "model.layers.13.mlp.experts.224.gate_proj", + "model.layers.13.mlp.experts.224.up_proj", + "model.layers.13.mlp.experts.225.down_proj", + "model.layers.13.mlp.experts.225.gate_proj", + "model.layers.13.mlp.experts.225.up_proj", + "model.layers.13.mlp.experts.226.down_proj", + "model.layers.13.mlp.experts.226.gate_proj", + "model.layers.13.mlp.experts.226.up_proj", + "model.layers.13.mlp.experts.227.down_proj", + "model.layers.13.mlp.experts.227.gate_proj", + "model.layers.13.mlp.experts.227.up_proj", + "model.layers.13.mlp.experts.228.down_proj", + "model.layers.13.mlp.experts.228.gate_proj", + "model.layers.13.mlp.experts.228.up_proj", + "model.layers.13.mlp.experts.229.down_proj", + "model.layers.13.mlp.experts.229.gate_proj", + "model.layers.13.mlp.experts.229.up_proj", + "model.layers.13.mlp.experts.23.down_proj", + "model.layers.13.mlp.experts.23.gate_proj", + "model.layers.13.mlp.experts.23.up_proj", + "model.layers.13.mlp.experts.230.down_proj", + "model.layers.13.mlp.experts.230.gate_proj", + "model.layers.13.mlp.experts.230.up_proj", + "model.layers.13.mlp.experts.231.down_proj", + "model.layers.13.mlp.experts.231.gate_proj", + "model.layers.13.mlp.experts.231.up_proj", + "model.layers.13.mlp.experts.232.down_proj", + "model.layers.13.mlp.experts.232.gate_proj", + "model.layers.13.mlp.experts.232.up_proj", + "model.layers.13.mlp.experts.233.down_proj", + "model.layers.13.mlp.experts.233.gate_proj", + "model.layers.13.mlp.experts.233.up_proj", + "model.layers.13.mlp.experts.234.down_proj", + "model.layers.13.mlp.experts.234.gate_proj", + "model.layers.13.mlp.experts.234.up_proj", + "model.layers.13.mlp.experts.235.down_proj", + "model.layers.13.mlp.experts.235.gate_proj", + "model.layers.13.mlp.experts.235.up_proj", + "model.layers.13.mlp.experts.236.down_proj", + "model.layers.13.mlp.experts.236.gate_proj", + "model.layers.13.mlp.experts.236.up_proj", + "model.layers.13.mlp.experts.237.down_proj", + "model.layers.13.mlp.experts.237.gate_proj", + "model.layers.13.mlp.experts.237.up_proj", + "model.layers.13.mlp.experts.238.down_proj", + "model.layers.13.mlp.experts.238.gate_proj", + "model.layers.13.mlp.experts.238.up_proj", + "model.layers.13.mlp.experts.239.down_proj", + "model.layers.13.mlp.experts.239.gate_proj", + "model.layers.13.mlp.experts.239.up_proj", + "model.layers.13.mlp.experts.24.down_proj", + "model.layers.13.mlp.experts.24.gate_proj", + "model.layers.13.mlp.experts.24.up_proj", + "model.layers.13.mlp.experts.240.down_proj", + "model.layers.13.mlp.experts.240.gate_proj", + "model.layers.13.mlp.experts.240.up_proj", + "model.layers.13.mlp.experts.241.down_proj", + "model.layers.13.mlp.experts.241.gate_proj", + "model.layers.13.mlp.experts.241.up_proj", + "model.layers.13.mlp.experts.242.down_proj", + "model.layers.13.mlp.experts.242.gate_proj", + "model.layers.13.mlp.experts.242.up_proj", + "model.layers.13.mlp.experts.243.down_proj", + "model.layers.13.mlp.experts.243.gate_proj", + "model.layers.13.mlp.experts.243.up_proj", + "model.layers.13.mlp.experts.244.down_proj", + "model.layers.13.mlp.experts.244.gate_proj", + "model.layers.13.mlp.experts.244.up_proj", + "model.layers.13.mlp.experts.245.down_proj", + "model.layers.13.mlp.experts.245.gate_proj", + "model.layers.13.mlp.experts.245.up_proj", + "model.layers.13.mlp.experts.246.down_proj", + "model.layers.13.mlp.experts.246.gate_proj", + "model.layers.13.mlp.experts.246.up_proj", + "model.layers.13.mlp.experts.247.down_proj", + "model.layers.13.mlp.experts.247.gate_proj", + "model.layers.13.mlp.experts.247.up_proj", + "model.layers.13.mlp.experts.248.down_proj", + "model.layers.13.mlp.experts.248.gate_proj", + "model.layers.13.mlp.experts.248.up_proj", + "model.layers.13.mlp.experts.249.down_proj", + "model.layers.13.mlp.experts.249.gate_proj", + "model.layers.13.mlp.experts.249.up_proj", + "model.layers.13.mlp.experts.25.down_proj", + "model.layers.13.mlp.experts.25.gate_proj", + "model.layers.13.mlp.experts.25.up_proj", + "model.layers.13.mlp.experts.250.down_proj", + "model.layers.13.mlp.experts.250.gate_proj", + "model.layers.13.mlp.experts.250.up_proj", + "model.layers.13.mlp.experts.251.down_proj", + "model.layers.13.mlp.experts.251.gate_proj", + "model.layers.13.mlp.experts.251.up_proj", + "model.layers.13.mlp.experts.252.down_proj", + "model.layers.13.mlp.experts.252.gate_proj", + "model.layers.13.mlp.experts.252.up_proj", + "model.layers.13.mlp.experts.253.down_proj", + "model.layers.13.mlp.experts.253.gate_proj", + "model.layers.13.mlp.experts.253.up_proj", + "model.layers.13.mlp.experts.254.down_proj", + "model.layers.13.mlp.experts.254.gate_proj", + "model.layers.13.mlp.experts.254.up_proj", + "model.layers.13.mlp.experts.255.down_proj", + "model.layers.13.mlp.experts.255.gate_proj", + "model.layers.13.mlp.experts.255.up_proj", + "model.layers.13.mlp.experts.26.down_proj", + "model.layers.13.mlp.experts.26.gate_proj", + "model.layers.13.mlp.experts.26.up_proj", + "model.layers.13.mlp.experts.27.down_proj", + "model.layers.13.mlp.experts.27.gate_proj", + "model.layers.13.mlp.experts.27.up_proj", + "model.layers.13.mlp.experts.28.down_proj", + "model.layers.13.mlp.experts.28.gate_proj", + "model.layers.13.mlp.experts.28.up_proj", + "model.layers.13.mlp.experts.29.down_proj", + "model.layers.13.mlp.experts.29.gate_proj", + "model.layers.13.mlp.experts.29.up_proj", + "model.layers.13.mlp.experts.3.down_proj", + "model.layers.13.mlp.experts.3.gate_proj", + "model.layers.13.mlp.experts.3.up_proj", + "model.layers.13.mlp.experts.30.down_proj", + "model.layers.13.mlp.experts.30.gate_proj", + "model.layers.13.mlp.experts.30.up_proj", + "model.layers.13.mlp.experts.31.down_proj", + "model.layers.13.mlp.experts.31.gate_proj", + "model.layers.13.mlp.experts.31.up_proj", + "model.layers.13.mlp.experts.32.down_proj", + "model.layers.13.mlp.experts.32.gate_proj", + "model.layers.13.mlp.experts.32.up_proj", + "model.layers.13.mlp.experts.33.down_proj", + "model.layers.13.mlp.experts.33.gate_proj", + "model.layers.13.mlp.experts.33.up_proj", + "model.layers.13.mlp.experts.34.down_proj", + "model.layers.13.mlp.experts.34.gate_proj", + "model.layers.13.mlp.experts.34.up_proj", + "model.layers.13.mlp.experts.35.down_proj", + "model.layers.13.mlp.experts.35.gate_proj", + "model.layers.13.mlp.experts.35.up_proj", + "model.layers.13.mlp.experts.36.down_proj", + "model.layers.13.mlp.experts.36.gate_proj", + "model.layers.13.mlp.experts.36.up_proj", + "model.layers.13.mlp.experts.37.down_proj", + "model.layers.13.mlp.experts.37.gate_proj", + "model.layers.13.mlp.experts.37.up_proj", + "model.layers.13.mlp.experts.38.down_proj", + "model.layers.13.mlp.experts.38.gate_proj", + "model.layers.13.mlp.experts.38.up_proj", + "model.layers.13.mlp.experts.39.down_proj", + "model.layers.13.mlp.experts.39.gate_proj", + "model.layers.13.mlp.experts.39.up_proj", + "model.layers.13.mlp.experts.4.down_proj", + "model.layers.13.mlp.experts.4.gate_proj", + "model.layers.13.mlp.experts.4.up_proj", + "model.layers.13.mlp.experts.40.down_proj", + "model.layers.13.mlp.experts.40.gate_proj", + "model.layers.13.mlp.experts.40.up_proj", + "model.layers.13.mlp.experts.41.down_proj", + "model.layers.13.mlp.experts.41.gate_proj", + "model.layers.13.mlp.experts.41.up_proj", + "model.layers.13.mlp.experts.42.down_proj", + "model.layers.13.mlp.experts.42.gate_proj", + "model.layers.13.mlp.experts.42.up_proj", + "model.layers.13.mlp.experts.43.down_proj", + "model.layers.13.mlp.experts.43.gate_proj", + "model.layers.13.mlp.experts.43.up_proj", + "model.layers.13.mlp.experts.44.down_proj", + "model.layers.13.mlp.experts.44.gate_proj", + "model.layers.13.mlp.experts.44.up_proj", + "model.layers.13.mlp.experts.45.down_proj", + "model.layers.13.mlp.experts.45.gate_proj", + "model.layers.13.mlp.experts.45.up_proj", + "model.layers.13.mlp.experts.46.down_proj", + "model.layers.13.mlp.experts.46.gate_proj", + "model.layers.13.mlp.experts.46.up_proj", + "model.layers.13.mlp.experts.47.down_proj", + "model.layers.13.mlp.experts.47.gate_proj", + "model.layers.13.mlp.experts.47.up_proj", + "model.layers.13.mlp.experts.48.down_proj", + "model.layers.13.mlp.experts.48.gate_proj", + "model.layers.13.mlp.experts.48.up_proj", + "model.layers.13.mlp.experts.49.down_proj", + "model.layers.13.mlp.experts.49.gate_proj", + "model.layers.13.mlp.experts.49.up_proj", + "model.layers.13.mlp.experts.5.down_proj", + "model.layers.13.mlp.experts.5.gate_proj", + "model.layers.13.mlp.experts.5.up_proj", + "model.layers.13.mlp.experts.50.down_proj", + "model.layers.13.mlp.experts.50.gate_proj", + "model.layers.13.mlp.experts.50.up_proj", + "model.layers.13.mlp.experts.51.down_proj", + "model.layers.13.mlp.experts.51.gate_proj", + "model.layers.13.mlp.experts.51.up_proj", + "model.layers.13.mlp.experts.52.down_proj", + "model.layers.13.mlp.experts.52.gate_proj", + "model.layers.13.mlp.experts.52.up_proj", + "model.layers.13.mlp.experts.53.down_proj", + "model.layers.13.mlp.experts.53.gate_proj", + "model.layers.13.mlp.experts.53.up_proj", + "model.layers.13.mlp.experts.54.down_proj", + "model.layers.13.mlp.experts.54.gate_proj", + "model.layers.13.mlp.experts.54.up_proj", + "model.layers.13.mlp.experts.55.down_proj", + "model.layers.13.mlp.experts.55.gate_proj", + "model.layers.13.mlp.experts.55.up_proj", + "model.layers.13.mlp.experts.56.down_proj", + "model.layers.13.mlp.experts.56.gate_proj", + "model.layers.13.mlp.experts.56.up_proj", + "model.layers.13.mlp.experts.57.down_proj", + "model.layers.13.mlp.experts.57.gate_proj", + "model.layers.13.mlp.experts.57.up_proj", + "model.layers.13.mlp.experts.58.down_proj", + "model.layers.13.mlp.experts.58.gate_proj", + "model.layers.13.mlp.experts.58.up_proj", + "model.layers.13.mlp.experts.59.down_proj", + "model.layers.13.mlp.experts.59.gate_proj", + "model.layers.13.mlp.experts.59.up_proj", + "model.layers.13.mlp.experts.6.down_proj", + "model.layers.13.mlp.experts.6.gate_proj", + "model.layers.13.mlp.experts.6.up_proj", + "model.layers.13.mlp.experts.60.down_proj", + "model.layers.13.mlp.experts.60.gate_proj", + "model.layers.13.mlp.experts.60.up_proj", + "model.layers.13.mlp.experts.61.down_proj", + "model.layers.13.mlp.experts.61.gate_proj", + "model.layers.13.mlp.experts.61.up_proj", + "model.layers.13.mlp.experts.62.down_proj", + "model.layers.13.mlp.experts.62.gate_proj", + "model.layers.13.mlp.experts.62.up_proj", + "model.layers.13.mlp.experts.63.down_proj", + "model.layers.13.mlp.experts.63.gate_proj", + "model.layers.13.mlp.experts.63.up_proj", + "model.layers.13.mlp.experts.64.down_proj", + "model.layers.13.mlp.experts.64.gate_proj", + "model.layers.13.mlp.experts.64.up_proj", + "model.layers.13.mlp.experts.65.down_proj", + "model.layers.13.mlp.experts.65.gate_proj", + "model.layers.13.mlp.experts.65.up_proj", + "model.layers.13.mlp.experts.66.down_proj", + "model.layers.13.mlp.experts.66.gate_proj", + "model.layers.13.mlp.experts.66.up_proj", + "model.layers.13.mlp.experts.67.down_proj", + "model.layers.13.mlp.experts.67.gate_proj", + "model.layers.13.mlp.experts.67.up_proj", + "model.layers.13.mlp.experts.68.down_proj", + "model.layers.13.mlp.experts.68.gate_proj", + "model.layers.13.mlp.experts.68.up_proj", + "model.layers.13.mlp.experts.69.down_proj", + "model.layers.13.mlp.experts.69.gate_proj", + "model.layers.13.mlp.experts.69.up_proj", + "model.layers.13.mlp.experts.7.down_proj", + "model.layers.13.mlp.experts.7.gate_proj", + "model.layers.13.mlp.experts.7.up_proj", + "model.layers.13.mlp.experts.70.down_proj", + "model.layers.13.mlp.experts.70.gate_proj", + "model.layers.13.mlp.experts.70.up_proj", + "model.layers.13.mlp.experts.71.down_proj", + "model.layers.13.mlp.experts.71.gate_proj", + "model.layers.13.mlp.experts.71.up_proj", + "model.layers.13.mlp.experts.72.down_proj", + "model.layers.13.mlp.experts.72.gate_proj", + "model.layers.13.mlp.experts.72.up_proj", + "model.layers.13.mlp.experts.73.down_proj", + "model.layers.13.mlp.experts.73.gate_proj", + "model.layers.13.mlp.experts.73.up_proj", + "model.layers.13.mlp.experts.74.down_proj", + "model.layers.13.mlp.experts.74.gate_proj", + "model.layers.13.mlp.experts.74.up_proj", + "model.layers.13.mlp.experts.75.down_proj", + "model.layers.13.mlp.experts.75.gate_proj", + "model.layers.13.mlp.experts.75.up_proj", + "model.layers.13.mlp.experts.76.down_proj", + "model.layers.13.mlp.experts.76.gate_proj", + "model.layers.13.mlp.experts.76.up_proj", + "model.layers.13.mlp.experts.77.down_proj", + "model.layers.13.mlp.experts.77.gate_proj", + "model.layers.13.mlp.experts.77.up_proj", + "model.layers.13.mlp.experts.78.down_proj", + "model.layers.13.mlp.experts.78.gate_proj", + "model.layers.13.mlp.experts.78.up_proj", + "model.layers.13.mlp.experts.79.down_proj", + "model.layers.13.mlp.experts.79.gate_proj", + "model.layers.13.mlp.experts.79.up_proj", + "model.layers.13.mlp.experts.8.down_proj", + "model.layers.13.mlp.experts.8.gate_proj", + "model.layers.13.mlp.experts.8.up_proj", + "model.layers.13.mlp.experts.80.down_proj", + "model.layers.13.mlp.experts.80.gate_proj", + "model.layers.13.mlp.experts.80.up_proj", + "model.layers.13.mlp.experts.81.down_proj", + "model.layers.13.mlp.experts.81.gate_proj", + "model.layers.13.mlp.experts.81.up_proj", + "model.layers.13.mlp.experts.82.down_proj", + "model.layers.13.mlp.experts.82.gate_proj", + "model.layers.13.mlp.experts.82.up_proj", + "model.layers.13.mlp.experts.83.down_proj", + "model.layers.13.mlp.experts.83.gate_proj", + "model.layers.13.mlp.experts.83.up_proj", + "model.layers.13.mlp.experts.84.down_proj", + "model.layers.13.mlp.experts.84.gate_proj", + "model.layers.13.mlp.experts.84.up_proj", + "model.layers.13.mlp.experts.85.down_proj", + "model.layers.13.mlp.experts.85.gate_proj", + "model.layers.13.mlp.experts.85.up_proj", + "model.layers.13.mlp.experts.86.down_proj", + "model.layers.13.mlp.experts.86.gate_proj", + "model.layers.13.mlp.experts.86.up_proj", + "model.layers.13.mlp.experts.87.down_proj", + "model.layers.13.mlp.experts.87.gate_proj", + "model.layers.13.mlp.experts.87.up_proj", + "model.layers.13.mlp.experts.88.down_proj", + "model.layers.13.mlp.experts.88.gate_proj", + "model.layers.13.mlp.experts.88.up_proj", + "model.layers.13.mlp.experts.89.down_proj", + "model.layers.13.mlp.experts.89.gate_proj", + "model.layers.13.mlp.experts.89.up_proj", + "model.layers.13.mlp.experts.9.down_proj", + "model.layers.13.mlp.experts.9.gate_proj", + "model.layers.13.mlp.experts.9.up_proj", + "model.layers.13.mlp.experts.90.down_proj", + "model.layers.13.mlp.experts.90.gate_proj", + "model.layers.13.mlp.experts.90.up_proj", + "model.layers.13.mlp.experts.91.down_proj", + "model.layers.13.mlp.experts.91.gate_proj", + "model.layers.13.mlp.experts.91.up_proj", + "model.layers.13.mlp.experts.92.down_proj", + "model.layers.13.mlp.experts.92.gate_proj", + "model.layers.13.mlp.experts.92.up_proj", + "model.layers.13.mlp.experts.93.down_proj", + "model.layers.13.mlp.experts.93.gate_proj", + "model.layers.13.mlp.experts.93.up_proj", + "model.layers.13.mlp.experts.94.down_proj", + "model.layers.13.mlp.experts.94.gate_proj", + "model.layers.13.mlp.experts.94.up_proj", + "model.layers.13.mlp.experts.95.down_proj", + "model.layers.13.mlp.experts.95.gate_proj", + "model.layers.13.mlp.experts.95.up_proj", + "model.layers.13.mlp.experts.96.down_proj", + "model.layers.13.mlp.experts.96.gate_proj", + "model.layers.13.mlp.experts.96.up_proj", + "model.layers.13.mlp.experts.97.down_proj", + "model.layers.13.mlp.experts.97.gate_proj", + "model.layers.13.mlp.experts.97.up_proj", + "model.layers.13.mlp.experts.98.down_proj", + "model.layers.13.mlp.experts.98.gate_proj", + "model.layers.13.mlp.experts.98.up_proj", + "model.layers.13.mlp.experts.99.down_proj", + "model.layers.13.mlp.experts.99.gate_proj", + "model.layers.13.mlp.experts.99.up_proj", + "model.layers.14.mlp.experts.0.down_proj", + "model.layers.14.mlp.experts.0.gate_proj", + "model.layers.14.mlp.experts.0.up_proj", + "model.layers.14.mlp.experts.1.down_proj", + "model.layers.14.mlp.experts.1.gate_proj", + "model.layers.14.mlp.experts.1.up_proj", + "model.layers.14.mlp.experts.10.down_proj", + "model.layers.14.mlp.experts.10.gate_proj", + "model.layers.14.mlp.experts.10.up_proj", + "model.layers.14.mlp.experts.100.down_proj", + "model.layers.14.mlp.experts.100.gate_proj", + "model.layers.14.mlp.experts.100.up_proj", + "model.layers.14.mlp.experts.101.down_proj", + "model.layers.14.mlp.experts.101.gate_proj", + "model.layers.14.mlp.experts.101.up_proj", + "model.layers.14.mlp.experts.102.down_proj", + "model.layers.14.mlp.experts.102.gate_proj", + "model.layers.14.mlp.experts.102.up_proj", + "model.layers.14.mlp.experts.103.down_proj", + "model.layers.14.mlp.experts.103.gate_proj", + "model.layers.14.mlp.experts.103.up_proj", + "model.layers.14.mlp.experts.104.down_proj", + "model.layers.14.mlp.experts.104.gate_proj", + "model.layers.14.mlp.experts.104.up_proj", + "model.layers.14.mlp.experts.105.down_proj", + "model.layers.14.mlp.experts.105.gate_proj", + "model.layers.14.mlp.experts.105.up_proj", + "model.layers.14.mlp.experts.106.down_proj", + "model.layers.14.mlp.experts.106.gate_proj", + "model.layers.14.mlp.experts.106.up_proj", + "model.layers.14.mlp.experts.107.down_proj", + "model.layers.14.mlp.experts.107.gate_proj", + "model.layers.14.mlp.experts.107.up_proj", + "model.layers.14.mlp.experts.108.down_proj", + "model.layers.14.mlp.experts.108.gate_proj", + "model.layers.14.mlp.experts.108.up_proj", + "model.layers.14.mlp.experts.109.down_proj", + "model.layers.14.mlp.experts.109.gate_proj", + "model.layers.14.mlp.experts.109.up_proj", + "model.layers.14.mlp.experts.11.down_proj", + "model.layers.14.mlp.experts.11.gate_proj", + "model.layers.14.mlp.experts.11.up_proj", + "model.layers.14.mlp.experts.110.down_proj", + "model.layers.14.mlp.experts.110.gate_proj", + "model.layers.14.mlp.experts.110.up_proj", + "model.layers.14.mlp.experts.111.down_proj", + "model.layers.14.mlp.experts.111.gate_proj", + "model.layers.14.mlp.experts.111.up_proj", + "model.layers.14.mlp.experts.112.down_proj", + "model.layers.14.mlp.experts.112.gate_proj", + "model.layers.14.mlp.experts.112.up_proj", + "model.layers.14.mlp.experts.113.down_proj", + "model.layers.14.mlp.experts.113.gate_proj", + "model.layers.14.mlp.experts.113.up_proj", + "model.layers.14.mlp.experts.114.down_proj", + "model.layers.14.mlp.experts.114.gate_proj", + "model.layers.14.mlp.experts.114.up_proj", + "model.layers.14.mlp.experts.115.down_proj", + "model.layers.14.mlp.experts.115.gate_proj", + "model.layers.14.mlp.experts.115.up_proj", + "model.layers.14.mlp.experts.116.down_proj", + "model.layers.14.mlp.experts.116.gate_proj", + "model.layers.14.mlp.experts.116.up_proj", + "model.layers.14.mlp.experts.117.down_proj", + "model.layers.14.mlp.experts.117.gate_proj", + "model.layers.14.mlp.experts.117.up_proj", + "model.layers.14.mlp.experts.118.down_proj", + "model.layers.14.mlp.experts.118.gate_proj", + "model.layers.14.mlp.experts.118.up_proj", + "model.layers.14.mlp.experts.119.down_proj", + "model.layers.14.mlp.experts.119.gate_proj", + "model.layers.14.mlp.experts.119.up_proj", + "model.layers.14.mlp.experts.12.down_proj", + "model.layers.14.mlp.experts.12.gate_proj", + "model.layers.14.mlp.experts.12.up_proj", + "model.layers.14.mlp.experts.120.down_proj", + "model.layers.14.mlp.experts.120.gate_proj", + "model.layers.14.mlp.experts.120.up_proj", + "model.layers.14.mlp.experts.121.down_proj", + "model.layers.14.mlp.experts.121.gate_proj", + "model.layers.14.mlp.experts.121.up_proj", + "model.layers.14.mlp.experts.122.down_proj", + "model.layers.14.mlp.experts.122.gate_proj", + "model.layers.14.mlp.experts.122.up_proj", + "model.layers.14.mlp.experts.123.down_proj", + "model.layers.14.mlp.experts.123.gate_proj", + "model.layers.14.mlp.experts.123.up_proj", + "model.layers.14.mlp.experts.124.down_proj", + "model.layers.14.mlp.experts.124.gate_proj", + "model.layers.14.mlp.experts.124.up_proj", + "model.layers.14.mlp.experts.125.down_proj", + "model.layers.14.mlp.experts.125.gate_proj", + "model.layers.14.mlp.experts.125.up_proj", + "model.layers.14.mlp.experts.126.down_proj", + "model.layers.14.mlp.experts.126.gate_proj", + "model.layers.14.mlp.experts.126.up_proj", + "model.layers.14.mlp.experts.127.down_proj", + "model.layers.14.mlp.experts.127.gate_proj", + "model.layers.14.mlp.experts.127.up_proj", + "model.layers.14.mlp.experts.128.down_proj", + "model.layers.14.mlp.experts.128.gate_proj", + "model.layers.14.mlp.experts.128.up_proj", + "model.layers.14.mlp.experts.129.down_proj", + "model.layers.14.mlp.experts.129.gate_proj", + "model.layers.14.mlp.experts.129.up_proj", + "model.layers.14.mlp.experts.13.down_proj", + "model.layers.14.mlp.experts.13.gate_proj", + "model.layers.14.mlp.experts.13.up_proj", + "model.layers.14.mlp.experts.130.down_proj", + "model.layers.14.mlp.experts.130.gate_proj", + "model.layers.14.mlp.experts.130.up_proj", + "model.layers.14.mlp.experts.131.down_proj", + "model.layers.14.mlp.experts.131.gate_proj", + "model.layers.14.mlp.experts.131.up_proj", + "model.layers.14.mlp.experts.132.down_proj", + "model.layers.14.mlp.experts.132.gate_proj", + "model.layers.14.mlp.experts.132.up_proj", + "model.layers.14.mlp.experts.133.down_proj", + "model.layers.14.mlp.experts.133.gate_proj", + "model.layers.14.mlp.experts.133.up_proj", + "model.layers.14.mlp.experts.134.down_proj", + "model.layers.14.mlp.experts.134.gate_proj", + "model.layers.14.mlp.experts.134.up_proj", + "model.layers.14.mlp.experts.135.down_proj", + "model.layers.14.mlp.experts.135.gate_proj", + "model.layers.14.mlp.experts.135.up_proj", + "model.layers.14.mlp.experts.136.down_proj", + "model.layers.14.mlp.experts.136.gate_proj", + "model.layers.14.mlp.experts.136.up_proj", + "model.layers.14.mlp.experts.137.down_proj", + "model.layers.14.mlp.experts.137.gate_proj", + "model.layers.14.mlp.experts.137.up_proj", + "model.layers.14.mlp.experts.138.down_proj", + "model.layers.14.mlp.experts.138.gate_proj", + "model.layers.14.mlp.experts.138.up_proj", + "model.layers.14.mlp.experts.139.down_proj", + "model.layers.14.mlp.experts.139.gate_proj", + "model.layers.14.mlp.experts.139.up_proj", + "model.layers.14.mlp.experts.14.down_proj", + "model.layers.14.mlp.experts.14.gate_proj", + "model.layers.14.mlp.experts.14.up_proj", + "model.layers.14.mlp.experts.140.down_proj", + "model.layers.14.mlp.experts.140.gate_proj", + "model.layers.14.mlp.experts.140.up_proj", + "model.layers.14.mlp.experts.141.down_proj", + "model.layers.14.mlp.experts.141.gate_proj", + "model.layers.14.mlp.experts.141.up_proj", + "model.layers.14.mlp.experts.142.down_proj", + "model.layers.14.mlp.experts.142.gate_proj", + "model.layers.14.mlp.experts.142.up_proj", + "model.layers.14.mlp.experts.143.down_proj", + "model.layers.14.mlp.experts.143.gate_proj", + "model.layers.14.mlp.experts.143.up_proj", + "model.layers.14.mlp.experts.144.down_proj", + "model.layers.14.mlp.experts.144.gate_proj", + "model.layers.14.mlp.experts.144.up_proj", + "model.layers.14.mlp.experts.145.down_proj", + "model.layers.14.mlp.experts.145.gate_proj", + "model.layers.14.mlp.experts.145.up_proj", + "model.layers.14.mlp.experts.146.down_proj", + "model.layers.14.mlp.experts.146.gate_proj", + "model.layers.14.mlp.experts.146.up_proj", + "model.layers.14.mlp.experts.147.down_proj", + "model.layers.14.mlp.experts.147.gate_proj", + "model.layers.14.mlp.experts.147.up_proj", + "model.layers.14.mlp.experts.148.down_proj", + "model.layers.14.mlp.experts.148.gate_proj", + "model.layers.14.mlp.experts.148.up_proj", + "model.layers.14.mlp.experts.149.down_proj", + "model.layers.14.mlp.experts.149.gate_proj", + "model.layers.14.mlp.experts.149.up_proj", + "model.layers.14.mlp.experts.15.down_proj", + "model.layers.14.mlp.experts.15.gate_proj", + "model.layers.14.mlp.experts.15.up_proj", + "model.layers.14.mlp.experts.150.down_proj", + "model.layers.14.mlp.experts.150.gate_proj", + "model.layers.14.mlp.experts.150.up_proj", + "model.layers.14.mlp.experts.151.down_proj", + "model.layers.14.mlp.experts.151.gate_proj", + "model.layers.14.mlp.experts.151.up_proj", + "model.layers.14.mlp.experts.152.down_proj", + "model.layers.14.mlp.experts.152.gate_proj", + "model.layers.14.mlp.experts.152.up_proj", + "model.layers.14.mlp.experts.153.down_proj", + "model.layers.14.mlp.experts.153.gate_proj", + "model.layers.14.mlp.experts.153.up_proj", + "model.layers.14.mlp.experts.154.down_proj", + "model.layers.14.mlp.experts.154.gate_proj", + "model.layers.14.mlp.experts.154.up_proj", + "model.layers.14.mlp.experts.155.down_proj", + "model.layers.14.mlp.experts.155.gate_proj", + "model.layers.14.mlp.experts.155.up_proj", + "model.layers.14.mlp.experts.156.down_proj", + "model.layers.14.mlp.experts.156.gate_proj", + "model.layers.14.mlp.experts.156.up_proj", + "model.layers.14.mlp.experts.157.down_proj", + "model.layers.14.mlp.experts.157.gate_proj", + "model.layers.14.mlp.experts.157.up_proj", + "model.layers.14.mlp.experts.158.down_proj", + "model.layers.14.mlp.experts.158.gate_proj", + "model.layers.14.mlp.experts.158.up_proj", + "model.layers.14.mlp.experts.159.down_proj", + "model.layers.14.mlp.experts.159.gate_proj", + "model.layers.14.mlp.experts.159.up_proj", + "model.layers.14.mlp.experts.16.down_proj", + "model.layers.14.mlp.experts.16.gate_proj", + "model.layers.14.mlp.experts.16.up_proj", + "model.layers.14.mlp.experts.160.down_proj", + "model.layers.14.mlp.experts.160.gate_proj", + "model.layers.14.mlp.experts.160.up_proj", + "model.layers.14.mlp.experts.161.down_proj", + "model.layers.14.mlp.experts.161.gate_proj", + "model.layers.14.mlp.experts.161.up_proj", + "model.layers.14.mlp.experts.162.down_proj", + "model.layers.14.mlp.experts.162.gate_proj", + "model.layers.14.mlp.experts.162.up_proj", + "model.layers.14.mlp.experts.163.down_proj", + "model.layers.14.mlp.experts.163.gate_proj", + "model.layers.14.mlp.experts.163.up_proj", + "model.layers.14.mlp.experts.164.down_proj", + "model.layers.14.mlp.experts.164.gate_proj", + "model.layers.14.mlp.experts.164.up_proj", + "model.layers.14.mlp.experts.165.down_proj", + "model.layers.14.mlp.experts.165.gate_proj", + "model.layers.14.mlp.experts.165.up_proj", + "model.layers.14.mlp.experts.166.down_proj", + "model.layers.14.mlp.experts.166.gate_proj", + "model.layers.14.mlp.experts.166.up_proj", + "model.layers.14.mlp.experts.167.down_proj", + "model.layers.14.mlp.experts.167.gate_proj", + "model.layers.14.mlp.experts.167.up_proj", + "model.layers.14.mlp.experts.168.down_proj", + "model.layers.14.mlp.experts.168.gate_proj", + "model.layers.14.mlp.experts.168.up_proj", + "model.layers.14.mlp.experts.169.down_proj", + "model.layers.14.mlp.experts.169.gate_proj", + "model.layers.14.mlp.experts.169.up_proj", + "model.layers.14.mlp.experts.17.down_proj", + "model.layers.14.mlp.experts.17.gate_proj", + "model.layers.14.mlp.experts.17.up_proj", + "model.layers.14.mlp.experts.170.down_proj", + "model.layers.14.mlp.experts.170.gate_proj", + "model.layers.14.mlp.experts.170.up_proj", + "model.layers.14.mlp.experts.171.down_proj", + "model.layers.14.mlp.experts.171.gate_proj", + "model.layers.14.mlp.experts.171.up_proj", + "model.layers.14.mlp.experts.172.down_proj", + "model.layers.14.mlp.experts.172.gate_proj", + "model.layers.14.mlp.experts.172.up_proj", + "model.layers.14.mlp.experts.173.down_proj", + "model.layers.14.mlp.experts.173.gate_proj", + "model.layers.14.mlp.experts.173.up_proj", + "model.layers.14.mlp.experts.174.down_proj", + "model.layers.14.mlp.experts.174.gate_proj", + "model.layers.14.mlp.experts.174.up_proj", + "model.layers.14.mlp.experts.175.down_proj", + "model.layers.14.mlp.experts.175.gate_proj", + "model.layers.14.mlp.experts.175.up_proj", + "model.layers.14.mlp.experts.176.down_proj", + "model.layers.14.mlp.experts.176.gate_proj", + "model.layers.14.mlp.experts.176.up_proj", + "model.layers.14.mlp.experts.177.down_proj", + "model.layers.14.mlp.experts.177.gate_proj", + "model.layers.14.mlp.experts.177.up_proj", + "model.layers.14.mlp.experts.178.down_proj", + "model.layers.14.mlp.experts.178.gate_proj", + "model.layers.14.mlp.experts.178.up_proj", + "model.layers.14.mlp.experts.179.down_proj", + "model.layers.14.mlp.experts.179.gate_proj", + "model.layers.14.mlp.experts.179.up_proj", + "model.layers.14.mlp.experts.18.down_proj", + "model.layers.14.mlp.experts.18.gate_proj", + "model.layers.14.mlp.experts.18.up_proj", + "model.layers.14.mlp.experts.180.down_proj", + "model.layers.14.mlp.experts.180.gate_proj", + "model.layers.14.mlp.experts.180.up_proj", + "model.layers.14.mlp.experts.181.down_proj", + "model.layers.14.mlp.experts.181.gate_proj", + "model.layers.14.mlp.experts.181.up_proj", + "model.layers.14.mlp.experts.182.down_proj", + "model.layers.14.mlp.experts.182.gate_proj", + "model.layers.14.mlp.experts.182.up_proj", + "model.layers.14.mlp.experts.183.down_proj", + "model.layers.14.mlp.experts.183.gate_proj", + "model.layers.14.mlp.experts.183.up_proj", + "model.layers.14.mlp.experts.184.down_proj", + "model.layers.14.mlp.experts.184.gate_proj", + "model.layers.14.mlp.experts.184.up_proj", + "model.layers.14.mlp.experts.185.down_proj", + "model.layers.14.mlp.experts.185.gate_proj", + "model.layers.14.mlp.experts.185.up_proj", + "model.layers.14.mlp.experts.186.down_proj", + "model.layers.14.mlp.experts.186.gate_proj", + "model.layers.14.mlp.experts.186.up_proj", + "model.layers.14.mlp.experts.187.down_proj", + "model.layers.14.mlp.experts.187.gate_proj", + "model.layers.14.mlp.experts.187.up_proj", + "model.layers.14.mlp.experts.188.down_proj", + "model.layers.14.mlp.experts.188.gate_proj", + "model.layers.14.mlp.experts.188.up_proj", + "model.layers.14.mlp.experts.189.down_proj", + "model.layers.14.mlp.experts.189.gate_proj", + "model.layers.14.mlp.experts.189.up_proj", + "model.layers.14.mlp.experts.19.down_proj", + "model.layers.14.mlp.experts.19.gate_proj", + "model.layers.14.mlp.experts.19.up_proj", + "model.layers.14.mlp.experts.190.down_proj", + "model.layers.14.mlp.experts.190.gate_proj", + "model.layers.14.mlp.experts.190.up_proj", + "model.layers.14.mlp.experts.191.down_proj", + "model.layers.14.mlp.experts.191.gate_proj", + "model.layers.14.mlp.experts.191.up_proj", + "model.layers.14.mlp.experts.192.down_proj", + "model.layers.14.mlp.experts.192.gate_proj", + "model.layers.14.mlp.experts.192.up_proj", + "model.layers.14.mlp.experts.193.down_proj", + "model.layers.14.mlp.experts.193.gate_proj", + "model.layers.14.mlp.experts.193.up_proj", + "model.layers.14.mlp.experts.194.down_proj", + "model.layers.14.mlp.experts.194.gate_proj", + "model.layers.14.mlp.experts.194.up_proj", + "model.layers.14.mlp.experts.195.down_proj", + "model.layers.14.mlp.experts.195.gate_proj", + "model.layers.14.mlp.experts.195.up_proj", + "model.layers.14.mlp.experts.196.down_proj", + "model.layers.14.mlp.experts.196.gate_proj", + "model.layers.14.mlp.experts.196.up_proj", + "model.layers.14.mlp.experts.197.down_proj", + "model.layers.14.mlp.experts.197.gate_proj", + "model.layers.14.mlp.experts.197.up_proj", + "model.layers.14.mlp.experts.198.down_proj", + "model.layers.14.mlp.experts.198.gate_proj", + "model.layers.14.mlp.experts.198.up_proj", + "model.layers.14.mlp.experts.199.down_proj", + "model.layers.14.mlp.experts.199.gate_proj", + "model.layers.14.mlp.experts.199.up_proj", + "model.layers.14.mlp.experts.2.down_proj", + "model.layers.14.mlp.experts.2.gate_proj", + "model.layers.14.mlp.experts.2.up_proj", + "model.layers.14.mlp.experts.20.down_proj", + "model.layers.14.mlp.experts.20.gate_proj", + "model.layers.14.mlp.experts.20.up_proj", + "model.layers.14.mlp.experts.200.down_proj", + "model.layers.14.mlp.experts.200.gate_proj", + "model.layers.14.mlp.experts.200.up_proj", + "model.layers.14.mlp.experts.201.down_proj", + "model.layers.14.mlp.experts.201.gate_proj", + "model.layers.14.mlp.experts.201.up_proj", + "model.layers.14.mlp.experts.202.down_proj", + "model.layers.14.mlp.experts.202.gate_proj", + "model.layers.14.mlp.experts.202.up_proj", + "model.layers.14.mlp.experts.203.down_proj", + "model.layers.14.mlp.experts.203.gate_proj", + "model.layers.14.mlp.experts.203.up_proj", + "model.layers.14.mlp.experts.204.down_proj", + "model.layers.14.mlp.experts.204.gate_proj", + "model.layers.14.mlp.experts.204.up_proj", + "model.layers.14.mlp.experts.205.down_proj", + "model.layers.14.mlp.experts.205.gate_proj", + "model.layers.14.mlp.experts.205.up_proj", + "model.layers.14.mlp.experts.206.down_proj", + "model.layers.14.mlp.experts.206.gate_proj", + "model.layers.14.mlp.experts.206.up_proj", + "model.layers.14.mlp.experts.207.down_proj", + "model.layers.14.mlp.experts.207.gate_proj", + "model.layers.14.mlp.experts.207.up_proj", + "model.layers.14.mlp.experts.208.down_proj", + "model.layers.14.mlp.experts.208.gate_proj", + "model.layers.14.mlp.experts.208.up_proj", + "model.layers.14.mlp.experts.209.down_proj", + "model.layers.14.mlp.experts.209.gate_proj", + "model.layers.14.mlp.experts.209.up_proj", + "model.layers.14.mlp.experts.21.down_proj", + "model.layers.14.mlp.experts.21.gate_proj", + "model.layers.14.mlp.experts.21.up_proj", + "model.layers.14.mlp.experts.210.down_proj", + "model.layers.14.mlp.experts.210.gate_proj", + "model.layers.14.mlp.experts.210.up_proj", + "model.layers.14.mlp.experts.211.down_proj", + "model.layers.14.mlp.experts.211.gate_proj", + "model.layers.14.mlp.experts.211.up_proj", + "model.layers.14.mlp.experts.212.down_proj", + "model.layers.14.mlp.experts.212.gate_proj", + "model.layers.14.mlp.experts.212.up_proj", + "model.layers.14.mlp.experts.213.down_proj", + "model.layers.14.mlp.experts.213.gate_proj", + "model.layers.14.mlp.experts.213.up_proj", + "model.layers.14.mlp.experts.214.down_proj", + "model.layers.14.mlp.experts.214.gate_proj", + "model.layers.14.mlp.experts.214.up_proj", + "model.layers.14.mlp.experts.215.down_proj", + "model.layers.14.mlp.experts.215.gate_proj", + "model.layers.14.mlp.experts.215.up_proj", + "model.layers.14.mlp.experts.216.down_proj", + "model.layers.14.mlp.experts.216.gate_proj", + "model.layers.14.mlp.experts.216.up_proj", + "model.layers.14.mlp.experts.217.down_proj", + "model.layers.14.mlp.experts.217.gate_proj", + "model.layers.14.mlp.experts.217.up_proj", + "model.layers.14.mlp.experts.218.down_proj", + "model.layers.14.mlp.experts.218.gate_proj", + "model.layers.14.mlp.experts.218.up_proj", + "model.layers.14.mlp.experts.219.down_proj", + "model.layers.14.mlp.experts.219.gate_proj", + "model.layers.14.mlp.experts.219.up_proj", + "model.layers.14.mlp.experts.22.down_proj", + "model.layers.14.mlp.experts.22.gate_proj", + "model.layers.14.mlp.experts.22.up_proj", + "model.layers.14.mlp.experts.220.down_proj", + "model.layers.14.mlp.experts.220.gate_proj", + "model.layers.14.mlp.experts.220.up_proj", + "model.layers.14.mlp.experts.221.down_proj", + "model.layers.14.mlp.experts.221.gate_proj", + "model.layers.14.mlp.experts.221.up_proj", + "model.layers.14.mlp.experts.222.down_proj", + "model.layers.14.mlp.experts.222.gate_proj", + "model.layers.14.mlp.experts.222.up_proj", + "model.layers.14.mlp.experts.223.down_proj", + "model.layers.14.mlp.experts.223.gate_proj", + "model.layers.14.mlp.experts.223.up_proj", + "model.layers.14.mlp.experts.224.down_proj", + "model.layers.14.mlp.experts.224.gate_proj", + "model.layers.14.mlp.experts.224.up_proj", + "model.layers.14.mlp.experts.225.down_proj", + "model.layers.14.mlp.experts.225.gate_proj", + "model.layers.14.mlp.experts.225.up_proj", + "model.layers.14.mlp.experts.226.down_proj", + "model.layers.14.mlp.experts.226.gate_proj", + "model.layers.14.mlp.experts.226.up_proj", + "model.layers.14.mlp.experts.227.down_proj", + "model.layers.14.mlp.experts.227.gate_proj", + "model.layers.14.mlp.experts.227.up_proj", + "model.layers.14.mlp.experts.228.down_proj", + "model.layers.14.mlp.experts.228.gate_proj", + "model.layers.14.mlp.experts.228.up_proj", + "model.layers.14.mlp.experts.229.down_proj", + "model.layers.14.mlp.experts.229.gate_proj", + "model.layers.14.mlp.experts.229.up_proj", + "model.layers.14.mlp.experts.23.down_proj", + "model.layers.14.mlp.experts.23.gate_proj", + "model.layers.14.mlp.experts.23.up_proj", + "model.layers.14.mlp.experts.230.down_proj", + "model.layers.14.mlp.experts.230.gate_proj", + "model.layers.14.mlp.experts.230.up_proj", + "model.layers.14.mlp.experts.231.down_proj", + "model.layers.14.mlp.experts.231.gate_proj", + "model.layers.14.mlp.experts.231.up_proj", + "model.layers.14.mlp.experts.232.down_proj", + "model.layers.14.mlp.experts.232.gate_proj", + "model.layers.14.mlp.experts.232.up_proj", + "model.layers.14.mlp.experts.233.down_proj", + "model.layers.14.mlp.experts.233.gate_proj", + "model.layers.14.mlp.experts.233.up_proj", + "model.layers.14.mlp.experts.234.down_proj", + "model.layers.14.mlp.experts.234.gate_proj", + "model.layers.14.mlp.experts.234.up_proj", + "model.layers.14.mlp.experts.235.down_proj", + "model.layers.14.mlp.experts.235.gate_proj", + "model.layers.14.mlp.experts.235.up_proj", + "model.layers.14.mlp.experts.236.down_proj", + "model.layers.14.mlp.experts.236.gate_proj", + "model.layers.14.mlp.experts.236.up_proj", + "model.layers.14.mlp.experts.237.down_proj", + "model.layers.14.mlp.experts.237.gate_proj", + "model.layers.14.mlp.experts.237.up_proj", + "model.layers.14.mlp.experts.238.down_proj", + "model.layers.14.mlp.experts.238.gate_proj", + "model.layers.14.mlp.experts.238.up_proj", + "model.layers.14.mlp.experts.239.down_proj", + "model.layers.14.mlp.experts.239.gate_proj", + "model.layers.14.mlp.experts.239.up_proj", + "model.layers.14.mlp.experts.24.down_proj", + "model.layers.14.mlp.experts.24.gate_proj", + "model.layers.14.mlp.experts.24.up_proj", + "model.layers.14.mlp.experts.240.down_proj", + "model.layers.14.mlp.experts.240.gate_proj", + "model.layers.14.mlp.experts.240.up_proj", + "model.layers.14.mlp.experts.241.down_proj", + "model.layers.14.mlp.experts.241.gate_proj", + "model.layers.14.mlp.experts.241.up_proj", + "model.layers.14.mlp.experts.242.down_proj", + "model.layers.14.mlp.experts.242.gate_proj", + "model.layers.14.mlp.experts.242.up_proj", + "model.layers.14.mlp.experts.243.down_proj", + "model.layers.14.mlp.experts.243.gate_proj", + "model.layers.14.mlp.experts.243.up_proj", + "model.layers.14.mlp.experts.244.down_proj", + "model.layers.14.mlp.experts.244.gate_proj", + "model.layers.14.mlp.experts.244.up_proj", + "model.layers.14.mlp.experts.245.down_proj", + "model.layers.14.mlp.experts.245.gate_proj", + "model.layers.14.mlp.experts.245.up_proj", + "model.layers.14.mlp.experts.246.down_proj", + "model.layers.14.mlp.experts.246.gate_proj", + "model.layers.14.mlp.experts.246.up_proj", + "model.layers.14.mlp.experts.247.down_proj", + "model.layers.14.mlp.experts.247.gate_proj", + "model.layers.14.mlp.experts.247.up_proj", + "model.layers.14.mlp.experts.248.down_proj", + "model.layers.14.mlp.experts.248.gate_proj", + "model.layers.14.mlp.experts.248.up_proj", + "model.layers.14.mlp.experts.249.down_proj", + "model.layers.14.mlp.experts.249.gate_proj", + "model.layers.14.mlp.experts.249.up_proj", + "model.layers.14.mlp.experts.25.down_proj", + "model.layers.14.mlp.experts.25.gate_proj", + "model.layers.14.mlp.experts.25.up_proj", + "model.layers.14.mlp.experts.250.down_proj", + "model.layers.14.mlp.experts.250.gate_proj", + "model.layers.14.mlp.experts.250.up_proj", + "model.layers.14.mlp.experts.251.down_proj", + "model.layers.14.mlp.experts.251.gate_proj", + "model.layers.14.mlp.experts.251.up_proj", + "model.layers.14.mlp.experts.252.down_proj", + "model.layers.14.mlp.experts.252.gate_proj", + "model.layers.14.mlp.experts.252.up_proj", + "model.layers.14.mlp.experts.253.down_proj", + "model.layers.14.mlp.experts.253.gate_proj", + "model.layers.14.mlp.experts.253.up_proj", + "model.layers.14.mlp.experts.254.down_proj", + "model.layers.14.mlp.experts.254.gate_proj", + "model.layers.14.mlp.experts.254.up_proj", + "model.layers.14.mlp.experts.255.down_proj", + "model.layers.14.mlp.experts.255.gate_proj", + "model.layers.14.mlp.experts.255.up_proj", + "model.layers.14.mlp.experts.26.down_proj", + "model.layers.14.mlp.experts.26.gate_proj", + "model.layers.14.mlp.experts.26.up_proj", + "model.layers.14.mlp.experts.27.down_proj", + "model.layers.14.mlp.experts.27.gate_proj", + "model.layers.14.mlp.experts.27.up_proj", + "model.layers.14.mlp.experts.28.down_proj", + "model.layers.14.mlp.experts.28.gate_proj", + "model.layers.14.mlp.experts.28.up_proj", + "model.layers.14.mlp.experts.29.down_proj", + "model.layers.14.mlp.experts.29.gate_proj", + "model.layers.14.mlp.experts.29.up_proj", + "model.layers.14.mlp.experts.3.down_proj", + "model.layers.14.mlp.experts.3.gate_proj", + "model.layers.14.mlp.experts.3.up_proj", + "model.layers.14.mlp.experts.30.down_proj", + "model.layers.14.mlp.experts.30.gate_proj", + "model.layers.14.mlp.experts.30.up_proj", + "model.layers.14.mlp.experts.31.down_proj", + "model.layers.14.mlp.experts.31.gate_proj", + "model.layers.14.mlp.experts.31.up_proj", + "model.layers.14.mlp.experts.32.down_proj", + "model.layers.14.mlp.experts.32.gate_proj", + "model.layers.14.mlp.experts.32.up_proj", + "model.layers.14.mlp.experts.33.down_proj", + "model.layers.14.mlp.experts.33.gate_proj", + "model.layers.14.mlp.experts.33.up_proj", + "model.layers.14.mlp.experts.34.down_proj", + "model.layers.14.mlp.experts.34.gate_proj", + "model.layers.14.mlp.experts.34.up_proj", + "model.layers.14.mlp.experts.35.down_proj", + "model.layers.14.mlp.experts.35.gate_proj", + "model.layers.14.mlp.experts.35.up_proj", + "model.layers.14.mlp.experts.36.down_proj", + "model.layers.14.mlp.experts.36.gate_proj", + "model.layers.14.mlp.experts.36.up_proj", + "model.layers.14.mlp.experts.37.down_proj", + "model.layers.14.mlp.experts.37.gate_proj", + "model.layers.14.mlp.experts.37.up_proj", + "model.layers.14.mlp.experts.38.down_proj", + "model.layers.14.mlp.experts.38.gate_proj", + "model.layers.14.mlp.experts.38.up_proj", + "model.layers.14.mlp.experts.39.down_proj", + "model.layers.14.mlp.experts.39.gate_proj", + "model.layers.14.mlp.experts.39.up_proj", + "model.layers.14.mlp.experts.4.down_proj", + "model.layers.14.mlp.experts.4.gate_proj", + "model.layers.14.mlp.experts.4.up_proj", + "model.layers.14.mlp.experts.40.down_proj", + "model.layers.14.mlp.experts.40.gate_proj", + "model.layers.14.mlp.experts.40.up_proj", + "model.layers.14.mlp.experts.41.down_proj", + "model.layers.14.mlp.experts.41.gate_proj", + "model.layers.14.mlp.experts.41.up_proj", + "model.layers.14.mlp.experts.42.down_proj", + "model.layers.14.mlp.experts.42.gate_proj", + "model.layers.14.mlp.experts.42.up_proj", + "model.layers.14.mlp.experts.43.down_proj", + "model.layers.14.mlp.experts.43.gate_proj", + "model.layers.14.mlp.experts.43.up_proj", + "model.layers.14.mlp.experts.44.down_proj", + "model.layers.14.mlp.experts.44.gate_proj", + "model.layers.14.mlp.experts.44.up_proj", + "model.layers.14.mlp.experts.45.down_proj", + "model.layers.14.mlp.experts.45.gate_proj", + "model.layers.14.mlp.experts.45.up_proj", + "model.layers.14.mlp.experts.46.down_proj", + "model.layers.14.mlp.experts.46.gate_proj", + "model.layers.14.mlp.experts.46.up_proj", + "model.layers.14.mlp.experts.47.down_proj", + "model.layers.14.mlp.experts.47.gate_proj", + "model.layers.14.mlp.experts.47.up_proj", + "model.layers.14.mlp.experts.48.down_proj", + "model.layers.14.mlp.experts.48.gate_proj", + "model.layers.14.mlp.experts.48.up_proj", + "model.layers.14.mlp.experts.49.down_proj", + "model.layers.14.mlp.experts.49.gate_proj", + "model.layers.14.mlp.experts.49.up_proj", + "model.layers.14.mlp.experts.5.down_proj", + "model.layers.14.mlp.experts.5.gate_proj", + "model.layers.14.mlp.experts.5.up_proj", + "model.layers.14.mlp.experts.50.down_proj", + "model.layers.14.mlp.experts.50.gate_proj", + "model.layers.14.mlp.experts.50.up_proj", + "model.layers.14.mlp.experts.51.down_proj", + "model.layers.14.mlp.experts.51.gate_proj", + "model.layers.14.mlp.experts.51.up_proj", + "model.layers.14.mlp.experts.52.down_proj", + "model.layers.14.mlp.experts.52.gate_proj", + "model.layers.14.mlp.experts.52.up_proj", + "model.layers.14.mlp.experts.53.down_proj", + "model.layers.14.mlp.experts.53.gate_proj", + "model.layers.14.mlp.experts.53.up_proj", + "model.layers.14.mlp.experts.54.down_proj", + "model.layers.14.mlp.experts.54.gate_proj", + "model.layers.14.mlp.experts.54.up_proj", + "model.layers.14.mlp.experts.55.down_proj", + "model.layers.14.mlp.experts.55.gate_proj", + "model.layers.14.mlp.experts.55.up_proj", + "model.layers.14.mlp.experts.56.down_proj", + "model.layers.14.mlp.experts.56.gate_proj", + "model.layers.14.mlp.experts.56.up_proj", + "model.layers.14.mlp.experts.57.down_proj", + "model.layers.14.mlp.experts.57.gate_proj", + "model.layers.14.mlp.experts.57.up_proj", + "model.layers.14.mlp.experts.58.down_proj", + "model.layers.14.mlp.experts.58.gate_proj", + "model.layers.14.mlp.experts.58.up_proj", + "model.layers.14.mlp.experts.59.down_proj", + "model.layers.14.mlp.experts.59.gate_proj", + "model.layers.14.mlp.experts.59.up_proj", + "model.layers.14.mlp.experts.6.down_proj", + "model.layers.14.mlp.experts.6.gate_proj", + "model.layers.14.mlp.experts.6.up_proj", + "model.layers.14.mlp.experts.60.down_proj", + "model.layers.14.mlp.experts.60.gate_proj", + "model.layers.14.mlp.experts.60.up_proj", + "model.layers.14.mlp.experts.61.down_proj", + "model.layers.14.mlp.experts.61.gate_proj", + "model.layers.14.mlp.experts.61.up_proj", + "model.layers.14.mlp.experts.62.down_proj", + "model.layers.14.mlp.experts.62.gate_proj", + "model.layers.14.mlp.experts.62.up_proj", + "model.layers.14.mlp.experts.63.down_proj", + "model.layers.14.mlp.experts.63.gate_proj", + "model.layers.14.mlp.experts.63.up_proj", + "model.layers.14.mlp.experts.64.down_proj", + "model.layers.14.mlp.experts.64.gate_proj", + "model.layers.14.mlp.experts.64.up_proj", + "model.layers.14.mlp.experts.65.down_proj", + "model.layers.14.mlp.experts.65.gate_proj", + "model.layers.14.mlp.experts.65.up_proj", + "model.layers.14.mlp.experts.66.down_proj", + "model.layers.14.mlp.experts.66.gate_proj", + "model.layers.14.mlp.experts.66.up_proj", + "model.layers.14.mlp.experts.67.down_proj", + "model.layers.14.mlp.experts.67.gate_proj", + "model.layers.14.mlp.experts.67.up_proj", + "model.layers.14.mlp.experts.68.down_proj", + "model.layers.14.mlp.experts.68.gate_proj", + "model.layers.14.mlp.experts.68.up_proj", + "model.layers.14.mlp.experts.69.down_proj", + "model.layers.14.mlp.experts.69.gate_proj", + "model.layers.14.mlp.experts.69.up_proj", + "model.layers.14.mlp.experts.7.down_proj", + "model.layers.14.mlp.experts.7.gate_proj", + "model.layers.14.mlp.experts.7.up_proj", + "model.layers.14.mlp.experts.70.down_proj", + "model.layers.14.mlp.experts.70.gate_proj", + "model.layers.14.mlp.experts.70.up_proj", + "model.layers.14.mlp.experts.71.down_proj", + "model.layers.14.mlp.experts.71.gate_proj", + "model.layers.14.mlp.experts.71.up_proj", + "model.layers.14.mlp.experts.72.down_proj", + "model.layers.14.mlp.experts.72.gate_proj", + "model.layers.14.mlp.experts.72.up_proj", + "model.layers.14.mlp.experts.73.down_proj", + "model.layers.14.mlp.experts.73.gate_proj", + "model.layers.14.mlp.experts.73.up_proj", + "model.layers.14.mlp.experts.74.down_proj", + "model.layers.14.mlp.experts.74.gate_proj", + "model.layers.14.mlp.experts.74.up_proj", + "model.layers.14.mlp.experts.75.down_proj", + "model.layers.14.mlp.experts.75.gate_proj", + "model.layers.14.mlp.experts.75.up_proj", + "model.layers.14.mlp.experts.76.down_proj", + "model.layers.14.mlp.experts.76.gate_proj", + "model.layers.14.mlp.experts.76.up_proj", + "model.layers.14.mlp.experts.77.down_proj", + "model.layers.14.mlp.experts.77.gate_proj", + "model.layers.14.mlp.experts.77.up_proj", + "model.layers.14.mlp.experts.78.down_proj", + "model.layers.14.mlp.experts.78.gate_proj", + "model.layers.14.mlp.experts.78.up_proj", + "model.layers.14.mlp.experts.79.down_proj", + "model.layers.14.mlp.experts.79.gate_proj", + "model.layers.14.mlp.experts.79.up_proj", + "model.layers.14.mlp.experts.8.down_proj", + "model.layers.14.mlp.experts.8.gate_proj", + "model.layers.14.mlp.experts.8.up_proj", + "model.layers.14.mlp.experts.80.down_proj", + "model.layers.14.mlp.experts.80.gate_proj", + "model.layers.14.mlp.experts.80.up_proj", + "model.layers.14.mlp.experts.81.down_proj", + "model.layers.14.mlp.experts.81.gate_proj", + "model.layers.14.mlp.experts.81.up_proj", + "model.layers.14.mlp.experts.82.down_proj", + "model.layers.14.mlp.experts.82.gate_proj", + "model.layers.14.mlp.experts.82.up_proj", + "model.layers.14.mlp.experts.83.down_proj", + "model.layers.14.mlp.experts.83.gate_proj", + "model.layers.14.mlp.experts.83.up_proj", + "model.layers.14.mlp.experts.84.down_proj", + "model.layers.14.mlp.experts.84.gate_proj", + "model.layers.14.mlp.experts.84.up_proj", + "model.layers.14.mlp.experts.85.down_proj", + "model.layers.14.mlp.experts.85.gate_proj", + "model.layers.14.mlp.experts.85.up_proj", + "model.layers.14.mlp.experts.86.down_proj", + "model.layers.14.mlp.experts.86.gate_proj", + "model.layers.14.mlp.experts.86.up_proj", + "model.layers.14.mlp.experts.87.down_proj", + "model.layers.14.mlp.experts.87.gate_proj", + "model.layers.14.mlp.experts.87.up_proj", + "model.layers.14.mlp.experts.88.down_proj", + "model.layers.14.mlp.experts.88.gate_proj", + "model.layers.14.mlp.experts.88.up_proj", + "model.layers.14.mlp.experts.89.down_proj", + "model.layers.14.mlp.experts.89.gate_proj", + "model.layers.14.mlp.experts.89.up_proj", + "model.layers.14.mlp.experts.9.down_proj", + "model.layers.14.mlp.experts.9.gate_proj", + "model.layers.14.mlp.experts.9.up_proj", + "model.layers.14.mlp.experts.90.down_proj", + "model.layers.14.mlp.experts.90.gate_proj", + "model.layers.14.mlp.experts.90.up_proj", + "model.layers.14.mlp.experts.91.down_proj", + "model.layers.14.mlp.experts.91.gate_proj", + "model.layers.14.mlp.experts.91.up_proj", + "model.layers.14.mlp.experts.92.down_proj", + "model.layers.14.mlp.experts.92.gate_proj", + "model.layers.14.mlp.experts.92.up_proj", + "model.layers.14.mlp.experts.93.down_proj", + "model.layers.14.mlp.experts.93.gate_proj", + "model.layers.14.mlp.experts.93.up_proj", + "model.layers.14.mlp.experts.94.down_proj", + "model.layers.14.mlp.experts.94.gate_proj", + "model.layers.14.mlp.experts.94.up_proj", + "model.layers.14.mlp.experts.95.down_proj", + "model.layers.14.mlp.experts.95.gate_proj", + "model.layers.14.mlp.experts.95.up_proj", + "model.layers.14.mlp.experts.96.down_proj", + "model.layers.14.mlp.experts.96.gate_proj", + "model.layers.14.mlp.experts.96.up_proj", + "model.layers.14.mlp.experts.97.down_proj", + "model.layers.14.mlp.experts.97.gate_proj", + "model.layers.14.mlp.experts.97.up_proj", + "model.layers.14.mlp.experts.98.down_proj", + "model.layers.14.mlp.experts.98.gate_proj", + "model.layers.14.mlp.experts.98.up_proj", + "model.layers.14.mlp.experts.99.down_proj", + "model.layers.14.mlp.experts.99.gate_proj", + "model.layers.14.mlp.experts.99.up_proj", + "model.layers.15.mlp.experts.0.down_proj", + "model.layers.15.mlp.experts.0.gate_proj", + "model.layers.15.mlp.experts.0.up_proj", + "model.layers.15.mlp.experts.1.down_proj", + "model.layers.15.mlp.experts.1.gate_proj", + "model.layers.15.mlp.experts.1.up_proj", + "model.layers.15.mlp.experts.10.down_proj", + "model.layers.15.mlp.experts.10.gate_proj", + "model.layers.15.mlp.experts.10.up_proj", + "model.layers.15.mlp.experts.100.down_proj", + "model.layers.15.mlp.experts.100.gate_proj", + "model.layers.15.mlp.experts.100.up_proj", + "model.layers.15.mlp.experts.101.down_proj", + "model.layers.15.mlp.experts.101.gate_proj", + "model.layers.15.mlp.experts.101.up_proj", + "model.layers.15.mlp.experts.102.down_proj", + "model.layers.15.mlp.experts.102.gate_proj", + "model.layers.15.mlp.experts.102.up_proj", + "model.layers.15.mlp.experts.103.down_proj", + "model.layers.15.mlp.experts.103.gate_proj", + "model.layers.15.mlp.experts.103.up_proj", + "model.layers.15.mlp.experts.104.down_proj", + "model.layers.15.mlp.experts.104.gate_proj", + "model.layers.15.mlp.experts.104.up_proj", + "model.layers.15.mlp.experts.105.down_proj", + "model.layers.15.mlp.experts.105.gate_proj", + "model.layers.15.mlp.experts.105.up_proj", + "model.layers.15.mlp.experts.106.down_proj", + "model.layers.15.mlp.experts.106.gate_proj", + "model.layers.15.mlp.experts.106.up_proj", + "model.layers.15.mlp.experts.107.down_proj", + "model.layers.15.mlp.experts.107.gate_proj", + "model.layers.15.mlp.experts.107.up_proj", + "model.layers.15.mlp.experts.108.down_proj", + "model.layers.15.mlp.experts.108.gate_proj", + "model.layers.15.mlp.experts.108.up_proj", + "model.layers.15.mlp.experts.109.down_proj", + "model.layers.15.mlp.experts.109.gate_proj", + "model.layers.15.mlp.experts.109.up_proj", + "model.layers.15.mlp.experts.11.down_proj", + "model.layers.15.mlp.experts.11.gate_proj", + "model.layers.15.mlp.experts.11.up_proj", + "model.layers.15.mlp.experts.110.down_proj", + "model.layers.15.mlp.experts.110.gate_proj", + "model.layers.15.mlp.experts.110.up_proj", + "model.layers.15.mlp.experts.111.down_proj", + "model.layers.15.mlp.experts.111.gate_proj", + "model.layers.15.mlp.experts.111.up_proj", + "model.layers.15.mlp.experts.112.down_proj", + "model.layers.15.mlp.experts.112.gate_proj", + "model.layers.15.mlp.experts.112.up_proj", + "model.layers.15.mlp.experts.113.down_proj", + "model.layers.15.mlp.experts.113.gate_proj", + "model.layers.15.mlp.experts.113.up_proj", + "model.layers.15.mlp.experts.114.down_proj", + "model.layers.15.mlp.experts.114.gate_proj", + "model.layers.15.mlp.experts.114.up_proj", + "model.layers.15.mlp.experts.115.down_proj", + "model.layers.15.mlp.experts.115.gate_proj", + "model.layers.15.mlp.experts.115.up_proj", + "model.layers.15.mlp.experts.116.down_proj", + "model.layers.15.mlp.experts.116.gate_proj", + "model.layers.15.mlp.experts.116.up_proj", + "model.layers.15.mlp.experts.117.down_proj", + "model.layers.15.mlp.experts.117.gate_proj", + "model.layers.15.mlp.experts.117.up_proj", + "model.layers.15.mlp.experts.118.down_proj", + "model.layers.15.mlp.experts.118.gate_proj", + "model.layers.15.mlp.experts.118.up_proj", + "model.layers.15.mlp.experts.119.down_proj", + "model.layers.15.mlp.experts.119.gate_proj", + "model.layers.15.mlp.experts.119.up_proj", + "model.layers.15.mlp.experts.12.down_proj", + "model.layers.15.mlp.experts.12.gate_proj", + "model.layers.15.mlp.experts.12.up_proj", + "model.layers.15.mlp.experts.120.down_proj", + "model.layers.15.mlp.experts.120.gate_proj", + "model.layers.15.mlp.experts.120.up_proj", + "model.layers.15.mlp.experts.121.down_proj", + "model.layers.15.mlp.experts.121.gate_proj", + "model.layers.15.mlp.experts.121.up_proj", + "model.layers.15.mlp.experts.122.down_proj", + "model.layers.15.mlp.experts.122.gate_proj", + "model.layers.15.mlp.experts.122.up_proj", + "model.layers.15.mlp.experts.123.down_proj", + "model.layers.15.mlp.experts.123.gate_proj", + "model.layers.15.mlp.experts.123.up_proj", + "model.layers.15.mlp.experts.124.down_proj", + "model.layers.15.mlp.experts.124.gate_proj", + "model.layers.15.mlp.experts.124.up_proj", + "model.layers.15.mlp.experts.125.down_proj", + "model.layers.15.mlp.experts.125.gate_proj", + "model.layers.15.mlp.experts.125.up_proj", + "model.layers.15.mlp.experts.126.down_proj", + "model.layers.15.mlp.experts.126.gate_proj", + "model.layers.15.mlp.experts.126.up_proj", + "model.layers.15.mlp.experts.127.down_proj", + "model.layers.15.mlp.experts.127.gate_proj", + "model.layers.15.mlp.experts.127.up_proj", + "model.layers.15.mlp.experts.128.down_proj", + "model.layers.15.mlp.experts.128.gate_proj", + "model.layers.15.mlp.experts.128.up_proj", + "model.layers.15.mlp.experts.129.down_proj", + "model.layers.15.mlp.experts.129.gate_proj", + "model.layers.15.mlp.experts.129.up_proj", + "model.layers.15.mlp.experts.13.down_proj", + "model.layers.15.mlp.experts.13.gate_proj", + "model.layers.15.mlp.experts.13.up_proj", + "model.layers.15.mlp.experts.130.down_proj", + "model.layers.15.mlp.experts.130.gate_proj", + "model.layers.15.mlp.experts.130.up_proj", + "model.layers.15.mlp.experts.131.down_proj", + "model.layers.15.mlp.experts.131.gate_proj", + "model.layers.15.mlp.experts.131.up_proj", + "model.layers.15.mlp.experts.132.down_proj", + "model.layers.15.mlp.experts.132.gate_proj", + "model.layers.15.mlp.experts.132.up_proj", + "model.layers.15.mlp.experts.133.down_proj", + "model.layers.15.mlp.experts.133.gate_proj", + "model.layers.15.mlp.experts.133.up_proj", + "model.layers.15.mlp.experts.134.down_proj", + "model.layers.15.mlp.experts.134.gate_proj", + "model.layers.15.mlp.experts.134.up_proj", + "model.layers.15.mlp.experts.135.down_proj", + "model.layers.15.mlp.experts.135.gate_proj", + "model.layers.15.mlp.experts.135.up_proj", + "model.layers.15.mlp.experts.136.down_proj", + "model.layers.15.mlp.experts.136.gate_proj", + "model.layers.15.mlp.experts.136.up_proj", + "model.layers.15.mlp.experts.137.down_proj", + "model.layers.15.mlp.experts.137.gate_proj", + "model.layers.15.mlp.experts.137.up_proj", + "model.layers.15.mlp.experts.138.down_proj", + "model.layers.15.mlp.experts.138.gate_proj", + "model.layers.15.mlp.experts.138.up_proj", + "model.layers.15.mlp.experts.139.down_proj", + "model.layers.15.mlp.experts.139.gate_proj", + "model.layers.15.mlp.experts.139.up_proj", + "model.layers.15.mlp.experts.14.down_proj", + "model.layers.15.mlp.experts.14.gate_proj", + "model.layers.15.mlp.experts.14.up_proj", + "model.layers.15.mlp.experts.140.down_proj", + "model.layers.15.mlp.experts.140.gate_proj", + "model.layers.15.mlp.experts.140.up_proj", + "model.layers.15.mlp.experts.141.down_proj", + "model.layers.15.mlp.experts.141.gate_proj", + "model.layers.15.mlp.experts.141.up_proj", + "model.layers.15.mlp.experts.142.down_proj", + "model.layers.15.mlp.experts.142.gate_proj", + "model.layers.15.mlp.experts.142.up_proj", + "model.layers.15.mlp.experts.143.down_proj", + "model.layers.15.mlp.experts.143.gate_proj", + "model.layers.15.mlp.experts.143.up_proj", + "model.layers.15.mlp.experts.144.down_proj", + "model.layers.15.mlp.experts.144.gate_proj", + "model.layers.15.mlp.experts.144.up_proj", + "model.layers.15.mlp.experts.145.down_proj", + "model.layers.15.mlp.experts.145.gate_proj", + "model.layers.15.mlp.experts.145.up_proj", + "model.layers.15.mlp.experts.146.down_proj", + "model.layers.15.mlp.experts.146.gate_proj", + "model.layers.15.mlp.experts.146.up_proj", + "model.layers.15.mlp.experts.147.down_proj", + "model.layers.15.mlp.experts.147.gate_proj", + "model.layers.15.mlp.experts.147.up_proj", + "model.layers.15.mlp.experts.148.down_proj", + "model.layers.15.mlp.experts.148.gate_proj", + "model.layers.15.mlp.experts.148.up_proj", + "model.layers.15.mlp.experts.149.down_proj", + "model.layers.15.mlp.experts.149.gate_proj", + "model.layers.15.mlp.experts.149.up_proj", + "model.layers.15.mlp.experts.15.down_proj", + "model.layers.15.mlp.experts.15.gate_proj", + "model.layers.15.mlp.experts.15.up_proj", + "model.layers.15.mlp.experts.150.down_proj", + "model.layers.15.mlp.experts.150.gate_proj", + "model.layers.15.mlp.experts.150.up_proj", + "model.layers.15.mlp.experts.151.down_proj", + "model.layers.15.mlp.experts.151.gate_proj", + "model.layers.15.mlp.experts.151.up_proj", + "model.layers.15.mlp.experts.152.down_proj", + "model.layers.15.mlp.experts.152.gate_proj", + "model.layers.15.mlp.experts.152.up_proj", + "model.layers.15.mlp.experts.153.down_proj", + "model.layers.15.mlp.experts.153.gate_proj", + "model.layers.15.mlp.experts.153.up_proj", + "model.layers.15.mlp.experts.154.down_proj", + "model.layers.15.mlp.experts.154.gate_proj", + "model.layers.15.mlp.experts.154.up_proj", + "model.layers.15.mlp.experts.155.down_proj", + "model.layers.15.mlp.experts.155.gate_proj", + "model.layers.15.mlp.experts.155.up_proj", + "model.layers.15.mlp.experts.156.down_proj", + "model.layers.15.mlp.experts.156.gate_proj", + "model.layers.15.mlp.experts.156.up_proj", + "model.layers.15.mlp.experts.157.down_proj", + "model.layers.15.mlp.experts.157.gate_proj", + "model.layers.15.mlp.experts.157.up_proj", + "model.layers.15.mlp.experts.158.down_proj", + "model.layers.15.mlp.experts.158.gate_proj", + "model.layers.15.mlp.experts.158.up_proj", + "model.layers.15.mlp.experts.159.down_proj", + "model.layers.15.mlp.experts.159.gate_proj", + "model.layers.15.mlp.experts.159.up_proj", + "model.layers.15.mlp.experts.16.down_proj", + "model.layers.15.mlp.experts.16.gate_proj", + "model.layers.15.mlp.experts.16.up_proj", + "model.layers.15.mlp.experts.160.down_proj", + "model.layers.15.mlp.experts.160.gate_proj", + "model.layers.15.mlp.experts.160.up_proj", + "model.layers.15.mlp.experts.161.down_proj", + "model.layers.15.mlp.experts.161.gate_proj", + "model.layers.15.mlp.experts.161.up_proj", + "model.layers.15.mlp.experts.162.down_proj", + "model.layers.15.mlp.experts.162.gate_proj", + "model.layers.15.mlp.experts.162.up_proj", + "model.layers.15.mlp.experts.163.down_proj", + "model.layers.15.mlp.experts.163.gate_proj", + "model.layers.15.mlp.experts.163.up_proj", + "model.layers.15.mlp.experts.164.down_proj", + "model.layers.15.mlp.experts.164.gate_proj", + "model.layers.15.mlp.experts.164.up_proj", + "model.layers.15.mlp.experts.165.down_proj", + "model.layers.15.mlp.experts.165.gate_proj", + "model.layers.15.mlp.experts.165.up_proj", + "model.layers.15.mlp.experts.166.down_proj", + "model.layers.15.mlp.experts.166.gate_proj", + "model.layers.15.mlp.experts.166.up_proj", + "model.layers.15.mlp.experts.167.down_proj", + "model.layers.15.mlp.experts.167.gate_proj", + "model.layers.15.mlp.experts.167.up_proj", + "model.layers.15.mlp.experts.168.down_proj", + "model.layers.15.mlp.experts.168.gate_proj", + "model.layers.15.mlp.experts.168.up_proj", + "model.layers.15.mlp.experts.169.down_proj", + "model.layers.15.mlp.experts.169.gate_proj", + "model.layers.15.mlp.experts.169.up_proj", + "model.layers.15.mlp.experts.17.down_proj", + "model.layers.15.mlp.experts.17.gate_proj", + "model.layers.15.mlp.experts.17.up_proj", + "model.layers.15.mlp.experts.170.down_proj", + "model.layers.15.mlp.experts.170.gate_proj", + "model.layers.15.mlp.experts.170.up_proj", + "model.layers.15.mlp.experts.171.down_proj", + "model.layers.15.mlp.experts.171.gate_proj", + "model.layers.15.mlp.experts.171.up_proj", + "model.layers.15.mlp.experts.172.down_proj", + "model.layers.15.mlp.experts.172.gate_proj", + "model.layers.15.mlp.experts.172.up_proj", + "model.layers.15.mlp.experts.173.down_proj", + "model.layers.15.mlp.experts.173.gate_proj", + "model.layers.15.mlp.experts.173.up_proj", + "model.layers.15.mlp.experts.174.down_proj", + "model.layers.15.mlp.experts.174.gate_proj", + "model.layers.15.mlp.experts.174.up_proj", + "model.layers.15.mlp.experts.175.down_proj", + "model.layers.15.mlp.experts.175.gate_proj", + "model.layers.15.mlp.experts.175.up_proj", + "model.layers.15.mlp.experts.176.down_proj", + "model.layers.15.mlp.experts.176.gate_proj", + "model.layers.15.mlp.experts.176.up_proj", + "model.layers.15.mlp.experts.177.down_proj", + "model.layers.15.mlp.experts.177.gate_proj", + "model.layers.15.mlp.experts.177.up_proj", + "model.layers.15.mlp.experts.178.down_proj", + "model.layers.15.mlp.experts.178.gate_proj", + "model.layers.15.mlp.experts.178.up_proj", + "model.layers.15.mlp.experts.179.down_proj", + "model.layers.15.mlp.experts.179.gate_proj", + "model.layers.15.mlp.experts.179.up_proj", + "model.layers.15.mlp.experts.18.down_proj", + "model.layers.15.mlp.experts.18.gate_proj", + "model.layers.15.mlp.experts.18.up_proj", + "model.layers.15.mlp.experts.180.down_proj", + "model.layers.15.mlp.experts.180.gate_proj", + "model.layers.15.mlp.experts.180.up_proj", + "model.layers.15.mlp.experts.181.down_proj", + "model.layers.15.mlp.experts.181.gate_proj", + "model.layers.15.mlp.experts.181.up_proj", + "model.layers.15.mlp.experts.182.down_proj", + "model.layers.15.mlp.experts.182.gate_proj", + "model.layers.15.mlp.experts.182.up_proj", + "model.layers.15.mlp.experts.183.down_proj", + "model.layers.15.mlp.experts.183.gate_proj", + "model.layers.15.mlp.experts.183.up_proj", + "model.layers.15.mlp.experts.184.down_proj", + "model.layers.15.mlp.experts.184.gate_proj", + "model.layers.15.mlp.experts.184.up_proj", + "model.layers.15.mlp.experts.185.down_proj", + "model.layers.15.mlp.experts.185.gate_proj", + "model.layers.15.mlp.experts.185.up_proj", + "model.layers.15.mlp.experts.186.down_proj", + "model.layers.15.mlp.experts.186.gate_proj", + "model.layers.15.mlp.experts.186.up_proj", + "model.layers.15.mlp.experts.187.down_proj", + "model.layers.15.mlp.experts.187.gate_proj", + "model.layers.15.mlp.experts.187.up_proj", + "model.layers.15.mlp.experts.188.down_proj", + "model.layers.15.mlp.experts.188.gate_proj", + "model.layers.15.mlp.experts.188.up_proj", + "model.layers.15.mlp.experts.189.down_proj", + "model.layers.15.mlp.experts.189.gate_proj", + "model.layers.15.mlp.experts.189.up_proj", + "model.layers.15.mlp.experts.19.down_proj", + "model.layers.15.mlp.experts.19.gate_proj", + "model.layers.15.mlp.experts.19.up_proj", + "model.layers.15.mlp.experts.190.down_proj", + "model.layers.15.mlp.experts.190.gate_proj", + "model.layers.15.mlp.experts.190.up_proj", + "model.layers.15.mlp.experts.191.down_proj", + "model.layers.15.mlp.experts.191.gate_proj", + "model.layers.15.mlp.experts.191.up_proj", + "model.layers.15.mlp.experts.192.down_proj", + "model.layers.15.mlp.experts.192.gate_proj", + "model.layers.15.mlp.experts.192.up_proj", + "model.layers.15.mlp.experts.193.down_proj", + "model.layers.15.mlp.experts.193.gate_proj", + "model.layers.15.mlp.experts.193.up_proj", + "model.layers.15.mlp.experts.194.down_proj", + "model.layers.15.mlp.experts.194.gate_proj", + "model.layers.15.mlp.experts.194.up_proj", + "model.layers.15.mlp.experts.195.down_proj", + "model.layers.15.mlp.experts.195.gate_proj", + "model.layers.15.mlp.experts.195.up_proj", + "model.layers.15.mlp.experts.196.down_proj", + "model.layers.15.mlp.experts.196.gate_proj", + "model.layers.15.mlp.experts.196.up_proj", + "model.layers.15.mlp.experts.197.down_proj", + "model.layers.15.mlp.experts.197.gate_proj", + "model.layers.15.mlp.experts.197.up_proj", + "model.layers.15.mlp.experts.198.down_proj", + "model.layers.15.mlp.experts.198.gate_proj", + "model.layers.15.mlp.experts.198.up_proj", + "model.layers.15.mlp.experts.199.down_proj", + "model.layers.15.mlp.experts.199.gate_proj", + "model.layers.15.mlp.experts.199.up_proj", + "model.layers.15.mlp.experts.2.down_proj", + "model.layers.15.mlp.experts.2.gate_proj", + "model.layers.15.mlp.experts.2.up_proj", + "model.layers.15.mlp.experts.20.down_proj", + "model.layers.15.mlp.experts.20.gate_proj", + "model.layers.15.mlp.experts.20.up_proj", + "model.layers.15.mlp.experts.200.down_proj", + "model.layers.15.mlp.experts.200.gate_proj", + "model.layers.15.mlp.experts.200.up_proj", + "model.layers.15.mlp.experts.201.down_proj", + "model.layers.15.mlp.experts.201.gate_proj", + "model.layers.15.mlp.experts.201.up_proj", + "model.layers.15.mlp.experts.202.down_proj", + "model.layers.15.mlp.experts.202.gate_proj", + "model.layers.15.mlp.experts.202.up_proj", + "model.layers.15.mlp.experts.203.down_proj", + "model.layers.15.mlp.experts.203.gate_proj", + "model.layers.15.mlp.experts.203.up_proj", + "model.layers.15.mlp.experts.204.down_proj", + "model.layers.15.mlp.experts.204.gate_proj", + "model.layers.15.mlp.experts.204.up_proj", + "model.layers.15.mlp.experts.205.down_proj", + "model.layers.15.mlp.experts.205.gate_proj", + "model.layers.15.mlp.experts.205.up_proj", + "model.layers.15.mlp.experts.206.down_proj", + "model.layers.15.mlp.experts.206.gate_proj", + "model.layers.15.mlp.experts.206.up_proj", + "model.layers.15.mlp.experts.207.down_proj", + "model.layers.15.mlp.experts.207.gate_proj", + "model.layers.15.mlp.experts.207.up_proj", + "model.layers.15.mlp.experts.208.down_proj", + "model.layers.15.mlp.experts.208.gate_proj", + "model.layers.15.mlp.experts.208.up_proj", + "model.layers.15.mlp.experts.209.down_proj", + "model.layers.15.mlp.experts.209.gate_proj", + "model.layers.15.mlp.experts.209.up_proj", + "model.layers.15.mlp.experts.21.down_proj", + "model.layers.15.mlp.experts.21.gate_proj", + "model.layers.15.mlp.experts.21.up_proj", + "model.layers.15.mlp.experts.210.down_proj", + "model.layers.15.mlp.experts.210.gate_proj", + "model.layers.15.mlp.experts.210.up_proj", + "model.layers.15.mlp.experts.211.down_proj", + "model.layers.15.mlp.experts.211.gate_proj", + "model.layers.15.mlp.experts.211.up_proj", + "model.layers.15.mlp.experts.212.down_proj", + "model.layers.15.mlp.experts.212.gate_proj", + "model.layers.15.mlp.experts.212.up_proj", + "model.layers.15.mlp.experts.213.down_proj", + "model.layers.15.mlp.experts.213.gate_proj", + "model.layers.15.mlp.experts.213.up_proj", + "model.layers.15.mlp.experts.214.down_proj", + "model.layers.15.mlp.experts.214.gate_proj", + "model.layers.15.mlp.experts.214.up_proj", + "model.layers.15.mlp.experts.215.down_proj", + "model.layers.15.mlp.experts.215.gate_proj", + "model.layers.15.mlp.experts.215.up_proj", + "model.layers.15.mlp.experts.216.down_proj", + "model.layers.15.mlp.experts.216.gate_proj", + "model.layers.15.mlp.experts.216.up_proj", + "model.layers.15.mlp.experts.217.down_proj", + "model.layers.15.mlp.experts.217.gate_proj", + "model.layers.15.mlp.experts.217.up_proj", + "model.layers.15.mlp.experts.218.down_proj", + "model.layers.15.mlp.experts.218.gate_proj", + "model.layers.15.mlp.experts.218.up_proj", + "model.layers.15.mlp.experts.219.down_proj", + "model.layers.15.mlp.experts.219.gate_proj", + "model.layers.15.mlp.experts.219.up_proj", + "model.layers.15.mlp.experts.22.down_proj", + "model.layers.15.mlp.experts.22.gate_proj", + "model.layers.15.mlp.experts.22.up_proj", + "model.layers.15.mlp.experts.220.down_proj", + "model.layers.15.mlp.experts.220.gate_proj", + "model.layers.15.mlp.experts.220.up_proj", + "model.layers.15.mlp.experts.221.down_proj", + "model.layers.15.mlp.experts.221.gate_proj", + "model.layers.15.mlp.experts.221.up_proj", + "model.layers.15.mlp.experts.222.down_proj", + "model.layers.15.mlp.experts.222.gate_proj", + "model.layers.15.mlp.experts.222.up_proj", + "model.layers.15.mlp.experts.223.down_proj", + "model.layers.15.mlp.experts.223.gate_proj", + "model.layers.15.mlp.experts.223.up_proj", + "model.layers.15.mlp.experts.224.down_proj", + "model.layers.15.mlp.experts.224.gate_proj", + "model.layers.15.mlp.experts.224.up_proj", + "model.layers.15.mlp.experts.225.down_proj", + "model.layers.15.mlp.experts.225.gate_proj", + "model.layers.15.mlp.experts.225.up_proj", + "model.layers.15.mlp.experts.226.down_proj", + "model.layers.15.mlp.experts.226.gate_proj", + "model.layers.15.mlp.experts.226.up_proj", + "model.layers.15.mlp.experts.227.down_proj", + "model.layers.15.mlp.experts.227.gate_proj", + "model.layers.15.mlp.experts.227.up_proj", + "model.layers.15.mlp.experts.228.down_proj", + "model.layers.15.mlp.experts.228.gate_proj", + "model.layers.15.mlp.experts.228.up_proj", + "model.layers.15.mlp.experts.229.down_proj", + "model.layers.15.mlp.experts.229.gate_proj", + "model.layers.15.mlp.experts.229.up_proj", + "model.layers.15.mlp.experts.23.down_proj", + "model.layers.15.mlp.experts.23.gate_proj", + "model.layers.15.mlp.experts.23.up_proj", + "model.layers.15.mlp.experts.230.down_proj", + "model.layers.15.mlp.experts.230.gate_proj", + "model.layers.15.mlp.experts.230.up_proj", + "model.layers.15.mlp.experts.231.down_proj", + "model.layers.15.mlp.experts.231.gate_proj", + "model.layers.15.mlp.experts.231.up_proj", + "model.layers.15.mlp.experts.232.down_proj", + "model.layers.15.mlp.experts.232.gate_proj", + "model.layers.15.mlp.experts.232.up_proj", + "model.layers.15.mlp.experts.233.down_proj", + "model.layers.15.mlp.experts.233.gate_proj", + "model.layers.15.mlp.experts.233.up_proj", + "model.layers.15.mlp.experts.234.down_proj", + "model.layers.15.mlp.experts.234.gate_proj", + "model.layers.15.mlp.experts.234.up_proj", + "model.layers.15.mlp.experts.235.down_proj", + "model.layers.15.mlp.experts.235.gate_proj", + "model.layers.15.mlp.experts.235.up_proj", + "model.layers.15.mlp.experts.236.down_proj", + "model.layers.15.mlp.experts.236.gate_proj", + "model.layers.15.mlp.experts.236.up_proj", + "model.layers.15.mlp.experts.237.down_proj", + "model.layers.15.mlp.experts.237.gate_proj", + "model.layers.15.mlp.experts.237.up_proj", + "model.layers.15.mlp.experts.238.down_proj", + "model.layers.15.mlp.experts.238.gate_proj", + "model.layers.15.mlp.experts.238.up_proj", + "model.layers.15.mlp.experts.239.down_proj", + "model.layers.15.mlp.experts.239.gate_proj", + "model.layers.15.mlp.experts.239.up_proj", + "model.layers.15.mlp.experts.24.down_proj", + "model.layers.15.mlp.experts.24.gate_proj", + "model.layers.15.mlp.experts.24.up_proj", + "model.layers.15.mlp.experts.240.down_proj", + "model.layers.15.mlp.experts.240.gate_proj", + "model.layers.15.mlp.experts.240.up_proj", + "model.layers.15.mlp.experts.241.down_proj", + "model.layers.15.mlp.experts.241.gate_proj", + "model.layers.15.mlp.experts.241.up_proj", + "model.layers.15.mlp.experts.242.down_proj", + "model.layers.15.mlp.experts.242.gate_proj", + "model.layers.15.mlp.experts.242.up_proj", + "model.layers.15.mlp.experts.243.down_proj", + "model.layers.15.mlp.experts.243.gate_proj", + "model.layers.15.mlp.experts.243.up_proj", + "model.layers.15.mlp.experts.244.down_proj", + "model.layers.15.mlp.experts.244.gate_proj", + "model.layers.15.mlp.experts.244.up_proj", + "model.layers.15.mlp.experts.245.down_proj", + "model.layers.15.mlp.experts.245.gate_proj", + "model.layers.15.mlp.experts.245.up_proj", + "model.layers.15.mlp.experts.246.down_proj", + "model.layers.15.mlp.experts.246.gate_proj", + "model.layers.15.mlp.experts.246.up_proj", + "model.layers.15.mlp.experts.247.down_proj", + "model.layers.15.mlp.experts.247.gate_proj", + "model.layers.15.mlp.experts.247.up_proj", + "model.layers.15.mlp.experts.248.down_proj", + "model.layers.15.mlp.experts.248.gate_proj", + "model.layers.15.mlp.experts.248.up_proj", + "model.layers.15.mlp.experts.249.down_proj", + "model.layers.15.mlp.experts.249.gate_proj", + "model.layers.15.mlp.experts.249.up_proj", + "model.layers.15.mlp.experts.25.down_proj", + "model.layers.15.mlp.experts.25.gate_proj", + "model.layers.15.mlp.experts.25.up_proj", + "model.layers.15.mlp.experts.250.down_proj", + "model.layers.15.mlp.experts.250.gate_proj", + "model.layers.15.mlp.experts.250.up_proj", + "model.layers.15.mlp.experts.251.down_proj", + "model.layers.15.mlp.experts.251.gate_proj", + "model.layers.15.mlp.experts.251.up_proj", + "model.layers.15.mlp.experts.252.down_proj", + "model.layers.15.mlp.experts.252.gate_proj", + "model.layers.15.mlp.experts.252.up_proj", + "model.layers.15.mlp.experts.253.down_proj", + "model.layers.15.mlp.experts.253.gate_proj", + "model.layers.15.mlp.experts.253.up_proj", + "model.layers.15.mlp.experts.254.down_proj", + "model.layers.15.mlp.experts.254.gate_proj", + "model.layers.15.mlp.experts.254.up_proj", + "model.layers.15.mlp.experts.255.down_proj", + "model.layers.15.mlp.experts.255.gate_proj", + "model.layers.15.mlp.experts.255.up_proj", + "model.layers.15.mlp.experts.26.down_proj", + "model.layers.15.mlp.experts.26.gate_proj", + "model.layers.15.mlp.experts.26.up_proj", + "model.layers.15.mlp.experts.27.down_proj", + "model.layers.15.mlp.experts.27.gate_proj", + "model.layers.15.mlp.experts.27.up_proj", + "model.layers.15.mlp.experts.28.down_proj", + "model.layers.15.mlp.experts.28.gate_proj", + "model.layers.15.mlp.experts.28.up_proj", + "model.layers.15.mlp.experts.29.down_proj", + "model.layers.15.mlp.experts.29.gate_proj", + "model.layers.15.mlp.experts.29.up_proj", + "model.layers.15.mlp.experts.3.down_proj", + "model.layers.15.mlp.experts.3.gate_proj", + "model.layers.15.mlp.experts.3.up_proj", + "model.layers.15.mlp.experts.30.down_proj", + "model.layers.15.mlp.experts.30.gate_proj", + "model.layers.15.mlp.experts.30.up_proj", + "model.layers.15.mlp.experts.31.down_proj", + "model.layers.15.mlp.experts.31.gate_proj", + "model.layers.15.mlp.experts.31.up_proj", + "model.layers.15.mlp.experts.32.down_proj", + "model.layers.15.mlp.experts.32.gate_proj", + "model.layers.15.mlp.experts.32.up_proj", + "model.layers.15.mlp.experts.33.down_proj", + "model.layers.15.mlp.experts.33.gate_proj", + "model.layers.15.mlp.experts.33.up_proj", + "model.layers.15.mlp.experts.34.down_proj", + "model.layers.15.mlp.experts.34.gate_proj", + "model.layers.15.mlp.experts.34.up_proj", + "model.layers.15.mlp.experts.35.down_proj", + "model.layers.15.mlp.experts.35.gate_proj", + "model.layers.15.mlp.experts.35.up_proj", + "model.layers.15.mlp.experts.36.down_proj", + "model.layers.15.mlp.experts.36.gate_proj", + "model.layers.15.mlp.experts.36.up_proj", + "model.layers.15.mlp.experts.37.down_proj", + "model.layers.15.mlp.experts.37.gate_proj", + "model.layers.15.mlp.experts.37.up_proj", + "model.layers.15.mlp.experts.38.down_proj", + "model.layers.15.mlp.experts.38.gate_proj", + "model.layers.15.mlp.experts.38.up_proj", + "model.layers.15.mlp.experts.39.down_proj", + "model.layers.15.mlp.experts.39.gate_proj", + "model.layers.15.mlp.experts.39.up_proj", + "model.layers.15.mlp.experts.4.down_proj", + "model.layers.15.mlp.experts.4.gate_proj", + "model.layers.15.mlp.experts.4.up_proj", + "model.layers.15.mlp.experts.40.down_proj", + "model.layers.15.mlp.experts.40.gate_proj", + "model.layers.15.mlp.experts.40.up_proj", + "model.layers.15.mlp.experts.41.down_proj", + "model.layers.15.mlp.experts.41.gate_proj", + "model.layers.15.mlp.experts.41.up_proj", + "model.layers.15.mlp.experts.42.down_proj", + "model.layers.15.mlp.experts.42.gate_proj", + "model.layers.15.mlp.experts.42.up_proj", + "model.layers.15.mlp.experts.43.down_proj", + "model.layers.15.mlp.experts.43.gate_proj", + "model.layers.15.mlp.experts.43.up_proj", + "model.layers.15.mlp.experts.44.down_proj", + "model.layers.15.mlp.experts.44.gate_proj", + "model.layers.15.mlp.experts.44.up_proj", + "model.layers.15.mlp.experts.45.down_proj", + "model.layers.15.mlp.experts.45.gate_proj", + "model.layers.15.mlp.experts.45.up_proj", + "model.layers.15.mlp.experts.46.down_proj", + "model.layers.15.mlp.experts.46.gate_proj", + "model.layers.15.mlp.experts.46.up_proj", + "model.layers.15.mlp.experts.47.down_proj", + "model.layers.15.mlp.experts.47.gate_proj", + "model.layers.15.mlp.experts.47.up_proj", + "model.layers.15.mlp.experts.48.down_proj", + "model.layers.15.mlp.experts.48.gate_proj", + "model.layers.15.mlp.experts.48.up_proj", + "model.layers.15.mlp.experts.49.down_proj", + "model.layers.15.mlp.experts.49.gate_proj", + "model.layers.15.mlp.experts.49.up_proj", + "model.layers.15.mlp.experts.5.down_proj", + "model.layers.15.mlp.experts.5.gate_proj", + "model.layers.15.mlp.experts.5.up_proj", + "model.layers.15.mlp.experts.50.down_proj", + "model.layers.15.mlp.experts.50.gate_proj", + "model.layers.15.mlp.experts.50.up_proj", + "model.layers.15.mlp.experts.51.down_proj", + "model.layers.15.mlp.experts.51.gate_proj", + "model.layers.15.mlp.experts.51.up_proj", + "model.layers.15.mlp.experts.52.down_proj", + "model.layers.15.mlp.experts.52.gate_proj", + "model.layers.15.mlp.experts.52.up_proj", + "model.layers.15.mlp.experts.53.down_proj", + "model.layers.15.mlp.experts.53.gate_proj", + "model.layers.15.mlp.experts.53.up_proj", + "model.layers.15.mlp.experts.54.down_proj", + "model.layers.15.mlp.experts.54.gate_proj", + "model.layers.15.mlp.experts.54.up_proj", + "model.layers.15.mlp.experts.55.down_proj", + "model.layers.15.mlp.experts.55.gate_proj", + "model.layers.15.mlp.experts.55.up_proj", + "model.layers.15.mlp.experts.56.down_proj", + "model.layers.15.mlp.experts.56.gate_proj", + "model.layers.15.mlp.experts.56.up_proj", + "model.layers.15.mlp.experts.57.down_proj", + "model.layers.15.mlp.experts.57.gate_proj", + "model.layers.15.mlp.experts.57.up_proj", + "model.layers.15.mlp.experts.58.down_proj", + "model.layers.15.mlp.experts.58.gate_proj", + "model.layers.15.mlp.experts.58.up_proj", + "model.layers.15.mlp.experts.59.down_proj", + "model.layers.15.mlp.experts.59.gate_proj", + "model.layers.15.mlp.experts.59.up_proj", + "model.layers.15.mlp.experts.6.down_proj", + "model.layers.15.mlp.experts.6.gate_proj", + "model.layers.15.mlp.experts.6.up_proj", + "model.layers.15.mlp.experts.60.down_proj", + "model.layers.15.mlp.experts.60.gate_proj", + "model.layers.15.mlp.experts.60.up_proj", + "model.layers.15.mlp.experts.61.down_proj", + "model.layers.15.mlp.experts.61.gate_proj", + "model.layers.15.mlp.experts.61.up_proj", + "model.layers.15.mlp.experts.62.down_proj", + "model.layers.15.mlp.experts.62.gate_proj", + "model.layers.15.mlp.experts.62.up_proj", + "model.layers.15.mlp.experts.63.down_proj", + "model.layers.15.mlp.experts.63.gate_proj", + "model.layers.15.mlp.experts.63.up_proj", + "model.layers.15.mlp.experts.64.down_proj", + "model.layers.15.mlp.experts.64.gate_proj", + "model.layers.15.mlp.experts.64.up_proj", + "model.layers.15.mlp.experts.65.down_proj", + "model.layers.15.mlp.experts.65.gate_proj", + "model.layers.15.mlp.experts.65.up_proj", + "model.layers.15.mlp.experts.66.down_proj", + "model.layers.15.mlp.experts.66.gate_proj", + "model.layers.15.mlp.experts.66.up_proj", + "model.layers.15.mlp.experts.67.down_proj", + "model.layers.15.mlp.experts.67.gate_proj", + "model.layers.15.mlp.experts.67.up_proj", + "model.layers.15.mlp.experts.68.down_proj", + "model.layers.15.mlp.experts.68.gate_proj", + "model.layers.15.mlp.experts.68.up_proj", + "model.layers.15.mlp.experts.69.down_proj", + "model.layers.15.mlp.experts.69.gate_proj", + "model.layers.15.mlp.experts.69.up_proj", + "model.layers.15.mlp.experts.7.down_proj", + "model.layers.15.mlp.experts.7.gate_proj", + "model.layers.15.mlp.experts.7.up_proj", + "model.layers.15.mlp.experts.70.down_proj", + "model.layers.15.mlp.experts.70.gate_proj", + "model.layers.15.mlp.experts.70.up_proj", + "model.layers.15.mlp.experts.71.down_proj", + "model.layers.15.mlp.experts.71.gate_proj", + "model.layers.15.mlp.experts.71.up_proj", + "model.layers.15.mlp.experts.72.down_proj", + "model.layers.15.mlp.experts.72.gate_proj", + "model.layers.15.mlp.experts.72.up_proj", + "model.layers.15.mlp.experts.73.down_proj", + "model.layers.15.mlp.experts.73.gate_proj", + "model.layers.15.mlp.experts.73.up_proj", + "model.layers.15.mlp.experts.74.down_proj", + "model.layers.15.mlp.experts.74.gate_proj", + "model.layers.15.mlp.experts.74.up_proj", + "model.layers.15.mlp.experts.75.down_proj", + "model.layers.15.mlp.experts.75.gate_proj", + "model.layers.15.mlp.experts.75.up_proj", + "model.layers.15.mlp.experts.76.down_proj", + "model.layers.15.mlp.experts.76.gate_proj", + "model.layers.15.mlp.experts.76.up_proj", + "model.layers.15.mlp.experts.77.down_proj", + "model.layers.15.mlp.experts.77.gate_proj", + "model.layers.15.mlp.experts.77.up_proj", + "model.layers.15.mlp.experts.78.down_proj", + "model.layers.15.mlp.experts.78.gate_proj", + "model.layers.15.mlp.experts.78.up_proj", + "model.layers.15.mlp.experts.79.down_proj", + "model.layers.15.mlp.experts.79.gate_proj", + "model.layers.15.mlp.experts.79.up_proj", + "model.layers.15.mlp.experts.8.down_proj", + "model.layers.15.mlp.experts.8.gate_proj", + "model.layers.15.mlp.experts.8.up_proj", + "model.layers.15.mlp.experts.80.down_proj", + "model.layers.15.mlp.experts.80.gate_proj", + "model.layers.15.mlp.experts.80.up_proj", + "model.layers.15.mlp.experts.81.down_proj", + "model.layers.15.mlp.experts.81.gate_proj", + "model.layers.15.mlp.experts.81.up_proj", + "model.layers.15.mlp.experts.82.down_proj", + "model.layers.15.mlp.experts.82.gate_proj", + "model.layers.15.mlp.experts.82.up_proj", + "model.layers.15.mlp.experts.83.down_proj", + "model.layers.15.mlp.experts.83.gate_proj", + "model.layers.15.mlp.experts.83.up_proj", + "model.layers.15.mlp.experts.84.down_proj", + "model.layers.15.mlp.experts.84.gate_proj", + "model.layers.15.mlp.experts.84.up_proj", + "model.layers.15.mlp.experts.85.down_proj", + "model.layers.15.mlp.experts.85.gate_proj", + "model.layers.15.mlp.experts.85.up_proj", + "model.layers.15.mlp.experts.86.down_proj", + "model.layers.15.mlp.experts.86.gate_proj", + "model.layers.15.mlp.experts.86.up_proj", + "model.layers.15.mlp.experts.87.down_proj", + "model.layers.15.mlp.experts.87.gate_proj", + "model.layers.15.mlp.experts.87.up_proj", + "model.layers.15.mlp.experts.88.down_proj", + "model.layers.15.mlp.experts.88.gate_proj", + "model.layers.15.mlp.experts.88.up_proj", + "model.layers.15.mlp.experts.89.down_proj", + "model.layers.15.mlp.experts.89.gate_proj", + "model.layers.15.mlp.experts.89.up_proj", + "model.layers.15.mlp.experts.9.down_proj", + "model.layers.15.mlp.experts.9.gate_proj", + "model.layers.15.mlp.experts.9.up_proj", + "model.layers.15.mlp.experts.90.down_proj", + "model.layers.15.mlp.experts.90.gate_proj", + "model.layers.15.mlp.experts.90.up_proj", + "model.layers.15.mlp.experts.91.down_proj", + "model.layers.15.mlp.experts.91.gate_proj", + "model.layers.15.mlp.experts.91.up_proj", + "model.layers.15.mlp.experts.92.down_proj", + "model.layers.15.mlp.experts.92.gate_proj", + "model.layers.15.mlp.experts.92.up_proj", + "model.layers.15.mlp.experts.93.down_proj", + "model.layers.15.mlp.experts.93.gate_proj", + "model.layers.15.mlp.experts.93.up_proj", + "model.layers.15.mlp.experts.94.down_proj", + "model.layers.15.mlp.experts.94.gate_proj", + "model.layers.15.mlp.experts.94.up_proj", + "model.layers.15.mlp.experts.95.down_proj", + "model.layers.15.mlp.experts.95.gate_proj", + "model.layers.15.mlp.experts.95.up_proj", + "model.layers.15.mlp.experts.96.down_proj", + "model.layers.15.mlp.experts.96.gate_proj", + "model.layers.15.mlp.experts.96.up_proj", + "model.layers.15.mlp.experts.97.down_proj", + "model.layers.15.mlp.experts.97.gate_proj", + "model.layers.15.mlp.experts.97.up_proj", + "model.layers.15.mlp.experts.98.down_proj", + "model.layers.15.mlp.experts.98.gate_proj", + "model.layers.15.mlp.experts.98.up_proj", + "model.layers.15.mlp.experts.99.down_proj", + "model.layers.15.mlp.experts.99.gate_proj", + "model.layers.15.mlp.experts.99.up_proj", + "model.layers.16.mlp.experts.0.down_proj", + "model.layers.16.mlp.experts.0.gate_proj", + "model.layers.16.mlp.experts.0.up_proj", + "model.layers.16.mlp.experts.1.down_proj", + "model.layers.16.mlp.experts.1.gate_proj", + "model.layers.16.mlp.experts.1.up_proj", + "model.layers.16.mlp.experts.10.down_proj", + "model.layers.16.mlp.experts.10.gate_proj", + "model.layers.16.mlp.experts.10.up_proj", + "model.layers.16.mlp.experts.100.down_proj", + "model.layers.16.mlp.experts.100.gate_proj", + "model.layers.16.mlp.experts.100.up_proj", + "model.layers.16.mlp.experts.101.down_proj", + "model.layers.16.mlp.experts.101.gate_proj", + "model.layers.16.mlp.experts.101.up_proj", + "model.layers.16.mlp.experts.102.down_proj", + "model.layers.16.mlp.experts.102.gate_proj", + "model.layers.16.mlp.experts.102.up_proj", + "model.layers.16.mlp.experts.103.down_proj", + "model.layers.16.mlp.experts.103.gate_proj", + "model.layers.16.mlp.experts.103.up_proj", + "model.layers.16.mlp.experts.104.down_proj", + "model.layers.16.mlp.experts.104.gate_proj", + "model.layers.16.mlp.experts.104.up_proj", + "model.layers.16.mlp.experts.105.down_proj", + "model.layers.16.mlp.experts.105.gate_proj", + "model.layers.16.mlp.experts.105.up_proj", + "model.layers.16.mlp.experts.106.down_proj", + "model.layers.16.mlp.experts.106.gate_proj", + "model.layers.16.mlp.experts.106.up_proj", + "model.layers.16.mlp.experts.107.down_proj", + "model.layers.16.mlp.experts.107.gate_proj", + "model.layers.16.mlp.experts.107.up_proj", + "model.layers.16.mlp.experts.108.down_proj", + "model.layers.16.mlp.experts.108.gate_proj", + "model.layers.16.mlp.experts.108.up_proj", + "model.layers.16.mlp.experts.109.down_proj", + "model.layers.16.mlp.experts.109.gate_proj", + "model.layers.16.mlp.experts.109.up_proj", + "model.layers.16.mlp.experts.11.down_proj", + "model.layers.16.mlp.experts.11.gate_proj", + "model.layers.16.mlp.experts.11.up_proj", + "model.layers.16.mlp.experts.110.down_proj", + "model.layers.16.mlp.experts.110.gate_proj", + "model.layers.16.mlp.experts.110.up_proj", + "model.layers.16.mlp.experts.111.down_proj", + "model.layers.16.mlp.experts.111.gate_proj", + "model.layers.16.mlp.experts.111.up_proj", + "model.layers.16.mlp.experts.112.down_proj", + "model.layers.16.mlp.experts.112.gate_proj", + "model.layers.16.mlp.experts.112.up_proj", + "model.layers.16.mlp.experts.113.down_proj", + "model.layers.16.mlp.experts.113.gate_proj", + "model.layers.16.mlp.experts.113.up_proj", + "model.layers.16.mlp.experts.114.down_proj", + "model.layers.16.mlp.experts.114.gate_proj", + "model.layers.16.mlp.experts.114.up_proj", + "model.layers.16.mlp.experts.115.down_proj", + "model.layers.16.mlp.experts.115.gate_proj", + "model.layers.16.mlp.experts.115.up_proj", + "model.layers.16.mlp.experts.116.down_proj", + "model.layers.16.mlp.experts.116.gate_proj", + "model.layers.16.mlp.experts.116.up_proj", + "model.layers.16.mlp.experts.117.down_proj", + "model.layers.16.mlp.experts.117.gate_proj", + "model.layers.16.mlp.experts.117.up_proj", + "model.layers.16.mlp.experts.118.down_proj", + "model.layers.16.mlp.experts.118.gate_proj", + "model.layers.16.mlp.experts.118.up_proj", + "model.layers.16.mlp.experts.119.down_proj", + "model.layers.16.mlp.experts.119.gate_proj", + "model.layers.16.mlp.experts.119.up_proj", + "model.layers.16.mlp.experts.12.down_proj", + "model.layers.16.mlp.experts.12.gate_proj", + "model.layers.16.mlp.experts.12.up_proj", + "model.layers.16.mlp.experts.120.down_proj", + "model.layers.16.mlp.experts.120.gate_proj", + "model.layers.16.mlp.experts.120.up_proj", + "model.layers.16.mlp.experts.121.down_proj", + "model.layers.16.mlp.experts.121.gate_proj", + "model.layers.16.mlp.experts.121.up_proj", + "model.layers.16.mlp.experts.122.down_proj", + "model.layers.16.mlp.experts.122.gate_proj", + "model.layers.16.mlp.experts.122.up_proj", + "model.layers.16.mlp.experts.123.down_proj", + "model.layers.16.mlp.experts.123.gate_proj", + "model.layers.16.mlp.experts.123.up_proj", + "model.layers.16.mlp.experts.124.down_proj", + "model.layers.16.mlp.experts.124.gate_proj", + "model.layers.16.mlp.experts.124.up_proj", + "model.layers.16.mlp.experts.125.down_proj", + "model.layers.16.mlp.experts.125.gate_proj", + "model.layers.16.mlp.experts.125.up_proj", + "model.layers.16.mlp.experts.126.down_proj", + "model.layers.16.mlp.experts.126.gate_proj", + "model.layers.16.mlp.experts.126.up_proj", + "model.layers.16.mlp.experts.127.down_proj", + "model.layers.16.mlp.experts.127.gate_proj", + "model.layers.16.mlp.experts.127.up_proj", + "model.layers.16.mlp.experts.128.down_proj", + "model.layers.16.mlp.experts.128.gate_proj", + "model.layers.16.mlp.experts.128.up_proj", + "model.layers.16.mlp.experts.129.down_proj", + "model.layers.16.mlp.experts.129.gate_proj", + "model.layers.16.mlp.experts.129.up_proj", + "model.layers.16.mlp.experts.13.down_proj", + "model.layers.16.mlp.experts.13.gate_proj", + "model.layers.16.mlp.experts.13.up_proj", + "model.layers.16.mlp.experts.130.down_proj", + "model.layers.16.mlp.experts.130.gate_proj", + "model.layers.16.mlp.experts.130.up_proj", + "model.layers.16.mlp.experts.131.down_proj", + "model.layers.16.mlp.experts.131.gate_proj", + "model.layers.16.mlp.experts.131.up_proj", + "model.layers.16.mlp.experts.132.down_proj", + "model.layers.16.mlp.experts.132.gate_proj", + "model.layers.16.mlp.experts.132.up_proj", + "model.layers.16.mlp.experts.133.down_proj", + "model.layers.16.mlp.experts.133.gate_proj", + "model.layers.16.mlp.experts.133.up_proj", + "model.layers.16.mlp.experts.134.down_proj", + "model.layers.16.mlp.experts.134.gate_proj", + "model.layers.16.mlp.experts.134.up_proj", + "model.layers.16.mlp.experts.135.down_proj", + "model.layers.16.mlp.experts.135.gate_proj", + "model.layers.16.mlp.experts.135.up_proj", + "model.layers.16.mlp.experts.136.down_proj", + "model.layers.16.mlp.experts.136.gate_proj", + "model.layers.16.mlp.experts.136.up_proj", + "model.layers.16.mlp.experts.137.down_proj", + "model.layers.16.mlp.experts.137.gate_proj", + "model.layers.16.mlp.experts.137.up_proj", + "model.layers.16.mlp.experts.138.down_proj", + "model.layers.16.mlp.experts.138.gate_proj", + "model.layers.16.mlp.experts.138.up_proj", + "model.layers.16.mlp.experts.139.down_proj", + "model.layers.16.mlp.experts.139.gate_proj", + "model.layers.16.mlp.experts.139.up_proj", + "model.layers.16.mlp.experts.14.down_proj", + "model.layers.16.mlp.experts.14.gate_proj", + "model.layers.16.mlp.experts.14.up_proj", + "model.layers.16.mlp.experts.140.down_proj", + "model.layers.16.mlp.experts.140.gate_proj", + "model.layers.16.mlp.experts.140.up_proj", + "model.layers.16.mlp.experts.141.down_proj", + "model.layers.16.mlp.experts.141.gate_proj", + "model.layers.16.mlp.experts.141.up_proj", + "model.layers.16.mlp.experts.142.down_proj", + "model.layers.16.mlp.experts.142.gate_proj", + "model.layers.16.mlp.experts.142.up_proj", + "model.layers.16.mlp.experts.143.down_proj", + "model.layers.16.mlp.experts.143.gate_proj", + "model.layers.16.mlp.experts.143.up_proj", + "model.layers.16.mlp.experts.144.down_proj", + "model.layers.16.mlp.experts.144.gate_proj", + "model.layers.16.mlp.experts.144.up_proj", + "model.layers.16.mlp.experts.145.down_proj", + "model.layers.16.mlp.experts.145.gate_proj", + "model.layers.16.mlp.experts.145.up_proj", + "model.layers.16.mlp.experts.146.down_proj", + "model.layers.16.mlp.experts.146.gate_proj", + "model.layers.16.mlp.experts.146.up_proj", + "model.layers.16.mlp.experts.147.down_proj", + "model.layers.16.mlp.experts.147.gate_proj", + "model.layers.16.mlp.experts.147.up_proj", + "model.layers.16.mlp.experts.148.down_proj", + "model.layers.16.mlp.experts.148.gate_proj", + "model.layers.16.mlp.experts.148.up_proj", + "model.layers.16.mlp.experts.149.down_proj", + "model.layers.16.mlp.experts.149.gate_proj", + "model.layers.16.mlp.experts.149.up_proj", + "model.layers.16.mlp.experts.15.down_proj", + "model.layers.16.mlp.experts.15.gate_proj", + "model.layers.16.mlp.experts.15.up_proj", + "model.layers.16.mlp.experts.150.down_proj", + "model.layers.16.mlp.experts.150.gate_proj", + "model.layers.16.mlp.experts.150.up_proj", + "model.layers.16.mlp.experts.151.down_proj", + "model.layers.16.mlp.experts.151.gate_proj", + "model.layers.16.mlp.experts.151.up_proj", + "model.layers.16.mlp.experts.152.down_proj", + "model.layers.16.mlp.experts.152.gate_proj", + "model.layers.16.mlp.experts.152.up_proj", + "model.layers.16.mlp.experts.153.down_proj", + "model.layers.16.mlp.experts.153.gate_proj", + "model.layers.16.mlp.experts.153.up_proj", + "model.layers.16.mlp.experts.154.down_proj", + "model.layers.16.mlp.experts.154.gate_proj", + "model.layers.16.mlp.experts.154.up_proj", + "model.layers.16.mlp.experts.155.down_proj", + "model.layers.16.mlp.experts.155.gate_proj", + "model.layers.16.mlp.experts.155.up_proj", + "model.layers.16.mlp.experts.156.down_proj", + "model.layers.16.mlp.experts.156.gate_proj", + "model.layers.16.mlp.experts.156.up_proj", + "model.layers.16.mlp.experts.157.down_proj", + "model.layers.16.mlp.experts.157.gate_proj", + "model.layers.16.mlp.experts.157.up_proj", + "model.layers.16.mlp.experts.158.down_proj", + "model.layers.16.mlp.experts.158.gate_proj", + "model.layers.16.mlp.experts.158.up_proj", + "model.layers.16.mlp.experts.159.down_proj", + "model.layers.16.mlp.experts.159.gate_proj", + "model.layers.16.mlp.experts.159.up_proj", + "model.layers.16.mlp.experts.16.down_proj", + "model.layers.16.mlp.experts.16.gate_proj", + "model.layers.16.mlp.experts.16.up_proj", + "model.layers.16.mlp.experts.160.down_proj", + "model.layers.16.mlp.experts.160.gate_proj", + "model.layers.16.mlp.experts.160.up_proj", + "model.layers.16.mlp.experts.161.down_proj", + "model.layers.16.mlp.experts.161.gate_proj", + "model.layers.16.mlp.experts.161.up_proj", + "model.layers.16.mlp.experts.162.down_proj", + "model.layers.16.mlp.experts.162.gate_proj", + "model.layers.16.mlp.experts.162.up_proj", + "model.layers.16.mlp.experts.163.down_proj", + "model.layers.16.mlp.experts.163.gate_proj", + "model.layers.16.mlp.experts.163.up_proj", + "model.layers.16.mlp.experts.164.down_proj", + "model.layers.16.mlp.experts.164.gate_proj", + "model.layers.16.mlp.experts.164.up_proj", + "model.layers.16.mlp.experts.165.down_proj", + "model.layers.16.mlp.experts.165.gate_proj", + "model.layers.16.mlp.experts.165.up_proj", + "model.layers.16.mlp.experts.166.down_proj", + "model.layers.16.mlp.experts.166.gate_proj", + "model.layers.16.mlp.experts.166.up_proj", + "model.layers.16.mlp.experts.167.down_proj", + "model.layers.16.mlp.experts.167.gate_proj", + "model.layers.16.mlp.experts.167.up_proj", + "model.layers.16.mlp.experts.168.down_proj", + "model.layers.16.mlp.experts.168.gate_proj", + "model.layers.16.mlp.experts.168.up_proj", + "model.layers.16.mlp.experts.169.down_proj", + "model.layers.16.mlp.experts.169.gate_proj", + "model.layers.16.mlp.experts.169.up_proj", + "model.layers.16.mlp.experts.17.down_proj", + "model.layers.16.mlp.experts.17.gate_proj", + "model.layers.16.mlp.experts.17.up_proj", + "model.layers.16.mlp.experts.170.down_proj", + "model.layers.16.mlp.experts.170.gate_proj", + "model.layers.16.mlp.experts.170.up_proj", + "model.layers.16.mlp.experts.171.down_proj", + "model.layers.16.mlp.experts.171.gate_proj", + "model.layers.16.mlp.experts.171.up_proj", + "model.layers.16.mlp.experts.172.down_proj", + "model.layers.16.mlp.experts.172.gate_proj", + "model.layers.16.mlp.experts.172.up_proj", + "model.layers.16.mlp.experts.173.down_proj", + "model.layers.16.mlp.experts.173.gate_proj", + "model.layers.16.mlp.experts.173.up_proj", + "model.layers.16.mlp.experts.174.down_proj", + "model.layers.16.mlp.experts.174.gate_proj", + "model.layers.16.mlp.experts.174.up_proj", + "model.layers.16.mlp.experts.175.down_proj", + "model.layers.16.mlp.experts.175.gate_proj", + "model.layers.16.mlp.experts.175.up_proj", + "model.layers.16.mlp.experts.176.down_proj", + "model.layers.16.mlp.experts.176.gate_proj", + "model.layers.16.mlp.experts.176.up_proj", + "model.layers.16.mlp.experts.177.down_proj", + "model.layers.16.mlp.experts.177.gate_proj", + "model.layers.16.mlp.experts.177.up_proj", + "model.layers.16.mlp.experts.178.down_proj", + "model.layers.16.mlp.experts.178.gate_proj", + "model.layers.16.mlp.experts.178.up_proj", + "model.layers.16.mlp.experts.179.down_proj", + "model.layers.16.mlp.experts.179.gate_proj", + "model.layers.16.mlp.experts.179.up_proj", + "model.layers.16.mlp.experts.18.down_proj", + "model.layers.16.mlp.experts.18.gate_proj", + "model.layers.16.mlp.experts.18.up_proj", + "model.layers.16.mlp.experts.180.down_proj", + "model.layers.16.mlp.experts.180.gate_proj", + "model.layers.16.mlp.experts.180.up_proj", + "model.layers.16.mlp.experts.181.down_proj", + "model.layers.16.mlp.experts.181.gate_proj", + "model.layers.16.mlp.experts.181.up_proj", + "model.layers.16.mlp.experts.182.down_proj", + "model.layers.16.mlp.experts.182.gate_proj", + "model.layers.16.mlp.experts.182.up_proj", + "model.layers.16.mlp.experts.183.down_proj", + "model.layers.16.mlp.experts.183.gate_proj", + "model.layers.16.mlp.experts.183.up_proj", + "model.layers.16.mlp.experts.184.down_proj", + "model.layers.16.mlp.experts.184.gate_proj", + "model.layers.16.mlp.experts.184.up_proj", + "model.layers.16.mlp.experts.185.down_proj", + "model.layers.16.mlp.experts.185.gate_proj", + "model.layers.16.mlp.experts.185.up_proj", + "model.layers.16.mlp.experts.186.down_proj", + "model.layers.16.mlp.experts.186.gate_proj", + "model.layers.16.mlp.experts.186.up_proj", + "model.layers.16.mlp.experts.187.down_proj", + "model.layers.16.mlp.experts.187.gate_proj", + "model.layers.16.mlp.experts.187.up_proj", + "model.layers.16.mlp.experts.188.down_proj", + "model.layers.16.mlp.experts.188.gate_proj", + "model.layers.16.mlp.experts.188.up_proj", + "model.layers.16.mlp.experts.189.down_proj", + "model.layers.16.mlp.experts.189.gate_proj", + "model.layers.16.mlp.experts.189.up_proj", + "model.layers.16.mlp.experts.19.down_proj", + "model.layers.16.mlp.experts.19.gate_proj", + "model.layers.16.mlp.experts.19.up_proj", + "model.layers.16.mlp.experts.190.down_proj", + "model.layers.16.mlp.experts.190.gate_proj", + "model.layers.16.mlp.experts.190.up_proj", + "model.layers.16.mlp.experts.191.down_proj", + "model.layers.16.mlp.experts.191.gate_proj", + "model.layers.16.mlp.experts.191.up_proj", + "model.layers.16.mlp.experts.192.down_proj", + "model.layers.16.mlp.experts.192.gate_proj", + "model.layers.16.mlp.experts.192.up_proj", + "model.layers.16.mlp.experts.193.down_proj", + "model.layers.16.mlp.experts.193.gate_proj", + "model.layers.16.mlp.experts.193.up_proj", + "model.layers.16.mlp.experts.194.down_proj", + "model.layers.16.mlp.experts.194.gate_proj", + "model.layers.16.mlp.experts.194.up_proj", + "model.layers.16.mlp.experts.195.down_proj", + "model.layers.16.mlp.experts.195.gate_proj", + "model.layers.16.mlp.experts.195.up_proj", + "model.layers.16.mlp.experts.196.down_proj", + "model.layers.16.mlp.experts.196.gate_proj", + "model.layers.16.mlp.experts.196.up_proj", + "model.layers.16.mlp.experts.197.down_proj", + "model.layers.16.mlp.experts.197.gate_proj", + "model.layers.16.mlp.experts.197.up_proj", + "model.layers.16.mlp.experts.198.down_proj", + "model.layers.16.mlp.experts.198.gate_proj", + "model.layers.16.mlp.experts.198.up_proj", + "model.layers.16.mlp.experts.199.down_proj", + "model.layers.16.mlp.experts.199.gate_proj", + "model.layers.16.mlp.experts.199.up_proj", + "model.layers.16.mlp.experts.2.down_proj", + "model.layers.16.mlp.experts.2.gate_proj", + "model.layers.16.mlp.experts.2.up_proj", + "model.layers.16.mlp.experts.20.down_proj", + "model.layers.16.mlp.experts.20.gate_proj", + "model.layers.16.mlp.experts.20.up_proj", + "model.layers.16.mlp.experts.200.down_proj", + "model.layers.16.mlp.experts.200.gate_proj", + "model.layers.16.mlp.experts.200.up_proj", + "model.layers.16.mlp.experts.201.down_proj", + "model.layers.16.mlp.experts.201.gate_proj", + "model.layers.16.mlp.experts.201.up_proj", + "model.layers.16.mlp.experts.202.down_proj", + "model.layers.16.mlp.experts.202.gate_proj", + "model.layers.16.mlp.experts.202.up_proj", + "model.layers.16.mlp.experts.203.down_proj", + "model.layers.16.mlp.experts.203.gate_proj", + "model.layers.16.mlp.experts.203.up_proj", + "model.layers.16.mlp.experts.204.down_proj", + "model.layers.16.mlp.experts.204.gate_proj", + "model.layers.16.mlp.experts.204.up_proj", + "model.layers.16.mlp.experts.205.down_proj", + "model.layers.16.mlp.experts.205.gate_proj", + "model.layers.16.mlp.experts.205.up_proj", + "model.layers.16.mlp.experts.206.down_proj", + "model.layers.16.mlp.experts.206.gate_proj", + "model.layers.16.mlp.experts.206.up_proj", + "model.layers.16.mlp.experts.207.down_proj", + "model.layers.16.mlp.experts.207.gate_proj", + "model.layers.16.mlp.experts.207.up_proj", + "model.layers.16.mlp.experts.208.down_proj", + "model.layers.16.mlp.experts.208.gate_proj", + "model.layers.16.mlp.experts.208.up_proj", + "model.layers.16.mlp.experts.209.down_proj", + "model.layers.16.mlp.experts.209.gate_proj", + "model.layers.16.mlp.experts.209.up_proj", + "model.layers.16.mlp.experts.21.down_proj", + "model.layers.16.mlp.experts.21.gate_proj", + "model.layers.16.mlp.experts.21.up_proj", + "model.layers.16.mlp.experts.210.down_proj", + "model.layers.16.mlp.experts.210.gate_proj", + "model.layers.16.mlp.experts.210.up_proj", + "model.layers.16.mlp.experts.211.down_proj", + "model.layers.16.mlp.experts.211.gate_proj", + "model.layers.16.mlp.experts.211.up_proj", + "model.layers.16.mlp.experts.212.down_proj", + "model.layers.16.mlp.experts.212.gate_proj", + "model.layers.16.mlp.experts.212.up_proj", + "model.layers.16.mlp.experts.213.down_proj", + "model.layers.16.mlp.experts.213.gate_proj", + "model.layers.16.mlp.experts.213.up_proj", + "model.layers.16.mlp.experts.214.down_proj", + "model.layers.16.mlp.experts.214.gate_proj", + "model.layers.16.mlp.experts.214.up_proj", + "model.layers.16.mlp.experts.215.down_proj", + "model.layers.16.mlp.experts.215.gate_proj", + "model.layers.16.mlp.experts.215.up_proj", + "model.layers.16.mlp.experts.216.down_proj", + "model.layers.16.mlp.experts.216.gate_proj", + "model.layers.16.mlp.experts.216.up_proj", + "model.layers.16.mlp.experts.217.down_proj", + "model.layers.16.mlp.experts.217.gate_proj", + "model.layers.16.mlp.experts.217.up_proj", + "model.layers.16.mlp.experts.218.down_proj", + "model.layers.16.mlp.experts.218.gate_proj", + "model.layers.16.mlp.experts.218.up_proj", + "model.layers.16.mlp.experts.219.down_proj", + "model.layers.16.mlp.experts.219.gate_proj", + "model.layers.16.mlp.experts.219.up_proj", + "model.layers.16.mlp.experts.22.down_proj", + "model.layers.16.mlp.experts.22.gate_proj", + "model.layers.16.mlp.experts.22.up_proj", + "model.layers.16.mlp.experts.220.down_proj", + "model.layers.16.mlp.experts.220.gate_proj", + "model.layers.16.mlp.experts.220.up_proj", + "model.layers.16.mlp.experts.221.down_proj", + "model.layers.16.mlp.experts.221.gate_proj", + "model.layers.16.mlp.experts.221.up_proj", + "model.layers.16.mlp.experts.222.down_proj", + "model.layers.16.mlp.experts.222.gate_proj", + "model.layers.16.mlp.experts.222.up_proj", + "model.layers.16.mlp.experts.223.down_proj", + "model.layers.16.mlp.experts.223.gate_proj", + "model.layers.16.mlp.experts.223.up_proj", + "model.layers.16.mlp.experts.224.down_proj", + "model.layers.16.mlp.experts.224.gate_proj", + "model.layers.16.mlp.experts.224.up_proj", + "model.layers.16.mlp.experts.225.down_proj", + "model.layers.16.mlp.experts.225.gate_proj", + "model.layers.16.mlp.experts.225.up_proj", + "model.layers.16.mlp.experts.226.down_proj", + "model.layers.16.mlp.experts.226.gate_proj", + "model.layers.16.mlp.experts.226.up_proj", + "model.layers.16.mlp.experts.227.down_proj", + "model.layers.16.mlp.experts.227.gate_proj", + "model.layers.16.mlp.experts.227.up_proj", + "model.layers.16.mlp.experts.228.down_proj", + "model.layers.16.mlp.experts.228.gate_proj", + "model.layers.16.mlp.experts.228.up_proj", + "model.layers.16.mlp.experts.229.down_proj", + "model.layers.16.mlp.experts.229.gate_proj", + "model.layers.16.mlp.experts.229.up_proj", + "model.layers.16.mlp.experts.23.down_proj", + "model.layers.16.mlp.experts.23.gate_proj", + "model.layers.16.mlp.experts.23.up_proj", + "model.layers.16.mlp.experts.230.down_proj", + "model.layers.16.mlp.experts.230.gate_proj", + "model.layers.16.mlp.experts.230.up_proj", + "model.layers.16.mlp.experts.231.down_proj", + "model.layers.16.mlp.experts.231.gate_proj", + "model.layers.16.mlp.experts.231.up_proj", + "model.layers.16.mlp.experts.232.down_proj", + "model.layers.16.mlp.experts.232.gate_proj", + "model.layers.16.mlp.experts.232.up_proj", + "model.layers.16.mlp.experts.233.down_proj", + "model.layers.16.mlp.experts.233.gate_proj", + "model.layers.16.mlp.experts.233.up_proj", + "model.layers.16.mlp.experts.234.down_proj", + "model.layers.16.mlp.experts.234.gate_proj", + "model.layers.16.mlp.experts.234.up_proj", + "model.layers.16.mlp.experts.235.down_proj", + "model.layers.16.mlp.experts.235.gate_proj", + "model.layers.16.mlp.experts.235.up_proj", + "model.layers.16.mlp.experts.236.down_proj", + "model.layers.16.mlp.experts.236.gate_proj", + "model.layers.16.mlp.experts.236.up_proj", + "model.layers.16.mlp.experts.237.down_proj", + "model.layers.16.mlp.experts.237.gate_proj", + "model.layers.16.mlp.experts.237.up_proj", + "model.layers.16.mlp.experts.238.down_proj", + "model.layers.16.mlp.experts.238.gate_proj", + "model.layers.16.mlp.experts.238.up_proj", + "model.layers.16.mlp.experts.239.down_proj", + "model.layers.16.mlp.experts.239.gate_proj", + "model.layers.16.mlp.experts.239.up_proj", + "model.layers.16.mlp.experts.24.down_proj", + "model.layers.16.mlp.experts.24.gate_proj", + "model.layers.16.mlp.experts.24.up_proj", + "model.layers.16.mlp.experts.240.down_proj", + "model.layers.16.mlp.experts.240.gate_proj", + "model.layers.16.mlp.experts.240.up_proj", + "model.layers.16.mlp.experts.241.down_proj", + "model.layers.16.mlp.experts.241.gate_proj", + "model.layers.16.mlp.experts.241.up_proj", + "model.layers.16.mlp.experts.242.down_proj", + "model.layers.16.mlp.experts.242.gate_proj", + "model.layers.16.mlp.experts.242.up_proj", + "model.layers.16.mlp.experts.243.down_proj", + "model.layers.16.mlp.experts.243.gate_proj", + "model.layers.16.mlp.experts.243.up_proj", + "model.layers.16.mlp.experts.244.down_proj", + "model.layers.16.mlp.experts.244.gate_proj", + "model.layers.16.mlp.experts.244.up_proj", + "model.layers.16.mlp.experts.245.down_proj", + "model.layers.16.mlp.experts.245.gate_proj", + "model.layers.16.mlp.experts.245.up_proj", + "model.layers.16.mlp.experts.246.down_proj", + "model.layers.16.mlp.experts.246.gate_proj", + "model.layers.16.mlp.experts.246.up_proj", + "model.layers.16.mlp.experts.247.down_proj", + "model.layers.16.mlp.experts.247.gate_proj", + "model.layers.16.mlp.experts.247.up_proj", + "model.layers.16.mlp.experts.248.down_proj", + "model.layers.16.mlp.experts.248.gate_proj", + "model.layers.16.mlp.experts.248.up_proj", + "model.layers.16.mlp.experts.249.down_proj", + "model.layers.16.mlp.experts.249.gate_proj", + "model.layers.16.mlp.experts.249.up_proj", + "model.layers.16.mlp.experts.25.down_proj", + "model.layers.16.mlp.experts.25.gate_proj", + "model.layers.16.mlp.experts.25.up_proj", + "model.layers.16.mlp.experts.250.down_proj", + "model.layers.16.mlp.experts.250.gate_proj", + "model.layers.16.mlp.experts.250.up_proj", + "model.layers.16.mlp.experts.251.down_proj", + "model.layers.16.mlp.experts.251.gate_proj", + "model.layers.16.mlp.experts.251.up_proj", + "model.layers.16.mlp.experts.252.down_proj", + "model.layers.16.mlp.experts.252.gate_proj", + "model.layers.16.mlp.experts.252.up_proj", + "model.layers.16.mlp.experts.253.down_proj", + "model.layers.16.mlp.experts.253.gate_proj", + "model.layers.16.mlp.experts.253.up_proj", + "model.layers.16.mlp.experts.254.down_proj", + "model.layers.16.mlp.experts.254.gate_proj", + "model.layers.16.mlp.experts.254.up_proj", + "model.layers.16.mlp.experts.255.down_proj", + "model.layers.16.mlp.experts.255.gate_proj", + "model.layers.16.mlp.experts.255.up_proj", + "model.layers.16.mlp.experts.26.down_proj", + "model.layers.16.mlp.experts.26.gate_proj", + "model.layers.16.mlp.experts.26.up_proj", + "model.layers.16.mlp.experts.27.down_proj", + "model.layers.16.mlp.experts.27.gate_proj", + "model.layers.16.mlp.experts.27.up_proj", + "model.layers.16.mlp.experts.28.down_proj", + "model.layers.16.mlp.experts.28.gate_proj", + "model.layers.16.mlp.experts.28.up_proj", + "model.layers.16.mlp.experts.29.down_proj", + "model.layers.16.mlp.experts.29.gate_proj", + "model.layers.16.mlp.experts.29.up_proj", + "model.layers.16.mlp.experts.3.down_proj", + "model.layers.16.mlp.experts.3.gate_proj", + "model.layers.16.mlp.experts.3.up_proj", + "model.layers.16.mlp.experts.30.down_proj", + "model.layers.16.mlp.experts.30.gate_proj", + "model.layers.16.mlp.experts.30.up_proj", + "model.layers.16.mlp.experts.31.down_proj", + "model.layers.16.mlp.experts.31.gate_proj", + "model.layers.16.mlp.experts.31.up_proj", + "model.layers.16.mlp.experts.32.down_proj", + "model.layers.16.mlp.experts.32.gate_proj", + "model.layers.16.mlp.experts.32.up_proj", + "model.layers.16.mlp.experts.33.down_proj", + "model.layers.16.mlp.experts.33.gate_proj", + "model.layers.16.mlp.experts.33.up_proj", + "model.layers.16.mlp.experts.34.down_proj", + "model.layers.16.mlp.experts.34.gate_proj", + "model.layers.16.mlp.experts.34.up_proj", + "model.layers.16.mlp.experts.35.down_proj", + "model.layers.16.mlp.experts.35.gate_proj", + "model.layers.16.mlp.experts.35.up_proj", + "model.layers.16.mlp.experts.36.down_proj", + "model.layers.16.mlp.experts.36.gate_proj", + "model.layers.16.mlp.experts.36.up_proj", + "model.layers.16.mlp.experts.37.down_proj", + "model.layers.16.mlp.experts.37.gate_proj", + "model.layers.16.mlp.experts.37.up_proj", + "model.layers.16.mlp.experts.38.down_proj", + "model.layers.16.mlp.experts.38.gate_proj", + "model.layers.16.mlp.experts.38.up_proj", + "model.layers.16.mlp.experts.39.down_proj", + "model.layers.16.mlp.experts.39.gate_proj", + "model.layers.16.mlp.experts.39.up_proj", + "model.layers.16.mlp.experts.4.down_proj", + "model.layers.16.mlp.experts.4.gate_proj", + "model.layers.16.mlp.experts.4.up_proj", + "model.layers.16.mlp.experts.40.down_proj", + "model.layers.16.mlp.experts.40.gate_proj", + "model.layers.16.mlp.experts.40.up_proj", + "model.layers.16.mlp.experts.41.down_proj", + "model.layers.16.mlp.experts.41.gate_proj", + "model.layers.16.mlp.experts.41.up_proj", + "model.layers.16.mlp.experts.42.down_proj", + "model.layers.16.mlp.experts.42.gate_proj", + "model.layers.16.mlp.experts.42.up_proj", + "model.layers.16.mlp.experts.43.down_proj", + "model.layers.16.mlp.experts.43.gate_proj", + "model.layers.16.mlp.experts.43.up_proj", + "model.layers.16.mlp.experts.44.down_proj", + "model.layers.16.mlp.experts.44.gate_proj", + "model.layers.16.mlp.experts.44.up_proj", + "model.layers.16.mlp.experts.45.down_proj", + "model.layers.16.mlp.experts.45.gate_proj", + "model.layers.16.mlp.experts.45.up_proj", + "model.layers.16.mlp.experts.46.down_proj", + "model.layers.16.mlp.experts.46.gate_proj", + "model.layers.16.mlp.experts.46.up_proj", + "model.layers.16.mlp.experts.47.down_proj", + "model.layers.16.mlp.experts.47.gate_proj", + "model.layers.16.mlp.experts.47.up_proj", + "model.layers.16.mlp.experts.48.down_proj", + "model.layers.16.mlp.experts.48.gate_proj", + "model.layers.16.mlp.experts.48.up_proj", + "model.layers.16.mlp.experts.49.down_proj", + "model.layers.16.mlp.experts.49.gate_proj", + "model.layers.16.mlp.experts.49.up_proj", + "model.layers.16.mlp.experts.5.down_proj", + "model.layers.16.mlp.experts.5.gate_proj", + "model.layers.16.mlp.experts.5.up_proj", + "model.layers.16.mlp.experts.50.down_proj", + "model.layers.16.mlp.experts.50.gate_proj", + "model.layers.16.mlp.experts.50.up_proj", + "model.layers.16.mlp.experts.51.down_proj", + "model.layers.16.mlp.experts.51.gate_proj", + "model.layers.16.mlp.experts.51.up_proj", + "model.layers.16.mlp.experts.52.down_proj", + "model.layers.16.mlp.experts.52.gate_proj", + "model.layers.16.mlp.experts.52.up_proj", + "model.layers.16.mlp.experts.53.down_proj", + "model.layers.16.mlp.experts.53.gate_proj", + "model.layers.16.mlp.experts.53.up_proj", + "model.layers.16.mlp.experts.54.down_proj", + "model.layers.16.mlp.experts.54.gate_proj", + "model.layers.16.mlp.experts.54.up_proj", + "model.layers.16.mlp.experts.55.down_proj", + "model.layers.16.mlp.experts.55.gate_proj", + "model.layers.16.mlp.experts.55.up_proj", + "model.layers.16.mlp.experts.56.down_proj", + "model.layers.16.mlp.experts.56.gate_proj", + "model.layers.16.mlp.experts.56.up_proj", + "model.layers.16.mlp.experts.57.down_proj", + "model.layers.16.mlp.experts.57.gate_proj", + "model.layers.16.mlp.experts.57.up_proj", + "model.layers.16.mlp.experts.58.down_proj", + "model.layers.16.mlp.experts.58.gate_proj", + "model.layers.16.mlp.experts.58.up_proj", + "model.layers.16.mlp.experts.59.down_proj", + "model.layers.16.mlp.experts.59.gate_proj", + "model.layers.16.mlp.experts.59.up_proj", + "model.layers.16.mlp.experts.6.down_proj", + "model.layers.16.mlp.experts.6.gate_proj", + "model.layers.16.mlp.experts.6.up_proj", + "model.layers.16.mlp.experts.60.down_proj", + "model.layers.16.mlp.experts.60.gate_proj", + "model.layers.16.mlp.experts.60.up_proj", + "model.layers.16.mlp.experts.61.down_proj", + "model.layers.16.mlp.experts.61.gate_proj", + "model.layers.16.mlp.experts.61.up_proj", + "model.layers.16.mlp.experts.62.down_proj", + "model.layers.16.mlp.experts.62.gate_proj", + "model.layers.16.mlp.experts.62.up_proj", + "model.layers.16.mlp.experts.63.down_proj", + "model.layers.16.mlp.experts.63.gate_proj", + "model.layers.16.mlp.experts.63.up_proj", + "model.layers.16.mlp.experts.64.down_proj", + "model.layers.16.mlp.experts.64.gate_proj", + "model.layers.16.mlp.experts.64.up_proj", + "model.layers.16.mlp.experts.65.down_proj", + "model.layers.16.mlp.experts.65.gate_proj", + "model.layers.16.mlp.experts.65.up_proj", + "model.layers.16.mlp.experts.66.down_proj", + "model.layers.16.mlp.experts.66.gate_proj", + "model.layers.16.mlp.experts.66.up_proj", + "model.layers.16.mlp.experts.67.down_proj", + "model.layers.16.mlp.experts.67.gate_proj", + "model.layers.16.mlp.experts.67.up_proj", + "model.layers.16.mlp.experts.68.down_proj", + "model.layers.16.mlp.experts.68.gate_proj", + "model.layers.16.mlp.experts.68.up_proj", + "model.layers.16.mlp.experts.69.down_proj", + "model.layers.16.mlp.experts.69.gate_proj", + "model.layers.16.mlp.experts.69.up_proj", + "model.layers.16.mlp.experts.7.down_proj", + "model.layers.16.mlp.experts.7.gate_proj", + "model.layers.16.mlp.experts.7.up_proj", + "model.layers.16.mlp.experts.70.down_proj", + "model.layers.16.mlp.experts.70.gate_proj", + "model.layers.16.mlp.experts.70.up_proj", + "model.layers.16.mlp.experts.71.down_proj", + "model.layers.16.mlp.experts.71.gate_proj", + "model.layers.16.mlp.experts.71.up_proj", + "model.layers.16.mlp.experts.72.down_proj", + "model.layers.16.mlp.experts.72.gate_proj", + "model.layers.16.mlp.experts.72.up_proj", + "model.layers.16.mlp.experts.73.down_proj", + "model.layers.16.mlp.experts.73.gate_proj", + "model.layers.16.mlp.experts.73.up_proj", + "model.layers.16.mlp.experts.74.down_proj", + "model.layers.16.mlp.experts.74.gate_proj", + "model.layers.16.mlp.experts.74.up_proj", + "model.layers.16.mlp.experts.75.down_proj", + "model.layers.16.mlp.experts.75.gate_proj", + "model.layers.16.mlp.experts.75.up_proj", + "model.layers.16.mlp.experts.76.down_proj", + "model.layers.16.mlp.experts.76.gate_proj", + "model.layers.16.mlp.experts.76.up_proj", + "model.layers.16.mlp.experts.77.down_proj", + "model.layers.16.mlp.experts.77.gate_proj", + "model.layers.16.mlp.experts.77.up_proj", + "model.layers.16.mlp.experts.78.down_proj", + "model.layers.16.mlp.experts.78.gate_proj", + "model.layers.16.mlp.experts.78.up_proj", + "model.layers.16.mlp.experts.79.down_proj", + "model.layers.16.mlp.experts.79.gate_proj", + "model.layers.16.mlp.experts.79.up_proj", + "model.layers.16.mlp.experts.8.down_proj", + "model.layers.16.mlp.experts.8.gate_proj", + "model.layers.16.mlp.experts.8.up_proj", + "model.layers.16.mlp.experts.80.down_proj", + "model.layers.16.mlp.experts.80.gate_proj", + "model.layers.16.mlp.experts.80.up_proj", + "model.layers.16.mlp.experts.81.down_proj", + "model.layers.16.mlp.experts.81.gate_proj", + "model.layers.16.mlp.experts.81.up_proj", + "model.layers.16.mlp.experts.82.down_proj", + "model.layers.16.mlp.experts.82.gate_proj", + "model.layers.16.mlp.experts.82.up_proj", + "model.layers.16.mlp.experts.83.down_proj", + "model.layers.16.mlp.experts.83.gate_proj", + "model.layers.16.mlp.experts.83.up_proj", + "model.layers.16.mlp.experts.84.down_proj", + "model.layers.16.mlp.experts.84.gate_proj", + "model.layers.16.mlp.experts.84.up_proj", + "model.layers.16.mlp.experts.85.down_proj", + "model.layers.16.mlp.experts.85.gate_proj", + "model.layers.16.mlp.experts.85.up_proj", + "model.layers.16.mlp.experts.86.down_proj", + "model.layers.16.mlp.experts.86.gate_proj", + "model.layers.16.mlp.experts.86.up_proj", + "model.layers.16.mlp.experts.87.down_proj", + "model.layers.16.mlp.experts.87.gate_proj", + "model.layers.16.mlp.experts.87.up_proj", + "model.layers.16.mlp.experts.88.down_proj", + "model.layers.16.mlp.experts.88.gate_proj", + "model.layers.16.mlp.experts.88.up_proj", + "model.layers.16.mlp.experts.89.down_proj", + "model.layers.16.mlp.experts.89.gate_proj", + "model.layers.16.mlp.experts.89.up_proj", + "model.layers.16.mlp.experts.9.down_proj", + "model.layers.16.mlp.experts.9.gate_proj", + "model.layers.16.mlp.experts.9.up_proj", + "model.layers.16.mlp.experts.90.down_proj", + "model.layers.16.mlp.experts.90.gate_proj", + "model.layers.16.mlp.experts.90.up_proj", + "model.layers.16.mlp.experts.91.down_proj", + "model.layers.16.mlp.experts.91.gate_proj", + "model.layers.16.mlp.experts.91.up_proj", + "model.layers.16.mlp.experts.92.down_proj", + "model.layers.16.mlp.experts.92.gate_proj", + "model.layers.16.mlp.experts.92.up_proj", + "model.layers.16.mlp.experts.93.down_proj", + "model.layers.16.mlp.experts.93.gate_proj", + "model.layers.16.mlp.experts.93.up_proj", + "model.layers.16.mlp.experts.94.down_proj", + "model.layers.16.mlp.experts.94.gate_proj", + "model.layers.16.mlp.experts.94.up_proj", + "model.layers.16.mlp.experts.95.down_proj", + "model.layers.16.mlp.experts.95.gate_proj", + "model.layers.16.mlp.experts.95.up_proj", + "model.layers.16.mlp.experts.96.down_proj", + "model.layers.16.mlp.experts.96.gate_proj", + "model.layers.16.mlp.experts.96.up_proj", + "model.layers.16.mlp.experts.97.down_proj", + "model.layers.16.mlp.experts.97.gate_proj", + "model.layers.16.mlp.experts.97.up_proj", + "model.layers.16.mlp.experts.98.down_proj", + "model.layers.16.mlp.experts.98.gate_proj", + "model.layers.16.mlp.experts.98.up_proj", + "model.layers.16.mlp.experts.99.down_proj", + "model.layers.16.mlp.experts.99.gate_proj", + "model.layers.16.mlp.experts.99.up_proj", + "model.layers.17.mlp.experts.0.down_proj", + "model.layers.17.mlp.experts.0.gate_proj", + "model.layers.17.mlp.experts.0.up_proj", + "model.layers.17.mlp.experts.1.down_proj", + "model.layers.17.mlp.experts.1.gate_proj", + "model.layers.17.mlp.experts.1.up_proj", + "model.layers.17.mlp.experts.10.down_proj", + "model.layers.17.mlp.experts.10.gate_proj", + "model.layers.17.mlp.experts.10.up_proj", + "model.layers.17.mlp.experts.100.down_proj", + "model.layers.17.mlp.experts.100.gate_proj", + "model.layers.17.mlp.experts.100.up_proj", + "model.layers.17.mlp.experts.101.down_proj", + "model.layers.17.mlp.experts.101.gate_proj", + "model.layers.17.mlp.experts.101.up_proj", + "model.layers.17.mlp.experts.102.down_proj", + "model.layers.17.mlp.experts.102.gate_proj", + "model.layers.17.mlp.experts.102.up_proj", + "model.layers.17.mlp.experts.103.down_proj", + "model.layers.17.mlp.experts.103.gate_proj", + "model.layers.17.mlp.experts.103.up_proj", + "model.layers.17.mlp.experts.104.down_proj", + "model.layers.17.mlp.experts.104.gate_proj", + "model.layers.17.mlp.experts.104.up_proj", + "model.layers.17.mlp.experts.105.down_proj", + "model.layers.17.mlp.experts.105.gate_proj", + "model.layers.17.mlp.experts.105.up_proj", + "model.layers.17.mlp.experts.106.down_proj", + "model.layers.17.mlp.experts.106.gate_proj", + "model.layers.17.mlp.experts.106.up_proj", + "model.layers.17.mlp.experts.107.down_proj", + "model.layers.17.mlp.experts.107.gate_proj", + "model.layers.17.mlp.experts.107.up_proj", + "model.layers.17.mlp.experts.108.down_proj", + "model.layers.17.mlp.experts.108.gate_proj", + "model.layers.17.mlp.experts.108.up_proj", + "model.layers.17.mlp.experts.109.down_proj", + "model.layers.17.mlp.experts.109.gate_proj", + "model.layers.17.mlp.experts.109.up_proj", + "model.layers.17.mlp.experts.11.down_proj", + "model.layers.17.mlp.experts.11.gate_proj", + "model.layers.17.mlp.experts.11.up_proj", + "model.layers.17.mlp.experts.110.down_proj", + "model.layers.17.mlp.experts.110.gate_proj", + "model.layers.17.mlp.experts.110.up_proj", + "model.layers.17.mlp.experts.111.down_proj", + "model.layers.17.mlp.experts.111.gate_proj", + "model.layers.17.mlp.experts.111.up_proj", + "model.layers.17.mlp.experts.112.down_proj", + "model.layers.17.mlp.experts.112.gate_proj", + "model.layers.17.mlp.experts.112.up_proj", + "model.layers.17.mlp.experts.113.down_proj", + "model.layers.17.mlp.experts.113.gate_proj", + "model.layers.17.mlp.experts.113.up_proj", + "model.layers.17.mlp.experts.114.down_proj", + "model.layers.17.mlp.experts.114.gate_proj", + "model.layers.17.mlp.experts.114.up_proj", + "model.layers.17.mlp.experts.115.down_proj", + "model.layers.17.mlp.experts.115.gate_proj", + "model.layers.17.mlp.experts.115.up_proj", + "model.layers.17.mlp.experts.116.down_proj", + "model.layers.17.mlp.experts.116.gate_proj", + "model.layers.17.mlp.experts.116.up_proj", + "model.layers.17.mlp.experts.117.down_proj", + "model.layers.17.mlp.experts.117.gate_proj", + "model.layers.17.mlp.experts.117.up_proj", + "model.layers.17.mlp.experts.118.down_proj", + "model.layers.17.mlp.experts.118.gate_proj", + "model.layers.17.mlp.experts.118.up_proj", + "model.layers.17.mlp.experts.119.down_proj", + "model.layers.17.mlp.experts.119.gate_proj", + "model.layers.17.mlp.experts.119.up_proj", + "model.layers.17.mlp.experts.12.down_proj", + "model.layers.17.mlp.experts.12.gate_proj", + "model.layers.17.mlp.experts.12.up_proj", + "model.layers.17.mlp.experts.120.down_proj", + "model.layers.17.mlp.experts.120.gate_proj", + "model.layers.17.mlp.experts.120.up_proj", + "model.layers.17.mlp.experts.121.down_proj", + "model.layers.17.mlp.experts.121.gate_proj", + "model.layers.17.mlp.experts.121.up_proj", + "model.layers.17.mlp.experts.122.down_proj", + "model.layers.17.mlp.experts.122.gate_proj", + "model.layers.17.mlp.experts.122.up_proj", + "model.layers.17.mlp.experts.123.down_proj", + "model.layers.17.mlp.experts.123.gate_proj", + "model.layers.17.mlp.experts.123.up_proj", + "model.layers.17.mlp.experts.124.down_proj", + "model.layers.17.mlp.experts.124.gate_proj", + "model.layers.17.mlp.experts.124.up_proj", + "model.layers.17.mlp.experts.125.down_proj", + "model.layers.17.mlp.experts.125.gate_proj", + "model.layers.17.mlp.experts.125.up_proj", + "model.layers.17.mlp.experts.126.down_proj", + "model.layers.17.mlp.experts.126.gate_proj", + "model.layers.17.mlp.experts.126.up_proj", + "model.layers.17.mlp.experts.127.down_proj", + "model.layers.17.mlp.experts.127.gate_proj", + "model.layers.17.mlp.experts.127.up_proj", + "model.layers.17.mlp.experts.128.down_proj", + "model.layers.17.mlp.experts.128.gate_proj", + "model.layers.17.mlp.experts.128.up_proj", + "model.layers.17.mlp.experts.129.down_proj", + "model.layers.17.mlp.experts.129.gate_proj", + "model.layers.17.mlp.experts.129.up_proj", + "model.layers.17.mlp.experts.13.down_proj", + "model.layers.17.mlp.experts.13.gate_proj", + "model.layers.17.mlp.experts.13.up_proj", + "model.layers.17.mlp.experts.130.down_proj", + "model.layers.17.mlp.experts.130.gate_proj", + "model.layers.17.mlp.experts.130.up_proj", + "model.layers.17.mlp.experts.131.down_proj", + "model.layers.17.mlp.experts.131.gate_proj", + "model.layers.17.mlp.experts.131.up_proj", + "model.layers.17.mlp.experts.132.down_proj", + "model.layers.17.mlp.experts.132.gate_proj", + "model.layers.17.mlp.experts.132.up_proj", + "model.layers.17.mlp.experts.133.down_proj", + "model.layers.17.mlp.experts.133.gate_proj", + "model.layers.17.mlp.experts.133.up_proj", + "model.layers.17.mlp.experts.134.down_proj", + "model.layers.17.mlp.experts.134.gate_proj", + "model.layers.17.mlp.experts.134.up_proj", + "model.layers.17.mlp.experts.135.down_proj", + "model.layers.17.mlp.experts.135.gate_proj", + "model.layers.17.mlp.experts.135.up_proj", + "model.layers.17.mlp.experts.136.down_proj", + "model.layers.17.mlp.experts.136.gate_proj", + "model.layers.17.mlp.experts.136.up_proj", + "model.layers.17.mlp.experts.137.down_proj", + "model.layers.17.mlp.experts.137.gate_proj", + "model.layers.17.mlp.experts.137.up_proj", + "model.layers.17.mlp.experts.138.down_proj", + "model.layers.17.mlp.experts.138.gate_proj", + "model.layers.17.mlp.experts.138.up_proj", + "model.layers.17.mlp.experts.139.down_proj", + "model.layers.17.mlp.experts.139.gate_proj", + "model.layers.17.mlp.experts.139.up_proj", + "model.layers.17.mlp.experts.14.down_proj", + "model.layers.17.mlp.experts.14.gate_proj", + "model.layers.17.mlp.experts.14.up_proj", + "model.layers.17.mlp.experts.140.down_proj", + "model.layers.17.mlp.experts.140.gate_proj", + "model.layers.17.mlp.experts.140.up_proj", + "model.layers.17.mlp.experts.141.down_proj", + "model.layers.17.mlp.experts.141.gate_proj", + "model.layers.17.mlp.experts.141.up_proj", + "model.layers.17.mlp.experts.142.down_proj", + "model.layers.17.mlp.experts.142.gate_proj", + "model.layers.17.mlp.experts.142.up_proj", + "model.layers.17.mlp.experts.143.down_proj", + "model.layers.17.mlp.experts.143.gate_proj", + "model.layers.17.mlp.experts.143.up_proj", + "model.layers.17.mlp.experts.144.down_proj", + "model.layers.17.mlp.experts.144.gate_proj", + "model.layers.17.mlp.experts.144.up_proj", + "model.layers.17.mlp.experts.145.down_proj", + "model.layers.17.mlp.experts.145.gate_proj", + "model.layers.17.mlp.experts.145.up_proj", + "model.layers.17.mlp.experts.146.down_proj", + "model.layers.17.mlp.experts.146.gate_proj", + "model.layers.17.mlp.experts.146.up_proj", + "model.layers.17.mlp.experts.147.down_proj", + "model.layers.17.mlp.experts.147.gate_proj", + "model.layers.17.mlp.experts.147.up_proj", + "model.layers.17.mlp.experts.148.down_proj", + "model.layers.17.mlp.experts.148.gate_proj", + "model.layers.17.mlp.experts.148.up_proj", + "model.layers.17.mlp.experts.149.down_proj", + "model.layers.17.mlp.experts.149.gate_proj", + "model.layers.17.mlp.experts.149.up_proj", + "model.layers.17.mlp.experts.15.down_proj", + "model.layers.17.mlp.experts.15.gate_proj", + "model.layers.17.mlp.experts.15.up_proj", + "model.layers.17.mlp.experts.150.down_proj", + "model.layers.17.mlp.experts.150.gate_proj", + "model.layers.17.mlp.experts.150.up_proj", + "model.layers.17.mlp.experts.151.down_proj", + "model.layers.17.mlp.experts.151.gate_proj", + "model.layers.17.mlp.experts.151.up_proj", + "model.layers.17.mlp.experts.152.down_proj", + "model.layers.17.mlp.experts.152.gate_proj", + "model.layers.17.mlp.experts.152.up_proj", + "model.layers.17.mlp.experts.153.down_proj", + "model.layers.17.mlp.experts.153.gate_proj", + "model.layers.17.mlp.experts.153.up_proj", + "model.layers.17.mlp.experts.154.down_proj", + "model.layers.17.mlp.experts.154.gate_proj", + "model.layers.17.mlp.experts.154.up_proj", + "model.layers.17.mlp.experts.155.down_proj", + "model.layers.17.mlp.experts.155.gate_proj", + "model.layers.17.mlp.experts.155.up_proj", + "model.layers.17.mlp.experts.156.down_proj", + "model.layers.17.mlp.experts.156.gate_proj", + "model.layers.17.mlp.experts.156.up_proj", + "model.layers.17.mlp.experts.157.down_proj", + "model.layers.17.mlp.experts.157.gate_proj", + "model.layers.17.mlp.experts.157.up_proj", + "model.layers.17.mlp.experts.158.down_proj", + "model.layers.17.mlp.experts.158.gate_proj", + "model.layers.17.mlp.experts.158.up_proj", + "model.layers.17.mlp.experts.159.down_proj", + "model.layers.17.mlp.experts.159.gate_proj", + "model.layers.17.mlp.experts.159.up_proj", + "model.layers.17.mlp.experts.16.down_proj", + "model.layers.17.mlp.experts.16.gate_proj", + "model.layers.17.mlp.experts.16.up_proj", + "model.layers.17.mlp.experts.160.down_proj", + "model.layers.17.mlp.experts.160.gate_proj", + "model.layers.17.mlp.experts.160.up_proj", + "model.layers.17.mlp.experts.161.down_proj", + "model.layers.17.mlp.experts.161.gate_proj", + "model.layers.17.mlp.experts.161.up_proj", + "model.layers.17.mlp.experts.162.down_proj", + "model.layers.17.mlp.experts.162.gate_proj", + "model.layers.17.mlp.experts.162.up_proj", + "model.layers.17.mlp.experts.163.down_proj", + "model.layers.17.mlp.experts.163.gate_proj", + "model.layers.17.mlp.experts.163.up_proj", + "model.layers.17.mlp.experts.164.down_proj", + "model.layers.17.mlp.experts.164.gate_proj", + "model.layers.17.mlp.experts.164.up_proj", + "model.layers.17.mlp.experts.165.down_proj", + "model.layers.17.mlp.experts.165.gate_proj", + "model.layers.17.mlp.experts.165.up_proj", + "model.layers.17.mlp.experts.166.down_proj", + "model.layers.17.mlp.experts.166.gate_proj", + "model.layers.17.mlp.experts.166.up_proj", + "model.layers.17.mlp.experts.167.down_proj", + "model.layers.17.mlp.experts.167.gate_proj", + "model.layers.17.mlp.experts.167.up_proj", + "model.layers.17.mlp.experts.168.down_proj", + "model.layers.17.mlp.experts.168.gate_proj", + "model.layers.17.mlp.experts.168.up_proj", + "model.layers.17.mlp.experts.169.down_proj", + "model.layers.17.mlp.experts.169.gate_proj", + "model.layers.17.mlp.experts.169.up_proj", + "model.layers.17.mlp.experts.17.down_proj", + "model.layers.17.mlp.experts.17.gate_proj", + "model.layers.17.mlp.experts.17.up_proj", + "model.layers.17.mlp.experts.170.down_proj", + "model.layers.17.mlp.experts.170.gate_proj", + "model.layers.17.mlp.experts.170.up_proj", + "model.layers.17.mlp.experts.171.down_proj", + "model.layers.17.mlp.experts.171.gate_proj", + "model.layers.17.mlp.experts.171.up_proj", + "model.layers.17.mlp.experts.172.down_proj", + "model.layers.17.mlp.experts.172.gate_proj", + "model.layers.17.mlp.experts.172.up_proj", + "model.layers.17.mlp.experts.173.down_proj", + "model.layers.17.mlp.experts.173.gate_proj", + "model.layers.17.mlp.experts.173.up_proj", + "model.layers.17.mlp.experts.174.down_proj", + "model.layers.17.mlp.experts.174.gate_proj", + "model.layers.17.mlp.experts.174.up_proj", + "model.layers.17.mlp.experts.175.down_proj", + "model.layers.17.mlp.experts.175.gate_proj", + "model.layers.17.mlp.experts.175.up_proj", + "model.layers.17.mlp.experts.176.down_proj", + "model.layers.17.mlp.experts.176.gate_proj", + "model.layers.17.mlp.experts.176.up_proj", + "model.layers.17.mlp.experts.177.down_proj", + "model.layers.17.mlp.experts.177.gate_proj", + "model.layers.17.mlp.experts.177.up_proj", + "model.layers.17.mlp.experts.178.down_proj", + "model.layers.17.mlp.experts.178.gate_proj", + "model.layers.17.mlp.experts.178.up_proj", + "model.layers.17.mlp.experts.179.down_proj", + "model.layers.17.mlp.experts.179.gate_proj", + "model.layers.17.mlp.experts.179.up_proj", + "model.layers.17.mlp.experts.18.down_proj", + "model.layers.17.mlp.experts.18.gate_proj", + "model.layers.17.mlp.experts.18.up_proj", + "model.layers.17.mlp.experts.180.down_proj", + "model.layers.17.mlp.experts.180.gate_proj", + "model.layers.17.mlp.experts.180.up_proj", + "model.layers.17.mlp.experts.181.down_proj", + "model.layers.17.mlp.experts.181.gate_proj", + "model.layers.17.mlp.experts.181.up_proj", + "model.layers.17.mlp.experts.182.down_proj", + "model.layers.17.mlp.experts.182.gate_proj", + "model.layers.17.mlp.experts.182.up_proj", + "model.layers.17.mlp.experts.183.down_proj", + "model.layers.17.mlp.experts.183.gate_proj", + "model.layers.17.mlp.experts.183.up_proj", + "model.layers.17.mlp.experts.184.down_proj", + "model.layers.17.mlp.experts.184.gate_proj", + "model.layers.17.mlp.experts.184.up_proj", + "model.layers.17.mlp.experts.185.down_proj", + "model.layers.17.mlp.experts.185.gate_proj", + "model.layers.17.mlp.experts.185.up_proj", + "model.layers.17.mlp.experts.186.down_proj", + "model.layers.17.mlp.experts.186.gate_proj", + "model.layers.17.mlp.experts.186.up_proj", + "model.layers.17.mlp.experts.187.down_proj", + "model.layers.17.mlp.experts.187.gate_proj", + "model.layers.17.mlp.experts.187.up_proj", + "model.layers.17.mlp.experts.188.down_proj", + "model.layers.17.mlp.experts.188.gate_proj", + "model.layers.17.mlp.experts.188.up_proj", + "model.layers.17.mlp.experts.189.down_proj", + "model.layers.17.mlp.experts.189.gate_proj", + "model.layers.17.mlp.experts.189.up_proj", + "model.layers.17.mlp.experts.19.down_proj", + "model.layers.17.mlp.experts.19.gate_proj", + "model.layers.17.mlp.experts.19.up_proj", + "model.layers.17.mlp.experts.190.down_proj", + "model.layers.17.mlp.experts.190.gate_proj", + "model.layers.17.mlp.experts.190.up_proj", + "model.layers.17.mlp.experts.191.down_proj", + "model.layers.17.mlp.experts.191.gate_proj", + "model.layers.17.mlp.experts.191.up_proj", + "model.layers.17.mlp.experts.192.down_proj", + "model.layers.17.mlp.experts.192.gate_proj", + "model.layers.17.mlp.experts.192.up_proj", + "model.layers.17.mlp.experts.193.down_proj", + "model.layers.17.mlp.experts.193.gate_proj", + "model.layers.17.mlp.experts.193.up_proj", + "model.layers.17.mlp.experts.194.down_proj", + "model.layers.17.mlp.experts.194.gate_proj", + "model.layers.17.mlp.experts.194.up_proj", + "model.layers.17.mlp.experts.195.down_proj", + "model.layers.17.mlp.experts.195.gate_proj", + "model.layers.17.mlp.experts.195.up_proj", + "model.layers.17.mlp.experts.196.down_proj", + "model.layers.17.mlp.experts.196.gate_proj", + "model.layers.17.mlp.experts.196.up_proj", + "model.layers.17.mlp.experts.197.down_proj", + "model.layers.17.mlp.experts.197.gate_proj", + "model.layers.17.mlp.experts.197.up_proj", + "model.layers.17.mlp.experts.198.down_proj", + "model.layers.17.mlp.experts.198.gate_proj", + "model.layers.17.mlp.experts.198.up_proj", + "model.layers.17.mlp.experts.199.down_proj", + "model.layers.17.mlp.experts.199.gate_proj", + "model.layers.17.mlp.experts.199.up_proj", + "model.layers.17.mlp.experts.2.down_proj", + "model.layers.17.mlp.experts.2.gate_proj", + "model.layers.17.mlp.experts.2.up_proj", + "model.layers.17.mlp.experts.20.down_proj", + "model.layers.17.mlp.experts.20.gate_proj", + "model.layers.17.mlp.experts.20.up_proj", + "model.layers.17.mlp.experts.200.down_proj", + "model.layers.17.mlp.experts.200.gate_proj", + "model.layers.17.mlp.experts.200.up_proj", + "model.layers.17.mlp.experts.201.down_proj", + "model.layers.17.mlp.experts.201.gate_proj", + "model.layers.17.mlp.experts.201.up_proj", + "model.layers.17.mlp.experts.202.down_proj", + "model.layers.17.mlp.experts.202.gate_proj", + "model.layers.17.mlp.experts.202.up_proj", + "model.layers.17.mlp.experts.203.down_proj", + "model.layers.17.mlp.experts.203.gate_proj", + "model.layers.17.mlp.experts.203.up_proj", + "model.layers.17.mlp.experts.204.down_proj", + "model.layers.17.mlp.experts.204.gate_proj", + "model.layers.17.mlp.experts.204.up_proj", + "model.layers.17.mlp.experts.205.down_proj", + "model.layers.17.mlp.experts.205.gate_proj", + "model.layers.17.mlp.experts.205.up_proj", + "model.layers.17.mlp.experts.206.down_proj", + "model.layers.17.mlp.experts.206.gate_proj", + "model.layers.17.mlp.experts.206.up_proj", + "model.layers.17.mlp.experts.207.down_proj", + "model.layers.17.mlp.experts.207.gate_proj", + "model.layers.17.mlp.experts.207.up_proj", + "model.layers.17.mlp.experts.208.down_proj", + "model.layers.17.mlp.experts.208.gate_proj", + "model.layers.17.mlp.experts.208.up_proj", + "model.layers.17.mlp.experts.209.down_proj", + "model.layers.17.mlp.experts.209.gate_proj", + "model.layers.17.mlp.experts.209.up_proj", + "model.layers.17.mlp.experts.21.down_proj", + "model.layers.17.mlp.experts.21.gate_proj", + "model.layers.17.mlp.experts.21.up_proj", + "model.layers.17.mlp.experts.210.down_proj", + "model.layers.17.mlp.experts.210.gate_proj", + "model.layers.17.mlp.experts.210.up_proj", + "model.layers.17.mlp.experts.211.down_proj", + "model.layers.17.mlp.experts.211.gate_proj", + "model.layers.17.mlp.experts.211.up_proj", + "model.layers.17.mlp.experts.212.down_proj", + "model.layers.17.mlp.experts.212.gate_proj", + "model.layers.17.mlp.experts.212.up_proj", + "model.layers.17.mlp.experts.213.down_proj", + "model.layers.17.mlp.experts.213.gate_proj", + "model.layers.17.mlp.experts.213.up_proj", + "model.layers.17.mlp.experts.214.down_proj", + "model.layers.17.mlp.experts.214.gate_proj", + "model.layers.17.mlp.experts.214.up_proj", + "model.layers.17.mlp.experts.215.down_proj", + "model.layers.17.mlp.experts.215.gate_proj", + "model.layers.17.mlp.experts.215.up_proj", + "model.layers.17.mlp.experts.216.down_proj", + "model.layers.17.mlp.experts.216.gate_proj", + "model.layers.17.mlp.experts.216.up_proj", + "model.layers.17.mlp.experts.217.down_proj", + "model.layers.17.mlp.experts.217.gate_proj", + "model.layers.17.mlp.experts.217.up_proj", + "model.layers.17.mlp.experts.218.down_proj", + "model.layers.17.mlp.experts.218.gate_proj", + "model.layers.17.mlp.experts.218.up_proj", + "model.layers.17.mlp.experts.219.down_proj", + "model.layers.17.mlp.experts.219.gate_proj", + "model.layers.17.mlp.experts.219.up_proj", + "model.layers.17.mlp.experts.22.down_proj", + "model.layers.17.mlp.experts.22.gate_proj", + "model.layers.17.mlp.experts.22.up_proj", + "model.layers.17.mlp.experts.220.down_proj", + "model.layers.17.mlp.experts.220.gate_proj", + "model.layers.17.mlp.experts.220.up_proj", + "model.layers.17.mlp.experts.221.down_proj", + "model.layers.17.mlp.experts.221.gate_proj", + "model.layers.17.mlp.experts.221.up_proj", + "model.layers.17.mlp.experts.222.down_proj", + "model.layers.17.mlp.experts.222.gate_proj", + "model.layers.17.mlp.experts.222.up_proj", + "model.layers.17.mlp.experts.223.down_proj", + "model.layers.17.mlp.experts.223.gate_proj", + "model.layers.17.mlp.experts.223.up_proj", + "model.layers.17.mlp.experts.224.down_proj", + "model.layers.17.mlp.experts.224.gate_proj", + "model.layers.17.mlp.experts.224.up_proj", + "model.layers.17.mlp.experts.225.down_proj", + "model.layers.17.mlp.experts.225.gate_proj", + "model.layers.17.mlp.experts.225.up_proj", + "model.layers.17.mlp.experts.226.down_proj", + "model.layers.17.mlp.experts.226.gate_proj", + "model.layers.17.mlp.experts.226.up_proj", + "model.layers.17.mlp.experts.227.down_proj", + "model.layers.17.mlp.experts.227.gate_proj", + "model.layers.17.mlp.experts.227.up_proj", + "model.layers.17.mlp.experts.228.down_proj", + "model.layers.17.mlp.experts.228.gate_proj", + "model.layers.17.mlp.experts.228.up_proj", + "model.layers.17.mlp.experts.229.down_proj", + "model.layers.17.mlp.experts.229.gate_proj", + "model.layers.17.mlp.experts.229.up_proj", + "model.layers.17.mlp.experts.23.down_proj", + "model.layers.17.mlp.experts.23.gate_proj", + "model.layers.17.mlp.experts.23.up_proj", + "model.layers.17.mlp.experts.230.down_proj", + "model.layers.17.mlp.experts.230.gate_proj", + "model.layers.17.mlp.experts.230.up_proj", + "model.layers.17.mlp.experts.231.down_proj", + "model.layers.17.mlp.experts.231.gate_proj", + "model.layers.17.mlp.experts.231.up_proj", + "model.layers.17.mlp.experts.232.down_proj", + "model.layers.17.mlp.experts.232.gate_proj", + "model.layers.17.mlp.experts.232.up_proj", + "model.layers.17.mlp.experts.233.down_proj", + "model.layers.17.mlp.experts.233.gate_proj", + "model.layers.17.mlp.experts.233.up_proj", + "model.layers.17.mlp.experts.234.down_proj", + "model.layers.17.mlp.experts.234.gate_proj", + "model.layers.17.mlp.experts.234.up_proj", + "model.layers.17.mlp.experts.235.down_proj", + "model.layers.17.mlp.experts.235.gate_proj", + "model.layers.17.mlp.experts.235.up_proj", + "model.layers.17.mlp.experts.236.down_proj", + "model.layers.17.mlp.experts.236.gate_proj", + "model.layers.17.mlp.experts.236.up_proj", + "model.layers.17.mlp.experts.237.down_proj", + "model.layers.17.mlp.experts.237.gate_proj", + "model.layers.17.mlp.experts.237.up_proj", + "model.layers.17.mlp.experts.238.down_proj", + "model.layers.17.mlp.experts.238.gate_proj", + "model.layers.17.mlp.experts.238.up_proj", + "model.layers.17.mlp.experts.239.down_proj", + "model.layers.17.mlp.experts.239.gate_proj", + "model.layers.17.mlp.experts.239.up_proj", + "model.layers.17.mlp.experts.24.down_proj", + "model.layers.17.mlp.experts.24.gate_proj", + "model.layers.17.mlp.experts.24.up_proj", + "model.layers.17.mlp.experts.240.down_proj", + "model.layers.17.mlp.experts.240.gate_proj", + "model.layers.17.mlp.experts.240.up_proj", + "model.layers.17.mlp.experts.241.down_proj", + "model.layers.17.mlp.experts.241.gate_proj", + "model.layers.17.mlp.experts.241.up_proj", + "model.layers.17.mlp.experts.242.down_proj", + "model.layers.17.mlp.experts.242.gate_proj", + "model.layers.17.mlp.experts.242.up_proj", + "model.layers.17.mlp.experts.243.down_proj", + "model.layers.17.mlp.experts.243.gate_proj", + "model.layers.17.mlp.experts.243.up_proj", + "model.layers.17.mlp.experts.244.down_proj", + "model.layers.17.mlp.experts.244.gate_proj", + "model.layers.17.mlp.experts.244.up_proj", + "model.layers.17.mlp.experts.245.down_proj", + "model.layers.17.mlp.experts.245.gate_proj", + "model.layers.17.mlp.experts.245.up_proj", + "model.layers.17.mlp.experts.246.down_proj", + "model.layers.17.mlp.experts.246.gate_proj", + "model.layers.17.mlp.experts.246.up_proj", + "model.layers.17.mlp.experts.247.down_proj", + "model.layers.17.mlp.experts.247.gate_proj", + "model.layers.17.mlp.experts.247.up_proj", + "model.layers.17.mlp.experts.248.down_proj", + "model.layers.17.mlp.experts.248.gate_proj", + "model.layers.17.mlp.experts.248.up_proj", + "model.layers.17.mlp.experts.249.down_proj", + "model.layers.17.mlp.experts.249.gate_proj", + "model.layers.17.mlp.experts.249.up_proj", + "model.layers.17.mlp.experts.25.down_proj", + "model.layers.17.mlp.experts.25.gate_proj", + "model.layers.17.mlp.experts.25.up_proj", + "model.layers.17.mlp.experts.250.down_proj", + "model.layers.17.mlp.experts.250.gate_proj", + "model.layers.17.mlp.experts.250.up_proj", + "model.layers.17.mlp.experts.251.down_proj", + "model.layers.17.mlp.experts.251.gate_proj", + "model.layers.17.mlp.experts.251.up_proj", + "model.layers.17.mlp.experts.252.down_proj", + "model.layers.17.mlp.experts.252.gate_proj", + "model.layers.17.mlp.experts.252.up_proj", + "model.layers.17.mlp.experts.253.down_proj", + "model.layers.17.mlp.experts.253.gate_proj", + "model.layers.17.mlp.experts.253.up_proj", + "model.layers.17.mlp.experts.254.down_proj", + "model.layers.17.mlp.experts.254.gate_proj", + "model.layers.17.mlp.experts.254.up_proj", + "model.layers.17.mlp.experts.255.down_proj", + "model.layers.17.mlp.experts.255.gate_proj", + "model.layers.17.mlp.experts.255.up_proj", + "model.layers.17.mlp.experts.26.down_proj", + "model.layers.17.mlp.experts.26.gate_proj", + "model.layers.17.mlp.experts.26.up_proj", + "model.layers.17.mlp.experts.27.down_proj", + "model.layers.17.mlp.experts.27.gate_proj", + "model.layers.17.mlp.experts.27.up_proj", + "model.layers.17.mlp.experts.28.down_proj", + "model.layers.17.mlp.experts.28.gate_proj", + "model.layers.17.mlp.experts.28.up_proj", + "model.layers.17.mlp.experts.29.down_proj", + "model.layers.17.mlp.experts.29.gate_proj", + "model.layers.17.mlp.experts.29.up_proj", + "model.layers.17.mlp.experts.3.down_proj", + "model.layers.17.mlp.experts.3.gate_proj", + "model.layers.17.mlp.experts.3.up_proj", + "model.layers.17.mlp.experts.30.down_proj", + "model.layers.17.mlp.experts.30.gate_proj", + "model.layers.17.mlp.experts.30.up_proj", + "model.layers.17.mlp.experts.31.down_proj", + "model.layers.17.mlp.experts.31.gate_proj", + "model.layers.17.mlp.experts.31.up_proj", + "model.layers.17.mlp.experts.32.down_proj", + "model.layers.17.mlp.experts.32.gate_proj", + "model.layers.17.mlp.experts.32.up_proj", + "model.layers.17.mlp.experts.33.down_proj", + "model.layers.17.mlp.experts.33.gate_proj", + "model.layers.17.mlp.experts.33.up_proj", + "model.layers.17.mlp.experts.34.down_proj", + "model.layers.17.mlp.experts.34.gate_proj", + "model.layers.17.mlp.experts.34.up_proj", + "model.layers.17.mlp.experts.35.down_proj", + "model.layers.17.mlp.experts.35.gate_proj", + "model.layers.17.mlp.experts.35.up_proj", + "model.layers.17.mlp.experts.36.down_proj", + "model.layers.17.mlp.experts.36.gate_proj", + "model.layers.17.mlp.experts.36.up_proj", + "model.layers.17.mlp.experts.37.down_proj", + "model.layers.17.mlp.experts.37.gate_proj", + "model.layers.17.mlp.experts.37.up_proj", + "model.layers.17.mlp.experts.38.down_proj", + "model.layers.17.mlp.experts.38.gate_proj", + "model.layers.17.mlp.experts.38.up_proj", + "model.layers.17.mlp.experts.39.down_proj", + "model.layers.17.mlp.experts.39.gate_proj", + "model.layers.17.mlp.experts.39.up_proj", + "model.layers.17.mlp.experts.4.down_proj", + "model.layers.17.mlp.experts.4.gate_proj", + "model.layers.17.mlp.experts.4.up_proj", + "model.layers.17.mlp.experts.40.down_proj", + "model.layers.17.mlp.experts.40.gate_proj", + "model.layers.17.mlp.experts.40.up_proj", + "model.layers.17.mlp.experts.41.down_proj", + "model.layers.17.mlp.experts.41.gate_proj", + "model.layers.17.mlp.experts.41.up_proj", + "model.layers.17.mlp.experts.42.down_proj", + "model.layers.17.mlp.experts.42.gate_proj", + "model.layers.17.mlp.experts.42.up_proj", + "model.layers.17.mlp.experts.43.down_proj", + "model.layers.17.mlp.experts.43.gate_proj", + "model.layers.17.mlp.experts.43.up_proj", + "model.layers.17.mlp.experts.44.down_proj", + "model.layers.17.mlp.experts.44.gate_proj", + "model.layers.17.mlp.experts.44.up_proj", + "model.layers.17.mlp.experts.45.down_proj", + "model.layers.17.mlp.experts.45.gate_proj", + "model.layers.17.mlp.experts.45.up_proj", + "model.layers.17.mlp.experts.46.down_proj", + "model.layers.17.mlp.experts.46.gate_proj", + "model.layers.17.mlp.experts.46.up_proj", + "model.layers.17.mlp.experts.47.down_proj", + "model.layers.17.mlp.experts.47.gate_proj", + "model.layers.17.mlp.experts.47.up_proj", + "model.layers.17.mlp.experts.48.down_proj", + "model.layers.17.mlp.experts.48.gate_proj", + "model.layers.17.mlp.experts.48.up_proj", + "model.layers.17.mlp.experts.49.down_proj", + "model.layers.17.mlp.experts.49.gate_proj", + "model.layers.17.mlp.experts.49.up_proj", + "model.layers.17.mlp.experts.5.down_proj", + "model.layers.17.mlp.experts.5.gate_proj", + "model.layers.17.mlp.experts.5.up_proj", + "model.layers.17.mlp.experts.50.down_proj", + "model.layers.17.mlp.experts.50.gate_proj", + "model.layers.17.mlp.experts.50.up_proj", + "model.layers.17.mlp.experts.51.down_proj", + "model.layers.17.mlp.experts.51.gate_proj", + "model.layers.17.mlp.experts.51.up_proj", + "model.layers.17.mlp.experts.52.down_proj", + "model.layers.17.mlp.experts.52.gate_proj", + "model.layers.17.mlp.experts.52.up_proj", + "model.layers.17.mlp.experts.53.down_proj", + "model.layers.17.mlp.experts.53.gate_proj", + "model.layers.17.mlp.experts.53.up_proj", + "model.layers.17.mlp.experts.54.down_proj", + "model.layers.17.mlp.experts.54.gate_proj", + "model.layers.17.mlp.experts.54.up_proj", + "model.layers.17.mlp.experts.55.down_proj", + "model.layers.17.mlp.experts.55.gate_proj", + "model.layers.17.mlp.experts.55.up_proj", + "model.layers.17.mlp.experts.56.down_proj", + "model.layers.17.mlp.experts.56.gate_proj", + "model.layers.17.mlp.experts.56.up_proj", + "model.layers.17.mlp.experts.57.down_proj", + "model.layers.17.mlp.experts.57.gate_proj", + "model.layers.17.mlp.experts.57.up_proj", + "model.layers.17.mlp.experts.58.down_proj", + "model.layers.17.mlp.experts.58.gate_proj", + "model.layers.17.mlp.experts.58.up_proj", + "model.layers.17.mlp.experts.59.down_proj", + "model.layers.17.mlp.experts.59.gate_proj", + "model.layers.17.mlp.experts.59.up_proj", + "model.layers.17.mlp.experts.6.down_proj", + "model.layers.17.mlp.experts.6.gate_proj", + "model.layers.17.mlp.experts.6.up_proj", + "model.layers.17.mlp.experts.60.down_proj", + "model.layers.17.mlp.experts.60.gate_proj", + "model.layers.17.mlp.experts.60.up_proj", + "model.layers.17.mlp.experts.61.down_proj", + "model.layers.17.mlp.experts.61.gate_proj", + "model.layers.17.mlp.experts.61.up_proj", + "model.layers.17.mlp.experts.62.down_proj", + "model.layers.17.mlp.experts.62.gate_proj", + "model.layers.17.mlp.experts.62.up_proj", + "model.layers.17.mlp.experts.63.down_proj", + "model.layers.17.mlp.experts.63.gate_proj", + "model.layers.17.mlp.experts.63.up_proj", + "model.layers.17.mlp.experts.64.down_proj", + "model.layers.17.mlp.experts.64.gate_proj", + "model.layers.17.mlp.experts.64.up_proj", + "model.layers.17.mlp.experts.65.down_proj", + "model.layers.17.mlp.experts.65.gate_proj", + "model.layers.17.mlp.experts.65.up_proj", + "model.layers.17.mlp.experts.66.down_proj", + "model.layers.17.mlp.experts.66.gate_proj", + "model.layers.17.mlp.experts.66.up_proj", + "model.layers.17.mlp.experts.67.down_proj", + "model.layers.17.mlp.experts.67.gate_proj", + "model.layers.17.mlp.experts.67.up_proj", + "model.layers.17.mlp.experts.68.down_proj", + "model.layers.17.mlp.experts.68.gate_proj", + "model.layers.17.mlp.experts.68.up_proj", + "model.layers.17.mlp.experts.69.down_proj", + "model.layers.17.mlp.experts.69.gate_proj", + "model.layers.17.mlp.experts.69.up_proj", + "model.layers.17.mlp.experts.7.down_proj", + "model.layers.17.mlp.experts.7.gate_proj", + "model.layers.17.mlp.experts.7.up_proj", + "model.layers.17.mlp.experts.70.down_proj", + "model.layers.17.mlp.experts.70.gate_proj", + "model.layers.17.mlp.experts.70.up_proj", + "model.layers.17.mlp.experts.71.down_proj", + "model.layers.17.mlp.experts.71.gate_proj", + "model.layers.17.mlp.experts.71.up_proj", + "model.layers.17.mlp.experts.72.down_proj", + "model.layers.17.mlp.experts.72.gate_proj", + "model.layers.17.mlp.experts.72.up_proj", + "model.layers.17.mlp.experts.73.down_proj", + "model.layers.17.mlp.experts.73.gate_proj", + "model.layers.17.mlp.experts.73.up_proj", + "model.layers.17.mlp.experts.74.down_proj", + "model.layers.17.mlp.experts.74.gate_proj", + "model.layers.17.mlp.experts.74.up_proj", + "model.layers.17.mlp.experts.75.down_proj", + "model.layers.17.mlp.experts.75.gate_proj", + "model.layers.17.mlp.experts.75.up_proj", + "model.layers.17.mlp.experts.76.down_proj", + "model.layers.17.mlp.experts.76.gate_proj", + "model.layers.17.mlp.experts.76.up_proj", + "model.layers.17.mlp.experts.77.down_proj", + "model.layers.17.mlp.experts.77.gate_proj", + "model.layers.17.mlp.experts.77.up_proj", + "model.layers.17.mlp.experts.78.down_proj", + "model.layers.17.mlp.experts.78.gate_proj", + "model.layers.17.mlp.experts.78.up_proj", + "model.layers.17.mlp.experts.79.down_proj", + "model.layers.17.mlp.experts.79.gate_proj", + "model.layers.17.mlp.experts.79.up_proj", + "model.layers.17.mlp.experts.8.down_proj", + "model.layers.17.mlp.experts.8.gate_proj", + "model.layers.17.mlp.experts.8.up_proj", + "model.layers.17.mlp.experts.80.down_proj", + "model.layers.17.mlp.experts.80.gate_proj", + "model.layers.17.mlp.experts.80.up_proj", + "model.layers.17.mlp.experts.81.down_proj", + "model.layers.17.mlp.experts.81.gate_proj", + "model.layers.17.mlp.experts.81.up_proj", + "model.layers.17.mlp.experts.82.down_proj", + "model.layers.17.mlp.experts.82.gate_proj", + "model.layers.17.mlp.experts.82.up_proj", + "model.layers.17.mlp.experts.83.down_proj", + "model.layers.17.mlp.experts.83.gate_proj", + "model.layers.17.mlp.experts.83.up_proj", + "model.layers.17.mlp.experts.84.down_proj", + "model.layers.17.mlp.experts.84.gate_proj", + "model.layers.17.mlp.experts.84.up_proj", + "model.layers.17.mlp.experts.85.down_proj", + "model.layers.17.mlp.experts.85.gate_proj", + "model.layers.17.mlp.experts.85.up_proj", + "model.layers.17.mlp.experts.86.down_proj", + "model.layers.17.mlp.experts.86.gate_proj", + "model.layers.17.mlp.experts.86.up_proj", + "model.layers.17.mlp.experts.87.down_proj", + "model.layers.17.mlp.experts.87.gate_proj", + "model.layers.17.mlp.experts.87.up_proj", + "model.layers.17.mlp.experts.88.down_proj", + "model.layers.17.mlp.experts.88.gate_proj", + "model.layers.17.mlp.experts.88.up_proj", + "model.layers.17.mlp.experts.89.down_proj", + "model.layers.17.mlp.experts.89.gate_proj", + "model.layers.17.mlp.experts.89.up_proj", + "model.layers.17.mlp.experts.9.down_proj", + "model.layers.17.mlp.experts.9.gate_proj", + "model.layers.17.mlp.experts.9.up_proj", + "model.layers.17.mlp.experts.90.down_proj", + "model.layers.17.mlp.experts.90.gate_proj", + "model.layers.17.mlp.experts.90.up_proj", + "model.layers.17.mlp.experts.91.down_proj", + "model.layers.17.mlp.experts.91.gate_proj", + "model.layers.17.mlp.experts.91.up_proj", + "model.layers.17.mlp.experts.92.down_proj", + "model.layers.17.mlp.experts.92.gate_proj", + "model.layers.17.mlp.experts.92.up_proj", + "model.layers.17.mlp.experts.93.down_proj", + "model.layers.17.mlp.experts.93.gate_proj", + "model.layers.17.mlp.experts.93.up_proj", + "model.layers.17.mlp.experts.94.down_proj", + "model.layers.17.mlp.experts.94.gate_proj", + "model.layers.17.mlp.experts.94.up_proj", + "model.layers.17.mlp.experts.95.down_proj", + "model.layers.17.mlp.experts.95.gate_proj", + "model.layers.17.mlp.experts.95.up_proj", + "model.layers.17.mlp.experts.96.down_proj", + "model.layers.17.mlp.experts.96.gate_proj", + "model.layers.17.mlp.experts.96.up_proj", + "model.layers.17.mlp.experts.97.down_proj", + "model.layers.17.mlp.experts.97.gate_proj", + "model.layers.17.mlp.experts.97.up_proj", + "model.layers.17.mlp.experts.98.down_proj", + "model.layers.17.mlp.experts.98.gate_proj", + "model.layers.17.mlp.experts.98.up_proj", + "model.layers.17.mlp.experts.99.down_proj", + "model.layers.17.mlp.experts.99.gate_proj", + "model.layers.17.mlp.experts.99.up_proj", + "model.layers.18.mlp.experts.0.down_proj", + "model.layers.18.mlp.experts.0.gate_proj", + "model.layers.18.mlp.experts.0.up_proj", + "model.layers.18.mlp.experts.1.down_proj", + "model.layers.18.mlp.experts.1.gate_proj", + "model.layers.18.mlp.experts.1.up_proj", + "model.layers.18.mlp.experts.10.down_proj", + "model.layers.18.mlp.experts.10.gate_proj", + "model.layers.18.mlp.experts.10.up_proj", + "model.layers.18.mlp.experts.100.down_proj", + "model.layers.18.mlp.experts.100.gate_proj", + "model.layers.18.mlp.experts.100.up_proj", + "model.layers.18.mlp.experts.101.down_proj", + "model.layers.18.mlp.experts.101.gate_proj", + "model.layers.18.mlp.experts.101.up_proj", + "model.layers.18.mlp.experts.102.down_proj", + "model.layers.18.mlp.experts.102.gate_proj", + "model.layers.18.mlp.experts.102.up_proj", + "model.layers.18.mlp.experts.103.down_proj", + "model.layers.18.mlp.experts.103.gate_proj", + "model.layers.18.mlp.experts.103.up_proj", + "model.layers.18.mlp.experts.104.down_proj", + "model.layers.18.mlp.experts.104.gate_proj", + "model.layers.18.mlp.experts.104.up_proj", + "model.layers.18.mlp.experts.105.down_proj", + "model.layers.18.mlp.experts.105.gate_proj", + "model.layers.18.mlp.experts.105.up_proj", + "model.layers.18.mlp.experts.106.down_proj", + "model.layers.18.mlp.experts.106.gate_proj", + "model.layers.18.mlp.experts.106.up_proj", + "model.layers.18.mlp.experts.107.down_proj", + "model.layers.18.mlp.experts.107.gate_proj", + "model.layers.18.mlp.experts.107.up_proj", + "model.layers.18.mlp.experts.108.down_proj", + "model.layers.18.mlp.experts.108.gate_proj", + "model.layers.18.mlp.experts.108.up_proj", + "model.layers.18.mlp.experts.109.down_proj", + "model.layers.18.mlp.experts.109.gate_proj", + "model.layers.18.mlp.experts.109.up_proj", + "model.layers.18.mlp.experts.11.down_proj", + "model.layers.18.mlp.experts.11.gate_proj", + "model.layers.18.mlp.experts.11.up_proj", + "model.layers.18.mlp.experts.110.down_proj", + "model.layers.18.mlp.experts.110.gate_proj", + "model.layers.18.mlp.experts.110.up_proj", + "model.layers.18.mlp.experts.111.down_proj", + "model.layers.18.mlp.experts.111.gate_proj", + "model.layers.18.mlp.experts.111.up_proj", + "model.layers.18.mlp.experts.112.down_proj", + "model.layers.18.mlp.experts.112.gate_proj", + "model.layers.18.mlp.experts.112.up_proj", + "model.layers.18.mlp.experts.113.down_proj", + "model.layers.18.mlp.experts.113.gate_proj", + "model.layers.18.mlp.experts.113.up_proj", + "model.layers.18.mlp.experts.114.down_proj", + "model.layers.18.mlp.experts.114.gate_proj", + "model.layers.18.mlp.experts.114.up_proj", + "model.layers.18.mlp.experts.115.down_proj", + "model.layers.18.mlp.experts.115.gate_proj", + "model.layers.18.mlp.experts.115.up_proj", + "model.layers.18.mlp.experts.116.down_proj", + "model.layers.18.mlp.experts.116.gate_proj", + "model.layers.18.mlp.experts.116.up_proj", + "model.layers.18.mlp.experts.117.down_proj", + "model.layers.18.mlp.experts.117.gate_proj", + "model.layers.18.mlp.experts.117.up_proj", + "model.layers.18.mlp.experts.118.down_proj", + "model.layers.18.mlp.experts.118.gate_proj", + "model.layers.18.mlp.experts.118.up_proj", + "model.layers.18.mlp.experts.119.down_proj", + "model.layers.18.mlp.experts.119.gate_proj", + "model.layers.18.mlp.experts.119.up_proj", + "model.layers.18.mlp.experts.12.down_proj", + "model.layers.18.mlp.experts.12.gate_proj", + "model.layers.18.mlp.experts.12.up_proj", + "model.layers.18.mlp.experts.120.down_proj", + "model.layers.18.mlp.experts.120.gate_proj", + "model.layers.18.mlp.experts.120.up_proj", + "model.layers.18.mlp.experts.121.down_proj", + "model.layers.18.mlp.experts.121.gate_proj", + "model.layers.18.mlp.experts.121.up_proj", + "model.layers.18.mlp.experts.122.down_proj", + "model.layers.18.mlp.experts.122.gate_proj", + "model.layers.18.mlp.experts.122.up_proj", + "model.layers.18.mlp.experts.123.down_proj", + "model.layers.18.mlp.experts.123.gate_proj", + "model.layers.18.mlp.experts.123.up_proj", + "model.layers.18.mlp.experts.124.down_proj", + "model.layers.18.mlp.experts.124.gate_proj", + "model.layers.18.mlp.experts.124.up_proj", + "model.layers.18.mlp.experts.125.down_proj", + "model.layers.18.mlp.experts.125.gate_proj", + "model.layers.18.mlp.experts.125.up_proj", + "model.layers.18.mlp.experts.126.down_proj", + "model.layers.18.mlp.experts.126.gate_proj", + "model.layers.18.mlp.experts.126.up_proj", + "model.layers.18.mlp.experts.127.down_proj", + "model.layers.18.mlp.experts.127.gate_proj", + "model.layers.18.mlp.experts.127.up_proj", + "model.layers.18.mlp.experts.128.down_proj", + "model.layers.18.mlp.experts.128.gate_proj", + "model.layers.18.mlp.experts.128.up_proj", + "model.layers.18.mlp.experts.129.down_proj", + "model.layers.18.mlp.experts.129.gate_proj", + "model.layers.18.mlp.experts.129.up_proj", + "model.layers.18.mlp.experts.13.down_proj", + "model.layers.18.mlp.experts.13.gate_proj", + "model.layers.18.mlp.experts.13.up_proj", + "model.layers.18.mlp.experts.130.down_proj", + "model.layers.18.mlp.experts.130.gate_proj", + "model.layers.18.mlp.experts.130.up_proj", + "model.layers.18.mlp.experts.131.down_proj", + "model.layers.18.mlp.experts.131.gate_proj", + "model.layers.18.mlp.experts.131.up_proj", + "model.layers.18.mlp.experts.132.down_proj", + "model.layers.18.mlp.experts.132.gate_proj", + "model.layers.18.mlp.experts.132.up_proj", + "model.layers.18.mlp.experts.133.down_proj", + "model.layers.18.mlp.experts.133.gate_proj", + "model.layers.18.mlp.experts.133.up_proj", + "model.layers.18.mlp.experts.134.down_proj", + "model.layers.18.mlp.experts.134.gate_proj", + "model.layers.18.mlp.experts.134.up_proj", + "model.layers.18.mlp.experts.135.down_proj", + "model.layers.18.mlp.experts.135.gate_proj", + "model.layers.18.mlp.experts.135.up_proj", + "model.layers.18.mlp.experts.136.down_proj", + "model.layers.18.mlp.experts.136.gate_proj", + "model.layers.18.mlp.experts.136.up_proj", + "model.layers.18.mlp.experts.137.down_proj", + "model.layers.18.mlp.experts.137.gate_proj", + "model.layers.18.mlp.experts.137.up_proj", + "model.layers.18.mlp.experts.138.down_proj", + "model.layers.18.mlp.experts.138.gate_proj", + "model.layers.18.mlp.experts.138.up_proj", + "model.layers.18.mlp.experts.139.down_proj", + "model.layers.18.mlp.experts.139.gate_proj", + "model.layers.18.mlp.experts.139.up_proj", + "model.layers.18.mlp.experts.14.down_proj", + "model.layers.18.mlp.experts.14.gate_proj", + "model.layers.18.mlp.experts.14.up_proj", + "model.layers.18.mlp.experts.140.down_proj", + "model.layers.18.mlp.experts.140.gate_proj", + "model.layers.18.mlp.experts.140.up_proj", + "model.layers.18.mlp.experts.141.down_proj", + "model.layers.18.mlp.experts.141.gate_proj", + "model.layers.18.mlp.experts.141.up_proj", + "model.layers.18.mlp.experts.142.down_proj", + "model.layers.18.mlp.experts.142.gate_proj", + "model.layers.18.mlp.experts.142.up_proj", + "model.layers.18.mlp.experts.143.down_proj", + "model.layers.18.mlp.experts.143.gate_proj", + "model.layers.18.mlp.experts.143.up_proj", + "model.layers.18.mlp.experts.144.down_proj", + "model.layers.18.mlp.experts.144.gate_proj", + "model.layers.18.mlp.experts.144.up_proj", + "model.layers.18.mlp.experts.145.down_proj", + "model.layers.18.mlp.experts.145.gate_proj", + "model.layers.18.mlp.experts.145.up_proj", + "model.layers.18.mlp.experts.146.down_proj", + "model.layers.18.mlp.experts.146.gate_proj", + "model.layers.18.mlp.experts.146.up_proj", + "model.layers.18.mlp.experts.147.down_proj", + "model.layers.18.mlp.experts.147.gate_proj", + "model.layers.18.mlp.experts.147.up_proj", + "model.layers.18.mlp.experts.148.down_proj", + "model.layers.18.mlp.experts.148.gate_proj", + "model.layers.18.mlp.experts.148.up_proj", + "model.layers.18.mlp.experts.149.down_proj", + "model.layers.18.mlp.experts.149.gate_proj", + "model.layers.18.mlp.experts.149.up_proj", + "model.layers.18.mlp.experts.15.down_proj", + "model.layers.18.mlp.experts.15.gate_proj", + "model.layers.18.mlp.experts.15.up_proj", + "model.layers.18.mlp.experts.150.down_proj", + "model.layers.18.mlp.experts.150.gate_proj", + "model.layers.18.mlp.experts.150.up_proj", + "model.layers.18.mlp.experts.151.down_proj", + "model.layers.18.mlp.experts.151.gate_proj", + "model.layers.18.mlp.experts.151.up_proj", + "model.layers.18.mlp.experts.152.down_proj", + "model.layers.18.mlp.experts.152.gate_proj", + "model.layers.18.mlp.experts.152.up_proj", + "model.layers.18.mlp.experts.153.down_proj", + "model.layers.18.mlp.experts.153.gate_proj", + "model.layers.18.mlp.experts.153.up_proj", + "model.layers.18.mlp.experts.154.down_proj", + "model.layers.18.mlp.experts.154.gate_proj", + "model.layers.18.mlp.experts.154.up_proj", + "model.layers.18.mlp.experts.155.down_proj", + "model.layers.18.mlp.experts.155.gate_proj", + "model.layers.18.mlp.experts.155.up_proj", + "model.layers.18.mlp.experts.156.down_proj", + "model.layers.18.mlp.experts.156.gate_proj", + "model.layers.18.mlp.experts.156.up_proj", + "model.layers.18.mlp.experts.157.down_proj", + "model.layers.18.mlp.experts.157.gate_proj", + "model.layers.18.mlp.experts.157.up_proj", + "model.layers.18.mlp.experts.158.down_proj", + "model.layers.18.mlp.experts.158.gate_proj", + "model.layers.18.mlp.experts.158.up_proj", + "model.layers.18.mlp.experts.159.down_proj", + "model.layers.18.mlp.experts.159.gate_proj", + "model.layers.18.mlp.experts.159.up_proj", + "model.layers.18.mlp.experts.16.down_proj", + "model.layers.18.mlp.experts.16.gate_proj", + "model.layers.18.mlp.experts.16.up_proj", + "model.layers.18.mlp.experts.160.down_proj", + "model.layers.18.mlp.experts.160.gate_proj", + "model.layers.18.mlp.experts.160.up_proj", + "model.layers.18.mlp.experts.161.down_proj", + "model.layers.18.mlp.experts.161.gate_proj", + "model.layers.18.mlp.experts.161.up_proj", + "model.layers.18.mlp.experts.162.down_proj", + "model.layers.18.mlp.experts.162.gate_proj", + "model.layers.18.mlp.experts.162.up_proj", + "model.layers.18.mlp.experts.163.down_proj", + "model.layers.18.mlp.experts.163.gate_proj", + "model.layers.18.mlp.experts.163.up_proj", + "model.layers.18.mlp.experts.164.down_proj", + "model.layers.18.mlp.experts.164.gate_proj", + "model.layers.18.mlp.experts.164.up_proj", + "model.layers.18.mlp.experts.165.down_proj", + "model.layers.18.mlp.experts.165.gate_proj", + "model.layers.18.mlp.experts.165.up_proj", + "model.layers.18.mlp.experts.166.down_proj", + "model.layers.18.mlp.experts.166.gate_proj", + "model.layers.18.mlp.experts.166.up_proj", + "model.layers.18.mlp.experts.167.down_proj", + "model.layers.18.mlp.experts.167.gate_proj", + "model.layers.18.mlp.experts.167.up_proj", + "model.layers.18.mlp.experts.168.down_proj", + "model.layers.18.mlp.experts.168.gate_proj", + "model.layers.18.mlp.experts.168.up_proj", + "model.layers.18.mlp.experts.169.down_proj", + "model.layers.18.mlp.experts.169.gate_proj", + "model.layers.18.mlp.experts.169.up_proj", + "model.layers.18.mlp.experts.17.down_proj", + "model.layers.18.mlp.experts.17.gate_proj", + "model.layers.18.mlp.experts.17.up_proj", + "model.layers.18.mlp.experts.170.down_proj", + "model.layers.18.mlp.experts.170.gate_proj", + "model.layers.18.mlp.experts.170.up_proj", + "model.layers.18.mlp.experts.171.down_proj", + "model.layers.18.mlp.experts.171.gate_proj", + "model.layers.18.mlp.experts.171.up_proj", + "model.layers.18.mlp.experts.172.down_proj", + "model.layers.18.mlp.experts.172.gate_proj", + "model.layers.18.mlp.experts.172.up_proj", + "model.layers.18.mlp.experts.173.down_proj", + "model.layers.18.mlp.experts.173.gate_proj", + "model.layers.18.mlp.experts.173.up_proj", + "model.layers.18.mlp.experts.174.down_proj", + "model.layers.18.mlp.experts.174.gate_proj", + "model.layers.18.mlp.experts.174.up_proj", + "model.layers.18.mlp.experts.175.down_proj", + "model.layers.18.mlp.experts.175.gate_proj", + "model.layers.18.mlp.experts.175.up_proj", + "model.layers.18.mlp.experts.176.down_proj", + "model.layers.18.mlp.experts.176.gate_proj", + "model.layers.18.mlp.experts.176.up_proj", + "model.layers.18.mlp.experts.177.down_proj", + "model.layers.18.mlp.experts.177.gate_proj", + "model.layers.18.mlp.experts.177.up_proj", + "model.layers.18.mlp.experts.178.down_proj", + "model.layers.18.mlp.experts.178.gate_proj", + "model.layers.18.mlp.experts.178.up_proj", + "model.layers.18.mlp.experts.179.down_proj", + "model.layers.18.mlp.experts.179.gate_proj", + "model.layers.18.mlp.experts.179.up_proj", + "model.layers.18.mlp.experts.18.down_proj", + "model.layers.18.mlp.experts.18.gate_proj", + "model.layers.18.mlp.experts.18.up_proj", + "model.layers.18.mlp.experts.180.down_proj", + "model.layers.18.mlp.experts.180.gate_proj", + "model.layers.18.mlp.experts.180.up_proj", + "model.layers.18.mlp.experts.181.down_proj", + "model.layers.18.mlp.experts.181.gate_proj", + "model.layers.18.mlp.experts.181.up_proj", + "model.layers.18.mlp.experts.182.down_proj", + "model.layers.18.mlp.experts.182.gate_proj", + "model.layers.18.mlp.experts.182.up_proj", + "model.layers.18.mlp.experts.183.down_proj", + "model.layers.18.mlp.experts.183.gate_proj", + "model.layers.18.mlp.experts.183.up_proj", + "model.layers.18.mlp.experts.184.down_proj", + "model.layers.18.mlp.experts.184.gate_proj", + "model.layers.18.mlp.experts.184.up_proj", + "model.layers.18.mlp.experts.185.down_proj", + "model.layers.18.mlp.experts.185.gate_proj", + "model.layers.18.mlp.experts.185.up_proj", + "model.layers.18.mlp.experts.186.down_proj", + "model.layers.18.mlp.experts.186.gate_proj", + "model.layers.18.mlp.experts.186.up_proj", + "model.layers.18.mlp.experts.187.down_proj", + "model.layers.18.mlp.experts.187.gate_proj", + "model.layers.18.mlp.experts.187.up_proj", + "model.layers.18.mlp.experts.188.down_proj", + "model.layers.18.mlp.experts.188.gate_proj", + "model.layers.18.mlp.experts.188.up_proj", + "model.layers.18.mlp.experts.189.down_proj", + "model.layers.18.mlp.experts.189.gate_proj", + "model.layers.18.mlp.experts.189.up_proj", + "model.layers.18.mlp.experts.19.down_proj", + "model.layers.18.mlp.experts.19.gate_proj", + "model.layers.18.mlp.experts.19.up_proj", + "model.layers.18.mlp.experts.190.down_proj", + "model.layers.18.mlp.experts.190.gate_proj", + "model.layers.18.mlp.experts.190.up_proj", + "model.layers.18.mlp.experts.191.down_proj", + "model.layers.18.mlp.experts.191.gate_proj", + "model.layers.18.mlp.experts.191.up_proj", + "model.layers.18.mlp.experts.192.down_proj", + "model.layers.18.mlp.experts.192.gate_proj", + "model.layers.18.mlp.experts.192.up_proj", + "model.layers.18.mlp.experts.193.down_proj", + "model.layers.18.mlp.experts.193.gate_proj", + "model.layers.18.mlp.experts.193.up_proj", + "model.layers.18.mlp.experts.194.down_proj", + "model.layers.18.mlp.experts.194.gate_proj", + "model.layers.18.mlp.experts.194.up_proj", + "model.layers.18.mlp.experts.195.down_proj", + "model.layers.18.mlp.experts.195.gate_proj", + "model.layers.18.mlp.experts.195.up_proj", + "model.layers.18.mlp.experts.196.down_proj", + "model.layers.18.mlp.experts.196.gate_proj", + "model.layers.18.mlp.experts.196.up_proj", + "model.layers.18.mlp.experts.197.down_proj", + "model.layers.18.mlp.experts.197.gate_proj", + "model.layers.18.mlp.experts.197.up_proj", + "model.layers.18.mlp.experts.198.down_proj", + "model.layers.18.mlp.experts.198.gate_proj", + "model.layers.18.mlp.experts.198.up_proj", + "model.layers.18.mlp.experts.199.down_proj", + "model.layers.18.mlp.experts.199.gate_proj", + "model.layers.18.mlp.experts.199.up_proj", + "model.layers.18.mlp.experts.2.down_proj", + "model.layers.18.mlp.experts.2.gate_proj", + "model.layers.18.mlp.experts.2.up_proj", + "model.layers.18.mlp.experts.20.down_proj", + "model.layers.18.mlp.experts.20.gate_proj", + "model.layers.18.mlp.experts.20.up_proj", + "model.layers.18.mlp.experts.200.down_proj", + "model.layers.18.mlp.experts.200.gate_proj", + "model.layers.18.mlp.experts.200.up_proj", + "model.layers.18.mlp.experts.201.down_proj", + "model.layers.18.mlp.experts.201.gate_proj", + "model.layers.18.mlp.experts.201.up_proj", + "model.layers.18.mlp.experts.202.down_proj", + "model.layers.18.mlp.experts.202.gate_proj", + "model.layers.18.mlp.experts.202.up_proj", + "model.layers.18.mlp.experts.203.down_proj", + "model.layers.18.mlp.experts.203.gate_proj", + "model.layers.18.mlp.experts.203.up_proj", + "model.layers.18.mlp.experts.204.down_proj", + "model.layers.18.mlp.experts.204.gate_proj", + "model.layers.18.mlp.experts.204.up_proj", + "model.layers.18.mlp.experts.205.down_proj", + "model.layers.18.mlp.experts.205.gate_proj", + "model.layers.18.mlp.experts.205.up_proj", + "model.layers.18.mlp.experts.206.down_proj", + "model.layers.18.mlp.experts.206.gate_proj", + "model.layers.18.mlp.experts.206.up_proj", + "model.layers.18.mlp.experts.207.down_proj", + "model.layers.18.mlp.experts.207.gate_proj", + "model.layers.18.mlp.experts.207.up_proj", + "model.layers.18.mlp.experts.208.down_proj", + "model.layers.18.mlp.experts.208.gate_proj", + "model.layers.18.mlp.experts.208.up_proj", + "model.layers.18.mlp.experts.209.down_proj", + "model.layers.18.mlp.experts.209.gate_proj", + "model.layers.18.mlp.experts.209.up_proj", + "model.layers.18.mlp.experts.21.down_proj", + "model.layers.18.mlp.experts.21.gate_proj", + "model.layers.18.mlp.experts.21.up_proj", + "model.layers.18.mlp.experts.210.down_proj", + "model.layers.18.mlp.experts.210.gate_proj", + "model.layers.18.mlp.experts.210.up_proj", + "model.layers.18.mlp.experts.211.down_proj", + "model.layers.18.mlp.experts.211.gate_proj", + "model.layers.18.mlp.experts.211.up_proj", + "model.layers.18.mlp.experts.212.down_proj", + "model.layers.18.mlp.experts.212.gate_proj", + "model.layers.18.mlp.experts.212.up_proj", + "model.layers.18.mlp.experts.213.down_proj", + "model.layers.18.mlp.experts.213.gate_proj", + "model.layers.18.mlp.experts.213.up_proj", + "model.layers.18.mlp.experts.214.down_proj", + "model.layers.18.mlp.experts.214.gate_proj", + "model.layers.18.mlp.experts.214.up_proj", + "model.layers.18.mlp.experts.215.down_proj", + "model.layers.18.mlp.experts.215.gate_proj", + "model.layers.18.mlp.experts.215.up_proj", + "model.layers.18.mlp.experts.216.down_proj", + "model.layers.18.mlp.experts.216.gate_proj", + "model.layers.18.mlp.experts.216.up_proj", + "model.layers.18.mlp.experts.217.down_proj", + "model.layers.18.mlp.experts.217.gate_proj", + "model.layers.18.mlp.experts.217.up_proj", + "model.layers.18.mlp.experts.218.down_proj", + "model.layers.18.mlp.experts.218.gate_proj", + "model.layers.18.mlp.experts.218.up_proj", + "model.layers.18.mlp.experts.219.down_proj", + "model.layers.18.mlp.experts.219.gate_proj", + "model.layers.18.mlp.experts.219.up_proj", + "model.layers.18.mlp.experts.22.down_proj", + "model.layers.18.mlp.experts.22.gate_proj", + "model.layers.18.mlp.experts.22.up_proj", + "model.layers.18.mlp.experts.220.down_proj", + "model.layers.18.mlp.experts.220.gate_proj", + "model.layers.18.mlp.experts.220.up_proj", + "model.layers.18.mlp.experts.221.down_proj", + "model.layers.18.mlp.experts.221.gate_proj", + "model.layers.18.mlp.experts.221.up_proj", + "model.layers.18.mlp.experts.222.down_proj", + "model.layers.18.mlp.experts.222.gate_proj", + "model.layers.18.mlp.experts.222.up_proj", + "model.layers.18.mlp.experts.223.down_proj", + "model.layers.18.mlp.experts.223.gate_proj", + "model.layers.18.mlp.experts.223.up_proj", + "model.layers.18.mlp.experts.224.down_proj", + "model.layers.18.mlp.experts.224.gate_proj", + "model.layers.18.mlp.experts.224.up_proj", + "model.layers.18.mlp.experts.225.down_proj", + "model.layers.18.mlp.experts.225.gate_proj", + "model.layers.18.mlp.experts.225.up_proj", + "model.layers.18.mlp.experts.226.down_proj", + "model.layers.18.mlp.experts.226.gate_proj", + "model.layers.18.mlp.experts.226.up_proj", + "model.layers.18.mlp.experts.227.down_proj", + "model.layers.18.mlp.experts.227.gate_proj", + "model.layers.18.mlp.experts.227.up_proj", + "model.layers.18.mlp.experts.228.down_proj", + "model.layers.18.mlp.experts.228.gate_proj", + "model.layers.18.mlp.experts.228.up_proj", + "model.layers.18.mlp.experts.229.down_proj", + "model.layers.18.mlp.experts.229.gate_proj", + "model.layers.18.mlp.experts.229.up_proj", + "model.layers.18.mlp.experts.23.down_proj", + "model.layers.18.mlp.experts.23.gate_proj", + "model.layers.18.mlp.experts.23.up_proj", + "model.layers.18.mlp.experts.230.down_proj", + "model.layers.18.mlp.experts.230.gate_proj", + "model.layers.18.mlp.experts.230.up_proj", + "model.layers.18.mlp.experts.231.down_proj", + "model.layers.18.mlp.experts.231.gate_proj", + "model.layers.18.mlp.experts.231.up_proj", + "model.layers.18.mlp.experts.232.down_proj", + "model.layers.18.mlp.experts.232.gate_proj", + "model.layers.18.mlp.experts.232.up_proj", + "model.layers.18.mlp.experts.233.down_proj", + "model.layers.18.mlp.experts.233.gate_proj", + "model.layers.18.mlp.experts.233.up_proj", + "model.layers.18.mlp.experts.234.down_proj", + "model.layers.18.mlp.experts.234.gate_proj", + "model.layers.18.mlp.experts.234.up_proj", + "model.layers.18.mlp.experts.235.down_proj", + "model.layers.18.mlp.experts.235.gate_proj", + "model.layers.18.mlp.experts.235.up_proj", + "model.layers.18.mlp.experts.236.down_proj", + "model.layers.18.mlp.experts.236.gate_proj", + "model.layers.18.mlp.experts.236.up_proj", + "model.layers.18.mlp.experts.237.down_proj", + "model.layers.18.mlp.experts.237.gate_proj", + "model.layers.18.mlp.experts.237.up_proj", + "model.layers.18.mlp.experts.238.down_proj", + "model.layers.18.mlp.experts.238.gate_proj", + "model.layers.18.mlp.experts.238.up_proj", + "model.layers.18.mlp.experts.239.down_proj", + "model.layers.18.mlp.experts.239.gate_proj", + "model.layers.18.mlp.experts.239.up_proj", + "model.layers.18.mlp.experts.24.down_proj", + "model.layers.18.mlp.experts.24.gate_proj", + "model.layers.18.mlp.experts.24.up_proj", + "model.layers.18.mlp.experts.240.down_proj", + "model.layers.18.mlp.experts.240.gate_proj", + "model.layers.18.mlp.experts.240.up_proj", + "model.layers.18.mlp.experts.241.down_proj", + "model.layers.18.mlp.experts.241.gate_proj", + "model.layers.18.mlp.experts.241.up_proj", + "model.layers.18.mlp.experts.242.down_proj", + "model.layers.18.mlp.experts.242.gate_proj", + "model.layers.18.mlp.experts.242.up_proj", + "model.layers.18.mlp.experts.243.down_proj", + "model.layers.18.mlp.experts.243.gate_proj", + "model.layers.18.mlp.experts.243.up_proj", + "model.layers.18.mlp.experts.244.down_proj", + "model.layers.18.mlp.experts.244.gate_proj", + "model.layers.18.mlp.experts.244.up_proj", + "model.layers.18.mlp.experts.245.down_proj", + "model.layers.18.mlp.experts.245.gate_proj", + "model.layers.18.mlp.experts.245.up_proj", + "model.layers.18.mlp.experts.246.down_proj", + "model.layers.18.mlp.experts.246.gate_proj", + "model.layers.18.mlp.experts.246.up_proj", + "model.layers.18.mlp.experts.247.down_proj", + "model.layers.18.mlp.experts.247.gate_proj", + "model.layers.18.mlp.experts.247.up_proj", + "model.layers.18.mlp.experts.248.down_proj", + "model.layers.18.mlp.experts.248.gate_proj", + "model.layers.18.mlp.experts.248.up_proj", + "model.layers.18.mlp.experts.249.down_proj", + "model.layers.18.mlp.experts.249.gate_proj", + "model.layers.18.mlp.experts.249.up_proj", + "model.layers.18.mlp.experts.25.down_proj", + "model.layers.18.mlp.experts.25.gate_proj", + "model.layers.18.mlp.experts.25.up_proj", + "model.layers.18.mlp.experts.250.down_proj", + "model.layers.18.mlp.experts.250.gate_proj", + "model.layers.18.mlp.experts.250.up_proj", + "model.layers.18.mlp.experts.251.down_proj", + "model.layers.18.mlp.experts.251.gate_proj", + "model.layers.18.mlp.experts.251.up_proj", + "model.layers.18.mlp.experts.252.down_proj", + "model.layers.18.mlp.experts.252.gate_proj", + "model.layers.18.mlp.experts.252.up_proj", + "model.layers.18.mlp.experts.253.down_proj", + "model.layers.18.mlp.experts.253.gate_proj", + "model.layers.18.mlp.experts.253.up_proj", + "model.layers.18.mlp.experts.254.down_proj", + "model.layers.18.mlp.experts.254.gate_proj", + "model.layers.18.mlp.experts.254.up_proj", + "model.layers.18.mlp.experts.255.down_proj", + "model.layers.18.mlp.experts.255.gate_proj", + "model.layers.18.mlp.experts.255.up_proj", + "model.layers.18.mlp.experts.26.down_proj", + "model.layers.18.mlp.experts.26.gate_proj", + "model.layers.18.mlp.experts.26.up_proj", + "model.layers.18.mlp.experts.27.down_proj", + "model.layers.18.mlp.experts.27.gate_proj", + "model.layers.18.mlp.experts.27.up_proj", + "model.layers.18.mlp.experts.28.down_proj", + "model.layers.18.mlp.experts.28.gate_proj", + "model.layers.18.mlp.experts.28.up_proj", + "model.layers.18.mlp.experts.29.down_proj", + "model.layers.18.mlp.experts.29.gate_proj", + "model.layers.18.mlp.experts.29.up_proj", + "model.layers.18.mlp.experts.3.down_proj", + "model.layers.18.mlp.experts.3.gate_proj", + "model.layers.18.mlp.experts.3.up_proj", + "model.layers.18.mlp.experts.30.down_proj", + "model.layers.18.mlp.experts.30.gate_proj", + "model.layers.18.mlp.experts.30.up_proj", + "model.layers.18.mlp.experts.31.down_proj", + "model.layers.18.mlp.experts.31.gate_proj", + "model.layers.18.mlp.experts.31.up_proj", + "model.layers.18.mlp.experts.32.down_proj", + "model.layers.18.mlp.experts.32.gate_proj", + "model.layers.18.mlp.experts.32.up_proj", + "model.layers.18.mlp.experts.33.down_proj", + "model.layers.18.mlp.experts.33.gate_proj", + "model.layers.18.mlp.experts.33.up_proj", + "model.layers.18.mlp.experts.34.down_proj", + "model.layers.18.mlp.experts.34.gate_proj", + "model.layers.18.mlp.experts.34.up_proj", + "model.layers.18.mlp.experts.35.down_proj", + "model.layers.18.mlp.experts.35.gate_proj", + "model.layers.18.mlp.experts.35.up_proj", + "model.layers.18.mlp.experts.36.down_proj", + "model.layers.18.mlp.experts.36.gate_proj", + "model.layers.18.mlp.experts.36.up_proj", + "model.layers.18.mlp.experts.37.down_proj", + "model.layers.18.mlp.experts.37.gate_proj", + "model.layers.18.mlp.experts.37.up_proj", + "model.layers.18.mlp.experts.38.down_proj", + "model.layers.18.mlp.experts.38.gate_proj", + "model.layers.18.mlp.experts.38.up_proj", + "model.layers.18.mlp.experts.39.down_proj", + "model.layers.18.mlp.experts.39.gate_proj", + "model.layers.18.mlp.experts.39.up_proj", + "model.layers.18.mlp.experts.4.down_proj", + "model.layers.18.mlp.experts.4.gate_proj", + "model.layers.18.mlp.experts.4.up_proj", + "model.layers.18.mlp.experts.40.down_proj", + "model.layers.18.mlp.experts.40.gate_proj", + "model.layers.18.mlp.experts.40.up_proj", + "model.layers.18.mlp.experts.41.down_proj", + "model.layers.18.mlp.experts.41.gate_proj", + "model.layers.18.mlp.experts.41.up_proj", + "model.layers.18.mlp.experts.42.down_proj", + "model.layers.18.mlp.experts.42.gate_proj", + "model.layers.18.mlp.experts.42.up_proj", + "model.layers.18.mlp.experts.43.down_proj", + "model.layers.18.mlp.experts.43.gate_proj", + "model.layers.18.mlp.experts.43.up_proj", + "model.layers.18.mlp.experts.44.down_proj", + "model.layers.18.mlp.experts.44.gate_proj", + "model.layers.18.mlp.experts.44.up_proj", + "model.layers.18.mlp.experts.45.down_proj", + "model.layers.18.mlp.experts.45.gate_proj", + "model.layers.18.mlp.experts.45.up_proj", + "model.layers.18.mlp.experts.46.down_proj", + "model.layers.18.mlp.experts.46.gate_proj", + "model.layers.18.mlp.experts.46.up_proj", + "model.layers.18.mlp.experts.47.down_proj", + "model.layers.18.mlp.experts.47.gate_proj", + "model.layers.18.mlp.experts.47.up_proj", + "model.layers.18.mlp.experts.48.down_proj", + "model.layers.18.mlp.experts.48.gate_proj", + "model.layers.18.mlp.experts.48.up_proj", + "model.layers.18.mlp.experts.49.down_proj", + "model.layers.18.mlp.experts.49.gate_proj", + "model.layers.18.mlp.experts.49.up_proj", + "model.layers.18.mlp.experts.5.down_proj", + "model.layers.18.mlp.experts.5.gate_proj", + "model.layers.18.mlp.experts.5.up_proj", + "model.layers.18.mlp.experts.50.down_proj", + "model.layers.18.mlp.experts.50.gate_proj", + "model.layers.18.mlp.experts.50.up_proj", + "model.layers.18.mlp.experts.51.down_proj", + "model.layers.18.mlp.experts.51.gate_proj", + "model.layers.18.mlp.experts.51.up_proj", + "model.layers.18.mlp.experts.52.down_proj", + "model.layers.18.mlp.experts.52.gate_proj", + "model.layers.18.mlp.experts.52.up_proj", + "model.layers.18.mlp.experts.53.down_proj", + "model.layers.18.mlp.experts.53.gate_proj", + "model.layers.18.mlp.experts.53.up_proj", + "model.layers.18.mlp.experts.54.down_proj", + "model.layers.18.mlp.experts.54.gate_proj", + "model.layers.18.mlp.experts.54.up_proj", + "model.layers.18.mlp.experts.55.down_proj", + "model.layers.18.mlp.experts.55.gate_proj", + "model.layers.18.mlp.experts.55.up_proj", + "model.layers.18.mlp.experts.56.down_proj", + "model.layers.18.mlp.experts.56.gate_proj", + "model.layers.18.mlp.experts.56.up_proj", + "model.layers.18.mlp.experts.57.down_proj", + "model.layers.18.mlp.experts.57.gate_proj", + "model.layers.18.mlp.experts.57.up_proj", + "model.layers.18.mlp.experts.58.down_proj", + "model.layers.18.mlp.experts.58.gate_proj", + "model.layers.18.mlp.experts.58.up_proj", + "model.layers.18.mlp.experts.59.down_proj", + "model.layers.18.mlp.experts.59.gate_proj", + "model.layers.18.mlp.experts.59.up_proj", + "model.layers.18.mlp.experts.6.down_proj", + "model.layers.18.mlp.experts.6.gate_proj", + "model.layers.18.mlp.experts.6.up_proj", + "model.layers.18.mlp.experts.60.down_proj", + "model.layers.18.mlp.experts.60.gate_proj", + "model.layers.18.mlp.experts.60.up_proj", + "model.layers.18.mlp.experts.61.down_proj", + "model.layers.18.mlp.experts.61.gate_proj", + "model.layers.18.mlp.experts.61.up_proj", + "model.layers.18.mlp.experts.62.down_proj", + "model.layers.18.mlp.experts.62.gate_proj", + "model.layers.18.mlp.experts.62.up_proj", + "model.layers.18.mlp.experts.63.down_proj", + "model.layers.18.mlp.experts.63.gate_proj", + "model.layers.18.mlp.experts.63.up_proj", + "model.layers.18.mlp.experts.64.down_proj", + "model.layers.18.mlp.experts.64.gate_proj", + "model.layers.18.mlp.experts.64.up_proj", + "model.layers.18.mlp.experts.65.down_proj", + "model.layers.18.mlp.experts.65.gate_proj", + "model.layers.18.mlp.experts.65.up_proj", + "model.layers.18.mlp.experts.66.down_proj", + "model.layers.18.mlp.experts.66.gate_proj", + "model.layers.18.mlp.experts.66.up_proj", + "model.layers.18.mlp.experts.67.down_proj", + "model.layers.18.mlp.experts.67.gate_proj", + "model.layers.18.mlp.experts.67.up_proj", + "model.layers.18.mlp.experts.68.down_proj", + "model.layers.18.mlp.experts.68.gate_proj", + "model.layers.18.mlp.experts.68.up_proj", + "model.layers.18.mlp.experts.69.down_proj", + "model.layers.18.mlp.experts.69.gate_proj", + "model.layers.18.mlp.experts.69.up_proj", + "model.layers.18.mlp.experts.7.down_proj", + "model.layers.18.mlp.experts.7.gate_proj", + "model.layers.18.mlp.experts.7.up_proj", + "model.layers.18.mlp.experts.70.down_proj", + "model.layers.18.mlp.experts.70.gate_proj", + "model.layers.18.mlp.experts.70.up_proj", + "model.layers.18.mlp.experts.71.down_proj", + "model.layers.18.mlp.experts.71.gate_proj", + "model.layers.18.mlp.experts.71.up_proj", + "model.layers.18.mlp.experts.72.down_proj", + "model.layers.18.mlp.experts.72.gate_proj", + "model.layers.18.mlp.experts.72.up_proj", + "model.layers.18.mlp.experts.73.down_proj", + "model.layers.18.mlp.experts.73.gate_proj", + "model.layers.18.mlp.experts.73.up_proj", + "model.layers.18.mlp.experts.74.down_proj", + "model.layers.18.mlp.experts.74.gate_proj", + "model.layers.18.mlp.experts.74.up_proj", + "model.layers.18.mlp.experts.75.down_proj", + "model.layers.18.mlp.experts.75.gate_proj", + "model.layers.18.mlp.experts.75.up_proj", + "model.layers.18.mlp.experts.76.down_proj", + "model.layers.18.mlp.experts.76.gate_proj", + "model.layers.18.mlp.experts.76.up_proj", + "model.layers.18.mlp.experts.77.down_proj", + "model.layers.18.mlp.experts.77.gate_proj", + "model.layers.18.mlp.experts.77.up_proj", + "model.layers.18.mlp.experts.78.down_proj", + "model.layers.18.mlp.experts.78.gate_proj", + "model.layers.18.mlp.experts.78.up_proj", + "model.layers.18.mlp.experts.79.down_proj", + "model.layers.18.mlp.experts.79.gate_proj", + "model.layers.18.mlp.experts.79.up_proj", + "model.layers.18.mlp.experts.8.down_proj", + "model.layers.18.mlp.experts.8.gate_proj", + "model.layers.18.mlp.experts.8.up_proj", + "model.layers.18.mlp.experts.80.down_proj", + "model.layers.18.mlp.experts.80.gate_proj", + "model.layers.18.mlp.experts.80.up_proj", + "model.layers.18.mlp.experts.81.down_proj", + "model.layers.18.mlp.experts.81.gate_proj", + "model.layers.18.mlp.experts.81.up_proj", + "model.layers.18.mlp.experts.82.down_proj", + "model.layers.18.mlp.experts.82.gate_proj", + "model.layers.18.mlp.experts.82.up_proj", + "model.layers.18.mlp.experts.83.down_proj", + "model.layers.18.mlp.experts.83.gate_proj", + "model.layers.18.mlp.experts.83.up_proj", + "model.layers.18.mlp.experts.84.down_proj", + "model.layers.18.mlp.experts.84.gate_proj", + "model.layers.18.mlp.experts.84.up_proj", + "model.layers.18.mlp.experts.85.down_proj", + "model.layers.18.mlp.experts.85.gate_proj", + "model.layers.18.mlp.experts.85.up_proj", + "model.layers.18.mlp.experts.86.down_proj", + "model.layers.18.mlp.experts.86.gate_proj", + "model.layers.18.mlp.experts.86.up_proj", + "model.layers.18.mlp.experts.87.down_proj", + "model.layers.18.mlp.experts.87.gate_proj", + "model.layers.18.mlp.experts.87.up_proj", + "model.layers.18.mlp.experts.88.down_proj", + "model.layers.18.mlp.experts.88.gate_proj", + "model.layers.18.mlp.experts.88.up_proj", + "model.layers.18.mlp.experts.89.down_proj", + "model.layers.18.mlp.experts.89.gate_proj", + "model.layers.18.mlp.experts.89.up_proj", + "model.layers.18.mlp.experts.9.down_proj", + "model.layers.18.mlp.experts.9.gate_proj", + "model.layers.18.mlp.experts.9.up_proj", + "model.layers.18.mlp.experts.90.down_proj", + "model.layers.18.mlp.experts.90.gate_proj", + "model.layers.18.mlp.experts.90.up_proj", + "model.layers.18.mlp.experts.91.down_proj", + "model.layers.18.mlp.experts.91.gate_proj", + "model.layers.18.mlp.experts.91.up_proj", + "model.layers.18.mlp.experts.92.down_proj", + "model.layers.18.mlp.experts.92.gate_proj", + "model.layers.18.mlp.experts.92.up_proj", + "model.layers.18.mlp.experts.93.down_proj", + "model.layers.18.mlp.experts.93.gate_proj", + "model.layers.18.mlp.experts.93.up_proj", + "model.layers.18.mlp.experts.94.down_proj", + "model.layers.18.mlp.experts.94.gate_proj", + "model.layers.18.mlp.experts.94.up_proj", + "model.layers.18.mlp.experts.95.down_proj", + "model.layers.18.mlp.experts.95.gate_proj", + "model.layers.18.mlp.experts.95.up_proj", + "model.layers.18.mlp.experts.96.down_proj", + "model.layers.18.mlp.experts.96.gate_proj", + "model.layers.18.mlp.experts.96.up_proj", + "model.layers.18.mlp.experts.97.down_proj", + "model.layers.18.mlp.experts.97.gate_proj", + "model.layers.18.mlp.experts.97.up_proj", + "model.layers.18.mlp.experts.98.down_proj", + "model.layers.18.mlp.experts.98.gate_proj", + "model.layers.18.mlp.experts.98.up_proj", + "model.layers.18.mlp.experts.99.down_proj", + "model.layers.18.mlp.experts.99.gate_proj", + "model.layers.18.mlp.experts.99.up_proj", + "model.layers.19.mlp.experts.0.down_proj", + "model.layers.19.mlp.experts.0.gate_proj", + "model.layers.19.mlp.experts.0.up_proj", + "model.layers.19.mlp.experts.1.down_proj", + "model.layers.19.mlp.experts.1.gate_proj", + "model.layers.19.mlp.experts.1.up_proj", + "model.layers.19.mlp.experts.10.down_proj", + "model.layers.19.mlp.experts.10.gate_proj", + "model.layers.19.mlp.experts.10.up_proj", + "model.layers.19.mlp.experts.100.down_proj", + "model.layers.19.mlp.experts.100.gate_proj", + "model.layers.19.mlp.experts.100.up_proj", + "model.layers.19.mlp.experts.101.down_proj", + "model.layers.19.mlp.experts.101.gate_proj", + "model.layers.19.mlp.experts.101.up_proj", + "model.layers.19.mlp.experts.102.down_proj", + "model.layers.19.mlp.experts.102.gate_proj", + "model.layers.19.mlp.experts.102.up_proj", + "model.layers.19.mlp.experts.103.down_proj", + "model.layers.19.mlp.experts.103.gate_proj", + "model.layers.19.mlp.experts.103.up_proj", + "model.layers.19.mlp.experts.104.down_proj", + "model.layers.19.mlp.experts.104.gate_proj", + "model.layers.19.mlp.experts.104.up_proj", + "model.layers.19.mlp.experts.105.down_proj", + "model.layers.19.mlp.experts.105.gate_proj", + "model.layers.19.mlp.experts.105.up_proj", + "model.layers.19.mlp.experts.106.down_proj", + "model.layers.19.mlp.experts.106.gate_proj", + "model.layers.19.mlp.experts.106.up_proj", + "model.layers.19.mlp.experts.107.down_proj", + "model.layers.19.mlp.experts.107.gate_proj", + "model.layers.19.mlp.experts.107.up_proj", + "model.layers.19.mlp.experts.108.down_proj", + "model.layers.19.mlp.experts.108.gate_proj", + "model.layers.19.mlp.experts.108.up_proj", + "model.layers.19.mlp.experts.109.down_proj", + "model.layers.19.mlp.experts.109.gate_proj", + "model.layers.19.mlp.experts.109.up_proj", + "model.layers.19.mlp.experts.11.down_proj", + "model.layers.19.mlp.experts.11.gate_proj", + "model.layers.19.mlp.experts.11.up_proj", + "model.layers.19.mlp.experts.110.down_proj", + "model.layers.19.mlp.experts.110.gate_proj", + "model.layers.19.mlp.experts.110.up_proj", + "model.layers.19.mlp.experts.111.down_proj", + "model.layers.19.mlp.experts.111.gate_proj", + "model.layers.19.mlp.experts.111.up_proj", + "model.layers.19.mlp.experts.112.down_proj", + "model.layers.19.mlp.experts.112.gate_proj", + "model.layers.19.mlp.experts.112.up_proj", + "model.layers.19.mlp.experts.113.down_proj", + "model.layers.19.mlp.experts.113.gate_proj", + "model.layers.19.mlp.experts.113.up_proj", + "model.layers.19.mlp.experts.114.down_proj", + "model.layers.19.mlp.experts.114.gate_proj", + "model.layers.19.mlp.experts.114.up_proj", + "model.layers.19.mlp.experts.115.down_proj", + "model.layers.19.mlp.experts.115.gate_proj", + "model.layers.19.mlp.experts.115.up_proj", + "model.layers.19.mlp.experts.116.down_proj", + "model.layers.19.mlp.experts.116.gate_proj", + "model.layers.19.mlp.experts.116.up_proj", + "model.layers.19.mlp.experts.117.down_proj", + "model.layers.19.mlp.experts.117.gate_proj", + "model.layers.19.mlp.experts.117.up_proj", + "model.layers.19.mlp.experts.118.down_proj", + "model.layers.19.mlp.experts.118.gate_proj", + "model.layers.19.mlp.experts.118.up_proj", + "model.layers.19.mlp.experts.119.down_proj", + "model.layers.19.mlp.experts.119.gate_proj", + "model.layers.19.mlp.experts.119.up_proj", + "model.layers.19.mlp.experts.12.down_proj", + "model.layers.19.mlp.experts.12.gate_proj", + "model.layers.19.mlp.experts.12.up_proj", + "model.layers.19.mlp.experts.120.down_proj", + "model.layers.19.mlp.experts.120.gate_proj", + "model.layers.19.mlp.experts.120.up_proj", + "model.layers.19.mlp.experts.121.down_proj", + "model.layers.19.mlp.experts.121.gate_proj", + "model.layers.19.mlp.experts.121.up_proj", + "model.layers.19.mlp.experts.122.down_proj", + "model.layers.19.mlp.experts.122.gate_proj", + "model.layers.19.mlp.experts.122.up_proj", + "model.layers.19.mlp.experts.123.down_proj", + "model.layers.19.mlp.experts.123.gate_proj", + "model.layers.19.mlp.experts.123.up_proj", + "model.layers.19.mlp.experts.124.down_proj", + "model.layers.19.mlp.experts.124.gate_proj", + "model.layers.19.mlp.experts.124.up_proj", + "model.layers.19.mlp.experts.125.down_proj", + "model.layers.19.mlp.experts.125.gate_proj", + "model.layers.19.mlp.experts.125.up_proj", + "model.layers.19.mlp.experts.126.down_proj", + "model.layers.19.mlp.experts.126.gate_proj", + "model.layers.19.mlp.experts.126.up_proj", + "model.layers.19.mlp.experts.127.down_proj", + "model.layers.19.mlp.experts.127.gate_proj", + "model.layers.19.mlp.experts.127.up_proj", + "model.layers.19.mlp.experts.128.down_proj", + "model.layers.19.mlp.experts.128.gate_proj", + "model.layers.19.mlp.experts.128.up_proj", + "model.layers.19.mlp.experts.129.down_proj", + "model.layers.19.mlp.experts.129.gate_proj", + "model.layers.19.mlp.experts.129.up_proj", + "model.layers.19.mlp.experts.13.down_proj", + "model.layers.19.mlp.experts.13.gate_proj", + "model.layers.19.mlp.experts.13.up_proj", + "model.layers.19.mlp.experts.130.down_proj", + "model.layers.19.mlp.experts.130.gate_proj", + "model.layers.19.mlp.experts.130.up_proj", + "model.layers.19.mlp.experts.131.down_proj", + "model.layers.19.mlp.experts.131.gate_proj", + "model.layers.19.mlp.experts.131.up_proj", + "model.layers.19.mlp.experts.132.down_proj", + "model.layers.19.mlp.experts.132.gate_proj", + "model.layers.19.mlp.experts.132.up_proj", + "model.layers.19.mlp.experts.133.down_proj", + "model.layers.19.mlp.experts.133.gate_proj", + "model.layers.19.mlp.experts.133.up_proj", + "model.layers.19.mlp.experts.134.down_proj", + "model.layers.19.mlp.experts.134.gate_proj", + "model.layers.19.mlp.experts.134.up_proj", + "model.layers.19.mlp.experts.135.down_proj", + "model.layers.19.mlp.experts.135.gate_proj", + "model.layers.19.mlp.experts.135.up_proj", + "model.layers.19.mlp.experts.136.down_proj", + "model.layers.19.mlp.experts.136.gate_proj", + "model.layers.19.mlp.experts.136.up_proj", + "model.layers.19.mlp.experts.137.down_proj", + "model.layers.19.mlp.experts.137.gate_proj", + "model.layers.19.mlp.experts.137.up_proj", + "model.layers.19.mlp.experts.138.down_proj", + "model.layers.19.mlp.experts.138.gate_proj", + "model.layers.19.mlp.experts.138.up_proj", + "model.layers.19.mlp.experts.139.down_proj", + "model.layers.19.mlp.experts.139.gate_proj", + "model.layers.19.mlp.experts.139.up_proj", + "model.layers.19.mlp.experts.14.down_proj", + "model.layers.19.mlp.experts.14.gate_proj", + "model.layers.19.mlp.experts.14.up_proj", + "model.layers.19.mlp.experts.140.down_proj", + "model.layers.19.mlp.experts.140.gate_proj", + "model.layers.19.mlp.experts.140.up_proj", + "model.layers.19.mlp.experts.141.down_proj", + "model.layers.19.mlp.experts.141.gate_proj", + "model.layers.19.mlp.experts.141.up_proj", + "model.layers.19.mlp.experts.142.down_proj", + "model.layers.19.mlp.experts.142.gate_proj", + "model.layers.19.mlp.experts.142.up_proj", + "model.layers.19.mlp.experts.143.down_proj", + "model.layers.19.mlp.experts.143.gate_proj", + "model.layers.19.mlp.experts.143.up_proj", + "model.layers.19.mlp.experts.144.down_proj", + "model.layers.19.mlp.experts.144.gate_proj", + "model.layers.19.mlp.experts.144.up_proj", + "model.layers.19.mlp.experts.145.down_proj", + "model.layers.19.mlp.experts.145.gate_proj", + "model.layers.19.mlp.experts.145.up_proj", + "model.layers.19.mlp.experts.146.down_proj", + "model.layers.19.mlp.experts.146.gate_proj", + "model.layers.19.mlp.experts.146.up_proj", + "model.layers.19.mlp.experts.147.down_proj", + "model.layers.19.mlp.experts.147.gate_proj", + "model.layers.19.mlp.experts.147.up_proj", + "model.layers.19.mlp.experts.148.down_proj", + "model.layers.19.mlp.experts.148.gate_proj", + "model.layers.19.mlp.experts.148.up_proj", + "model.layers.19.mlp.experts.149.down_proj", + "model.layers.19.mlp.experts.149.gate_proj", + "model.layers.19.mlp.experts.149.up_proj", + "model.layers.19.mlp.experts.15.down_proj", + "model.layers.19.mlp.experts.15.gate_proj", + "model.layers.19.mlp.experts.15.up_proj", + "model.layers.19.mlp.experts.150.down_proj", + "model.layers.19.mlp.experts.150.gate_proj", + "model.layers.19.mlp.experts.150.up_proj", + "model.layers.19.mlp.experts.151.down_proj", + "model.layers.19.mlp.experts.151.gate_proj", + "model.layers.19.mlp.experts.151.up_proj", + "model.layers.19.mlp.experts.152.down_proj", + "model.layers.19.mlp.experts.152.gate_proj", + "model.layers.19.mlp.experts.152.up_proj", + "model.layers.19.mlp.experts.153.down_proj", + "model.layers.19.mlp.experts.153.gate_proj", + "model.layers.19.mlp.experts.153.up_proj", + "model.layers.19.mlp.experts.154.down_proj", + "model.layers.19.mlp.experts.154.gate_proj", + "model.layers.19.mlp.experts.154.up_proj", + "model.layers.19.mlp.experts.155.down_proj", + "model.layers.19.mlp.experts.155.gate_proj", + "model.layers.19.mlp.experts.155.up_proj", + "model.layers.19.mlp.experts.156.down_proj", + "model.layers.19.mlp.experts.156.gate_proj", + "model.layers.19.mlp.experts.156.up_proj", + "model.layers.19.mlp.experts.157.down_proj", + "model.layers.19.mlp.experts.157.gate_proj", + "model.layers.19.mlp.experts.157.up_proj", + "model.layers.19.mlp.experts.158.down_proj", + "model.layers.19.mlp.experts.158.gate_proj", + "model.layers.19.mlp.experts.158.up_proj", + "model.layers.19.mlp.experts.159.down_proj", + "model.layers.19.mlp.experts.159.gate_proj", + "model.layers.19.mlp.experts.159.up_proj", + "model.layers.19.mlp.experts.16.down_proj", + "model.layers.19.mlp.experts.16.gate_proj", + "model.layers.19.mlp.experts.16.up_proj", + "model.layers.19.mlp.experts.160.down_proj", + "model.layers.19.mlp.experts.160.gate_proj", + "model.layers.19.mlp.experts.160.up_proj", + "model.layers.19.mlp.experts.161.down_proj", + "model.layers.19.mlp.experts.161.gate_proj", + "model.layers.19.mlp.experts.161.up_proj", + "model.layers.19.mlp.experts.162.down_proj", + "model.layers.19.mlp.experts.162.gate_proj", + "model.layers.19.mlp.experts.162.up_proj", + "model.layers.19.mlp.experts.163.down_proj", + "model.layers.19.mlp.experts.163.gate_proj", + "model.layers.19.mlp.experts.163.up_proj", + "model.layers.19.mlp.experts.164.down_proj", + "model.layers.19.mlp.experts.164.gate_proj", + "model.layers.19.mlp.experts.164.up_proj", + "model.layers.19.mlp.experts.165.down_proj", + "model.layers.19.mlp.experts.165.gate_proj", + "model.layers.19.mlp.experts.165.up_proj", + "model.layers.19.mlp.experts.166.down_proj", + "model.layers.19.mlp.experts.166.gate_proj", + "model.layers.19.mlp.experts.166.up_proj", + "model.layers.19.mlp.experts.167.down_proj", + "model.layers.19.mlp.experts.167.gate_proj", + "model.layers.19.mlp.experts.167.up_proj", + "model.layers.19.mlp.experts.168.down_proj", + "model.layers.19.mlp.experts.168.gate_proj", + "model.layers.19.mlp.experts.168.up_proj", + "model.layers.19.mlp.experts.169.down_proj", + "model.layers.19.mlp.experts.169.gate_proj", + "model.layers.19.mlp.experts.169.up_proj", + "model.layers.19.mlp.experts.17.down_proj", + "model.layers.19.mlp.experts.17.gate_proj", + "model.layers.19.mlp.experts.17.up_proj", + "model.layers.19.mlp.experts.170.down_proj", + "model.layers.19.mlp.experts.170.gate_proj", + "model.layers.19.mlp.experts.170.up_proj", + "model.layers.19.mlp.experts.171.down_proj", + "model.layers.19.mlp.experts.171.gate_proj", + "model.layers.19.mlp.experts.171.up_proj", + "model.layers.19.mlp.experts.172.down_proj", + "model.layers.19.mlp.experts.172.gate_proj", + "model.layers.19.mlp.experts.172.up_proj", + "model.layers.19.mlp.experts.173.down_proj", + "model.layers.19.mlp.experts.173.gate_proj", + "model.layers.19.mlp.experts.173.up_proj", + "model.layers.19.mlp.experts.174.down_proj", + "model.layers.19.mlp.experts.174.gate_proj", + "model.layers.19.mlp.experts.174.up_proj", + "model.layers.19.mlp.experts.175.down_proj", + "model.layers.19.mlp.experts.175.gate_proj", + "model.layers.19.mlp.experts.175.up_proj", + "model.layers.19.mlp.experts.176.down_proj", + "model.layers.19.mlp.experts.176.gate_proj", + "model.layers.19.mlp.experts.176.up_proj", + "model.layers.19.mlp.experts.177.down_proj", + "model.layers.19.mlp.experts.177.gate_proj", + "model.layers.19.mlp.experts.177.up_proj", + "model.layers.19.mlp.experts.178.down_proj", + "model.layers.19.mlp.experts.178.gate_proj", + "model.layers.19.mlp.experts.178.up_proj", + "model.layers.19.mlp.experts.179.down_proj", + "model.layers.19.mlp.experts.179.gate_proj", + "model.layers.19.mlp.experts.179.up_proj", + "model.layers.19.mlp.experts.18.down_proj", + "model.layers.19.mlp.experts.18.gate_proj", + "model.layers.19.mlp.experts.18.up_proj", + "model.layers.19.mlp.experts.180.down_proj", + "model.layers.19.mlp.experts.180.gate_proj", + "model.layers.19.mlp.experts.180.up_proj", + "model.layers.19.mlp.experts.181.down_proj", + "model.layers.19.mlp.experts.181.gate_proj", + "model.layers.19.mlp.experts.181.up_proj", + "model.layers.19.mlp.experts.182.down_proj", + "model.layers.19.mlp.experts.182.gate_proj", + "model.layers.19.mlp.experts.182.up_proj", + "model.layers.19.mlp.experts.183.down_proj", + "model.layers.19.mlp.experts.183.gate_proj", + "model.layers.19.mlp.experts.183.up_proj", + "model.layers.19.mlp.experts.184.down_proj", + "model.layers.19.mlp.experts.184.gate_proj", + "model.layers.19.mlp.experts.184.up_proj", + "model.layers.19.mlp.experts.185.down_proj", + "model.layers.19.mlp.experts.185.gate_proj", + "model.layers.19.mlp.experts.185.up_proj", + "model.layers.19.mlp.experts.186.down_proj", + "model.layers.19.mlp.experts.186.gate_proj", + "model.layers.19.mlp.experts.186.up_proj", + "model.layers.19.mlp.experts.187.down_proj", + "model.layers.19.mlp.experts.187.gate_proj", + "model.layers.19.mlp.experts.187.up_proj", + "model.layers.19.mlp.experts.188.down_proj", + "model.layers.19.mlp.experts.188.gate_proj", + "model.layers.19.mlp.experts.188.up_proj", + "model.layers.19.mlp.experts.189.down_proj", + "model.layers.19.mlp.experts.189.gate_proj", + "model.layers.19.mlp.experts.189.up_proj", + "model.layers.19.mlp.experts.19.down_proj", + "model.layers.19.mlp.experts.19.gate_proj", + "model.layers.19.mlp.experts.19.up_proj", + "model.layers.19.mlp.experts.190.down_proj", + "model.layers.19.mlp.experts.190.gate_proj", + "model.layers.19.mlp.experts.190.up_proj", + "model.layers.19.mlp.experts.191.down_proj", + "model.layers.19.mlp.experts.191.gate_proj", + "model.layers.19.mlp.experts.191.up_proj", + "model.layers.19.mlp.experts.192.down_proj", + "model.layers.19.mlp.experts.192.gate_proj", + "model.layers.19.mlp.experts.192.up_proj", + "model.layers.19.mlp.experts.193.down_proj", + "model.layers.19.mlp.experts.193.gate_proj", + "model.layers.19.mlp.experts.193.up_proj", + "model.layers.19.mlp.experts.194.down_proj", + "model.layers.19.mlp.experts.194.gate_proj", + "model.layers.19.mlp.experts.194.up_proj", + "model.layers.19.mlp.experts.195.down_proj", + "model.layers.19.mlp.experts.195.gate_proj", + "model.layers.19.mlp.experts.195.up_proj", + "model.layers.19.mlp.experts.196.down_proj", + "model.layers.19.mlp.experts.196.gate_proj", + "model.layers.19.mlp.experts.196.up_proj", + "model.layers.19.mlp.experts.197.down_proj", + "model.layers.19.mlp.experts.197.gate_proj", + "model.layers.19.mlp.experts.197.up_proj", + "model.layers.19.mlp.experts.198.down_proj", + "model.layers.19.mlp.experts.198.gate_proj", + "model.layers.19.mlp.experts.198.up_proj", + "model.layers.19.mlp.experts.199.down_proj", + "model.layers.19.mlp.experts.199.gate_proj", + "model.layers.19.mlp.experts.199.up_proj", + "model.layers.19.mlp.experts.2.down_proj", + "model.layers.19.mlp.experts.2.gate_proj", + "model.layers.19.mlp.experts.2.up_proj", + "model.layers.19.mlp.experts.20.down_proj", + "model.layers.19.mlp.experts.20.gate_proj", + "model.layers.19.mlp.experts.20.up_proj", + "model.layers.19.mlp.experts.200.down_proj", + "model.layers.19.mlp.experts.200.gate_proj", + "model.layers.19.mlp.experts.200.up_proj", + "model.layers.19.mlp.experts.201.down_proj", + "model.layers.19.mlp.experts.201.gate_proj", + "model.layers.19.mlp.experts.201.up_proj", + "model.layers.19.mlp.experts.202.down_proj", + "model.layers.19.mlp.experts.202.gate_proj", + "model.layers.19.mlp.experts.202.up_proj", + "model.layers.19.mlp.experts.203.down_proj", + "model.layers.19.mlp.experts.203.gate_proj", + "model.layers.19.mlp.experts.203.up_proj", + "model.layers.19.mlp.experts.204.down_proj", + "model.layers.19.mlp.experts.204.gate_proj", + "model.layers.19.mlp.experts.204.up_proj", + "model.layers.19.mlp.experts.205.down_proj", + "model.layers.19.mlp.experts.205.gate_proj", + "model.layers.19.mlp.experts.205.up_proj", + "model.layers.19.mlp.experts.206.down_proj", + "model.layers.19.mlp.experts.206.gate_proj", + "model.layers.19.mlp.experts.206.up_proj", + "model.layers.19.mlp.experts.207.down_proj", + "model.layers.19.mlp.experts.207.gate_proj", + "model.layers.19.mlp.experts.207.up_proj", + "model.layers.19.mlp.experts.208.down_proj", + "model.layers.19.mlp.experts.208.gate_proj", + "model.layers.19.mlp.experts.208.up_proj", + "model.layers.19.mlp.experts.209.down_proj", + "model.layers.19.mlp.experts.209.gate_proj", + "model.layers.19.mlp.experts.209.up_proj", + "model.layers.19.mlp.experts.21.down_proj", + "model.layers.19.mlp.experts.21.gate_proj", + "model.layers.19.mlp.experts.21.up_proj", + "model.layers.19.mlp.experts.210.down_proj", + "model.layers.19.mlp.experts.210.gate_proj", + "model.layers.19.mlp.experts.210.up_proj", + "model.layers.19.mlp.experts.211.down_proj", + "model.layers.19.mlp.experts.211.gate_proj", + "model.layers.19.mlp.experts.211.up_proj", + "model.layers.19.mlp.experts.212.down_proj", + "model.layers.19.mlp.experts.212.gate_proj", + "model.layers.19.mlp.experts.212.up_proj", + "model.layers.19.mlp.experts.213.down_proj", + "model.layers.19.mlp.experts.213.gate_proj", + "model.layers.19.mlp.experts.213.up_proj", + "model.layers.19.mlp.experts.214.down_proj", + "model.layers.19.mlp.experts.214.gate_proj", + "model.layers.19.mlp.experts.214.up_proj", + "model.layers.19.mlp.experts.215.down_proj", + "model.layers.19.mlp.experts.215.gate_proj", + "model.layers.19.mlp.experts.215.up_proj", + "model.layers.19.mlp.experts.216.down_proj", + "model.layers.19.mlp.experts.216.gate_proj", + "model.layers.19.mlp.experts.216.up_proj", + "model.layers.19.mlp.experts.217.down_proj", + "model.layers.19.mlp.experts.217.gate_proj", + "model.layers.19.mlp.experts.217.up_proj", + "model.layers.19.mlp.experts.218.down_proj", + "model.layers.19.mlp.experts.218.gate_proj", + "model.layers.19.mlp.experts.218.up_proj", + "model.layers.19.mlp.experts.219.down_proj", + "model.layers.19.mlp.experts.219.gate_proj", + "model.layers.19.mlp.experts.219.up_proj", + "model.layers.19.mlp.experts.22.down_proj", + "model.layers.19.mlp.experts.22.gate_proj", + "model.layers.19.mlp.experts.22.up_proj", + "model.layers.19.mlp.experts.220.down_proj", + "model.layers.19.mlp.experts.220.gate_proj", + "model.layers.19.mlp.experts.220.up_proj", + "model.layers.19.mlp.experts.221.down_proj", + "model.layers.19.mlp.experts.221.gate_proj", + "model.layers.19.mlp.experts.221.up_proj", + "model.layers.19.mlp.experts.222.down_proj", + "model.layers.19.mlp.experts.222.gate_proj", + "model.layers.19.mlp.experts.222.up_proj", + "model.layers.19.mlp.experts.223.down_proj", + "model.layers.19.mlp.experts.223.gate_proj", + "model.layers.19.mlp.experts.223.up_proj", + "model.layers.19.mlp.experts.224.down_proj", + "model.layers.19.mlp.experts.224.gate_proj", + "model.layers.19.mlp.experts.224.up_proj", + "model.layers.19.mlp.experts.225.down_proj", + "model.layers.19.mlp.experts.225.gate_proj", + "model.layers.19.mlp.experts.225.up_proj", + "model.layers.19.mlp.experts.226.down_proj", + "model.layers.19.mlp.experts.226.gate_proj", + "model.layers.19.mlp.experts.226.up_proj", + "model.layers.19.mlp.experts.227.down_proj", + "model.layers.19.mlp.experts.227.gate_proj", + "model.layers.19.mlp.experts.227.up_proj", + "model.layers.19.mlp.experts.228.down_proj", + "model.layers.19.mlp.experts.228.gate_proj", + "model.layers.19.mlp.experts.228.up_proj", + "model.layers.19.mlp.experts.229.down_proj", + "model.layers.19.mlp.experts.229.gate_proj", + "model.layers.19.mlp.experts.229.up_proj", + "model.layers.19.mlp.experts.23.down_proj", + "model.layers.19.mlp.experts.23.gate_proj", + "model.layers.19.mlp.experts.23.up_proj", + "model.layers.19.mlp.experts.230.down_proj", + "model.layers.19.mlp.experts.230.gate_proj", + "model.layers.19.mlp.experts.230.up_proj", + "model.layers.19.mlp.experts.231.down_proj", + "model.layers.19.mlp.experts.231.gate_proj", + "model.layers.19.mlp.experts.231.up_proj", + "model.layers.19.mlp.experts.232.down_proj", + "model.layers.19.mlp.experts.232.gate_proj", + "model.layers.19.mlp.experts.232.up_proj", + "model.layers.19.mlp.experts.233.down_proj", + "model.layers.19.mlp.experts.233.gate_proj", + "model.layers.19.mlp.experts.233.up_proj", + "model.layers.19.mlp.experts.234.down_proj", + "model.layers.19.mlp.experts.234.gate_proj", + "model.layers.19.mlp.experts.234.up_proj", + "model.layers.19.mlp.experts.235.down_proj", + "model.layers.19.mlp.experts.235.gate_proj", + "model.layers.19.mlp.experts.235.up_proj", + "model.layers.19.mlp.experts.236.down_proj", + "model.layers.19.mlp.experts.236.gate_proj", + "model.layers.19.mlp.experts.236.up_proj", + "model.layers.19.mlp.experts.237.down_proj", + "model.layers.19.mlp.experts.237.gate_proj", + "model.layers.19.mlp.experts.237.up_proj", + "model.layers.19.mlp.experts.238.down_proj", + "model.layers.19.mlp.experts.238.gate_proj", + "model.layers.19.mlp.experts.238.up_proj", + "model.layers.19.mlp.experts.239.down_proj", + "model.layers.19.mlp.experts.239.gate_proj", + "model.layers.19.mlp.experts.239.up_proj", + "model.layers.19.mlp.experts.24.down_proj", + "model.layers.19.mlp.experts.24.gate_proj", + "model.layers.19.mlp.experts.24.up_proj", + "model.layers.19.mlp.experts.240.down_proj", + "model.layers.19.mlp.experts.240.gate_proj", + "model.layers.19.mlp.experts.240.up_proj", + "model.layers.19.mlp.experts.241.down_proj", + "model.layers.19.mlp.experts.241.gate_proj", + "model.layers.19.mlp.experts.241.up_proj", + "model.layers.19.mlp.experts.242.down_proj", + "model.layers.19.mlp.experts.242.gate_proj", + "model.layers.19.mlp.experts.242.up_proj", + "model.layers.19.mlp.experts.243.down_proj", + "model.layers.19.mlp.experts.243.gate_proj", + "model.layers.19.mlp.experts.243.up_proj", + "model.layers.19.mlp.experts.244.down_proj", + "model.layers.19.mlp.experts.244.gate_proj", + "model.layers.19.mlp.experts.244.up_proj", + "model.layers.19.mlp.experts.245.down_proj", + "model.layers.19.mlp.experts.245.gate_proj", + "model.layers.19.mlp.experts.245.up_proj", + "model.layers.19.mlp.experts.246.down_proj", + "model.layers.19.mlp.experts.246.gate_proj", + "model.layers.19.mlp.experts.246.up_proj", + "model.layers.19.mlp.experts.247.down_proj", + "model.layers.19.mlp.experts.247.gate_proj", + "model.layers.19.mlp.experts.247.up_proj", + "model.layers.19.mlp.experts.248.down_proj", + "model.layers.19.mlp.experts.248.gate_proj", + "model.layers.19.mlp.experts.248.up_proj", + "model.layers.19.mlp.experts.249.down_proj", + "model.layers.19.mlp.experts.249.gate_proj", + "model.layers.19.mlp.experts.249.up_proj", + "model.layers.19.mlp.experts.25.down_proj", + "model.layers.19.mlp.experts.25.gate_proj", + "model.layers.19.mlp.experts.25.up_proj", + "model.layers.19.mlp.experts.250.down_proj", + "model.layers.19.mlp.experts.250.gate_proj", + "model.layers.19.mlp.experts.250.up_proj", + "model.layers.19.mlp.experts.251.down_proj", + "model.layers.19.mlp.experts.251.gate_proj", + "model.layers.19.mlp.experts.251.up_proj", + "model.layers.19.mlp.experts.252.down_proj", + "model.layers.19.mlp.experts.252.gate_proj", + "model.layers.19.mlp.experts.252.up_proj", + "model.layers.19.mlp.experts.253.down_proj", + "model.layers.19.mlp.experts.253.gate_proj", + "model.layers.19.mlp.experts.253.up_proj", + "model.layers.19.mlp.experts.254.down_proj", + "model.layers.19.mlp.experts.254.gate_proj", + "model.layers.19.mlp.experts.254.up_proj", + "model.layers.19.mlp.experts.255.down_proj", + "model.layers.19.mlp.experts.255.gate_proj", + "model.layers.19.mlp.experts.255.up_proj", + "model.layers.19.mlp.experts.26.down_proj", + "model.layers.19.mlp.experts.26.gate_proj", + "model.layers.19.mlp.experts.26.up_proj", + "model.layers.19.mlp.experts.27.down_proj", + "model.layers.19.mlp.experts.27.gate_proj", + "model.layers.19.mlp.experts.27.up_proj", + "model.layers.19.mlp.experts.28.down_proj", + "model.layers.19.mlp.experts.28.gate_proj", + "model.layers.19.mlp.experts.28.up_proj", + "model.layers.19.mlp.experts.29.down_proj", + "model.layers.19.mlp.experts.29.gate_proj", + "model.layers.19.mlp.experts.29.up_proj", + "model.layers.19.mlp.experts.3.down_proj", + "model.layers.19.mlp.experts.3.gate_proj", + "model.layers.19.mlp.experts.3.up_proj", + "model.layers.19.mlp.experts.30.down_proj", + "model.layers.19.mlp.experts.30.gate_proj", + "model.layers.19.mlp.experts.30.up_proj", + "model.layers.19.mlp.experts.31.down_proj", + "model.layers.19.mlp.experts.31.gate_proj", + "model.layers.19.mlp.experts.31.up_proj", + "model.layers.19.mlp.experts.32.down_proj", + "model.layers.19.mlp.experts.32.gate_proj", + "model.layers.19.mlp.experts.32.up_proj", + "model.layers.19.mlp.experts.33.down_proj", + "model.layers.19.mlp.experts.33.gate_proj", + "model.layers.19.mlp.experts.33.up_proj", + "model.layers.19.mlp.experts.34.down_proj", + "model.layers.19.mlp.experts.34.gate_proj", + "model.layers.19.mlp.experts.34.up_proj", + "model.layers.19.mlp.experts.35.down_proj", + "model.layers.19.mlp.experts.35.gate_proj", + "model.layers.19.mlp.experts.35.up_proj", + "model.layers.19.mlp.experts.36.down_proj", + "model.layers.19.mlp.experts.36.gate_proj", + "model.layers.19.mlp.experts.36.up_proj", + "model.layers.19.mlp.experts.37.down_proj", + "model.layers.19.mlp.experts.37.gate_proj", + "model.layers.19.mlp.experts.37.up_proj", + "model.layers.19.mlp.experts.38.down_proj", + "model.layers.19.mlp.experts.38.gate_proj", + "model.layers.19.mlp.experts.38.up_proj", + "model.layers.19.mlp.experts.39.down_proj", + "model.layers.19.mlp.experts.39.gate_proj", + "model.layers.19.mlp.experts.39.up_proj", + "model.layers.19.mlp.experts.4.down_proj", + "model.layers.19.mlp.experts.4.gate_proj", + "model.layers.19.mlp.experts.4.up_proj", + "model.layers.19.mlp.experts.40.down_proj", + "model.layers.19.mlp.experts.40.gate_proj", + "model.layers.19.mlp.experts.40.up_proj", + "model.layers.19.mlp.experts.41.down_proj", + "model.layers.19.mlp.experts.41.gate_proj", + "model.layers.19.mlp.experts.41.up_proj", + "model.layers.19.mlp.experts.42.down_proj", + "model.layers.19.mlp.experts.42.gate_proj", + "model.layers.19.mlp.experts.42.up_proj", + "model.layers.19.mlp.experts.43.down_proj", + "model.layers.19.mlp.experts.43.gate_proj", + "model.layers.19.mlp.experts.43.up_proj", + "model.layers.19.mlp.experts.44.down_proj", + "model.layers.19.mlp.experts.44.gate_proj", + "model.layers.19.mlp.experts.44.up_proj", + "model.layers.19.mlp.experts.45.down_proj", + "model.layers.19.mlp.experts.45.gate_proj", + "model.layers.19.mlp.experts.45.up_proj", + "model.layers.19.mlp.experts.46.down_proj", + "model.layers.19.mlp.experts.46.gate_proj", + "model.layers.19.mlp.experts.46.up_proj", + "model.layers.19.mlp.experts.47.down_proj", + "model.layers.19.mlp.experts.47.gate_proj", + "model.layers.19.mlp.experts.47.up_proj", + "model.layers.19.mlp.experts.48.down_proj", + "model.layers.19.mlp.experts.48.gate_proj", + "model.layers.19.mlp.experts.48.up_proj", + "model.layers.19.mlp.experts.49.down_proj", + "model.layers.19.mlp.experts.49.gate_proj", + "model.layers.19.mlp.experts.49.up_proj", + "model.layers.19.mlp.experts.5.down_proj", + "model.layers.19.mlp.experts.5.gate_proj", + "model.layers.19.mlp.experts.5.up_proj", + "model.layers.19.mlp.experts.50.down_proj", + "model.layers.19.mlp.experts.50.gate_proj", + "model.layers.19.mlp.experts.50.up_proj", + "model.layers.19.mlp.experts.51.down_proj", + "model.layers.19.mlp.experts.51.gate_proj", + "model.layers.19.mlp.experts.51.up_proj", + "model.layers.19.mlp.experts.52.down_proj", + "model.layers.19.mlp.experts.52.gate_proj", + "model.layers.19.mlp.experts.52.up_proj", + "model.layers.19.mlp.experts.53.down_proj", + "model.layers.19.mlp.experts.53.gate_proj", + "model.layers.19.mlp.experts.53.up_proj", + "model.layers.19.mlp.experts.54.down_proj", + "model.layers.19.mlp.experts.54.gate_proj", + "model.layers.19.mlp.experts.54.up_proj", + "model.layers.19.mlp.experts.55.down_proj", + "model.layers.19.mlp.experts.55.gate_proj", + "model.layers.19.mlp.experts.55.up_proj", + "model.layers.19.mlp.experts.56.down_proj", + "model.layers.19.mlp.experts.56.gate_proj", + "model.layers.19.mlp.experts.56.up_proj", + "model.layers.19.mlp.experts.57.down_proj", + "model.layers.19.mlp.experts.57.gate_proj", + "model.layers.19.mlp.experts.57.up_proj", + "model.layers.19.mlp.experts.58.down_proj", + "model.layers.19.mlp.experts.58.gate_proj", + "model.layers.19.mlp.experts.58.up_proj", + "model.layers.19.mlp.experts.59.down_proj", + "model.layers.19.mlp.experts.59.gate_proj", + "model.layers.19.mlp.experts.59.up_proj", + "model.layers.19.mlp.experts.6.down_proj", + "model.layers.19.mlp.experts.6.gate_proj", + "model.layers.19.mlp.experts.6.up_proj", + "model.layers.19.mlp.experts.60.down_proj", + "model.layers.19.mlp.experts.60.gate_proj", + "model.layers.19.mlp.experts.60.up_proj", + "model.layers.19.mlp.experts.61.down_proj", + "model.layers.19.mlp.experts.61.gate_proj", + "model.layers.19.mlp.experts.61.up_proj", + "model.layers.19.mlp.experts.62.down_proj", + "model.layers.19.mlp.experts.62.gate_proj", + "model.layers.19.mlp.experts.62.up_proj", + "model.layers.19.mlp.experts.63.down_proj", + "model.layers.19.mlp.experts.63.gate_proj", + "model.layers.19.mlp.experts.63.up_proj", + "model.layers.19.mlp.experts.64.down_proj", + "model.layers.19.mlp.experts.64.gate_proj", + "model.layers.19.mlp.experts.64.up_proj", + "model.layers.19.mlp.experts.65.down_proj", + "model.layers.19.mlp.experts.65.gate_proj", + "model.layers.19.mlp.experts.65.up_proj", + "model.layers.19.mlp.experts.66.down_proj", + "model.layers.19.mlp.experts.66.gate_proj", + "model.layers.19.mlp.experts.66.up_proj", + "model.layers.19.mlp.experts.67.down_proj", + "model.layers.19.mlp.experts.67.gate_proj", + "model.layers.19.mlp.experts.67.up_proj", + "model.layers.19.mlp.experts.68.down_proj", + "model.layers.19.mlp.experts.68.gate_proj", + "model.layers.19.mlp.experts.68.up_proj", + "model.layers.19.mlp.experts.69.down_proj", + "model.layers.19.mlp.experts.69.gate_proj", + "model.layers.19.mlp.experts.69.up_proj", + "model.layers.19.mlp.experts.7.down_proj", + "model.layers.19.mlp.experts.7.gate_proj", + "model.layers.19.mlp.experts.7.up_proj", + "model.layers.19.mlp.experts.70.down_proj", + "model.layers.19.mlp.experts.70.gate_proj", + "model.layers.19.mlp.experts.70.up_proj", + "model.layers.19.mlp.experts.71.down_proj", + "model.layers.19.mlp.experts.71.gate_proj", + "model.layers.19.mlp.experts.71.up_proj", + "model.layers.19.mlp.experts.72.down_proj", + "model.layers.19.mlp.experts.72.gate_proj", + "model.layers.19.mlp.experts.72.up_proj", + "model.layers.19.mlp.experts.73.down_proj", + "model.layers.19.mlp.experts.73.gate_proj", + "model.layers.19.mlp.experts.73.up_proj", + "model.layers.19.mlp.experts.74.down_proj", + "model.layers.19.mlp.experts.74.gate_proj", + "model.layers.19.mlp.experts.74.up_proj", + "model.layers.19.mlp.experts.75.down_proj", + "model.layers.19.mlp.experts.75.gate_proj", + "model.layers.19.mlp.experts.75.up_proj", + "model.layers.19.mlp.experts.76.down_proj", + "model.layers.19.mlp.experts.76.gate_proj", + "model.layers.19.mlp.experts.76.up_proj", + "model.layers.19.mlp.experts.77.down_proj", + "model.layers.19.mlp.experts.77.gate_proj", + "model.layers.19.mlp.experts.77.up_proj", + "model.layers.19.mlp.experts.78.down_proj", + "model.layers.19.mlp.experts.78.gate_proj", + "model.layers.19.mlp.experts.78.up_proj", + "model.layers.19.mlp.experts.79.down_proj", + "model.layers.19.mlp.experts.79.gate_proj", + "model.layers.19.mlp.experts.79.up_proj", + "model.layers.19.mlp.experts.8.down_proj", + "model.layers.19.mlp.experts.8.gate_proj", + "model.layers.19.mlp.experts.8.up_proj", + "model.layers.19.mlp.experts.80.down_proj", + "model.layers.19.mlp.experts.80.gate_proj", + "model.layers.19.mlp.experts.80.up_proj", + "model.layers.19.mlp.experts.81.down_proj", + "model.layers.19.mlp.experts.81.gate_proj", + "model.layers.19.mlp.experts.81.up_proj", + "model.layers.19.mlp.experts.82.down_proj", + "model.layers.19.mlp.experts.82.gate_proj", + "model.layers.19.mlp.experts.82.up_proj", + "model.layers.19.mlp.experts.83.down_proj", + "model.layers.19.mlp.experts.83.gate_proj", + "model.layers.19.mlp.experts.83.up_proj", + "model.layers.19.mlp.experts.84.down_proj", + "model.layers.19.mlp.experts.84.gate_proj", + "model.layers.19.mlp.experts.84.up_proj", + "model.layers.19.mlp.experts.85.down_proj", + "model.layers.19.mlp.experts.85.gate_proj", + "model.layers.19.mlp.experts.85.up_proj", + "model.layers.19.mlp.experts.86.down_proj", + "model.layers.19.mlp.experts.86.gate_proj", + "model.layers.19.mlp.experts.86.up_proj", + "model.layers.19.mlp.experts.87.down_proj", + "model.layers.19.mlp.experts.87.gate_proj", + "model.layers.19.mlp.experts.87.up_proj", + "model.layers.19.mlp.experts.88.down_proj", + "model.layers.19.mlp.experts.88.gate_proj", + "model.layers.19.mlp.experts.88.up_proj", + "model.layers.19.mlp.experts.89.down_proj", + "model.layers.19.mlp.experts.89.gate_proj", + "model.layers.19.mlp.experts.89.up_proj", + "model.layers.19.mlp.experts.9.down_proj", + "model.layers.19.mlp.experts.9.gate_proj", + "model.layers.19.mlp.experts.9.up_proj", + "model.layers.19.mlp.experts.90.down_proj", + "model.layers.19.mlp.experts.90.gate_proj", + "model.layers.19.mlp.experts.90.up_proj", + "model.layers.19.mlp.experts.91.down_proj", + "model.layers.19.mlp.experts.91.gate_proj", + "model.layers.19.mlp.experts.91.up_proj", + "model.layers.19.mlp.experts.92.down_proj", + "model.layers.19.mlp.experts.92.gate_proj", + "model.layers.19.mlp.experts.92.up_proj", + "model.layers.19.mlp.experts.93.down_proj", + "model.layers.19.mlp.experts.93.gate_proj", + "model.layers.19.mlp.experts.93.up_proj", + "model.layers.19.mlp.experts.94.down_proj", + "model.layers.19.mlp.experts.94.gate_proj", + "model.layers.19.mlp.experts.94.up_proj", + "model.layers.19.mlp.experts.95.down_proj", + "model.layers.19.mlp.experts.95.gate_proj", + "model.layers.19.mlp.experts.95.up_proj", + "model.layers.19.mlp.experts.96.down_proj", + "model.layers.19.mlp.experts.96.gate_proj", + "model.layers.19.mlp.experts.96.up_proj", + "model.layers.19.mlp.experts.97.down_proj", + "model.layers.19.mlp.experts.97.gate_proj", + "model.layers.19.mlp.experts.97.up_proj", + "model.layers.19.mlp.experts.98.down_proj", + "model.layers.19.mlp.experts.98.gate_proj", + "model.layers.19.mlp.experts.98.up_proj", + "model.layers.19.mlp.experts.99.down_proj", + "model.layers.19.mlp.experts.99.gate_proj", + "model.layers.19.mlp.experts.99.up_proj", + "model.layers.20.mlp.experts.0.down_proj", + "model.layers.20.mlp.experts.0.gate_proj", + "model.layers.20.mlp.experts.0.up_proj", + "model.layers.20.mlp.experts.1.down_proj", + "model.layers.20.mlp.experts.1.gate_proj", + "model.layers.20.mlp.experts.1.up_proj", + "model.layers.20.mlp.experts.10.down_proj", + "model.layers.20.mlp.experts.10.gate_proj", + "model.layers.20.mlp.experts.10.up_proj", + "model.layers.20.mlp.experts.100.down_proj", + "model.layers.20.mlp.experts.100.gate_proj", + "model.layers.20.mlp.experts.100.up_proj", + "model.layers.20.mlp.experts.101.down_proj", + "model.layers.20.mlp.experts.101.gate_proj", + "model.layers.20.mlp.experts.101.up_proj", + "model.layers.20.mlp.experts.102.down_proj", + "model.layers.20.mlp.experts.102.gate_proj", + "model.layers.20.mlp.experts.102.up_proj", + "model.layers.20.mlp.experts.103.down_proj", + "model.layers.20.mlp.experts.103.gate_proj", + "model.layers.20.mlp.experts.103.up_proj", + "model.layers.20.mlp.experts.104.down_proj", + "model.layers.20.mlp.experts.104.gate_proj", + "model.layers.20.mlp.experts.104.up_proj", + "model.layers.20.mlp.experts.105.down_proj", + "model.layers.20.mlp.experts.105.gate_proj", + "model.layers.20.mlp.experts.105.up_proj", + "model.layers.20.mlp.experts.106.down_proj", + "model.layers.20.mlp.experts.106.gate_proj", + "model.layers.20.mlp.experts.106.up_proj", + "model.layers.20.mlp.experts.107.down_proj", + "model.layers.20.mlp.experts.107.gate_proj", + "model.layers.20.mlp.experts.107.up_proj", + "model.layers.20.mlp.experts.108.down_proj", + "model.layers.20.mlp.experts.108.gate_proj", + "model.layers.20.mlp.experts.108.up_proj", + "model.layers.20.mlp.experts.109.down_proj", + "model.layers.20.mlp.experts.109.gate_proj", + "model.layers.20.mlp.experts.109.up_proj", + "model.layers.20.mlp.experts.11.down_proj", + "model.layers.20.mlp.experts.11.gate_proj", + "model.layers.20.mlp.experts.11.up_proj", + "model.layers.20.mlp.experts.110.down_proj", + "model.layers.20.mlp.experts.110.gate_proj", + "model.layers.20.mlp.experts.110.up_proj", + "model.layers.20.mlp.experts.111.down_proj", + "model.layers.20.mlp.experts.111.gate_proj", + "model.layers.20.mlp.experts.111.up_proj", + "model.layers.20.mlp.experts.112.down_proj", + "model.layers.20.mlp.experts.112.gate_proj", + "model.layers.20.mlp.experts.112.up_proj", + "model.layers.20.mlp.experts.113.down_proj", + "model.layers.20.mlp.experts.113.gate_proj", + "model.layers.20.mlp.experts.113.up_proj", + "model.layers.20.mlp.experts.114.down_proj", + "model.layers.20.mlp.experts.114.gate_proj", + "model.layers.20.mlp.experts.114.up_proj", + "model.layers.20.mlp.experts.115.down_proj", + "model.layers.20.mlp.experts.115.gate_proj", + "model.layers.20.mlp.experts.115.up_proj", + "model.layers.20.mlp.experts.116.down_proj", + "model.layers.20.mlp.experts.116.gate_proj", + "model.layers.20.mlp.experts.116.up_proj", + "model.layers.20.mlp.experts.117.down_proj", + "model.layers.20.mlp.experts.117.gate_proj", + "model.layers.20.mlp.experts.117.up_proj", + "model.layers.20.mlp.experts.118.down_proj", + "model.layers.20.mlp.experts.118.gate_proj", + "model.layers.20.mlp.experts.118.up_proj", + "model.layers.20.mlp.experts.119.down_proj", + "model.layers.20.mlp.experts.119.gate_proj", + "model.layers.20.mlp.experts.119.up_proj", + "model.layers.20.mlp.experts.12.down_proj", + "model.layers.20.mlp.experts.12.gate_proj", + "model.layers.20.mlp.experts.12.up_proj", + "model.layers.20.mlp.experts.120.down_proj", + "model.layers.20.mlp.experts.120.gate_proj", + "model.layers.20.mlp.experts.120.up_proj", + "model.layers.20.mlp.experts.121.down_proj", + "model.layers.20.mlp.experts.121.gate_proj", + "model.layers.20.mlp.experts.121.up_proj", + "model.layers.20.mlp.experts.122.down_proj", + "model.layers.20.mlp.experts.122.gate_proj", + "model.layers.20.mlp.experts.122.up_proj", + "model.layers.20.mlp.experts.123.down_proj", + "model.layers.20.mlp.experts.123.gate_proj", + "model.layers.20.mlp.experts.123.up_proj", + "model.layers.20.mlp.experts.124.down_proj", + "model.layers.20.mlp.experts.124.gate_proj", + "model.layers.20.mlp.experts.124.up_proj", + "model.layers.20.mlp.experts.125.down_proj", + "model.layers.20.mlp.experts.125.gate_proj", + "model.layers.20.mlp.experts.125.up_proj", + "model.layers.20.mlp.experts.126.down_proj", + "model.layers.20.mlp.experts.126.gate_proj", + "model.layers.20.mlp.experts.126.up_proj", + "model.layers.20.mlp.experts.127.down_proj", + "model.layers.20.mlp.experts.127.gate_proj", + "model.layers.20.mlp.experts.127.up_proj", + "model.layers.20.mlp.experts.128.down_proj", + "model.layers.20.mlp.experts.128.gate_proj", + "model.layers.20.mlp.experts.128.up_proj", + "model.layers.20.mlp.experts.129.down_proj", + "model.layers.20.mlp.experts.129.gate_proj", + "model.layers.20.mlp.experts.129.up_proj", + "model.layers.20.mlp.experts.13.down_proj", + "model.layers.20.mlp.experts.13.gate_proj", + "model.layers.20.mlp.experts.13.up_proj", + "model.layers.20.mlp.experts.130.down_proj", + "model.layers.20.mlp.experts.130.gate_proj", + "model.layers.20.mlp.experts.130.up_proj", + "model.layers.20.mlp.experts.131.down_proj", + "model.layers.20.mlp.experts.131.gate_proj", + "model.layers.20.mlp.experts.131.up_proj", + "model.layers.20.mlp.experts.132.down_proj", + "model.layers.20.mlp.experts.132.gate_proj", + "model.layers.20.mlp.experts.132.up_proj", + "model.layers.20.mlp.experts.133.down_proj", + "model.layers.20.mlp.experts.133.gate_proj", + "model.layers.20.mlp.experts.133.up_proj", + "model.layers.20.mlp.experts.134.down_proj", + "model.layers.20.mlp.experts.134.gate_proj", + "model.layers.20.mlp.experts.134.up_proj", + "model.layers.20.mlp.experts.135.down_proj", + "model.layers.20.mlp.experts.135.gate_proj", + "model.layers.20.mlp.experts.135.up_proj", + "model.layers.20.mlp.experts.136.down_proj", + "model.layers.20.mlp.experts.136.gate_proj", + "model.layers.20.mlp.experts.136.up_proj", + "model.layers.20.mlp.experts.137.down_proj", + "model.layers.20.mlp.experts.137.gate_proj", + "model.layers.20.mlp.experts.137.up_proj", + "model.layers.20.mlp.experts.138.down_proj", + "model.layers.20.mlp.experts.138.gate_proj", + "model.layers.20.mlp.experts.138.up_proj", + "model.layers.20.mlp.experts.139.down_proj", + "model.layers.20.mlp.experts.139.gate_proj", + "model.layers.20.mlp.experts.139.up_proj", + "model.layers.20.mlp.experts.14.down_proj", + "model.layers.20.mlp.experts.14.gate_proj", + "model.layers.20.mlp.experts.14.up_proj", + "model.layers.20.mlp.experts.140.down_proj", + "model.layers.20.mlp.experts.140.gate_proj", + "model.layers.20.mlp.experts.140.up_proj", + "model.layers.20.mlp.experts.141.down_proj", + "model.layers.20.mlp.experts.141.gate_proj", + "model.layers.20.mlp.experts.141.up_proj", + "model.layers.20.mlp.experts.142.down_proj", + "model.layers.20.mlp.experts.142.gate_proj", + "model.layers.20.mlp.experts.142.up_proj", + "model.layers.20.mlp.experts.143.down_proj", + "model.layers.20.mlp.experts.143.gate_proj", + "model.layers.20.mlp.experts.143.up_proj", + "model.layers.20.mlp.experts.144.down_proj", + "model.layers.20.mlp.experts.144.gate_proj", + "model.layers.20.mlp.experts.144.up_proj", + "model.layers.20.mlp.experts.145.down_proj", + "model.layers.20.mlp.experts.145.gate_proj", + "model.layers.20.mlp.experts.145.up_proj", + "model.layers.20.mlp.experts.146.down_proj", + "model.layers.20.mlp.experts.146.gate_proj", + "model.layers.20.mlp.experts.146.up_proj", + "model.layers.20.mlp.experts.147.down_proj", + "model.layers.20.mlp.experts.147.gate_proj", + "model.layers.20.mlp.experts.147.up_proj", + "model.layers.20.mlp.experts.148.down_proj", + "model.layers.20.mlp.experts.148.gate_proj", + "model.layers.20.mlp.experts.148.up_proj", + "model.layers.20.mlp.experts.149.down_proj", + "model.layers.20.mlp.experts.149.gate_proj", + "model.layers.20.mlp.experts.149.up_proj", + "model.layers.20.mlp.experts.15.down_proj", + "model.layers.20.mlp.experts.15.gate_proj", + "model.layers.20.mlp.experts.15.up_proj", + "model.layers.20.mlp.experts.150.down_proj", + "model.layers.20.mlp.experts.150.gate_proj", + "model.layers.20.mlp.experts.150.up_proj", + "model.layers.20.mlp.experts.151.down_proj", + "model.layers.20.mlp.experts.151.gate_proj", + "model.layers.20.mlp.experts.151.up_proj", + "model.layers.20.mlp.experts.152.down_proj", + "model.layers.20.mlp.experts.152.gate_proj", + "model.layers.20.mlp.experts.152.up_proj", + "model.layers.20.mlp.experts.153.down_proj", + "model.layers.20.mlp.experts.153.gate_proj", + "model.layers.20.mlp.experts.153.up_proj", + "model.layers.20.mlp.experts.154.down_proj", + "model.layers.20.mlp.experts.154.gate_proj", + "model.layers.20.mlp.experts.154.up_proj", + "model.layers.20.mlp.experts.155.down_proj", + "model.layers.20.mlp.experts.155.gate_proj", + "model.layers.20.mlp.experts.155.up_proj", + "model.layers.20.mlp.experts.156.down_proj", + "model.layers.20.mlp.experts.156.gate_proj", + "model.layers.20.mlp.experts.156.up_proj", + "model.layers.20.mlp.experts.157.down_proj", + "model.layers.20.mlp.experts.157.gate_proj", + "model.layers.20.mlp.experts.157.up_proj", + "model.layers.20.mlp.experts.158.down_proj", + "model.layers.20.mlp.experts.158.gate_proj", + "model.layers.20.mlp.experts.158.up_proj", + "model.layers.20.mlp.experts.159.down_proj", + "model.layers.20.mlp.experts.159.gate_proj", + "model.layers.20.mlp.experts.159.up_proj", + "model.layers.20.mlp.experts.16.down_proj", + "model.layers.20.mlp.experts.16.gate_proj", + "model.layers.20.mlp.experts.16.up_proj", + "model.layers.20.mlp.experts.160.down_proj", + "model.layers.20.mlp.experts.160.gate_proj", + "model.layers.20.mlp.experts.160.up_proj", + "model.layers.20.mlp.experts.161.down_proj", + "model.layers.20.mlp.experts.161.gate_proj", + "model.layers.20.mlp.experts.161.up_proj", + "model.layers.20.mlp.experts.162.down_proj", + "model.layers.20.mlp.experts.162.gate_proj", + "model.layers.20.mlp.experts.162.up_proj", + "model.layers.20.mlp.experts.163.down_proj", + "model.layers.20.mlp.experts.163.gate_proj", + "model.layers.20.mlp.experts.163.up_proj", + "model.layers.20.mlp.experts.164.down_proj", + "model.layers.20.mlp.experts.164.gate_proj", + "model.layers.20.mlp.experts.164.up_proj", + "model.layers.20.mlp.experts.165.down_proj", + "model.layers.20.mlp.experts.165.gate_proj", + "model.layers.20.mlp.experts.165.up_proj", + "model.layers.20.mlp.experts.166.down_proj", + "model.layers.20.mlp.experts.166.gate_proj", + "model.layers.20.mlp.experts.166.up_proj", + "model.layers.20.mlp.experts.167.down_proj", + "model.layers.20.mlp.experts.167.gate_proj", + "model.layers.20.mlp.experts.167.up_proj", + "model.layers.20.mlp.experts.168.down_proj", + "model.layers.20.mlp.experts.168.gate_proj", + "model.layers.20.mlp.experts.168.up_proj", + "model.layers.20.mlp.experts.169.down_proj", + "model.layers.20.mlp.experts.169.gate_proj", + "model.layers.20.mlp.experts.169.up_proj", + "model.layers.20.mlp.experts.17.down_proj", + "model.layers.20.mlp.experts.17.gate_proj", + "model.layers.20.mlp.experts.17.up_proj", + "model.layers.20.mlp.experts.170.down_proj", + "model.layers.20.mlp.experts.170.gate_proj", + "model.layers.20.mlp.experts.170.up_proj", + "model.layers.20.mlp.experts.171.down_proj", + "model.layers.20.mlp.experts.171.gate_proj", + "model.layers.20.mlp.experts.171.up_proj", + "model.layers.20.mlp.experts.172.down_proj", + "model.layers.20.mlp.experts.172.gate_proj", + "model.layers.20.mlp.experts.172.up_proj", + "model.layers.20.mlp.experts.173.down_proj", + "model.layers.20.mlp.experts.173.gate_proj", + "model.layers.20.mlp.experts.173.up_proj", + "model.layers.20.mlp.experts.174.down_proj", + "model.layers.20.mlp.experts.174.gate_proj", + "model.layers.20.mlp.experts.174.up_proj", + "model.layers.20.mlp.experts.175.down_proj", + "model.layers.20.mlp.experts.175.gate_proj", + "model.layers.20.mlp.experts.175.up_proj", + "model.layers.20.mlp.experts.176.down_proj", + "model.layers.20.mlp.experts.176.gate_proj", + "model.layers.20.mlp.experts.176.up_proj", + "model.layers.20.mlp.experts.177.down_proj", + "model.layers.20.mlp.experts.177.gate_proj", + "model.layers.20.mlp.experts.177.up_proj", + "model.layers.20.mlp.experts.178.down_proj", + "model.layers.20.mlp.experts.178.gate_proj", + "model.layers.20.mlp.experts.178.up_proj", + "model.layers.20.mlp.experts.179.down_proj", + "model.layers.20.mlp.experts.179.gate_proj", + "model.layers.20.mlp.experts.179.up_proj", + "model.layers.20.mlp.experts.18.down_proj", + "model.layers.20.mlp.experts.18.gate_proj", + "model.layers.20.mlp.experts.18.up_proj", + "model.layers.20.mlp.experts.180.down_proj", + "model.layers.20.mlp.experts.180.gate_proj", + "model.layers.20.mlp.experts.180.up_proj", + "model.layers.20.mlp.experts.181.down_proj", + "model.layers.20.mlp.experts.181.gate_proj", + "model.layers.20.mlp.experts.181.up_proj", + "model.layers.20.mlp.experts.182.down_proj", + "model.layers.20.mlp.experts.182.gate_proj", + "model.layers.20.mlp.experts.182.up_proj", + "model.layers.20.mlp.experts.183.down_proj", + "model.layers.20.mlp.experts.183.gate_proj", + "model.layers.20.mlp.experts.183.up_proj", + "model.layers.20.mlp.experts.184.down_proj", + "model.layers.20.mlp.experts.184.gate_proj", + "model.layers.20.mlp.experts.184.up_proj", + "model.layers.20.mlp.experts.185.down_proj", + "model.layers.20.mlp.experts.185.gate_proj", + "model.layers.20.mlp.experts.185.up_proj", + "model.layers.20.mlp.experts.186.down_proj", + "model.layers.20.mlp.experts.186.gate_proj", + "model.layers.20.mlp.experts.186.up_proj", + "model.layers.20.mlp.experts.187.down_proj", + "model.layers.20.mlp.experts.187.gate_proj", + "model.layers.20.mlp.experts.187.up_proj", + "model.layers.20.mlp.experts.188.down_proj", + "model.layers.20.mlp.experts.188.gate_proj", + "model.layers.20.mlp.experts.188.up_proj", + "model.layers.20.mlp.experts.189.down_proj", + "model.layers.20.mlp.experts.189.gate_proj", + "model.layers.20.mlp.experts.189.up_proj", + "model.layers.20.mlp.experts.19.down_proj", + "model.layers.20.mlp.experts.19.gate_proj", + "model.layers.20.mlp.experts.19.up_proj", + "model.layers.20.mlp.experts.190.down_proj", + "model.layers.20.mlp.experts.190.gate_proj", + "model.layers.20.mlp.experts.190.up_proj", + "model.layers.20.mlp.experts.191.down_proj", + "model.layers.20.mlp.experts.191.gate_proj", + "model.layers.20.mlp.experts.191.up_proj", + "model.layers.20.mlp.experts.192.down_proj", + "model.layers.20.mlp.experts.192.gate_proj", + "model.layers.20.mlp.experts.192.up_proj", + "model.layers.20.mlp.experts.193.down_proj", + "model.layers.20.mlp.experts.193.gate_proj", + "model.layers.20.mlp.experts.193.up_proj", + "model.layers.20.mlp.experts.194.down_proj", + "model.layers.20.mlp.experts.194.gate_proj", + "model.layers.20.mlp.experts.194.up_proj", + "model.layers.20.mlp.experts.195.down_proj", + "model.layers.20.mlp.experts.195.gate_proj", + "model.layers.20.mlp.experts.195.up_proj", + "model.layers.20.mlp.experts.196.down_proj", + "model.layers.20.mlp.experts.196.gate_proj", + "model.layers.20.mlp.experts.196.up_proj", + "model.layers.20.mlp.experts.197.down_proj", + "model.layers.20.mlp.experts.197.gate_proj", + "model.layers.20.mlp.experts.197.up_proj", + "model.layers.20.mlp.experts.198.down_proj", + "model.layers.20.mlp.experts.198.gate_proj", + "model.layers.20.mlp.experts.198.up_proj", + "model.layers.20.mlp.experts.199.down_proj", + "model.layers.20.mlp.experts.199.gate_proj", + "model.layers.20.mlp.experts.199.up_proj", + "model.layers.20.mlp.experts.2.down_proj", + "model.layers.20.mlp.experts.2.gate_proj", + "model.layers.20.mlp.experts.2.up_proj", + "model.layers.20.mlp.experts.20.down_proj", + "model.layers.20.mlp.experts.20.gate_proj", + "model.layers.20.mlp.experts.20.up_proj", + "model.layers.20.mlp.experts.200.down_proj", + "model.layers.20.mlp.experts.200.gate_proj", + "model.layers.20.mlp.experts.200.up_proj", + "model.layers.20.mlp.experts.201.down_proj", + "model.layers.20.mlp.experts.201.gate_proj", + "model.layers.20.mlp.experts.201.up_proj", + "model.layers.20.mlp.experts.202.down_proj", + "model.layers.20.mlp.experts.202.gate_proj", + "model.layers.20.mlp.experts.202.up_proj", + "model.layers.20.mlp.experts.203.down_proj", + "model.layers.20.mlp.experts.203.gate_proj", + "model.layers.20.mlp.experts.203.up_proj", + "model.layers.20.mlp.experts.204.down_proj", + "model.layers.20.mlp.experts.204.gate_proj", + "model.layers.20.mlp.experts.204.up_proj", + "model.layers.20.mlp.experts.205.down_proj", + "model.layers.20.mlp.experts.205.gate_proj", + "model.layers.20.mlp.experts.205.up_proj", + "model.layers.20.mlp.experts.206.down_proj", + "model.layers.20.mlp.experts.206.gate_proj", + "model.layers.20.mlp.experts.206.up_proj", + "model.layers.20.mlp.experts.207.down_proj", + "model.layers.20.mlp.experts.207.gate_proj", + "model.layers.20.mlp.experts.207.up_proj", + "model.layers.20.mlp.experts.208.down_proj", + "model.layers.20.mlp.experts.208.gate_proj", + "model.layers.20.mlp.experts.208.up_proj", + "model.layers.20.mlp.experts.209.down_proj", + "model.layers.20.mlp.experts.209.gate_proj", + "model.layers.20.mlp.experts.209.up_proj", + "model.layers.20.mlp.experts.21.down_proj", + "model.layers.20.mlp.experts.21.gate_proj", + "model.layers.20.mlp.experts.21.up_proj", + "model.layers.20.mlp.experts.210.down_proj", + "model.layers.20.mlp.experts.210.gate_proj", + "model.layers.20.mlp.experts.210.up_proj", + "model.layers.20.mlp.experts.211.down_proj", + "model.layers.20.mlp.experts.211.gate_proj", + "model.layers.20.mlp.experts.211.up_proj", + "model.layers.20.mlp.experts.212.down_proj", + "model.layers.20.mlp.experts.212.gate_proj", + "model.layers.20.mlp.experts.212.up_proj", + "model.layers.20.mlp.experts.213.down_proj", + "model.layers.20.mlp.experts.213.gate_proj", + "model.layers.20.mlp.experts.213.up_proj", + "model.layers.20.mlp.experts.214.down_proj", + "model.layers.20.mlp.experts.214.gate_proj", + "model.layers.20.mlp.experts.214.up_proj", + "model.layers.20.mlp.experts.215.down_proj", + "model.layers.20.mlp.experts.215.gate_proj", + "model.layers.20.mlp.experts.215.up_proj", + "model.layers.20.mlp.experts.216.down_proj", + "model.layers.20.mlp.experts.216.gate_proj", + "model.layers.20.mlp.experts.216.up_proj", + "model.layers.20.mlp.experts.217.down_proj", + "model.layers.20.mlp.experts.217.gate_proj", + "model.layers.20.mlp.experts.217.up_proj", + "model.layers.20.mlp.experts.218.down_proj", + "model.layers.20.mlp.experts.218.gate_proj", + "model.layers.20.mlp.experts.218.up_proj", + "model.layers.20.mlp.experts.219.down_proj", + "model.layers.20.mlp.experts.219.gate_proj", + "model.layers.20.mlp.experts.219.up_proj", + "model.layers.20.mlp.experts.22.down_proj", + "model.layers.20.mlp.experts.22.gate_proj", + "model.layers.20.mlp.experts.22.up_proj", + "model.layers.20.mlp.experts.220.down_proj", + "model.layers.20.mlp.experts.220.gate_proj", + "model.layers.20.mlp.experts.220.up_proj", + "model.layers.20.mlp.experts.221.down_proj", + "model.layers.20.mlp.experts.221.gate_proj", + "model.layers.20.mlp.experts.221.up_proj", + "model.layers.20.mlp.experts.222.down_proj", + "model.layers.20.mlp.experts.222.gate_proj", + "model.layers.20.mlp.experts.222.up_proj", + "model.layers.20.mlp.experts.223.down_proj", + "model.layers.20.mlp.experts.223.gate_proj", + "model.layers.20.mlp.experts.223.up_proj", + "model.layers.20.mlp.experts.224.down_proj", + "model.layers.20.mlp.experts.224.gate_proj", + "model.layers.20.mlp.experts.224.up_proj", + "model.layers.20.mlp.experts.225.down_proj", + "model.layers.20.mlp.experts.225.gate_proj", + "model.layers.20.mlp.experts.225.up_proj", + "model.layers.20.mlp.experts.226.down_proj", + "model.layers.20.mlp.experts.226.gate_proj", + "model.layers.20.mlp.experts.226.up_proj", + "model.layers.20.mlp.experts.227.down_proj", + "model.layers.20.mlp.experts.227.gate_proj", + "model.layers.20.mlp.experts.227.up_proj", + "model.layers.20.mlp.experts.228.down_proj", + "model.layers.20.mlp.experts.228.gate_proj", + "model.layers.20.mlp.experts.228.up_proj", + "model.layers.20.mlp.experts.229.down_proj", + "model.layers.20.mlp.experts.229.gate_proj", + "model.layers.20.mlp.experts.229.up_proj", + "model.layers.20.mlp.experts.23.down_proj", + "model.layers.20.mlp.experts.23.gate_proj", + "model.layers.20.mlp.experts.23.up_proj", + "model.layers.20.mlp.experts.230.down_proj", + "model.layers.20.mlp.experts.230.gate_proj", + "model.layers.20.mlp.experts.230.up_proj", + "model.layers.20.mlp.experts.231.down_proj", + "model.layers.20.mlp.experts.231.gate_proj", + "model.layers.20.mlp.experts.231.up_proj", + "model.layers.20.mlp.experts.232.down_proj", + "model.layers.20.mlp.experts.232.gate_proj", + "model.layers.20.mlp.experts.232.up_proj", + "model.layers.20.mlp.experts.233.down_proj", + "model.layers.20.mlp.experts.233.gate_proj", + "model.layers.20.mlp.experts.233.up_proj", + "model.layers.20.mlp.experts.234.down_proj", + "model.layers.20.mlp.experts.234.gate_proj", + "model.layers.20.mlp.experts.234.up_proj", + "model.layers.20.mlp.experts.235.down_proj", + "model.layers.20.mlp.experts.235.gate_proj", + "model.layers.20.mlp.experts.235.up_proj", + "model.layers.20.mlp.experts.236.down_proj", + "model.layers.20.mlp.experts.236.gate_proj", + "model.layers.20.mlp.experts.236.up_proj", + "model.layers.20.mlp.experts.237.down_proj", + "model.layers.20.mlp.experts.237.gate_proj", + "model.layers.20.mlp.experts.237.up_proj", + "model.layers.20.mlp.experts.238.down_proj", + "model.layers.20.mlp.experts.238.gate_proj", + "model.layers.20.mlp.experts.238.up_proj", + "model.layers.20.mlp.experts.239.down_proj", + "model.layers.20.mlp.experts.239.gate_proj", + "model.layers.20.mlp.experts.239.up_proj", + "model.layers.20.mlp.experts.24.down_proj", + "model.layers.20.mlp.experts.24.gate_proj", + "model.layers.20.mlp.experts.24.up_proj", + "model.layers.20.mlp.experts.240.down_proj", + "model.layers.20.mlp.experts.240.gate_proj", + "model.layers.20.mlp.experts.240.up_proj", + "model.layers.20.mlp.experts.241.down_proj", + "model.layers.20.mlp.experts.241.gate_proj", + "model.layers.20.mlp.experts.241.up_proj", + "model.layers.20.mlp.experts.242.down_proj", + "model.layers.20.mlp.experts.242.gate_proj", + "model.layers.20.mlp.experts.242.up_proj", + "model.layers.20.mlp.experts.243.down_proj", + "model.layers.20.mlp.experts.243.gate_proj", + "model.layers.20.mlp.experts.243.up_proj", + "model.layers.20.mlp.experts.244.down_proj", + "model.layers.20.mlp.experts.244.gate_proj", + "model.layers.20.mlp.experts.244.up_proj", + "model.layers.20.mlp.experts.245.down_proj", + "model.layers.20.mlp.experts.245.gate_proj", + "model.layers.20.mlp.experts.245.up_proj", + "model.layers.20.mlp.experts.246.down_proj", + "model.layers.20.mlp.experts.246.gate_proj", + "model.layers.20.mlp.experts.246.up_proj", + "model.layers.20.mlp.experts.247.down_proj", + "model.layers.20.mlp.experts.247.gate_proj", + "model.layers.20.mlp.experts.247.up_proj", + "model.layers.20.mlp.experts.248.down_proj", + "model.layers.20.mlp.experts.248.gate_proj", + "model.layers.20.mlp.experts.248.up_proj", + "model.layers.20.mlp.experts.249.down_proj", + "model.layers.20.mlp.experts.249.gate_proj", + "model.layers.20.mlp.experts.249.up_proj", + "model.layers.20.mlp.experts.25.down_proj", + "model.layers.20.mlp.experts.25.gate_proj", + "model.layers.20.mlp.experts.25.up_proj", + "model.layers.20.mlp.experts.250.down_proj", + "model.layers.20.mlp.experts.250.gate_proj", + "model.layers.20.mlp.experts.250.up_proj", + "model.layers.20.mlp.experts.251.down_proj", + "model.layers.20.mlp.experts.251.gate_proj", + "model.layers.20.mlp.experts.251.up_proj", + "model.layers.20.mlp.experts.252.down_proj", + "model.layers.20.mlp.experts.252.gate_proj", + "model.layers.20.mlp.experts.252.up_proj", + "model.layers.20.mlp.experts.253.down_proj", + "model.layers.20.mlp.experts.253.gate_proj", + "model.layers.20.mlp.experts.253.up_proj", + "model.layers.20.mlp.experts.254.down_proj", + "model.layers.20.mlp.experts.254.gate_proj", + "model.layers.20.mlp.experts.254.up_proj", + "model.layers.20.mlp.experts.255.down_proj", + "model.layers.20.mlp.experts.255.gate_proj", + "model.layers.20.mlp.experts.255.up_proj", + "model.layers.20.mlp.experts.26.down_proj", + "model.layers.20.mlp.experts.26.gate_proj", + "model.layers.20.mlp.experts.26.up_proj", + "model.layers.20.mlp.experts.27.down_proj", + "model.layers.20.mlp.experts.27.gate_proj", + "model.layers.20.mlp.experts.27.up_proj", + "model.layers.20.mlp.experts.28.down_proj", + "model.layers.20.mlp.experts.28.gate_proj", + "model.layers.20.mlp.experts.28.up_proj", + "model.layers.20.mlp.experts.29.down_proj", + "model.layers.20.mlp.experts.29.gate_proj", + "model.layers.20.mlp.experts.29.up_proj", + "model.layers.20.mlp.experts.3.down_proj", + "model.layers.20.mlp.experts.3.gate_proj", + "model.layers.20.mlp.experts.3.up_proj", + "model.layers.20.mlp.experts.30.down_proj", + "model.layers.20.mlp.experts.30.gate_proj", + "model.layers.20.mlp.experts.30.up_proj", + "model.layers.20.mlp.experts.31.down_proj", + "model.layers.20.mlp.experts.31.gate_proj", + "model.layers.20.mlp.experts.31.up_proj", + "model.layers.20.mlp.experts.32.down_proj", + "model.layers.20.mlp.experts.32.gate_proj", + "model.layers.20.mlp.experts.32.up_proj", + "model.layers.20.mlp.experts.33.down_proj", + "model.layers.20.mlp.experts.33.gate_proj", + "model.layers.20.mlp.experts.33.up_proj", + "model.layers.20.mlp.experts.34.down_proj", + "model.layers.20.mlp.experts.34.gate_proj", + "model.layers.20.mlp.experts.34.up_proj", + "model.layers.20.mlp.experts.35.down_proj", + "model.layers.20.mlp.experts.35.gate_proj", + "model.layers.20.mlp.experts.35.up_proj", + "model.layers.20.mlp.experts.36.down_proj", + "model.layers.20.mlp.experts.36.gate_proj", + "model.layers.20.mlp.experts.36.up_proj", + "model.layers.20.mlp.experts.37.down_proj", + "model.layers.20.mlp.experts.37.gate_proj", + "model.layers.20.mlp.experts.37.up_proj", + "model.layers.20.mlp.experts.38.down_proj", + "model.layers.20.mlp.experts.38.gate_proj", + "model.layers.20.mlp.experts.38.up_proj", + "model.layers.20.mlp.experts.39.down_proj", + "model.layers.20.mlp.experts.39.gate_proj", + "model.layers.20.mlp.experts.39.up_proj", + "model.layers.20.mlp.experts.4.down_proj", + "model.layers.20.mlp.experts.4.gate_proj", + "model.layers.20.mlp.experts.4.up_proj", + "model.layers.20.mlp.experts.40.down_proj", + "model.layers.20.mlp.experts.40.gate_proj", + "model.layers.20.mlp.experts.40.up_proj", + "model.layers.20.mlp.experts.41.down_proj", + "model.layers.20.mlp.experts.41.gate_proj", + "model.layers.20.mlp.experts.41.up_proj", + "model.layers.20.mlp.experts.42.down_proj", + "model.layers.20.mlp.experts.42.gate_proj", + "model.layers.20.mlp.experts.42.up_proj", + "model.layers.20.mlp.experts.43.down_proj", + "model.layers.20.mlp.experts.43.gate_proj", + "model.layers.20.mlp.experts.43.up_proj", + "model.layers.20.mlp.experts.44.down_proj", + "model.layers.20.mlp.experts.44.gate_proj", + "model.layers.20.mlp.experts.44.up_proj", + "model.layers.20.mlp.experts.45.down_proj", + "model.layers.20.mlp.experts.45.gate_proj", + "model.layers.20.mlp.experts.45.up_proj", + "model.layers.20.mlp.experts.46.down_proj", + "model.layers.20.mlp.experts.46.gate_proj", + "model.layers.20.mlp.experts.46.up_proj", + "model.layers.20.mlp.experts.47.down_proj", + "model.layers.20.mlp.experts.47.gate_proj", + "model.layers.20.mlp.experts.47.up_proj", + "model.layers.20.mlp.experts.48.down_proj", + "model.layers.20.mlp.experts.48.gate_proj", + "model.layers.20.mlp.experts.48.up_proj", + "model.layers.20.mlp.experts.49.down_proj", + "model.layers.20.mlp.experts.49.gate_proj", + "model.layers.20.mlp.experts.49.up_proj", + "model.layers.20.mlp.experts.5.down_proj", + "model.layers.20.mlp.experts.5.gate_proj", + "model.layers.20.mlp.experts.5.up_proj", + "model.layers.20.mlp.experts.50.down_proj", + "model.layers.20.mlp.experts.50.gate_proj", + "model.layers.20.mlp.experts.50.up_proj", + "model.layers.20.mlp.experts.51.down_proj", + "model.layers.20.mlp.experts.51.gate_proj", + "model.layers.20.mlp.experts.51.up_proj", + "model.layers.20.mlp.experts.52.down_proj", + "model.layers.20.mlp.experts.52.gate_proj", + "model.layers.20.mlp.experts.52.up_proj", + "model.layers.20.mlp.experts.53.down_proj", + "model.layers.20.mlp.experts.53.gate_proj", + "model.layers.20.mlp.experts.53.up_proj", + "model.layers.20.mlp.experts.54.down_proj", + "model.layers.20.mlp.experts.54.gate_proj", + "model.layers.20.mlp.experts.54.up_proj", + "model.layers.20.mlp.experts.55.down_proj", + "model.layers.20.mlp.experts.55.gate_proj", + "model.layers.20.mlp.experts.55.up_proj", + "model.layers.20.mlp.experts.56.down_proj", + "model.layers.20.mlp.experts.56.gate_proj", + "model.layers.20.mlp.experts.56.up_proj", + "model.layers.20.mlp.experts.57.down_proj", + "model.layers.20.mlp.experts.57.gate_proj", + "model.layers.20.mlp.experts.57.up_proj", + "model.layers.20.mlp.experts.58.down_proj", + "model.layers.20.mlp.experts.58.gate_proj", + "model.layers.20.mlp.experts.58.up_proj", + "model.layers.20.mlp.experts.59.down_proj", + "model.layers.20.mlp.experts.59.gate_proj", + "model.layers.20.mlp.experts.59.up_proj", + "model.layers.20.mlp.experts.6.down_proj", + "model.layers.20.mlp.experts.6.gate_proj", + "model.layers.20.mlp.experts.6.up_proj", + "model.layers.20.mlp.experts.60.down_proj", + "model.layers.20.mlp.experts.60.gate_proj", + "model.layers.20.mlp.experts.60.up_proj", + "model.layers.20.mlp.experts.61.down_proj", + "model.layers.20.mlp.experts.61.gate_proj", + "model.layers.20.mlp.experts.61.up_proj", + "model.layers.20.mlp.experts.62.down_proj", + "model.layers.20.mlp.experts.62.gate_proj", + "model.layers.20.mlp.experts.62.up_proj", + "model.layers.20.mlp.experts.63.down_proj", + "model.layers.20.mlp.experts.63.gate_proj", + "model.layers.20.mlp.experts.63.up_proj", + "model.layers.20.mlp.experts.64.down_proj", + "model.layers.20.mlp.experts.64.gate_proj", + "model.layers.20.mlp.experts.64.up_proj", + "model.layers.20.mlp.experts.65.down_proj", + "model.layers.20.mlp.experts.65.gate_proj", + "model.layers.20.mlp.experts.65.up_proj", + "model.layers.20.mlp.experts.66.down_proj", + "model.layers.20.mlp.experts.66.gate_proj", + "model.layers.20.mlp.experts.66.up_proj", + "model.layers.20.mlp.experts.67.down_proj", + "model.layers.20.mlp.experts.67.gate_proj", + "model.layers.20.mlp.experts.67.up_proj", + "model.layers.20.mlp.experts.68.down_proj", + "model.layers.20.mlp.experts.68.gate_proj", + "model.layers.20.mlp.experts.68.up_proj", + "model.layers.20.mlp.experts.69.down_proj", + "model.layers.20.mlp.experts.69.gate_proj", + "model.layers.20.mlp.experts.69.up_proj", + "model.layers.20.mlp.experts.7.down_proj", + "model.layers.20.mlp.experts.7.gate_proj", + "model.layers.20.mlp.experts.7.up_proj", + "model.layers.20.mlp.experts.70.down_proj", + "model.layers.20.mlp.experts.70.gate_proj", + "model.layers.20.mlp.experts.70.up_proj", + "model.layers.20.mlp.experts.71.down_proj", + "model.layers.20.mlp.experts.71.gate_proj", + "model.layers.20.mlp.experts.71.up_proj", + "model.layers.20.mlp.experts.72.down_proj", + "model.layers.20.mlp.experts.72.gate_proj", + "model.layers.20.mlp.experts.72.up_proj", + "model.layers.20.mlp.experts.73.down_proj", + "model.layers.20.mlp.experts.73.gate_proj", + "model.layers.20.mlp.experts.73.up_proj", + "model.layers.20.mlp.experts.74.down_proj", + "model.layers.20.mlp.experts.74.gate_proj", + "model.layers.20.mlp.experts.74.up_proj", + "model.layers.20.mlp.experts.75.down_proj", + "model.layers.20.mlp.experts.75.gate_proj", + "model.layers.20.mlp.experts.75.up_proj", + "model.layers.20.mlp.experts.76.down_proj", + "model.layers.20.mlp.experts.76.gate_proj", + "model.layers.20.mlp.experts.76.up_proj", + "model.layers.20.mlp.experts.77.down_proj", + "model.layers.20.mlp.experts.77.gate_proj", + "model.layers.20.mlp.experts.77.up_proj", + "model.layers.20.mlp.experts.78.down_proj", + "model.layers.20.mlp.experts.78.gate_proj", + "model.layers.20.mlp.experts.78.up_proj", + "model.layers.20.mlp.experts.79.down_proj", + "model.layers.20.mlp.experts.79.gate_proj", + "model.layers.20.mlp.experts.79.up_proj", + "model.layers.20.mlp.experts.8.down_proj", + "model.layers.20.mlp.experts.8.gate_proj", + "model.layers.20.mlp.experts.8.up_proj", + "model.layers.20.mlp.experts.80.down_proj", + "model.layers.20.mlp.experts.80.gate_proj", + "model.layers.20.mlp.experts.80.up_proj", + "model.layers.20.mlp.experts.81.down_proj", + "model.layers.20.mlp.experts.81.gate_proj", + "model.layers.20.mlp.experts.81.up_proj", + "model.layers.20.mlp.experts.82.down_proj", + "model.layers.20.mlp.experts.82.gate_proj", + "model.layers.20.mlp.experts.82.up_proj", + "model.layers.20.mlp.experts.83.down_proj", + "model.layers.20.mlp.experts.83.gate_proj", + "model.layers.20.mlp.experts.83.up_proj", + "model.layers.20.mlp.experts.84.down_proj", + "model.layers.20.mlp.experts.84.gate_proj", + "model.layers.20.mlp.experts.84.up_proj", + "model.layers.20.mlp.experts.85.down_proj", + "model.layers.20.mlp.experts.85.gate_proj", + "model.layers.20.mlp.experts.85.up_proj", + "model.layers.20.mlp.experts.86.down_proj", + "model.layers.20.mlp.experts.86.gate_proj", + "model.layers.20.mlp.experts.86.up_proj", + "model.layers.20.mlp.experts.87.down_proj", + "model.layers.20.mlp.experts.87.gate_proj", + "model.layers.20.mlp.experts.87.up_proj", + "model.layers.20.mlp.experts.88.down_proj", + "model.layers.20.mlp.experts.88.gate_proj", + "model.layers.20.mlp.experts.88.up_proj", + "model.layers.20.mlp.experts.89.down_proj", + "model.layers.20.mlp.experts.89.gate_proj", + "model.layers.20.mlp.experts.89.up_proj", + "model.layers.20.mlp.experts.9.down_proj", + "model.layers.20.mlp.experts.9.gate_proj", + "model.layers.20.mlp.experts.9.up_proj", + "model.layers.20.mlp.experts.90.down_proj", + "model.layers.20.mlp.experts.90.gate_proj", + "model.layers.20.mlp.experts.90.up_proj", + "model.layers.20.mlp.experts.91.down_proj", + "model.layers.20.mlp.experts.91.gate_proj", + "model.layers.20.mlp.experts.91.up_proj", + "model.layers.20.mlp.experts.92.down_proj", + "model.layers.20.mlp.experts.92.gate_proj", + "model.layers.20.mlp.experts.92.up_proj", + "model.layers.20.mlp.experts.93.down_proj", + "model.layers.20.mlp.experts.93.gate_proj", + "model.layers.20.mlp.experts.93.up_proj", + "model.layers.20.mlp.experts.94.down_proj", + "model.layers.20.mlp.experts.94.gate_proj", + "model.layers.20.mlp.experts.94.up_proj", + "model.layers.20.mlp.experts.95.down_proj", + "model.layers.20.mlp.experts.95.gate_proj", + "model.layers.20.mlp.experts.95.up_proj", + "model.layers.20.mlp.experts.96.down_proj", + "model.layers.20.mlp.experts.96.gate_proj", + "model.layers.20.mlp.experts.96.up_proj", + "model.layers.20.mlp.experts.97.down_proj", + "model.layers.20.mlp.experts.97.gate_proj", + "model.layers.20.mlp.experts.97.up_proj", + "model.layers.20.mlp.experts.98.down_proj", + "model.layers.20.mlp.experts.98.gate_proj", + "model.layers.20.mlp.experts.98.up_proj", + "model.layers.20.mlp.experts.99.down_proj", + "model.layers.20.mlp.experts.99.gate_proj", + "model.layers.20.mlp.experts.99.up_proj", + "model.layers.21.mlp.experts.0.down_proj", + "model.layers.21.mlp.experts.0.gate_proj", + "model.layers.21.mlp.experts.0.up_proj", + "model.layers.21.mlp.experts.1.down_proj", + "model.layers.21.mlp.experts.1.gate_proj", + "model.layers.21.mlp.experts.1.up_proj", + "model.layers.21.mlp.experts.10.down_proj", + "model.layers.21.mlp.experts.10.gate_proj", + "model.layers.21.mlp.experts.10.up_proj", + "model.layers.21.mlp.experts.100.down_proj", + "model.layers.21.mlp.experts.100.gate_proj", + "model.layers.21.mlp.experts.100.up_proj", + "model.layers.21.mlp.experts.101.down_proj", + "model.layers.21.mlp.experts.101.gate_proj", + "model.layers.21.mlp.experts.101.up_proj", + "model.layers.21.mlp.experts.102.down_proj", + "model.layers.21.mlp.experts.102.gate_proj", + "model.layers.21.mlp.experts.102.up_proj", + "model.layers.21.mlp.experts.103.down_proj", + "model.layers.21.mlp.experts.103.gate_proj", + "model.layers.21.mlp.experts.103.up_proj", + "model.layers.21.mlp.experts.104.down_proj", + "model.layers.21.mlp.experts.104.gate_proj", + "model.layers.21.mlp.experts.104.up_proj", + "model.layers.21.mlp.experts.105.down_proj", + "model.layers.21.mlp.experts.105.gate_proj", + "model.layers.21.mlp.experts.105.up_proj", + "model.layers.21.mlp.experts.106.down_proj", + "model.layers.21.mlp.experts.106.gate_proj", + "model.layers.21.mlp.experts.106.up_proj", + "model.layers.21.mlp.experts.107.down_proj", + "model.layers.21.mlp.experts.107.gate_proj", + "model.layers.21.mlp.experts.107.up_proj", + "model.layers.21.mlp.experts.108.down_proj", + "model.layers.21.mlp.experts.108.gate_proj", + "model.layers.21.mlp.experts.108.up_proj", + "model.layers.21.mlp.experts.109.down_proj", + "model.layers.21.mlp.experts.109.gate_proj", + "model.layers.21.mlp.experts.109.up_proj", + "model.layers.21.mlp.experts.11.down_proj", + "model.layers.21.mlp.experts.11.gate_proj", + "model.layers.21.mlp.experts.11.up_proj", + "model.layers.21.mlp.experts.110.down_proj", + "model.layers.21.mlp.experts.110.gate_proj", + "model.layers.21.mlp.experts.110.up_proj", + "model.layers.21.mlp.experts.111.down_proj", + "model.layers.21.mlp.experts.111.gate_proj", + "model.layers.21.mlp.experts.111.up_proj", + "model.layers.21.mlp.experts.112.down_proj", + "model.layers.21.mlp.experts.112.gate_proj", + "model.layers.21.mlp.experts.112.up_proj", + "model.layers.21.mlp.experts.113.down_proj", + "model.layers.21.mlp.experts.113.gate_proj", + "model.layers.21.mlp.experts.113.up_proj", + "model.layers.21.mlp.experts.114.down_proj", + "model.layers.21.mlp.experts.114.gate_proj", + "model.layers.21.mlp.experts.114.up_proj", + "model.layers.21.mlp.experts.115.down_proj", + "model.layers.21.mlp.experts.115.gate_proj", + "model.layers.21.mlp.experts.115.up_proj", + "model.layers.21.mlp.experts.116.down_proj", + "model.layers.21.mlp.experts.116.gate_proj", + "model.layers.21.mlp.experts.116.up_proj", + "model.layers.21.mlp.experts.117.down_proj", + "model.layers.21.mlp.experts.117.gate_proj", + "model.layers.21.mlp.experts.117.up_proj", + "model.layers.21.mlp.experts.118.down_proj", + "model.layers.21.mlp.experts.118.gate_proj", + "model.layers.21.mlp.experts.118.up_proj", + "model.layers.21.mlp.experts.119.down_proj", + "model.layers.21.mlp.experts.119.gate_proj", + "model.layers.21.mlp.experts.119.up_proj", + "model.layers.21.mlp.experts.12.down_proj", + "model.layers.21.mlp.experts.12.gate_proj", + "model.layers.21.mlp.experts.12.up_proj", + "model.layers.21.mlp.experts.120.down_proj", + "model.layers.21.mlp.experts.120.gate_proj", + "model.layers.21.mlp.experts.120.up_proj", + "model.layers.21.mlp.experts.121.down_proj", + "model.layers.21.mlp.experts.121.gate_proj", + "model.layers.21.mlp.experts.121.up_proj", + "model.layers.21.mlp.experts.122.down_proj", + "model.layers.21.mlp.experts.122.gate_proj", + "model.layers.21.mlp.experts.122.up_proj", + "model.layers.21.mlp.experts.123.down_proj", + "model.layers.21.mlp.experts.123.gate_proj", + "model.layers.21.mlp.experts.123.up_proj", + "model.layers.21.mlp.experts.124.down_proj", + "model.layers.21.mlp.experts.124.gate_proj", + "model.layers.21.mlp.experts.124.up_proj", + "model.layers.21.mlp.experts.125.down_proj", + "model.layers.21.mlp.experts.125.gate_proj", + "model.layers.21.mlp.experts.125.up_proj", + "model.layers.21.mlp.experts.126.down_proj", + "model.layers.21.mlp.experts.126.gate_proj", + "model.layers.21.mlp.experts.126.up_proj", + "model.layers.21.mlp.experts.127.down_proj", + "model.layers.21.mlp.experts.127.gate_proj", + "model.layers.21.mlp.experts.127.up_proj", + "model.layers.21.mlp.experts.128.down_proj", + "model.layers.21.mlp.experts.128.gate_proj", + "model.layers.21.mlp.experts.128.up_proj", + "model.layers.21.mlp.experts.129.down_proj", + "model.layers.21.mlp.experts.129.gate_proj", + "model.layers.21.mlp.experts.129.up_proj", + "model.layers.21.mlp.experts.13.down_proj", + "model.layers.21.mlp.experts.13.gate_proj", + "model.layers.21.mlp.experts.13.up_proj", + "model.layers.21.mlp.experts.130.down_proj", + "model.layers.21.mlp.experts.130.gate_proj", + "model.layers.21.mlp.experts.130.up_proj", + "model.layers.21.mlp.experts.131.down_proj", + "model.layers.21.mlp.experts.131.gate_proj", + "model.layers.21.mlp.experts.131.up_proj", + "model.layers.21.mlp.experts.132.down_proj", + "model.layers.21.mlp.experts.132.gate_proj", + "model.layers.21.mlp.experts.132.up_proj", + "model.layers.21.mlp.experts.133.down_proj", + "model.layers.21.mlp.experts.133.gate_proj", + "model.layers.21.mlp.experts.133.up_proj", + "model.layers.21.mlp.experts.134.down_proj", + "model.layers.21.mlp.experts.134.gate_proj", + "model.layers.21.mlp.experts.134.up_proj", + "model.layers.21.mlp.experts.135.down_proj", + "model.layers.21.mlp.experts.135.gate_proj", + "model.layers.21.mlp.experts.135.up_proj", + "model.layers.21.mlp.experts.136.down_proj", + "model.layers.21.mlp.experts.136.gate_proj", + "model.layers.21.mlp.experts.136.up_proj", + "model.layers.21.mlp.experts.137.down_proj", + "model.layers.21.mlp.experts.137.gate_proj", + "model.layers.21.mlp.experts.137.up_proj", + "model.layers.21.mlp.experts.138.down_proj", + "model.layers.21.mlp.experts.138.gate_proj", + "model.layers.21.mlp.experts.138.up_proj", + "model.layers.21.mlp.experts.139.down_proj", + "model.layers.21.mlp.experts.139.gate_proj", + "model.layers.21.mlp.experts.139.up_proj", + "model.layers.21.mlp.experts.14.down_proj", + "model.layers.21.mlp.experts.14.gate_proj", + "model.layers.21.mlp.experts.14.up_proj", + "model.layers.21.mlp.experts.140.down_proj", + "model.layers.21.mlp.experts.140.gate_proj", + "model.layers.21.mlp.experts.140.up_proj", + "model.layers.21.mlp.experts.141.down_proj", + "model.layers.21.mlp.experts.141.gate_proj", + "model.layers.21.mlp.experts.141.up_proj", + "model.layers.21.mlp.experts.142.down_proj", + "model.layers.21.mlp.experts.142.gate_proj", + "model.layers.21.mlp.experts.142.up_proj", + "model.layers.21.mlp.experts.143.down_proj", + "model.layers.21.mlp.experts.143.gate_proj", + "model.layers.21.mlp.experts.143.up_proj", + "model.layers.21.mlp.experts.144.down_proj", + "model.layers.21.mlp.experts.144.gate_proj", + "model.layers.21.mlp.experts.144.up_proj", + "model.layers.21.mlp.experts.145.down_proj", + "model.layers.21.mlp.experts.145.gate_proj", + "model.layers.21.mlp.experts.145.up_proj", + "model.layers.21.mlp.experts.146.down_proj", + "model.layers.21.mlp.experts.146.gate_proj", + "model.layers.21.mlp.experts.146.up_proj", + "model.layers.21.mlp.experts.147.down_proj", + "model.layers.21.mlp.experts.147.gate_proj", + "model.layers.21.mlp.experts.147.up_proj", + "model.layers.21.mlp.experts.148.down_proj", + "model.layers.21.mlp.experts.148.gate_proj", + "model.layers.21.mlp.experts.148.up_proj", + "model.layers.21.mlp.experts.149.down_proj", + "model.layers.21.mlp.experts.149.gate_proj", + "model.layers.21.mlp.experts.149.up_proj", + "model.layers.21.mlp.experts.15.down_proj", + "model.layers.21.mlp.experts.15.gate_proj", + "model.layers.21.mlp.experts.15.up_proj", + "model.layers.21.mlp.experts.150.down_proj", + "model.layers.21.mlp.experts.150.gate_proj", + "model.layers.21.mlp.experts.150.up_proj", + "model.layers.21.mlp.experts.151.down_proj", + "model.layers.21.mlp.experts.151.gate_proj", + "model.layers.21.mlp.experts.151.up_proj", + "model.layers.21.mlp.experts.152.down_proj", + "model.layers.21.mlp.experts.152.gate_proj", + "model.layers.21.mlp.experts.152.up_proj", + "model.layers.21.mlp.experts.153.down_proj", + "model.layers.21.mlp.experts.153.gate_proj", + "model.layers.21.mlp.experts.153.up_proj", + "model.layers.21.mlp.experts.154.down_proj", + "model.layers.21.mlp.experts.154.gate_proj", + "model.layers.21.mlp.experts.154.up_proj", + "model.layers.21.mlp.experts.155.down_proj", + "model.layers.21.mlp.experts.155.gate_proj", + "model.layers.21.mlp.experts.155.up_proj", + "model.layers.21.mlp.experts.156.down_proj", + "model.layers.21.mlp.experts.156.gate_proj", + "model.layers.21.mlp.experts.156.up_proj", + "model.layers.21.mlp.experts.157.down_proj", + "model.layers.21.mlp.experts.157.gate_proj", + "model.layers.21.mlp.experts.157.up_proj", + "model.layers.21.mlp.experts.158.down_proj", + "model.layers.21.mlp.experts.158.gate_proj", + "model.layers.21.mlp.experts.158.up_proj", + "model.layers.21.mlp.experts.159.down_proj", + "model.layers.21.mlp.experts.159.gate_proj", + "model.layers.21.mlp.experts.159.up_proj", + "model.layers.21.mlp.experts.16.down_proj", + "model.layers.21.mlp.experts.16.gate_proj", + "model.layers.21.mlp.experts.16.up_proj", + "model.layers.21.mlp.experts.160.down_proj", + "model.layers.21.mlp.experts.160.gate_proj", + "model.layers.21.mlp.experts.160.up_proj", + "model.layers.21.mlp.experts.161.down_proj", + "model.layers.21.mlp.experts.161.gate_proj", + "model.layers.21.mlp.experts.161.up_proj", + "model.layers.21.mlp.experts.162.down_proj", + "model.layers.21.mlp.experts.162.gate_proj", + "model.layers.21.mlp.experts.162.up_proj", + "model.layers.21.mlp.experts.163.down_proj", + "model.layers.21.mlp.experts.163.gate_proj", + "model.layers.21.mlp.experts.163.up_proj", + "model.layers.21.mlp.experts.164.down_proj", + "model.layers.21.mlp.experts.164.gate_proj", + "model.layers.21.mlp.experts.164.up_proj", + "model.layers.21.mlp.experts.165.down_proj", + "model.layers.21.mlp.experts.165.gate_proj", + "model.layers.21.mlp.experts.165.up_proj", + "model.layers.21.mlp.experts.166.down_proj", + "model.layers.21.mlp.experts.166.gate_proj", + "model.layers.21.mlp.experts.166.up_proj", + "model.layers.21.mlp.experts.167.down_proj", + "model.layers.21.mlp.experts.167.gate_proj", + "model.layers.21.mlp.experts.167.up_proj", + "model.layers.21.mlp.experts.168.down_proj", + "model.layers.21.mlp.experts.168.gate_proj", + "model.layers.21.mlp.experts.168.up_proj", + "model.layers.21.mlp.experts.169.down_proj", + "model.layers.21.mlp.experts.169.gate_proj", + "model.layers.21.mlp.experts.169.up_proj", + "model.layers.21.mlp.experts.17.down_proj", + "model.layers.21.mlp.experts.17.gate_proj", + "model.layers.21.mlp.experts.17.up_proj", + "model.layers.21.mlp.experts.170.down_proj", + "model.layers.21.mlp.experts.170.gate_proj", + "model.layers.21.mlp.experts.170.up_proj", + "model.layers.21.mlp.experts.171.down_proj", + "model.layers.21.mlp.experts.171.gate_proj", + "model.layers.21.mlp.experts.171.up_proj", + "model.layers.21.mlp.experts.172.down_proj", + "model.layers.21.mlp.experts.172.gate_proj", + "model.layers.21.mlp.experts.172.up_proj", + "model.layers.21.mlp.experts.173.down_proj", + "model.layers.21.mlp.experts.173.gate_proj", + "model.layers.21.mlp.experts.173.up_proj", + "model.layers.21.mlp.experts.174.down_proj", + "model.layers.21.mlp.experts.174.gate_proj", + "model.layers.21.mlp.experts.174.up_proj", + "model.layers.21.mlp.experts.175.down_proj", + "model.layers.21.mlp.experts.175.gate_proj", + "model.layers.21.mlp.experts.175.up_proj", + "model.layers.21.mlp.experts.176.down_proj", + "model.layers.21.mlp.experts.176.gate_proj", + "model.layers.21.mlp.experts.176.up_proj", + "model.layers.21.mlp.experts.177.down_proj", + "model.layers.21.mlp.experts.177.gate_proj", + "model.layers.21.mlp.experts.177.up_proj", + "model.layers.21.mlp.experts.178.down_proj", + "model.layers.21.mlp.experts.178.gate_proj", + "model.layers.21.mlp.experts.178.up_proj", + "model.layers.21.mlp.experts.179.down_proj", + "model.layers.21.mlp.experts.179.gate_proj", + "model.layers.21.mlp.experts.179.up_proj", + "model.layers.21.mlp.experts.18.down_proj", + "model.layers.21.mlp.experts.18.gate_proj", + "model.layers.21.mlp.experts.18.up_proj", + "model.layers.21.mlp.experts.180.down_proj", + "model.layers.21.mlp.experts.180.gate_proj", + "model.layers.21.mlp.experts.180.up_proj", + "model.layers.21.mlp.experts.181.down_proj", + "model.layers.21.mlp.experts.181.gate_proj", + "model.layers.21.mlp.experts.181.up_proj", + "model.layers.21.mlp.experts.182.down_proj", + "model.layers.21.mlp.experts.182.gate_proj", + "model.layers.21.mlp.experts.182.up_proj", + "model.layers.21.mlp.experts.183.down_proj", + "model.layers.21.mlp.experts.183.gate_proj", + "model.layers.21.mlp.experts.183.up_proj", + "model.layers.21.mlp.experts.184.down_proj", + "model.layers.21.mlp.experts.184.gate_proj", + "model.layers.21.mlp.experts.184.up_proj", + "model.layers.21.mlp.experts.185.down_proj", + "model.layers.21.mlp.experts.185.gate_proj", + "model.layers.21.mlp.experts.185.up_proj", + "model.layers.21.mlp.experts.186.down_proj", + "model.layers.21.mlp.experts.186.gate_proj", + "model.layers.21.mlp.experts.186.up_proj", + "model.layers.21.mlp.experts.187.down_proj", + "model.layers.21.mlp.experts.187.gate_proj", + "model.layers.21.mlp.experts.187.up_proj", + "model.layers.21.mlp.experts.188.down_proj", + "model.layers.21.mlp.experts.188.gate_proj", + "model.layers.21.mlp.experts.188.up_proj", + "model.layers.21.mlp.experts.189.down_proj", + "model.layers.21.mlp.experts.189.gate_proj", + "model.layers.21.mlp.experts.189.up_proj", + "model.layers.21.mlp.experts.19.down_proj", + "model.layers.21.mlp.experts.19.gate_proj", + "model.layers.21.mlp.experts.19.up_proj", + "model.layers.21.mlp.experts.190.down_proj", + "model.layers.21.mlp.experts.190.gate_proj", + "model.layers.21.mlp.experts.190.up_proj", + "model.layers.21.mlp.experts.191.down_proj", + "model.layers.21.mlp.experts.191.gate_proj", + "model.layers.21.mlp.experts.191.up_proj", + "model.layers.21.mlp.experts.192.down_proj", + "model.layers.21.mlp.experts.192.gate_proj", + "model.layers.21.mlp.experts.192.up_proj", + "model.layers.21.mlp.experts.193.down_proj", + "model.layers.21.mlp.experts.193.gate_proj", + "model.layers.21.mlp.experts.193.up_proj", + "model.layers.21.mlp.experts.194.down_proj", + "model.layers.21.mlp.experts.194.gate_proj", + "model.layers.21.mlp.experts.194.up_proj", + "model.layers.21.mlp.experts.195.down_proj", + "model.layers.21.mlp.experts.195.gate_proj", + "model.layers.21.mlp.experts.195.up_proj", + "model.layers.21.mlp.experts.196.down_proj", + "model.layers.21.mlp.experts.196.gate_proj", + "model.layers.21.mlp.experts.196.up_proj", + "model.layers.21.mlp.experts.197.down_proj", + "model.layers.21.mlp.experts.197.gate_proj", + "model.layers.21.mlp.experts.197.up_proj", + "model.layers.21.mlp.experts.198.down_proj", + "model.layers.21.mlp.experts.198.gate_proj", + "model.layers.21.mlp.experts.198.up_proj", + "model.layers.21.mlp.experts.199.down_proj", + "model.layers.21.mlp.experts.199.gate_proj", + "model.layers.21.mlp.experts.199.up_proj", + "model.layers.21.mlp.experts.2.down_proj", + "model.layers.21.mlp.experts.2.gate_proj", + "model.layers.21.mlp.experts.2.up_proj", + "model.layers.21.mlp.experts.20.down_proj", + "model.layers.21.mlp.experts.20.gate_proj", + "model.layers.21.mlp.experts.20.up_proj", + "model.layers.21.mlp.experts.200.down_proj", + "model.layers.21.mlp.experts.200.gate_proj", + "model.layers.21.mlp.experts.200.up_proj", + "model.layers.21.mlp.experts.201.down_proj", + "model.layers.21.mlp.experts.201.gate_proj", + "model.layers.21.mlp.experts.201.up_proj", + "model.layers.21.mlp.experts.202.down_proj", + "model.layers.21.mlp.experts.202.gate_proj", + "model.layers.21.mlp.experts.202.up_proj", + "model.layers.21.mlp.experts.203.down_proj", + "model.layers.21.mlp.experts.203.gate_proj", + "model.layers.21.mlp.experts.203.up_proj", + "model.layers.21.mlp.experts.204.down_proj", + "model.layers.21.mlp.experts.204.gate_proj", + "model.layers.21.mlp.experts.204.up_proj", + "model.layers.21.mlp.experts.205.down_proj", + "model.layers.21.mlp.experts.205.gate_proj", + "model.layers.21.mlp.experts.205.up_proj", + "model.layers.21.mlp.experts.206.down_proj", + "model.layers.21.mlp.experts.206.gate_proj", + "model.layers.21.mlp.experts.206.up_proj", + "model.layers.21.mlp.experts.207.down_proj", + "model.layers.21.mlp.experts.207.gate_proj", + "model.layers.21.mlp.experts.207.up_proj", + "model.layers.21.mlp.experts.208.down_proj", + "model.layers.21.mlp.experts.208.gate_proj", + "model.layers.21.mlp.experts.208.up_proj", + "model.layers.21.mlp.experts.209.down_proj", + "model.layers.21.mlp.experts.209.gate_proj", + "model.layers.21.mlp.experts.209.up_proj", + "model.layers.21.mlp.experts.21.down_proj", + "model.layers.21.mlp.experts.21.gate_proj", + "model.layers.21.mlp.experts.21.up_proj", + "model.layers.21.mlp.experts.210.down_proj", + "model.layers.21.mlp.experts.210.gate_proj", + "model.layers.21.mlp.experts.210.up_proj", + "model.layers.21.mlp.experts.211.down_proj", + "model.layers.21.mlp.experts.211.gate_proj", + "model.layers.21.mlp.experts.211.up_proj", + "model.layers.21.mlp.experts.212.down_proj", + "model.layers.21.mlp.experts.212.gate_proj", + "model.layers.21.mlp.experts.212.up_proj", + "model.layers.21.mlp.experts.213.down_proj", + "model.layers.21.mlp.experts.213.gate_proj", + "model.layers.21.mlp.experts.213.up_proj", + "model.layers.21.mlp.experts.214.down_proj", + "model.layers.21.mlp.experts.214.gate_proj", + "model.layers.21.mlp.experts.214.up_proj", + "model.layers.21.mlp.experts.215.down_proj", + "model.layers.21.mlp.experts.215.gate_proj", + "model.layers.21.mlp.experts.215.up_proj", + "model.layers.21.mlp.experts.216.down_proj", + "model.layers.21.mlp.experts.216.gate_proj", + "model.layers.21.mlp.experts.216.up_proj", + "model.layers.21.mlp.experts.217.down_proj", + "model.layers.21.mlp.experts.217.gate_proj", + "model.layers.21.mlp.experts.217.up_proj", + "model.layers.21.mlp.experts.218.down_proj", + "model.layers.21.mlp.experts.218.gate_proj", + "model.layers.21.mlp.experts.218.up_proj", + "model.layers.21.mlp.experts.219.down_proj", + "model.layers.21.mlp.experts.219.gate_proj", + "model.layers.21.mlp.experts.219.up_proj", + "model.layers.21.mlp.experts.22.down_proj", + "model.layers.21.mlp.experts.22.gate_proj", + "model.layers.21.mlp.experts.22.up_proj", + "model.layers.21.mlp.experts.220.down_proj", + "model.layers.21.mlp.experts.220.gate_proj", + "model.layers.21.mlp.experts.220.up_proj", + "model.layers.21.mlp.experts.221.down_proj", + "model.layers.21.mlp.experts.221.gate_proj", + "model.layers.21.mlp.experts.221.up_proj", + "model.layers.21.mlp.experts.222.down_proj", + "model.layers.21.mlp.experts.222.gate_proj", + "model.layers.21.mlp.experts.222.up_proj", + "model.layers.21.mlp.experts.223.down_proj", + "model.layers.21.mlp.experts.223.gate_proj", + "model.layers.21.mlp.experts.223.up_proj", + "model.layers.21.mlp.experts.224.down_proj", + "model.layers.21.mlp.experts.224.gate_proj", + "model.layers.21.mlp.experts.224.up_proj", + "model.layers.21.mlp.experts.225.down_proj", + "model.layers.21.mlp.experts.225.gate_proj", + "model.layers.21.mlp.experts.225.up_proj", + "model.layers.21.mlp.experts.226.down_proj", + "model.layers.21.mlp.experts.226.gate_proj", + "model.layers.21.mlp.experts.226.up_proj", + "model.layers.21.mlp.experts.227.down_proj", + "model.layers.21.mlp.experts.227.gate_proj", + "model.layers.21.mlp.experts.227.up_proj", + "model.layers.21.mlp.experts.228.down_proj", + "model.layers.21.mlp.experts.228.gate_proj", + "model.layers.21.mlp.experts.228.up_proj", + "model.layers.21.mlp.experts.229.down_proj", + "model.layers.21.mlp.experts.229.gate_proj", + "model.layers.21.mlp.experts.229.up_proj", + "model.layers.21.mlp.experts.23.down_proj", + "model.layers.21.mlp.experts.23.gate_proj", + "model.layers.21.mlp.experts.23.up_proj", + "model.layers.21.mlp.experts.230.down_proj", + "model.layers.21.mlp.experts.230.gate_proj", + "model.layers.21.mlp.experts.230.up_proj", + "model.layers.21.mlp.experts.231.down_proj", + "model.layers.21.mlp.experts.231.gate_proj", + "model.layers.21.mlp.experts.231.up_proj", + "model.layers.21.mlp.experts.232.down_proj", + "model.layers.21.mlp.experts.232.gate_proj", + "model.layers.21.mlp.experts.232.up_proj", + "model.layers.21.mlp.experts.233.down_proj", + "model.layers.21.mlp.experts.233.gate_proj", + "model.layers.21.mlp.experts.233.up_proj", + "model.layers.21.mlp.experts.234.down_proj", + "model.layers.21.mlp.experts.234.gate_proj", + "model.layers.21.mlp.experts.234.up_proj", + "model.layers.21.mlp.experts.235.down_proj", + "model.layers.21.mlp.experts.235.gate_proj", + "model.layers.21.mlp.experts.235.up_proj", + "model.layers.21.mlp.experts.236.down_proj", + "model.layers.21.mlp.experts.236.gate_proj", + "model.layers.21.mlp.experts.236.up_proj", + "model.layers.21.mlp.experts.237.down_proj", + "model.layers.21.mlp.experts.237.gate_proj", + "model.layers.21.mlp.experts.237.up_proj", + "model.layers.21.mlp.experts.238.down_proj", + "model.layers.21.mlp.experts.238.gate_proj", + "model.layers.21.mlp.experts.238.up_proj", + "model.layers.21.mlp.experts.239.down_proj", + "model.layers.21.mlp.experts.239.gate_proj", + "model.layers.21.mlp.experts.239.up_proj", + "model.layers.21.mlp.experts.24.down_proj", + "model.layers.21.mlp.experts.24.gate_proj", + "model.layers.21.mlp.experts.24.up_proj", + "model.layers.21.mlp.experts.240.down_proj", + "model.layers.21.mlp.experts.240.gate_proj", + "model.layers.21.mlp.experts.240.up_proj", + "model.layers.21.mlp.experts.241.down_proj", + "model.layers.21.mlp.experts.241.gate_proj", + "model.layers.21.mlp.experts.241.up_proj", + "model.layers.21.mlp.experts.242.down_proj", + "model.layers.21.mlp.experts.242.gate_proj", + "model.layers.21.mlp.experts.242.up_proj", + "model.layers.21.mlp.experts.243.down_proj", + "model.layers.21.mlp.experts.243.gate_proj", + "model.layers.21.mlp.experts.243.up_proj", + "model.layers.21.mlp.experts.244.down_proj", + "model.layers.21.mlp.experts.244.gate_proj", + "model.layers.21.mlp.experts.244.up_proj", + "model.layers.21.mlp.experts.245.down_proj", + "model.layers.21.mlp.experts.245.gate_proj", + "model.layers.21.mlp.experts.245.up_proj", + "model.layers.21.mlp.experts.246.down_proj", + "model.layers.21.mlp.experts.246.gate_proj", + "model.layers.21.mlp.experts.246.up_proj", + "model.layers.21.mlp.experts.247.down_proj", + "model.layers.21.mlp.experts.247.gate_proj", + "model.layers.21.mlp.experts.247.up_proj", + "model.layers.21.mlp.experts.248.down_proj", + "model.layers.21.mlp.experts.248.gate_proj", + "model.layers.21.mlp.experts.248.up_proj", + "model.layers.21.mlp.experts.249.down_proj", + "model.layers.21.mlp.experts.249.gate_proj", + "model.layers.21.mlp.experts.249.up_proj", + "model.layers.21.mlp.experts.25.down_proj", + "model.layers.21.mlp.experts.25.gate_proj", + "model.layers.21.mlp.experts.25.up_proj", + "model.layers.21.mlp.experts.250.down_proj", + "model.layers.21.mlp.experts.250.gate_proj", + "model.layers.21.mlp.experts.250.up_proj", + "model.layers.21.mlp.experts.251.down_proj", + "model.layers.21.mlp.experts.251.gate_proj", + "model.layers.21.mlp.experts.251.up_proj", + "model.layers.21.mlp.experts.252.down_proj", + "model.layers.21.mlp.experts.252.gate_proj", + "model.layers.21.mlp.experts.252.up_proj", + "model.layers.21.mlp.experts.253.down_proj", + "model.layers.21.mlp.experts.253.gate_proj", + "model.layers.21.mlp.experts.253.up_proj", + "model.layers.21.mlp.experts.254.down_proj", + "model.layers.21.mlp.experts.254.gate_proj", + "model.layers.21.mlp.experts.254.up_proj", + "model.layers.21.mlp.experts.255.down_proj", + "model.layers.21.mlp.experts.255.gate_proj", + "model.layers.21.mlp.experts.255.up_proj", + "model.layers.21.mlp.experts.26.down_proj", + "model.layers.21.mlp.experts.26.gate_proj", + "model.layers.21.mlp.experts.26.up_proj", + "model.layers.21.mlp.experts.27.down_proj", + "model.layers.21.mlp.experts.27.gate_proj", + "model.layers.21.mlp.experts.27.up_proj", + "model.layers.21.mlp.experts.28.down_proj", + "model.layers.21.mlp.experts.28.gate_proj", + "model.layers.21.mlp.experts.28.up_proj", + "model.layers.21.mlp.experts.29.down_proj", + "model.layers.21.mlp.experts.29.gate_proj", + "model.layers.21.mlp.experts.29.up_proj", + "model.layers.21.mlp.experts.3.down_proj", + "model.layers.21.mlp.experts.3.gate_proj", + "model.layers.21.mlp.experts.3.up_proj", + "model.layers.21.mlp.experts.30.down_proj", + "model.layers.21.mlp.experts.30.gate_proj", + "model.layers.21.mlp.experts.30.up_proj", + "model.layers.21.mlp.experts.31.down_proj", + "model.layers.21.mlp.experts.31.gate_proj", + "model.layers.21.mlp.experts.31.up_proj", + "model.layers.21.mlp.experts.32.down_proj", + "model.layers.21.mlp.experts.32.gate_proj", + "model.layers.21.mlp.experts.32.up_proj", + "model.layers.21.mlp.experts.33.down_proj", + "model.layers.21.mlp.experts.33.gate_proj", + "model.layers.21.mlp.experts.33.up_proj", + "model.layers.21.mlp.experts.34.down_proj", + "model.layers.21.mlp.experts.34.gate_proj", + "model.layers.21.mlp.experts.34.up_proj", + "model.layers.21.mlp.experts.35.down_proj", + "model.layers.21.mlp.experts.35.gate_proj", + "model.layers.21.mlp.experts.35.up_proj", + "model.layers.21.mlp.experts.36.down_proj", + "model.layers.21.mlp.experts.36.gate_proj", + "model.layers.21.mlp.experts.36.up_proj", + "model.layers.21.mlp.experts.37.down_proj", + "model.layers.21.mlp.experts.37.gate_proj", + "model.layers.21.mlp.experts.37.up_proj", + "model.layers.21.mlp.experts.38.down_proj", + "model.layers.21.mlp.experts.38.gate_proj", + "model.layers.21.mlp.experts.38.up_proj", + "model.layers.21.mlp.experts.39.down_proj", + "model.layers.21.mlp.experts.39.gate_proj", + "model.layers.21.mlp.experts.39.up_proj", + "model.layers.21.mlp.experts.4.down_proj", + "model.layers.21.mlp.experts.4.gate_proj", + "model.layers.21.mlp.experts.4.up_proj", + "model.layers.21.mlp.experts.40.down_proj", + "model.layers.21.mlp.experts.40.gate_proj", + "model.layers.21.mlp.experts.40.up_proj", + "model.layers.21.mlp.experts.41.down_proj", + "model.layers.21.mlp.experts.41.gate_proj", + "model.layers.21.mlp.experts.41.up_proj", + "model.layers.21.mlp.experts.42.down_proj", + "model.layers.21.mlp.experts.42.gate_proj", + "model.layers.21.mlp.experts.42.up_proj", + "model.layers.21.mlp.experts.43.down_proj", + "model.layers.21.mlp.experts.43.gate_proj", + "model.layers.21.mlp.experts.43.up_proj", + "model.layers.21.mlp.experts.44.down_proj", + "model.layers.21.mlp.experts.44.gate_proj", + "model.layers.21.mlp.experts.44.up_proj", + "model.layers.21.mlp.experts.45.down_proj", + "model.layers.21.mlp.experts.45.gate_proj", + "model.layers.21.mlp.experts.45.up_proj", + "model.layers.21.mlp.experts.46.down_proj", + "model.layers.21.mlp.experts.46.gate_proj", + "model.layers.21.mlp.experts.46.up_proj", + "model.layers.21.mlp.experts.47.down_proj", + "model.layers.21.mlp.experts.47.gate_proj", + "model.layers.21.mlp.experts.47.up_proj", + "model.layers.21.mlp.experts.48.down_proj", + "model.layers.21.mlp.experts.48.gate_proj", + "model.layers.21.mlp.experts.48.up_proj", + "model.layers.21.mlp.experts.49.down_proj", + "model.layers.21.mlp.experts.49.gate_proj", + "model.layers.21.mlp.experts.49.up_proj", + "model.layers.21.mlp.experts.5.down_proj", + "model.layers.21.mlp.experts.5.gate_proj", + "model.layers.21.mlp.experts.5.up_proj", + "model.layers.21.mlp.experts.50.down_proj", + "model.layers.21.mlp.experts.50.gate_proj", + "model.layers.21.mlp.experts.50.up_proj", + "model.layers.21.mlp.experts.51.down_proj", + "model.layers.21.mlp.experts.51.gate_proj", + "model.layers.21.mlp.experts.51.up_proj", + "model.layers.21.mlp.experts.52.down_proj", + "model.layers.21.mlp.experts.52.gate_proj", + "model.layers.21.mlp.experts.52.up_proj", + "model.layers.21.mlp.experts.53.down_proj", + "model.layers.21.mlp.experts.53.gate_proj", + "model.layers.21.mlp.experts.53.up_proj", + "model.layers.21.mlp.experts.54.down_proj", + "model.layers.21.mlp.experts.54.gate_proj", + "model.layers.21.mlp.experts.54.up_proj", + "model.layers.21.mlp.experts.55.down_proj", + "model.layers.21.mlp.experts.55.gate_proj", + "model.layers.21.mlp.experts.55.up_proj", + "model.layers.21.mlp.experts.56.down_proj", + "model.layers.21.mlp.experts.56.gate_proj", + "model.layers.21.mlp.experts.56.up_proj", + "model.layers.21.mlp.experts.57.down_proj", + "model.layers.21.mlp.experts.57.gate_proj", + "model.layers.21.mlp.experts.57.up_proj", + "model.layers.21.mlp.experts.58.down_proj", + "model.layers.21.mlp.experts.58.gate_proj", + "model.layers.21.mlp.experts.58.up_proj", + "model.layers.21.mlp.experts.59.down_proj", + "model.layers.21.mlp.experts.59.gate_proj", + "model.layers.21.mlp.experts.59.up_proj", + "model.layers.21.mlp.experts.6.down_proj", + "model.layers.21.mlp.experts.6.gate_proj", + "model.layers.21.mlp.experts.6.up_proj", + "model.layers.21.mlp.experts.60.down_proj", + "model.layers.21.mlp.experts.60.gate_proj", + "model.layers.21.mlp.experts.60.up_proj", + "model.layers.21.mlp.experts.61.down_proj", + "model.layers.21.mlp.experts.61.gate_proj", + "model.layers.21.mlp.experts.61.up_proj", + "model.layers.21.mlp.experts.62.down_proj", + "model.layers.21.mlp.experts.62.gate_proj", + "model.layers.21.mlp.experts.62.up_proj", + "model.layers.21.mlp.experts.63.down_proj", + "model.layers.21.mlp.experts.63.gate_proj", + "model.layers.21.mlp.experts.63.up_proj", + "model.layers.21.mlp.experts.64.down_proj", + "model.layers.21.mlp.experts.64.gate_proj", + "model.layers.21.mlp.experts.64.up_proj", + "model.layers.21.mlp.experts.65.down_proj", + "model.layers.21.mlp.experts.65.gate_proj", + "model.layers.21.mlp.experts.65.up_proj", + "model.layers.21.mlp.experts.66.down_proj", + "model.layers.21.mlp.experts.66.gate_proj", + "model.layers.21.mlp.experts.66.up_proj", + "model.layers.21.mlp.experts.67.down_proj", + "model.layers.21.mlp.experts.67.gate_proj", + "model.layers.21.mlp.experts.67.up_proj", + "model.layers.21.mlp.experts.68.down_proj", + "model.layers.21.mlp.experts.68.gate_proj", + "model.layers.21.mlp.experts.68.up_proj", + "model.layers.21.mlp.experts.69.down_proj", + "model.layers.21.mlp.experts.69.gate_proj", + "model.layers.21.mlp.experts.69.up_proj", + "model.layers.21.mlp.experts.7.down_proj", + "model.layers.21.mlp.experts.7.gate_proj", + "model.layers.21.mlp.experts.7.up_proj", + "model.layers.21.mlp.experts.70.down_proj", + "model.layers.21.mlp.experts.70.gate_proj", + "model.layers.21.mlp.experts.70.up_proj", + "model.layers.21.mlp.experts.71.down_proj", + "model.layers.21.mlp.experts.71.gate_proj", + "model.layers.21.mlp.experts.71.up_proj", + "model.layers.21.mlp.experts.72.down_proj", + "model.layers.21.mlp.experts.72.gate_proj", + "model.layers.21.mlp.experts.72.up_proj", + "model.layers.21.mlp.experts.73.down_proj", + "model.layers.21.mlp.experts.73.gate_proj", + "model.layers.21.mlp.experts.73.up_proj", + "model.layers.21.mlp.experts.74.down_proj", + "model.layers.21.mlp.experts.74.gate_proj", + "model.layers.21.mlp.experts.74.up_proj", + "model.layers.21.mlp.experts.75.down_proj", + "model.layers.21.mlp.experts.75.gate_proj", + "model.layers.21.mlp.experts.75.up_proj", + "model.layers.21.mlp.experts.76.down_proj", + "model.layers.21.mlp.experts.76.gate_proj", + "model.layers.21.mlp.experts.76.up_proj", + "model.layers.21.mlp.experts.77.down_proj", + "model.layers.21.mlp.experts.77.gate_proj", + "model.layers.21.mlp.experts.77.up_proj", + "model.layers.21.mlp.experts.78.down_proj", + "model.layers.21.mlp.experts.78.gate_proj", + "model.layers.21.mlp.experts.78.up_proj", + "model.layers.21.mlp.experts.79.down_proj", + "model.layers.21.mlp.experts.79.gate_proj", + "model.layers.21.mlp.experts.79.up_proj", + "model.layers.21.mlp.experts.8.down_proj", + "model.layers.21.mlp.experts.8.gate_proj", + "model.layers.21.mlp.experts.8.up_proj", + "model.layers.21.mlp.experts.80.down_proj", + "model.layers.21.mlp.experts.80.gate_proj", + "model.layers.21.mlp.experts.80.up_proj", + "model.layers.21.mlp.experts.81.down_proj", + "model.layers.21.mlp.experts.81.gate_proj", + "model.layers.21.mlp.experts.81.up_proj", + "model.layers.21.mlp.experts.82.down_proj", + "model.layers.21.mlp.experts.82.gate_proj", + "model.layers.21.mlp.experts.82.up_proj", + "model.layers.21.mlp.experts.83.down_proj", + "model.layers.21.mlp.experts.83.gate_proj", + "model.layers.21.mlp.experts.83.up_proj", + "model.layers.21.mlp.experts.84.down_proj", + "model.layers.21.mlp.experts.84.gate_proj", + "model.layers.21.mlp.experts.84.up_proj", + "model.layers.21.mlp.experts.85.down_proj", + "model.layers.21.mlp.experts.85.gate_proj", + "model.layers.21.mlp.experts.85.up_proj", + "model.layers.21.mlp.experts.86.down_proj", + "model.layers.21.mlp.experts.86.gate_proj", + "model.layers.21.mlp.experts.86.up_proj", + "model.layers.21.mlp.experts.87.down_proj", + "model.layers.21.mlp.experts.87.gate_proj", + "model.layers.21.mlp.experts.87.up_proj", + "model.layers.21.mlp.experts.88.down_proj", + "model.layers.21.mlp.experts.88.gate_proj", + "model.layers.21.mlp.experts.88.up_proj", + "model.layers.21.mlp.experts.89.down_proj", + "model.layers.21.mlp.experts.89.gate_proj", + "model.layers.21.mlp.experts.89.up_proj", + "model.layers.21.mlp.experts.9.down_proj", + "model.layers.21.mlp.experts.9.gate_proj", + "model.layers.21.mlp.experts.9.up_proj", + "model.layers.21.mlp.experts.90.down_proj", + "model.layers.21.mlp.experts.90.gate_proj", + "model.layers.21.mlp.experts.90.up_proj", + "model.layers.21.mlp.experts.91.down_proj", + "model.layers.21.mlp.experts.91.gate_proj", + "model.layers.21.mlp.experts.91.up_proj", + "model.layers.21.mlp.experts.92.down_proj", + "model.layers.21.mlp.experts.92.gate_proj", + "model.layers.21.mlp.experts.92.up_proj", + "model.layers.21.mlp.experts.93.down_proj", + "model.layers.21.mlp.experts.93.gate_proj", + "model.layers.21.mlp.experts.93.up_proj", + "model.layers.21.mlp.experts.94.down_proj", + "model.layers.21.mlp.experts.94.gate_proj", + "model.layers.21.mlp.experts.94.up_proj", + "model.layers.21.mlp.experts.95.down_proj", + "model.layers.21.mlp.experts.95.gate_proj", + "model.layers.21.mlp.experts.95.up_proj", + "model.layers.21.mlp.experts.96.down_proj", + "model.layers.21.mlp.experts.96.gate_proj", + "model.layers.21.mlp.experts.96.up_proj", + "model.layers.21.mlp.experts.97.down_proj", + "model.layers.21.mlp.experts.97.gate_proj", + "model.layers.21.mlp.experts.97.up_proj", + "model.layers.21.mlp.experts.98.down_proj", + "model.layers.21.mlp.experts.98.gate_proj", + "model.layers.21.mlp.experts.98.up_proj", + "model.layers.21.mlp.experts.99.down_proj", + "model.layers.21.mlp.experts.99.gate_proj", + "model.layers.21.mlp.experts.99.up_proj", + "model.layers.22.mlp.experts.0.down_proj", + "model.layers.22.mlp.experts.0.gate_proj", + "model.layers.22.mlp.experts.0.up_proj", + "model.layers.22.mlp.experts.1.down_proj", + "model.layers.22.mlp.experts.1.gate_proj", + "model.layers.22.mlp.experts.1.up_proj", + "model.layers.22.mlp.experts.10.down_proj", + "model.layers.22.mlp.experts.10.gate_proj", + "model.layers.22.mlp.experts.10.up_proj", + "model.layers.22.mlp.experts.100.down_proj", + "model.layers.22.mlp.experts.100.gate_proj", + "model.layers.22.mlp.experts.100.up_proj", + "model.layers.22.mlp.experts.101.down_proj", + "model.layers.22.mlp.experts.101.gate_proj", + "model.layers.22.mlp.experts.101.up_proj", + "model.layers.22.mlp.experts.102.down_proj", + "model.layers.22.mlp.experts.102.gate_proj", + "model.layers.22.mlp.experts.102.up_proj", + "model.layers.22.mlp.experts.103.down_proj", + "model.layers.22.mlp.experts.103.gate_proj", + "model.layers.22.mlp.experts.103.up_proj", + "model.layers.22.mlp.experts.104.down_proj", + "model.layers.22.mlp.experts.104.gate_proj", + "model.layers.22.mlp.experts.104.up_proj", + "model.layers.22.mlp.experts.105.down_proj", + "model.layers.22.mlp.experts.105.gate_proj", + "model.layers.22.mlp.experts.105.up_proj", + "model.layers.22.mlp.experts.106.down_proj", + "model.layers.22.mlp.experts.106.gate_proj", + "model.layers.22.mlp.experts.106.up_proj", + "model.layers.22.mlp.experts.107.down_proj", + "model.layers.22.mlp.experts.107.gate_proj", + "model.layers.22.mlp.experts.107.up_proj", + "model.layers.22.mlp.experts.108.down_proj", + "model.layers.22.mlp.experts.108.gate_proj", + "model.layers.22.mlp.experts.108.up_proj", + "model.layers.22.mlp.experts.109.down_proj", + "model.layers.22.mlp.experts.109.gate_proj", + "model.layers.22.mlp.experts.109.up_proj", + "model.layers.22.mlp.experts.11.down_proj", + "model.layers.22.mlp.experts.11.gate_proj", + "model.layers.22.mlp.experts.11.up_proj", + "model.layers.22.mlp.experts.110.down_proj", + "model.layers.22.mlp.experts.110.gate_proj", + "model.layers.22.mlp.experts.110.up_proj", + "model.layers.22.mlp.experts.111.down_proj", + "model.layers.22.mlp.experts.111.gate_proj", + "model.layers.22.mlp.experts.111.up_proj", + "model.layers.22.mlp.experts.112.down_proj", + "model.layers.22.mlp.experts.112.gate_proj", + "model.layers.22.mlp.experts.112.up_proj", + "model.layers.22.mlp.experts.113.down_proj", + "model.layers.22.mlp.experts.113.gate_proj", + "model.layers.22.mlp.experts.113.up_proj", + "model.layers.22.mlp.experts.114.down_proj", + "model.layers.22.mlp.experts.114.gate_proj", + "model.layers.22.mlp.experts.114.up_proj", + "model.layers.22.mlp.experts.115.down_proj", + "model.layers.22.mlp.experts.115.gate_proj", + "model.layers.22.mlp.experts.115.up_proj", + "model.layers.22.mlp.experts.116.down_proj", + "model.layers.22.mlp.experts.116.gate_proj", + "model.layers.22.mlp.experts.116.up_proj", + "model.layers.22.mlp.experts.117.down_proj", + "model.layers.22.mlp.experts.117.gate_proj", + "model.layers.22.mlp.experts.117.up_proj", + "model.layers.22.mlp.experts.118.down_proj", + "model.layers.22.mlp.experts.118.gate_proj", + "model.layers.22.mlp.experts.118.up_proj", + "model.layers.22.mlp.experts.119.down_proj", + "model.layers.22.mlp.experts.119.gate_proj", + "model.layers.22.mlp.experts.119.up_proj", + "model.layers.22.mlp.experts.12.down_proj", + "model.layers.22.mlp.experts.12.gate_proj", + "model.layers.22.mlp.experts.12.up_proj", + "model.layers.22.mlp.experts.120.down_proj", + "model.layers.22.mlp.experts.120.gate_proj", + "model.layers.22.mlp.experts.120.up_proj", + "model.layers.22.mlp.experts.121.down_proj", + "model.layers.22.mlp.experts.121.gate_proj", + "model.layers.22.mlp.experts.121.up_proj", + "model.layers.22.mlp.experts.122.down_proj", + "model.layers.22.mlp.experts.122.gate_proj", + "model.layers.22.mlp.experts.122.up_proj", + "model.layers.22.mlp.experts.123.down_proj", + "model.layers.22.mlp.experts.123.gate_proj", + "model.layers.22.mlp.experts.123.up_proj", + "model.layers.22.mlp.experts.124.down_proj", + "model.layers.22.mlp.experts.124.gate_proj", + "model.layers.22.mlp.experts.124.up_proj", + "model.layers.22.mlp.experts.125.down_proj", + "model.layers.22.mlp.experts.125.gate_proj", + "model.layers.22.mlp.experts.125.up_proj", + "model.layers.22.mlp.experts.126.down_proj", + "model.layers.22.mlp.experts.126.gate_proj", + "model.layers.22.mlp.experts.126.up_proj", + "model.layers.22.mlp.experts.127.down_proj", + "model.layers.22.mlp.experts.127.gate_proj", + "model.layers.22.mlp.experts.127.up_proj", + "model.layers.22.mlp.experts.128.down_proj", + "model.layers.22.mlp.experts.128.gate_proj", + "model.layers.22.mlp.experts.128.up_proj", + "model.layers.22.mlp.experts.129.down_proj", + "model.layers.22.mlp.experts.129.gate_proj", + "model.layers.22.mlp.experts.129.up_proj", + "model.layers.22.mlp.experts.13.down_proj", + "model.layers.22.mlp.experts.13.gate_proj", + "model.layers.22.mlp.experts.13.up_proj", + "model.layers.22.mlp.experts.130.down_proj", + "model.layers.22.mlp.experts.130.gate_proj", + "model.layers.22.mlp.experts.130.up_proj", + "model.layers.22.mlp.experts.131.down_proj", + "model.layers.22.mlp.experts.131.gate_proj", + "model.layers.22.mlp.experts.131.up_proj", + "model.layers.22.mlp.experts.132.down_proj", + "model.layers.22.mlp.experts.132.gate_proj", + "model.layers.22.mlp.experts.132.up_proj", + "model.layers.22.mlp.experts.133.down_proj", + "model.layers.22.mlp.experts.133.gate_proj", + "model.layers.22.mlp.experts.133.up_proj", + "model.layers.22.mlp.experts.134.down_proj", + "model.layers.22.mlp.experts.134.gate_proj", + "model.layers.22.mlp.experts.134.up_proj", + "model.layers.22.mlp.experts.135.down_proj", + "model.layers.22.mlp.experts.135.gate_proj", + "model.layers.22.mlp.experts.135.up_proj", + "model.layers.22.mlp.experts.136.down_proj", + "model.layers.22.mlp.experts.136.gate_proj", + "model.layers.22.mlp.experts.136.up_proj", + "model.layers.22.mlp.experts.137.down_proj", + "model.layers.22.mlp.experts.137.gate_proj", + "model.layers.22.mlp.experts.137.up_proj", + "model.layers.22.mlp.experts.138.down_proj", + "model.layers.22.mlp.experts.138.gate_proj", + "model.layers.22.mlp.experts.138.up_proj", + "model.layers.22.mlp.experts.139.down_proj", + "model.layers.22.mlp.experts.139.gate_proj", + "model.layers.22.mlp.experts.139.up_proj", + "model.layers.22.mlp.experts.14.down_proj", + "model.layers.22.mlp.experts.14.gate_proj", + "model.layers.22.mlp.experts.14.up_proj", + "model.layers.22.mlp.experts.140.down_proj", + "model.layers.22.mlp.experts.140.gate_proj", + "model.layers.22.mlp.experts.140.up_proj", + "model.layers.22.mlp.experts.141.down_proj", + "model.layers.22.mlp.experts.141.gate_proj", + "model.layers.22.mlp.experts.141.up_proj", + "model.layers.22.mlp.experts.142.down_proj", + "model.layers.22.mlp.experts.142.gate_proj", + "model.layers.22.mlp.experts.142.up_proj", + "model.layers.22.mlp.experts.143.down_proj", + "model.layers.22.mlp.experts.143.gate_proj", + "model.layers.22.mlp.experts.143.up_proj", + "model.layers.22.mlp.experts.144.down_proj", + "model.layers.22.mlp.experts.144.gate_proj", + "model.layers.22.mlp.experts.144.up_proj", + "model.layers.22.mlp.experts.145.down_proj", + "model.layers.22.mlp.experts.145.gate_proj", + "model.layers.22.mlp.experts.145.up_proj", + "model.layers.22.mlp.experts.146.down_proj", + "model.layers.22.mlp.experts.146.gate_proj", + "model.layers.22.mlp.experts.146.up_proj", + "model.layers.22.mlp.experts.147.down_proj", + "model.layers.22.mlp.experts.147.gate_proj", + "model.layers.22.mlp.experts.147.up_proj", + "model.layers.22.mlp.experts.148.down_proj", + "model.layers.22.mlp.experts.148.gate_proj", + "model.layers.22.mlp.experts.148.up_proj", + "model.layers.22.mlp.experts.149.down_proj", + "model.layers.22.mlp.experts.149.gate_proj", + "model.layers.22.mlp.experts.149.up_proj", + "model.layers.22.mlp.experts.15.down_proj", + "model.layers.22.mlp.experts.15.gate_proj", + "model.layers.22.mlp.experts.15.up_proj", + "model.layers.22.mlp.experts.150.down_proj", + "model.layers.22.mlp.experts.150.gate_proj", + "model.layers.22.mlp.experts.150.up_proj", + "model.layers.22.mlp.experts.151.down_proj", + "model.layers.22.mlp.experts.151.gate_proj", + "model.layers.22.mlp.experts.151.up_proj", + "model.layers.22.mlp.experts.152.down_proj", + "model.layers.22.mlp.experts.152.gate_proj", + "model.layers.22.mlp.experts.152.up_proj", + "model.layers.22.mlp.experts.153.down_proj", + "model.layers.22.mlp.experts.153.gate_proj", + "model.layers.22.mlp.experts.153.up_proj", + "model.layers.22.mlp.experts.154.down_proj", + "model.layers.22.mlp.experts.154.gate_proj", + "model.layers.22.mlp.experts.154.up_proj", + "model.layers.22.mlp.experts.155.down_proj", + "model.layers.22.mlp.experts.155.gate_proj", + "model.layers.22.mlp.experts.155.up_proj", + "model.layers.22.mlp.experts.156.down_proj", + "model.layers.22.mlp.experts.156.gate_proj", + "model.layers.22.mlp.experts.156.up_proj", + "model.layers.22.mlp.experts.157.down_proj", + "model.layers.22.mlp.experts.157.gate_proj", + "model.layers.22.mlp.experts.157.up_proj", + "model.layers.22.mlp.experts.158.down_proj", + "model.layers.22.mlp.experts.158.gate_proj", + "model.layers.22.mlp.experts.158.up_proj", + "model.layers.22.mlp.experts.159.down_proj", + "model.layers.22.mlp.experts.159.gate_proj", + "model.layers.22.mlp.experts.159.up_proj", + "model.layers.22.mlp.experts.16.down_proj", + "model.layers.22.mlp.experts.16.gate_proj", + "model.layers.22.mlp.experts.16.up_proj", + "model.layers.22.mlp.experts.160.down_proj", + "model.layers.22.mlp.experts.160.gate_proj", + "model.layers.22.mlp.experts.160.up_proj", + "model.layers.22.mlp.experts.161.down_proj", + "model.layers.22.mlp.experts.161.gate_proj", + "model.layers.22.mlp.experts.161.up_proj", + "model.layers.22.mlp.experts.162.down_proj", + "model.layers.22.mlp.experts.162.gate_proj", + "model.layers.22.mlp.experts.162.up_proj", + "model.layers.22.mlp.experts.163.down_proj", + "model.layers.22.mlp.experts.163.gate_proj", + "model.layers.22.mlp.experts.163.up_proj", + "model.layers.22.mlp.experts.164.down_proj", + "model.layers.22.mlp.experts.164.gate_proj", + "model.layers.22.mlp.experts.164.up_proj", + "model.layers.22.mlp.experts.165.down_proj", + "model.layers.22.mlp.experts.165.gate_proj", + "model.layers.22.mlp.experts.165.up_proj", + "model.layers.22.mlp.experts.166.down_proj", + "model.layers.22.mlp.experts.166.gate_proj", + "model.layers.22.mlp.experts.166.up_proj", + "model.layers.22.mlp.experts.167.down_proj", + "model.layers.22.mlp.experts.167.gate_proj", + "model.layers.22.mlp.experts.167.up_proj", + "model.layers.22.mlp.experts.168.down_proj", + "model.layers.22.mlp.experts.168.gate_proj", + "model.layers.22.mlp.experts.168.up_proj", + "model.layers.22.mlp.experts.169.down_proj", + "model.layers.22.mlp.experts.169.gate_proj", + "model.layers.22.mlp.experts.169.up_proj", + "model.layers.22.mlp.experts.17.down_proj", + "model.layers.22.mlp.experts.17.gate_proj", + "model.layers.22.mlp.experts.17.up_proj", + "model.layers.22.mlp.experts.170.down_proj", + "model.layers.22.mlp.experts.170.gate_proj", + "model.layers.22.mlp.experts.170.up_proj", + "model.layers.22.mlp.experts.171.down_proj", + "model.layers.22.mlp.experts.171.gate_proj", + "model.layers.22.mlp.experts.171.up_proj", + "model.layers.22.mlp.experts.172.down_proj", + "model.layers.22.mlp.experts.172.gate_proj", + "model.layers.22.mlp.experts.172.up_proj", + "model.layers.22.mlp.experts.173.down_proj", + "model.layers.22.mlp.experts.173.gate_proj", + "model.layers.22.mlp.experts.173.up_proj", + "model.layers.22.mlp.experts.174.down_proj", + "model.layers.22.mlp.experts.174.gate_proj", + "model.layers.22.mlp.experts.174.up_proj", + "model.layers.22.mlp.experts.175.down_proj", + "model.layers.22.mlp.experts.175.gate_proj", + "model.layers.22.mlp.experts.175.up_proj", + "model.layers.22.mlp.experts.176.down_proj", + "model.layers.22.mlp.experts.176.gate_proj", + "model.layers.22.mlp.experts.176.up_proj", + "model.layers.22.mlp.experts.177.down_proj", + "model.layers.22.mlp.experts.177.gate_proj", + "model.layers.22.mlp.experts.177.up_proj", + "model.layers.22.mlp.experts.178.down_proj", + "model.layers.22.mlp.experts.178.gate_proj", + "model.layers.22.mlp.experts.178.up_proj", + "model.layers.22.mlp.experts.179.down_proj", + "model.layers.22.mlp.experts.179.gate_proj", + "model.layers.22.mlp.experts.179.up_proj", + "model.layers.22.mlp.experts.18.down_proj", + "model.layers.22.mlp.experts.18.gate_proj", + "model.layers.22.mlp.experts.18.up_proj", + "model.layers.22.mlp.experts.180.down_proj", + "model.layers.22.mlp.experts.180.gate_proj", + "model.layers.22.mlp.experts.180.up_proj", + "model.layers.22.mlp.experts.181.down_proj", + "model.layers.22.mlp.experts.181.gate_proj", + "model.layers.22.mlp.experts.181.up_proj", + "model.layers.22.mlp.experts.182.down_proj", + "model.layers.22.mlp.experts.182.gate_proj", + "model.layers.22.mlp.experts.182.up_proj", + "model.layers.22.mlp.experts.183.down_proj", + "model.layers.22.mlp.experts.183.gate_proj", + "model.layers.22.mlp.experts.183.up_proj", + "model.layers.22.mlp.experts.184.down_proj", + "model.layers.22.mlp.experts.184.gate_proj", + "model.layers.22.mlp.experts.184.up_proj", + "model.layers.22.mlp.experts.185.down_proj", + "model.layers.22.mlp.experts.185.gate_proj", + "model.layers.22.mlp.experts.185.up_proj", + "model.layers.22.mlp.experts.186.down_proj", + "model.layers.22.mlp.experts.186.gate_proj", + "model.layers.22.mlp.experts.186.up_proj", + "model.layers.22.mlp.experts.187.down_proj", + "model.layers.22.mlp.experts.187.gate_proj", + "model.layers.22.mlp.experts.187.up_proj", + "model.layers.22.mlp.experts.188.down_proj", + "model.layers.22.mlp.experts.188.gate_proj", + "model.layers.22.mlp.experts.188.up_proj", + "model.layers.22.mlp.experts.189.down_proj", + "model.layers.22.mlp.experts.189.gate_proj", + "model.layers.22.mlp.experts.189.up_proj", + "model.layers.22.mlp.experts.19.down_proj", + "model.layers.22.mlp.experts.19.gate_proj", + "model.layers.22.mlp.experts.19.up_proj", + "model.layers.22.mlp.experts.190.down_proj", + "model.layers.22.mlp.experts.190.gate_proj", + "model.layers.22.mlp.experts.190.up_proj", + "model.layers.22.mlp.experts.191.down_proj", + "model.layers.22.mlp.experts.191.gate_proj", + "model.layers.22.mlp.experts.191.up_proj", + "model.layers.22.mlp.experts.192.down_proj", + "model.layers.22.mlp.experts.192.gate_proj", + "model.layers.22.mlp.experts.192.up_proj", + "model.layers.22.mlp.experts.193.down_proj", + "model.layers.22.mlp.experts.193.gate_proj", + "model.layers.22.mlp.experts.193.up_proj", + "model.layers.22.mlp.experts.194.down_proj", + "model.layers.22.mlp.experts.194.gate_proj", + "model.layers.22.mlp.experts.194.up_proj", + "model.layers.22.mlp.experts.195.down_proj", + "model.layers.22.mlp.experts.195.gate_proj", + "model.layers.22.mlp.experts.195.up_proj", + "model.layers.22.mlp.experts.196.down_proj", + "model.layers.22.mlp.experts.196.gate_proj", + "model.layers.22.mlp.experts.196.up_proj", + "model.layers.22.mlp.experts.197.down_proj", + "model.layers.22.mlp.experts.197.gate_proj", + "model.layers.22.mlp.experts.197.up_proj", + "model.layers.22.mlp.experts.198.down_proj", + "model.layers.22.mlp.experts.198.gate_proj", + "model.layers.22.mlp.experts.198.up_proj", + "model.layers.22.mlp.experts.199.down_proj", + "model.layers.22.mlp.experts.199.gate_proj", + "model.layers.22.mlp.experts.199.up_proj", + "model.layers.22.mlp.experts.2.down_proj", + "model.layers.22.mlp.experts.2.gate_proj", + "model.layers.22.mlp.experts.2.up_proj", + "model.layers.22.mlp.experts.20.down_proj", + "model.layers.22.mlp.experts.20.gate_proj", + "model.layers.22.mlp.experts.20.up_proj", + "model.layers.22.mlp.experts.200.down_proj", + "model.layers.22.mlp.experts.200.gate_proj", + "model.layers.22.mlp.experts.200.up_proj", + "model.layers.22.mlp.experts.201.down_proj", + "model.layers.22.mlp.experts.201.gate_proj", + "model.layers.22.mlp.experts.201.up_proj", + "model.layers.22.mlp.experts.202.down_proj", + "model.layers.22.mlp.experts.202.gate_proj", + "model.layers.22.mlp.experts.202.up_proj", + "model.layers.22.mlp.experts.203.down_proj", + "model.layers.22.mlp.experts.203.gate_proj", + "model.layers.22.mlp.experts.203.up_proj", + "model.layers.22.mlp.experts.204.down_proj", + "model.layers.22.mlp.experts.204.gate_proj", + "model.layers.22.mlp.experts.204.up_proj", + "model.layers.22.mlp.experts.205.down_proj", + "model.layers.22.mlp.experts.205.gate_proj", + "model.layers.22.mlp.experts.205.up_proj", + "model.layers.22.mlp.experts.206.down_proj", + "model.layers.22.mlp.experts.206.gate_proj", + "model.layers.22.mlp.experts.206.up_proj", + "model.layers.22.mlp.experts.207.down_proj", + "model.layers.22.mlp.experts.207.gate_proj", + "model.layers.22.mlp.experts.207.up_proj", + "model.layers.22.mlp.experts.208.down_proj", + "model.layers.22.mlp.experts.208.gate_proj", + "model.layers.22.mlp.experts.208.up_proj", + "model.layers.22.mlp.experts.209.down_proj", + "model.layers.22.mlp.experts.209.gate_proj", + "model.layers.22.mlp.experts.209.up_proj", + "model.layers.22.mlp.experts.21.down_proj", + "model.layers.22.mlp.experts.21.gate_proj", + "model.layers.22.mlp.experts.21.up_proj", + "model.layers.22.mlp.experts.210.down_proj", + "model.layers.22.mlp.experts.210.gate_proj", + "model.layers.22.mlp.experts.210.up_proj", + "model.layers.22.mlp.experts.211.down_proj", + "model.layers.22.mlp.experts.211.gate_proj", + "model.layers.22.mlp.experts.211.up_proj", + "model.layers.22.mlp.experts.212.down_proj", + "model.layers.22.mlp.experts.212.gate_proj", + "model.layers.22.mlp.experts.212.up_proj", + "model.layers.22.mlp.experts.213.down_proj", + "model.layers.22.mlp.experts.213.gate_proj", + "model.layers.22.mlp.experts.213.up_proj", + "model.layers.22.mlp.experts.214.down_proj", + "model.layers.22.mlp.experts.214.gate_proj", + "model.layers.22.mlp.experts.214.up_proj", + "model.layers.22.mlp.experts.215.down_proj", + "model.layers.22.mlp.experts.215.gate_proj", + "model.layers.22.mlp.experts.215.up_proj", + "model.layers.22.mlp.experts.216.down_proj", + "model.layers.22.mlp.experts.216.gate_proj", + "model.layers.22.mlp.experts.216.up_proj", + "model.layers.22.mlp.experts.217.down_proj", + "model.layers.22.mlp.experts.217.gate_proj", + "model.layers.22.mlp.experts.217.up_proj", + "model.layers.22.mlp.experts.218.down_proj", + "model.layers.22.mlp.experts.218.gate_proj", + "model.layers.22.mlp.experts.218.up_proj", + "model.layers.22.mlp.experts.219.down_proj", + "model.layers.22.mlp.experts.219.gate_proj", + "model.layers.22.mlp.experts.219.up_proj", + "model.layers.22.mlp.experts.22.down_proj", + "model.layers.22.mlp.experts.22.gate_proj", + "model.layers.22.mlp.experts.22.up_proj", + "model.layers.22.mlp.experts.220.down_proj", + "model.layers.22.mlp.experts.220.gate_proj", + "model.layers.22.mlp.experts.220.up_proj", + "model.layers.22.mlp.experts.221.down_proj", + "model.layers.22.mlp.experts.221.gate_proj", + "model.layers.22.mlp.experts.221.up_proj", + "model.layers.22.mlp.experts.222.down_proj", + "model.layers.22.mlp.experts.222.gate_proj", + "model.layers.22.mlp.experts.222.up_proj", + "model.layers.22.mlp.experts.223.down_proj", + "model.layers.22.mlp.experts.223.gate_proj", + "model.layers.22.mlp.experts.223.up_proj", + "model.layers.22.mlp.experts.224.down_proj", + "model.layers.22.mlp.experts.224.gate_proj", + "model.layers.22.mlp.experts.224.up_proj", + "model.layers.22.mlp.experts.225.down_proj", + "model.layers.22.mlp.experts.225.gate_proj", + "model.layers.22.mlp.experts.225.up_proj", + "model.layers.22.mlp.experts.226.down_proj", + "model.layers.22.mlp.experts.226.gate_proj", + "model.layers.22.mlp.experts.226.up_proj", + "model.layers.22.mlp.experts.227.down_proj", + "model.layers.22.mlp.experts.227.gate_proj", + "model.layers.22.mlp.experts.227.up_proj", + "model.layers.22.mlp.experts.228.down_proj", + "model.layers.22.mlp.experts.228.gate_proj", + "model.layers.22.mlp.experts.228.up_proj", + "model.layers.22.mlp.experts.229.down_proj", + "model.layers.22.mlp.experts.229.gate_proj", + "model.layers.22.mlp.experts.229.up_proj", + "model.layers.22.mlp.experts.23.down_proj", + "model.layers.22.mlp.experts.23.gate_proj", + "model.layers.22.mlp.experts.23.up_proj", + "model.layers.22.mlp.experts.230.down_proj", + "model.layers.22.mlp.experts.230.gate_proj", + "model.layers.22.mlp.experts.230.up_proj", + "model.layers.22.mlp.experts.231.down_proj", + "model.layers.22.mlp.experts.231.gate_proj", + "model.layers.22.mlp.experts.231.up_proj", + "model.layers.22.mlp.experts.232.down_proj", + "model.layers.22.mlp.experts.232.gate_proj", + "model.layers.22.mlp.experts.232.up_proj", + "model.layers.22.mlp.experts.233.down_proj", + "model.layers.22.mlp.experts.233.gate_proj", + "model.layers.22.mlp.experts.233.up_proj", + "model.layers.22.mlp.experts.234.down_proj", + "model.layers.22.mlp.experts.234.gate_proj", + "model.layers.22.mlp.experts.234.up_proj", + "model.layers.22.mlp.experts.235.down_proj", + "model.layers.22.mlp.experts.235.gate_proj", + "model.layers.22.mlp.experts.235.up_proj", + "model.layers.22.mlp.experts.236.down_proj", + "model.layers.22.mlp.experts.236.gate_proj", + "model.layers.22.mlp.experts.236.up_proj", + "model.layers.22.mlp.experts.237.down_proj", + "model.layers.22.mlp.experts.237.gate_proj", + "model.layers.22.mlp.experts.237.up_proj", + "model.layers.22.mlp.experts.238.down_proj", + "model.layers.22.mlp.experts.238.gate_proj", + "model.layers.22.mlp.experts.238.up_proj", + "model.layers.22.mlp.experts.239.down_proj", + "model.layers.22.mlp.experts.239.gate_proj", + "model.layers.22.mlp.experts.239.up_proj", + "model.layers.22.mlp.experts.24.down_proj", + "model.layers.22.mlp.experts.24.gate_proj", + "model.layers.22.mlp.experts.24.up_proj", + "model.layers.22.mlp.experts.240.down_proj", + "model.layers.22.mlp.experts.240.gate_proj", + "model.layers.22.mlp.experts.240.up_proj", + "model.layers.22.mlp.experts.241.down_proj", + "model.layers.22.mlp.experts.241.gate_proj", + "model.layers.22.mlp.experts.241.up_proj", + "model.layers.22.mlp.experts.242.down_proj", + "model.layers.22.mlp.experts.242.gate_proj", + "model.layers.22.mlp.experts.242.up_proj", + "model.layers.22.mlp.experts.243.down_proj", + "model.layers.22.mlp.experts.243.gate_proj", + "model.layers.22.mlp.experts.243.up_proj", + "model.layers.22.mlp.experts.244.down_proj", + "model.layers.22.mlp.experts.244.gate_proj", + "model.layers.22.mlp.experts.244.up_proj", + "model.layers.22.mlp.experts.245.down_proj", + "model.layers.22.mlp.experts.245.gate_proj", + "model.layers.22.mlp.experts.245.up_proj", + "model.layers.22.mlp.experts.246.down_proj", + "model.layers.22.mlp.experts.246.gate_proj", + "model.layers.22.mlp.experts.246.up_proj", + "model.layers.22.mlp.experts.247.down_proj", + "model.layers.22.mlp.experts.247.gate_proj", + "model.layers.22.mlp.experts.247.up_proj", + "model.layers.22.mlp.experts.248.down_proj", + "model.layers.22.mlp.experts.248.gate_proj", + "model.layers.22.mlp.experts.248.up_proj", + "model.layers.22.mlp.experts.249.down_proj", + "model.layers.22.mlp.experts.249.gate_proj", + "model.layers.22.mlp.experts.249.up_proj", + "model.layers.22.mlp.experts.25.down_proj", + "model.layers.22.mlp.experts.25.gate_proj", + "model.layers.22.mlp.experts.25.up_proj", + "model.layers.22.mlp.experts.250.down_proj", + "model.layers.22.mlp.experts.250.gate_proj", + "model.layers.22.mlp.experts.250.up_proj", + "model.layers.22.mlp.experts.251.down_proj", + "model.layers.22.mlp.experts.251.gate_proj", + "model.layers.22.mlp.experts.251.up_proj", + "model.layers.22.mlp.experts.252.down_proj", + "model.layers.22.mlp.experts.252.gate_proj", + "model.layers.22.mlp.experts.252.up_proj", + "model.layers.22.mlp.experts.253.down_proj", + "model.layers.22.mlp.experts.253.gate_proj", + "model.layers.22.mlp.experts.253.up_proj", + "model.layers.22.mlp.experts.254.down_proj", + "model.layers.22.mlp.experts.254.gate_proj", + "model.layers.22.mlp.experts.254.up_proj", + "model.layers.22.mlp.experts.255.down_proj", + "model.layers.22.mlp.experts.255.gate_proj", + "model.layers.22.mlp.experts.255.up_proj", + "model.layers.22.mlp.experts.26.down_proj", + "model.layers.22.mlp.experts.26.gate_proj", + "model.layers.22.mlp.experts.26.up_proj", + "model.layers.22.mlp.experts.27.down_proj", + "model.layers.22.mlp.experts.27.gate_proj", + "model.layers.22.mlp.experts.27.up_proj", + "model.layers.22.mlp.experts.28.down_proj", + "model.layers.22.mlp.experts.28.gate_proj", + "model.layers.22.mlp.experts.28.up_proj", + "model.layers.22.mlp.experts.29.down_proj", + "model.layers.22.mlp.experts.29.gate_proj", + "model.layers.22.mlp.experts.29.up_proj", + "model.layers.22.mlp.experts.3.down_proj", + "model.layers.22.mlp.experts.3.gate_proj", + "model.layers.22.mlp.experts.3.up_proj", + "model.layers.22.mlp.experts.30.down_proj", + "model.layers.22.mlp.experts.30.gate_proj", + "model.layers.22.mlp.experts.30.up_proj", + "model.layers.22.mlp.experts.31.down_proj", + "model.layers.22.mlp.experts.31.gate_proj", + "model.layers.22.mlp.experts.31.up_proj", + "model.layers.22.mlp.experts.32.down_proj", + "model.layers.22.mlp.experts.32.gate_proj", + "model.layers.22.mlp.experts.32.up_proj", + "model.layers.22.mlp.experts.33.down_proj", + "model.layers.22.mlp.experts.33.gate_proj", + "model.layers.22.mlp.experts.33.up_proj", + "model.layers.22.mlp.experts.34.down_proj", + "model.layers.22.mlp.experts.34.gate_proj", + "model.layers.22.mlp.experts.34.up_proj", + "model.layers.22.mlp.experts.35.down_proj", + "model.layers.22.mlp.experts.35.gate_proj", + "model.layers.22.mlp.experts.35.up_proj", + "model.layers.22.mlp.experts.36.down_proj", + "model.layers.22.mlp.experts.36.gate_proj", + "model.layers.22.mlp.experts.36.up_proj", + "model.layers.22.mlp.experts.37.down_proj", + "model.layers.22.mlp.experts.37.gate_proj", + "model.layers.22.mlp.experts.37.up_proj", + "model.layers.22.mlp.experts.38.down_proj", + "model.layers.22.mlp.experts.38.gate_proj", + "model.layers.22.mlp.experts.38.up_proj", + "model.layers.22.mlp.experts.39.down_proj", + "model.layers.22.mlp.experts.39.gate_proj", + "model.layers.22.mlp.experts.39.up_proj", + "model.layers.22.mlp.experts.4.down_proj", + "model.layers.22.mlp.experts.4.gate_proj", + "model.layers.22.mlp.experts.4.up_proj", + "model.layers.22.mlp.experts.40.down_proj", + "model.layers.22.mlp.experts.40.gate_proj", + "model.layers.22.mlp.experts.40.up_proj", + "model.layers.22.mlp.experts.41.down_proj", + "model.layers.22.mlp.experts.41.gate_proj", + "model.layers.22.mlp.experts.41.up_proj", + "model.layers.22.mlp.experts.42.down_proj", + "model.layers.22.mlp.experts.42.gate_proj", + "model.layers.22.mlp.experts.42.up_proj", + "model.layers.22.mlp.experts.43.down_proj", + "model.layers.22.mlp.experts.43.gate_proj", + "model.layers.22.mlp.experts.43.up_proj", + "model.layers.22.mlp.experts.44.down_proj", + "model.layers.22.mlp.experts.44.gate_proj", + "model.layers.22.mlp.experts.44.up_proj", + "model.layers.22.mlp.experts.45.down_proj", + "model.layers.22.mlp.experts.45.gate_proj", + "model.layers.22.mlp.experts.45.up_proj", + "model.layers.22.mlp.experts.46.down_proj", + "model.layers.22.mlp.experts.46.gate_proj", + "model.layers.22.mlp.experts.46.up_proj", + "model.layers.22.mlp.experts.47.down_proj", + "model.layers.22.mlp.experts.47.gate_proj", + "model.layers.22.mlp.experts.47.up_proj", + "model.layers.22.mlp.experts.48.down_proj", + "model.layers.22.mlp.experts.48.gate_proj", + "model.layers.22.mlp.experts.48.up_proj", + "model.layers.22.mlp.experts.49.down_proj", + "model.layers.22.mlp.experts.49.gate_proj", + "model.layers.22.mlp.experts.49.up_proj", + "model.layers.22.mlp.experts.5.down_proj", + "model.layers.22.mlp.experts.5.gate_proj", + "model.layers.22.mlp.experts.5.up_proj", + "model.layers.22.mlp.experts.50.down_proj", + "model.layers.22.mlp.experts.50.gate_proj", + "model.layers.22.mlp.experts.50.up_proj", + "model.layers.22.mlp.experts.51.down_proj", + "model.layers.22.mlp.experts.51.gate_proj", + "model.layers.22.mlp.experts.51.up_proj", + "model.layers.22.mlp.experts.52.down_proj", + "model.layers.22.mlp.experts.52.gate_proj", + "model.layers.22.mlp.experts.52.up_proj", + "model.layers.22.mlp.experts.53.down_proj", + "model.layers.22.mlp.experts.53.gate_proj", + "model.layers.22.mlp.experts.53.up_proj", + "model.layers.22.mlp.experts.54.down_proj", + "model.layers.22.mlp.experts.54.gate_proj", + "model.layers.22.mlp.experts.54.up_proj", + "model.layers.22.mlp.experts.55.down_proj", + "model.layers.22.mlp.experts.55.gate_proj", + "model.layers.22.mlp.experts.55.up_proj", + "model.layers.22.mlp.experts.56.down_proj", + "model.layers.22.mlp.experts.56.gate_proj", + "model.layers.22.mlp.experts.56.up_proj", + "model.layers.22.mlp.experts.57.down_proj", + "model.layers.22.mlp.experts.57.gate_proj", + "model.layers.22.mlp.experts.57.up_proj", + "model.layers.22.mlp.experts.58.down_proj", + "model.layers.22.mlp.experts.58.gate_proj", + "model.layers.22.mlp.experts.58.up_proj", + "model.layers.22.mlp.experts.59.down_proj", + "model.layers.22.mlp.experts.59.gate_proj", + "model.layers.22.mlp.experts.59.up_proj", + "model.layers.22.mlp.experts.6.down_proj", + "model.layers.22.mlp.experts.6.gate_proj", + "model.layers.22.mlp.experts.6.up_proj", + "model.layers.22.mlp.experts.60.down_proj", + "model.layers.22.mlp.experts.60.gate_proj", + "model.layers.22.mlp.experts.60.up_proj", + "model.layers.22.mlp.experts.61.down_proj", + "model.layers.22.mlp.experts.61.gate_proj", + "model.layers.22.mlp.experts.61.up_proj", + "model.layers.22.mlp.experts.62.down_proj", + "model.layers.22.mlp.experts.62.gate_proj", + "model.layers.22.mlp.experts.62.up_proj", + "model.layers.22.mlp.experts.63.down_proj", + "model.layers.22.mlp.experts.63.gate_proj", + "model.layers.22.mlp.experts.63.up_proj", + "model.layers.22.mlp.experts.64.down_proj", + "model.layers.22.mlp.experts.64.gate_proj", + "model.layers.22.mlp.experts.64.up_proj", + "model.layers.22.mlp.experts.65.down_proj", + "model.layers.22.mlp.experts.65.gate_proj", + "model.layers.22.mlp.experts.65.up_proj", + "model.layers.22.mlp.experts.66.down_proj", + "model.layers.22.mlp.experts.66.gate_proj", + "model.layers.22.mlp.experts.66.up_proj", + "model.layers.22.mlp.experts.67.down_proj", + "model.layers.22.mlp.experts.67.gate_proj", + "model.layers.22.mlp.experts.67.up_proj", + "model.layers.22.mlp.experts.68.down_proj", + "model.layers.22.mlp.experts.68.gate_proj", + "model.layers.22.mlp.experts.68.up_proj", + "model.layers.22.mlp.experts.69.down_proj", + "model.layers.22.mlp.experts.69.gate_proj", + "model.layers.22.mlp.experts.69.up_proj", + "model.layers.22.mlp.experts.7.down_proj", + "model.layers.22.mlp.experts.7.gate_proj", + "model.layers.22.mlp.experts.7.up_proj", + "model.layers.22.mlp.experts.70.down_proj", + "model.layers.22.mlp.experts.70.gate_proj", + "model.layers.22.mlp.experts.70.up_proj", + "model.layers.22.mlp.experts.71.down_proj", + "model.layers.22.mlp.experts.71.gate_proj", + "model.layers.22.mlp.experts.71.up_proj", + "model.layers.22.mlp.experts.72.down_proj", + "model.layers.22.mlp.experts.72.gate_proj", + "model.layers.22.mlp.experts.72.up_proj", + "model.layers.22.mlp.experts.73.down_proj", + "model.layers.22.mlp.experts.73.gate_proj", + "model.layers.22.mlp.experts.73.up_proj", + "model.layers.22.mlp.experts.74.down_proj", + "model.layers.22.mlp.experts.74.gate_proj", + "model.layers.22.mlp.experts.74.up_proj", + "model.layers.22.mlp.experts.75.down_proj", + "model.layers.22.mlp.experts.75.gate_proj", + "model.layers.22.mlp.experts.75.up_proj", + "model.layers.22.mlp.experts.76.down_proj", + "model.layers.22.mlp.experts.76.gate_proj", + "model.layers.22.mlp.experts.76.up_proj", + "model.layers.22.mlp.experts.77.down_proj", + "model.layers.22.mlp.experts.77.gate_proj", + "model.layers.22.mlp.experts.77.up_proj", + "model.layers.22.mlp.experts.78.down_proj", + "model.layers.22.mlp.experts.78.gate_proj", + "model.layers.22.mlp.experts.78.up_proj", + "model.layers.22.mlp.experts.79.down_proj", + "model.layers.22.mlp.experts.79.gate_proj", + "model.layers.22.mlp.experts.79.up_proj", + "model.layers.22.mlp.experts.8.down_proj", + "model.layers.22.mlp.experts.8.gate_proj", + "model.layers.22.mlp.experts.8.up_proj", + "model.layers.22.mlp.experts.80.down_proj", + "model.layers.22.mlp.experts.80.gate_proj", + "model.layers.22.mlp.experts.80.up_proj", + "model.layers.22.mlp.experts.81.down_proj", + "model.layers.22.mlp.experts.81.gate_proj", + "model.layers.22.mlp.experts.81.up_proj", + "model.layers.22.mlp.experts.82.down_proj", + "model.layers.22.mlp.experts.82.gate_proj", + "model.layers.22.mlp.experts.82.up_proj", + "model.layers.22.mlp.experts.83.down_proj", + "model.layers.22.mlp.experts.83.gate_proj", + "model.layers.22.mlp.experts.83.up_proj", + "model.layers.22.mlp.experts.84.down_proj", + "model.layers.22.mlp.experts.84.gate_proj", + "model.layers.22.mlp.experts.84.up_proj", + "model.layers.22.mlp.experts.85.down_proj", + "model.layers.22.mlp.experts.85.gate_proj", + "model.layers.22.mlp.experts.85.up_proj", + "model.layers.22.mlp.experts.86.down_proj", + "model.layers.22.mlp.experts.86.gate_proj", + "model.layers.22.mlp.experts.86.up_proj", + "model.layers.22.mlp.experts.87.down_proj", + "model.layers.22.mlp.experts.87.gate_proj", + "model.layers.22.mlp.experts.87.up_proj", + "model.layers.22.mlp.experts.88.down_proj", + "model.layers.22.mlp.experts.88.gate_proj", + "model.layers.22.mlp.experts.88.up_proj", + "model.layers.22.mlp.experts.89.down_proj", + "model.layers.22.mlp.experts.89.gate_proj", + "model.layers.22.mlp.experts.89.up_proj", + "model.layers.22.mlp.experts.9.down_proj", + "model.layers.22.mlp.experts.9.gate_proj", + "model.layers.22.mlp.experts.9.up_proj", + "model.layers.22.mlp.experts.90.down_proj", + "model.layers.22.mlp.experts.90.gate_proj", + "model.layers.22.mlp.experts.90.up_proj", + "model.layers.22.mlp.experts.91.down_proj", + "model.layers.22.mlp.experts.91.gate_proj", + "model.layers.22.mlp.experts.91.up_proj", + "model.layers.22.mlp.experts.92.down_proj", + "model.layers.22.mlp.experts.92.gate_proj", + "model.layers.22.mlp.experts.92.up_proj", + "model.layers.22.mlp.experts.93.down_proj", + "model.layers.22.mlp.experts.93.gate_proj", + "model.layers.22.mlp.experts.93.up_proj", + "model.layers.22.mlp.experts.94.down_proj", + "model.layers.22.mlp.experts.94.gate_proj", + "model.layers.22.mlp.experts.94.up_proj", + "model.layers.22.mlp.experts.95.down_proj", + "model.layers.22.mlp.experts.95.gate_proj", + "model.layers.22.mlp.experts.95.up_proj", + "model.layers.22.mlp.experts.96.down_proj", + "model.layers.22.mlp.experts.96.gate_proj", + "model.layers.22.mlp.experts.96.up_proj", + "model.layers.22.mlp.experts.97.down_proj", + "model.layers.22.mlp.experts.97.gate_proj", + "model.layers.22.mlp.experts.97.up_proj", + "model.layers.22.mlp.experts.98.down_proj", + "model.layers.22.mlp.experts.98.gate_proj", + "model.layers.22.mlp.experts.98.up_proj", + "model.layers.22.mlp.experts.99.down_proj", + "model.layers.22.mlp.experts.99.gate_proj", + "model.layers.22.mlp.experts.99.up_proj", + "model.layers.23.mlp.experts.0.down_proj", + "model.layers.23.mlp.experts.0.gate_proj", + "model.layers.23.mlp.experts.0.up_proj", + "model.layers.23.mlp.experts.1.down_proj", + "model.layers.23.mlp.experts.1.gate_proj", + "model.layers.23.mlp.experts.1.up_proj", + "model.layers.23.mlp.experts.10.down_proj", + "model.layers.23.mlp.experts.10.gate_proj", + "model.layers.23.mlp.experts.10.up_proj", + "model.layers.23.mlp.experts.100.down_proj", + "model.layers.23.mlp.experts.100.gate_proj", + "model.layers.23.mlp.experts.100.up_proj", + "model.layers.23.mlp.experts.101.down_proj", + "model.layers.23.mlp.experts.101.gate_proj", + "model.layers.23.mlp.experts.101.up_proj", + "model.layers.23.mlp.experts.102.down_proj", + "model.layers.23.mlp.experts.102.gate_proj", + "model.layers.23.mlp.experts.102.up_proj", + "model.layers.23.mlp.experts.103.down_proj", + "model.layers.23.mlp.experts.103.gate_proj", + "model.layers.23.mlp.experts.103.up_proj", + "model.layers.23.mlp.experts.104.down_proj", + "model.layers.23.mlp.experts.104.gate_proj", + "model.layers.23.mlp.experts.104.up_proj", + "model.layers.23.mlp.experts.105.down_proj", + "model.layers.23.mlp.experts.105.gate_proj", + "model.layers.23.mlp.experts.105.up_proj", + "model.layers.23.mlp.experts.106.down_proj", + "model.layers.23.mlp.experts.106.gate_proj", + "model.layers.23.mlp.experts.106.up_proj", + "model.layers.23.mlp.experts.107.down_proj", + "model.layers.23.mlp.experts.107.gate_proj", + "model.layers.23.mlp.experts.107.up_proj", + "model.layers.23.mlp.experts.108.down_proj", + "model.layers.23.mlp.experts.108.gate_proj", + "model.layers.23.mlp.experts.108.up_proj", + "model.layers.23.mlp.experts.109.down_proj", + "model.layers.23.mlp.experts.109.gate_proj", + "model.layers.23.mlp.experts.109.up_proj", + "model.layers.23.mlp.experts.11.down_proj", + "model.layers.23.mlp.experts.11.gate_proj", + "model.layers.23.mlp.experts.11.up_proj", + "model.layers.23.mlp.experts.110.down_proj", + "model.layers.23.mlp.experts.110.gate_proj", + "model.layers.23.mlp.experts.110.up_proj", + "model.layers.23.mlp.experts.111.down_proj", + "model.layers.23.mlp.experts.111.gate_proj", + "model.layers.23.mlp.experts.111.up_proj", + "model.layers.23.mlp.experts.112.down_proj", + "model.layers.23.mlp.experts.112.gate_proj", + "model.layers.23.mlp.experts.112.up_proj", + "model.layers.23.mlp.experts.113.down_proj", + "model.layers.23.mlp.experts.113.gate_proj", + "model.layers.23.mlp.experts.113.up_proj", + "model.layers.23.mlp.experts.114.down_proj", + "model.layers.23.mlp.experts.114.gate_proj", + "model.layers.23.mlp.experts.114.up_proj", + "model.layers.23.mlp.experts.115.down_proj", + "model.layers.23.mlp.experts.115.gate_proj", + "model.layers.23.mlp.experts.115.up_proj", + "model.layers.23.mlp.experts.116.down_proj", + "model.layers.23.mlp.experts.116.gate_proj", + "model.layers.23.mlp.experts.116.up_proj", + "model.layers.23.mlp.experts.117.down_proj", + "model.layers.23.mlp.experts.117.gate_proj", + "model.layers.23.mlp.experts.117.up_proj", + "model.layers.23.mlp.experts.118.down_proj", + "model.layers.23.mlp.experts.118.gate_proj", + "model.layers.23.mlp.experts.118.up_proj", + "model.layers.23.mlp.experts.119.down_proj", + "model.layers.23.mlp.experts.119.gate_proj", + "model.layers.23.mlp.experts.119.up_proj", + "model.layers.23.mlp.experts.12.down_proj", + "model.layers.23.mlp.experts.12.gate_proj", + "model.layers.23.mlp.experts.12.up_proj", + "model.layers.23.mlp.experts.120.down_proj", + "model.layers.23.mlp.experts.120.gate_proj", + "model.layers.23.mlp.experts.120.up_proj", + "model.layers.23.mlp.experts.121.down_proj", + "model.layers.23.mlp.experts.121.gate_proj", + "model.layers.23.mlp.experts.121.up_proj", + "model.layers.23.mlp.experts.122.down_proj", + "model.layers.23.mlp.experts.122.gate_proj", + "model.layers.23.mlp.experts.122.up_proj", + "model.layers.23.mlp.experts.123.down_proj", + "model.layers.23.mlp.experts.123.gate_proj", + "model.layers.23.mlp.experts.123.up_proj", + "model.layers.23.mlp.experts.124.down_proj", + "model.layers.23.mlp.experts.124.gate_proj", + "model.layers.23.mlp.experts.124.up_proj", + "model.layers.23.mlp.experts.125.down_proj", + "model.layers.23.mlp.experts.125.gate_proj", + "model.layers.23.mlp.experts.125.up_proj", + "model.layers.23.mlp.experts.126.down_proj", + "model.layers.23.mlp.experts.126.gate_proj", + "model.layers.23.mlp.experts.126.up_proj", + "model.layers.23.mlp.experts.127.down_proj", + "model.layers.23.mlp.experts.127.gate_proj", + "model.layers.23.mlp.experts.127.up_proj", + "model.layers.23.mlp.experts.128.down_proj", + "model.layers.23.mlp.experts.128.gate_proj", + "model.layers.23.mlp.experts.128.up_proj", + "model.layers.23.mlp.experts.129.down_proj", + "model.layers.23.mlp.experts.129.gate_proj", + "model.layers.23.mlp.experts.129.up_proj", + "model.layers.23.mlp.experts.13.down_proj", + "model.layers.23.mlp.experts.13.gate_proj", + "model.layers.23.mlp.experts.13.up_proj", + "model.layers.23.mlp.experts.130.down_proj", + "model.layers.23.mlp.experts.130.gate_proj", + "model.layers.23.mlp.experts.130.up_proj", + "model.layers.23.mlp.experts.131.down_proj", + "model.layers.23.mlp.experts.131.gate_proj", + "model.layers.23.mlp.experts.131.up_proj", + "model.layers.23.mlp.experts.132.down_proj", + "model.layers.23.mlp.experts.132.gate_proj", + "model.layers.23.mlp.experts.132.up_proj", + "model.layers.23.mlp.experts.133.down_proj", + "model.layers.23.mlp.experts.133.gate_proj", + "model.layers.23.mlp.experts.133.up_proj", + "model.layers.23.mlp.experts.134.down_proj", + "model.layers.23.mlp.experts.134.gate_proj", + "model.layers.23.mlp.experts.134.up_proj", + "model.layers.23.mlp.experts.135.down_proj", + "model.layers.23.mlp.experts.135.gate_proj", + "model.layers.23.mlp.experts.135.up_proj", + "model.layers.23.mlp.experts.136.down_proj", + "model.layers.23.mlp.experts.136.gate_proj", + "model.layers.23.mlp.experts.136.up_proj", + "model.layers.23.mlp.experts.137.down_proj", + "model.layers.23.mlp.experts.137.gate_proj", + "model.layers.23.mlp.experts.137.up_proj", + "model.layers.23.mlp.experts.138.down_proj", + "model.layers.23.mlp.experts.138.gate_proj", + "model.layers.23.mlp.experts.138.up_proj", + "model.layers.23.mlp.experts.139.down_proj", + "model.layers.23.mlp.experts.139.gate_proj", + "model.layers.23.mlp.experts.139.up_proj", + "model.layers.23.mlp.experts.14.down_proj", + "model.layers.23.mlp.experts.14.gate_proj", + "model.layers.23.mlp.experts.14.up_proj", + "model.layers.23.mlp.experts.140.down_proj", + "model.layers.23.mlp.experts.140.gate_proj", + "model.layers.23.mlp.experts.140.up_proj", + "model.layers.23.mlp.experts.141.down_proj", + "model.layers.23.mlp.experts.141.gate_proj", + "model.layers.23.mlp.experts.141.up_proj", + "model.layers.23.mlp.experts.142.down_proj", + "model.layers.23.mlp.experts.142.gate_proj", + "model.layers.23.mlp.experts.142.up_proj", + "model.layers.23.mlp.experts.143.down_proj", + "model.layers.23.mlp.experts.143.gate_proj", + "model.layers.23.mlp.experts.143.up_proj", + "model.layers.23.mlp.experts.144.down_proj", + "model.layers.23.mlp.experts.144.gate_proj", + "model.layers.23.mlp.experts.144.up_proj", + "model.layers.23.mlp.experts.145.down_proj", + "model.layers.23.mlp.experts.145.gate_proj", + "model.layers.23.mlp.experts.145.up_proj", + "model.layers.23.mlp.experts.146.down_proj", + "model.layers.23.mlp.experts.146.gate_proj", + "model.layers.23.mlp.experts.146.up_proj", + "model.layers.23.mlp.experts.147.down_proj", + "model.layers.23.mlp.experts.147.gate_proj", + "model.layers.23.mlp.experts.147.up_proj", + "model.layers.23.mlp.experts.148.down_proj", + "model.layers.23.mlp.experts.148.gate_proj", + "model.layers.23.mlp.experts.148.up_proj", + "model.layers.23.mlp.experts.149.down_proj", + "model.layers.23.mlp.experts.149.gate_proj", + "model.layers.23.mlp.experts.149.up_proj", + "model.layers.23.mlp.experts.15.down_proj", + "model.layers.23.mlp.experts.15.gate_proj", + "model.layers.23.mlp.experts.15.up_proj", + "model.layers.23.mlp.experts.150.down_proj", + "model.layers.23.mlp.experts.150.gate_proj", + "model.layers.23.mlp.experts.150.up_proj", + "model.layers.23.mlp.experts.151.down_proj", + "model.layers.23.mlp.experts.151.gate_proj", + "model.layers.23.mlp.experts.151.up_proj", + "model.layers.23.mlp.experts.152.down_proj", + "model.layers.23.mlp.experts.152.gate_proj", + "model.layers.23.mlp.experts.152.up_proj", + "model.layers.23.mlp.experts.153.down_proj", + "model.layers.23.mlp.experts.153.gate_proj", + "model.layers.23.mlp.experts.153.up_proj", + "model.layers.23.mlp.experts.154.down_proj", + "model.layers.23.mlp.experts.154.gate_proj", + "model.layers.23.mlp.experts.154.up_proj", + "model.layers.23.mlp.experts.155.down_proj", + "model.layers.23.mlp.experts.155.gate_proj", + "model.layers.23.mlp.experts.155.up_proj", + "model.layers.23.mlp.experts.156.down_proj", + "model.layers.23.mlp.experts.156.gate_proj", + "model.layers.23.mlp.experts.156.up_proj", + "model.layers.23.mlp.experts.157.down_proj", + "model.layers.23.mlp.experts.157.gate_proj", + "model.layers.23.mlp.experts.157.up_proj", + "model.layers.23.mlp.experts.158.down_proj", + "model.layers.23.mlp.experts.158.gate_proj", + "model.layers.23.mlp.experts.158.up_proj", + "model.layers.23.mlp.experts.159.down_proj", + "model.layers.23.mlp.experts.159.gate_proj", + "model.layers.23.mlp.experts.159.up_proj", + "model.layers.23.mlp.experts.16.down_proj", + "model.layers.23.mlp.experts.16.gate_proj", + "model.layers.23.mlp.experts.16.up_proj", + "model.layers.23.mlp.experts.160.down_proj", + "model.layers.23.mlp.experts.160.gate_proj", + "model.layers.23.mlp.experts.160.up_proj", + "model.layers.23.mlp.experts.161.down_proj", + "model.layers.23.mlp.experts.161.gate_proj", + "model.layers.23.mlp.experts.161.up_proj", + "model.layers.23.mlp.experts.162.down_proj", + "model.layers.23.mlp.experts.162.gate_proj", + "model.layers.23.mlp.experts.162.up_proj", + "model.layers.23.mlp.experts.163.down_proj", + "model.layers.23.mlp.experts.163.gate_proj", + "model.layers.23.mlp.experts.163.up_proj", + "model.layers.23.mlp.experts.164.down_proj", + "model.layers.23.mlp.experts.164.gate_proj", + "model.layers.23.mlp.experts.164.up_proj", + "model.layers.23.mlp.experts.165.down_proj", + "model.layers.23.mlp.experts.165.gate_proj", + "model.layers.23.mlp.experts.165.up_proj", + "model.layers.23.mlp.experts.166.down_proj", + "model.layers.23.mlp.experts.166.gate_proj", + "model.layers.23.mlp.experts.166.up_proj", + "model.layers.23.mlp.experts.167.down_proj", + "model.layers.23.mlp.experts.167.gate_proj", + "model.layers.23.mlp.experts.167.up_proj", + "model.layers.23.mlp.experts.168.down_proj", + "model.layers.23.mlp.experts.168.gate_proj", + "model.layers.23.mlp.experts.168.up_proj", + "model.layers.23.mlp.experts.169.down_proj", + "model.layers.23.mlp.experts.169.gate_proj", + "model.layers.23.mlp.experts.169.up_proj", + "model.layers.23.mlp.experts.17.down_proj", + "model.layers.23.mlp.experts.17.gate_proj", + "model.layers.23.mlp.experts.17.up_proj", + "model.layers.23.mlp.experts.170.down_proj", + "model.layers.23.mlp.experts.170.gate_proj", + "model.layers.23.mlp.experts.170.up_proj", + "model.layers.23.mlp.experts.171.down_proj", + "model.layers.23.mlp.experts.171.gate_proj", + "model.layers.23.mlp.experts.171.up_proj", + "model.layers.23.mlp.experts.172.down_proj", + "model.layers.23.mlp.experts.172.gate_proj", + "model.layers.23.mlp.experts.172.up_proj", + "model.layers.23.mlp.experts.173.down_proj", + "model.layers.23.mlp.experts.173.gate_proj", + "model.layers.23.mlp.experts.173.up_proj", + "model.layers.23.mlp.experts.174.down_proj", + "model.layers.23.mlp.experts.174.gate_proj", + "model.layers.23.mlp.experts.174.up_proj", + "model.layers.23.mlp.experts.175.down_proj", + "model.layers.23.mlp.experts.175.gate_proj", + "model.layers.23.mlp.experts.175.up_proj", + "model.layers.23.mlp.experts.176.down_proj", + "model.layers.23.mlp.experts.176.gate_proj", + "model.layers.23.mlp.experts.176.up_proj", + "model.layers.23.mlp.experts.177.down_proj", + "model.layers.23.mlp.experts.177.gate_proj", + "model.layers.23.mlp.experts.177.up_proj", + "model.layers.23.mlp.experts.178.down_proj", + "model.layers.23.mlp.experts.178.gate_proj", + "model.layers.23.mlp.experts.178.up_proj", + "model.layers.23.mlp.experts.179.down_proj", + "model.layers.23.mlp.experts.179.gate_proj", + "model.layers.23.mlp.experts.179.up_proj", + "model.layers.23.mlp.experts.18.down_proj", + "model.layers.23.mlp.experts.18.gate_proj", + "model.layers.23.mlp.experts.18.up_proj", + "model.layers.23.mlp.experts.180.down_proj", + "model.layers.23.mlp.experts.180.gate_proj", + "model.layers.23.mlp.experts.180.up_proj", + "model.layers.23.mlp.experts.181.down_proj", + "model.layers.23.mlp.experts.181.gate_proj", + "model.layers.23.mlp.experts.181.up_proj", + "model.layers.23.mlp.experts.182.down_proj", + "model.layers.23.mlp.experts.182.gate_proj", + "model.layers.23.mlp.experts.182.up_proj", + "model.layers.23.mlp.experts.183.down_proj", + "model.layers.23.mlp.experts.183.gate_proj", + "model.layers.23.mlp.experts.183.up_proj", + "model.layers.23.mlp.experts.184.down_proj", + "model.layers.23.mlp.experts.184.gate_proj", + "model.layers.23.mlp.experts.184.up_proj", + "model.layers.23.mlp.experts.185.down_proj", + "model.layers.23.mlp.experts.185.gate_proj", + "model.layers.23.mlp.experts.185.up_proj", + "model.layers.23.mlp.experts.186.down_proj", + "model.layers.23.mlp.experts.186.gate_proj", + "model.layers.23.mlp.experts.186.up_proj", + "model.layers.23.mlp.experts.187.down_proj", + "model.layers.23.mlp.experts.187.gate_proj", + "model.layers.23.mlp.experts.187.up_proj", + "model.layers.23.mlp.experts.188.down_proj", + "model.layers.23.mlp.experts.188.gate_proj", + "model.layers.23.mlp.experts.188.up_proj", + "model.layers.23.mlp.experts.189.down_proj", + "model.layers.23.mlp.experts.189.gate_proj", + "model.layers.23.mlp.experts.189.up_proj", + "model.layers.23.mlp.experts.19.down_proj", + "model.layers.23.mlp.experts.19.gate_proj", + "model.layers.23.mlp.experts.19.up_proj", + "model.layers.23.mlp.experts.190.down_proj", + "model.layers.23.mlp.experts.190.gate_proj", + "model.layers.23.mlp.experts.190.up_proj", + "model.layers.23.mlp.experts.191.down_proj", + "model.layers.23.mlp.experts.191.gate_proj", + "model.layers.23.mlp.experts.191.up_proj", + "model.layers.23.mlp.experts.192.down_proj", + "model.layers.23.mlp.experts.192.gate_proj", + "model.layers.23.mlp.experts.192.up_proj", + "model.layers.23.mlp.experts.193.down_proj", + "model.layers.23.mlp.experts.193.gate_proj", + "model.layers.23.mlp.experts.193.up_proj", + "model.layers.23.mlp.experts.194.down_proj", + "model.layers.23.mlp.experts.194.gate_proj", + "model.layers.23.mlp.experts.194.up_proj", + "model.layers.23.mlp.experts.195.down_proj", + "model.layers.23.mlp.experts.195.gate_proj", + "model.layers.23.mlp.experts.195.up_proj", + "model.layers.23.mlp.experts.196.down_proj", + "model.layers.23.mlp.experts.196.gate_proj", + "model.layers.23.mlp.experts.196.up_proj", + "model.layers.23.mlp.experts.197.down_proj", + "model.layers.23.mlp.experts.197.gate_proj", + "model.layers.23.mlp.experts.197.up_proj", + "model.layers.23.mlp.experts.198.down_proj", + "model.layers.23.mlp.experts.198.gate_proj", + "model.layers.23.mlp.experts.198.up_proj", + "model.layers.23.mlp.experts.199.down_proj", + "model.layers.23.mlp.experts.199.gate_proj", + "model.layers.23.mlp.experts.199.up_proj", + "model.layers.23.mlp.experts.2.down_proj", + "model.layers.23.mlp.experts.2.gate_proj", + "model.layers.23.mlp.experts.2.up_proj", + "model.layers.23.mlp.experts.20.down_proj", + "model.layers.23.mlp.experts.20.gate_proj", + "model.layers.23.mlp.experts.20.up_proj", + "model.layers.23.mlp.experts.200.down_proj", + "model.layers.23.mlp.experts.200.gate_proj", + "model.layers.23.mlp.experts.200.up_proj", + "model.layers.23.mlp.experts.201.down_proj", + "model.layers.23.mlp.experts.201.gate_proj", + "model.layers.23.mlp.experts.201.up_proj", + "model.layers.23.mlp.experts.202.down_proj", + "model.layers.23.mlp.experts.202.gate_proj", + "model.layers.23.mlp.experts.202.up_proj", + "model.layers.23.mlp.experts.203.down_proj", + "model.layers.23.mlp.experts.203.gate_proj", + "model.layers.23.mlp.experts.203.up_proj", + "model.layers.23.mlp.experts.204.down_proj", + "model.layers.23.mlp.experts.204.gate_proj", + "model.layers.23.mlp.experts.204.up_proj", + "model.layers.23.mlp.experts.205.down_proj", + "model.layers.23.mlp.experts.205.gate_proj", + "model.layers.23.mlp.experts.205.up_proj", + "model.layers.23.mlp.experts.206.down_proj", + "model.layers.23.mlp.experts.206.gate_proj", + "model.layers.23.mlp.experts.206.up_proj", + "model.layers.23.mlp.experts.207.down_proj", + "model.layers.23.mlp.experts.207.gate_proj", + "model.layers.23.mlp.experts.207.up_proj", + "model.layers.23.mlp.experts.208.down_proj", + "model.layers.23.mlp.experts.208.gate_proj", + "model.layers.23.mlp.experts.208.up_proj", + "model.layers.23.mlp.experts.209.down_proj", + "model.layers.23.mlp.experts.209.gate_proj", + "model.layers.23.mlp.experts.209.up_proj", + "model.layers.23.mlp.experts.21.down_proj", + "model.layers.23.mlp.experts.21.gate_proj", + "model.layers.23.mlp.experts.21.up_proj", + "model.layers.23.mlp.experts.210.down_proj", + "model.layers.23.mlp.experts.210.gate_proj", + "model.layers.23.mlp.experts.210.up_proj", + "model.layers.23.mlp.experts.211.down_proj", + "model.layers.23.mlp.experts.211.gate_proj", + "model.layers.23.mlp.experts.211.up_proj", + "model.layers.23.mlp.experts.212.down_proj", + "model.layers.23.mlp.experts.212.gate_proj", + "model.layers.23.mlp.experts.212.up_proj", + "model.layers.23.mlp.experts.213.down_proj", + "model.layers.23.mlp.experts.213.gate_proj", + "model.layers.23.mlp.experts.213.up_proj", + "model.layers.23.mlp.experts.214.down_proj", + "model.layers.23.mlp.experts.214.gate_proj", + "model.layers.23.mlp.experts.214.up_proj", + "model.layers.23.mlp.experts.215.down_proj", + "model.layers.23.mlp.experts.215.gate_proj", + "model.layers.23.mlp.experts.215.up_proj", + "model.layers.23.mlp.experts.216.down_proj", + "model.layers.23.mlp.experts.216.gate_proj", + "model.layers.23.mlp.experts.216.up_proj", + "model.layers.23.mlp.experts.217.down_proj", + "model.layers.23.mlp.experts.217.gate_proj", + "model.layers.23.mlp.experts.217.up_proj", + "model.layers.23.mlp.experts.218.down_proj", + "model.layers.23.mlp.experts.218.gate_proj", + "model.layers.23.mlp.experts.218.up_proj", + "model.layers.23.mlp.experts.219.down_proj", + "model.layers.23.mlp.experts.219.gate_proj", + "model.layers.23.mlp.experts.219.up_proj", + "model.layers.23.mlp.experts.22.down_proj", + "model.layers.23.mlp.experts.22.gate_proj", + "model.layers.23.mlp.experts.22.up_proj", + "model.layers.23.mlp.experts.220.down_proj", + "model.layers.23.mlp.experts.220.gate_proj", + "model.layers.23.mlp.experts.220.up_proj", + "model.layers.23.mlp.experts.221.down_proj", + "model.layers.23.mlp.experts.221.gate_proj", + "model.layers.23.mlp.experts.221.up_proj", + "model.layers.23.mlp.experts.222.down_proj", + "model.layers.23.mlp.experts.222.gate_proj", + "model.layers.23.mlp.experts.222.up_proj", + "model.layers.23.mlp.experts.223.down_proj", + "model.layers.23.mlp.experts.223.gate_proj", + "model.layers.23.mlp.experts.223.up_proj", + "model.layers.23.mlp.experts.224.down_proj", + "model.layers.23.mlp.experts.224.gate_proj", + "model.layers.23.mlp.experts.224.up_proj", + "model.layers.23.mlp.experts.225.down_proj", + "model.layers.23.mlp.experts.225.gate_proj", + "model.layers.23.mlp.experts.225.up_proj", + "model.layers.23.mlp.experts.226.down_proj", + "model.layers.23.mlp.experts.226.gate_proj", + "model.layers.23.mlp.experts.226.up_proj", + "model.layers.23.mlp.experts.227.down_proj", + "model.layers.23.mlp.experts.227.gate_proj", + "model.layers.23.mlp.experts.227.up_proj", + "model.layers.23.mlp.experts.228.down_proj", + "model.layers.23.mlp.experts.228.gate_proj", + "model.layers.23.mlp.experts.228.up_proj", + "model.layers.23.mlp.experts.229.down_proj", + "model.layers.23.mlp.experts.229.gate_proj", + "model.layers.23.mlp.experts.229.up_proj", + "model.layers.23.mlp.experts.23.down_proj", + "model.layers.23.mlp.experts.23.gate_proj", + "model.layers.23.mlp.experts.23.up_proj", + "model.layers.23.mlp.experts.230.down_proj", + "model.layers.23.mlp.experts.230.gate_proj", + "model.layers.23.mlp.experts.230.up_proj", + "model.layers.23.mlp.experts.231.down_proj", + "model.layers.23.mlp.experts.231.gate_proj", + "model.layers.23.mlp.experts.231.up_proj", + "model.layers.23.mlp.experts.232.down_proj", + "model.layers.23.mlp.experts.232.gate_proj", + "model.layers.23.mlp.experts.232.up_proj", + "model.layers.23.mlp.experts.233.down_proj", + "model.layers.23.mlp.experts.233.gate_proj", + "model.layers.23.mlp.experts.233.up_proj", + "model.layers.23.mlp.experts.234.down_proj", + "model.layers.23.mlp.experts.234.gate_proj", + "model.layers.23.mlp.experts.234.up_proj", + "model.layers.23.mlp.experts.235.down_proj", + "model.layers.23.mlp.experts.235.gate_proj", + "model.layers.23.mlp.experts.235.up_proj", + "model.layers.23.mlp.experts.236.down_proj", + "model.layers.23.mlp.experts.236.gate_proj", + "model.layers.23.mlp.experts.236.up_proj", + "model.layers.23.mlp.experts.237.down_proj", + "model.layers.23.mlp.experts.237.gate_proj", + "model.layers.23.mlp.experts.237.up_proj", + "model.layers.23.mlp.experts.238.down_proj", + "model.layers.23.mlp.experts.238.gate_proj", + "model.layers.23.mlp.experts.238.up_proj", + "model.layers.23.mlp.experts.239.down_proj", + "model.layers.23.mlp.experts.239.gate_proj", + "model.layers.23.mlp.experts.239.up_proj", + "model.layers.23.mlp.experts.24.down_proj", + "model.layers.23.mlp.experts.24.gate_proj", + "model.layers.23.mlp.experts.24.up_proj", + "model.layers.23.mlp.experts.240.down_proj", + "model.layers.23.mlp.experts.240.gate_proj", + "model.layers.23.mlp.experts.240.up_proj", + "model.layers.23.mlp.experts.241.down_proj", + "model.layers.23.mlp.experts.241.gate_proj", + "model.layers.23.mlp.experts.241.up_proj", + "model.layers.23.mlp.experts.242.down_proj", + "model.layers.23.mlp.experts.242.gate_proj", + "model.layers.23.mlp.experts.242.up_proj", + "model.layers.23.mlp.experts.243.down_proj", + "model.layers.23.mlp.experts.243.gate_proj", + "model.layers.23.mlp.experts.243.up_proj", + "model.layers.23.mlp.experts.244.down_proj", + "model.layers.23.mlp.experts.244.gate_proj", + "model.layers.23.mlp.experts.244.up_proj", + "model.layers.23.mlp.experts.245.down_proj", + "model.layers.23.mlp.experts.245.gate_proj", + "model.layers.23.mlp.experts.245.up_proj", + "model.layers.23.mlp.experts.246.down_proj", + "model.layers.23.mlp.experts.246.gate_proj", + "model.layers.23.mlp.experts.246.up_proj", + "model.layers.23.mlp.experts.247.down_proj", + "model.layers.23.mlp.experts.247.gate_proj", + "model.layers.23.mlp.experts.247.up_proj", + "model.layers.23.mlp.experts.248.down_proj", + "model.layers.23.mlp.experts.248.gate_proj", + "model.layers.23.mlp.experts.248.up_proj", + "model.layers.23.mlp.experts.249.down_proj", + "model.layers.23.mlp.experts.249.gate_proj", + "model.layers.23.mlp.experts.249.up_proj", + "model.layers.23.mlp.experts.25.down_proj", + "model.layers.23.mlp.experts.25.gate_proj", + "model.layers.23.mlp.experts.25.up_proj", + "model.layers.23.mlp.experts.250.down_proj", + "model.layers.23.mlp.experts.250.gate_proj", + "model.layers.23.mlp.experts.250.up_proj", + "model.layers.23.mlp.experts.251.down_proj", + "model.layers.23.mlp.experts.251.gate_proj", + "model.layers.23.mlp.experts.251.up_proj", + "model.layers.23.mlp.experts.252.down_proj", + "model.layers.23.mlp.experts.252.gate_proj", + "model.layers.23.mlp.experts.252.up_proj", + "model.layers.23.mlp.experts.253.down_proj", + "model.layers.23.mlp.experts.253.gate_proj", + "model.layers.23.mlp.experts.253.up_proj", + "model.layers.23.mlp.experts.254.down_proj", + "model.layers.23.mlp.experts.254.gate_proj", + "model.layers.23.mlp.experts.254.up_proj", + "model.layers.23.mlp.experts.255.down_proj", + "model.layers.23.mlp.experts.255.gate_proj", + "model.layers.23.mlp.experts.255.up_proj", + "model.layers.23.mlp.experts.26.down_proj", + "model.layers.23.mlp.experts.26.gate_proj", + "model.layers.23.mlp.experts.26.up_proj", + "model.layers.23.mlp.experts.27.down_proj", + "model.layers.23.mlp.experts.27.gate_proj", + "model.layers.23.mlp.experts.27.up_proj", + "model.layers.23.mlp.experts.28.down_proj", + "model.layers.23.mlp.experts.28.gate_proj", + "model.layers.23.mlp.experts.28.up_proj", + "model.layers.23.mlp.experts.29.down_proj", + "model.layers.23.mlp.experts.29.gate_proj", + "model.layers.23.mlp.experts.29.up_proj", + "model.layers.23.mlp.experts.3.down_proj", + "model.layers.23.mlp.experts.3.gate_proj", + "model.layers.23.mlp.experts.3.up_proj", + "model.layers.23.mlp.experts.30.down_proj", + "model.layers.23.mlp.experts.30.gate_proj", + "model.layers.23.mlp.experts.30.up_proj", + "model.layers.23.mlp.experts.31.down_proj", + "model.layers.23.mlp.experts.31.gate_proj", + "model.layers.23.mlp.experts.31.up_proj", + "model.layers.23.mlp.experts.32.down_proj", + "model.layers.23.mlp.experts.32.gate_proj", + "model.layers.23.mlp.experts.32.up_proj", + "model.layers.23.mlp.experts.33.down_proj", + "model.layers.23.mlp.experts.33.gate_proj", + "model.layers.23.mlp.experts.33.up_proj", + "model.layers.23.mlp.experts.34.down_proj", + "model.layers.23.mlp.experts.34.gate_proj", + "model.layers.23.mlp.experts.34.up_proj", + "model.layers.23.mlp.experts.35.down_proj", + "model.layers.23.mlp.experts.35.gate_proj", + "model.layers.23.mlp.experts.35.up_proj", + "model.layers.23.mlp.experts.36.down_proj", + "model.layers.23.mlp.experts.36.gate_proj", + "model.layers.23.mlp.experts.36.up_proj", + "model.layers.23.mlp.experts.37.down_proj", + "model.layers.23.mlp.experts.37.gate_proj", + "model.layers.23.mlp.experts.37.up_proj", + "model.layers.23.mlp.experts.38.down_proj", + "model.layers.23.mlp.experts.38.gate_proj", + "model.layers.23.mlp.experts.38.up_proj", + "model.layers.23.mlp.experts.39.down_proj", + "model.layers.23.mlp.experts.39.gate_proj", + "model.layers.23.mlp.experts.39.up_proj", + "model.layers.23.mlp.experts.4.down_proj", + "model.layers.23.mlp.experts.4.gate_proj", + "model.layers.23.mlp.experts.4.up_proj", + "model.layers.23.mlp.experts.40.down_proj", + "model.layers.23.mlp.experts.40.gate_proj", + "model.layers.23.mlp.experts.40.up_proj", + "model.layers.23.mlp.experts.41.down_proj", + "model.layers.23.mlp.experts.41.gate_proj", + "model.layers.23.mlp.experts.41.up_proj", + "model.layers.23.mlp.experts.42.down_proj", + "model.layers.23.mlp.experts.42.gate_proj", + "model.layers.23.mlp.experts.42.up_proj", + "model.layers.23.mlp.experts.43.down_proj", + "model.layers.23.mlp.experts.43.gate_proj", + "model.layers.23.mlp.experts.43.up_proj", + "model.layers.23.mlp.experts.44.down_proj", + "model.layers.23.mlp.experts.44.gate_proj", + "model.layers.23.mlp.experts.44.up_proj", + "model.layers.23.mlp.experts.45.down_proj", + "model.layers.23.mlp.experts.45.gate_proj", + "model.layers.23.mlp.experts.45.up_proj", + "model.layers.23.mlp.experts.46.down_proj", + "model.layers.23.mlp.experts.46.gate_proj", + "model.layers.23.mlp.experts.46.up_proj", + "model.layers.23.mlp.experts.47.down_proj", + "model.layers.23.mlp.experts.47.gate_proj", + "model.layers.23.mlp.experts.47.up_proj", + "model.layers.23.mlp.experts.48.down_proj", + "model.layers.23.mlp.experts.48.gate_proj", + "model.layers.23.mlp.experts.48.up_proj", + "model.layers.23.mlp.experts.49.down_proj", + "model.layers.23.mlp.experts.49.gate_proj", + "model.layers.23.mlp.experts.49.up_proj", + "model.layers.23.mlp.experts.5.down_proj", + "model.layers.23.mlp.experts.5.gate_proj", + "model.layers.23.mlp.experts.5.up_proj", + "model.layers.23.mlp.experts.50.down_proj", + "model.layers.23.mlp.experts.50.gate_proj", + "model.layers.23.mlp.experts.50.up_proj", + "model.layers.23.mlp.experts.51.down_proj", + "model.layers.23.mlp.experts.51.gate_proj", + "model.layers.23.mlp.experts.51.up_proj", + "model.layers.23.mlp.experts.52.down_proj", + "model.layers.23.mlp.experts.52.gate_proj", + "model.layers.23.mlp.experts.52.up_proj", + "model.layers.23.mlp.experts.53.down_proj", + "model.layers.23.mlp.experts.53.gate_proj", + "model.layers.23.mlp.experts.53.up_proj", + "model.layers.23.mlp.experts.54.down_proj", + "model.layers.23.mlp.experts.54.gate_proj", + "model.layers.23.mlp.experts.54.up_proj", + "model.layers.23.mlp.experts.55.down_proj", + "model.layers.23.mlp.experts.55.gate_proj", + "model.layers.23.mlp.experts.55.up_proj", + "model.layers.23.mlp.experts.56.down_proj", + "model.layers.23.mlp.experts.56.gate_proj", + "model.layers.23.mlp.experts.56.up_proj", + "model.layers.23.mlp.experts.57.down_proj", + "model.layers.23.mlp.experts.57.gate_proj", + "model.layers.23.mlp.experts.57.up_proj", + "model.layers.23.mlp.experts.58.down_proj", + "model.layers.23.mlp.experts.58.gate_proj", + "model.layers.23.mlp.experts.58.up_proj", + "model.layers.23.mlp.experts.59.down_proj", + "model.layers.23.mlp.experts.59.gate_proj", + "model.layers.23.mlp.experts.59.up_proj", + "model.layers.23.mlp.experts.6.down_proj", + "model.layers.23.mlp.experts.6.gate_proj", + "model.layers.23.mlp.experts.6.up_proj", + "model.layers.23.mlp.experts.60.down_proj", + "model.layers.23.mlp.experts.60.gate_proj", + "model.layers.23.mlp.experts.60.up_proj", + "model.layers.23.mlp.experts.61.down_proj", + "model.layers.23.mlp.experts.61.gate_proj", + "model.layers.23.mlp.experts.61.up_proj", + "model.layers.23.mlp.experts.62.down_proj", + "model.layers.23.mlp.experts.62.gate_proj", + "model.layers.23.mlp.experts.62.up_proj", + "model.layers.23.mlp.experts.63.down_proj", + "model.layers.23.mlp.experts.63.gate_proj", + "model.layers.23.mlp.experts.63.up_proj", + "model.layers.23.mlp.experts.64.down_proj", + "model.layers.23.mlp.experts.64.gate_proj", + "model.layers.23.mlp.experts.64.up_proj", + "model.layers.23.mlp.experts.65.down_proj", + "model.layers.23.mlp.experts.65.gate_proj", + "model.layers.23.mlp.experts.65.up_proj", + "model.layers.23.mlp.experts.66.down_proj", + "model.layers.23.mlp.experts.66.gate_proj", + "model.layers.23.mlp.experts.66.up_proj", + "model.layers.23.mlp.experts.67.down_proj", + "model.layers.23.mlp.experts.67.gate_proj", + "model.layers.23.mlp.experts.67.up_proj", + "model.layers.23.mlp.experts.68.down_proj", + "model.layers.23.mlp.experts.68.gate_proj", + "model.layers.23.mlp.experts.68.up_proj", + "model.layers.23.mlp.experts.69.down_proj", + "model.layers.23.mlp.experts.69.gate_proj", + "model.layers.23.mlp.experts.69.up_proj", + "model.layers.23.mlp.experts.7.down_proj", + "model.layers.23.mlp.experts.7.gate_proj", + "model.layers.23.mlp.experts.7.up_proj", + "model.layers.23.mlp.experts.70.down_proj", + "model.layers.23.mlp.experts.70.gate_proj", + "model.layers.23.mlp.experts.70.up_proj", + "model.layers.23.mlp.experts.71.down_proj", + "model.layers.23.mlp.experts.71.gate_proj", + "model.layers.23.mlp.experts.71.up_proj", + "model.layers.23.mlp.experts.72.down_proj", + "model.layers.23.mlp.experts.72.gate_proj", + "model.layers.23.mlp.experts.72.up_proj", + "model.layers.23.mlp.experts.73.down_proj", + "model.layers.23.mlp.experts.73.gate_proj", + "model.layers.23.mlp.experts.73.up_proj", + "model.layers.23.mlp.experts.74.down_proj", + "model.layers.23.mlp.experts.74.gate_proj", + "model.layers.23.mlp.experts.74.up_proj", + "model.layers.23.mlp.experts.75.down_proj", + "model.layers.23.mlp.experts.75.gate_proj", + "model.layers.23.mlp.experts.75.up_proj", + "model.layers.23.mlp.experts.76.down_proj", + "model.layers.23.mlp.experts.76.gate_proj", + "model.layers.23.mlp.experts.76.up_proj", + "model.layers.23.mlp.experts.77.down_proj", + "model.layers.23.mlp.experts.77.gate_proj", + "model.layers.23.mlp.experts.77.up_proj", + "model.layers.23.mlp.experts.78.down_proj", + "model.layers.23.mlp.experts.78.gate_proj", + "model.layers.23.mlp.experts.78.up_proj", + "model.layers.23.mlp.experts.79.down_proj", + "model.layers.23.mlp.experts.79.gate_proj", + "model.layers.23.mlp.experts.79.up_proj", + "model.layers.23.mlp.experts.8.down_proj", + "model.layers.23.mlp.experts.8.gate_proj", + "model.layers.23.mlp.experts.8.up_proj", + "model.layers.23.mlp.experts.80.down_proj", + "model.layers.23.mlp.experts.80.gate_proj", + "model.layers.23.mlp.experts.80.up_proj", + "model.layers.23.mlp.experts.81.down_proj", + "model.layers.23.mlp.experts.81.gate_proj", + "model.layers.23.mlp.experts.81.up_proj", + "model.layers.23.mlp.experts.82.down_proj", + "model.layers.23.mlp.experts.82.gate_proj", + "model.layers.23.mlp.experts.82.up_proj", + "model.layers.23.mlp.experts.83.down_proj", + "model.layers.23.mlp.experts.83.gate_proj", + "model.layers.23.mlp.experts.83.up_proj", + "model.layers.23.mlp.experts.84.down_proj", + "model.layers.23.mlp.experts.84.gate_proj", + "model.layers.23.mlp.experts.84.up_proj", + "model.layers.23.mlp.experts.85.down_proj", + "model.layers.23.mlp.experts.85.gate_proj", + "model.layers.23.mlp.experts.85.up_proj", + "model.layers.23.mlp.experts.86.down_proj", + "model.layers.23.mlp.experts.86.gate_proj", + "model.layers.23.mlp.experts.86.up_proj", + "model.layers.23.mlp.experts.87.down_proj", + "model.layers.23.mlp.experts.87.gate_proj", + "model.layers.23.mlp.experts.87.up_proj", + "model.layers.23.mlp.experts.88.down_proj", + "model.layers.23.mlp.experts.88.gate_proj", + "model.layers.23.mlp.experts.88.up_proj", + "model.layers.23.mlp.experts.89.down_proj", + "model.layers.23.mlp.experts.89.gate_proj", + "model.layers.23.mlp.experts.89.up_proj", + "model.layers.23.mlp.experts.9.down_proj", + "model.layers.23.mlp.experts.9.gate_proj", + "model.layers.23.mlp.experts.9.up_proj", + "model.layers.23.mlp.experts.90.down_proj", + "model.layers.23.mlp.experts.90.gate_proj", + "model.layers.23.mlp.experts.90.up_proj", + "model.layers.23.mlp.experts.91.down_proj", + "model.layers.23.mlp.experts.91.gate_proj", + "model.layers.23.mlp.experts.91.up_proj", + "model.layers.23.mlp.experts.92.down_proj", + "model.layers.23.mlp.experts.92.gate_proj", + "model.layers.23.mlp.experts.92.up_proj", + "model.layers.23.mlp.experts.93.down_proj", + "model.layers.23.mlp.experts.93.gate_proj", + "model.layers.23.mlp.experts.93.up_proj", + "model.layers.23.mlp.experts.94.down_proj", + "model.layers.23.mlp.experts.94.gate_proj", + "model.layers.23.mlp.experts.94.up_proj", + "model.layers.23.mlp.experts.95.down_proj", + "model.layers.23.mlp.experts.95.gate_proj", + "model.layers.23.mlp.experts.95.up_proj", + "model.layers.23.mlp.experts.96.down_proj", + "model.layers.23.mlp.experts.96.gate_proj", + "model.layers.23.mlp.experts.96.up_proj", + "model.layers.23.mlp.experts.97.down_proj", + "model.layers.23.mlp.experts.97.gate_proj", + "model.layers.23.mlp.experts.97.up_proj", + "model.layers.23.mlp.experts.98.down_proj", + "model.layers.23.mlp.experts.98.gate_proj", + "model.layers.23.mlp.experts.98.up_proj", + "model.layers.23.mlp.experts.99.down_proj", + "model.layers.23.mlp.experts.99.gate_proj", + "model.layers.23.mlp.experts.99.up_proj", + "model.layers.24.mlp.experts.0.down_proj", + "model.layers.24.mlp.experts.0.gate_proj", + "model.layers.24.mlp.experts.0.up_proj", + "model.layers.24.mlp.experts.1.down_proj", + "model.layers.24.mlp.experts.1.gate_proj", + "model.layers.24.mlp.experts.1.up_proj", + "model.layers.24.mlp.experts.10.down_proj", + "model.layers.24.mlp.experts.10.gate_proj", + "model.layers.24.mlp.experts.10.up_proj", + "model.layers.24.mlp.experts.100.down_proj", + "model.layers.24.mlp.experts.100.gate_proj", + "model.layers.24.mlp.experts.100.up_proj", + "model.layers.24.mlp.experts.101.down_proj", + "model.layers.24.mlp.experts.101.gate_proj", + "model.layers.24.mlp.experts.101.up_proj", + "model.layers.24.mlp.experts.102.down_proj", + "model.layers.24.mlp.experts.102.gate_proj", + "model.layers.24.mlp.experts.102.up_proj", + "model.layers.24.mlp.experts.103.down_proj", + "model.layers.24.mlp.experts.103.gate_proj", + "model.layers.24.mlp.experts.103.up_proj", + "model.layers.24.mlp.experts.104.down_proj", + "model.layers.24.mlp.experts.104.gate_proj", + "model.layers.24.mlp.experts.104.up_proj", + "model.layers.24.mlp.experts.105.down_proj", + "model.layers.24.mlp.experts.105.gate_proj", + "model.layers.24.mlp.experts.105.up_proj", + "model.layers.24.mlp.experts.106.down_proj", + "model.layers.24.mlp.experts.106.gate_proj", + "model.layers.24.mlp.experts.106.up_proj", + "model.layers.24.mlp.experts.107.down_proj", + "model.layers.24.mlp.experts.107.gate_proj", + "model.layers.24.mlp.experts.107.up_proj", + "model.layers.24.mlp.experts.108.down_proj", + "model.layers.24.mlp.experts.108.gate_proj", + "model.layers.24.mlp.experts.108.up_proj", + "model.layers.24.mlp.experts.109.down_proj", + "model.layers.24.mlp.experts.109.gate_proj", + "model.layers.24.mlp.experts.109.up_proj", + "model.layers.24.mlp.experts.11.down_proj", + "model.layers.24.mlp.experts.11.gate_proj", + "model.layers.24.mlp.experts.11.up_proj", + "model.layers.24.mlp.experts.110.down_proj", + "model.layers.24.mlp.experts.110.gate_proj", + "model.layers.24.mlp.experts.110.up_proj", + "model.layers.24.mlp.experts.111.down_proj", + "model.layers.24.mlp.experts.111.gate_proj", + "model.layers.24.mlp.experts.111.up_proj", + "model.layers.24.mlp.experts.112.down_proj", + "model.layers.24.mlp.experts.112.gate_proj", + "model.layers.24.mlp.experts.112.up_proj", + "model.layers.24.mlp.experts.113.down_proj", + "model.layers.24.mlp.experts.113.gate_proj", + "model.layers.24.mlp.experts.113.up_proj", + "model.layers.24.mlp.experts.114.down_proj", + "model.layers.24.mlp.experts.114.gate_proj", + "model.layers.24.mlp.experts.114.up_proj", + "model.layers.24.mlp.experts.115.down_proj", + "model.layers.24.mlp.experts.115.gate_proj", + "model.layers.24.mlp.experts.115.up_proj", + "model.layers.24.mlp.experts.116.down_proj", + "model.layers.24.mlp.experts.116.gate_proj", + "model.layers.24.mlp.experts.116.up_proj", + "model.layers.24.mlp.experts.117.down_proj", + "model.layers.24.mlp.experts.117.gate_proj", + "model.layers.24.mlp.experts.117.up_proj", + "model.layers.24.mlp.experts.118.down_proj", + "model.layers.24.mlp.experts.118.gate_proj", + "model.layers.24.mlp.experts.118.up_proj", + "model.layers.24.mlp.experts.119.down_proj", + "model.layers.24.mlp.experts.119.gate_proj", + "model.layers.24.mlp.experts.119.up_proj", + "model.layers.24.mlp.experts.12.down_proj", + "model.layers.24.mlp.experts.12.gate_proj", + "model.layers.24.mlp.experts.12.up_proj", + "model.layers.24.mlp.experts.120.down_proj", + "model.layers.24.mlp.experts.120.gate_proj", + "model.layers.24.mlp.experts.120.up_proj", + "model.layers.24.mlp.experts.121.down_proj", + "model.layers.24.mlp.experts.121.gate_proj", + "model.layers.24.mlp.experts.121.up_proj", + "model.layers.24.mlp.experts.122.down_proj", + "model.layers.24.mlp.experts.122.gate_proj", + "model.layers.24.mlp.experts.122.up_proj", + "model.layers.24.mlp.experts.123.down_proj", + "model.layers.24.mlp.experts.123.gate_proj", + "model.layers.24.mlp.experts.123.up_proj", + "model.layers.24.mlp.experts.124.down_proj", + "model.layers.24.mlp.experts.124.gate_proj", + "model.layers.24.mlp.experts.124.up_proj", + "model.layers.24.mlp.experts.125.down_proj", + "model.layers.24.mlp.experts.125.gate_proj", + "model.layers.24.mlp.experts.125.up_proj", + "model.layers.24.mlp.experts.126.down_proj", + "model.layers.24.mlp.experts.126.gate_proj", + "model.layers.24.mlp.experts.126.up_proj", + "model.layers.24.mlp.experts.127.down_proj", + "model.layers.24.mlp.experts.127.gate_proj", + "model.layers.24.mlp.experts.127.up_proj", + "model.layers.24.mlp.experts.128.down_proj", + "model.layers.24.mlp.experts.128.gate_proj", + "model.layers.24.mlp.experts.128.up_proj", + "model.layers.24.mlp.experts.129.down_proj", + "model.layers.24.mlp.experts.129.gate_proj", + "model.layers.24.mlp.experts.129.up_proj", + "model.layers.24.mlp.experts.13.down_proj", + "model.layers.24.mlp.experts.13.gate_proj", + "model.layers.24.mlp.experts.13.up_proj", + "model.layers.24.mlp.experts.130.down_proj", + "model.layers.24.mlp.experts.130.gate_proj", + "model.layers.24.mlp.experts.130.up_proj", + "model.layers.24.mlp.experts.131.down_proj", + "model.layers.24.mlp.experts.131.gate_proj", + "model.layers.24.mlp.experts.131.up_proj", + "model.layers.24.mlp.experts.132.down_proj", + "model.layers.24.mlp.experts.132.gate_proj", + "model.layers.24.mlp.experts.132.up_proj", + "model.layers.24.mlp.experts.133.down_proj", + "model.layers.24.mlp.experts.133.gate_proj", + "model.layers.24.mlp.experts.133.up_proj", + "model.layers.24.mlp.experts.134.down_proj", + "model.layers.24.mlp.experts.134.gate_proj", + "model.layers.24.mlp.experts.134.up_proj", + "model.layers.24.mlp.experts.135.down_proj", + "model.layers.24.mlp.experts.135.gate_proj", + "model.layers.24.mlp.experts.135.up_proj", + "model.layers.24.mlp.experts.136.down_proj", + "model.layers.24.mlp.experts.136.gate_proj", + "model.layers.24.mlp.experts.136.up_proj", + "model.layers.24.mlp.experts.137.down_proj", + "model.layers.24.mlp.experts.137.gate_proj", + "model.layers.24.mlp.experts.137.up_proj", + "model.layers.24.mlp.experts.138.down_proj", + "model.layers.24.mlp.experts.138.gate_proj", + "model.layers.24.mlp.experts.138.up_proj", + "model.layers.24.mlp.experts.139.down_proj", + "model.layers.24.mlp.experts.139.gate_proj", + "model.layers.24.mlp.experts.139.up_proj", + "model.layers.24.mlp.experts.14.down_proj", + "model.layers.24.mlp.experts.14.gate_proj", + "model.layers.24.mlp.experts.14.up_proj", + "model.layers.24.mlp.experts.140.down_proj", + "model.layers.24.mlp.experts.140.gate_proj", + "model.layers.24.mlp.experts.140.up_proj", + "model.layers.24.mlp.experts.141.down_proj", + "model.layers.24.mlp.experts.141.gate_proj", + "model.layers.24.mlp.experts.141.up_proj", + "model.layers.24.mlp.experts.142.down_proj", + "model.layers.24.mlp.experts.142.gate_proj", + "model.layers.24.mlp.experts.142.up_proj", + "model.layers.24.mlp.experts.143.down_proj", + "model.layers.24.mlp.experts.143.gate_proj", + "model.layers.24.mlp.experts.143.up_proj", + "model.layers.24.mlp.experts.144.down_proj", + "model.layers.24.mlp.experts.144.gate_proj", + "model.layers.24.mlp.experts.144.up_proj", + "model.layers.24.mlp.experts.145.down_proj", + "model.layers.24.mlp.experts.145.gate_proj", + "model.layers.24.mlp.experts.145.up_proj", + "model.layers.24.mlp.experts.146.down_proj", + "model.layers.24.mlp.experts.146.gate_proj", + "model.layers.24.mlp.experts.146.up_proj", + "model.layers.24.mlp.experts.147.down_proj", + "model.layers.24.mlp.experts.147.gate_proj", + "model.layers.24.mlp.experts.147.up_proj", + "model.layers.24.mlp.experts.148.down_proj", + "model.layers.24.mlp.experts.148.gate_proj", + "model.layers.24.mlp.experts.148.up_proj", + "model.layers.24.mlp.experts.149.down_proj", + "model.layers.24.mlp.experts.149.gate_proj", + "model.layers.24.mlp.experts.149.up_proj", + "model.layers.24.mlp.experts.15.down_proj", + "model.layers.24.mlp.experts.15.gate_proj", + "model.layers.24.mlp.experts.15.up_proj", + "model.layers.24.mlp.experts.150.down_proj", + "model.layers.24.mlp.experts.150.gate_proj", + "model.layers.24.mlp.experts.150.up_proj", + "model.layers.24.mlp.experts.151.down_proj", + "model.layers.24.mlp.experts.151.gate_proj", + "model.layers.24.mlp.experts.151.up_proj", + "model.layers.24.mlp.experts.152.down_proj", + "model.layers.24.mlp.experts.152.gate_proj", + "model.layers.24.mlp.experts.152.up_proj", + "model.layers.24.mlp.experts.153.down_proj", + "model.layers.24.mlp.experts.153.gate_proj", + "model.layers.24.mlp.experts.153.up_proj", + "model.layers.24.mlp.experts.154.down_proj", + "model.layers.24.mlp.experts.154.gate_proj", + "model.layers.24.mlp.experts.154.up_proj", + "model.layers.24.mlp.experts.155.down_proj", + "model.layers.24.mlp.experts.155.gate_proj", + "model.layers.24.mlp.experts.155.up_proj", + "model.layers.24.mlp.experts.156.down_proj", + "model.layers.24.mlp.experts.156.gate_proj", + "model.layers.24.mlp.experts.156.up_proj", + "model.layers.24.mlp.experts.157.down_proj", + "model.layers.24.mlp.experts.157.gate_proj", + "model.layers.24.mlp.experts.157.up_proj", + "model.layers.24.mlp.experts.158.down_proj", + "model.layers.24.mlp.experts.158.gate_proj", + "model.layers.24.mlp.experts.158.up_proj", + "model.layers.24.mlp.experts.159.down_proj", + "model.layers.24.mlp.experts.159.gate_proj", + "model.layers.24.mlp.experts.159.up_proj", + "model.layers.24.mlp.experts.16.down_proj", + "model.layers.24.mlp.experts.16.gate_proj", + "model.layers.24.mlp.experts.16.up_proj", + "model.layers.24.mlp.experts.160.down_proj", + "model.layers.24.mlp.experts.160.gate_proj", + "model.layers.24.mlp.experts.160.up_proj", + "model.layers.24.mlp.experts.161.down_proj", + "model.layers.24.mlp.experts.161.gate_proj", + "model.layers.24.mlp.experts.161.up_proj", + "model.layers.24.mlp.experts.162.down_proj", + "model.layers.24.mlp.experts.162.gate_proj", + "model.layers.24.mlp.experts.162.up_proj", + "model.layers.24.mlp.experts.163.down_proj", + "model.layers.24.mlp.experts.163.gate_proj", + "model.layers.24.mlp.experts.163.up_proj", + "model.layers.24.mlp.experts.164.down_proj", + "model.layers.24.mlp.experts.164.gate_proj", + "model.layers.24.mlp.experts.164.up_proj", + "model.layers.24.mlp.experts.165.down_proj", + "model.layers.24.mlp.experts.165.gate_proj", + "model.layers.24.mlp.experts.165.up_proj", + "model.layers.24.mlp.experts.166.down_proj", + "model.layers.24.mlp.experts.166.gate_proj", + "model.layers.24.mlp.experts.166.up_proj", + "model.layers.24.mlp.experts.167.down_proj", + "model.layers.24.mlp.experts.167.gate_proj", + "model.layers.24.mlp.experts.167.up_proj", + "model.layers.24.mlp.experts.168.down_proj", + "model.layers.24.mlp.experts.168.gate_proj", + "model.layers.24.mlp.experts.168.up_proj", + "model.layers.24.mlp.experts.169.down_proj", + "model.layers.24.mlp.experts.169.gate_proj", + "model.layers.24.mlp.experts.169.up_proj", + "model.layers.24.mlp.experts.17.down_proj", + "model.layers.24.mlp.experts.17.gate_proj", + "model.layers.24.mlp.experts.17.up_proj", + "model.layers.24.mlp.experts.170.down_proj", + "model.layers.24.mlp.experts.170.gate_proj", + "model.layers.24.mlp.experts.170.up_proj", + "model.layers.24.mlp.experts.171.down_proj", + "model.layers.24.mlp.experts.171.gate_proj", + "model.layers.24.mlp.experts.171.up_proj", + "model.layers.24.mlp.experts.172.down_proj", + "model.layers.24.mlp.experts.172.gate_proj", + "model.layers.24.mlp.experts.172.up_proj", + "model.layers.24.mlp.experts.173.down_proj", + "model.layers.24.mlp.experts.173.gate_proj", + "model.layers.24.mlp.experts.173.up_proj", + "model.layers.24.mlp.experts.174.down_proj", + "model.layers.24.mlp.experts.174.gate_proj", + "model.layers.24.mlp.experts.174.up_proj", + "model.layers.24.mlp.experts.175.down_proj", + "model.layers.24.mlp.experts.175.gate_proj", + "model.layers.24.mlp.experts.175.up_proj", + "model.layers.24.mlp.experts.176.down_proj", + "model.layers.24.mlp.experts.176.gate_proj", + "model.layers.24.mlp.experts.176.up_proj", + "model.layers.24.mlp.experts.177.down_proj", + "model.layers.24.mlp.experts.177.gate_proj", + "model.layers.24.mlp.experts.177.up_proj", + "model.layers.24.mlp.experts.178.down_proj", + "model.layers.24.mlp.experts.178.gate_proj", + "model.layers.24.mlp.experts.178.up_proj", + "model.layers.24.mlp.experts.179.down_proj", + "model.layers.24.mlp.experts.179.gate_proj", + "model.layers.24.mlp.experts.179.up_proj", + "model.layers.24.mlp.experts.18.down_proj", + "model.layers.24.mlp.experts.18.gate_proj", + "model.layers.24.mlp.experts.18.up_proj", + "model.layers.24.mlp.experts.180.down_proj", + "model.layers.24.mlp.experts.180.gate_proj", + "model.layers.24.mlp.experts.180.up_proj", + "model.layers.24.mlp.experts.181.down_proj", + "model.layers.24.mlp.experts.181.gate_proj", + "model.layers.24.mlp.experts.181.up_proj", + "model.layers.24.mlp.experts.182.down_proj", + "model.layers.24.mlp.experts.182.gate_proj", + "model.layers.24.mlp.experts.182.up_proj", + "model.layers.24.mlp.experts.183.down_proj", + "model.layers.24.mlp.experts.183.gate_proj", + "model.layers.24.mlp.experts.183.up_proj", + "model.layers.24.mlp.experts.184.down_proj", + "model.layers.24.mlp.experts.184.gate_proj", + "model.layers.24.mlp.experts.184.up_proj", + "model.layers.24.mlp.experts.185.down_proj", + "model.layers.24.mlp.experts.185.gate_proj", + "model.layers.24.mlp.experts.185.up_proj", + "model.layers.24.mlp.experts.186.down_proj", + "model.layers.24.mlp.experts.186.gate_proj", + "model.layers.24.mlp.experts.186.up_proj", + "model.layers.24.mlp.experts.187.down_proj", + "model.layers.24.mlp.experts.187.gate_proj", + "model.layers.24.mlp.experts.187.up_proj", + "model.layers.24.mlp.experts.188.down_proj", + "model.layers.24.mlp.experts.188.gate_proj", + "model.layers.24.mlp.experts.188.up_proj", + "model.layers.24.mlp.experts.189.down_proj", + "model.layers.24.mlp.experts.189.gate_proj", + "model.layers.24.mlp.experts.189.up_proj", + "model.layers.24.mlp.experts.19.down_proj", + "model.layers.24.mlp.experts.19.gate_proj", + "model.layers.24.mlp.experts.19.up_proj", + "model.layers.24.mlp.experts.190.down_proj", + "model.layers.24.mlp.experts.190.gate_proj", + "model.layers.24.mlp.experts.190.up_proj", + "model.layers.24.mlp.experts.191.down_proj", + "model.layers.24.mlp.experts.191.gate_proj", + "model.layers.24.mlp.experts.191.up_proj", + "model.layers.24.mlp.experts.192.down_proj", + "model.layers.24.mlp.experts.192.gate_proj", + "model.layers.24.mlp.experts.192.up_proj", + "model.layers.24.mlp.experts.193.down_proj", + "model.layers.24.mlp.experts.193.gate_proj", + "model.layers.24.mlp.experts.193.up_proj", + "model.layers.24.mlp.experts.194.down_proj", + "model.layers.24.mlp.experts.194.gate_proj", + "model.layers.24.mlp.experts.194.up_proj", + "model.layers.24.mlp.experts.195.down_proj", + "model.layers.24.mlp.experts.195.gate_proj", + "model.layers.24.mlp.experts.195.up_proj", + "model.layers.24.mlp.experts.196.down_proj", + "model.layers.24.mlp.experts.196.gate_proj", + "model.layers.24.mlp.experts.196.up_proj", + "model.layers.24.mlp.experts.197.down_proj", + "model.layers.24.mlp.experts.197.gate_proj", + "model.layers.24.mlp.experts.197.up_proj", + "model.layers.24.mlp.experts.198.down_proj", + "model.layers.24.mlp.experts.198.gate_proj", + "model.layers.24.mlp.experts.198.up_proj", + "model.layers.24.mlp.experts.199.down_proj", + "model.layers.24.mlp.experts.199.gate_proj", + "model.layers.24.mlp.experts.199.up_proj", + "model.layers.24.mlp.experts.2.down_proj", + "model.layers.24.mlp.experts.2.gate_proj", + "model.layers.24.mlp.experts.2.up_proj", + "model.layers.24.mlp.experts.20.down_proj", + "model.layers.24.mlp.experts.20.gate_proj", + "model.layers.24.mlp.experts.20.up_proj", + "model.layers.24.mlp.experts.200.down_proj", + "model.layers.24.mlp.experts.200.gate_proj", + "model.layers.24.mlp.experts.200.up_proj", + "model.layers.24.mlp.experts.201.down_proj", + "model.layers.24.mlp.experts.201.gate_proj", + "model.layers.24.mlp.experts.201.up_proj", + "model.layers.24.mlp.experts.202.down_proj", + "model.layers.24.mlp.experts.202.gate_proj", + "model.layers.24.mlp.experts.202.up_proj", + "model.layers.24.mlp.experts.203.down_proj", + "model.layers.24.mlp.experts.203.gate_proj", + "model.layers.24.mlp.experts.203.up_proj", + "model.layers.24.mlp.experts.204.down_proj", + "model.layers.24.mlp.experts.204.gate_proj", + "model.layers.24.mlp.experts.204.up_proj", + "model.layers.24.mlp.experts.205.down_proj", + "model.layers.24.mlp.experts.205.gate_proj", + "model.layers.24.mlp.experts.205.up_proj", + "model.layers.24.mlp.experts.206.down_proj", + "model.layers.24.mlp.experts.206.gate_proj", + "model.layers.24.mlp.experts.206.up_proj", + "model.layers.24.mlp.experts.207.down_proj", + "model.layers.24.mlp.experts.207.gate_proj", + "model.layers.24.mlp.experts.207.up_proj", + "model.layers.24.mlp.experts.208.down_proj", + "model.layers.24.mlp.experts.208.gate_proj", + "model.layers.24.mlp.experts.208.up_proj", + "model.layers.24.mlp.experts.209.down_proj", + "model.layers.24.mlp.experts.209.gate_proj", + "model.layers.24.mlp.experts.209.up_proj", + "model.layers.24.mlp.experts.21.down_proj", + "model.layers.24.mlp.experts.21.gate_proj", + "model.layers.24.mlp.experts.21.up_proj", + "model.layers.24.mlp.experts.210.down_proj", + "model.layers.24.mlp.experts.210.gate_proj", + "model.layers.24.mlp.experts.210.up_proj", + "model.layers.24.mlp.experts.211.down_proj", + "model.layers.24.mlp.experts.211.gate_proj", + "model.layers.24.mlp.experts.211.up_proj", + "model.layers.24.mlp.experts.212.down_proj", + "model.layers.24.mlp.experts.212.gate_proj", + "model.layers.24.mlp.experts.212.up_proj", + "model.layers.24.mlp.experts.213.down_proj", + "model.layers.24.mlp.experts.213.gate_proj", + "model.layers.24.mlp.experts.213.up_proj", + "model.layers.24.mlp.experts.214.down_proj", + "model.layers.24.mlp.experts.214.gate_proj", + "model.layers.24.mlp.experts.214.up_proj", + "model.layers.24.mlp.experts.215.down_proj", + "model.layers.24.mlp.experts.215.gate_proj", + "model.layers.24.mlp.experts.215.up_proj", + "model.layers.24.mlp.experts.216.down_proj", + "model.layers.24.mlp.experts.216.gate_proj", + "model.layers.24.mlp.experts.216.up_proj", + "model.layers.24.mlp.experts.217.down_proj", + "model.layers.24.mlp.experts.217.gate_proj", + "model.layers.24.mlp.experts.217.up_proj", + "model.layers.24.mlp.experts.218.down_proj", + "model.layers.24.mlp.experts.218.gate_proj", + "model.layers.24.mlp.experts.218.up_proj", + "model.layers.24.mlp.experts.219.down_proj", + "model.layers.24.mlp.experts.219.gate_proj", + "model.layers.24.mlp.experts.219.up_proj", + "model.layers.24.mlp.experts.22.down_proj", + "model.layers.24.mlp.experts.22.gate_proj", + "model.layers.24.mlp.experts.22.up_proj", + "model.layers.24.mlp.experts.220.down_proj", + "model.layers.24.mlp.experts.220.gate_proj", + "model.layers.24.mlp.experts.220.up_proj", + "model.layers.24.mlp.experts.221.down_proj", + "model.layers.24.mlp.experts.221.gate_proj", + "model.layers.24.mlp.experts.221.up_proj", + "model.layers.24.mlp.experts.222.down_proj", + "model.layers.24.mlp.experts.222.gate_proj", + "model.layers.24.mlp.experts.222.up_proj", + "model.layers.24.mlp.experts.223.down_proj", + "model.layers.24.mlp.experts.223.gate_proj", + "model.layers.24.mlp.experts.223.up_proj", + "model.layers.24.mlp.experts.224.down_proj", + "model.layers.24.mlp.experts.224.gate_proj", + "model.layers.24.mlp.experts.224.up_proj", + "model.layers.24.mlp.experts.225.down_proj", + "model.layers.24.mlp.experts.225.gate_proj", + "model.layers.24.mlp.experts.225.up_proj", + "model.layers.24.mlp.experts.226.down_proj", + "model.layers.24.mlp.experts.226.gate_proj", + "model.layers.24.mlp.experts.226.up_proj", + "model.layers.24.mlp.experts.227.down_proj", + "model.layers.24.mlp.experts.227.gate_proj", + "model.layers.24.mlp.experts.227.up_proj", + "model.layers.24.mlp.experts.228.down_proj", + "model.layers.24.mlp.experts.228.gate_proj", + "model.layers.24.mlp.experts.228.up_proj", + "model.layers.24.mlp.experts.229.down_proj", + "model.layers.24.mlp.experts.229.gate_proj", + "model.layers.24.mlp.experts.229.up_proj", + "model.layers.24.mlp.experts.23.down_proj", + "model.layers.24.mlp.experts.23.gate_proj", + "model.layers.24.mlp.experts.23.up_proj", + "model.layers.24.mlp.experts.230.down_proj", + "model.layers.24.mlp.experts.230.gate_proj", + "model.layers.24.mlp.experts.230.up_proj", + "model.layers.24.mlp.experts.231.down_proj", + "model.layers.24.mlp.experts.231.gate_proj", + "model.layers.24.mlp.experts.231.up_proj", + "model.layers.24.mlp.experts.232.down_proj", + "model.layers.24.mlp.experts.232.gate_proj", + "model.layers.24.mlp.experts.232.up_proj", + "model.layers.24.mlp.experts.233.down_proj", + "model.layers.24.mlp.experts.233.gate_proj", + "model.layers.24.mlp.experts.233.up_proj", + "model.layers.24.mlp.experts.234.down_proj", + "model.layers.24.mlp.experts.234.gate_proj", + "model.layers.24.mlp.experts.234.up_proj", + "model.layers.24.mlp.experts.235.down_proj", + "model.layers.24.mlp.experts.235.gate_proj", + "model.layers.24.mlp.experts.235.up_proj", + "model.layers.24.mlp.experts.236.down_proj", + "model.layers.24.mlp.experts.236.gate_proj", + "model.layers.24.mlp.experts.236.up_proj", + "model.layers.24.mlp.experts.237.down_proj", + "model.layers.24.mlp.experts.237.gate_proj", + "model.layers.24.mlp.experts.237.up_proj", + "model.layers.24.mlp.experts.238.down_proj", + "model.layers.24.mlp.experts.238.gate_proj", + "model.layers.24.mlp.experts.238.up_proj", + "model.layers.24.mlp.experts.239.down_proj", + "model.layers.24.mlp.experts.239.gate_proj", + "model.layers.24.mlp.experts.239.up_proj", + "model.layers.24.mlp.experts.24.down_proj", + "model.layers.24.mlp.experts.24.gate_proj", + "model.layers.24.mlp.experts.24.up_proj", + "model.layers.24.mlp.experts.240.down_proj", + "model.layers.24.mlp.experts.240.gate_proj", + "model.layers.24.mlp.experts.240.up_proj", + "model.layers.24.mlp.experts.241.down_proj", + "model.layers.24.mlp.experts.241.gate_proj", + "model.layers.24.mlp.experts.241.up_proj", + "model.layers.24.mlp.experts.242.down_proj", + "model.layers.24.mlp.experts.242.gate_proj", + "model.layers.24.mlp.experts.242.up_proj", + "model.layers.24.mlp.experts.243.down_proj", + "model.layers.24.mlp.experts.243.gate_proj", + "model.layers.24.mlp.experts.243.up_proj", + "model.layers.24.mlp.experts.244.down_proj", + "model.layers.24.mlp.experts.244.gate_proj", + "model.layers.24.mlp.experts.244.up_proj", + "model.layers.24.mlp.experts.245.down_proj", + "model.layers.24.mlp.experts.245.gate_proj", + "model.layers.24.mlp.experts.245.up_proj", + "model.layers.24.mlp.experts.246.down_proj", + "model.layers.24.mlp.experts.246.gate_proj", + "model.layers.24.mlp.experts.246.up_proj", + "model.layers.24.mlp.experts.247.down_proj", + "model.layers.24.mlp.experts.247.gate_proj", + "model.layers.24.mlp.experts.247.up_proj", + "model.layers.24.mlp.experts.248.down_proj", + "model.layers.24.mlp.experts.248.gate_proj", + "model.layers.24.mlp.experts.248.up_proj", + "model.layers.24.mlp.experts.249.down_proj", + "model.layers.24.mlp.experts.249.gate_proj", + "model.layers.24.mlp.experts.249.up_proj", + "model.layers.24.mlp.experts.25.down_proj", + "model.layers.24.mlp.experts.25.gate_proj", + "model.layers.24.mlp.experts.25.up_proj", + "model.layers.24.mlp.experts.250.down_proj", + "model.layers.24.mlp.experts.250.gate_proj", + "model.layers.24.mlp.experts.250.up_proj", + "model.layers.24.mlp.experts.251.down_proj", + "model.layers.24.mlp.experts.251.gate_proj", + "model.layers.24.mlp.experts.251.up_proj", + "model.layers.24.mlp.experts.252.down_proj", + "model.layers.24.mlp.experts.252.gate_proj", + "model.layers.24.mlp.experts.252.up_proj", + "model.layers.24.mlp.experts.253.down_proj", + "model.layers.24.mlp.experts.253.gate_proj", + "model.layers.24.mlp.experts.253.up_proj", + "model.layers.24.mlp.experts.254.down_proj", + "model.layers.24.mlp.experts.254.gate_proj", + "model.layers.24.mlp.experts.254.up_proj", + "model.layers.24.mlp.experts.255.down_proj", + "model.layers.24.mlp.experts.255.gate_proj", + "model.layers.24.mlp.experts.255.up_proj", + "model.layers.24.mlp.experts.26.down_proj", + "model.layers.24.mlp.experts.26.gate_proj", + "model.layers.24.mlp.experts.26.up_proj", + "model.layers.24.mlp.experts.27.down_proj", + "model.layers.24.mlp.experts.27.gate_proj", + "model.layers.24.mlp.experts.27.up_proj", + "model.layers.24.mlp.experts.28.down_proj", + "model.layers.24.mlp.experts.28.gate_proj", + "model.layers.24.mlp.experts.28.up_proj", + "model.layers.24.mlp.experts.29.down_proj", + "model.layers.24.mlp.experts.29.gate_proj", + "model.layers.24.mlp.experts.29.up_proj", + "model.layers.24.mlp.experts.3.down_proj", + "model.layers.24.mlp.experts.3.gate_proj", + "model.layers.24.mlp.experts.3.up_proj", + "model.layers.24.mlp.experts.30.down_proj", + "model.layers.24.mlp.experts.30.gate_proj", + "model.layers.24.mlp.experts.30.up_proj", + "model.layers.24.mlp.experts.31.down_proj", + "model.layers.24.mlp.experts.31.gate_proj", + "model.layers.24.mlp.experts.31.up_proj", + "model.layers.24.mlp.experts.32.down_proj", + "model.layers.24.mlp.experts.32.gate_proj", + "model.layers.24.mlp.experts.32.up_proj", + "model.layers.24.mlp.experts.33.down_proj", + "model.layers.24.mlp.experts.33.gate_proj", + "model.layers.24.mlp.experts.33.up_proj", + "model.layers.24.mlp.experts.34.down_proj", + "model.layers.24.mlp.experts.34.gate_proj", + "model.layers.24.mlp.experts.34.up_proj", + "model.layers.24.mlp.experts.35.down_proj", + "model.layers.24.mlp.experts.35.gate_proj", + "model.layers.24.mlp.experts.35.up_proj", + "model.layers.24.mlp.experts.36.down_proj", + "model.layers.24.mlp.experts.36.gate_proj", + "model.layers.24.mlp.experts.36.up_proj", + "model.layers.24.mlp.experts.37.down_proj", + "model.layers.24.mlp.experts.37.gate_proj", + "model.layers.24.mlp.experts.37.up_proj", + "model.layers.24.mlp.experts.38.down_proj", + "model.layers.24.mlp.experts.38.gate_proj", + "model.layers.24.mlp.experts.38.up_proj", + "model.layers.24.mlp.experts.39.down_proj", + "model.layers.24.mlp.experts.39.gate_proj", + "model.layers.24.mlp.experts.39.up_proj", + "model.layers.24.mlp.experts.4.down_proj", + "model.layers.24.mlp.experts.4.gate_proj", + "model.layers.24.mlp.experts.4.up_proj", + "model.layers.24.mlp.experts.40.down_proj", + "model.layers.24.mlp.experts.40.gate_proj", + "model.layers.24.mlp.experts.40.up_proj", + "model.layers.24.mlp.experts.41.down_proj", + "model.layers.24.mlp.experts.41.gate_proj", + "model.layers.24.mlp.experts.41.up_proj", + "model.layers.24.mlp.experts.42.down_proj", + "model.layers.24.mlp.experts.42.gate_proj", + "model.layers.24.mlp.experts.42.up_proj", + "model.layers.24.mlp.experts.43.down_proj", + "model.layers.24.mlp.experts.43.gate_proj", + "model.layers.24.mlp.experts.43.up_proj", + "model.layers.24.mlp.experts.44.down_proj", + "model.layers.24.mlp.experts.44.gate_proj", + "model.layers.24.mlp.experts.44.up_proj", + "model.layers.24.mlp.experts.45.down_proj", + "model.layers.24.mlp.experts.45.gate_proj", + "model.layers.24.mlp.experts.45.up_proj", + "model.layers.24.mlp.experts.46.down_proj", + "model.layers.24.mlp.experts.46.gate_proj", + "model.layers.24.mlp.experts.46.up_proj", + "model.layers.24.mlp.experts.47.down_proj", + "model.layers.24.mlp.experts.47.gate_proj", + "model.layers.24.mlp.experts.47.up_proj", + "model.layers.24.mlp.experts.48.down_proj", + "model.layers.24.mlp.experts.48.gate_proj", + "model.layers.24.mlp.experts.48.up_proj", + "model.layers.24.mlp.experts.49.down_proj", + "model.layers.24.mlp.experts.49.gate_proj", + "model.layers.24.mlp.experts.49.up_proj", + "model.layers.24.mlp.experts.5.down_proj", + "model.layers.24.mlp.experts.5.gate_proj", + "model.layers.24.mlp.experts.5.up_proj", + "model.layers.24.mlp.experts.50.down_proj", + "model.layers.24.mlp.experts.50.gate_proj", + "model.layers.24.mlp.experts.50.up_proj", + "model.layers.24.mlp.experts.51.down_proj", + "model.layers.24.mlp.experts.51.gate_proj", + "model.layers.24.mlp.experts.51.up_proj", + "model.layers.24.mlp.experts.52.down_proj", + "model.layers.24.mlp.experts.52.gate_proj", + "model.layers.24.mlp.experts.52.up_proj", + "model.layers.24.mlp.experts.53.down_proj", + "model.layers.24.mlp.experts.53.gate_proj", + "model.layers.24.mlp.experts.53.up_proj", + "model.layers.24.mlp.experts.54.down_proj", + "model.layers.24.mlp.experts.54.gate_proj", + "model.layers.24.mlp.experts.54.up_proj", + "model.layers.24.mlp.experts.55.down_proj", + "model.layers.24.mlp.experts.55.gate_proj", + "model.layers.24.mlp.experts.55.up_proj", + "model.layers.24.mlp.experts.56.down_proj", + "model.layers.24.mlp.experts.56.gate_proj", + "model.layers.24.mlp.experts.56.up_proj", + "model.layers.24.mlp.experts.57.down_proj", + "model.layers.24.mlp.experts.57.gate_proj", + "model.layers.24.mlp.experts.57.up_proj", + "model.layers.24.mlp.experts.58.down_proj", + "model.layers.24.mlp.experts.58.gate_proj", + "model.layers.24.mlp.experts.58.up_proj", + "model.layers.24.mlp.experts.59.down_proj", + "model.layers.24.mlp.experts.59.gate_proj", + "model.layers.24.mlp.experts.59.up_proj", + "model.layers.24.mlp.experts.6.down_proj", + "model.layers.24.mlp.experts.6.gate_proj", + "model.layers.24.mlp.experts.6.up_proj", + "model.layers.24.mlp.experts.60.down_proj", + "model.layers.24.mlp.experts.60.gate_proj", + "model.layers.24.mlp.experts.60.up_proj", + "model.layers.24.mlp.experts.61.down_proj", + "model.layers.24.mlp.experts.61.gate_proj", + "model.layers.24.mlp.experts.61.up_proj", + "model.layers.24.mlp.experts.62.down_proj", + "model.layers.24.mlp.experts.62.gate_proj", + "model.layers.24.mlp.experts.62.up_proj", + "model.layers.24.mlp.experts.63.down_proj", + "model.layers.24.mlp.experts.63.gate_proj", + "model.layers.24.mlp.experts.63.up_proj", + "model.layers.24.mlp.experts.64.down_proj", + "model.layers.24.mlp.experts.64.gate_proj", + "model.layers.24.mlp.experts.64.up_proj", + "model.layers.24.mlp.experts.65.down_proj", + "model.layers.24.mlp.experts.65.gate_proj", + "model.layers.24.mlp.experts.65.up_proj", + "model.layers.24.mlp.experts.66.down_proj", + "model.layers.24.mlp.experts.66.gate_proj", + "model.layers.24.mlp.experts.66.up_proj", + "model.layers.24.mlp.experts.67.down_proj", + "model.layers.24.mlp.experts.67.gate_proj", + "model.layers.24.mlp.experts.67.up_proj", + "model.layers.24.mlp.experts.68.down_proj", + "model.layers.24.mlp.experts.68.gate_proj", + "model.layers.24.mlp.experts.68.up_proj", + "model.layers.24.mlp.experts.69.down_proj", + "model.layers.24.mlp.experts.69.gate_proj", + "model.layers.24.mlp.experts.69.up_proj", + "model.layers.24.mlp.experts.7.down_proj", + "model.layers.24.mlp.experts.7.gate_proj", + "model.layers.24.mlp.experts.7.up_proj", + "model.layers.24.mlp.experts.70.down_proj", + "model.layers.24.mlp.experts.70.gate_proj", + "model.layers.24.mlp.experts.70.up_proj", + "model.layers.24.mlp.experts.71.down_proj", + "model.layers.24.mlp.experts.71.gate_proj", + "model.layers.24.mlp.experts.71.up_proj", + "model.layers.24.mlp.experts.72.down_proj", + "model.layers.24.mlp.experts.72.gate_proj", + "model.layers.24.mlp.experts.72.up_proj", + "model.layers.24.mlp.experts.73.down_proj", + "model.layers.24.mlp.experts.73.gate_proj", + "model.layers.24.mlp.experts.73.up_proj", + "model.layers.24.mlp.experts.74.down_proj", + "model.layers.24.mlp.experts.74.gate_proj", + "model.layers.24.mlp.experts.74.up_proj", + "model.layers.24.mlp.experts.75.down_proj", + "model.layers.24.mlp.experts.75.gate_proj", + "model.layers.24.mlp.experts.75.up_proj", + "model.layers.24.mlp.experts.76.down_proj", + "model.layers.24.mlp.experts.76.gate_proj", + "model.layers.24.mlp.experts.76.up_proj", + "model.layers.24.mlp.experts.77.down_proj", + "model.layers.24.mlp.experts.77.gate_proj", + "model.layers.24.mlp.experts.77.up_proj", + "model.layers.24.mlp.experts.78.down_proj", + "model.layers.24.mlp.experts.78.gate_proj", + "model.layers.24.mlp.experts.78.up_proj", + "model.layers.24.mlp.experts.79.down_proj", + "model.layers.24.mlp.experts.79.gate_proj", + "model.layers.24.mlp.experts.79.up_proj", + "model.layers.24.mlp.experts.8.down_proj", + "model.layers.24.mlp.experts.8.gate_proj", + "model.layers.24.mlp.experts.8.up_proj", + "model.layers.24.mlp.experts.80.down_proj", + "model.layers.24.mlp.experts.80.gate_proj", + "model.layers.24.mlp.experts.80.up_proj", + "model.layers.24.mlp.experts.81.down_proj", + "model.layers.24.mlp.experts.81.gate_proj", + "model.layers.24.mlp.experts.81.up_proj", + "model.layers.24.mlp.experts.82.down_proj", + "model.layers.24.mlp.experts.82.gate_proj", + "model.layers.24.mlp.experts.82.up_proj", + "model.layers.24.mlp.experts.83.down_proj", + "model.layers.24.mlp.experts.83.gate_proj", + "model.layers.24.mlp.experts.83.up_proj", + "model.layers.24.mlp.experts.84.down_proj", + "model.layers.24.mlp.experts.84.gate_proj", + "model.layers.24.mlp.experts.84.up_proj", + "model.layers.24.mlp.experts.85.down_proj", + "model.layers.24.mlp.experts.85.gate_proj", + "model.layers.24.mlp.experts.85.up_proj", + "model.layers.24.mlp.experts.86.down_proj", + "model.layers.24.mlp.experts.86.gate_proj", + "model.layers.24.mlp.experts.86.up_proj", + "model.layers.24.mlp.experts.87.down_proj", + "model.layers.24.mlp.experts.87.gate_proj", + "model.layers.24.mlp.experts.87.up_proj", + "model.layers.24.mlp.experts.88.down_proj", + "model.layers.24.mlp.experts.88.gate_proj", + "model.layers.24.mlp.experts.88.up_proj", + "model.layers.24.mlp.experts.89.down_proj", + "model.layers.24.mlp.experts.89.gate_proj", + "model.layers.24.mlp.experts.89.up_proj", + "model.layers.24.mlp.experts.9.down_proj", + "model.layers.24.mlp.experts.9.gate_proj", + "model.layers.24.mlp.experts.9.up_proj", + "model.layers.24.mlp.experts.90.down_proj", + "model.layers.24.mlp.experts.90.gate_proj", + "model.layers.24.mlp.experts.90.up_proj", + "model.layers.24.mlp.experts.91.down_proj", + "model.layers.24.mlp.experts.91.gate_proj", + "model.layers.24.mlp.experts.91.up_proj", + "model.layers.24.mlp.experts.92.down_proj", + "model.layers.24.mlp.experts.92.gate_proj", + "model.layers.24.mlp.experts.92.up_proj", + "model.layers.24.mlp.experts.93.down_proj", + "model.layers.24.mlp.experts.93.gate_proj", + "model.layers.24.mlp.experts.93.up_proj", + "model.layers.24.mlp.experts.94.down_proj", + "model.layers.24.mlp.experts.94.gate_proj", + "model.layers.24.mlp.experts.94.up_proj", + "model.layers.24.mlp.experts.95.down_proj", + "model.layers.24.mlp.experts.95.gate_proj", + "model.layers.24.mlp.experts.95.up_proj", + "model.layers.24.mlp.experts.96.down_proj", + "model.layers.24.mlp.experts.96.gate_proj", + "model.layers.24.mlp.experts.96.up_proj", + "model.layers.24.mlp.experts.97.down_proj", + "model.layers.24.mlp.experts.97.gate_proj", + "model.layers.24.mlp.experts.97.up_proj", + "model.layers.24.mlp.experts.98.down_proj", + "model.layers.24.mlp.experts.98.gate_proj", + "model.layers.24.mlp.experts.98.up_proj", + "model.layers.24.mlp.experts.99.down_proj", + "model.layers.24.mlp.experts.99.gate_proj", + "model.layers.24.mlp.experts.99.up_proj", + "model.layers.25.mlp.experts.0.down_proj", + "model.layers.25.mlp.experts.0.gate_proj", + "model.layers.25.mlp.experts.0.up_proj", + "model.layers.25.mlp.experts.1.down_proj", + "model.layers.25.mlp.experts.1.gate_proj", + "model.layers.25.mlp.experts.1.up_proj", + "model.layers.25.mlp.experts.10.down_proj", + "model.layers.25.mlp.experts.10.gate_proj", + "model.layers.25.mlp.experts.10.up_proj", + "model.layers.25.mlp.experts.100.down_proj", + "model.layers.25.mlp.experts.100.gate_proj", + "model.layers.25.mlp.experts.100.up_proj", + "model.layers.25.mlp.experts.101.down_proj", + "model.layers.25.mlp.experts.101.gate_proj", + "model.layers.25.mlp.experts.101.up_proj", + "model.layers.25.mlp.experts.102.down_proj", + "model.layers.25.mlp.experts.102.gate_proj", + "model.layers.25.mlp.experts.102.up_proj", + "model.layers.25.mlp.experts.103.down_proj", + "model.layers.25.mlp.experts.103.gate_proj", + "model.layers.25.mlp.experts.103.up_proj", + "model.layers.25.mlp.experts.104.down_proj", + "model.layers.25.mlp.experts.104.gate_proj", + "model.layers.25.mlp.experts.104.up_proj", + "model.layers.25.mlp.experts.105.down_proj", + "model.layers.25.mlp.experts.105.gate_proj", + "model.layers.25.mlp.experts.105.up_proj", + "model.layers.25.mlp.experts.106.down_proj", + "model.layers.25.mlp.experts.106.gate_proj", + "model.layers.25.mlp.experts.106.up_proj", + "model.layers.25.mlp.experts.107.down_proj", + "model.layers.25.mlp.experts.107.gate_proj", + "model.layers.25.mlp.experts.107.up_proj", + "model.layers.25.mlp.experts.108.down_proj", + "model.layers.25.mlp.experts.108.gate_proj", + "model.layers.25.mlp.experts.108.up_proj", + "model.layers.25.mlp.experts.109.down_proj", + "model.layers.25.mlp.experts.109.gate_proj", + "model.layers.25.mlp.experts.109.up_proj", + "model.layers.25.mlp.experts.11.down_proj", + "model.layers.25.mlp.experts.11.gate_proj", + "model.layers.25.mlp.experts.11.up_proj", + "model.layers.25.mlp.experts.110.down_proj", + "model.layers.25.mlp.experts.110.gate_proj", + "model.layers.25.mlp.experts.110.up_proj", + "model.layers.25.mlp.experts.111.down_proj", + "model.layers.25.mlp.experts.111.gate_proj", + "model.layers.25.mlp.experts.111.up_proj", + "model.layers.25.mlp.experts.112.down_proj", + "model.layers.25.mlp.experts.112.gate_proj", + "model.layers.25.mlp.experts.112.up_proj", + "model.layers.25.mlp.experts.113.down_proj", + "model.layers.25.mlp.experts.113.gate_proj", + "model.layers.25.mlp.experts.113.up_proj", + "model.layers.25.mlp.experts.114.down_proj", + "model.layers.25.mlp.experts.114.gate_proj", + "model.layers.25.mlp.experts.114.up_proj", + "model.layers.25.mlp.experts.115.down_proj", + "model.layers.25.mlp.experts.115.gate_proj", + "model.layers.25.mlp.experts.115.up_proj", + "model.layers.25.mlp.experts.116.down_proj", + "model.layers.25.mlp.experts.116.gate_proj", + "model.layers.25.mlp.experts.116.up_proj", + "model.layers.25.mlp.experts.117.down_proj", + "model.layers.25.mlp.experts.117.gate_proj", + "model.layers.25.mlp.experts.117.up_proj", + "model.layers.25.mlp.experts.118.down_proj", + "model.layers.25.mlp.experts.118.gate_proj", + "model.layers.25.mlp.experts.118.up_proj", + "model.layers.25.mlp.experts.119.down_proj", + "model.layers.25.mlp.experts.119.gate_proj", + "model.layers.25.mlp.experts.119.up_proj", + "model.layers.25.mlp.experts.12.down_proj", + "model.layers.25.mlp.experts.12.gate_proj", + "model.layers.25.mlp.experts.12.up_proj", + "model.layers.25.mlp.experts.120.down_proj", + "model.layers.25.mlp.experts.120.gate_proj", + "model.layers.25.mlp.experts.120.up_proj", + "model.layers.25.mlp.experts.121.down_proj", + "model.layers.25.mlp.experts.121.gate_proj", + "model.layers.25.mlp.experts.121.up_proj", + "model.layers.25.mlp.experts.122.down_proj", + "model.layers.25.mlp.experts.122.gate_proj", + "model.layers.25.mlp.experts.122.up_proj", + "model.layers.25.mlp.experts.123.down_proj", + "model.layers.25.mlp.experts.123.gate_proj", + "model.layers.25.mlp.experts.123.up_proj", + "model.layers.25.mlp.experts.124.down_proj", + "model.layers.25.mlp.experts.124.gate_proj", + "model.layers.25.mlp.experts.124.up_proj", + "model.layers.25.mlp.experts.125.down_proj", + "model.layers.25.mlp.experts.125.gate_proj", + "model.layers.25.mlp.experts.125.up_proj", + "model.layers.25.mlp.experts.126.down_proj", + "model.layers.25.mlp.experts.126.gate_proj", + "model.layers.25.mlp.experts.126.up_proj", + "model.layers.25.mlp.experts.127.down_proj", + "model.layers.25.mlp.experts.127.gate_proj", + "model.layers.25.mlp.experts.127.up_proj", + "model.layers.25.mlp.experts.128.down_proj", + "model.layers.25.mlp.experts.128.gate_proj", + "model.layers.25.mlp.experts.128.up_proj", + "model.layers.25.mlp.experts.129.down_proj", + "model.layers.25.mlp.experts.129.gate_proj", + "model.layers.25.mlp.experts.129.up_proj", + "model.layers.25.mlp.experts.13.down_proj", + "model.layers.25.mlp.experts.13.gate_proj", + "model.layers.25.mlp.experts.13.up_proj", + "model.layers.25.mlp.experts.130.down_proj", + "model.layers.25.mlp.experts.130.gate_proj", + "model.layers.25.mlp.experts.130.up_proj", + "model.layers.25.mlp.experts.131.down_proj", + "model.layers.25.mlp.experts.131.gate_proj", + "model.layers.25.mlp.experts.131.up_proj", + "model.layers.25.mlp.experts.132.down_proj", + "model.layers.25.mlp.experts.132.gate_proj", + "model.layers.25.mlp.experts.132.up_proj", + "model.layers.25.mlp.experts.133.down_proj", + "model.layers.25.mlp.experts.133.gate_proj", + "model.layers.25.mlp.experts.133.up_proj", + "model.layers.25.mlp.experts.134.down_proj", + "model.layers.25.mlp.experts.134.gate_proj", + "model.layers.25.mlp.experts.134.up_proj", + "model.layers.25.mlp.experts.135.down_proj", + "model.layers.25.mlp.experts.135.gate_proj", + "model.layers.25.mlp.experts.135.up_proj", + "model.layers.25.mlp.experts.136.down_proj", + "model.layers.25.mlp.experts.136.gate_proj", + "model.layers.25.mlp.experts.136.up_proj", + "model.layers.25.mlp.experts.137.down_proj", + "model.layers.25.mlp.experts.137.gate_proj", + "model.layers.25.mlp.experts.137.up_proj", + "model.layers.25.mlp.experts.138.down_proj", + "model.layers.25.mlp.experts.138.gate_proj", + "model.layers.25.mlp.experts.138.up_proj", + "model.layers.25.mlp.experts.139.down_proj", + "model.layers.25.mlp.experts.139.gate_proj", + "model.layers.25.mlp.experts.139.up_proj", + "model.layers.25.mlp.experts.14.down_proj", + "model.layers.25.mlp.experts.14.gate_proj", + "model.layers.25.mlp.experts.14.up_proj", + "model.layers.25.mlp.experts.140.down_proj", + "model.layers.25.mlp.experts.140.gate_proj", + "model.layers.25.mlp.experts.140.up_proj", + "model.layers.25.mlp.experts.141.down_proj", + "model.layers.25.mlp.experts.141.gate_proj", + "model.layers.25.mlp.experts.141.up_proj", + "model.layers.25.mlp.experts.142.down_proj", + "model.layers.25.mlp.experts.142.gate_proj", + "model.layers.25.mlp.experts.142.up_proj", + "model.layers.25.mlp.experts.143.down_proj", + "model.layers.25.mlp.experts.143.gate_proj", + "model.layers.25.mlp.experts.143.up_proj", + "model.layers.25.mlp.experts.144.down_proj", + "model.layers.25.mlp.experts.144.gate_proj", + "model.layers.25.mlp.experts.144.up_proj", + "model.layers.25.mlp.experts.145.down_proj", + "model.layers.25.mlp.experts.145.gate_proj", + "model.layers.25.mlp.experts.145.up_proj", + "model.layers.25.mlp.experts.146.down_proj", + "model.layers.25.mlp.experts.146.gate_proj", + "model.layers.25.mlp.experts.146.up_proj", + "model.layers.25.mlp.experts.147.down_proj", + "model.layers.25.mlp.experts.147.gate_proj", + "model.layers.25.mlp.experts.147.up_proj", + "model.layers.25.mlp.experts.148.down_proj", + "model.layers.25.mlp.experts.148.gate_proj", + "model.layers.25.mlp.experts.148.up_proj", + "model.layers.25.mlp.experts.149.down_proj", + "model.layers.25.mlp.experts.149.gate_proj", + "model.layers.25.mlp.experts.149.up_proj", + "model.layers.25.mlp.experts.15.down_proj", + "model.layers.25.mlp.experts.15.gate_proj", + "model.layers.25.mlp.experts.15.up_proj", + "model.layers.25.mlp.experts.150.down_proj", + "model.layers.25.mlp.experts.150.gate_proj", + "model.layers.25.mlp.experts.150.up_proj", + "model.layers.25.mlp.experts.151.down_proj", + "model.layers.25.mlp.experts.151.gate_proj", + "model.layers.25.mlp.experts.151.up_proj", + "model.layers.25.mlp.experts.152.down_proj", + "model.layers.25.mlp.experts.152.gate_proj", + "model.layers.25.mlp.experts.152.up_proj", + "model.layers.25.mlp.experts.153.down_proj", + "model.layers.25.mlp.experts.153.gate_proj", + "model.layers.25.mlp.experts.153.up_proj", + "model.layers.25.mlp.experts.154.down_proj", + "model.layers.25.mlp.experts.154.gate_proj", + "model.layers.25.mlp.experts.154.up_proj", + "model.layers.25.mlp.experts.155.down_proj", + "model.layers.25.mlp.experts.155.gate_proj", + "model.layers.25.mlp.experts.155.up_proj", + "model.layers.25.mlp.experts.156.down_proj", + "model.layers.25.mlp.experts.156.gate_proj", + "model.layers.25.mlp.experts.156.up_proj", + "model.layers.25.mlp.experts.157.down_proj", + "model.layers.25.mlp.experts.157.gate_proj", + "model.layers.25.mlp.experts.157.up_proj", + "model.layers.25.mlp.experts.158.down_proj", + "model.layers.25.mlp.experts.158.gate_proj", + "model.layers.25.mlp.experts.158.up_proj", + "model.layers.25.mlp.experts.159.down_proj", + "model.layers.25.mlp.experts.159.gate_proj", + "model.layers.25.mlp.experts.159.up_proj", + "model.layers.25.mlp.experts.16.down_proj", + "model.layers.25.mlp.experts.16.gate_proj", + "model.layers.25.mlp.experts.16.up_proj", + "model.layers.25.mlp.experts.160.down_proj", + "model.layers.25.mlp.experts.160.gate_proj", + "model.layers.25.mlp.experts.160.up_proj", + "model.layers.25.mlp.experts.161.down_proj", + "model.layers.25.mlp.experts.161.gate_proj", + "model.layers.25.mlp.experts.161.up_proj", + "model.layers.25.mlp.experts.162.down_proj", + "model.layers.25.mlp.experts.162.gate_proj", + "model.layers.25.mlp.experts.162.up_proj", + "model.layers.25.mlp.experts.163.down_proj", + "model.layers.25.mlp.experts.163.gate_proj", + "model.layers.25.mlp.experts.163.up_proj", + "model.layers.25.mlp.experts.164.down_proj", + "model.layers.25.mlp.experts.164.gate_proj", + "model.layers.25.mlp.experts.164.up_proj", + "model.layers.25.mlp.experts.165.down_proj", + "model.layers.25.mlp.experts.165.gate_proj", + "model.layers.25.mlp.experts.165.up_proj", + "model.layers.25.mlp.experts.166.down_proj", + "model.layers.25.mlp.experts.166.gate_proj", + "model.layers.25.mlp.experts.166.up_proj", + "model.layers.25.mlp.experts.167.down_proj", + "model.layers.25.mlp.experts.167.gate_proj", + "model.layers.25.mlp.experts.167.up_proj", + "model.layers.25.mlp.experts.168.down_proj", + "model.layers.25.mlp.experts.168.gate_proj", + "model.layers.25.mlp.experts.168.up_proj", + "model.layers.25.mlp.experts.169.down_proj", + "model.layers.25.mlp.experts.169.gate_proj", + "model.layers.25.mlp.experts.169.up_proj", + "model.layers.25.mlp.experts.17.down_proj", + "model.layers.25.mlp.experts.17.gate_proj", + "model.layers.25.mlp.experts.17.up_proj", + "model.layers.25.mlp.experts.170.down_proj", + "model.layers.25.mlp.experts.170.gate_proj", + "model.layers.25.mlp.experts.170.up_proj", + "model.layers.25.mlp.experts.171.down_proj", + "model.layers.25.mlp.experts.171.gate_proj", + "model.layers.25.mlp.experts.171.up_proj", + "model.layers.25.mlp.experts.172.down_proj", + "model.layers.25.mlp.experts.172.gate_proj", + "model.layers.25.mlp.experts.172.up_proj", + "model.layers.25.mlp.experts.173.down_proj", + "model.layers.25.mlp.experts.173.gate_proj", + "model.layers.25.mlp.experts.173.up_proj", + "model.layers.25.mlp.experts.174.down_proj", + "model.layers.25.mlp.experts.174.gate_proj", + "model.layers.25.mlp.experts.174.up_proj", + "model.layers.25.mlp.experts.175.down_proj", + "model.layers.25.mlp.experts.175.gate_proj", + "model.layers.25.mlp.experts.175.up_proj", + "model.layers.25.mlp.experts.176.down_proj", + "model.layers.25.mlp.experts.176.gate_proj", + "model.layers.25.mlp.experts.176.up_proj", + "model.layers.25.mlp.experts.177.down_proj", + "model.layers.25.mlp.experts.177.gate_proj", + "model.layers.25.mlp.experts.177.up_proj", + "model.layers.25.mlp.experts.178.down_proj", + "model.layers.25.mlp.experts.178.gate_proj", + "model.layers.25.mlp.experts.178.up_proj", + "model.layers.25.mlp.experts.179.down_proj", + "model.layers.25.mlp.experts.179.gate_proj", + "model.layers.25.mlp.experts.179.up_proj", + "model.layers.25.mlp.experts.18.down_proj", + "model.layers.25.mlp.experts.18.gate_proj", + "model.layers.25.mlp.experts.18.up_proj", + "model.layers.25.mlp.experts.180.down_proj", + "model.layers.25.mlp.experts.180.gate_proj", + "model.layers.25.mlp.experts.180.up_proj", + "model.layers.25.mlp.experts.181.down_proj", + "model.layers.25.mlp.experts.181.gate_proj", + "model.layers.25.mlp.experts.181.up_proj", + "model.layers.25.mlp.experts.182.down_proj", + "model.layers.25.mlp.experts.182.gate_proj", + "model.layers.25.mlp.experts.182.up_proj", + "model.layers.25.mlp.experts.183.down_proj", + "model.layers.25.mlp.experts.183.gate_proj", + "model.layers.25.mlp.experts.183.up_proj", + "model.layers.25.mlp.experts.184.down_proj", + "model.layers.25.mlp.experts.184.gate_proj", + "model.layers.25.mlp.experts.184.up_proj", + "model.layers.25.mlp.experts.185.down_proj", + "model.layers.25.mlp.experts.185.gate_proj", + "model.layers.25.mlp.experts.185.up_proj", + "model.layers.25.mlp.experts.186.down_proj", + "model.layers.25.mlp.experts.186.gate_proj", + "model.layers.25.mlp.experts.186.up_proj", + "model.layers.25.mlp.experts.187.down_proj", + "model.layers.25.mlp.experts.187.gate_proj", + "model.layers.25.mlp.experts.187.up_proj", + "model.layers.25.mlp.experts.188.down_proj", + "model.layers.25.mlp.experts.188.gate_proj", + "model.layers.25.mlp.experts.188.up_proj", + "model.layers.25.mlp.experts.189.down_proj", + "model.layers.25.mlp.experts.189.gate_proj", + "model.layers.25.mlp.experts.189.up_proj", + "model.layers.25.mlp.experts.19.down_proj", + "model.layers.25.mlp.experts.19.gate_proj", + "model.layers.25.mlp.experts.19.up_proj", + "model.layers.25.mlp.experts.190.down_proj", + "model.layers.25.mlp.experts.190.gate_proj", + "model.layers.25.mlp.experts.190.up_proj", + "model.layers.25.mlp.experts.191.down_proj", + "model.layers.25.mlp.experts.191.gate_proj", + "model.layers.25.mlp.experts.191.up_proj", + "model.layers.25.mlp.experts.192.down_proj", + "model.layers.25.mlp.experts.192.gate_proj", + "model.layers.25.mlp.experts.192.up_proj", + "model.layers.25.mlp.experts.193.down_proj", + "model.layers.25.mlp.experts.193.gate_proj", + "model.layers.25.mlp.experts.193.up_proj", + "model.layers.25.mlp.experts.194.down_proj", + "model.layers.25.mlp.experts.194.gate_proj", + "model.layers.25.mlp.experts.194.up_proj", + "model.layers.25.mlp.experts.195.down_proj", + "model.layers.25.mlp.experts.195.gate_proj", + "model.layers.25.mlp.experts.195.up_proj", + "model.layers.25.mlp.experts.196.down_proj", + "model.layers.25.mlp.experts.196.gate_proj", + "model.layers.25.mlp.experts.196.up_proj", + "model.layers.25.mlp.experts.197.down_proj", + "model.layers.25.mlp.experts.197.gate_proj", + "model.layers.25.mlp.experts.197.up_proj", + "model.layers.25.mlp.experts.198.down_proj", + "model.layers.25.mlp.experts.198.gate_proj", + "model.layers.25.mlp.experts.198.up_proj", + "model.layers.25.mlp.experts.199.down_proj", + "model.layers.25.mlp.experts.199.gate_proj", + "model.layers.25.mlp.experts.199.up_proj", + "model.layers.25.mlp.experts.2.down_proj", + "model.layers.25.mlp.experts.2.gate_proj", + "model.layers.25.mlp.experts.2.up_proj", + "model.layers.25.mlp.experts.20.down_proj", + "model.layers.25.mlp.experts.20.gate_proj", + "model.layers.25.mlp.experts.20.up_proj", + "model.layers.25.mlp.experts.200.down_proj", + "model.layers.25.mlp.experts.200.gate_proj", + "model.layers.25.mlp.experts.200.up_proj", + "model.layers.25.mlp.experts.201.down_proj", + "model.layers.25.mlp.experts.201.gate_proj", + "model.layers.25.mlp.experts.201.up_proj", + "model.layers.25.mlp.experts.202.down_proj", + "model.layers.25.mlp.experts.202.gate_proj", + "model.layers.25.mlp.experts.202.up_proj", + "model.layers.25.mlp.experts.203.down_proj", + "model.layers.25.mlp.experts.203.gate_proj", + "model.layers.25.mlp.experts.203.up_proj", + "model.layers.25.mlp.experts.204.down_proj", + "model.layers.25.mlp.experts.204.gate_proj", + "model.layers.25.mlp.experts.204.up_proj", + "model.layers.25.mlp.experts.205.down_proj", + "model.layers.25.mlp.experts.205.gate_proj", + "model.layers.25.mlp.experts.205.up_proj", + "model.layers.25.mlp.experts.206.down_proj", + "model.layers.25.mlp.experts.206.gate_proj", + "model.layers.25.mlp.experts.206.up_proj", + "model.layers.25.mlp.experts.207.down_proj", + "model.layers.25.mlp.experts.207.gate_proj", + "model.layers.25.mlp.experts.207.up_proj", + "model.layers.25.mlp.experts.208.down_proj", + "model.layers.25.mlp.experts.208.gate_proj", + "model.layers.25.mlp.experts.208.up_proj", + "model.layers.25.mlp.experts.209.down_proj", + "model.layers.25.mlp.experts.209.gate_proj", + "model.layers.25.mlp.experts.209.up_proj", + "model.layers.25.mlp.experts.21.down_proj", + "model.layers.25.mlp.experts.21.gate_proj", + "model.layers.25.mlp.experts.21.up_proj", + "model.layers.25.mlp.experts.210.down_proj", + "model.layers.25.mlp.experts.210.gate_proj", + "model.layers.25.mlp.experts.210.up_proj", + "model.layers.25.mlp.experts.211.down_proj", + "model.layers.25.mlp.experts.211.gate_proj", + "model.layers.25.mlp.experts.211.up_proj", + "model.layers.25.mlp.experts.212.down_proj", + "model.layers.25.mlp.experts.212.gate_proj", + "model.layers.25.mlp.experts.212.up_proj", + "model.layers.25.mlp.experts.213.down_proj", + "model.layers.25.mlp.experts.213.gate_proj", + "model.layers.25.mlp.experts.213.up_proj", + "model.layers.25.mlp.experts.214.down_proj", + "model.layers.25.mlp.experts.214.gate_proj", + "model.layers.25.mlp.experts.214.up_proj", + "model.layers.25.mlp.experts.215.down_proj", + "model.layers.25.mlp.experts.215.gate_proj", + "model.layers.25.mlp.experts.215.up_proj", + "model.layers.25.mlp.experts.216.down_proj", + "model.layers.25.mlp.experts.216.gate_proj", + "model.layers.25.mlp.experts.216.up_proj", + "model.layers.25.mlp.experts.217.down_proj", + "model.layers.25.mlp.experts.217.gate_proj", + "model.layers.25.mlp.experts.217.up_proj", + "model.layers.25.mlp.experts.218.down_proj", + "model.layers.25.mlp.experts.218.gate_proj", + "model.layers.25.mlp.experts.218.up_proj", + "model.layers.25.mlp.experts.219.down_proj", + "model.layers.25.mlp.experts.219.gate_proj", + "model.layers.25.mlp.experts.219.up_proj", + "model.layers.25.mlp.experts.22.down_proj", + "model.layers.25.mlp.experts.22.gate_proj", + "model.layers.25.mlp.experts.22.up_proj", + "model.layers.25.mlp.experts.220.down_proj", + "model.layers.25.mlp.experts.220.gate_proj", + "model.layers.25.mlp.experts.220.up_proj", + "model.layers.25.mlp.experts.221.down_proj", + "model.layers.25.mlp.experts.221.gate_proj", + "model.layers.25.mlp.experts.221.up_proj", + "model.layers.25.mlp.experts.222.down_proj", + "model.layers.25.mlp.experts.222.gate_proj", + "model.layers.25.mlp.experts.222.up_proj", + "model.layers.25.mlp.experts.223.down_proj", + "model.layers.25.mlp.experts.223.gate_proj", + "model.layers.25.mlp.experts.223.up_proj", + "model.layers.25.mlp.experts.224.down_proj", + "model.layers.25.mlp.experts.224.gate_proj", + "model.layers.25.mlp.experts.224.up_proj", + "model.layers.25.mlp.experts.225.down_proj", + "model.layers.25.mlp.experts.225.gate_proj", + "model.layers.25.mlp.experts.225.up_proj", + "model.layers.25.mlp.experts.226.down_proj", + "model.layers.25.mlp.experts.226.gate_proj", + "model.layers.25.mlp.experts.226.up_proj", + "model.layers.25.mlp.experts.227.down_proj", + "model.layers.25.mlp.experts.227.gate_proj", + "model.layers.25.mlp.experts.227.up_proj", + "model.layers.25.mlp.experts.228.down_proj", + "model.layers.25.mlp.experts.228.gate_proj", + "model.layers.25.mlp.experts.228.up_proj", + "model.layers.25.mlp.experts.229.down_proj", + "model.layers.25.mlp.experts.229.gate_proj", + "model.layers.25.mlp.experts.229.up_proj", + "model.layers.25.mlp.experts.23.down_proj", + "model.layers.25.mlp.experts.23.gate_proj", + "model.layers.25.mlp.experts.23.up_proj", + "model.layers.25.mlp.experts.230.down_proj", + "model.layers.25.mlp.experts.230.gate_proj", + "model.layers.25.mlp.experts.230.up_proj", + "model.layers.25.mlp.experts.231.down_proj", + "model.layers.25.mlp.experts.231.gate_proj", + "model.layers.25.mlp.experts.231.up_proj", + "model.layers.25.mlp.experts.232.down_proj", + "model.layers.25.mlp.experts.232.gate_proj", + "model.layers.25.mlp.experts.232.up_proj", + "model.layers.25.mlp.experts.233.down_proj", + "model.layers.25.mlp.experts.233.gate_proj", + "model.layers.25.mlp.experts.233.up_proj", + "model.layers.25.mlp.experts.234.down_proj", + "model.layers.25.mlp.experts.234.gate_proj", + "model.layers.25.mlp.experts.234.up_proj", + "model.layers.25.mlp.experts.235.down_proj", + "model.layers.25.mlp.experts.235.gate_proj", + "model.layers.25.mlp.experts.235.up_proj", + "model.layers.25.mlp.experts.236.down_proj", + "model.layers.25.mlp.experts.236.gate_proj", + "model.layers.25.mlp.experts.236.up_proj", + "model.layers.25.mlp.experts.237.down_proj", + "model.layers.25.mlp.experts.237.gate_proj", + "model.layers.25.mlp.experts.237.up_proj", + "model.layers.25.mlp.experts.238.down_proj", + "model.layers.25.mlp.experts.238.gate_proj", + "model.layers.25.mlp.experts.238.up_proj", + "model.layers.25.mlp.experts.239.down_proj", + "model.layers.25.mlp.experts.239.gate_proj", + "model.layers.25.mlp.experts.239.up_proj", + "model.layers.25.mlp.experts.24.down_proj", + "model.layers.25.mlp.experts.24.gate_proj", + "model.layers.25.mlp.experts.24.up_proj", + "model.layers.25.mlp.experts.240.down_proj", + "model.layers.25.mlp.experts.240.gate_proj", + "model.layers.25.mlp.experts.240.up_proj", + "model.layers.25.mlp.experts.241.down_proj", + "model.layers.25.mlp.experts.241.gate_proj", + "model.layers.25.mlp.experts.241.up_proj", + "model.layers.25.mlp.experts.242.down_proj", + "model.layers.25.mlp.experts.242.gate_proj", + "model.layers.25.mlp.experts.242.up_proj", + "model.layers.25.mlp.experts.243.down_proj", + "model.layers.25.mlp.experts.243.gate_proj", + "model.layers.25.mlp.experts.243.up_proj", + "model.layers.25.mlp.experts.244.down_proj", + "model.layers.25.mlp.experts.244.gate_proj", + "model.layers.25.mlp.experts.244.up_proj", + "model.layers.25.mlp.experts.245.down_proj", + "model.layers.25.mlp.experts.245.gate_proj", + "model.layers.25.mlp.experts.245.up_proj", + "model.layers.25.mlp.experts.246.down_proj", + "model.layers.25.mlp.experts.246.gate_proj", + "model.layers.25.mlp.experts.246.up_proj", + "model.layers.25.mlp.experts.247.down_proj", + "model.layers.25.mlp.experts.247.gate_proj", + "model.layers.25.mlp.experts.247.up_proj", + "model.layers.25.mlp.experts.248.down_proj", + "model.layers.25.mlp.experts.248.gate_proj", + "model.layers.25.mlp.experts.248.up_proj", + "model.layers.25.mlp.experts.249.down_proj", + "model.layers.25.mlp.experts.249.gate_proj", + "model.layers.25.mlp.experts.249.up_proj", + "model.layers.25.mlp.experts.25.down_proj", + "model.layers.25.mlp.experts.25.gate_proj", + "model.layers.25.mlp.experts.25.up_proj", + "model.layers.25.mlp.experts.250.down_proj", + "model.layers.25.mlp.experts.250.gate_proj", + "model.layers.25.mlp.experts.250.up_proj", + "model.layers.25.mlp.experts.251.down_proj", + "model.layers.25.mlp.experts.251.gate_proj", + "model.layers.25.mlp.experts.251.up_proj", + "model.layers.25.mlp.experts.252.down_proj", + "model.layers.25.mlp.experts.252.gate_proj", + "model.layers.25.mlp.experts.252.up_proj", + "model.layers.25.mlp.experts.253.down_proj", + "model.layers.25.mlp.experts.253.gate_proj", + "model.layers.25.mlp.experts.253.up_proj", + "model.layers.25.mlp.experts.254.down_proj", + "model.layers.25.mlp.experts.254.gate_proj", + "model.layers.25.mlp.experts.254.up_proj", + "model.layers.25.mlp.experts.255.down_proj", + "model.layers.25.mlp.experts.255.gate_proj", + "model.layers.25.mlp.experts.255.up_proj", + "model.layers.25.mlp.experts.26.down_proj", + "model.layers.25.mlp.experts.26.gate_proj", + "model.layers.25.mlp.experts.26.up_proj", + "model.layers.25.mlp.experts.27.down_proj", + "model.layers.25.mlp.experts.27.gate_proj", + "model.layers.25.mlp.experts.27.up_proj", + "model.layers.25.mlp.experts.28.down_proj", + "model.layers.25.mlp.experts.28.gate_proj", + "model.layers.25.mlp.experts.28.up_proj", + "model.layers.25.mlp.experts.29.down_proj", + "model.layers.25.mlp.experts.29.gate_proj", + "model.layers.25.mlp.experts.29.up_proj", + "model.layers.25.mlp.experts.3.down_proj", + "model.layers.25.mlp.experts.3.gate_proj", + "model.layers.25.mlp.experts.3.up_proj", + "model.layers.25.mlp.experts.30.down_proj", + "model.layers.25.mlp.experts.30.gate_proj", + "model.layers.25.mlp.experts.30.up_proj", + "model.layers.25.mlp.experts.31.down_proj", + "model.layers.25.mlp.experts.31.gate_proj", + "model.layers.25.mlp.experts.31.up_proj", + "model.layers.25.mlp.experts.32.down_proj", + "model.layers.25.mlp.experts.32.gate_proj", + "model.layers.25.mlp.experts.32.up_proj", + "model.layers.25.mlp.experts.33.down_proj", + "model.layers.25.mlp.experts.33.gate_proj", + "model.layers.25.mlp.experts.33.up_proj", + "model.layers.25.mlp.experts.34.down_proj", + "model.layers.25.mlp.experts.34.gate_proj", + "model.layers.25.mlp.experts.34.up_proj", + "model.layers.25.mlp.experts.35.down_proj", + "model.layers.25.mlp.experts.35.gate_proj", + "model.layers.25.mlp.experts.35.up_proj", + "model.layers.25.mlp.experts.36.down_proj", + "model.layers.25.mlp.experts.36.gate_proj", + "model.layers.25.mlp.experts.36.up_proj", + "model.layers.25.mlp.experts.37.down_proj", + "model.layers.25.mlp.experts.37.gate_proj", + "model.layers.25.mlp.experts.37.up_proj", + "model.layers.25.mlp.experts.38.down_proj", + "model.layers.25.mlp.experts.38.gate_proj", + "model.layers.25.mlp.experts.38.up_proj", + "model.layers.25.mlp.experts.39.down_proj", + "model.layers.25.mlp.experts.39.gate_proj", + "model.layers.25.mlp.experts.39.up_proj", + "model.layers.25.mlp.experts.4.down_proj", + "model.layers.25.mlp.experts.4.gate_proj", + "model.layers.25.mlp.experts.4.up_proj", + "model.layers.25.mlp.experts.40.down_proj", + "model.layers.25.mlp.experts.40.gate_proj", + "model.layers.25.mlp.experts.40.up_proj", + "model.layers.25.mlp.experts.41.down_proj", + "model.layers.25.mlp.experts.41.gate_proj", + "model.layers.25.mlp.experts.41.up_proj", + "model.layers.25.mlp.experts.42.down_proj", + "model.layers.25.mlp.experts.42.gate_proj", + "model.layers.25.mlp.experts.42.up_proj", + "model.layers.25.mlp.experts.43.down_proj", + "model.layers.25.mlp.experts.43.gate_proj", + "model.layers.25.mlp.experts.43.up_proj", + "model.layers.25.mlp.experts.44.down_proj", + "model.layers.25.mlp.experts.44.gate_proj", + "model.layers.25.mlp.experts.44.up_proj", + "model.layers.25.mlp.experts.45.down_proj", + "model.layers.25.mlp.experts.45.gate_proj", + "model.layers.25.mlp.experts.45.up_proj", + "model.layers.25.mlp.experts.46.down_proj", + "model.layers.25.mlp.experts.46.gate_proj", + "model.layers.25.mlp.experts.46.up_proj", + "model.layers.25.mlp.experts.47.down_proj", + "model.layers.25.mlp.experts.47.gate_proj", + "model.layers.25.mlp.experts.47.up_proj", + "model.layers.25.mlp.experts.48.down_proj", + "model.layers.25.mlp.experts.48.gate_proj", + "model.layers.25.mlp.experts.48.up_proj", + "model.layers.25.mlp.experts.49.down_proj", + "model.layers.25.mlp.experts.49.gate_proj", + "model.layers.25.mlp.experts.49.up_proj", + "model.layers.25.mlp.experts.5.down_proj", + "model.layers.25.mlp.experts.5.gate_proj", + "model.layers.25.mlp.experts.5.up_proj", + "model.layers.25.mlp.experts.50.down_proj", + "model.layers.25.mlp.experts.50.gate_proj", + "model.layers.25.mlp.experts.50.up_proj", + "model.layers.25.mlp.experts.51.down_proj", + "model.layers.25.mlp.experts.51.gate_proj", + "model.layers.25.mlp.experts.51.up_proj", + "model.layers.25.mlp.experts.52.down_proj", + "model.layers.25.mlp.experts.52.gate_proj", + "model.layers.25.mlp.experts.52.up_proj", + "model.layers.25.mlp.experts.53.down_proj", + "model.layers.25.mlp.experts.53.gate_proj", + "model.layers.25.mlp.experts.53.up_proj", + "model.layers.25.mlp.experts.54.down_proj", + "model.layers.25.mlp.experts.54.gate_proj", + "model.layers.25.mlp.experts.54.up_proj", + "model.layers.25.mlp.experts.55.down_proj", + "model.layers.25.mlp.experts.55.gate_proj", + "model.layers.25.mlp.experts.55.up_proj", + "model.layers.25.mlp.experts.56.down_proj", + "model.layers.25.mlp.experts.56.gate_proj", + "model.layers.25.mlp.experts.56.up_proj", + "model.layers.25.mlp.experts.57.down_proj", + "model.layers.25.mlp.experts.57.gate_proj", + "model.layers.25.mlp.experts.57.up_proj", + "model.layers.25.mlp.experts.58.down_proj", + "model.layers.25.mlp.experts.58.gate_proj", + "model.layers.25.mlp.experts.58.up_proj", + "model.layers.25.mlp.experts.59.down_proj", + "model.layers.25.mlp.experts.59.gate_proj", + "model.layers.25.mlp.experts.59.up_proj", + "model.layers.25.mlp.experts.6.down_proj", + "model.layers.25.mlp.experts.6.gate_proj", + "model.layers.25.mlp.experts.6.up_proj", + "model.layers.25.mlp.experts.60.down_proj", + "model.layers.25.mlp.experts.60.gate_proj", + "model.layers.25.mlp.experts.60.up_proj", + "model.layers.25.mlp.experts.61.down_proj", + "model.layers.25.mlp.experts.61.gate_proj", + "model.layers.25.mlp.experts.61.up_proj", + "model.layers.25.mlp.experts.62.down_proj", + "model.layers.25.mlp.experts.62.gate_proj", + "model.layers.25.mlp.experts.62.up_proj", + "model.layers.25.mlp.experts.63.down_proj", + "model.layers.25.mlp.experts.63.gate_proj", + "model.layers.25.mlp.experts.63.up_proj", + "model.layers.25.mlp.experts.64.down_proj", + "model.layers.25.mlp.experts.64.gate_proj", + "model.layers.25.mlp.experts.64.up_proj", + "model.layers.25.mlp.experts.65.down_proj", + "model.layers.25.mlp.experts.65.gate_proj", + "model.layers.25.mlp.experts.65.up_proj", + "model.layers.25.mlp.experts.66.down_proj", + "model.layers.25.mlp.experts.66.gate_proj", + "model.layers.25.mlp.experts.66.up_proj", + "model.layers.25.mlp.experts.67.down_proj", + "model.layers.25.mlp.experts.67.gate_proj", + "model.layers.25.mlp.experts.67.up_proj", + "model.layers.25.mlp.experts.68.down_proj", + "model.layers.25.mlp.experts.68.gate_proj", + "model.layers.25.mlp.experts.68.up_proj", + "model.layers.25.mlp.experts.69.down_proj", + "model.layers.25.mlp.experts.69.gate_proj", + "model.layers.25.mlp.experts.69.up_proj", + "model.layers.25.mlp.experts.7.down_proj", + "model.layers.25.mlp.experts.7.gate_proj", + "model.layers.25.mlp.experts.7.up_proj", + "model.layers.25.mlp.experts.70.down_proj", + "model.layers.25.mlp.experts.70.gate_proj", + "model.layers.25.mlp.experts.70.up_proj", + "model.layers.25.mlp.experts.71.down_proj", + "model.layers.25.mlp.experts.71.gate_proj", + "model.layers.25.mlp.experts.71.up_proj", + "model.layers.25.mlp.experts.72.down_proj", + "model.layers.25.mlp.experts.72.gate_proj", + "model.layers.25.mlp.experts.72.up_proj", + "model.layers.25.mlp.experts.73.down_proj", + "model.layers.25.mlp.experts.73.gate_proj", + "model.layers.25.mlp.experts.73.up_proj", + "model.layers.25.mlp.experts.74.down_proj", + "model.layers.25.mlp.experts.74.gate_proj", + "model.layers.25.mlp.experts.74.up_proj", + "model.layers.25.mlp.experts.75.down_proj", + "model.layers.25.mlp.experts.75.gate_proj", + "model.layers.25.mlp.experts.75.up_proj", + "model.layers.25.mlp.experts.76.down_proj", + "model.layers.25.mlp.experts.76.gate_proj", + "model.layers.25.mlp.experts.76.up_proj", + "model.layers.25.mlp.experts.77.down_proj", + "model.layers.25.mlp.experts.77.gate_proj", + "model.layers.25.mlp.experts.77.up_proj", + "model.layers.25.mlp.experts.78.down_proj", + "model.layers.25.mlp.experts.78.gate_proj", + "model.layers.25.mlp.experts.78.up_proj", + "model.layers.25.mlp.experts.79.down_proj", + "model.layers.25.mlp.experts.79.gate_proj", + "model.layers.25.mlp.experts.79.up_proj", + "model.layers.25.mlp.experts.8.down_proj", + "model.layers.25.mlp.experts.8.gate_proj", + "model.layers.25.mlp.experts.8.up_proj", + "model.layers.25.mlp.experts.80.down_proj", + "model.layers.25.mlp.experts.80.gate_proj", + "model.layers.25.mlp.experts.80.up_proj", + "model.layers.25.mlp.experts.81.down_proj", + "model.layers.25.mlp.experts.81.gate_proj", + "model.layers.25.mlp.experts.81.up_proj", + "model.layers.25.mlp.experts.82.down_proj", + "model.layers.25.mlp.experts.82.gate_proj", + "model.layers.25.mlp.experts.82.up_proj", + "model.layers.25.mlp.experts.83.down_proj", + "model.layers.25.mlp.experts.83.gate_proj", + "model.layers.25.mlp.experts.83.up_proj", + "model.layers.25.mlp.experts.84.down_proj", + "model.layers.25.mlp.experts.84.gate_proj", + "model.layers.25.mlp.experts.84.up_proj", + "model.layers.25.mlp.experts.85.down_proj", + "model.layers.25.mlp.experts.85.gate_proj", + "model.layers.25.mlp.experts.85.up_proj", + "model.layers.25.mlp.experts.86.down_proj", + "model.layers.25.mlp.experts.86.gate_proj", + "model.layers.25.mlp.experts.86.up_proj", + "model.layers.25.mlp.experts.87.down_proj", + "model.layers.25.mlp.experts.87.gate_proj", + "model.layers.25.mlp.experts.87.up_proj", + "model.layers.25.mlp.experts.88.down_proj", + "model.layers.25.mlp.experts.88.gate_proj", + "model.layers.25.mlp.experts.88.up_proj", + "model.layers.25.mlp.experts.89.down_proj", + "model.layers.25.mlp.experts.89.gate_proj", + "model.layers.25.mlp.experts.89.up_proj", + "model.layers.25.mlp.experts.9.down_proj", + "model.layers.25.mlp.experts.9.gate_proj", + "model.layers.25.mlp.experts.9.up_proj", + "model.layers.25.mlp.experts.90.down_proj", + "model.layers.25.mlp.experts.90.gate_proj", + "model.layers.25.mlp.experts.90.up_proj", + "model.layers.25.mlp.experts.91.down_proj", + "model.layers.25.mlp.experts.91.gate_proj", + "model.layers.25.mlp.experts.91.up_proj", + "model.layers.25.mlp.experts.92.down_proj", + "model.layers.25.mlp.experts.92.gate_proj", + "model.layers.25.mlp.experts.92.up_proj", + "model.layers.25.mlp.experts.93.down_proj", + "model.layers.25.mlp.experts.93.gate_proj", + "model.layers.25.mlp.experts.93.up_proj", + "model.layers.25.mlp.experts.94.down_proj", + "model.layers.25.mlp.experts.94.gate_proj", + "model.layers.25.mlp.experts.94.up_proj", + "model.layers.25.mlp.experts.95.down_proj", + "model.layers.25.mlp.experts.95.gate_proj", + "model.layers.25.mlp.experts.95.up_proj", + "model.layers.25.mlp.experts.96.down_proj", + "model.layers.25.mlp.experts.96.gate_proj", + "model.layers.25.mlp.experts.96.up_proj", + "model.layers.25.mlp.experts.97.down_proj", + "model.layers.25.mlp.experts.97.gate_proj", + "model.layers.25.mlp.experts.97.up_proj", + "model.layers.25.mlp.experts.98.down_proj", + "model.layers.25.mlp.experts.98.gate_proj", + "model.layers.25.mlp.experts.98.up_proj", + "model.layers.25.mlp.experts.99.down_proj", + "model.layers.25.mlp.experts.99.gate_proj", + "model.layers.25.mlp.experts.99.up_proj", + "model.layers.26.mlp.experts.0.down_proj", + "model.layers.26.mlp.experts.0.gate_proj", + "model.layers.26.mlp.experts.0.up_proj", + "model.layers.26.mlp.experts.1.down_proj", + "model.layers.26.mlp.experts.1.gate_proj", + "model.layers.26.mlp.experts.1.up_proj", + "model.layers.26.mlp.experts.10.down_proj", + "model.layers.26.mlp.experts.10.gate_proj", + "model.layers.26.mlp.experts.10.up_proj", + "model.layers.26.mlp.experts.100.down_proj", + "model.layers.26.mlp.experts.100.gate_proj", + "model.layers.26.mlp.experts.100.up_proj", + "model.layers.26.mlp.experts.101.down_proj", + "model.layers.26.mlp.experts.101.gate_proj", + "model.layers.26.mlp.experts.101.up_proj", + "model.layers.26.mlp.experts.102.down_proj", + "model.layers.26.mlp.experts.102.gate_proj", + "model.layers.26.mlp.experts.102.up_proj", + "model.layers.26.mlp.experts.103.down_proj", + "model.layers.26.mlp.experts.103.gate_proj", + "model.layers.26.mlp.experts.103.up_proj", + "model.layers.26.mlp.experts.104.down_proj", + "model.layers.26.mlp.experts.104.gate_proj", + "model.layers.26.mlp.experts.104.up_proj", + "model.layers.26.mlp.experts.105.down_proj", + "model.layers.26.mlp.experts.105.gate_proj", + "model.layers.26.mlp.experts.105.up_proj", + "model.layers.26.mlp.experts.106.down_proj", + "model.layers.26.mlp.experts.106.gate_proj", + "model.layers.26.mlp.experts.106.up_proj", + "model.layers.26.mlp.experts.107.down_proj", + "model.layers.26.mlp.experts.107.gate_proj", + "model.layers.26.mlp.experts.107.up_proj", + "model.layers.26.mlp.experts.108.down_proj", + "model.layers.26.mlp.experts.108.gate_proj", + "model.layers.26.mlp.experts.108.up_proj", + "model.layers.26.mlp.experts.109.down_proj", + "model.layers.26.mlp.experts.109.gate_proj", + "model.layers.26.mlp.experts.109.up_proj", + "model.layers.26.mlp.experts.11.down_proj", + "model.layers.26.mlp.experts.11.gate_proj", + "model.layers.26.mlp.experts.11.up_proj", + "model.layers.26.mlp.experts.110.down_proj", + "model.layers.26.mlp.experts.110.gate_proj", + "model.layers.26.mlp.experts.110.up_proj", + "model.layers.26.mlp.experts.111.down_proj", + "model.layers.26.mlp.experts.111.gate_proj", + "model.layers.26.mlp.experts.111.up_proj", + "model.layers.26.mlp.experts.112.down_proj", + "model.layers.26.mlp.experts.112.gate_proj", + "model.layers.26.mlp.experts.112.up_proj", + "model.layers.26.mlp.experts.113.down_proj", + "model.layers.26.mlp.experts.113.gate_proj", + "model.layers.26.mlp.experts.113.up_proj", + "model.layers.26.mlp.experts.114.down_proj", + "model.layers.26.mlp.experts.114.gate_proj", + "model.layers.26.mlp.experts.114.up_proj", + "model.layers.26.mlp.experts.115.down_proj", + "model.layers.26.mlp.experts.115.gate_proj", + "model.layers.26.mlp.experts.115.up_proj", + "model.layers.26.mlp.experts.116.down_proj", + "model.layers.26.mlp.experts.116.gate_proj", + "model.layers.26.mlp.experts.116.up_proj", + "model.layers.26.mlp.experts.117.down_proj", + "model.layers.26.mlp.experts.117.gate_proj", + "model.layers.26.mlp.experts.117.up_proj", + "model.layers.26.mlp.experts.118.down_proj", + "model.layers.26.mlp.experts.118.gate_proj", + "model.layers.26.mlp.experts.118.up_proj", + "model.layers.26.mlp.experts.119.down_proj", + "model.layers.26.mlp.experts.119.gate_proj", + "model.layers.26.mlp.experts.119.up_proj", + "model.layers.26.mlp.experts.12.down_proj", + "model.layers.26.mlp.experts.12.gate_proj", + "model.layers.26.mlp.experts.12.up_proj", + "model.layers.26.mlp.experts.120.down_proj", + "model.layers.26.mlp.experts.120.gate_proj", + "model.layers.26.mlp.experts.120.up_proj", + "model.layers.26.mlp.experts.121.down_proj", + "model.layers.26.mlp.experts.121.gate_proj", + "model.layers.26.mlp.experts.121.up_proj", + "model.layers.26.mlp.experts.122.down_proj", + "model.layers.26.mlp.experts.122.gate_proj", + "model.layers.26.mlp.experts.122.up_proj", + "model.layers.26.mlp.experts.123.down_proj", + "model.layers.26.mlp.experts.123.gate_proj", + "model.layers.26.mlp.experts.123.up_proj", + "model.layers.26.mlp.experts.124.down_proj", + "model.layers.26.mlp.experts.124.gate_proj", + "model.layers.26.mlp.experts.124.up_proj", + "model.layers.26.mlp.experts.125.down_proj", + "model.layers.26.mlp.experts.125.gate_proj", + "model.layers.26.mlp.experts.125.up_proj", + "model.layers.26.mlp.experts.126.down_proj", + "model.layers.26.mlp.experts.126.gate_proj", + "model.layers.26.mlp.experts.126.up_proj", + "model.layers.26.mlp.experts.127.down_proj", + "model.layers.26.mlp.experts.127.gate_proj", + "model.layers.26.mlp.experts.127.up_proj", + "model.layers.26.mlp.experts.128.down_proj", + "model.layers.26.mlp.experts.128.gate_proj", + "model.layers.26.mlp.experts.128.up_proj", + "model.layers.26.mlp.experts.129.down_proj", + "model.layers.26.mlp.experts.129.gate_proj", + "model.layers.26.mlp.experts.129.up_proj", + "model.layers.26.mlp.experts.13.down_proj", + "model.layers.26.mlp.experts.13.gate_proj", + "model.layers.26.mlp.experts.13.up_proj", + "model.layers.26.mlp.experts.130.down_proj", + "model.layers.26.mlp.experts.130.gate_proj", + "model.layers.26.mlp.experts.130.up_proj", + "model.layers.26.mlp.experts.131.down_proj", + "model.layers.26.mlp.experts.131.gate_proj", + "model.layers.26.mlp.experts.131.up_proj", + "model.layers.26.mlp.experts.132.down_proj", + "model.layers.26.mlp.experts.132.gate_proj", + "model.layers.26.mlp.experts.132.up_proj", + "model.layers.26.mlp.experts.133.down_proj", + "model.layers.26.mlp.experts.133.gate_proj", + "model.layers.26.mlp.experts.133.up_proj", + "model.layers.26.mlp.experts.134.down_proj", + "model.layers.26.mlp.experts.134.gate_proj", + "model.layers.26.mlp.experts.134.up_proj", + "model.layers.26.mlp.experts.135.down_proj", + "model.layers.26.mlp.experts.135.gate_proj", + "model.layers.26.mlp.experts.135.up_proj", + "model.layers.26.mlp.experts.136.down_proj", + "model.layers.26.mlp.experts.136.gate_proj", + "model.layers.26.mlp.experts.136.up_proj", + "model.layers.26.mlp.experts.137.down_proj", + "model.layers.26.mlp.experts.137.gate_proj", + "model.layers.26.mlp.experts.137.up_proj", + "model.layers.26.mlp.experts.138.down_proj", + "model.layers.26.mlp.experts.138.gate_proj", + "model.layers.26.mlp.experts.138.up_proj", + "model.layers.26.mlp.experts.139.down_proj", + "model.layers.26.mlp.experts.139.gate_proj", + "model.layers.26.mlp.experts.139.up_proj", + "model.layers.26.mlp.experts.14.down_proj", + "model.layers.26.mlp.experts.14.gate_proj", + "model.layers.26.mlp.experts.14.up_proj", + "model.layers.26.mlp.experts.140.down_proj", + "model.layers.26.mlp.experts.140.gate_proj", + "model.layers.26.mlp.experts.140.up_proj", + "model.layers.26.mlp.experts.141.down_proj", + "model.layers.26.mlp.experts.141.gate_proj", + "model.layers.26.mlp.experts.141.up_proj", + "model.layers.26.mlp.experts.142.down_proj", + "model.layers.26.mlp.experts.142.gate_proj", + "model.layers.26.mlp.experts.142.up_proj", + "model.layers.26.mlp.experts.143.down_proj", + "model.layers.26.mlp.experts.143.gate_proj", + "model.layers.26.mlp.experts.143.up_proj", + "model.layers.26.mlp.experts.144.down_proj", + "model.layers.26.mlp.experts.144.gate_proj", + "model.layers.26.mlp.experts.144.up_proj", + "model.layers.26.mlp.experts.145.down_proj", + "model.layers.26.mlp.experts.145.gate_proj", + "model.layers.26.mlp.experts.145.up_proj", + "model.layers.26.mlp.experts.146.down_proj", + "model.layers.26.mlp.experts.146.gate_proj", + "model.layers.26.mlp.experts.146.up_proj", + "model.layers.26.mlp.experts.147.down_proj", + "model.layers.26.mlp.experts.147.gate_proj", + "model.layers.26.mlp.experts.147.up_proj", + "model.layers.26.mlp.experts.148.down_proj", + "model.layers.26.mlp.experts.148.gate_proj", + "model.layers.26.mlp.experts.148.up_proj", + "model.layers.26.mlp.experts.149.down_proj", + "model.layers.26.mlp.experts.149.gate_proj", + "model.layers.26.mlp.experts.149.up_proj", + "model.layers.26.mlp.experts.15.down_proj", + "model.layers.26.mlp.experts.15.gate_proj", + "model.layers.26.mlp.experts.15.up_proj", + "model.layers.26.mlp.experts.150.down_proj", + "model.layers.26.mlp.experts.150.gate_proj", + "model.layers.26.mlp.experts.150.up_proj", + "model.layers.26.mlp.experts.151.down_proj", + "model.layers.26.mlp.experts.151.gate_proj", + "model.layers.26.mlp.experts.151.up_proj", + "model.layers.26.mlp.experts.152.down_proj", + "model.layers.26.mlp.experts.152.gate_proj", + "model.layers.26.mlp.experts.152.up_proj", + "model.layers.26.mlp.experts.153.down_proj", + "model.layers.26.mlp.experts.153.gate_proj", + "model.layers.26.mlp.experts.153.up_proj", + "model.layers.26.mlp.experts.154.down_proj", + "model.layers.26.mlp.experts.154.gate_proj", + "model.layers.26.mlp.experts.154.up_proj", + "model.layers.26.mlp.experts.155.down_proj", + "model.layers.26.mlp.experts.155.gate_proj", + "model.layers.26.mlp.experts.155.up_proj", + "model.layers.26.mlp.experts.156.down_proj", + "model.layers.26.mlp.experts.156.gate_proj", + "model.layers.26.mlp.experts.156.up_proj", + "model.layers.26.mlp.experts.157.down_proj", + "model.layers.26.mlp.experts.157.gate_proj", + "model.layers.26.mlp.experts.157.up_proj", + "model.layers.26.mlp.experts.158.down_proj", + "model.layers.26.mlp.experts.158.gate_proj", + "model.layers.26.mlp.experts.158.up_proj", + "model.layers.26.mlp.experts.159.down_proj", + "model.layers.26.mlp.experts.159.gate_proj", + "model.layers.26.mlp.experts.159.up_proj", + "model.layers.26.mlp.experts.16.down_proj", + "model.layers.26.mlp.experts.16.gate_proj", + "model.layers.26.mlp.experts.16.up_proj", + "model.layers.26.mlp.experts.160.down_proj", + "model.layers.26.mlp.experts.160.gate_proj", + "model.layers.26.mlp.experts.160.up_proj", + "model.layers.26.mlp.experts.161.down_proj", + "model.layers.26.mlp.experts.161.gate_proj", + "model.layers.26.mlp.experts.161.up_proj", + "model.layers.26.mlp.experts.162.down_proj", + "model.layers.26.mlp.experts.162.gate_proj", + "model.layers.26.mlp.experts.162.up_proj", + "model.layers.26.mlp.experts.163.down_proj", + "model.layers.26.mlp.experts.163.gate_proj", + "model.layers.26.mlp.experts.163.up_proj", + "model.layers.26.mlp.experts.164.down_proj", + "model.layers.26.mlp.experts.164.gate_proj", + "model.layers.26.mlp.experts.164.up_proj", + "model.layers.26.mlp.experts.165.down_proj", + "model.layers.26.mlp.experts.165.gate_proj", + "model.layers.26.mlp.experts.165.up_proj", + "model.layers.26.mlp.experts.166.down_proj", + "model.layers.26.mlp.experts.166.gate_proj", + "model.layers.26.mlp.experts.166.up_proj", + "model.layers.26.mlp.experts.167.down_proj", + "model.layers.26.mlp.experts.167.gate_proj", + "model.layers.26.mlp.experts.167.up_proj", + "model.layers.26.mlp.experts.168.down_proj", + "model.layers.26.mlp.experts.168.gate_proj", + "model.layers.26.mlp.experts.168.up_proj", + "model.layers.26.mlp.experts.169.down_proj", + "model.layers.26.mlp.experts.169.gate_proj", + "model.layers.26.mlp.experts.169.up_proj", + "model.layers.26.mlp.experts.17.down_proj", + "model.layers.26.mlp.experts.17.gate_proj", + "model.layers.26.mlp.experts.17.up_proj", + "model.layers.26.mlp.experts.170.down_proj", + "model.layers.26.mlp.experts.170.gate_proj", + "model.layers.26.mlp.experts.170.up_proj", + "model.layers.26.mlp.experts.171.down_proj", + "model.layers.26.mlp.experts.171.gate_proj", + "model.layers.26.mlp.experts.171.up_proj", + "model.layers.26.mlp.experts.172.down_proj", + "model.layers.26.mlp.experts.172.gate_proj", + "model.layers.26.mlp.experts.172.up_proj", + "model.layers.26.mlp.experts.173.down_proj", + "model.layers.26.mlp.experts.173.gate_proj", + "model.layers.26.mlp.experts.173.up_proj", + "model.layers.26.mlp.experts.174.down_proj", + "model.layers.26.mlp.experts.174.gate_proj", + "model.layers.26.mlp.experts.174.up_proj", + "model.layers.26.mlp.experts.175.down_proj", + "model.layers.26.mlp.experts.175.gate_proj", + "model.layers.26.mlp.experts.175.up_proj", + "model.layers.26.mlp.experts.176.down_proj", + "model.layers.26.mlp.experts.176.gate_proj", + "model.layers.26.mlp.experts.176.up_proj", + "model.layers.26.mlp.experts.177.down_proj", + "model.layers.26.mlp.experts.177.gate_proj", + "model.layers.26.mlp.experts.177.up_proj", + "model.layers.26.mlp.experts.178.down_proj", + "model.layers.26.mlp.experts.178.gate_proj", + "model.layers.26.mlp.experts.178.up_proj", + "model.layers.26.mlp.experts.179.down_proj", + "model.layers.26.mlp.experts.179.gate_proj", + "model.layers.26.mlp.experts.179.up_proj", + "model.layers.26.mlp.experts.18.down_proj", + "model.layers.26.mlp.experts.18.gate_proj", + "model.layers.26.mlp.experts.18.up_proj", + "model.layers.26.mlp.experts.180.down_proj", + "model.layers.26.mlp.experts.180.gate_proj", + "model.layers.26.mlp.experts.180.up_proj", + "model.layers.26.mlp.experts.181.down_proj", + "model.layers.26.mlp.experts.181.gate_proj", + "model.layers.26.mlp.experts.181.up_proj", + "model.layers.26.mlp.experts.182.down_proj", + "model.layers.26.mlp.experts.182.gate_proj", + "model.layers.26.mlp.experts.182.up_proj", + "model.layers.26.mlp.experts.183.down_proj", + "model.layers.26.mlp.experts.183.gate_proj", + "model.layers.26.mlp.experts.183.up_proj", + "model.layers.26.mlp.experts.184.down_proj", + "model.layers.26.mlp.experts.184.gate_proj", + "model.layers.26.mlp.experts.184.up_proj", + "model.layers.26.mlp.experts.185.down_proj", + "model.layers.26.mlp.experts.185.gate_proj", + "model.layers.26.mlp.experts.185.up_proj", + "model.layers.26.mlp.experts.186.down_proj", + "model.layers.26.mlp.experts.186.gate_proj", + "model.layers.26.mlp.experts.186.up_proj", + "model.layers.26.mlp.experts.187.down_proj", + "model.layers.26.mlp.experts.187.gate_proj", + "model.layers.26.mlp.experts.187.up_proj", + "model.layers.26.mlp.experts.188.down_proj", + "model.layers.26.mlp.experts.188.gate_proj", + "model.layers.26.mlp.experts.188.up_proj", + "model.layers.26.mlp.experts.189.down_proj", + "model.layers.26.mlp.experts.189.gate_proj", + "model.layers.26.mlp.experts.189.up_proj", + "model.layers.26.mlp.experts.19.down_proj", + "model.layers.26.mlp.experts.19.gate_proj", + "model.layers.26.mlp.experts.19.up_proj", + "model.layers.26.mlp.experts.190.down_proj", + "model.layers.26.mlp.experts.190.gate_proj", + "model.layers.26.mlp.experts.190.up_proj", + "model.layers.26.mlp.experts.191.down_proj", + "model.layers.26.mlp.experts.191.gate_proj", + "model.layers.26.mlp.experts.191.up_proj", + "model.layers.26.mlp.experts.192.down_proj", + "model.layers.26.mlp.experts.192.gate_proj", + "model.layers.26.mlp.experts.192.up_proj", + "model.layers.26.mlp.experts.193.down_proj", + "model.layers.26.mlp.experts.193.gate_proj", + "model.layers.26.mlp.experts.193.up_proj", + "model.layers.26.mlp.experts.194.down_proj", + "model.layers.26.mlp.experts.194.gate_proj", + "model.layers.26.mlp.experts.194.up_proj", + "model.layers.26.mlp.experts.195.down_proj", + "model.layers.26.mlp.experts.195.gate_proj", + "model.layers.26.mlp.experts.195.up_proj", + "model.layers.26.mlp.experts.196.down_proj", + "model.layers.26.mlp.experts.196.gate_proj", + "model.layers.26.mlp.experts.196.up_proj", + "model.layers.26.mlp.experts.197.down_proj", + "model.layers.26.mlp.experts.197.gate_proj", + "model.layers.26.mlp.experts.197.up_proj", + "model.layers.26.mlp.experts.198.down_proj", + "model.layers.26.mlp.experts.198.gate_proj", + "model.layers.26.mlp.experts.198.up_proj", + "model.layers.26.mlp.experts.199.down_proj", + "model.layers.26.mlp.experts.199.gate_proj", + "model.layers.26.mlp.experts.199.up_proj", + "model.layers.26.mlp.experts.2.down_proj", + "model.layers.26.mlp.experts.2.gate_proj", + "model.layers.26.mlp.experts.2.up_proj", + "model.layers.26.mlp.experts.20.down_proj", + "model.layers.26.mlp.experts.20.gate_proj", + "model.layers.26.mlp.experts.20.up_proj", + "model.layers.26.mlp.experts.200.down_proj", + "model.layers.26.mlp.experts.200.gate_proj", + "model.layers.26.mlp.experts.200.up_proj", + "model.layers.26.mlp.experts.201.down_proj", + "model.layers.26.mlp.experts.201.gate_proj", + "model.layers.26.mlp.experts.201.up_proj", + "model.layers.26.mlp.experts.202.down_proj", + "model.layers.26.mlp.experts.202.gate_proj", + "model.layers.26.mlp.experts.202.up_proj", + "model.layers.26.mlp.experts.203.down_proj", + "model.layers.26.mlp.experts.203.gate_proj", + "model.layers.26.mlp.experts.203.up_proj", + "model.layers.26.mlp.experts.204.down_proj", + "model.layers.26.mlp.experts.204.gate_proj", + "model.layers.26.mlp.experts.204.up_proj", + "model.layers.26.mlp.experts.205.down_proj", + "model.layers.26.mlp.experts.205.gate_proj", + "model.layers.26.mlp.experts.205.up_proj", + "model.layers.26.mlp.experts.206.down_proj", + "model.layers.26.mlp.experts.206.gate_proj", + "model.layers.26.mlp.experts.206.up_proj", + "model.layers.26.mlp.experts.207.down_proj", + "model.layers.26.mlp.experts.207.gate_proj", + "model.layers.26.mlp.experts.207.up_proj", + "model.layers.26.mlp.experts.208.down_proj", + "model.layers.26.mlp.experts.208.gate_proj", + "model.layers.26.mlp.experts.208.up_proj", + "model.layers.26.mlp.experts.209.down_proj", + "model.layers.26.mlp.experts.209.gate_proj", + "model.layers.26.mlp.experts.209.up_proj", + "model.layers.26.mlp.experts.21.down_proj", + "model.layers.26.mlp.experts.21.gate_proj", + "model.layers.26.mlp.experts.21.up_proj", + "model.layers.26.mlp.experts.210.down_proj", + "model.layers.26.mlp.experts.210.gate_proj", + "model.layers.26.mlp.experts.210.up_proj", + "model.layers.26.mlp.experts.211.down_proj", + "model.layers.26.mlp.experts.211.gate_proj", + "model.layers.26.mlp.experts.211.up_proj", + "model.layers.26.mlp.experts.212.down_proj", + "model.layers.26.mlp.experts.212.gate_proj", + "model.layers.26.mlp.experts.212.up_proj", + "model.layers.26.mlp.experts.213.down_proj", + "model.layers.26.mlp.experts.213.gate_proj", + "model.layers.26.mlp.experts.213.up_proj", + "model.layers.26.mlp.experts.214.down_proj", + "model.layers.26.mlp.experts.214.gate_proj", + "model.layers.26.mlp.experts.214.up_proj", + "model.layers.26.mlp.experts.215.down_proj", + "model.layers.26.mlp.experts.215.gate_proj", + "model.layers.26.mlp.experts.215.up_proj", + "model.layers.26.mlp.experts.216.down_proj", + "model.layers.26.mlp.experts.216.gate_proj", + "model.layers.26.mlp.experts.216.up_proj", + "model.layers.26.mlp.experts.217.down_proj", + "model.layers.26.mlp.experts.217.gate_proj", + "model.layers.26.mlp.experts.217.up_proj", + "model.layers.26.mlp.experts.218.down_proj", + "model.layers.26.mlp.experts.218.gate_proj", + "model.layers.26.mlp.experts.218.up_proj", + "model.layers.26.mlp.experts.219.down_proj", + "model.layers.26.mlp.experts.219.gate_proj", + "model.layers.26.mlp.experts.219.up_proj", + "model.layers.26.mlp.experts.22.down_proj", + "model.layers.26.mlp.experts.22.gate_proj", + "model.layers.26.mlp.experts.22.up_proj", + "model.layers.26.mlp.experts.220.down_proj", + "model.layers.26.mlp.experts.220.gate_proj", + "model.layers.26.mlp.experts.220.up_proj", + "model.layers.26.mlp.experts.221.down_proj", + "model.layers.26.mlp.experts.221.gate_proj", + "model.layers.26.mlp.experts.221.up_proj", + "model.layers.26.mlp.experts.222.down_proj", + "model.layers.26.mlp.experts.222.gate_proj", + "model.layers.26.mlp.experts.222.up_proj", + "model.layers.26.mlp.experts.223.down_proj", + "model.layers.26.mlp.experts.223.gate_proj", + "model.layers.26.mlp.experts.223.up_proj", + "model.layers.26.mlp.experts.224.down_proj", + "model.layers.26.mlp.experts.224.gate_proj", + "model.layers.26.mlp.experts.224.up_proj", + "model.layers.26.mlp.experts.225.down_proj", + "model.layers.26.mlp.experts.225.gate_proj", + "model.layers.26.mlp.experts.225.up_proj", + "model.layers.26.mlp.experts.226.down_proj", + "model.layers.26.mlp.experts.226.gate_proj", + "model.layers.26.mlp.experts.226.up_proj", + "model.layers.26.mlp.experts.227.down_proj", + "model.layers.26.mlp.experts.227.gate_proj", + "model.layers.26.mlp.experts.227.up_proj", + "model.layers.26.mlp.experts.228.down_proj", + "model.layers.26.mlp.experts.228.gate_proj", + "model.layers.26.mlp.experts.228.up_proj", + "model.layers.26.mlp.experts.229.down_proj", + "model.layers.26.mlp.experts.229.gate_proj", + "model.layers.26.mlp.experts.229.up_proj", + "model.layers.26.mlp.experts.23.down_proj", + "model.layers.26.mlp.experts.23.gate_proj", + "model.layers.26.mlp.experts.23.up_proj", + "model.layers.26.mlp.experts.230.down_proj", + "model.layers.26.mlp.experts.230.gate_proj", + "model.layers.26.mlp.experts.230.up_proj", + "model.layers.26.mlp.experts.231.down_proj", + "model.layers.26.mlp.experts.231.gate_proj", + "model.layers.26.mlp.experts.231.up_proj", + "model.layers.26.mlp.experts.232.down_proj", + "model.layers.26.mlp.experts.232.gate_proj", + "model.layers.26.mlp.experts.232.up_proj", + "model.layers.26.mlp.experts.233.down_proj", + "model.layers.26.mlp.experts.233.gate_proj", + "model.layers.26.mlp.experts.233.up_proj", + "model.layers.26.mlp.experts.234.down_proj", + "model.layers.26.mlp.experts.234.gate_proj", + "model.layers.26.mlp.experts.234.up_proj", + "model.layers.26.mlp.experts.235.down_proj", + "model.layers.26.mlp.experts.235.gate_proj", + "model.layers.26.mlp.experts.235.up_proj", + "model.layers.26.mlp.experts.236.down_proj", + "model.layers.26.mlp.experts.236.gate_proj", + "model.layers.26.mlp.experts.236.up_proj", + "model.layers.26.mlp.experts.237.down_proj", + "model.layers.26.mlp.experts.237.gate_proj", + "model.layers.26.mlp.experts.237.up_proj", + "model.layers.26.mlp.experts.238.down_proj", + "model.layers.26.mlp.experts.238.gate_proj", + "model.layers.26.mlp.experts.238.up_proj", + "model.layers.26.mlp.experts.239.down_proj", + "model.layers.26.mlp.experts.239.gate_proj", + "model.layers.26.mlp.experts.239.up_proj", + "model.layers.26.mlp.experts.24.down_proj", + "model.layers.26.mlp.experts.24.gate_proj", + "model.layers.26.mlp.experts.24.up_proj", + "model.layers.26.mlp.experts.240.down_proj", + "model.layers.26.mlp.experts.240.gate_proj", + "model.layers.26.mlp.experts.240.up_proj", + "model.layers.26.mlp.experts.241.down_proj", + "model.layers.26.mlp.experts.241.gate_proj", + "model.layers.26.mlp.experts.241.up_proj", + "model.layers.26.mlp.experts.242.down_proj", + "model.layers.26.mlp.experts.242.gate_proj", + "model.layers.26.mlp.experts.242.up_proj", + "model.layers.26.mlp.experts.243.down_proj", + "model.layers.26.mlp.experts.243.gate_proj", + "model.layers.26.mlp.experts.243.up_proj", + "model.layers.26.mlp.experts.244.down_proj", + "model.layers.26.mlp.experts.244.gate_proj", + "model.layers.26.mlp.experts.244.up_proj", + "model.layers.26.mlp.experts.245.down_proj", + "model.layers.26.mlp.experts.245.gate_proj", + "model.layers.26.mlp.experts.245.up_proj", + "model.layers.26.mlp.experts.246.down_proj", + "model.layers.26.mlp.experts.246.gate_proj", + "model.layers.26.mlp.experts.246.up_proj", + "model.layers.26.mlp.experts.247.down_proj", + "model.layers.26.mlp.experts.247.gate_proj", + "model.layers.26.mlp.experts.247.up_proj", + "model.layers.26.mlp.experts.248.down_proj", + "model.layers.26.mlp.experts.248.gate_proj", + "model.layers.26.mlp.experts.248.up_proj", + "model.layers.26.mlp.experts.249.down_proj", + "model.layers.26.mlp.experts.249.gate_proj", + "model.layers.26.mlp.experts.249.up_proj", + "model.layers.26.mlp.experts.25.down_proj", + "model.layers.26.mlp.experts.25.gate_proj", + "model.layers.26.mlp.experts.25.up_proj", + "model.layers.26.mlp.experts.250.down_proj", + "model.layers.26.mlp.experts.250.gate_proj", + "model.layers.26.mlp.experts.250.up_proj", + "model.layers.26.mlp.experts.251.down_proj", + "model.layers.26.mlp.experts.251.gate_proj", + "model.layers.26.mlp.experts.251.up_proj", + "model.layers.26.mlp.experts.252.down_proj", + "model.layers.26.mlp.experts.252.gate_proj", + "model.layers.26.mlp.experts.252.up_proj", + "model.layers.26.mlp.experts.253.down_proj", + "model.layers.26.mlp.experts.253.gate_proj", + "model.layers.26.mlp.experts.253.up_proj", + "model.layers.26.mlp.experts.254.down_proj", + "model.layers.26.mlp.experts.254.gate_proj", + "model.layers.26.mlp.experts.254.up_proj", + "model.layers.26.mlp.experts.255.down_proj", + "model.layers.26.mlp.experts.255.gate_proj", + "model.layers.26.mlp.experts.255.up_proj", + "model.layers.26.mlp.experts.26.down_proj", + "model.layers.26.mlp.experts.26.gate_proj", + "model.layers.26.mlp.experts.26.up_proj", + "model.layers.26.mlp.experts.27.down_proj", + "model.layers.26.mlp.experts.27.gate_proj", + "model.layers.26.mlp.experts.27.up_proj", + "model.layers.26.mlp.experts.28.down_proj", + "model.layers.26.mlp.experts.28.gate_proj", + "model.layers.26.mlp.experts.28.up_proj", + "model.layers.26.mlp.experts.29.down_proj", + "model.layers.26.mlp.experts.29.gate_proj", + "model.layers.26.mlp.experts.29.up_proj", + "model.layers.26.mlp.experts.3.down_proj", + "model.layers.26.mlp.experts.3.gate_proj", + "model.layers.26.mlp.experts.3.up_proj", + "model.layers.26.mlp.experts.30.down_proj", + "model.layers.26.mlp.experts.30.gate_proj", + "model.layers.26.mlp.experts.30.up_proj", + "model.layers.26.mlp.experts.31.down_proj", + "model.layers.26.mlp.experts.31.gate_proj", + "model.layers.26.mlp.experts.31.up_proj", + "model.layers.26.mlp.experts.32.down_proj", + "model.layers.26.mlp.experts.32.gate_proj", + "model.layers.26.mlp.experts.32.up_proj", + "model.layers.26.mlp.experts.33.down_proj", + "model.layers.26.mlp.experts.33.gate_proj", + "model.layers.26.mlp.experts.33.up_proj", + "model.layers.26.mlp.experts.34.down_proj", + "model.layers.26.mlp.experts.34.gate_proj", + "model.layers.26.mlp.experts.34.up_proj", + "model.layers.26.mlp.experts.35.down_proj", + "model.layers.26.mlp.experts.35.gate_proj", + "model.layers.26.mlp.experts.35.up_proj", + "model.layers.26.mlp.experts.36.down_proj", + "model.layers.26.mlp.experts.36.gate_proj", + "model.layers.26.mlp.experts.36.up_proj", + "model.layers.26.mlp.experts.37.down_proj", + "model.layers.26.mlp.experts.37.gate_proj", + "model.layers.26.mlp.experts.37.up_proj", + "model.layers.26.mlp.experts.38.down_proj", + "model.layers.26.mlp.experts.38.gate_proj", + "model.layers.26.mlp.experts.38.up_proj", + "model.layers.26.mlp.experts.39.down_proj", + "model.layers.26.mlp.experts.39.gate_proj", + "model.layers.26.mlp.experts.39.up_proj", + "model.layers.26.mlp.experts.4.down_proj", + "model.layers.26.mlp.experts.4.gate_proj", + "model.layers.26.mlp.experts.4.up_proj", + "model.layers.26.mlp.experts.40.down_proj", + "model.layers.26.mlp.experts.40.gate_proj", + "model.layers.26.mlp.experts.40.up_proj", + "model.layers.26.mlp.experts.41.down_proj", + "model.layers.26.mlp.experts.41.gate_proj", + "model.layers.26.mlp.experts.41.up_proj", + "model.layers.26.mlp.experts.42.down_proj", + "model.layers.26.mlp.experts.42.gate_proj", + "model.layers.26.mlp.experts.42.up_proj", + "model.layers.26.mlp.experts.43.down_proj", + "model.layers.26.mlp.experts.43.gate_proj", + "model.layers.26.mlp.experts.43.up_proj", + "model.layers.26.mlp.experts.44.down_proj", + "model.layers.26.mlp.experts.44.gate_proj", + "model.layers.26.mlp.experts.44.up_proj", + "model.layers.26.mlp.experts.45.down_proj", + "model.layers.26.mlp.experts.45.gate_proj", + "model.layers.26.mlp.experts.45.up_proj", + "model.layers.26.mlp.experts.46.down_proj", + "model.layers.26.mlp.experts.46.gate_proj", + "model.layers.26.mlp.experts.46.up_proj", + "model.layers.26.mlp.experts.47.down_proj", + "model.layers.26.mlp.experts.47.gate_proj", + "model.layers.26.mlp.experts.47.up_proj", + "model.layers.26.mlp.experts.48.down_proj", + "model.layers.26.mlp.experts.48.gate_proj", + "model.layers.26.mlp.experts.48.up_proj", + "model.layers.26.mlp.experts.49.down_proj", + "model.layers.26.mlp.experts.49.gate_proj", + "model.layers.26.mlp.experts.49.up_proj", + "model.layers.26.mlp.experts.5.down_proj", + "model.layers.26.mlp.experts.5.gate_proj", + "model.layers.26.mlp.experts.5.up_proj", + "model.layers.26.mlp.experts.50.down_proj", + "model.layers.26.mlp.experts.50.gate_proj", + "model.layers.26.mlp.experts.50.up_proj", + "model.layers.26.mlp.experts.51.down_proj", + "model.layers.26.mlp.experts.51.gate_proj", + "model.layers.26.mlp.experts.51.up_proj", + "model.layers.26.mlp.experts.52.down_proj", + "model.layers.26.mlp.experts.52.gate_proj", + "model.layers.26.mlp.experts.52.up_proj", + "model.layers.26.mlp.experts.53.down_proj", + "model.layers.26.mlp.experts.53.gate_proj", + "model.layers.26.mlp.experts.53.up_proj", + "model.layers.26.mlp.experts.54.down_proj", + "model.layers.26.mlp.experts.54.gate_proj", + "model.layers.26.mlp.experts.54.up_proj", + "model.layers.26.mlp.experts.55.down_proj", + "model.layers.26.mlp.experts.55.gate_proj", + "model.layers.26.mlp.experts.55.up_proj", + "model.layers.26.mlp.experts.56.down_proj", + "model.layers.26.mlp.experts.56.gate_proj", + "model.layers.26.mlp.experts.56.up_proj", + "model.layers.26.mlp.experts.57.down_proj", + "model.layers.26.mlp.experts.57.gate_proj", + "model.layers.26.mlp.experts.57.up_proj", + "model.layers.26.mlp.experts.58.down_proj", + "model.layers.26.mlp.experts.58.gate_proj", + "model.layers.26.mlp.experts.58.up_proj", + "model.layers.26.mlp.experts.59.down_proj", + "model.layers.26.mlp.experts.59.gate_proj", + "model.layers.26.mlp.experts.59.up_proj", + "model.layers.26.mlp.experts.6.down_proj", + "model.layers.26.mlp.experts.6.gate_proj", + "model.layers.26.mlp.experts.6.up_proj", + "model.layers.26.mlp.experts.60.down_proj", + "model.layers.26.mlp.experts.60.gate_proj", + "model.layers.26.mlp.experts.60.up_proj", + "model.layers.26.mlp.experts.61.down_proj", + "model.layers.26.mlp.experts.61.gate_proj", + "model.layers.26.mlp.experts.61.up_proj", + "model.layers.26.mlp.experts.62.down_proj", + "model.layers.26.mlp.experts.62.gate_proj", + "model.layers.26.mlp.experts.62.up_proj", + "model.layers.26.mlp.experts.63.down_proj", + "model.layers.26.mlp.experts.63.gate_proj", + "model.layers.26.mlp.experts.63.up_proj", + "model.layers.26.mlp.experts.64.down_proj", + "model.layers.26.mlp.experts.64.gate_proj", + "model.layers.26.mlp.experts.64.up_proj", + "model.layers.26.mlp.experts.65.down_proj", + "model.layers.26.mlp.experts.65.gate_proj", + "model.layers.26.mlp.experts.65.up_proj", + "model.layers.26.mlp.experts.66.down_proj", + "model.layers.26.mlp.experts.66.gate_proj", + "model.layers.26.mlp.experts.66.up_proj", + "model.layers.26.mlp.experts.67.down_proj", + "model.layers.26.mlp.experts.67.gate_proj", + "model.layers.26.mlp.experts.67.up_proj", + "model.layers.26.mlp.experts.68.down_proj", + "model.layers.26.mlp.experts.68.gate_proj", + "model.layers.26.mlp.experts.68.up_proj", + "model.layers.26.mlp.experts.69.down_proj", + "model.layers.26.mlp.experts.69.gate_proj", + "model.layers.26.mlp.experts.69.up_proj", + "model.layers.26.mlp.experts.7.down_proj", + "model.layers.26.mlp.experts.7.gate_proj", + "model.layers.26.mlp.experts.7.up_proj", + "model.layers.26.mlp.experts.70.down_proj", + "model.layers.26.mlp.experts.70.gate_proj", + "model.layers.26.mlp.experts.70.up_proj", + "model.layers.26.mlp.experts.71.down_proj", + "model.layers.26.mlp.experts.71.gate_proj", + "model.layers.26.mlp.experts.71.up_proj", + "model.layers.26.mlp.experts.72.down_proj", + "model.layers.26.mlp.experts.72.gate_proj", + "model.layers.26.mlp.experts.72.up_proj", + "model.layers.26.mlp.experts.73.down_proj", + "model.layers.26.mlp.experts.73.gate_proj", + "model.layers.26.mlp.experts.73.up_proj", + "model.layers.26.mlp.experts.74.down_proj", + "model.layers.26.mlp.experts.74.gate_proj", + "model.layers.26.mlp.experts.74.up_proj", + "model.layers.26.mlp.experts.75.down_proj", + "model.layers.26.mlp.experts.75.gate_proj", + "model.layers.26.mlp.experts.75.up_proj", + "model.layers.26.mlp.experts.76.down_proj", + "model.layers.26.mlp.experts.76.gate_proj", + "model.layers.26.mlp.experts.76.up_proj", + "model.layers.26.mlp.experts.77.down_proj", + "model.layers.26.mlp.experts.77.gate_proj", + "model.layers.26.mlp.experts.77.up_proj", + "model.layers.26.mlp.experts.78.down_proj", + "model.layers.26.mlp.experts.78.gate_proj", + "model.layers.26.mlp.experts.78.up_proj", + "model.layers.26.mlp.experts.79.down_proj", + "model.layers.26.mlp.experts.79.gate_proj", + "model.layers.26.mlp.experts.79.up_proj", + "model.layers.26.mlp.experts.8.down_proj", + "model.layers.26.mlp.experts.8.gate_proj", + "model.layers.26.mlp.experts.8.up_proj", + "model.layers.26.mlp.experts.80.down_proj", + "model.layers.26.mlp.experts.80.gate_proj", + "model.layers.26.mlp.experts.80.up_proj", + "model.layers.26.mlp.experts.81.down_proj", + "model.layers.26.mlp.experts.81.gate_proj", + "model.layers.26.mlp.experts.81.up_proj", + "model.layers.26.mlp.experts.82.down_proj", + "model.layers.26.mlp.experts.82.gate_proj", + "model.layers.26.mlp.experts.82.up_proj", + "model.layers.26.mlp.experts.83.down_proj", + "model.layers.26.mlp.experts.83.gate_proj", + "model.layers.26.mlp.experts.83.up_proj", + "model.layers.26.mlp.experts.84.down_proj", + "model.layers.26.mlp.experts.84.gate_proj", + "model.layers.26.mlp.experts.84.up_proj", + "model.layers.26.mlp.experts.85.down_proj", + "model.layers.26.mlp.experts.85.gate_proj", + "model.layers.26.mlp.experts.85.up_proj", + "model.layers.26.mlp.experts.86.down_proj", + "model.layers.26.mlp.experts.86.gate_proj", + "model.layers.26.mlp.experts.86.up_proj", + "model.layers.26.mlp.experts.87.down_proj", + "model.layers.26.mlp.experts.87.gate_proj", + "model.layers.26.mlp.experts.87.up_proj", + "model.layers.26.mlp.experts.88.down_proj", + "model.layers.26.mlp.experts.88.gate_proj", + "model.layers.26.mlp.experts.88.up_proj", + "model.layers.26.mlp.experts.89.down_proj", + "model.layers.26.mlp.experts.89.gate_proj", + "model.layers.26.mlp.experts.89.up_proj", + "model.layers.26.mlp.experts.9.down_proj", + "model.layers.26.mlp.experts.9.gate_proj", + "model.layers.26.mlp.experts.9.up_proj", + "model.layers.26.mlp.experts.90.down_proj", + "model.layers.26.mlp.experts.90.gate_proj", + "model.layers.26.mlp.experts.90.up_proj", + "model.layers.26.mlp.experts.91.down_proj", + "model.layers.26.mlp.experts.91.gate_proj", + "model.layers.26.mlp.experts.91.up_proj", + "model.layers.26.mlp.experts.92.down_proj", + "model.layers.26.mlp.experts.92.gate_proj", + "model.layers.26.mlp.experts.92.up_proj", + "model.layers.26.mlp.experts.93.down_proj", + "model.layers.26.mlp.experts.93.gate_proj", + "model.layers.26.mlp.experts.93.up_proj", + "model.layers.26.mlp.experts.94.down_proj", + "model.layers.26.mlp.experts.94.gate_proj", + "model.layers.26.mlp.experts.94.up_proj", + "model.layers.26.mlp.experts.95.down_proj", + "model.layers.26.mlp.experts.95.gate_proj", + "model.layers.26.mlp.experts.95.up_proj", + "model.layers.26.mlp.experts.96.down_proj", + "model.layers.26.mlp.experts.96.gate_proj", + "model.layers.26.mlp.experts.96.up_proj", + "model.layers.26.mlp.experts.97.down_proj", + "model.layers.26.mlp.experts.97.gate_proj", + "model.layers.26.mlp.experts.97.up_proj", + "model.layers.26.mlp.experts.98.down_proj", + "model.layers.26.mlp.experts.98.gate_proj", + "model.layers.26.mlp.experts.98.up_proj", + "model.layers.26.mlp.experts.99.down_proj", + "model.layers.26.mlp.experts.99.gate_proj", + "model.layers.26.mlp.experts.99.up_proj", + "model.layers.27.mlp.experts.0.down_proj", + "model.layers.27.mlp.experts.0.gate_proj", + "model.layers.27.mlp.experts.0.up_proj", + "model.layers.27.mlp.experts.1.down_proj", + "model.layers.27.mlp.experts.1.gate_proj", + "model.layers.27.mlp.experts.1.up_proj", + "model.layers.27.mlp.experts.10.down_proj", + "model.layers.27.mlp.experts.10.gate_proj", + "model.layers.27.mlp.experts.10.up_proj", + "model.layers.27.mlp.experts.100.down_proj", + "model.layers.27.mlp.experts.100.gate_proj", + "model.layers.27.mlp.experts.100.up_proj", + "model.layers.27.mlp.experts.101.down_proj", + "model.layers.27.mlp.experts.101.gate_proj", + "model.layers.27.mlp.experts.101.up_proj", + "model.layers.27.mlp.experts.102.down_proj", + "model.layers.27.mlp.experts.102.gate_proj", + "model.layers.27.mlp.experts.102.up_proj", + "model.layers.27.mlp.experts.103.down_proj", + "model.layers.27.mlp.experts.103.gate_proj", + "model.layers.27.mlp.experts.103.up_proj", + "model.layers.27.mlp.experts.104.down_proj", + "model.layers.27.mlp.experts.104.gate_proj", + "model.layers.27.mlp.experts.104.up_proj", + "model.layers.27.mlp.experts.105.down_proj", + "model.layers.27.mlp.experts.105.gate_proj", + "model.layers.27.mlp.experts.105.up_proj", + "model.layers.27.mlp.experts.106.down_proj", + "model.layers.27.mlp.experts.106.gate_proj", + "model.layers.27.mlp.experts.106.up_proj", + "model.layers.27.mlp.experts.107.down_proj", + "model.layers.27.mlp.experts.107.gate_proj", + "model.layers.27.mlp.experts.107.up_proj", + "model.layers.27.mlp.experts.108.down_proj", + "model.layers.27.mlp.experts.108.gate_proj", + "model.layers.27.mlp.experts.108.up_proj", + "model.layers.27.mlp.experts.109.down_proj", + "model.layers.27.mlp.experts.109.gate_proj", + "model.layers.27.mlp.experts.109.up_proj", + "model.layers.27.mlp.experts.11.down_proj", + "model.layers.27.mlp.experts.11.gate_proj", + "model.layers.27.mlp.experts.11.up_proj", + "model.layers.27.mlp.experts.110.down_proj", + "model.layers.27.mlp.experts.110.gate_proj", + "model.layers.27.mlp.experts.110.up_proj", + "model.layers.27.mlp.experts.111.down_proj", + "model.layers.27.mlp.experts.111.gate_proj", + "model.layers.27.mlp.experts.111.up_proj", + "model.layers.27.mlp.experts.112.down_proj", + "model.layers.27.mlp.experts.112.gate_proj", + "model.layers.27.mlp.experts.112.up_proj", + "model.layers.27.mlp.experts.113.down_proj", + "model.layers.27.mlp.experts.113.gate_proj", + "model.layers.27.mlp.experts.113.up_proj", + "model.layers.27.mlp.experts.114.down_proj", + "model.layers.27.mlp.experts.114.gate_proj", + "model.layers.27.mlp.experts.114.up_proj", + "model.layers.27.mlp.experts.115.down_proj", + "model.layers.27.mlp.experts.115.gate_proj", + "model.layers.27.mlp.experts.115.up_proj", + "model.layers.27.mlp.experts.116.down_proj", + "model.layers.27.mlp.experts.116.gate_proj", + "model.layers.27.mlp.experts.116.up_proj", + "model.layers.27.mlp.experts.117.down_proj", + "model.layers.27.mlp.experts.117.gate_proj", + "model.layers.27.mlp.experts.117.up_proj", + "model.layers.27.mlp.experts.118.down_proj", + "model.layers.27.mlp.experts.118.gate_proj", + "model.layers.27.mlp.experts.118.up_proj", + "model.layers.27.mlp.experts.119.down_proj", + "model.layers.27.mlp.experts.119.gate_proj", + "model.layers.27.mlp.experts.119.up_proj", + "model.layers.27.mlp.experts.12.down_proj", + "model.layers.27.mlp.experts.12.gate_proj", + "model.layers.27.mlp.experts.12.up_proj", + "model.layers.27.mlp.experts.120.down_proj", + "model.layers.27.mlp.experts.120.gate_proj", + "model.layers.27.mlp.experts.120.up_proj", + "model.layers.27.mlp.experts.121.down_proj", + "model.layers.27.mlp.experts.121.gate_proj", + "model.layers.27.mlp.experts.121.up_proj", + "model.layers.27.mlp.experts.122.down_proj", + "model.layers.27.mlp.experts.122.gate_proj", + "model.layers.27.mlp.experts.122.up_proj", + "model.layers.27.mlp.experts.123.down_proj", + "model.layers.27.mlp.experts.123.gate_proj", + "model.layers.27.mlp.experts.123.up_proj", + "model.layers.27.mlp.experts.124.down_proj", + "model.layers.27.mlp.experts.124.gate_proj", + "model.layers.27.mlp.experts.124.up_proj", + "model.layers.27.mlp.experts.125.down_proj", + "model.layers.27.mlp.experts.125.gate_proj", + "model.layers.27.mlp.experts.125.up_proj", + "model.layers.27.mlp.experts.126.down_proj", + "model.layers.27.mlp.experts.126.gate_proj", + "model.layers.27.mlp.experts.126.up_proj", + "model.layers.27.mlp.experts.127.down_proj", + "model.layers.27.mlp.experts.127.gate_proj", + "model.layers.27.mlp.experts.127.up_proj", + "model.layers.27.mlp.experts.128.down_proj", + "model.layers.27.mlp.experts.128.gate_proj", + "model.layers.27.mlp.experts.128.up_proj", + "model.layers.27.mlp.experts.129.down_proj", + "model.layers.27.mlp.experts.129.gate_proj", + "model.layers.27.mlp.experts.129.up_proj", + "model.layers.27.mlp.experts.13.down_proj", + "model.layers.27.mlp.experts.13.gate_proj", + "model.layers.27.mlp.experts.13.up_proj", + "model.layers.27.mlp.experts.130.down_proj", + "model.layers.27.mlp.experts.130.gate_proj", + "model.layers.27.mlp.experts.130.up_proj", + "model.layers.27.mlp.experts.131.down_proj", + "model.layers.27.mlp.experts.131.gate_proj", + "model.layers.27.mlp.experts.131.up_proj", + "model.layers.27.mlp.experts.132.down_proj", + "model.layers.27.mlp.experts.132.gate_proj", + "model.layers.27.mlp.experts.132.up_proj", + "model.layers.27.mlp.experts.133.down_proj", + "model.layers.27.mlp.experts.133.gate_proj", + "model.layers.27.mlp.experts.133.up_proj", + "model.layers.27.mlp.experts.134.down_proj", + "model.layers.27.mlp.experts.134.gate_proj", + "model.layers.27.mlp.experts.134.up_proj", + "model.layers.27.mlp.experts.135.down_proj", + "model.layers.27.mlp.experts.135.gate_proj", + "model.layers.27.mlp.experts.135.up_proj", + "model.layers.27.mlp.experts.136.down_proj", + "model.layers.27.mlp.experts.136.gate_proj", + "model.layers.27.mlp.experts.136.up_proj", + "model.layers.27.mlp.experts.137.down_proj", + "model.layers.27.mlp.experts.137.gate_proj", + "model.layers.27.mlp.experts.137.up_proj", + "model.layers.27.mlp.experts.138.down_proj", + "model.layers.27.mlp.experts.138.gate_proj", + "model.layers.27.mlp.experts.138.up_proj", + "model.layers.27.mlp.experts.139.down_proj", + "model.layers.27.mlp.experts.139.gate_proj", + "model.layers.27.mlp.experts.139.up_proj", + "model.layers.27.mlp.experts.14.down_proj", + "model.layers.27.mlp.experts.14.gate_proj", + "model.layers.27.mlp.experts.14.up_proj", + "model.layers.27.mlp.experts.140.down_proj", + "model.layers.27.mlp.experts.140.gate_proj", + "model.layers.27.mlp.experts.140.up_proj", + "model.layers.27.mlp.experts.141.down_proj", + "model.layers.27.mlp.experts.141.gate_proj", + "model.layers.27.mlp.experts.141.up_proj", + "model.layers.27.mlp.experts.142.down_proj", + "model.layers.27.mlp.experts.142.gate_proj", + "model.layers.27.mlp.experts.142.up_proj", + "model.layers.27.mlp.experts.143.down_proj", + "model.layers.27.mlp.experts.143.gate_proj", + "model.layers.27.mlp.experts.143.up_proj", + "model.layers.27.mlp.experts.144.down_proj", + "model.layers.27.mlp.experts.144.gate_proj", + "model.layers.27.mlp.experts.144.up_proj", + "model.layers.27.mlp.experts.145.down_proj", + "model.layers.27.mlp.experts.145.gate_proj", + "model.layers.27.mlp.experts.145.up_proj", + "model.layers.27.mlp.experts.146.down_proj", + "model.layers.27.mlp.experts.146.gate_proj", + "model.layers.27.mlp.experts.146.up_proj", + "model.layers.27.mlp.experts.147.down_proj", + "model.layers.27.mlp.experts.147.gate_proj", + "model.layers.27.mlp.experts.147.up_proj", + "model.layers.27.mlp.experts.148.down_proj", + "model.layers.27.mlp.experts.148.gate_proj", + "model.layers.27.mlp.experts.148.up_proj", + "model.layers.27.mlp.experts.149.down_proj", + "model.layers.27.mlp.experts.149.gate_proj", + "model.layers.27.mlp.experts.149.up_proj", + "model.layers.27.mlp.experts.15.down_proj", + "model.layers.27.mlp.experts.15.gate_proj", + "model.layers.27.mlp.experts.15.up_proj", + "model.layers.27.mlp.experts.150.down_proj", + "model.layers.27.mlp.experts.150.gate_proj", + "model.layers.27.mlp.experts.150.up_proj", + "model.layers.27.mlp.experts.151.down_proj", + "model.layers.27.mlp.experts.151.gate_proj", + "model.layers.27.mlp.experts.151.up_proj", + "model.layers.27.mlp.experts.152.down_proj", + "model.layers.27.mlp.experts.152.gate_proj", + "model.layers.27.mlp.experts.152.up_proj", + "model.layers.27.mlp.experts.153.down_proj", + "model.layers.27.mlp.experts.153.gate_proj", + "model.layers.27.mlp.experts.153.up_proj", + "model.layers.27.mlp.experts.154.down_proj", + "model.layers.27.mlp.experts.154.gate_proj", + "model.layers.27.mlp.experts.154.up_proj", + "model.layers.27.mlp.experts.155.down_proj", + "model.layers.27.mlp.experts.155.gate_proj", + "model.layers.27.mlp.experts.155.up_proj", + "model.layers.27.mlp.experts.156.down_proj", + "model.layers.27.mlp.experts.156.gate_proj", + "model.layers.27.mlp.experts.156.up_proj", + "model.layers.27.mlp.experts.157.down_proj", + "model.layers.27.mlp.experts.157.gate_proj", + "model.layers.27.mlp.experts.157.up_proj", + "model.layers.27.mlp.experts.158.down_proj", + "model.layers.27.mlp.experts.158.gate_proj", + "model.layers.27.mlp.experts.158.up_proj", + "model.layers.27.mlp.experts.159.down_proj", + "model.layers.27.mlp.experts.159.gate_proj", + "model.layers.27.mlp.experts.159.up_proj", + "model.layers.27.mlp.experts.16.down_proj", + "model.layers.27.mlp.experts.16.gate_proj", + "model.layers.27.mlp.experts.16.up_proj", + "model.layers.27.mlp.experts.160.down_proj", + "model.layers.27.mlp.experts.160.gate_proj", + "model.layers.27.mlp.experts.160.up_proj", + "model.layers.27.mlp.experts.161.down_proj", + "model.layers.27.mlp.experts.161.gate_proj", + "model.layers.27.mlp.experts.161.up_proj", + "model.layers.27.mlp.experts.162.down_proj", + "model.layers.27.mlp.experts.162.gate_proj", + "model.layers.27.mlp.experts.162.up_proj", + "model.layers.27.mlp.experts.163.down_proj", + "model.layers.27.mlp.experts.163.gate_proj", + "model.layers.27.mlp.experts.163.up_proj", + "model.layers.27.mlp.experts.164.down_proj", + "model.layers.27.mlp.experts.164.gate_proj", + "model.layers.27.mlp.experts.164.up_proj", + "model.layers.27.mlp.experts.165.down_proj", + "model.layers.27.mlp.experts.165.gate_proj", + "model.layers.27.mlp.experts.165.up_proj", + "model.layers.27.mlp.experts.166.down_proj", + "model.layers.27.mlp.experts.166.gate_proj", + "model.layers.27.mlp.experts.166.up_proj", + "model.layers.27.mlp.experts.167.down_proj", + "model.layers.27.mlp.experts.167.gate_proj", + "model.layers.27.mlp.experts.167.up_proj", + "model.layers.27.mlp.experts.168.down_proj", + "model.layers.27.mlp.experts.168.gate_proj", + "model.layers.27.mlp.experts.168.up_proj", + "model.layers.27.mlp.experts.169.down_proj", + "model.layers.27.mlp.experts.169.gate_proj", + "model.layers.27.mlp.experts.169.up_proj", + "model.layers.27.mlp.experts.17.down_proj", + "model.layers.27.mlp.experts.17.gate_proj", + "model.layers.27.mlp.experts.17.up_proj", + "model.layers.27.mlp.experts.170.down_proj", + "model.layers.27.mlp.experts.170.gate_proj", + "model.layers.27.mlp.experts.170.up_proj", + "model.layers.27.mlp.experts.171.down_proj", + "model.layers.27.mlp.experts.171.gate_proj", + "model.layers.27.mlp.experts.171.up_proj", + "model.layers.27.mlp.experts.172.down_proj", + "model.layers.27.mlp.experts.172.gate_proj", + "model.layers.27.mlp.experts.172.up_proj", + "model.layers.27.mlp.experts.173.down_proj", + "model.layers.27.mlp.experts.173.gate_proj", + "model.layers.27.mlp.experts.173.up_proj", + "model.layers.27.mlp.experts.174.down_proj", + "model.layers.27.mlp.experts.174.gate_proj", + "model.layers.27.mlp.experts.174.up_proj", + "model.layers.27.mlp.experts.175.down_proj", + "model.layers.27.mlp.experts.175.gate_proj", + "model.layers.27.mlp.experts.175.up_proj", + "model.layers.27.mlp.experts.176.down_proj", + "model.layers.27.mlp.experts.176.gate_proj", + "model.layers.27.mlp.experts.176.up_proj", + "model.layers.27.mlp.experts.177.down_proj", + "model.layers.27.mlp.experts.177.gate_proj", + "model.layers.27.mlp.experts.177.up_proj", + "model.layers.27.mlp.experts.178.down_proj", + "model.layers.27.mlp.experts.178.gate_proj", + "model.layers.27.mlp.experts.178.up_proj", + "model.layers.27.mlp.experts.179.down_proj", + "model.layers.27.mlp.experts.179.gate_proj", + "model.layers.27.mlp.experts.179.up_proj", + "model.layers.27.mlp.experts.18.down_proj", + "model.layers.27.mlp.experts.18.gate_proj", + "model.layers.27.mlp.experts.18.up_proj", + "model.layers.27.mlp.experts.180.down_proj", + "model.layers.27.mlp.experts.180.gate_proj", + "model.layers.27.mlp.experts.180.up_proj", + "model.layers.27.mlp.experts.181.down_proj", + "model.layers.27.mlp.experts.181.gate_proj", + "model.layers.27.mlp.experts.181.up_proj", + "model.layers.27.mlp.experts.182.down_proj", + "model.layers.27.mlp.experts.182.gate_proj", + "model.layers.27.mlp.experts.182.up_proj", + "model.layers.27.mlp.experts.183.down_proj", + "model.layers.27.mlp.experts.183.gate_proj", + "model.layers.27.mlp.experts.183.up_proj", + "model.layers.27.mlp.experts.184.down_proj", + "model.layers.27.mlp.experts.184.gate_proj", + "model.layers.27.mlp.experts.184.up_proj", + "model.layers.27.mlp.experts.185.down_proj", + "model.layers.27.mlp.experts.185.gate_proj", + "model.layers.27.mlp.experts.185.up_proj", + "model.layers.27.mlp.experts.186.down_proj", + "model.layers.27.mlp.experts.186.gate_proj", + "model.layers.27.mlp.experts.186.up_proj", + "model.layers.27.mlp.experts.187.down_proj", + "model.layers.27.mlp.experts.187.gate_proj", + "model.layers.27.mlp.experts.187.up_proj", + "model.layers.27.mlp.experts.188.down_proj", + "model.layers.27.mlp.experts.188.gate_proj", + "model.layers.27.mlp.experts.188.up_proj", + "model.layers.27.mlp.experts.189.down_proj", + "model.layers.27.mlp.experts.189.gate_proj", + "model.layers.27.mlp.experts.189.up_proj", + "model.layers.27.mlp.experts.19.down_proj", + "model.layers.27.mlp.experts.19.gate_proj", + "model.layers.27.mlp.experts.19.up_proj", + "model.layers.27.mlp.experts.190.down_proj", + "model.layers.27.mlp.experts.190.gate_proj", + "model.layers.27.mlp.experts.190.up_proj", + "model.layers.27.mlp.experts.191.down_proj", + "model.layers.27.mlp.experts.191.gate_proj", + "model.layers.27.mlp.experts.191.up_proj", + "model.layers.27.mlp.experts.192.down_proj", + "model.layers.27.mlp.experts.192.gate_proj", + "model.layers.27.mlp.experts.192.up_proj", + "model.layers.27.mlp.experts.193.down_proj", + "model.layers.27.mlp.experts.193.gate_proj", + "model.layers.27.mlp.experts.193.up_proj", + "model.layers.27.mlp.experts.194.down_proj", + "model.layers.27.mlp.experts.194.gate_proj", + "model.layers.27.mlp.experts.194.up_proj", + "model.layers.27.mlp.experts.195.down_proj", + "model.layers.27.mlp.experts.195.gate_proj", + "model.layers.27.mlp.experts.195.up_proj", + "model.layers.27.mlp.experts.196.down_proj", + "model.layers.27.mlp.experts.196.gate_proj", + "model.layers.27.mlp.experts.196.up_proj", + "model.layers.27.mlp.experts.197.down_proj", + "model.layers.27.mlp.experts.197.gate_proj", + "model.layers.27.mlp.experts.197.up_proj", + "model.layers.27.mlp.experts.198.down_proj", + "model.layers.27.mlp.experts.198.gate_proj", + "model.layers.27.mlp.experts.198.up_proj", + "model.layers.27.mlp.experts.199.down_proj", + "model.layers.27.mlp.experts.199.gate_proj", + "model.layers.27.mlp.experts.199.up_proj", + "model.layers.27.mlp.experts.2.down_proj", + "model.layers.27.mlp.experts.2.gate_proj", + "model.layers.27.mlp.experts.2.up_proj", + "model.layers.27.mlp.experts.20.down_proj", + "model.layers.27.mlp.experts.20.gate_proj", + "model.layers.27.mlp.experts.20.up_proj", + "model.layers.27.mlp.experts.200.down_proj", + "model.layers.27.mlp.experts.200.gate_proj", + "model.layers.27.mlp.experts.200.up_proj", + "model.layers.27.mlp.experts.201.down_proj", + "model.layers.27.mlp.experts.201.gate_proj", + "model.layers.27.mlp.experts.201.up_proj", + "model.layers.27.mlp.experts.202.down_proj", + "model.layers.27.mlp.experts.202.gate_proj", + "model.layers.27.mlp.experts.202.up_proj", + "model.layers.27.mlp.experts.203.down_proj", + "model.layers.27.mlp.experts.203.gate_proj", + "model.layers.27.mlp.experts.203.up_proj", + "model.layers.27.mlp.experts.204.down_proj", + "model.layers.27.mlp.experts.204.gate_proj", + "model.layers.27.mlp.experts.204.up_proj", + "model.layers.27.mlp.experts.205.down_proj", + "model.layers.27.mlp.experts.205.gate_proj", + "model.layers.27.mlp.experts.205.up_proj", + "model.layers.27.mlp.experts.206.down_proj", + "model.layers.27.mlp.experts.206.gate_proj", + "model.layers.27.mlp.experts.206.up_proj", + "model.layers.27.mlp.experts.207.down_proj", + "model.layers.27.mlp.experts.207.gate_proj", + "model.layers.27.mlp.experts.207.up_proj", + "model.layers.27.mlp.experts.208.down_proj", + "model.layers.27.mlp.experts.208.gate_proj", + "model.layers.27.mlp.experts.208.up_proj", + "model.layers.27.mlp.experts.209.down_proj", + "model.layers.27.mlp.experts.209.gate_proj", + "model.layers.27.mlp.experts.209.up_proj", + "model.layers.27.mlp.experts.21.down_proj", + "model.layers.27.mlp.experts.21.gate_proj", + "model.layers.27.mlp.experts.21.up_proj", + "model.layers.27.mlp.experts.210.down_proj", + "model.layers.27.mlp.experts.210.gate_proj", + "model.layers.27.mlp.experts.210.up_proj", + "model.layers.27.mlp.experts.211.down_proj", + "model.layers.27.mlp.experts.211.gate_proj", + "model.layers.27.mlp.experts.211.up_proj", + "model.layers.27.mlp.experts.212.down_proj", + "model.layers.27.mlp.experts.212.gate_proj", + "model.layers.27.mlp.experts.212.up_proj", + "model.layers.27.mlp.experts.213.down_proj", + "model.layers.27.mlp.experts.213.gate_proj", + "model.layers.27.mlp.experts.213.up_proj", + "model.layers.27.mlp.experts.214.down_proj", + "model.layers.27.mlp.experts.214.gate_proj", + "model.layers.27.mlp.experts.214.up_proj", + "model.layers.27.mlp.experts.215.down_proj", + "model.layers.27.mlp.experts.215.gate_proj", + "model.layers.27.mlp.experts.215.up_proj", + "model.layers.27.mlp.experts.216.down_proj", + "model.layers.27.mlp.experts.216.gate_proj", + "model.layers.27.mlp.experts.216.up_proj", + "model.layers.27.mlp.experts.217.down_proj", + "model.layers.27.mlp.experts.217.gate_proj", + "model.layers.27.mlp.experts.217.up_proj", + "model.layers.27.mlp.experts.218.down_proj", + "model.layers.27.mlp.experts.218.gate_proj", + "model.layers.27.mlp.experts.218.up_proj", + "model.layers.27.mlp.experts.219.down_proj", + "model.layers.27.mlp.experts.219.gate_proj", + "model.layers.27.mlp.experts.219.up_proj", + "model.layers.27.mlp.experts.22.down_proj", + "model.layers.27.mlp.experts.22.gate_proj", + "model.layers.27.mlp.experts.22.up_proj", + "model.layers.27.mlp.experts.220.down_proj", + "model.layers.27.mlp.experts.220.gate_proj", + "model.layers.27.mlp.experts.220.up_proj", + "model.layers.27.mlp.experts.221.down_proj", + "model.layers.27.mlp.experts.221.gate_proj", + "model.layers.27.mlp.experts.221.up_proj", + "model.layers.27.mlp.experts.222.down_proj", + "model.layers.27.mlp.experts.222.gate_proj", + "model.layers.27.mlp.experts.222.up_proj", + "model.layers.27.mlp.experts.223.down_proj", + "model.layers.27.mlp.experts.223.gate_proj", + "model.layers.27.mlp.experts.223.up_proj", + "model.layers.27.mlp.experts.224.down_proj", + "model.layers.27.mlp.experts.224.gate_proj", + "model.layers.27.mlp.experts.224.up_proj", + "model.layers.27.mlp.experts.225.down_proj", + "model.layers.27.mlp.experts.225.gate_proj", + "model.layers.27.mlp.experts.225.up_proj", + "model.layers.27.mlp.experts.226.down_proj", + "model.layers.27.mlp.experts.226.gate_proj", + "model.layers.27.mlp.experts.226.up_proj", + "model.layers.27.mlp.experts.227.down_proj", + "model.layers.27.mlp.experts.227.gate_proj", + "model.layers.27.mlp.experts.227.up_proj", + "model.layers.27.mlp.experts.228.down_proj", + "model.layers.27.mlp.experts.228.gate_proj", + "model.layers.27.mlp.experts.228.up_proj", + "model.layers.27.mlp.experts.229.down_proj", + "model.layers.27.mlp.experts.229.gate_proj", + "model.layers.27.mlp.experts.229.up_proj", + "model.layers.27.mlp.experts.23.down_proj", + "model.layers.27.mlp.experts.23.gate_proj", + "model.layers.27.mlp.experts.23.up_proj", + "model.layers.27.mlp.experts.230.down_proj", + "model.layers.27.mlp.experts.230.gate_proj", + "model.layers.27.mlp.experts.230.up_proj", + "model.layers.27.mlp.experts.231.down_proj", + "model.layers.27.mlp.experts.231.gate_proj", + "model.layers.27.mlp.experts.231.up_proj", + "model.layers.27.mlp.experts.232.down_proj", + "model.layers.27.mlp.experts.232.gate_proj", + "model.layers.27.mlp.experts.232.up_proj", + "model.layers.27.mlp.experts.233.down_proj", + "model.layers.27.mlp.experts.233.gate_proj", + "model.layers.27.mlp.experts.233.up_proj", + "model.layers.27.mlp.experts.234.down_proj", + "model.layers.27.mlp.experts.234.gate_proj", + "model.layers.27.mlp.experts.234.up_proj", + "model.layers.27.mlp.experts.235.down_proj", + "model.layers.27.mlp.experts.235.gate_proj", + "model.layers.27.mlp.experts.235.up_proj", + "model.layers.27.mlp.experts.236.down_proj", + "model.layers.27.mlp.experts.236.gate_proj", + "model.layers.27.mlp.experts.236.up_proj", + "model.layers.27.mlp.experts.237.down_proj", + "model.layers.27.mlp.experts.237.gate_proj", + "model.layers.27.mlp.experts.237.up_proj", + "model.layers.27.mlp.experts.238.down_proj", + "model.layers.27.mlp.experts.238.gate_proj", + "model.layers.27.mlp.experts.238.up_proj", + "model.layers.27.mlp.experts.239.down_proj", + "model.layers.27.mlp.experts.239.gate_proj", + "model.layers.27.mlp.experts.239.up_proj", + "model.layers.27.mlp.experts.24.down_proj", + "model.layers.27.mlp.experts.24.gate_proj", + "model.layers.27.mlp.experts.24.up_proj", + "model.layers.27.mlp.experts.240.down_proj", + "model.layers.27.mlp.experts.240.gate_proj", + "model.layers.27.mlp.experts.240.up_proj", + "model.layers.27.mlp.experts.241.down_proj", + "model.layers.27.mlp.experts.241.gate_proj", + "model.layers.27.mlp.experts.241.up_proj", + "model.layers.27.mlp.experts.242.down_proj", + "model.layers.27.mlp.experts.242.gate_proj", + "model.layers.27.mlp.experts.242.up_proj", + "model.layers.27.mlp.experts.243.down_proj", + "model.layers.27.mlp.experts.243.gate_proj", + "model.layers.27.mlp.experts.243.up_proj", + "model.layers.27.mlp.experts.244.down_proj", + "model.layers.27.mlp.experts.244.gate_proj", + "model.layers.27.mlp.experts.244.up_proj", + "model.layers.27.mlp.experts.245.down_proj", + "model.layers.27.mlp.experts.245.gate_proj", + "model.layers.27.mlp.experts.245.up_proj", + "model.layers.27.mlp.experts.246.down_proj", + "model.layers.27.mlp.experts.246.gate_proj", + "model.layers.27.mlp.experts.246.up_proj", + "model.layers.27.mlp.experts.247.down_proj", + "model.layers.27.mlp.experts.247.gate_proj", + "model.layers.27.mlp.experts.247.up_proj", + "model.layers.27.mlp.experts.248.down_proj", + "model.layers.27.mlp.experts.248.gate_proj", + "model.layers.27.mlp.experts.248.up_proj", + "model.layers.27.mlp.experts.249.down_proj", + "model.layers.27.mlp.experts.249.gate_proj", + "model.layers.27.mlp.experts.249.up_proj", + "model.layers.27.mlp.experts.25.down_proj", + "model.layers.27.mlp.experts.25.gate_proj", + "model.layers.27.mlp.experts.25.up_proj", + "model.layers.27.mlp.experts.250.down_proj", + "model.layers.27.mlp.experts.250.gate_proj", + "model.layers.27.mlp.experts.250.up_proj", + "model.layers.27.mlp.experts.251.down_proj", + "model.layers.27.mlp.experts.251.gate_proj", + "model.layers.27.mlp.experts.251.up_proj", + "model.layers.27.mlp.experts.252.down_proj", + "model.layers.27.mlp.experts.252.gate_proj", + "model.layers.27.mlp.experts.252.up_proj", + "model.layers.27.mlp.experts.253.down_proj", + "model.layers.27.mlp.experts.253.gate_proj", + "model.layers.27.mlp.experts.253.up_proj", + "model.layers.27.mlp.experts.254.down_proj", + "model.layers.27.mlp.experts.254.gate_proj", + "model.layers.27.mlp.experts.254.up_proj", + "model.layers.27.mlp.experts.255.down_proj", + "model.layers.27.mlp.experts.255.gate_proj", + "model.layers.27.mlp.experts.255.up_proj", + "model.layers.27.mlp.experts.26.down_proj", + "model.layers.27.mlp.experts.26.gate_proj", + "model.layers.27.mlp.experts.26.up_proj", + "model.layers.27.mlp.experts.27.down_proj", + "model.layers.27.mlp.experts.27.gate_proj", + "model.layers.27.mlp.experts.27.up_proj", + "model.layers.27.mlp.experts.28.down_proj", + "model.layers.27.mlp.experts.28.gate_proj", + "model.layers.27.mlp.experts.28.up_proj", + "model.layers.27.mlp.experts.29.down_proj", + "model.layers.27.mlp.experts.29.gate_proj", + "model.layers.27.mlp.experts.29.up_proj", + "model.layers.27.mlp.experts.3.down_proj", + "model.layers.27.mlp.experts.3.gate_proj", + "model.layers.27.mlp.experts.3.up_proj", + "model.layers.27.mlp.experts.30.down_proj", + "model.layers.27.mlp.experts.30.gate_proj", + "model.layers.27.mlp.experts.30.up_proj", + "model.layers.27.mlp.experts.31.down_proj", + "model.layers.27.mlp.experts.31.gate_proj", + "model.layers.27.mlp.experts.31.up_proj", + "model.layers.27.mlp.experts.32.down_proj", + "model.layers.27.mlp.experts.32.gate_proj", + "model.layers.27.mlp.experts.32.up_proj", + "model.layers.27.mlp.experts.33.down_proj", + "model.layers.27.mlp.experts.33.gate_proj", + "model.layers.27.mlp.experts.33.up_proj", + "model.layers.27.mlp.experts.34.down_proj", + "model.layers.27.mlp.experts.34.gate_proj", + "model.layers.27.mlp.experts.34.up_proj", + "model.layers.27.mlp.experts.35.down_proj", + "model.layers.27.mlp.experts.35.gate_proj", + "model.layers.27.mlp.experts.35.up_proj", + "model.layers.27.mlp.experts.36.down_proj", + "model.layers.27.mlp.experts.36.gate_proj", + "model.layers.27.mlp.experts.36.up_proj", + "model.layers.27.mlp.experts.37.down_proj", + "model.layers.27.mlp.experts.37.gate_proj", + "model.layers.27.mlp.experts.37.up_proj", + "model.layers.27.mlp.experts.38.down_proj", + "model.layers.27.mlp.experts.38.gate_proj", + "model.layers.27.mlp.experts.38.up_proj", + "model.layers.27.mlp.experts.39.down_proj", + "model.layers.27.mlp.experts.39.gate_proj", + "model.layers.27.mlp.experts.39.up_proj", + "model.layers.27.mlp.experts.4.down_proj", + "model.layers.27.mlp.experts.4.gate_proj", + "model.layers.27.mlp.experts.4.up_proj", + "model.layers.27.mlp.experts.40.down_proj", + "model.layers.27.mlp.experts.40.gate_proj", + "model.layers.27.mlp.experts.40.up_proj", + "model.layers.27.mlp.experts.41.down_proj", + "model.layers.27.mlp.experts.41.gate_proj", + "model.layers.27.mlp.experts.41.up_proj", + "model.layers.27.mlp.experts.42.down_proj", + "model.layers.27.mlp.experts.42.gate_proj", + "model.layers.27.mlp.experts.42.up_proj", + "model.layers.27.mlp.experts.43.down_proj", + "model.layers.27.mlp.experts.43.gate_proj", + "model.layers.27.mlp.experts.43.up_proj", + "model.layers.27.mlp.experts.44.down_proj", + "model.layers.27.mlp.experts.44.gate_proj", + "model.layers.27.mlp.experts.44.up_proj", + "model.layers.27.mlp.experts.45.down_proj", + "model.layers.27.mlp.experts.45.gate_proj", + "model.layers.27.mlp.experts.45.up_proj", + "model.layers.27.mlp.experts.46.down_proj", + "model.layers.27.mlp.experts.46.gate_proj", + "model.layers.27.mlp.experts.46.up_proj", + "model.layers.27.mlp.experts.47.down_proj", + "model.layers.27.mlp.experts.47.gate_proj", + "model.layers.27.mlp.experts.47.up_proj", + "model.layers.27.mlp.experts.48.down_proj", + "model.layers.27.mlp.experts.48.gate_proj", + "model.layers.27.mlp.experts.48.up_proj", + "model.layers.27.mlp.experts.49.down_proj", + "model.layers.27.mlp.experts.49.gate_proj", + "model.layers.27.mlp.experts.49.up_proj", + "model.layers.27.mlp.experts.5.down_proj", + "model.layers.27.mlp.experts.5.gate_proj", + "model.layers.27.mlp.experts.5.up_proj", + "model.layers.27.mlp.experts.50.down_proj", + "model.layers.27.mlp.experts.50.gate_proj", + "model.layers.27.mlp.experts.50.up_proj", + "model.layers.27.mlp.experts.51.down_proj", + "model.layers.27.mlp.experts.51.gate_proj", + "model.layers.27.mlp.experts.51.up_proj", + "model.layers.27.mlp.experts.52.down_proj", + "model.layers.27.mlp.experts.52.gate_proj", + "model.layers.27.mlp.experts.52.up_proj", + "model.layers.27.mlp.experts.53.down_proj", + "model.layers.27.mlp.experts.53.gate_proj", + "model.layers.27.mlp.experts.53.up_proj", + "model.layers.27.mlp.experts.54.down_proj", + "model.layers.27.mlp.experts.54.gate_proj", + "model.layers.27.mlp.experts.54.up_proj", + "model.layers.27.mlp.experts.55.down_proj", + "model.layers.27.mlp.experts.55.gate_proj", + "model.layers.27.mlp.experts.55.up_proj", + "model.layers.27.mlp.experts.56.down_proj", + "model.layers.27.mlp.experts.56.gate_proj", + "model.layers.27.mlp.experts.56.up_proj", + "model.layers.27.mlp.experts.57.down_proj", + "model.layers.27.mlp.experts.57.gate_proj", + "model.layers.27.mlp.experts.57.up_proj", + "model.layers.27.mlp.experts.58.down_proj", + "model.layers.27.mlp.experts.58.gate_proj", + "model.layers.27.mlp.experts.58.up_proj", + "model.layers.27.mlp.experts.59.down_proj", + "model.layers.27.mlp.experts.59.gate_proj", + "model.layers.27.mlp.experts.59.up_proj", + "model.layers.27.mlp.experts.6.down_proj", + "model.layers.27.mlp.experts.6.gate_proj", + "model.layers.27.mlp.experts.6.up_proj", + "model.layers.27.mlp.experts.60.down_proj", + "model.layers.27.mlp.experts.60.gate_proj", + "model.layers.27.mlp.experts.60.up_proj", + "model.layers.27.mlp.experts.61.down_proj", + "model.layers.27.mlp.experts.61.gate_proj", + "model.layers.27.mlp.experts.61.up_proj", + "model.layers.27.mlp.experts.62.down_proj", + "model.layers.27.mlp.experts.62.gate_proj", + "model.layers.27.mlp.experts.62.up_proj", + "model.layers.27.mlp.experts.63.down_proj", + "model.layers.27.mlp.experts.63.gate_proj", + "model.layers.27.mlp.experts.63.up_proj", + "model.layers.27.mlp.experts.64.down_proj", + "model.layers.27.mlp.experts.64.gate_proj", + "model.layers.27.mlp.experts.64.up_proj", + "model.layers.27.mlp.experts.65.down_proj", + "model.layers.27.mlp.experts.65.gate_proj", + "model.layers.27.mlp.experts.65.up_proj", + "model.layers.27.mlp.experts.66.down_proj", + "model.layers.27.mlp.experts.66.gate_proj", + "model.layers.27.mlp.experts.66.up_proj", + "model.layers.27.mlp.experts.67.down_proj", + "model.layers.27.mlp.experts.67.gate_proj", + "model.layers.27.mlp.experts.67.up_proj", + "model.layers.27.mlp.experts.68.down_proj", + "model.layers.27.mlp.experts.68.gate_proj", + "model.layers.27.mlp.experts.68.up_proj", + "model.layers.27.mlp.experts.69.down_proj", + "model.layers.27.mlp.experts.69.gate_proj", + "model.layers.27.mlp.experts.69.up_proj", + "model.layers.27.mlp.experts.7.down_proj", + "model.layers.27.mlp.experts.7.gate_proj", + "model.layers.27.mlp.experts.7.up_proj", + "model.layers.27.mlp.experts.70.down_proj", + "model.layers.27.mlp.experts.70.gate_proj", + "model.layers.27.mlp.experts.70.up_proj", + "model.layers.27.mlp.experts.71.down_proj", + "model.layers.27.mlp.experts.71.gate_proj", + "model.layers.27.mlp.experts.71.up_proj", + "model.layers.27.mlp.experts.72.down_proj", + "model.layers.27.mlp.experts.72.gate_proj", + "model.layers.27.mlp.experts.72.up_proj", + "model.layers.27.mlp.experts.73.down_proj", + "model.layers.27.mlp.experts.73.gate_proj", + "model.layers.27.mlp.experts.73.up_proj", + "model.layers.27.mlp.experts.74.down_proj", + "model.layers.27.mlp.experts.74.gate_proj", + "model.layers.27.mlp.experts.74.up_proj", + "model.layers.27.mlp.experts.75.down_proj", + "model.layers.27.mlp.experts.75.gate_proj", + "model.layers.27.mlp.experts.75.up_proj", + "model.layers.27.mlp.experts.76.down_proj", + "model.layers.27.mlp.experts.76.gate_proj", + "model.layers.27.mlp.experts.76.up_proj", + "model.layers.27.mlp.experts.77.down_proj", + "model.layers.27.mlp.experts.77.gate_proj", + "model.layers.27.mlp.experts.77.up_proj", + "model.layers.27.mlp.experts.78.down_proj", + "model.layers.27.mlp.experts.78.gate_proj", + "model.layers.27.mlp.experts.78.up_proj", + "model.layers.27.mlp.experts.79.down_proj", + "model.layers.27.mlp.experts.79.gate_proj", + "model.layers.27.mlp.experts.79.up_proj", + "model.layers.27.mlp.experts.8.down_proj", + "model.layers.27.mlp.experts.8.gate_proj", + "model.layers.27.mlp.experts.8.up_proj", + "model.layers.27.mlp.experts.80.down_proj", + "model.layers.27.mlp.experts.80.gate_proj", + "model.layers.27.mlp.experts.80.up_proj", + "model.layers.27.mlp.experts.81.down_proj", + "model.layers.27.mlp.experts.81.gate_proj", + "model.layers.27.mlp.experts.81.up_proj", + "model.layers.27.mlp.experts.82.down_proj", + "model.layers.27.mlp.experts.82.gate_proj", + "model.layers.27.mlp.experts.82.up_proj", + "model.layers.27.mlp.experts.83.down_proj", + "model.layers.27.mlp.experts.83.gate_proj", + "model.layers.27.mlp.experts.83.up_proj", + "model.layers.27.mlp.experts.84.down_proj", + "model.layers.27.mlp.experts.84.gate_proj", + "model.layers.27.mlp.experts.84.up_proj", + "model.layers.27.mlp.experts.85.down_proj", + "model.layers.27.mlp.experts.85.gate_proj", + "model.layers.27.mlp.experts.85.up_proj", + "model.layers.27.mlp.experts.86.down_proj", + "model.layers.27.mlp.experts.86.gate_proj", + "model.layers.27.mlp.experts.86.up_proj", + "model.layers.27.mlp.experts.87.down_proj", + "model.layers.27.mlp.experts.87.gate_proj", + "model.layers.27.mlp.experts.87.up_proj", + "model.layers.27.mlp.experts.88.down_proj", + "model.layers.27.mlp.experts.88.gate_proj", + "model.layers.27.mlp.experts.88.up_proj", + "model.layers.27.mlp.experts.89.down_proj", + "model.layers.27.mlp.experts.89.gate_proj", + "model.layers.27.mlp.experts.89.up_proj", + "model.layers.27.mlp.experts.9.down_proj", + "model.layers.27.mlp.experts.9.gate_proj", + "model.layers.27.mlp.experts.9.up_proj", + "model.layers.27.mlp.experts.90.down_proj", + "model.layers.27.mlp.experts.90.gate_proj", + "model.layers.27.mlp.experts.90.up_proj", + "model.layers.27.mlp.experts.91.down_proj", + "model.layers.27.mlp.experts.91.gate_proj", + "model.layers.27.mlp.experts.91.up_proj", + "model.layers.27.mlp.experts.92.down_proj", + "model.layers.27.mlp.experts.92.gate_proj", + "model.layers.27.mlp.experts.92.up_proj", + "model.layers.27.mlp.experts.93.down_proj", + "model.layers.27.mlp.experts.93.gate_proj", + "model.layers.27.mlp.experts.93.up_proj", + "model.layers.27.mlp.experts.94.down_proj", + "model.layers.27.mlp.experts.94.gate_proj", + "model.layers.27.mlp.experts.94.up_proj", + "model.layers.27.mlp.experts.95.down_proj", + "model.layers.27.mlp.experts.95.gate_proj", + "model.layers.27.mlp.experts.95.up_proj", + "model.layers.27.mlp.experts.96.down_proj", + "model.layers.27.mlp.experts.96.gate_proj", + "model.layers.27.mlp.experts.96.up_proj", + "model.layers.27.mlp.experts.97.down_proj", + "model.layers.27.mlp.experts.97.gate_proj", + "model.layers.27.mlp.experts.97.up_proj", + "model.layers.27.mlp.experts.98.down_proj", + "model.layers.27.mlp.experts.98.gate_proj", + "model.layers.27.mlp.experts.98.up_proj", + "model.layers.27.mlp.experts.99.down_proj", + "model.layers.27.mlp.experts.99.gate_proj", + "model.layers.27.mlp.experts.99.up_proj", + "model.layers.28.mlp.experts.0.down_proj", + "model.layers.28.mlp.experts.0.gate_proj", + "model.layers.28.mlp.experts.0.up_proj", + "model.layers.28.mlp.experts.1.down_proj", + "model.layers.28.mlp.experts.1.gate_proj", + "model.layers.28.mlp.experts.1.up_proj", + "model.layers.28.mlp.experts.10.down_proj", + "model.layers.28.mlp.experts.10.gate_proj", + "model.layers.28.mlp.experts.10.up_proj", + "model.layers.28.mlp.experts.100.down_proj", + "model.layers.28.mlp.experts.100.gate_proj", + "model.layers.28.mlp.experts.100.up_proj", + "model.layers.28.mlp.experts.101.down_proj", + "model.layers.28.mlp.experts.101.gate_proj", + "model.layers.28.mlp.experts.101.up_proj", + "model.layers.28.mlp.experts.102.down_proj", + "model.layers.28.mlp.experts.102.gate_proj", + "model.layers.28.mlp.experts.102.up_proj", + "model.layers.28.mlp.experts.103.down_proj", + "model.layers.28.mlp.experts.103.gate_proj", + "model.layers.28.mlp.experts.103.up_proj", + "model.layers.28.mlp.experts.104.down_proj", + "model.layers.28.mlp.experts.104.gate_proj", + "model.layers.28.mlp.experts.104.up_proj", + "model.layers.28.mlp.experts.105.down_proj", + "model.layers.28.mlp.experts.105.gate_proj", + "model.layers.28.mlp.experts.105.up_proj", + "model.layers.28.mlp.experts.106.down_proj", + "model.layers.28.mlp.experts.106.gate_proj", + "model.layers.28.mlp.experts.106.up_proj", + "model.layers.28.mlp.experts.107.down_proj", + "model.layers.28.mlp.experts.107.gate_proj", + "model.layers.28.mlp.experts.107.up_proj", + "model.layers.28.mlp.experts.108.down_proj", + "model.layers.28.mlp.experts.108.gate_proj", + "model.layers.28.mlp.experts.108.up_proj", + "model.layers.28.mlp.experts.109.down_proj", + "model.layers.28.mlp.experts.109.gate_proj", + "model.layers.28.mlp.experts.109.up_proj", + "model.layers.28.mlp.experts.11.down_proj", + "model.layers.28.mlp.experts.11.gate_proj", + "model.layers.28.mlp.experts.11.up_proj", + "model.layers.28.mlp.experts.110.down_proj", + "model.layers.28.mlp.experts.110.gate_proj", + "model.layers.28.mlp.experts.110.up_proj", + "model.layers.28.mlp.experts.111.down_proj", + "model.layers.28.mlp.experts.111.gate_proj", + "model.layers.28.mlp.experts.111.up_proj", + "model.layers.28.mlp.experts.112.down_proj", + "model.layers.28.mlp.experts.112.gate_proj", + "model.layers.28.mlp.experts.112.up_proj", + "model.layers.28.mlp.experts.113.down_proj", + "model.layers.28.mlp.experts.113.gate_proj", + "model.layers.28.mlp.experts.113.up_proj", + "model.layers.28.mlp.experts.114.down_proj", + "model.layers.28.mlp.experts.114.gate_proj", + "model.layers.28.mlp.experts.114.up_proj", + "model.layers.28.mlp.experts.115.down_proj", + "model.layers.28.mlp.experts.115.gate_proj", + "model.layers.28.mlp.experts.115.up_proj", + "model.layers.28.mlp.experts.116.down_proj", + "model.layers.28.mlp.experts.116.gate_proj", + "model.layers.28.mlp.experts.116.up_proj", + "model.layers.28.mlp.experts.117.down_proj", + "model.layers.28.mlp.experts.117.gate_proj", + "model.layers.28.mlp.experts.117.up_proj", + "model.layers.28.mlp.experts.118.down_proj", + "model.layers.28.mlp.experts.118.gate_proj", + "model.layers.28.mlp.experts.118.up_proj", + "model.layers.28.mlp.experts.119.down_proj", + "model.layers.28.mlp.experts.119.gate_proj", + "model.layers.28.mlp.experts.119.up_proj", + "model.layers.28.mlp.experts.12.down_proj", + "model.layers.28.mlp.experts.12.gate_proj", + "model.layers.28.mlp.experts.12.up_proj", + "model.layers.28.mlp.experts.120.down_proj", + "model.layers.28.mlp.experts.120.gate_proj", + "model.layers.28.mlp.experts.120.up_proj", + "model.layers.28.mlp.experts.121.down_proj", + "model.layers.28.mlp.experts.121.gate_proj", + "model.layers.28.mlp.experts.121.up_proj", + "model.layers.28.mlp.experts.122.down_proj", + "model.layers.28.mlp.experts.122.gate_proj", + "model.layers.28.mlp.experts.122.up_proj", + "model.layers.28.mlp.experts.123.down_proj", + "model.layers.28.mlp.experts.123.gate_proj", + "model.layers.28.mlp.experts.123.up_proj", + "model.layers.28.mlp.experts.124.down_proj", + "model.layers.28.mlp.experts.124.gate_proj", + "model.layers.28.mlp.experts.124.up_proj", + "model.layers.28.mlp.experts.125.down_proj", + "model.layers.28.mlp.experts.125.gate_proj", + "model.layers.28.mlp.experts.125.up_proj", + "model.layers.28.mlp.experts.126.down_proj", + "model.layers.28.mlp.experts.126.gate_proj", + "model.layers.28.mlp.experts.126.up_proj", + "model.layers.28.mlp.experts.127.down_proj", + "model.layers.28.mlp.experts.127.gate_proj", + "model.layers.28.mlp.experts.127.up_proj", + "model.layers.28.mlp.experts.128.down_proj", + "model.layers.28.mlp.experts.128.gate_proj", + "model.layers.28.mlp.experts.128.up_proj", + "model.layers.28.mlp.experts.129.down_proj", + "model.layers.28.mlp.experts.129.gate_proj", + "model.layers.28.mlp.experts.129.up_proj", + "model.layers.28.mlp.experts.13.down_proj", + "model.layers.28.mlp.experts.13.gate_proj", + "model.layers.28.mlp.experts.13.up_proj", + "model.layers.28.mlp.experts.130.down_proj", + "model.layers.28.mlp.experts.130.gate_proj", + "model.layers.28.mlp.experts.130.up_proj", + "model.layers.28.mlp.experts.131.down_proj", + "model.layers.28.mlp.experts.131.gate_proj", + "model.layers.28.mlp.experts.131.up_proj", + "model.layers.28.mlp.experts.132.down_proj", + "model.layers.28.mlp.experts.132.gate_proj", + "model.layers.28.mlp.experts.132.up_proj", + "model.layers.28.mlp.experts.133.down_proj", + "model.layers.28.mlp.experts.133.gate_proj", + "model.layers.28.mlp.experts.133.up_proj", + "model.layers.28.mlp.experts.134.down_proj", + "model.layers.28.mlp.experts.134.gate_proj", + "model.layers.28.mlp.experts.134.up_proj", + "model.layers.28.mlp.experts.135.down_proj", + "model.layers.28.mlp.experts.135.gate_proj", + "model.layers.28.mlp.experts.135.up_proj", + "model.layers.28.mlp.experts.136.down_proj", + "model.layers.28.mlp.experts.136.gate_proj", + "model.layers.28.mlp.experts.136.up_proj", + "model.layers.28.mlp.experts.137.down_proj", + "model.layers.28.mlp.experts.137.gate_proj", + "model.layers.28.mlp.experts.137.up_proj", + "model.layers.28.mlp.experts.138.down_proj", + "model.layers.28.mlp.experts.138.gate_proj", + "model.layers.28.mlp.experts.138.up_proj", + "model.layers.28.mlp.experts.139.down_proj", + "model.layers.28.mlp.experts.139.gate_proj", + "model.layers.28.mlp.experts.139.up_proj", + "model.layers.28.mlp.experts.14.down_proj", + "model.layers.28.mlp.experts.14.gate_proj", + "model.layers.28.mlp.experts.14.up_proj", + "model.layers.28.mlp.experts.140.down_proj", + "model.layers.28.mlp.experts.140.gate_proj", + "model.layers.28.mlp.experts.140.up_proj", + "model.layers.28.mlp.experts.141.down_proj", + "model.layers.28.mlp.experts.141.gate_proj", + "model.layers.28.mlp.experts.141.up_proj", + "model.layers.28.mlp.experts.142.down_proj", + "model.layers.28.mlp.experts.142.gate_proj", + "model.layers.28.mlp.experts.142.up_proj", + "model.layers.28.mlp.experts.143.down_proj", + "model.layers.28.mlp.experts.143.gate_proj", + "model.layers.28.mlp.experts.143.up_proj", + "model.layers.28.mlp.experts.144.down_proj", + "model.layers.28.mlp.experts.144.gate_proj", + "model.layers.28.mlp.experts.144.up_proj", + "model.layers.28.mlp.experts.145.down_proj", + "model.layers.28.mlp.experts.145.gate_proj", + "model.layers.28.mlp.experts.145.up_proj", + "model.layers.28.mlp.experts.146.down_proj", + "model.layers.28.mlp.experts.146.gate_proj", + "model.layers.28.mlp.experts.146.up_proj", + "model.layers.28.mlp.experts.147.down_proj", + "model.layers.28.mlp.experts.147.gate_proj", + "model.layers.28.mlp.experts.147.up_proj", + "model.layers.28.mlp.experts.148.down_proj", + "model.layers.28.mlp.experts.148.gate_proj", + "model.layers.28.mlp.experts.148.up_proj", + "model.layers.28.mlp.experts.149.down_proj", + "model.layers.28.mlp.experts.149.gate_proj", + "model.layers.28.mlp.experts.149.up_proj", + "model.layers.28.mlp.experts.15.down_proj", + "model.layers.28.mlp.experts.15.gate_proj", + "model.layers.28.mlp.experts.15.up_proj", + "model.layers.28.mlp.experts.150.down_proj", + "model.layers.28.mlp.experts.150.gate_proj", + "model.layers.28.mlp.experts.150.up_proj", + "model.layers.28.mlp.experts.151.down_proj", + "model.layers.28.mlp.experts.151.gate_proj", + "model.layers.28.mlp.experts.151.up_proj", + "model.layers.28.mlp.experts.152.down_proj", + "model.layers.28.mlp.experts.152.gate_proj", + "model.layers.28.mlp.experts.152.up_proj", + "model.layers.28.mlp.experts.153.down_proj", + "model.layers.28.mlp.experts.153.gate_proj", + "model.layers.28.mlp.experts.153.up_proj", + "model.layers.28.mlp.experts.154.down_proj", + "model.layers.28.mlp.experts.154.gate_proj", + "model.layers.28.mlp.experts.154.up_proj", + "model.layers.28.mlp.experts.155.down_proj", + "model.layers.28.mlp.experts.155.gate_proj", + "model.layers.28.mlp.experts.155.up_proj", + "model.layers.28.mlp.experts.156.down_proj", + "model.layers.28.mlp.experts.156.gate_proj", + "model.layers.28.mlp.experts.156.up_proj", + "model.layers.28.mlp.experts.157.down_proj", + "model.layers.28.mlp.experts.157.gate_proj", + "model.layers.28.mlp.experts.157.up_proj", + "model.layers.28.mlp.experts.158.down_proj", + "model.layers.28.mlp.experts.158.gate_proj", + "model.layers.28.mlp.experts.158.up_proj", + "model.layers.28.mlp.experts.159.down_proj", + "model.layers.28.mlp.experts.159.gate_proj", + "model.layers.28.mlp.experts.159.up_proj", + "model.layers.28.mlp.experts.16.down_proj", + "model.layers.28.mlp.experts.16.gate_proj", + "model.layers.28.mlp.experts.16.up_proj", + "model.layers.28.mlp.experts.160.down_proj", + "model.layers.28.mlp.experts.160.gate_proj", + "model.layers.28.mlp.experts.160.up_proj", + "model.layers.28.mlp.experts.161.down_proj", + "model.layers.28.mlp.experts.161.gate_proj", + "model.layers.28.mlp.experts.161.up_proj", + "model.layers.28.mlp.experts.162.down_proj", + "model.layers.28.mlp.experts.162.gate_proj", + "model.layers.28.mlp.experts.162.up_proj", + "model.layers.28.mlp.experts.163.down_proj", + "model.layers.28.mlp.experts.163.gate_proj", + "model.layers.28.mlp.experts.163.up_proj", + "model.layers.28.mlp.experts.164.down_proj", + "model.layers.28.mlp.experts.164.gate_proj", + "model.layers.28.mlp.experts.164.up_proj", + "model.layers.28.mlp.experts.165.down_proj", + "model.layers.28.mlp.experts.165.gate_proj", + "model.layers.28.mlp.experts.165.up_proj", + "model.layers.28.mlp.experts.166.down_proj", + "model.layers.28.mlp.experts.166.gate_proj", + "model.layers.28.mlp.experts.166.up_proj", + "model.layers.28.mlp.experts.167.down_proj", + "model.layers.28.mlp.experts.167.gate_proj", + "model.layers.28.mlp.experts.167.up_proj", + "model.layers.28.mlp.experts.168.down_proj", + "model.layers.28.mlp.experts.168.gate_proj", + "model.layers.28.mlp.experts.168.up_proj", + "model.layers.28.mlp.experts.169.down_proj", + "model.layers.28.mlp.experts.169.gate_proj", + "model.layers.28.mlp.experts.169.up_proj", + "model.layers.28.mlp.experts.17.down_proj", + "model.layers.28.mlp.experts.17.gate_proj", + "model.layers.28.mlp.experts.17.up_proj", + "model.layers.28.mlp.experts.170.down_proj", + "model.layers.28.mlp.experts.170.gate_proj", + "model.layers.28.mlp.experts.170.up_proj", + "model.layers.28.mlp.experts.171.down_proj", + "model.layers.28.mlp.experts.171.gate_proj", + "model.layers.28.mlp.experts.171.up_proj", + "model.layers.28.mlp.experts.172.down_proj", + "model.layers.28.mlp.experts.172.gate_proj", + "model.layers.28.mlp.experts.172.up_proj", + "model.layers.28.mlp.experts.173.down_proj", + "model.layers.28.mlp.experts.173.gate_proj", + "model.layers.28.mlp.experts.173.up_proj", + "model.layers.28.mlp.experts.174.down_proj", + "model.layers.28.mlp.experts.174.gate_proj", + "model.layers.28.mlp.experts.174.up_proj", + "model.layers.28.mlp.experts.175.down_proj", + "model.layers.28.mlp.experts.175.gate_proj", + "model.layers.28.mlp.experts.175.up_proj", + "model.layers.28.mlp.experts.176.down_proj", + "model.layers.28.mlp.experts.176.gate_proj", + "model.layers.28.mlp.experts.176.up_proj", + "model.layers.28.mlp.experts.177.down_proj", + "model.layers.28.mlp.experts.177.gate_proj", + "model.layers.28.mlp.experts.177.up_proj", + "model.layers.28.mlp.experts.178.down_proj", + "model.layers.28.mlp.experts.178.gate_proj", + "model.layers.28.mlp.experts.178.up_proj", + "model.layers.28.mlp.experts.179.down_proj", + "model.layers.28.mlp.experts.179.gate_proj", + "model.layers.28.mlp.experts.179.up_proj", + "model.layers.28.mlp.experts.18.down_proj", + "model.layers.28.mlp.experts.18.gate_proj", + "model.layers.28.mlp.experts.18.up_proj", + "model.layers.28.mlp.experts.180.down_proj", + "model.layers.28.mlp.experts.180.gate_proj", + "model.layers.28.mlp.experts.180.up_proj", + "model.layers.28.mlp.experts.181.down_proj", + "model.layers.28.mlp.experts.181.gate_proj", + "model.layers.28.mlp.experts.181.up_proj", + "model.layers.28.mlp.experts.182.down_proj", + "model.layers.28.mlp.experts.182.gate_proj", + "model.layers.28.mlp.experts.182.up_proj", + "model.layers.28.mlp.experts.183.down_proj", + "model.layers.28.mlp.experts.183.gate_proj", + "model.layers.28.mlp.experts.183.up_proj", + "model.layers.28.mlp.experts.184.down_proj", + "model.layers.28.mlp.experts.184.gate_proj", + "model.layers.28.mlp.experts.184.up_proj", + "model.layers.28.mlp.experts.185.down_proj", + "model.layers.28.mlp.experts.185.gate_proj", + "model.layers.28.mlp.experts.185.up_proj", + "model.layers.28.mlp.experts.186.down_proj", + "model.layers.28.mlp.experts.186.gate_proj", + "model.layers.28.mlp.experts.186.up_proj", + "model.layers.28.mlp.experts.187.down_proj", + "model.layers.28.mlp.experts.187.gate_proj", + "model.layers.28.mlp.experts.187.up_proj", + "model.layers.28.mlp.experts.188.down_proj", + "model.layers.28.mlp.experts.188.gate_proj", + "model.layers.28.mlp.experts.188.up_proj", + "model.layers.28.mlp.experts.189.down_proj", + "model.layers.28.mlp.experts.189.gate_proj", + "model.layers.28.mlp.experts.189.up_proj", + "model.layers.28.mlp.experts.19.down_proj", + "model.layers.28.mlp.experts.19.gate_proj", + "model.layers.28.mlp.experts.19.up_proj", + "model.layers.28.mlp.experts.190.down_proj", + "model.layers.28.mlp.experts.190.gate_proj", + "model.layers.28.mlp.experts.190.up_proj", + "model.layers.28.mlp.experts.191.down_proj", + "model.layers.28.mlp.experts.191.gate_proj", + "model.layers.28.mlp.experts.191.up_proj", + "model.layers.28.mlp.experts.192.down_proj", + "model.layers.28.mlp.experts.192.gate_proj", + "model.layers.28.mlp.experts.192.up_proj", + "model.layers.28.mlp.experts.193.down_proj", + "model.layers.28.mlp.experts.193.gate_proj", + "model.layers.28.mlp.experts.193.up_proj", + "model.layers.28.mlp.experts.194.down_proj", + "model.layers.28.mlp.experts.194.gate_proj", + "model.layers.28.mlp.experts.194.up_proj", + "model.layers.28.mlp.experts.195.down_proj", + "model.layers.28.mlp.experts.195.gate_proj", + "model.layers.28.mlp.experts.195.up_proj", + "model.layers.28.mlp.experts.196.down_proj", + "model.layers.28.mlp.experts.196.gate_proj", + "model.layers.28.mlp.experts.196.up_proj", + "model.layers.28.mlp.experts.197.down_proj", + "model.layers.28.mlp.experts.197.gate_proj", + "model.layers.28.mlp.experts.197.up_proj", + "model.layers.28.mlp.experts.198.down_proj", + "model.layers.28.mlp.experts.198.gate_proj", + "model.layers.28.mlp.experts.198.up_proj", + "model.layers.28.mlp.experts.199.down_proj", + "model.layers.28.mlp.experts.199.gate_proj", + "model.layers.28.mlp.experts.199.up_proj", + "model.layers.28.mlp.experts.2.down_proj", + "model.layers.28.mlp.experts.2.gate_proj", + "model.layers.28.mlp.experts.2.up_proj", + "model.layers.28.mlp.experts.20.down_proj", + "model.layers.28.mlp.experts.20.gate_proj", + "model.layers.28.mlp.experts.20.up_proj", + "model.layers.28.mlp.experts.200.down_proj", + "model.layers.28.mlp.experts.200.gate_proj", + "model.layers.28.mlp.experts.200.up_proj", + "model.layers.28.mlp.experts.201.down_proj", + "model.layers.28.mlp.experts.201.gate_proj", + "model.layers.28.mlp.experts.201.up_proj", + "model.layers.28.mlp.experts.202.down_proj", + "model.layers.28.mlp.experts.202.gate_proj", + "model.layers.28.mlp.experts.202.up_proj", + "model.layers.28.mlp.experts.203.down_proj", + "model.layers.28.mlp.experts.203.gate_proj", + "model.layers.28.mlp.experts.203.up_proj", + "model.layers.28.mlp.experts.204.down_proj", + "model.layers.28.mlp.experts.204.gate_proj", + "model.layers.28.mlp.experts.204.up_proj", + "model.layers.28.mlp.experts.205.down_proj", + "model.layers.28.mlp.experts.205.gate_proj", + "model.layers.28.mlp.experts.205.up_proj", + "model.layers.28.mlp.experts.206.down_proj", + "model.layers.28.mlp.experts.206.gate_proj", + "model.layers.28.mlp.experts.206.up_proj", + "model.layers.28.mlp.experts.207.down_proj", + "model.layers.28.mlp.experts.207.gate_proj", + "model.layers.28.mlp.experts.207.up_proj", + "model.layers.28.mlp.experts.208.down_proj", + "model.layers.28.mlp.experts.208.gate_proj", + "model.layers.28.mlp.experts.208.up_proj", + "model.layers.28.mlp.experts.209.down_proj", + "model.layers.28.mlp.experts.209.gate_proj", + "model.layers.28.mlp.experts.209.up_proj", + "model.layers.28.mlp.experts.21.down_proj", + "model.layers.28.mlp.experts.21.gate_proj", + "model.layers.28.mlp.experts.21.up_proj", + "model.layers.28.mlp.experts.210.down_proj", + "model.layers.28.mlp.experts.210.gate_proj", + "model.layers.28.mlp.experts.210.up_proj", + "model.layers.28.mlp.experts.211.down_proj", + "model.layers.28.mlp.experts.211.gate_proj", + "model.layers.28.mlp.experts.211.up_proj", + "model.layers.28.mlp.experts.212.down_proj", + "model.layers.28.mlp.experts.212.gate_proj", + "model.layers.28.mlp.experts.212.up_proj", + "model.layers.28.mlp.experts.213.down_proj", + "model.layers.28.mlp.experts.213.gate_proj", + "model.layers.28.mlp.experts.213.up_proj", + "model.layers.28.mlp.experts.214.down_proj", + "model.layers.28.mlp.experts.214.gate_proj", + "model.layers.28.mlp.experts.214.up_proj", + "model.layers.28.mlp.experts.215.down_proj", + "model.layers.28.mlp.experts.215.gate_proj", + "model.layers.28.mlp.experts.215.up_proj", + "model.layers.28.mlp.experts.216.down_proj", + "model.layers.28.mlp.experts.216.gate_proj", + "model.layers.28.mlp.experts.216.up_proj", + "model.layers.28.mlp.experts.217.down_proj", + "model.layers.28.mlp.experts.217.gate_proj", + "model.layers.28.mlp.experts.217.up_proj", + "model.layers.28.mlp.experts.218.down_proj", + "model.layers.28.mlp.experts.218.gate_proj", + "model.layers.28.mlp.experts.218.up_proj", + "model.layers.28.mlp.experts.219.down_proj", + "model.layers.28.mlp.experts.219.gate_proj", + "model.layers.28.mlp.experts.219.up_proj", + "model.layers.28.mlp.experts.22.down_proj", + "model.layers.28.mlp.experts.22.gate_proj", + "model.layers.28.mlp.experts.22.up_proj", + "model.layers.28.mlp.experts.220.down_proj", + "model.layers.28.mlp.experts.220.gate_proj", + "model.layers.28.mlp.experts.220.up_proj", + "model.layers.28.mlp.experts.221.down_proj", + "model.layers.28.mlp.experts.221.gate_proj", + "model.layers.28.mlp.experts.221.up_proj", + "model.layers.28.mlp.experts.222.down_proj", + "model.layers.28.mlp.experts.222.gate_proj", + "model.layers.28.mlp.experts.222.up_proj", + "model.layers.28.mlp.experts.223.down_proj", + "model.layers.28.mlp.experts.223.gate_proj", + "model.layers.28.mlp.experts.223.up_proj", + "model.layers.28.mlp.experts.224.down_proj", + "model.layers.28.mlp.experts.224.gate_proj", + "model.layers.28.mlp.experts.224.up_proj", + "model.layers.28.mlp.experts.225.down_proj", + "model.layers.28.mlp.experts.225.gate_proj", + "model.layers.28.mlp.experts.225.up_proj", + "model.layers.28.mlp.experts.226.down_proj", + "model.layers.28.mlp.experts.226.gate_proj", + "model.layers.28.mlp.experts.226.up_proj", + "model.layers.28.mlp.experts.227.down_proj", + "model.layers.28.mlp.experts.227.gate_proj", + "model.layers.28.mlp.experts.227.up_proj", + "model.layers.28.mlp.experts.228.down_proj", + "model.layers.28.mlp.experts.228.gate_proj", + "model.layers.28.mlp.experts.228.up_proj", + "model.layers.28.mlp.experts.229.down_proj", + "model.layers.28.mlp.experts.229.gate_proj", + "model.layers.28.mlp.experts.229.up_proj", + "model.layers.28.mlp.experts.23.down_proj", + "model.layers.28.mlp.experts.23.gate_proj", + "model.layers.28.mlp.experts.23.up_proj", + "model.layers.28.mlp.experts.230.down_proj", + "model.layers.28.mlp.experts.230.gate_proj", + "model.layers.28.mlp.experts.230.up_proj", + "model.layers.28.mlp.experts.231.down_proj", + "model.layers.28.mlp.experts.231.gate_proj", + "model.layers.28.mlp.experts.231.up_proj", + "model.layers.28.mlp.experts.232.down_proj", + "model.layers.28.mlp.experts.232.gate_proj", + "model.layers.28.mlp.experts.232.up_proj", + "model.layers.28.mlp.experts.233.down_proj", + "model.layers.28.mlp.experts.233.gate_proj", + "model.layers.28.mlp.experts.233.up_proj", + "model.layers.28.mlp.experts.234.down_proj", + "model.layers.28.mlp.experts.234.gate_proj", + "model.layers.28.mlp.experts.234.up_proj", + "model.layers.28.mlp.experts.235.down_proj", + "model.layers.28.mlp.experts.235.gate_proj", + "model.layers.28.mlp.experts.235.up_proj", + "model.layers.28.mlp.experts.236.down_proj", + "model.layers.28.mlp.experts.236.gate_proj", + "model.layers.28.mlp.experts.236.up_proj", + "model.layers.28.mlp.experts.237.down_proj", + "model.layers.28.mlp.experts.237.gate_proj", + "model.layers.28.mlp.experts.237.up_proj", + "model.layers.28.mlp.experts.238.down_proj", + "model.layers.28.mlp.experts.238.gate_proj", + "model.layers.28.mlp.experts.238.up_proj", + "model.layers.28.mlp.experts.239.down_proj", + "model.layers.28.mlp.experts.239.gate_proj", + "model.layers.28.mlp.experts.239.up_proj", + "model.layers.28.mlp.experts.24.down_proj", + "model.layers.28.mlp.experts.24.gate_proj", + "model.layers.28.mlp.experts.24.up_proj", + "model.layers.28.mlp.experts.240.down_proj", + "model.layers.28.mlp.experts.240.gate_proj", + "model.layers.28.mlp.experts.240.up_proj", + "model.layers.28.mlp.experts.241.down_proj", + "model.layers.28.mlp.experts.241.gate_proj", + "model.layers.28.mlp.experts.241.up_proj", + "model.layers.28.mlp.experts.242.down_proj", + "model.layers.28.mlp.experts.242.gate_proj", + "model.layers.28.mlp.experts.242.up_proj", + "model.layers.28.mlp.experts.243.down_proj", + "model.layers.28.mlp.experts.243.gate_proj", + "model.layers.28.mlp.experts.243.up_proj", + "model.layers.28.mlp.experts.244.down_proj", + "model.layers.28.mlp.experts.244.gate_proj", + "model.layers.28.mlp.experts.244.up_proj", + "model.layers.28.mlp.experts.245.down_proj", + "model.layers.28.mlp.experts.245.gate_proj", + "model.layers.28.mlp.experts.245.up_proj", + "model.layers.28.mlp.experts.246.down_proj", + "model.layers.28.mlp.experts.246.gate_proj", + "model.layers.28.mlp.experts.246.up_proj", + "model.layers.28.mlp.experts.247.down_proj", + "model.layers.28.mlp.experts.247.gate_proj", + "model.layers.28.mlp.experts.247.up_proj", + "model.layers.28.mlp.experts.248.down_proj", + "model.layers.28.mlp.experts.248.gate_proj", + "model.layers.28.mlp.experts.248.up_proj", + "model.layers.28.mlp.experts.249.down_proj", + "model.layers.28.mlp.experts.249.gate_proj", + "model.layers.28.mlp.experts.249.up_proj", + "model.layers.28.mlp.experts.25.down_proj", + "model.layers.28.mlp.experts.25.gate_proj", + "model.layers.28.mlp.experts.25.up_proj", + "model.layers.28.mlp.experts.250.down_proj", + "model.layers.28.mlp.experts.250.gate_proj", + "model.layers.28.mlp.experts.250.up_proj", + "model.layers.28.mlp.experts.251.down_proj", + "model.layers.28.mlp.experts.251.gate_proj", + "model.layers.28.mlp.experts.251.up_proj", + "model.layers.28.mlp.experts.252.down_proj", + "model.layers.28.mlp.experts.252.gate_proj", + "model.layers.28.mlp.experts.252.up_proj", + "model.layers.28.mlp.experts.253.down_proj", + "model.layers.28.mlp.experts.253.gate_proj", + "model.layers.28.mlp.experts.253.up_proj", + "model.layers.28.mlp.experts.254.down_proj", + "model.layers.28.mlp.experts.254.gate_proj", + "model.layers.28.mlp.experts.254.up_proj", + "model.layers.28.mlp.experts.255.down_proj", + "model.layers.28.mlp.experts.255.gate_proj", + "model.layers.28.mlp.experts.255.up_proj", + "model.layers.28.mlp.experts.26.down_proj", + "model.layers.28.mlp.experts.26.gate_proj", + "model.layers.28.mlp.experts.26.up_proj", + "model.layers.28.mlp.experts.27.down_proj", + "model.layers.28.mlp.experts.27.gate_proj", + "model.layers.28.mlp.experts.27.up_proj", + "model.layers.28.mlp.experts.28.down_proj", + "model.layers.28.mlp.experts.28.gate_proj", + "model.layers.28.mlp.experts.28.up_proj", + "model.layers.28.mlp.experts.29.down_proj", + "model.layers.28.mlp.experts.29.gate_proj", + "model.layers.28.mlp.experts.29.up_proj", + "model.layers.28.mlp.experts.3.down_proj", + "model.layers.28.mlp.experts.3.gate_proj", + "model.layers.28.mlp.experts.3.up_proj", + "model.layers.28.mlp.experts.30.down_proj", + "model.layers.28.mlp.experts.30.gate_proj", + "model.layers.28.mlp.experts.30.up_proj", + "model.layers.28.mlp.experts.31.down_proj", + "model.layers.28.mlp.experts.31.gate_proj", + "model.layers.28.mlp.experts.31.up_proj", + "model.layers.28.mlp.experts.32.down_proj", + "model.layers.28.mlp.experts.32.gate_proj", + "model.layers.28.mlp.experts.32.up_proj", + "model.layers.28.mlp.experts.33.down_proj", + "model.layers.28.mlp.experts.33.gate_proj", + "model.layers.28.mlp.experts.33.up_proj", + "model.layers.28.mlp.experts.34.down_proj", + "model.layers.28.mlp.experts.34.gate_proj", + "model.layers.28.mlp.experts.34.up_proj", + "model.layers.28.mlp.experts.35.down_proj", + "model.layers.28.mlp.experts.35.gate_proj", + "model.layers.28.mlp.experts.35.up_proj", + "model.layers.28.mlp.experts.36.down_proj", + "model.layers.28.mlp.experts.36.gate_proj", + "model.layers.28.mlp.experts.36.up_proj", + "model.layers.28.mlp.experts.37.down_proj", + "model.layers.28.mlp.experts.37.gate_proj", + "model.layers.28.mlp.experts.37.up_proj", + "model.layers.28.mlp.experts.38.down_proj", + "model.layers.28.mlp.experts.38.gate_proj", + "model.layers.28.mlp.experts.38.up_proj", + "model.layers.28.mlp.experts.39.down_proj", + "model.layers.28.mlp.experts.39.gate_proj", + "model.layers.28.mlp.experts.39.up_proj", + "model.layers.28.mlp.experts.4.down_proj", + "model.layers.28.mlp.experts.4.gate_proj", + "model.layers.28.mlp.experts.4.up_proj", + "model.layers.28.mlp.experts.40.down_proj", + "model.layers.28.mlp.experts.40.gate_proj", + "model.layers.28.mlp.experts.40.up_proj", + "model.layers.28.mlp.experts.41.down_proj", + "model.layers.28.mlp.experts.41.gate_proj", + "model.layers.28.mlp.experts.41.up_proj", + "model.layers.28.mlp.experts.42.down_proj", + "model.layers.28.mlp.experts.42.gate_proj", + "model.layers.28.mlp.experts.42.up_proj", + "model.layers.28.mlp.experts.43.down_proj", + "model.layers.28.mlp.experts.43.gate_proj", + "model.layers.28.mlp.experts.43.up_proj", + "model.layers.28.mlp.experts.44.down_proj", + "model.layers.28.mlp.experts.44.gate_proj", + "model.layers.28.mlp.experts.44.up_proj", + "model.layers.28.mlp.experts.45.down_proj", + "model.layers.28.mlp.experts.45.gate_proj", + "model.layers.28.mlp.experts.45.up_proj", + "model.layers.28.mlp.experts.46.down_proj", + "model.layers.28.mlp.experts.46.gate_proj", + "model.layers.28.mlp.experts.46.up_proj", + "model.layers.28.mlp.experts.47.down_proj", + "model.layers.28.mlp.experts.47.gate_proj", + "model.layers.28.mlp.experts.47.up_proj", + "model.layers.28.mlp.experts.48.down_proj", + "model.layers.28.mlp.experts.48.gate_proj", + "model.layers.28.mlp.experts.48.up_proj", + "model.layers.28.mlp.experts.49.down_proj", + "model.layers.28.mlp.experts.49.gate_proj", + "model.layers.28.mlp.experts.49.up_proj", + "model.layers.28.mlp.experts.5.down_proj", + "model.layers.28.mlp.experts.5.gate_proj", + "model.layers.28.mlp.experts.5.up_proj", + "model.layers.28.mlp.experts.50.down_proj", + "model.layers.28.mlp.experts.50.gate_proj", + "model.layers.28.mlp.experts.50.up_proj", + "model.layers.28.mlp.experts.51.down_proj", + "model.layers.28.mlp.experts.51.gate_proj", + "model.layers.28.mlp.experts.51.up_proj", + "model.layers.28.mlp.experts.52.down_proj", + "model.layers.28.mlp.experts.52.gate_proj", + "model.layers.28.mlp.experts.52.up_proj", + "model.layers.28.mlp.experts.53.down_proj", + "model.layers.28.mlp.experts.53.gate_proj", + "model.layers.28.mlp.experts.53.up_proj", + "model.layers.28.mlp.experts.54.down_proj", + "model.layers.28.mlp.experts.54.gate_proj", + "model.layers.28.mlp.experts.54.up_proj", + "model.layers.28.mlp.experts.55.down_proj", + "model.layers.28.mlp.experts.55.gate_proj", + "model.layers.28.mlp.experts.55.up_proj", + "model.layers.28.mlp.experts.56.down_proj", + "model.layers.28.mlp.experts.56.gate_proj", + "model.layers.28.mlp.experts.56.up_proj", + "model.layers.28.mlp.experts.57.down_proj", + "model.layers.28.mlp.experts.57.gate_proj", + "model.layers.28.mlp.experts.57.up_proj", + "model.layers.28.mlp.experts.58.down_proj", + "model.layers.28.mlp.experts.58.gate_proj", + "model.layers.28.mlp.experts.58.up_proj", + "model.layers.28.mlp.experts.59.down_proj", + "model.layers.28.mlp.experts.59.gate_proj", + "model.layers.28.mlp.experts.59.up_proj", + "model.layers.28.mlp.experts.6.down_proj", + "model.layers.28.mlp.experts.6.gate_proj", + "model.layers.28.mlp.experts.6.up_proj", + "model.layers.28.mlp.experts.60.down_proj", + "model.layers.28.mlp.experts.60.gate_proj", + "model.layers.28.mlp.experts.60.up_proj", + "model.layers.28.mlp.experts.61.down_proj", + "model.layers.28.mlp.experts.61.gate_proj", + "model.layers.28.mlp.experts.61.up_proj", + "model.layers.28.mlp.experts.62.down_proj", + "model.layers.28.mlp.experts.62.gate_proj", + "model.layers.28.mlp.experts.62.up_proj", + "model.layers.28.mlp.experts.63.down_proj", + "model.layers.28.mlp.experts.63.gate_proj", + "model.layers.28.mlp.experts.63.up_proj", + "model.layers.28.mlp.experts.64.down_proj", + "model.layers.28.mlp.experts.64.gate_proj", + "model.layers.28.mlp.experts.64.up_proj", + "model.layers.28.mlp.experts.65.down_proj", + "model.layers.28.mlp.experts.65.gate_proj", + "model.layers.28.mlp.experts.65.up_proj", + "model.layers.28.mlp.experts.66.down_proj", + "model.layers.28.mlp.experts.66.gate_proj", + "model.layers.28.mlp.experts.66.up_proj", + "model.layers.28.mlp.experts.67.down_proj", + "model.layers.28.mlp.experts.67.gate_proj", + "model.layers.28.mlp.experts.67.up_proj", + "model.layers.28.mlp.experts.68.down_proj", + "model.layers.28.mlp.experts.68.gate_proj", + "model.layers.28.mlp.experts.68.up_proj", + "model.layers.28.mlp.experts.69.down_proj", + "model.layers.28.mlp.experts.69.gate_proj", + "model.layers.28.mlp.experts.69.up_proj", + "model.layers.28.mlp.experts.7.down_proj", + "model.layers.28.mlp.experts.7.gate_proj", + "model.layers.28.mlp.experts.7.up_proj", + "model.layers.28.mlp.experts.70.down_proj", + "model.layers.28.mlp.experts.70.gate_proj", + "model.layers.28.mlp.experts.70.up_proj", + "model.layers.28.mlp.experts.71.down_proj", + "model.layers.28.mlp.experts.71.gate_proj", + "model.layers.28.mlp.experts.71.up_proj", + "model.layers.28.mlp.experts.72.down_proj", + "model.layers.28.mlp.experts.72.gate_proj", + "model.layers.28.mlp.experts.72.up_proj", + "model.layers.28.mlp.experts.73.down_proj", + "model.layers.28.mlp.experts.73.gate_proj", + "model.layers.28.mlp.experts.73.up_proj", + "model.layers.28.mlp.experts.74.down_proj", + "model.layers.28.mlp.experts.74.gate_proj", + "model.layers.28.mlp.experts.74.up_proj", + "model.layers.28.mlp.experts.75.down_proj", + "model.layers.28.mlp.experts.75.gate_proj", + "model.layers.28.mlp.experts.75.up_proj", + "model.layers.28.mlp.experts.76.down_proj", + "model.layers.28.mlp.experts.76.gate_proj", + "model.layers.28.mlp.experts.76.up_proj", + "model.layers.28.mlp.experts.77.down_proj", + "model.layers.28.mlp.experts.77.gate_proj", + "model.layers.28.mlp.experts.77.up_proj", + "model.layers.28.mlp.experts.78.down_proj", + "model.layers.28.mlp.experts.78.gate_proj", + "model.layers.28.mlp.experts.78.up_proj", + "model.layers.28.mlp.experts.79.down_proj", + "model.layers.28.mlp.experts.79.gate_proj", + "model.layers.28.mlp.experts.79.up_proj", + "model.layers.28.mlp.experts.8.down_proj", + "model.layers.28.mlp.experts.8.gate_proj", + "model.layers.28.mlp.experts.8.up_proj", + "model.layers.28.mlp.experts.80.down_proj", + "model.layers.28.mlp.experts.80.gate_proj", + "model.layers.28.mlp.experts.80.up_proj", + "model.layers.28.mlp.experts.81.down_proj", + "model.layers.28.mlp.experts.81.gate_proj", + "model.layers.28.mlp.experts.81.up_proj", + "model.layers.28.mlp.experts.82.down_proj", + "model.layers.28.mlp.experts.82.gate_proj", + "model.layers.28.mlp.experts.82.up_proj", + "model.layers.28.mlp.experts.83.down_proj", + "model.layers.28.mlp.experts.83.gate_proj", + "model.layers.28.mlp.experts.83.up_proj", + "model.layers.28.mlp.experts.84.down_proj", + "model.layers.28.mlp.experts.84.gate_proj", + "model.layers.28.mlp.experts.84.up_proj", + "model.layers.28.mlp.experts.85.down_proj", + "model.layers.28.mlp.experts.85.gate_proj", + "model.layers.28.mlp.experts.85.up_proj", + "model.layers.28.mlp.experts.86.down_proj", + "model.layers.28.mlp.experts.86.gate_proj", + "model.layers.28.mlp.experts.86.up_proj", + "model.layers.28.mlp.experts.87.down_proj", + "model.layers.28.mlp.experts.87.gate_proj", + "model.layers.28.mlp.experts.87.up_proj", + "model.layers.28.mlp.experts.88.down_proj", + "model.layers.28.mlp.experts.88.gate_proj", + "model.layers.28.mlp.experts.88.up_proj", + "model.layers.28.mlp.experts.89.down_proj", + "model.layers.28.mlp.experts.89.gate_proj", + "model.layers.28.mlp.experts.89.up_proj", + "model.layers.28.mlp.experts.9.down_proj", + "model.layers.28.mlp.experts.9.gate_proj", + "model.layers.28.mlp.experts.9.up_proj", + "model.layers.28.mlp.experts.90.down_proj", + "model.layers.28.mlp.experts.90.gate_proj", + "model.layers.28.mlp.experts.90.up_proj", + "model.layers.28.mlp.experts.91.down_proj", + "model.layers.28.mlp.experts.91.gate_proj", + "model.layers.28.mlp.experts.91.up_proj", + "model.layers.28.mlp.experts.92.down_proj", + "model.layers.28.mlp.experts.92.gate_proj", + "model.layers.28.mlp.experts.92.up_proj", + "model.layers.28.mlp.experts.93.down_proj", + "model.layers.28.mlp.experts.93.gate_proj", + "model.layers.28.mlp.experts.93.up_proj", + "model.layers.28.mlp.experts.94.down_proj", + "model.layers.28.mlp.experts.94.gate_proj", + "model.layers.28.mlp.experts.94.up_proj", + "model.layers.28.mlp.experts.95.down_proj", + "model.layers.28.mlp.experts.95.gate_proj", + "model.layers.28.mlp.experts.95.up_proj", + "model.layers.28.mlp.experts.96.down_proj", + "model.layers.28.mlp.experts.96.gate_proj", + "model.layers.28.mlp.experts.96.up_proj", + "model.layers.28.mlp.experts.97.down_proj", + "model.layers.28.mlp.experts.97.gate_proj", + "model.layers.28.mlp.experts.97.up_proj", + "model.layers.28.mlp.experts.98.down_proj", + "model.layers.28.mlp.experts.98.gate_proj", + "model.layers.28.mlp.experts.98.up_proj", + "model.layers.28.mlp.experts.99.down_proj", + "model.layers.28.mlp.experts.99.gate_proj", + "model.layers.28.mlp.experts.99.up_proj", + "model.layers.29.mlp.experts.0.down_proj", + "model.layers.29.mlp.experts.0.gate_proj", + "model.layers.29.mlp.experts.0.up_proj", + "model.layers.29.mlp.experts.1.down_proj", + "model.layers.29.mlp.experts.1.gate_proj", + "model.layers.29.mlp.experts.1.up_proj", + "model.layers.29.mlp.experts.10.down_proj", + "model.layers.29.mlp.experts.10.gate_proj", + "model.layers.29.mlp.experts.10.up_proj", + "model.layers.29.mlp.experts.100.down_proj", + "model.layers.29.mlp.experts.100.gate_proj", + "model.layers.29.mlp.experts.100.up_proj", + "model.layers.29.mlp.experts.101.down_proj", + "model.layers.29.mlp.experts.101.gate_proj", + "model.layers.29.mlp.experts.101.up_proj", + "model.layers.29.mlp.experts.102.down_proj", + "model.layers.29.mlp.experts.102.gate_proj", + "model.layers.29.mlp.experts.102.up_proj", + "model.layers.29.mlp.experts.103.down_proj", + "model.layers.29.mlp.experts.103.gate_proj", + "model.layers.29.mlp.experts.103.up_proj", + "model.layers.29.mlp.experts.104.down_proj", + "model.layers.29.mlp.experts.104.gate_proj", + "model.layers.29.mlp.experts.104.up_proj", + "model.layers.29.mlp.experts.105.down_proj", + "model.layers.29.mlp.experts.105.gate_proj", + "model.layers.29.mlp.experts.105.up_proj", + "model.layers.29.mlp.experts.106.down_proj", + "model.layers.29.mlp.experts.106.gate_proj", + "model.layers.29.mlp.experts.106.up_proj", + "model.layers.29.mlp.experts.107.down_proj", + "model.layers.29.mlp.experts.107.gate_proj", + "model.layers.29.mlp.experts.107.up_proj", + "model.layers.29.mlp.experts.108.down_proj", + "model.layers.29.mlp.experts.108.gate_proj", + "model.layers.29.mlp.experts.108.up_proj", + "model.layers.29.mlp.experts.109.down_proj", + "model.layers.29.mlp.experts.109.gate_proj", + "model.layers.29.mlp.experts.109.up_proj", + "model.layers.29.mlp.experts.11.down_proj", + "model.layers.29.mlp.experts.11.gate_proj", + "model.layers.29.mlp.experts.11.up_proj", + "model.layers.29.mlp.experts.110.down_proj", + "model.layers.29.mlp.experts.110.gate_proj", + "model.layers.29.mlp.experts.110.up_proj", + "model.layers.29.mlp.experts.111.down_proj", + "model.layers.29.mlp.experts.111.gate_proj", + "model.layers.29.mlp.experts.111.up_proj", + "model.layers.29.mlp.experts.112.down_proj", + "model.layers.29.mlp.experts.112.gate_proj", + "model.layers.29.mlp.experts.112.up_proj", + "model.layers.29.mlp.experts.113.down_proj", + "model.layers.29.mlp.experts.113.gate_proj", + "model.layers.29.mlp.experts.113.up_proj", + "model.layers.29.mlp.experts.114.down_proj", + "model.layers.29.mlp.experts.114.gate_proj", + "model.layers.29.mlp.experts.114.up_proj", + "model.layers.29.mlp.experts.115.down_proj", + "model.layers.29.mlp.experts.115.gate_proj", + "model.layers.29.mlp.experts.115.up_proj", + "model.layers.29.mlp.experts.116.down_proj", + "model.layers.29.mlp.experts.116.gate_proj", + "model.layers.29.mlp.experts.116.up_proj", + "model.layers.29.mlp.experts.117.down_proj", + "model.layers.29.mlp.experts.117.gate_proj", + "model.layers.29.mlp.experts.117.up_proj", + "model.layers.29.mlp.experts.118.down_proj", + "model.layers.29.mlp.experts.118.gate_proj", + "model.layers.29.mlp.experts.118.up_proj", + "model.layers.29.mlp.experts.119.down_proj", + "model.layers.29.mlp.experts.119.gate_proj", + "model.layers.29.mlp.experts.119.up_proj", + "model.layers.29.mlp.experts.12.down_proj", + "model.layers.29.mlp.experts.12.gate_proj", + "model.layers.29.mlp.experts.12.up_proj", + "model.layers.29.mlp.experts.120.down_proj", + "model.layers.29.mlp.experts.120.gate_proj", + "model.layers.29.mlp.experts.120.up_proj", + "model.layers.29.mlp.experts.121.down_proj", + "model.layers.29.mlp.experts.121.gate_proj", + "model.layers.29.mlp.experts.121.up_proj", + "model.layers.29.mlp.experts.122.down_proj", + "model.layers.29.mlp.experts.122.gate_proj", + "model.layers.29.mlp.experts.122.up_proj", + "model.layers.29.mlp.experts.123.down_proj", + "model.layers.29.mlp.experts.123.gate_proj", + "model.layers.29.mlp.experts.123.up_proj", + "model.layers.29.mlp.experts.124.down_proj", + "model.layers.29.mlp.experts.124.gate_proj", + "model.layers.29.mlp.experts.124.up_proj", + "model.layers.29.mlp.experts.125.down_proj", + "model.layers.29.mlp.experts.125.gate_proj", + "model.layers.29.mlp.experts.125.up_proj", + "model.layers.29.mlp.experts.126.down_proj", + "model.layers.29.mlp.experts.126.gate_proj", + "model.layers.29.mlp.experts.126.up_proj", + "model.layers.29.mlp.experts.127.down_proj", + "model.layers.29.mlp.experts.127.gate_proj", + "model.layers.29.mlp.experts.127.up_proj", + "model.layers.29.mlp.experts.128.down_proj", + "model.layers.29.mlp.experts.128.gate_proj", + "model.layers.29.mlp.experts.128.up_proj", + "model.layers.29.mlp.experts.129.down_proj", + "model.layers.29.mlp.experts.129.gate_proj", + "model.layers.29.mlp.experts.129.up_proj", + "model.layers.29.mlp.experts.13.down_proj", + "model.layers.29.mlp.experts.13.gate_proj", + "model.layers.29.mlp.experts.13.up_proj", + "model.layers.29.mlp.experts.130.down_proj", + "model.layers.29.mlp.experts.130.gate_proj", + "model.layers.29.mlp.experts.130.up_proj", + "model.layers.29.mlp.experts.131.down_proj", + "model.layers.29.mlp.experts.131.gate_proj", + "model.layers.29.mlp.experts.131.up_proj", + "model.layers.29.mlp.experts.132.down_proj", + "model.layers.29.mlp.experts.132.gate_proj", + "model.layers.29.mlp.experts.132.up_proj", + "model.layers.29.mlp.experts.133.down_proj", + "model.layers.29.mlp.experts.133.gate_proj", + "model.layers.29.mlp.experts.133.up_proj", + "model.layers.29.mlp.experts.134.down_proj", + "model.layers.29.mlp.experts.134.gate_proj", + "model.layers.29.mlp.experts.134.up_proj", + "model.layers.29.mlp.experts.135.down_proj", + "model.layers.29.mlp.experts.135.gate_proj", + "model.layers.29.mlp.experts.135.up_proj", + "model.layers.29.mlp.experts.136.down_proj", + "model.layers.29.mlp.experts.136.gate_proj", + "model.layers.29.mlp.experts.136.up_proj", + "model.layers.29.mlp.experts.137.down_proj", + "model.layers.29.mlp.experts.137.gate_proj", + "model.layers.29.mlp.experts.137.up_proj", + "model.layers.29.mlp.experts.138.down_proj", + "model.layers.29.mlp.experts.138.gate_proj", + "model.layers.29.mlp.experts.138.up_proj", + "model.layers.29.mlp.experts.139.down_proj", + "model.layers.29.mlp.experts.139.gate_proj", + "model.layers.29.mlp.experts.139.up_proj", + "model.layers.29.mlp.experts.14.down_proj", + "model.layers.29.mlp.experts.14.gate_proj", + "model.layers.29.mlp.experts.14.up_proj", + "model.layers.29.mlp.experts.140.down_proj", + "model.layers.29.mlp.experts.140.gate_proj", + "model.layers.29.mlp.experts.140.up_proj", + "model.layers.29.mlp.experts.141.down_proj", + "model.layers.29.mlp.experts.141.gate_proj", + "model.layers.29.mlp.experts.141.up_proj", + "model.layers.29.mlp.experts.142.down_proj", + "model.layers.29.mlp.experts.142.gate_proj", + "model.layers.29.mlp.experts.142.up_proj", + "model.layers.29.mlp.experts.143.down_proj", + "model.layers.29.mlp.experts.143.gate_proj", + "model.layers.29.mlp.experts.143.up_proj", + "model.layers.29.mlp.experts.144.down_proj", + "model.layers.29.mlp.experts.144.gate_proj", + "model.layers.29.mlp.experts.144.up_proj", + "model.layers.29.mlp.experts.145.down_proj", + "model.layers.29.mlp.experts.145.gate_proj", + "model.layers.29.mlp.experts.145.up_proj", + "model.layers.29.mlp.experts.146.down_proj", + "model.layers.29.mlp.experts.146.gate_proj", + "model.layers.29.mlp.experts.146.up_proj", + "model.layers.29.mlp.experts.147.down_proj", + "model.layers.29.mlp.experts.147.gate_proj", + "model.layers.29.mlp.experts.147.up_proj", + "model.layers.29.mlp.experts.148.down_proj", + "model.layers.29.mlp.experts.148.gate_proj", + "model.layers.29.mlp.experts.148.up_proj", + "model.layers.29.mlp.experts.149.down_proj", + "model.layers.29.mlp.experts.149.gate_proj", + "model.layers.29.mlp.experts.149.up_proj", + "model.layers.29.mlp.experts.15.down_proj", + "model.layers.29.mlp.experts.15.gate_proj", + "model.layers.29.mlp.experts.15.up_proj", + "model.layers.29.mlp.experts.150.down_proj", + "model.layers.29.mlp.experts.150.gate_proj", + "model.layers.29.mlp.experts.150.up_proj", + "model.layers.29.mlp.experts.151.down_proj", + "model.layers.29.mlp.experts.151.gate_proj", + "model.layers.29.mlp.experts.151.up_proj", + "model.layers.29.mlp.experts.152.down_proj", + "model.layers.29.mlp.experts.152.gate_proj", + "model.layers.29.mlp.experts.152.up_proj", + "model.layers.29.mlp.experts.153.down_proj", + "model.layers.29.mlp.experts.153.gate_proj", + "model.layers.29.mlp.experts.153.up_proj", + "model.layers.29.mlp.experts.154.down_proj", + "model.layers.29.mlp.experts.154.gate_proj", + "model.layers.29.mlp.experts.154.up_proj", + "model.layers.29.mlp.experts.155.down_proj", + "model.layers.29.mlp.experts.155.gate_proj", + "model.layers.29.mlp.experts.155.up_proj", + "model.layers.29.mlp.experts.156.down_proj", + "model.layers.29.mlp.experts.156.gate_proj", + "model.layers.29.mlp.experts.156.up_proj", + "model.layers.29.mlp.experts.157.down_proj", + "model.layers.29.mlp.experts.157.gate_proj", + "model.layers.29.mlp.experts.157.up_proj", + "model.layers.29.mlp.experts.158.down_proj", + "model.layers.29.mlp.experts.158.gate_proj", + "model.layers.29.mlp.experts.158.up_proj", + "model.layers.29.mlp.experts.159.down_proj", + "model.layers.29.mlp.experts.159.gate_proj", + "model.layers.29.mlp.experts.159.up_proj", + "model.layers.29.mlp.experts.16.down_proj", + "model.layers.29.mlp.experts.16.gate_proj", + "model.layers.29.mlp.experts.16.up_proj", + "model.layers.29.mlp.experts.160.down_proj", + "model.layers.29.mlp.experts.160.gate_proj", + "model.layers.29.mlp.experts.160.up_proj", + "model.layers.29.mlp.experts.161.down_proj", + "model.layers.29.mlp.experts.161.gate_proj", + "model.layers.29.mlp.experts.161.up_proj", + "model.layers.29.mlp.experts.162.down_proj", + "model.layers.29.mlp.experts.162.gate_proj", + "model.layers.29.mlp.experts.162.up_proj", + "model.layers.29.mlp.experts.163.down_proj", + "model.layers.29.mlp.experts.163.gate_proj", + "model.layers.29.mlp.experts.163.up_proj", + "model.layers.29.mlp.experts.164.down_proj", + "model.layers.29.mlp.experts.164.gate_proj", + "model.layers.29.mlp.experts.164.up_proj", + "model.layers.29.mlp.experts.165.down_proj", + "model.layers.29.mlp.experts.165.gate_proj", + "model.layers.29.mlp.experts.165.up_proj", + "model.layers.29.mlp.experts.166.down_proj", + "model.layers.29.mlp.experts.166.gate_proj", + "model.layers.29.mlp.experts.166.up_proj", + "model.layers.29.mlp.experts.167.down_proj", + "model.layers.29.mlp.experts.167.gate_proj", + "model.layers.29.mlp.experts.167.up_proj", + "model.layers.29.mlp.experts.168.down_proj", + "model.layers.29.mlp.experts.168.gate_proj", + "model.layers.29.mlp.experts.168.up_proj", + "model.layers.29.mlp.experts.169.down_proj", + "model.layers.29.mlp.experts.169.gate_proj", + "model.layers.29.mlp.experts.169.up_proj", + "model.layers.29.mlp.experts.17.down_proj", + "model.layers.29.mlp.experts.17.gate_proj", + "model.layers.29.mlp.experts.17.up_proj", + "model.layers.29.mlp.experts.170.down_proj", + "model.layers.29.mlp.experts.170.gate_proj", + "model.layers.29.mlp.experts.170.up_proj", + "model.layers.29.mlp.experts.171.down_proj", + "model.layers.29.mlp.experts.171.gate_proj", + "model.layers.29.mlp.experts.171.up_proj", + "model.layers.29.mlp.experts.172.down_proj", + "model.layers.29.mlp.experts.172.gate_proj", + "model.layers.29.mlp.experts.172.up_proj", + "model.layers.29.mlp.experts.173.down_proj", + "model.layers.29.mlp.experts.173.gate_proj", + "model.layers.29.mlp.experts.173.up_proj", + "model.layers.29.mlp.experts.174.down_proj", + "model.layers.29.mlp.experts.174.gate_proj", + "model.layers.29.mlp.experts.174.up_proj", + "model.layers.29.mlp.experts.175.down_proj", + "model.layers.29.mlp.experts.175.gate_proj", + "model.layers.29.mlp.experts.175.up_proj", + "model.layers.29.mlp.experts.176.down_proj", + "model.layers.29.mlp.experts.176.gate_proj", + "model.layers.29.mlp.experts.176.up_proj", + "model.layers.29.mlp.experts.177.down_proj", + "model.layers.29.mlp.experts.177.gate_proj", + "model.layers.29.mlp.experts.177.up_proj", + "model.layers.29.mlp.experts.178.down_proj", + "model.layers.29.mlp.experts.178.gate_proj", + "model.layers.29.mlp.experts.178.up_proj", + "model.layers.29.mlp.experts.179.down_proj", + "model.layers.29.mlp.experts.179.gate_proj", + "model.layers.29.mlp.experts.179.up_proj", + "model.layers.29.mlp.experts.18.down_proj", + "model.layers.29.mlp.experts.18.gate_proj", + "model.layers.29.mlp.experts.18.up_proj", + "model.layers.29.mlp.experts.180.down_proj", + "model.layers.29.mlp.experts.180.gate_proj", + "model.layers.29.mlp.experts.180.up_proj", + "model.layers.29.mlp.experts.181.down_proj", + "model.layers.29.mlp.experts.181.gate_proj", + "model.layers.29.mlp.experts.181.up_proj", + "model.layers.29.mlp.experts.182.down_proj", + "model.layers.29.mlp.experts.182.gate_proj", + "model.layers.29.mlp.experts.182.up_proj", + "model.layers.29.mlp.experts.183.down_proj", + "model.layers.29.mlp.experts.183.gate_proj", + "model.layers.29.mlp.experts.183.up_proj", + "model.layers.29.mlp.experts.184.down_proj", + "model.layers.29.mlp.experts.184.gate_proj", + "model.layers.29.mlp.experts.184.up_proj", + "model.layers.29.mlp.experts.185.down_proj", + "model.layers.29.mlp.experts.185.gate_proj", + "model.layers.29.mlp.experts.185.up_proj", + "model.layers.29.mlp.experts.186.down_proj", + "model.layers.29.mlp.experts.186.gate_proj", + "model.layers.29.mlp.experts.186.up_proj", + "model.layers.29.mlp.experts.187.down_proj", + "model.layers.29.mlp.experts.187.gate_proj", + "model.layers.29.mlp.experts.187.up_proj", + "model.layers.29.mlp.experts.188.down_proj", + "model.layers.29.mlp.experts.188.gate_proj", + "model.layers.29.mlp.experts.188.up_proj", + "model.layers.29.mlp.experts.189.down_proj", + "model.layers.29.mlp.experts.189.gate_proj", + "model.layers.29.mlp.experts.189.up_proj", + "model.layers.29.mlp.experts.19.down_proj", + "model.layers.29.mlp.experts.19.gate_proj", + "model.layers.29.mlp.experts.19.up_proj", + "model.layers.29.mlp.experts.190.down_proj", + "model.layers.29.mlp.experts.190.gate_proj", + "model.layers.29.mlp.experts.190.up_proj", + "model.layers.29.mlp.experts.191.down_proj", + "model.layers.29.mlp.experts.191.gate_proj", + "model.layers.29.mlp.experts.191.up_proj", + "model.layers.29.mlp.experts.192.down_proj", + "model.layers.29.mlp.experts.192.gate_proj", + "model.layers.29.mlp.experts.192.up_proj", + "model.layers.29.mlp.experts.193.down_proj", + "model.layers.29.mlp.experts.193.gate_proj", + "model.layers.29.mlp.experts.193.up_proj", + "model.layers.29.mlp.experts.194.down_proj", + "model.layers.29.mlp.experts.194.gate_proj", + "model.layers.29.mlp.experts.194.up_proj", + "model.layers.29.mlp.experts.195.down_proj", + "model.layers.29.mlp.experts.195.gate_proj", + "model.layers.29.mlp.experts.195.up_proj", + "model.layers.29.mlp.experts.196.down_proj", + "model.layers.29.mlp.experts.196.gate_proj", + "model.layers.29.mlp.experts.196.up_proj", + "model.layers.29.mlp.experts.197.down_proj", + "model.layers.29.mlp.experts.197.gate_proj", + "model.layers.29.mlp.experts.197.up_proj", + "model.layers.29.mlp.experts.198.down_proj", + "model.layers.29.mlp.experts.198.gate_proj", + "model.layers.29.mlp.experts.198.up_proj", + "model.layers.29.mlp.experts.199.down_proj", + "model.layers.29.mlp.experts.199.gate_proj", + "model.layers.29.mlp.experts.199.up_proj", + "model.layers.29.mlp.experts.2.down_proj", + "model.layers.29.mlp.experts.2.gate_proj", + "model.layers.29.mlp.experts.2.up_proj", + "model.layers.29.mlp.experts.20.down_proj", + "model.layers.29.mlp.experts.20.gate_proj", + "model.layers.29.mlp.experts.20.up_proj", + "model.layers.29.mlp.experts.200.down_proj", + "model.layers.29.mlp.experts.200.gate_proj", + "model.layers.29.mlp.experts.200.up_proj", + "model.layers.29.mlp.experts.201.down_proj", + "model.layers.29.mlp.experts.201.gate_proj", + "model.layers.29.mlp.experts.201.up_proj", + "model.layers.29.mlp.experts.202.down_proj", + "model.layers.29.mlp.experts.202.gate_proj", + "model.layers.29.mlp.experts.202.up_proj", + "model.layers.29.mlp.experts.203.down_proj", + "model.layers.29.mlp.experts.203.gate_proj", + "model.layers.29.mlp.experts.203.up_proj", + "model.layers.29.mlp.experts.204.down_proj", + "model.layers.29.mlp.experts.204.gate_proj", + "model.layers.29.mlp.experts.204.up_proj", + "model.layers.29.mlp.experts.205.down_proj", + "model.layers.29.mlp.experts.205.gate_proj", + "model.layers.29.mlp.experts.205.up_proj", + "model.layers.29.mlp.experts.206.down_proj", + "model.layers.29.mlp.experts.206.gate_proj", + "model.layers.29.mlp.experts.206.up_proj", + "model.layers.29.mlp.experts.207.down_proj", + "model.layers.29.mlp.experts.207.gate_proj", + "model.layers.29.mlp.experts.207.up_proj", + "model.layers.29.mlp.experts.208.down_proj", + "model.layers.29.mlp.experts.208.gate_proj", + "model.layers.29.mlp.experts.208.up_proj", + "model.layers.29.mlp.experts.209.down_proj", + "model.layers.29.mlp.experts.209.gate_proj", + "model.layers.29.mlp.experts.209.up_proj", + "model.layers.29.mlp.experts.21.down_proj", + "model.layers.29.mlp.experts.21.gate_proj", + "model.layers.29.mlp.experts.21.up_proj", + "model.layers.29.mlp.experts.210.down_proj", + "model.layers.29.mlp.experts.210.gate_proj", + "model.layers.29.mlp.experts.210.up_proj", + "model.layers.29.mlp.experts.211.down_proj", + "model.layers.29.mlp.experts.211.gate_proj", + "model.layers.29.mlp.experts.211.up_proj", + "model.layers.29.mlp.experts.212.down_proj", + "model.layers.29.mlp.experts.212.gate_proj", + "model.layers.29.mlp.experts.212.up_proj", + "model.layers.29.mlp.experts.213.down_proj", + "model.layers.29.mlp.experts.213.gate_proj", + "model.layers.29.mlp.experts.213.up_proj", + "model.layers.29.mlp.experts.214.down_proj", + "model.layers.29.mlp.experts.214.gate_proj", + "model.layers.29.mlp.experts.214.up_proj", + "model.layers.29.mlp.experts.215.down_proj", + "model.layers.29.mlp.experts.215.gate_proj", + "model.layers.29.mlp.experts.215.up_proj", + "model.layers.29.mlp.experts.216.down_proj", + "model.layers.29.mlp.experts.216.gate_proj", + "model.layers.29.mlp.experts.216.up_proj", + "model.layers.29.mlp.experts.217.down_proj", + "model.layers.29.mlp.experts.217.gate_proj", + "model.layers.29.mlp.experts.217.up_proj", + "model.layers.29.mlp.experts.218.down_proj", + "model.layers.29.mlp.experts.218.gate_proj", + "model.layers.29.mlp.experts.218.up_proj", + "model.layers.29.mlp.experts.219.down_proj", + "model.layers.29.mlp.experts.219.gate_proj", + "model.layers.29.mlp.experts.219.up_proj", + "model.layers.29.mlp.experts.22.down_proj", + "model.layers.29.mlp.experts.22.gate_proj", + "model.layers.29.mlp.experts.22.up_proj", + "model.layers.29.mlp.experts.220.down_proj", + "model.layers.29.mlp.experts.220.gate_proj", + "model.layers.29.mlp.experts.220.up_proj", + "model.layers.29.mlp.experts.221.down_proj", + "model.layers.29.mlp.experts.221.gate_proj", + "model.layers.29.mlp.experts.221.up_proj", + "model.layers.29.mlp.experts.222.down_proj", + "model.layers.29.mlp.experts.222.gate_proj", + "model.layers.29.mlp.experts.222.up_proj", + "model.layers.29.mlp.experts.223.down_proj", + "model.layers.29.mlp.experts.223.gate_proj", + "model.layers.29.mlp.experts.223.up_proj", + "model.layers.29.mlp.experts.224.down_proj", + "model.layers.29.mlp.experts.224.gate_proj", + "model.layers.29.mlp.experts.224.up_proj", + "model.layers.29.mlp.experts.225.down_proj", + "model.layers.29.mlp.experts.225.gate_proj", + "model.layers.29.mlp.experts.225.up_proj", + "model.layers.29.mlp.experts.226.down_proj", + "model.layers.29.mlp.experts.226.gate_proj", + "model.layers.29.mlp.experts.226.up_proj", + "model.layers.29.mlp.experts.227.down_proj", + "model.layers.29.mlp.experts.227.gate_proj", + "model.layers.29.mlp.experts.227.up_proj", + "model.layers.29.mlp.experts.228.down_proj", + "model.layers.29.mlp.experts.228.gate_proj", + "model.layers.29.mlp.experts.228.up_proj", + "model.layers.29.mlp.experts.229.down_proj", + "model.layers.29.mlp.experts.229.gate_proj", + "model.layers.29.mlp.experts.229.up_proj", + "model.layers.29.mlp.experts.23.down_proj", + "model.layers.29.mlp.experts.23.gate_proj", + "model.layers.29.mlp.experts.23.up_proj", + "model.layers.29.mlp.experts.230.down_proj", + "model.layers.29.mlp.experts.230.gate_proj", + "model.layers.29.mlp.experts.230.up_proj", + "model.layers.29.mlp.experts.231.down_proj", + "model.layers.29.mlp.experts.231.gate_proj", + "model.layers.29.mlp.experts.231.up_proj", + "model.layers.29.mlp.experts.232.down_proj", + "model.layers.29.mlp.experts.232.gate_proj", + "model.layers.29.mlp.experts.232.up_proj", + "model.layers.29.mlp.experts.233.down_proj", + "model.layers.29.mlp.experts.233.gate_proj", + "model.layers.29.mlp.experts.233.up_proj", + "model.layers.29.mlp.experts.234.down_proj", + "model.layers.29.mlp.experts.234.gate_proj", + "model.layers.29.mlp.experts.234.up_proj", + "model.layers.29.mlp.experts.235.down_proj", + "model.layers.29.mlp.experts.235.gate_proj", + "model.layers.29.mlp.experts.235.up_proj", + "model.layers.29.mlp.experts.236.down_proj", + "model.layers.29.mlp.experts.236.gate_proj", + "model.layers.29.mlp.experts.236.up_proj", + "model.layers.29.mlp.experts.237.down_proj", + "model.layers.29.mlp.experts.237.gate_proj", + "model.layers.29.mlp.experts.237.up_proj", + "model.layers.29.mlp.experts.238.down_proj", + "model.layers.29.mlp.experts.238.gate_proj", + "model.layers.29.mlp.experts.238.up_proj", + "model.layers.29.mlp.experts.239.down_proj", + "model.layers.29.mlp.experts.239.gate_proj", + "model.layers.29.mlp.experts.239.up_proj", + "model.layers.29.mlp.experts.24.down_proj", + "model.layers.29.mlp.experts.24.gate_proj", + "model.layers.29.mlp.experts.24.up_proj", + "model.layers.29.mlp.experts.240.down_proj", + "model.layers.29.mlp.experts.240.gate_proj", + "model.layers.29.mlp.experts.240.up_proj", + "model.layers.29.mlp.experts.241.down_proj", + "model.layers.29.mlp.experts.241.gate_proj", + "model.layers.29.mlp.experts.241.up_proj", + "model.layers.29.mlp.experts.242.down_proj", + "model.layers.29.mlp.experts.242.gate_proj", + "model.layers.29.mlp.experts.242.up_proj", + "model.layers.29.mlp.experts.243.down_proj", + "model.layers.29.mlp.experts.243.gate_proj", + "model.layers.29.mlp.experts.243.up_proj", + "model.layers.29.mlp.experts.244.down_proj", + "model.layers.29.mlp.experts.244.gate_proj", + "model.layers.29.mlp.experts.244.up_proj", + "model.layers.29.mlp.experts.245.down_proj", + "model.layers.29.mlp.experts.245.gate_proj", + "model.layers.29.mlp.experts.245.up_proj", + "model.layers.29.mlp.experts.246.down_proj", + "model.layers.29.mlp.experts.246.gate_proj", + "model.layers.29.mlp.experts.246.up_proj", + "model.layers.29.mlp.experts.247.down_proj", + "model.layers.29.mlp.experts.247.gate_proj", + "model.layers.29.mlp.experts.247.up_proj", + "model.layers.29.mlp.experts.248.down_proj", + "model.layers.29.mlp.experts.248.gate_proj", + "model.layers.29.mlp.experts.248.up_proj", + "model.layers.29.mlp.experts.249.down_proj", + "model.layers.29.mlp.experts.249.gate_proj", + "model.layers.29.mlp.experts.249.up_proj", + "model.layers.29.mlp.experts.25.down_proj", + "model.layers.29.mlp.experts.25.gate_proj", + "model.layers.29.mlp.experts.25.up_proj", + "model.layers.29.mlp.experts.250.down_proj", + "model.layers.29.mlp.experts.250.gate_proj", + "model.layers.29.mlp.experts.250.up_proj", + "model.layers.29.mlp.experts.251.down_proj", + "model.layers.29.mlp.experts.251.gate_proj", + "model.layers.29.mlp.experts.251.up_proj", + "model.layers.29.mlp.experts.252.down_proj", + "model.layers.29.mlp.experts.252.gate_proj", + "model.layers.29.mlp.experts.252.up_proj", + "model.layers.29.mlp.experts.253.down_proj", + "model.layers.29.mlp.experts.253.gate_proj", + "model.layers.29.mlp.experts.253.up_proj", + "model.layers.29.mlp.experts.254.down_proj", + "model.layers.29.mlp.experts.254.gate_proj", + "model.layers.29.mlp.experts.254.up_proj", + "model.layers.29.mlp.experts.255.down_proj", + "model.layers.29.mlp.experts.255.gate_proj", + "model.layers.29.mlp.experts.255.up_proj", + "model.layers.29.mlp.experts.26.down_proj", + "model.layers.29.mlp.experts.26.gate_proj", + "model.layers.29.mlp.experts.26.up_proj", + "model.layers.29.mlp.experts.27.down_proj", + "model.layers.29.mlp.experts.27.gate_proj", + "model.layers.29.mlp.experts.27.up_proj", + "model.layers.29.mlp.experts.28.down_proj", + "model.layers.29.mlp.experts.28.gate_proj", + "model.layers.29.mlp.experts.28.up_proj", + "model.layers.29.mlp.experts.29.down_proj", + "model.layers.29.mlp.experts.29.gate_proj", + "model.layers.29.mlp.experts.29.up_proj", + "model.layers.29.mlp.experts.3.down_proj", + "model.layers.29.mlp.experts.3.gate_proj", + "model.layers.29.mlp.experts.3.up_proj", + "model.layers.29.mlp.experts.30.down_proj", + "model.layers.29.mlp.experts.30.gate_proj", + "model.layers.29.mlp.experts.30.up_proj", + "model.layers.29.mlp.experts.31.down_proj", + "model.layers.29.mlp.experts.31.gate_proj", + "model.layers.29.mlp.experts.31.up_proj", + "model.layers.29.mlp.experts.32.down_proj", + "model.layers.29.mlp.experts.32.gate_proj", + "model.layers.29.mlp.experts.32.up_proj", + "model.layers.29.mlp.experts.33.down_proj", + "model.layers.29.mlp.experts.33.gate_proj", + "model.layers.29.mlp.experts.33.up_proj", + "model.layers.29.mlp.experts.34.down_proj", + "model.layers.29.mlp.experts.34.gate_proj", + "model.layers.29.mlp.experts.34.up_proj", + "model.layers.29.mlp.experts.35.down_proj", + "model.layers.29.mlp.experts.35.gate_proj", + "model.layers.29.mlp.experts.35.up_proj", + "model.layers.29.mlp.experts.36.down_proj", + "model.layers.29.mlp.experts.36.gate_proj", + "model.layers.29.mlp.experts.36.up_proj", + "model.layers.29.mlp.experts.37.down_proj", + "model.layers.29.mlp.experts.37.gate_proj", + "model.layers.29.mlp.experts.37.up_proj", + "model.layers.29.mlp.experts.38.down_proj", + "model.layers.29.mlp.experts.38.gate_proj", + "model.layers.29.mlp.experts.38.up_proj", + "model.layers.29.mlp.experts.39.down_proj", + "model.layers.29.mlp.experts.39.gate_proj", + "model.layers.29.mlp.experts.39.up_proj", + "model.layers.29.mlp.experts.4.down_proj", + "model.layers.29.mlp.experts.4.gate_proj", + "model.layers.29.mlp.experts.4.up_proj", + "model.layers.29.mlp.experts.40.down_proj", + "model.layers.29.mlp.experts.40.gate_proj", + "model.layers.29.mlp.experts.40.up_proj", + "model.layers.29.mlp.experts.41.down_proj", + "model.layers.29.mlp.experts.41.gate_proj", + "model.layers.29.mlp.experts.41.up_proj", + "model.layers.29.mlp.experts.42.down_proj", + "model.layers.29.mlp.experts.42.gate_proj", + "model.layers.29.mlp.experts.42.up_proj", + "model.layers.29.mlp.experts.43.down_proj", + "model.layers.29.mlp.experts.43.gate_proj", + "model.layers.29.mlp.experts.43.up_proj", + "model.layers.29.mlp.experts.44.down_proj", + "model.layers.29.mlp.experts.44.gate_proj", + "model.layers.29.mlp.experts.44.up_proj", + "model.layers.29.mlp.experts.45.down_proj", + "model.layers.29.mlp.experts.45.gate_proj", + "model.layers.29.mlp.experts.45.up_proj", + "model.layers.29.mlp.experts.46.down_proj", + "model.layers.29.mlp.experts.46.gate_proj", + "model.layers.29.mlp.experts.46.up_proj", + "model.layers.29.mlp.experts.47.down_proj", + "model.layers.29.mlp.experts.47.gate_proj", + "model.layers.29.mlp.experts.47.up_proj", + "model.layers.29.mlp.experts.48.down_proj", + "model.layers.29.mlp.experts.48.gate_proj", + "model.layers.29.mlp.experts.48.up_proj", + "model.layers.29.mlp.experts.49.down_proj", + "model.layers.29.mlp.experts.49.gate_proj", + "model.layers.29.mlp.experts.49.up_proj", + "model.layers.29.mlp.experts.5.down_proj", + "model.layers.29.mlp.experts.5.gate_proj", + "model.layers.29.mlp.experts.5.up_proj", + "model.layers.29.mlp.experts.50.down_proj", + "model.layers.29.mlp.experts.50.gate_proj", + "model.layers.29.mlp.experts.50.up_proj", + "model.layers.29.mlp.experts.51.down_proj", + "model.layers.29.mlp.experts.51.gate_proj", + "model.layers.29.mlp.experts.51.up_proj", + "model.layers.29.mlp.experts.52.down_proj", + "model.layers.29.mlp.experts.52.gate_proj", + "model.layers.29.mlp.experts.52.up_proj", + "model.layers.29.mlp.experts.53.down_proj", + "model.layers.29.mlp.experts.53.gate_proj", + "model.layers.29.mlp.experts.53.up_proj", + "model.layers.29.mlp.experts.54.down_proj", + "model.layers.29.mlp.experts.54.gate_proj", + "model.layers.29.mlp.experts.54.up_proj", + "model.layers.29.mlp.experts.55.down_proj", + "model.layers.29.mlp.experts.55.gate_proj", + "model.layers.29.mlp.experts.55.up_proj", + "model.layers.29.mlp.experts.56.down_proj", + "model.layers.29.mlp.experts.56.gate_proj", + "model.layers.29.mlp.experts.56.up_proj", + "model.layers.29.mlp.experts.57.down_proj", + "model.layers.29.mlp.experts.57.gate_proj", + "model.layers.29.mlp.experts.57.up_proj", + "model.layers.29.mlp.experts.58.down_proj", + "model.layers.29.mlp.experts.58.gate_proj", + "model.layers.29.mlp.experts.58.up_proj", + "model.layers.29.mlp.experts.59.down_proj", + "model.layers.29.mlp.experts.59.gate_proj", + "model.layers.29.mlp.experts.59.up_proj", + "model.layers.29.mlp.experts.6.down_proj", + "model.layers.29.mlp.experts.6.gate_proj", + "model.layers.29.mlp.experts.6.up_proj", + "model.layers.29.mlp.experts.60.down_proj", + "model.layers.29.mlp.experts.60.gate_proj", + "model.layers.29.mlp.experts.60.up_proj", + "model.layers.29.mlp.experts.61.down_proj", + "model.layers.29.mlp.experts.61.gate_proj", + "model.layers.29.mlp.experts.61.up_proj", + "model.layers.29.mlp.experts.62.down_proj", + "model.layers.29.mlp.experts.62.gate_proj", + "model.layers.29.mlp.experts.62.up_proj", + "model.layers.29.mlp.experts.63.down_proj", + "model.layers.29.mlp.experts.63.gate_proj", + "model.layers.29.mlp.experts.63.up_proj", + "model.layers.29.mlp.experts.64.down_proj", + "model.layers.29.mlp.experts.64.gate_proj", + "model.layers.29.mlp.experts.64.up_proj", + "model.layers.29.mlp.experts.65.down_proj", + "model.layers.29.mlp.experts.65.gate_proj", + "model.layers.29.mlp.experts.65.up_proj", + "model.layers.29.mlp.experts.66.down_proj", + "model.layers.29.mlp.experts.66.gate_proj", + "model.layers.29.mlp.experts.66.up_proj", + "model.layers.29.mlp.experts.67.down_proj", + "model.layers.29.mlp.experts.67.gate_proj", + "model.layers.29.mlp.experts.67.up_proj", + "model.layers.29.mlp.experts.68.down_proj", + "model.layers.29.mlp.experts.68.gate_proj", + "model.layers.29.mlp.experts.68.up_proj", + "model.layers.29.mlp.experts.69.down_proj", + "model.layers.29.mlp.experts.69.gate_proj", + "model.layers.29.mlp.experts.69.up_proj", + "model.layers.29.mlp.experts.7.down_proj", + "model.layers.29.mlp.experts.7.gate_proj", + "model.layers.29.mlp.experts.7.up_proj", + "model.layers.29.mlp.experts.70.down_proj", + "model.layers.29.mlp.experts.70.gate_proj", + "model.layers.29.mlp.experts.70.up_proj", + "model.layers.29.mlp.experts.71.down_proj", + "model.layers.29.mlp.experts.71.gate_proj", + "model.layers.29.mlp.experts.71.up_proj", + "model.layers.29.mlp.experts.72.down_proj", + "model.layers.29.mlp.experts.72.gate_proj", + "model.layers.29.mlp.experts.72.up_proj", + "model.layers.29.mlp.experts.73.down_proj", + "model.layers.29.mlp.experts.73.gate_proj", + "model.layers.29.mlp.experts.73.up_proj", + "model.layers.29.mlp.experts.74.down_proj", + "model.layers.29.mlp.experts.74.gate_proj", + "model.layers.29.mlp.experts.74.up_proj", + "model.layers.29.mlp.experts.75.down_proj", + "model.layers.29.mlp.experts.75.gate_proj", + "model.layers.29.mlp.experts.75.up_proj", + "model.layers.29.mlp.experts.76.down_proj", + "model.layers.29.mlp.experts.76.gate_proj", + "model.layers.29.mlp.experts.76.up_proj", + "model.layers.29.mlp.experts.77.down_proj", + "model.layers.29.mlp.experts.77.gate_proj", + "model.layers.29.mlp.experts.77.up_proj", + "model.layers.29.mlp.experts.78.down_proj", + "model.layers.29.mlp.experts.78.gate_proj", + "model.layers.29.mlp.experts.78.up_proj", + "model.layers.29.mlp.experts.79.down_proj", + "model.layers.29.mlp.experts.79.gate_proj", + "model.layers.29.mlp.experts.79.up_proj", + "model.layers.29.mlp.experts.8.down_proj", + "model.layers.29.mlp.experts.8.gate_proj", + "model.layers.29.mlp.experts.8.up_proj", + "model.layers.29.mlp.experts.80.down_proj", + "model.layers.29.mlp.experts.80.gate_proj", + "model.layers.29.mlp.experts.80.up_proj", + "model.layers.29.mlp.experts.81.down_proj", + "model.layers.29.mlp.experts.81.gate_proj", + "model.layers.29.mlp.experts.81.up_proj", + "model.layers.29.mlp.experts.82.down_proj", + "model.layers.29.mlp.experts.82.gate_proj", + "model.layers.29.mlp.experts.82.up_proj", + "model.layers.29.mlp.experts.83.down_proj", + "model.layers.29.mlp.experts.83.gate_proj", + "model.layers.29.mlp.experts.83.up_proj", + "model.layers.29.mlp.experts.84.down_proj", + "model.layers.29.mlp.experts.84.gate_proj", + "model.layers.29.mlp.experts.84.up_proj", + "model.layers.29.mlp.experts.85.down_proj", + "model.layers.29.mlp.experts.85.gate_proj", + "model.layers.29.mlp.experts.85.up_proj", + "model.layers.29.mlp.experts.86.down_proj", + "model.layers.29.mlp.experts.86.gate_proj", + "model.layers.29.mlp.experts.86.up_proj", + "model.layers.29.mlp.experts.87.down_proj", + "model.layers.29.mlp.experts.87.gate_proj", + "model.layers.29.mlp.experts.87.up_proj", + "model.layers.29.mlp.experts.88.down_proj", + "model.layers.29.mlp.experts.88.gate_proj", + "model.layers.29.mlp.experts.88.up_proj", + "model.layers.29.mlp.experts.89.down_proj", + "model.layers.29.mlp.experts.89.gate_proj", + "model.layers.29.mlp.experts.89.up_proj", + "model.layers.29.mlp.experts.9.down_proj", + "model.layers.29.mlp.experts.9.gate_proj", + "model.layers.29.mlp.experts.9.up_proj", + "model.layers.29.mlp.experts.90.down_proj", + "model.layers.29.mlp.experts.90.gate_proj", + "model.layers.29.mlp.experts.90.up_proj", + "model.layers.29.mlp.experts.91.down_proj", + "model.layers.29.mlp.experts.91.gate_proj", + "model.layers.29.mlp.experts.91.up_proj", + "model.layers.29.mlp.experts.92.down_proj", + "model.layers.29.mlp.experts.92.gate_proj", + "model.layers.29.mlp.experts.92.up_proj", + "model.layers.29.mlp.experts.93.down_proj", + "model.layers.29.mlp.experts.93.gate_proj", + "model.layers.29.mlp.experts.93.up_proj", + "model.layers.29.mlp.experts.94.down_proj", + "model.layers.29.mlp.experts.94.gate_proj", + "model.layers.29.mlp.experts.94.up_proj", + "model.layers.29.mlp.experts.95.down_proj", + "model.layers.29.mlp.experts.95.gate_proj", + "model.layers.29.mlp.experts.95.up_proj", + "model.layers.29.mlp.experts.96.down_proj", + "model.layers.29.mlp.experts.96.gate_proj", + "model.layers.29.mlp.experts.96.up_proj", + "model.layers.29.mlp.experts.97.down_proj", + "model.layers.29.mlp.experts.97.gate_proj", + "model.layers.29.mlp.experts.97.up_proj", + "model.layers.29.mlp.experts.98.down_proj", + "model.layers.29.mlp.experts.98.gate_proj", + "model.layers.29.mlp.experts.98.up_proj", + "model.layers.29.mlp.experts.99.down_proj", + "model.layers.29.mlp.experts.99.gate_proj", + "model.layers.29.mlp.experts.99.up_proj", + "model.layers.3.mlp.experts.0.down_proj", + "model.layers.3.mlp.experts.0.gate_proj", + "model.layers.3.mlp.experts.0.up_proj", + "model.layers.3.mlp.experts.1.down_proj", + "model.layers.3.mlp.experts.1.gate_proj", + "model.layers.3.mlp.experts.1.up_proj", + "model.layers.3.mlp.experts.10.down_proj", + "model.layers.3.mlp.experts.10.gate_proj", + "model.layers.3.mlp.experts.10.up_proj", + "model.layers.3.mlp.experts.100.down_proj", + "model.layers.3.mlp.experts.100.gate_proj", + "model.layers.3.mlp.experts.100.up_proj", + "model.layers.3.mlp.experts.101.down_proj", + "model.layers.3.mlp.experts.101.gate_proj", + "model.layers.3.mlp.experts.101.up_proj", + "model.layers.3.mlp.experts.102.down_proj", + "model.layers.3.mlp.experts.102.gate_proj", + "model.layers.3.mlp.experts.102.up_proj", + "model.layers.3.mlp.experts.103.down_proj", + "model.layers.3.mlp.experts.103.gate_proj", + "model.layers.3.mlp.experts.103.up_proj", + "model.layers.3.mlp.experts.104.down_proj", + "model.layers.3.mlp.experts.104.gate_proj", + "model.layers.3.mlp.experts.104.up_proj", + "model.layers.3.mlp.experts.105.down_proj", + "model.layers.3.mlp.experts.105.gate_proj", + "model.layers.3.mlp.experts.105.up_proj", + "model.layers.3.mlp.experts.106.down_proj", + "model.layers.3.mlp.experts.106.gate_proj", + "model.layers.3.mlp.experts.106.up_proj", + "model.layers.3.mlp.experts.107.down_proj", + "model.layers.3.mlp.experts.107.gate_proj", + "model.layers.3.mlp.experts.107.up_proj", + "model.layers.3.mlp.experts.108.down_proj", + "model.layers.3.mlp.experts.108.gate_proj", + "model.layers.3.mlp.experts.108.up_proj", + "model.layers.3.mlp.experts.109.down_proj", + "model.layers.3.mlp.experts.109.gate_proj", + "model.layers.3.mlp.experts.109.up_proj", + "model.layers.3.mlp.experts.11.down_proj", + "model.layers.3.mlp.experts.11.gate_proj", + "model.layers.3.mlp.experts.11.up_proj", + "model.layers.3.mlp.experts.110.down_proj", + "model.layers.3.mlp.experts.110.gate_proj", + "model.layers.3.mlp.experts.110.up_proj", + "model.layers.3.mlp.experts.111.down_proj", + "model.layers.3.mlp.experts.111.gate_proj", + "model.layers.3.mlp.experts.111.up_proj", + "model.layers.3.mlp.experts.112.down_proj", + "model.layers.3.mlp.experts.112.gate_proj", + "model.layers.3.mlp.experts.112.up_proj", + "model.layers.3.mlp.experts.113.down_proj", + "model.layers.3.mlp.experts.113.gate_proj", + "model.layers.3.mlp.experts.113.up_proj", + "model.layers.3.mlp.experts.114.down_proj", + "model.layers.3.mlp.experts.114.gate_proj", + "model.layers.3.mlp.experts.114.up_proj", + "model.layers.3.mlp.experts.115.down_proj", + "model.layers.3.mlp.experts.115.gate_proj", + "model.layers.3.mlp.experts.115.up_proj", + "model.layers.3.mlp.experts.116.down_proj", + "model.layers.3.mlp.experts.116.gate_proj", + "model.layers.3.mlp.experts.116.up_proj", + "model.layers.3.mlp.experts.117.down_proj", + "model.layers.3.mlp.experts.117.gate_proj", + "model.layers.3.mlp.experts.117.up_proj", + "model.layers.3.mlp.experts.118.down_proj", + "model.layers.3.mlp.experts.118.gate_proj", + "model.layers.3.mlp.experts.118.up_proj", + "model.layers.3.mlp.experts.119.down_proj", + "model.layers.3.mlp.experts.119.gate_proj", + "model.layers.3.mlp.experts.119.up_proj", + "model.layers.3.mlp.experts.12.down_proj", + "model.layers.3.mlp.experts.12.gate_proj", + "model.layers.3.mlp.experts.12.up_proj", + "model.layers.3.mlp.experts.120.down_proj", + "model.layers.3.mlp.experts.120.gate_proj", + "model.layers.3.mlp.experts.120.up_proj", + "model.layers.3.mlp.experts.121.down_proj", + "model.layers.3.mlp.experts.121.gate_proj", + "model.layers.3.mlp.experts.121.up_proj", + "model.layers.3.mlp.experts.122.down_proj", + "model.layers.3.mlp.experts.122.gate_proj", + "model.layers.3.mlp.experts.122.up_proj", + "model.layers.3.mlp.experts.123.down_proj", + "model.layers.3.mlp.experts.123.gate_proj", + "model.layers.3.mlp.experts.123.up_proj", + "model.layers.3.mlp.experts.124.down_proj", + "model.layers.3.mlp.experts.124.gate_proj", + "model.layers.3.mlp.experts.124.up_proj", + "model.layers.3.mlp.experts.125.down_proj", + "model.layers.3.mlp.experts.125.gate_proj", + "model.layers.3.mlp.experts.125.up_proj", + "model.layers.3.mlp.experts.126.down_proj", + "model.layers.3.mlp.experts.126.gate_proj", + "model.layers.3.mlp.experts.126.up_proj", + "model.layers.3.mlp.experts.127.down_proj", + "model.layers.3.mlp.experts.127.gate_proj", + "model.layers.3.mlp.experts.127.up_proj", + "model.layers.3.mlp.experts.128.down_proj", + "model.layers.3.mlp.experts.128.gate_proj", + "model.layers.3.mlp.experts.128.up_proj", + "model.layers.3.mlp.experts.129.down_proj", + "model.layers.3.mlp.experts.129.gate_proj", + "model.layers.3.mlp.experts.129.up_proj", + "model.layers.3.mlp.experts.13.down_proj", + "model.layers.3.mlp.experts.13.gate_proj", + "model.layers.3.mlp.experts.13.up_proj", + "model.layers.3.mlp.experts.130.down_proj", + "model.layers.3.mlp.experts.130.gate_proj", + "model.layers.3.mlp.experts.130.up_proj", + "model.layers.3.mlp.experts.131.down_proj", + "model.layers.3.mlp.experts.131.gate_proj", + "model.layers.3.mlp.experts.131.up_proj", + "model.layers.3.mlp.experts.132.down_proj", + "model.layers.3.mlp.experts.132.gate_proj", + "model.layers.3.mlp.experts.132.up_proj", + "model.layers.3.mlp.experts.133.down_proj", + "model.layers.3.mlp.experts.133.gate_proj", + "model.layers.3.mlp.experts.133.up_proj", + "model.layers.3.mlp.experts.134.down_proj", + "model.layers.3.mlp.experts.134.gate_proj", + "model.layers.3.mlp.experts.134.up_proj", + "model.layers.3.mlp.experts.135.down_proj", + "model.layers.3.mlp.experts.135.gate_proj", + "model.layers.3.mlp.experts.135.up_proj", + "model.layers.3.mlp.experts.136.down_proj", + "model.layers.3.mlp.experts.136.gate_proj", + "model.layers.3.mlp.experts.136.up_proj", + "model.layers.3.mlp.experts.137.down_proj", + "model.layers.3.mlp.experts.137.gate_proj", + "model.layers.3.mlp.experts.137.up_proj", + "model.layers.3.mlp.experts.138.down_proj", + "model.layers.3.mlp.experts.138.gate_proj", + "model.layers.3.mlp.experts.138.up_proj", + "model.layers.3.mlp.experts.139.down_proj", + "model.layers.3.mlp.experts.139.gate_proj", + "model.layers.3.mlp.experts.139.up_proj", + "model.layers.3.mlp.experts.14.down_proj", + "model.layers.3.mlp.experts.14.gate_proj", + "model.layers.3.mlp.experts.14.up_proj", + "model.layers.3.mlp.experts.140.down_proj", + "model.layers.3.mlp.experts.140.gate_proj", + "model.layers.3.mlp.experts.140.up_proj", + "model.layers.3.mlp.experts.141.down_proj", + "model.layers.3.mlp.experts.141.gate_proj", + "model.layers.3.mlp.experts.141.up_proj", + "model.layers.3.mlp.experts.142.down_proj", + "model.layers.3.mlp.experts.142.gate_proj", + "model.layers.3.mlp.experts.142.up_proj", + "model.layers.3.mlp.experts.143.down_proj", + "model.layers.3.mlp.experts.143.gate_proj", + "model.layers.3.mlp.experts.143.up_proj", + "model.layers.3.mlp.experts.144.down_proj", + "model.layers.3.mlp.experts.144.gate_proj", + "model.layers.3.mlp.experts.144.up_proj", + "model.layers.3.mlp.experts.145.down_proj", + "model.layers.3.mlp.experts.145.gate_proj", + "model.layers.3.mlp.experts.145.up_proj", + "model.layers.3.mlp.experts.146.down_proj", + "model.layers.3.mlp.experts.146.gate_proj", + "model.layers.3.mlp.experts.146.up_proj", + "model.layers.3.mlp.experts.147.down_proj", + "model.layers.3.mlp.experts.147.gate_proj", + "model.layers.3.mlp.experts.147.up_proj", + "model.layers.3.mlp.experts.148.down_proj", + "model.layers.3.mlp.experts.148.gate_proj", + "model.layers.3.mlp.experts.148.up_proj", + "model.layers.3.mlp.experts.149.down_proj", + "model.layers.3.mlp.experts.149.gate_proj", + "model.layers.3.mlp.experts.149.up_proj", + "model.layers.3.mlp.experts.15.down_proj", + "model.layers.3.mlp.experts.15.gate_proj", + "model.layers.3.mlp.experts.15.up_proj", + "model.layers.3.mlp.experts.150.down_proj", + "model.layers.3.mlp.experts.150.gate_proj", + "model.layers.3.mlp.experts.150.up_proj", + "model.layers.3.mlp.experts.151.down_proj", + "model.layers.3.mlp.experts.151.gate_proj", + "model.layers.3.mlp.experts.151.up_proj", + "model.layers.3.mlp.experts.152.down_proj", + "model.layers.3.mlp.experts.152.gate_proj", + "model.layers.3.mlp.experts.152.up_proj", + "model.layers.3.mlp.experts.153.down_proj", + "model.layers.3.mlp.experts.153.gate_proj", + "model.layers.3.mlp.experts.153.up_proj", + "model.layers.3.mlp.experts.154.down_proj", + "model.layers.3.mlp.experts.154.gate_proj", + "model.layers.3.mlp.experts.154.up_proj", + "model.layers.3.mlp.experts.155.down_proj", + "model.layers.3.mlp.experts.155.gate_proj", + "model.layers.3.mlp.experts.155.up_proj", + "model.layers.3.mlp.experts.156.down_proj", + "model.layers.3.mlp.experts.156.gate_proj", + "model.layers.3.mlp.experts.156.up_proj", + "model.layers.3.mlp.experts.157.down_proj", + "model.layers.3.mlp.experts.157.gate_proj", + "model.layers.3.mlp.experts.157.up_proj", + "model.layers.3.mlp.experts.158.down_proj", + "model.layers.3.mlp.experts.158.gate_proj", + "model.layers.3.mlp.experts.158.up_proj", + "model.layers.3.mlp.experts.159.down_proj", + "model.layers.3.mlp.experts.159.gate_proj", + "model.layers.3.mlp.experts.159.up_proj", + "model.layers.3.mlp.experts.16.down_proj", + "model.layers.3.mlp.experts.16.gate_proj", + "model.layers.3.mlp.experts.16.up_proj", + "model.layers.3.mlp.experts.160.down_proj", + "model.layers.3.mlp.experts.160.gate_proj", + "model.layers.3.mlp.experts.160.up_proj", + "model.layers.3.mlp.experts.161.down_proj", + "model.layers.3.mlp.experts.161.gate_proj", + "model.layers.3.mlp.experts.161.up_proj", + "model.layers.3.mlp.experts.162.down_proj", + "model.layers.3.mlp.experts.162.gate_proj", + "model.layers.3.mlp.experts.162.up_proj", + "model.layers.3.mlp.experts.163.down_proj", + "model.layers.3.mlp.experts.163.gate_proj", + "model.layers.3.mlp.experts.163.up_proj", + "model.layers.3.mlp.experts.164.down_proj", + "model.layers.3.mlp.experts.164.gate_proj", + "model.layers.3.mlp.experts.164.up_proj", + "model.layers.3.mlp.experts.165.down_proj", + "model.layers.3.mlp.experts.165.gate_proj", + "model.layers.3.mlp.experts.165.up_proj", + "model.layers.3.mlp.experts.166.down_proj", + "model.layers.3.mlp.experts.166.gate_proj", + "model.layers.3.mlp.experts.166.up_proj", + "model.layers.3.mlp.experts.167.down_proj", + "model.layers.3.mlp.experts.167.gate_proj", + "model.layers.3.mlp.experts.167.up_proj", + "model.layers.3.mlp.experts.168.down_proj", + "model.layers.3.mlp.experts.168.gate_proj", + "model.layers.3.mlp.experts.168.up_proj", + "model.layers.3.mlp.experts.169.down_proj", + "model.layers.3.mlp.experts.169.gate_proj", + "model.layers.3.mlp.experts.169.up_proj", + "model.layers.3.mlp.experts.17.down_proj", + "model.layers.3.mlp.experts.17.gate_proj", + "model.layers.3.mlp.experts.17.up_proj", + "model.layers.3.mlp.experts.170.down_proj", + "model.layers.3.mlp.experts.170.gate_proj", + "model.layers.3.mlp.experts.170.up_proj", + "model.layers.3.mlp.experts.171.down_proj", + "model.layers.3.mlp.experts.171.gate_proj", + "model.layers.3.mlp.experts.171.up_proj", + "model.layers.3.mlp.experts.172.down_proj", + "model.layers.3.mlp.experts.172.gate_proj", + "model.layers.3.mlp.experts.172.up_proj", + "model.layers.3.mlp.experts.173.down_proj", + "model.layers.3.mlp.experts.173.gate_proj", + "model.layers.3.mlp.experts.173.up_proj", + "model.layers.3.mlp.experts.174.down_proj", + "model.layers.3.mlp.experts.174.gate_proj", + "model.layers.3.mlp.experts.174.up_proj", + "model.layers.3.mlp.experts.175.down_proj", + "model.layers.3.mlp.experts.175.gate_proj", + "model.layers.3.mlp.experts.175.up_proj", + "model.layers.3.mlp.experts.176.down_proj", + "model.layers.3.mlp.experts.176.gate_proj", + "model.layers.3.mlp.experts.176.up_proj", + "model.layers.3.mlp.experts.177.down_proj", + "model.layers.3.mlp.experts.177.gate_proj", + "model.layers.3.mlp.experts.177.up_proj", + "model.layers.3.mlp.experts.178.down_proj", + "model.layers.3.mlp.experts.178.gate_proj", + "model.layers.3.mlp.experts.178.up_proj", + "model.layers.3.mlp.experts.179.down_proj", + "model.layers.3.mlp.experts.179.gate_proj", + "model.layers.3.mlp.experts.179.up_proj", + "model.layers.3.mlp.experts.18.down_proj", + "model.layers.3.mlp.experts.18.gate_proj", + "model.layers.3.mlp.experts.18.up_proj", + "model.layers.3.mlp.experts.180.down_proj", + "model.layers.3.mlp.experts.180.gate_proj", + "model.layers.3.mlp.experts.180.up_proj", + "model.layers.3.mlp.experts.181.down_proj", + "model.layers.3.mlp.experts.181.gate_proj", + "model.layers.3.mlp.experts.181.up_proj", + "model.layers.3.mlp.experts.182.down_proj", + "model.layers.3.mlp.experts.182.gate_proj", + "model.layers.3.mlp.experts.182.up_proj", + "model.layers.3.mlp.experts.183.down_proj", + "model.layers.3.mlp.experts.183.gate_proj", + "model.layers.3.mlp.experts.183.up_proj", + "model.layers.3.mlp.experts.184.down_proj", + "model.layers.3.mlp.experts.184.gate_proj", + "model.layers.3.mlp.experts.184.up_proj", + "model.layers.3.mlp.experts.185.down_proj", + "model.layers.3.mlp.experts.185.gate_proj", + "model.layers.3.mlp.experts.185.up_proj", + "model.layers.3.mlp.experts.186.down_proj", + "model.layers.3.mlp.experts.186.gate_proj", + "model.layers.3.mlp.experts.186.up_proj", + "model.layers.3.mlp.experts.187.down_proj", + "model.layers.3.mlp.experts.187.gate_proj", + "model.layers.3.mlp.experts.187.up_proj", + "model.layers.3.mlp.experts.188.down_proj", + "model.layers.3.mlp.experts.188.gate_proj", + "model.layers.3.mlp.experts.188.up_proj", + "model.layers.3.mlp.experts.189.down_proj", + "model.layers.3.mlp.experts.189.gate_proj", + "model.layers.3.mlp.experts.189.up_proj", + "model.layers.3.mlp.experts.19.down_proj", + "model.layers.3.mlp.experts.19.gate_proj", + "model.layers.3.mlp.experts.19.up_proj", + "model.layers.3.mlp.experts.190.down_proj", + "model.layers.3.mlp.experts.190.gate_proj", + "model.layers.3.mlp.experts.190.up_proj", + "model.layers.3.mlp.experts.191.down_proj", + "model.layers.3.mlp.experts.191.gate_proj", + "model.layers.3.mlp.experts.191.up_proj", + "model.layers.3.mlp.experts.192.down_proj", + "model.layers.3.mlp.experts.192.gate_proj", + "model.layers.3.mlp.experts.192.up_proj", + "model.layers.3.mlp.experts.193.down_proj", + "model.layers.3.mlp.experts.193.gate_proj", + "model.layers.3.mlp.experts.193.up_proj", + "model.layers.3.mlp.experts.194.down_proj", + "model.layers.3.mlp.experts.194.gate_proj", + "model.layers.3.mlp.experts.194.up_proj", + "model.layers.3.mlp.experts.195.down_proj", + "model.layers.3.mlp.experts.195.gate_proj", + "model.layers.3.mlp.experts.195.up_proj", + "model.layers.3.mlp.experts.196.down_proj", + "model.layers.3.mlp.experts.196.gate_proj", + "model.layers.3.mlp.experts.196.up_proj", + "model.layers.3.mlp.experts.197.down_proj", + "model.layers.3.mlp.experts.197.gate_proj", + "model.layers.3.mlp.experts.197.up_proj", + "model.layers.3.mlp.experts.198.down_proj", + "model.layers.3.mlp.experts.198.gate_proj", + "model.layers.3.mlp.experts.198.up_proj", + "model.layers.3.mlp.experts.199.down_proj", + "model.layers.3.mlp.experts.199.gate_proj", + "model.layers.3.mlp.experts.199.up_proj", + "model.layers.3.mlp.experts.2.down_proj", + "model.layers.3.mlp.experts.2.gate_proj", + "model.layers.3.mlp.experts.2.up_proj", + "model.layers.3.mlp.experts.20.down_proj", + "model.layers.3.mlp.experts.20.gate_proj", + "model.layers.3.mlp.experts.20.up_proj", + "model.layers.3.mlp.experts.200.down_proj", + "model.layers.3.mlp.experts.200.gate_proj", + "model.layers.3.mlp.experts.200.up_proj", + "model.layers.3.mlp.experts.201.down_proj", + "model.layers.3.mlp.experts.201.gate_proj", + "model.layers.3.mlp.experts.201.up_proj", + "model.layers.3.mlp.experts.202.down_proj", + "model.layers.3.mlp.experts.202.gate_proj", + "model.layers.3.mlp.experts.202.up_proj", + "model.layers.3.mlp.experts.203.down_proj", + "model.layers.3.mlp.experts.203.gate_proj", + "model.layers.3.mlp.experts.203.up_proj", + "model.layers.3.mlp.experts.204.down_proj", + "model.layers.3.mlp.experts.204.gate_proj", + "model.layers.3.mlp.experts.204.up_proj", + "model.layers.3.mlp.experts.205.down_proj", + "model.layers.3.mlp.experts.205.gate_proj", + "model.layers.3.mlp.experts.205.up_proj", + "model.layers.3.mlp.experts.206.down_proj", + "model.layers.3.mlp.experts.206.gate_proj", + "model.layers.3.mlp.experts.206.up_proj", + "model.layers.3.mlp.experts.207.down_proj", + "model.layers.3.mlp.experts.207.gate_proj", + "model.layers.3.mlp.experts.207.up_proj", + "model.layers.3.mlp.experts.208.down_proj", + "model.layers.3.mlp.experts.208.gate_proj", + "model.layers.3.mlp.experts.208.up_proj", + "model.layers.3.mlp.experts.209.down_proj", + "model.layers.3.mlp.experts.209.gate_proj", + "model.layers.3.mlp.experts.209.up_proj", + "model.layers.3.mlp.experts.21.down_proj", + "model.layers.3.mlp.experts.21.gate_proj", + "model.layers.3.mlp.experts.21.up_proj", + "model.layers.3.mlp.experts.210.down_proj", + "model.layers.3.mlp.experts.210.gate_proj", + "model.layers.3.mlp.experts.210.up_proj", + "model.layers.3.mlp.experts.211.down_proj", + "model.layers.3.mlp.experts.211.gate_proj", + "model.layers.3.mlp.experts.211.up_proj", + "model.layers.3.mlp.experts.212.down_proj", + "model.layers.3.mlp.experts.212.gate_proj", + "model.layers.3.mlp.experts.212.up_proj", + "model.layers.3.mlp.experts.213.down_proj", + "model.layers.3.mlp.experts.213.gate_proj", + "model.layers.3.mlp.experts.213.up_proj", + "model.layers.3.mlp.experts.214.down_proj", + "model.layers.3.mlp.experts.214.gate_proj", + "model.layers.3.mlp.experts.214.up_proj", + "model.layers.3.mlp.experts.215.down_proj", + "model.layers.3.mlp.experts.215.gate_proj", + "model.layers.3.mlp.experts.215.up_proj", + "model.layers.3.mlp.experts.216.down_proj", + "model.layers.3.mlp.experts.216.gate_proj", + "model.layers.3.mlp.experts.216.up_proj", + "model.layers.3.mlp.experts.217.down_proj", + "model.layers.3.mlp.experts.217.gate_proj", + "model.layers.3.mlp.experts.217.up_proj", + "model.layers.3.mlp.experts.218.down_proj", + "model.layers.3.mlp.experts.218.gate_proj", + "model.layers.3.mlp.experts.218.up_proj", + "model.layers.3.mlp.experts.219.down_proj", + "model.layers.3.mlp.experts.219.gate_proj", + "model.layers.3.mlp.experts.219.up_proj", + "model.layers.3.mlp.experts.22.down_proj", + "model.layers.3.mlp.experts.22.gate_proj", + "model.layers.3.mlp.experts.22.up_proj", + "model.layers.3.mlp.experts.220.down_proj", + "model.layers.3.mlp.experts.220.gate_proj", + "model.layers.3.mlp.experts.220.up_proj", + "model.layers.3.mlp.experts.221.down_proj", + "model.layers.3.mlp.experts.221.gate_proj", + "model.layers.3.mlp.experts.221.up_proj", + "model.layers.3.mlp.experts.222.down_proj", + "model.layers.3.mlp.experts.222.gate_proj", + "model.layers.3.mlp.experts.222.up_proj", + "model.layers.3.mlp.experts.223.down_proj", + "model.layers.3.mlp.experts.223.gate_proj", + "model.layers.3.mlp.experts.223.up_proj", + "model.layers.3.mlp.experts.224.down_proj", + "model.layers.3.mlp.experts.224.gate_proj", + "model.layers.3.mlp.experts.224.up_proj", + "model.layers.3.mlp.experts.225.down_proj", + "model.layers.3.mlp.experts.225.gate_proj", + "model.layers.3.mlp.experts.225.up_proj", + "model.layers.3.mlp.experts.226.down_proj", + "model.layers.3.mlp.experts.226.gate_proj", + "model.layers.3.mlp.experts.226.up_proj", + "model.layers.3.mlp.experts.227.down_proj", + "model.layers.3.mlp.experts.227.gate_proj", + "model.layers.3.mlp.experts.227.up_proj", + "model.layers.3.mlp.experts.228.down_proj", + "model.layers.3.mlp.experts.228.gate_proj", + "model.layers.3.mlp.experts.228.up_proj", + "model.layers.3.mlp.experts.229.down_proj", + "model.layers.3.mlp.experts.229.gate_proj", + "model.layers.3.mlp.experts.229.up_proj", + "model.layers.3.mlp.experts.23.down_proj", + "model.layers.3.mlp.experts.23.gate_proj", + "model.layers.3.mlp.experts.23.up_proj", + "model.layers.3.mlp.experts.230.down_proj", + "model.layers.3.mlp.experts.230.gate_proj", + "model.layers.3.mlp.experts.230.up_proj", + "model.layers.3.mlp.experts.231.down_proj", + "model.layers.3.mlp.experts.231.gate_proj", + "model.layers.3.mlp.experts.231.up_proj", + "model.layers.3.mlp.experts.232.down_proj", + "model.layers.3.mlp.experts.232.gate_proj", + "model.layers.3.mlp.experts.232.up_proj", + "model.layers.3.mlp.experts.233.down_proj", + "model.layers.3.mlp.experts.233.gate_proj", + "model.layers.3.mlp.experts.233.up_proj", + "model.layers.3.mlp.experts.234.down_proj", + "model.layers.3.mlp.experts.234.gate_proj", + "model.layers.3.mlp.experts.234.up_proj", + "model.layers.3.mlp.experts.235.down_proj", + "model.layers.3.mlp.experts.235.gate_proj", + "model.layers.3.mlp.experts.235.up_proj", + "model.layers.3.mlp.experts.236.down_proj", + "model.layers.3.mlp.experts.236.gate_proj", + "model.layers.3.mlp.experts.236.up_proj", + "model.layers.3.mlp.experts.237.down_proj", + "model.layers.3.mlp.experts.237.gate_proj", + "model.layers.3.mlp.experts.237.up_proj", + "model.layers.3.mlp.experts.238.down_proj", + "model.layers.3.mlp.experts.238.gate_proj", + "model.layers.3.mlp.experts.238.up_proj", + "model.layers.3.mlp.experts.239.down_proj", + "model.layers.3.mlp.experts.239.gate_proj", + "model.layers.3.mlp.experts.239.up_proj", + "model.layers.3.mlp.experts.24.down_proj", + "model.layers.3.mlp.experts.24.gate_proj", + "model.layers.3.mlp.experts.24.up_proj", + "model.layers.3.mlp.experts.240.down_proj", + "model.layers.3.mlp.experts.240.gate_proj", + "model.layers.3.mlp.experts.240.up_proj", + "model.layers.3.mlp.experts.241.down_proj", + "model.layers.3.mlp.experts.241.gate_proj", + "model.layers.3.mlp.experts.241.up_proj", + "model.layers.3.mlp.experts.242.down_proj", + "model.layers.3.mlp.experts.242.gate_proj", + "model.layers.3.mlp.experts.242.up_proj", + "model.layers.3.mlp.experts.243.down_proj", + "model.layers.3.mlp.experts.243.gate_proj", + "model.layers.3.mlp.experts.243.up_proj", + "model.layers.3.mlp.experts.244.down_proj", + "model.layers.3.mlp.experts.244.gate_proj", + "model.layers.3.mlp.experts.244.up_proj", + "model.layers.3.mlp.experts.245.down_proj", + "model.layers.3.mlp.experts.245.gate_proj", + "model.layers.3.mlp.experts.245.up_proj", + "model.layers.3.mlp.experts.246.down_proj", + "model.layers.3.mlp.experts.246.gate_proj", + "model.layers.3.mlp.experts.246.up_proj", + "model.layers.3.mlp.experts.247.down_proj", + "model.layers.3.mlp.experts.247.gate_proj", + "model.layers.3.mlp.experts.247.up_proj", + "model.layers.3.mlp.experts.248.down_proj", + "model.layers.3.mlp.experts.248.gate_proj", + "model.layers.3.mlp.experts.248.up_proj", + "model.layers.3.mlp.experts.249.down_proj", + "model.layers.3.mlp.experts.249.gate_proj", + "model.layers.3.mlp.experts.249.up_proj", + "model.layers.3.mlp.experts.25.down_proj", + "model.layers.3.mlp.experts.25.gate_proj", + "model.layers.3.mlp.experts.25.up_proj", + "model.layers.3.mlp.experts.250.down_proj", + "model.layers.3.mlp.experts.250.gate_proj", + "model.layers.3.mlp.experts.250.up_proj", + "model.layers.3.mlp.experts.251.down_proj", + "model.layers.3.mlp.experts.251.gate_proj", + "model.layers.3.mlp.experts.251.up_proj", + "model.layers.3.mlp.experts.252.down_proj", + "model.layers.3.mlp.experts.252.gate_proj", + "model.layers.3.mlp.experts.252.up_proj", + "model.layers.3.mlp.experts.253.down_proj", + "model.layers.3.mlp.experts.253.gate_proj", + "model.layers.3.mlp.experts.253.up_proj", + "model.layers.3.mlp.experts.254.down_proj", + "model.layers.3.mlp.experts.254.gate_proj", + "model.layers.3.mlp.experts.254.up_proj", + "model.layers.3.mlp.experts.255.down_proj", + "model.layers.3.mlp.experts.255.gate_proj", + "model.layers.3.mlp.experts.255.up_proj", + "model.layers.3.mlp.experts.26.down_proj", + "model.layers.3.mlp.experts.26.gate_proj", + "model.layers.3.mlp.experts.26.up_proj", + "model.layers.3.mlp.experts.27.down_proj", + "model.layers.3.mlp.experts.27.gate_proj", + "model.layers.3.mlp.experts.27.up_proj", + "model.layers.3.mlp.experts.28.down_proj", + "model.layers.3.mlp.experts.28.gate_proj", + "model.layers.3.mlp.experts.28.up_proj", + "model.layers.3.mlp.experts.29.down_proj", + "model.layers.3.mlp.experts.29.gate_proj", + "model.layers.3.mlp.experts.29.up_proj", + "model.layers.3.mlp.experts.3.down_proj", + "model.layers.3.mlp.experts.3.gate_proj", + "model.layers.3.mlp.experts.3.up_proj", + "model.layers.3.mlp.experts.30.down_proj", + "model.layers.3.mlp.experts.30.gate_proj", + "model.layers.3.mlp.experts.30.up_proj", + "model.layers.3.mlp.experts.31.down_proj", + "model.layers.3.mlp.experts.31.gate_proj", + "model.layers.3.mlp.experts.31.up_proj", + "model.layers.3.mlp.experts.32.down_proj", + "model.layers.3.mlp.experts.32.gate_proj", + "model.layers.3.mlp.experts.32.up_proj", + "model.layers.3.mlp.experts.33.down_proj", + "model.layers.3.mlp.experts.33.gate_proj", + "model.layers.3.mlp.experts.33.up_proj", + "model.layers.3.mlp.experts.34.down_proj", + "model.layers.3.mlp.experts.34.gate_proj", + "model.layers.3.mlp.experts.34.up_proj", + "model.layers.3.mlp.experts.35.down_proj", + "model.layers.3.mlp.experts.35.gate_proj", + "model.layers.3.mlp.experts.35.up_proj", + "model.layers.3.mlp.experts.36.down_proj", + "model.layers.3.mlp.experts.36.gate_proj", + "model.layers.3.mlp.experts.36.up_proj", + "model.layers.3.mlp.experts.37.down_proj", + "model.layers.3.mlp.experts.37.gate_proj", + "model.layers.3.mlp.experts.37.up_proj", + "model.layers.3.mlp.experts.38.down_proj", + "model.layers.3.mlp.experts.38.gate_proj", + "model.layers.3.mlp.experts.38.up_proj", + "model.layers.3.mlp.experts.39.down_proj", + "model.layers.3.mlp.experts.39.gate_proj", + "model.layers.3.mlp.experts.39.up_proj", + "model.layers.3.mlp.experts.4.down_proj", + "model.layers.3.mlp.experts.4.gate_proj", + "model.layers.3.mlp.experts.4.up_proj", + "model.layers.3.mlp.experts.40.down_proj", + "model.layers.3.mlp.experts.40.gate_proj", + "model.layers.3.mlp.experts.40.up_proj", + "model.layers.3.mlp.experts.41.down_proj", + "model.layers.3.mlp.experts.41.gate_proj", + "model.layers.3.mlp.experts.41.up_proj", + "model.layers.3.mlp.experts.42.down_proj", + "model.layers.3.mlp.experts.42.gate_proj", + "model.layers.3.mlp.experts.42.up_proj", + "model.layers.3.mlp.experts.43.down_proj", + "model.layers.3.mlp.experts.43.gate_proj", + "model.layers.3.mlp.experts.43.up_proj", + "model.layers.3.mlp.experts.44.down_proj", + "model.layers.3.mlp.experts.44.gate_proj", + "model.layers.3.mlp.experts.44.up_proj", + "model.layers.3.mlp.experts.45.down_proj", + "model.layers.3.mlp.experts.45.gate_proj", + "model.layers.3.mlp.experts.45.up_proj", + "model.layers.3.mlp.experts.46.down_proj", + "model.layers.3.mlp.experts.46.gate_proj", + "model.layers.3.mlp.experts.46.up_proj", + "model.layers.3.mlp.experts.47.down_proj", + "model.layers.3.mlp.experts.47.gate_proj", + "model.layers.3.mlp.experts.47.up_proj", + "model.layers.3.mlp.experts.48.down_proj", + "model.layers.3.mlp.experts.48.gate_proj", + "model.layers.3.mlp.experts.48.up_proj", + "model.layers.3.mlp.experts.49.down_proj", + "model.layers.3.mlp.experts.49.gate_proj", + "model.layers.3.mlp.experts.49.up_proj", + "model.layers.3.mlp.experts.5.down_proj", + "model.layers.3.mlp.experts.5.gate_proj", + "model.layers.3.mlp.experts.5.up_proj", + "model.layers.3.mlp.experts.50.down_proj", + "model.layers.3.mlp.experts.50.gate_proj", + "model.layers.3.mlp.experts.50.up_proj", + "model.layers.3.mlp.experts.51.down_proj", + "model.layers.3.mlp.experts.51.gate_proj", + "model.layers.3.mlp.experts.51.up_proj", + "model.layers.3.mlp.experts.52.down_proj", + "model.layers.3.mlp.experts.52.gate_proj", + "model.layers.3.mlp.experts.52.up_proj", + "model.layers.3.mlp.experts.53.down_proj", + "model.layers.3.mlp.experts.53.gate_proj", + "model.layers.3.mlp.experts.53.up_proj", + "model.layers.3.mlp.experts.54.down_proj", + "model.layers.3.mlp.experts.54.gate_proj", + "model.layers.3.mlp.experts.54.up_proj", + "model.layers.3.mlp.experts.55.down_proj", + "model.layers.3.mlp.experts.55.gate_proj", + "model.layers.3.mlp.experts.55.up_proj", + "model.layers.3.mlp.experts.56.down_proj", + "model.layers.3.mlp.experts.56.gate_proj", + "model.layers.3.mlp.experts.56.up_proj", + "model.layers.3.mlp.experts.57.down_proj", + "model.layers.3.mlp.experts.57.gate_proj", + "model.layers.3.mlp.experts.57.up_proj", + "model.layers.3.mlp.experts.58.down_proj", + "model.layers.3.mlp.experts.58.gate_proj", + "model.layers.3.mlp.experts.58.up_proj", + "model.layers.3.mlp.experts.59.down_proj", + "model.layers.3.mlp.experts.59.gate_proj", + "model.layers.3.mlp.experts.59.up_proj", + "model.layers.3.mlp.experts.6.down_proj", + "model.layers.3.mlp.experts.6.gate_proj", + "model.layers.3.mlp.experts.6.up_proj", + "model.layers.3.mlp.experts.60.down_proj", + "model.layers.3.mlp.experts.60.gate_proj", + "model.layers.3.mlp.experts.60.up_proj", + "model.layers.3.mlp.experts.61.down_proj", + "model.layers.3.mlp.experts.61.gate_proj", + "model.layers.3.mlp.experts.61.up_proj", + "model.layers.3.mlp.experts.62.down_proj", + "model.layers.3.mlp.experts.62.gate_proj", + "model.layers.3.mlp.experts.62.up_proj", + "model.layers.3.mlp.experts.63.down_proj", + "model.layers.3.mlp.experts.63.gate_proj", + "model.layers.3.mlp.experts.63.up_proj", + "model.layers.3.mlp.experts.64.down_proj", + "model.layers.3.mlp.experts.64.gate_proj", + "model.layers.3.mlp.experts.64.up_proj", + "model.layers.3.mlp.experts.65.down_proj", + "model.layers.3.mlp.experts.65.gate_proj", + "model.layers.3.mlp.experts.65.up_proj", + "model.layers.3.mlp.experts.66.down_proj", + "model.layers.3.mlp.experts.66.gate_proj", + "model.layers.3.mlp.experts.66.up_proj", + "model.layers.3.mlp.experts.67.down_proj", + "model.layers.3.mlp.experts.67.gate_proj", + "model.layers.3.mlp.experts.67.up_proj", + "model.layers.3.mlp.experts.68.down_proj", + "model.layers.3.mlp.experts.68.gate_proj", + "model.layers.3.mlp.experts.68.up_proj", + "model.layers.3.mlp.experts.69.down_proj", + "model.layers.3.mlp.experts.69.gate_proj", + "model.layers.3.mlp.experts.69.up_proj", + "model.layers.3.mlp.experts.7.down_proj", + "model.layers.3.mlp.experts.7.gate_proj", + "model.layers.3.mlp.experts.7.up_proj", + "model.layers.3.mlp.experts.70.down_proj", + "model.layers.3.mlp.experts.70.gate_proj", + "model.layers.3.mlp.experts.70.up_proj", + "model.layers.3.mlp.experts.71.down_proj", + "model.layers.3.mlp.experts.71.gate_proj", + "model.layers.3.mlp.experts.71.up_proj", + "model.layers.3.mlp.experts.72.down_proj", + "model.layers.3.mlp.experts.72.gate_proj", + "model.layers.3.mlp.experts.72.up_proj", + "model.layers.3.mlp.experts.73.down_proj", + "model.layers.3.mlp.experts.73.gate_proj", + "model.layers.3.mlp.experts.73.up_proj", + "model.layers.3.mlp.experts.74.down_proj", + "model.layers.3.mlp.experts.74.gate_proj", + "model.layers.3.mlp.experts.74.up_proj", + "model.layers.3.mlp.experts.75.down_proj", + "model.layers.3.mlp.experts.75.gate_proj", + "model.layers.3.mlp.experts.75.up_proj", + "model.layers.3.mlp.experts.76.down_proj", + "model.layers.3.mlp.experts.76.gate_proj", + "model.layers.3.mlp.experts.76.up_proj", + "model.layers.3.mlp.experts.77.down_proj", + "model.layers.3.mlp.experts.77.gate_proj", + "model.layers.3.mlp.experts.77.up_proj", + "model.layers.3.mlp.experts.78.down_proj", + "model.layers.3.mlp.experts.78.gate_proj", + "model.layers.3.mlp.experts.78.up_proj", + "model.layers.3.mlp.experts.79.down_proj", + "model.layers.3.mlp.experts.79.gate_proj", + "model.layers.3.mlp.experts.79.up_proj", + "model.layers.3.mlp.experts.8.down_proj", + "model.layers.3.mlp.experts.8.gate_proj", + "model.layers.3.mlp.experts.8.up_proj", + "model.layers.3.mlp.experts.80.down_proj", + "model.layers.3.mlp.experts.80.gate_proj", + "model.layers.3.mlp.experts.80.up_proj", + "model.layers.3.mlp.experts.81.down_proj", + "model.layers.3.mlp.experts.81.gate_proj", + "model.layers.3.mlp.experts.81.up_proj", + "model.layers.3.mlp.experts.82.down_proj", + "model.layers.3.mlp.experts.82.gate_proj", + "model.layers.3.mlp.experts.82.up_proj", + "model.layers.3.mlp.experts.83.down_proj", + "model.layers.3.mlp.experts.83.gate_proj", + "model.layers.3.mlp.experts.83.up_proj", + "model.layers.3.mlp.experts.84.down_proj", + "model.layers.3.mlp.experts.84.gate_proj", + "model.layers.3.mlp.experts.84.up_proj", + "model.layers.3.mlp.experts.85.down_proj", + "model.layers.3.mlp.experts.85.gate_proj", + "model.layers.3.mlp.experts.85.up_proj", + "model.layers.3.mlp.experts.86.down_proj", + "model.layers.3.mlp.experts.86.gate_proj", + "model.layers.3.mlp.experts.86.up_proj", + "model.layers.3.mlp.experts.87.down_proj", + "model.layers.3.mlp.experts.87.gate_proj", + "model.layers.3.mlp.experts.87.up_proj", + "model.layers.3.mlp.experts.88.down_proj", + "model.layers.3.mlp.experts.88.gate_proj", + "model.layers.3.mlp.experts.88.up_proj", + "model.layers.3.mlp.experts.89.down_proj", + "model.layers.3.mlp.experts.89.gate_proj", + "model.layers.3.mlp.experts.89.up_proj", + "model.layers.3.mlp.experts.9.down_proj", + "model.layers.3.mlp.experts.9.gate_proj", + "model.layers.3.mlp.experts.9.up_proj", + "model.layers.3.mlp.experts.90.down_proj", + "model.layers.3.mlp.experts.90.gate_proj", + "model.layers.3.mlp.experts.90.up_proj", + "model.layers.3.mlp.experts.91.down_proj", + "model.layers.3.mlp.experts.91.gate_proj", + "model.layers.3.mlp.experts.91.up_proj", + "model.layers.3.mlp.experts.92.down_proj", + "model.layers.3.mlp.experts.92.gate_proj", + "model.layers.3.mlp.experts.92.up_proj", + "model.layers.3.mlp.experts.93.down_proj", + "model.layers.3.mlp.experts.93.gate_proj", + "model.layers.3.mlp.experts.93.up_proj", + "model.layers.3.mlp.experts.94.down_proj", + "model.layers.3.mlp.experts.94.gate_proj", + "model.layers.3.mlp.experts.94.up_proj", + "model.layers.3.mlp.experts.95.down_proj", + "model.layers.3.mlp.experts.95.gate_proj", + "model.layers.3.mlp.experts.95.up_proj", + "model.layers.3.mlp.experts.96.down_proj", + "model.layers.3.mlp.experts.96.gate_proj", + "model.layers.3.mlp.experts.96.up_proj", + "model.layers.3.mlp.experts.97.down_proj", + "model.layers.3.mlp.experts.97.gate_proj", + "model.layers.3.mlp.experts.97.up_proj", + "model.layers.3.mlp.experts.98.down_proj", + "model.layers.3.mlp.experts.98.gate_proj", + "model.layers.3.mlp.experts.98.up_proj", + "model.layers.3.mlp.experts.99.down_proj", + "model.layers.3.mlp.experts.99.gate_proj", + "model.layers.3.mlp.experts.99.up_proj", + "model.layers.30.mlp.experts.0.down_proj", + "model.layers.30.mlp.experts.0.gate_proj", + "model.layers.30.mlp.experts.0.up_proj", + "model.layers.30.mlp.experts.1.down_proj", + "model.layers.30.mlp.experts.1.gate_proj", + "model.layers.30.mlp.experts.1.up_proj", + "model.layers.30.mlp.experts.10.down_proj", + "model.layers.30.mlp.experts.10.gate_proj", + "model.layers.30.mlp.experts.10.up_proj", + "model.layers.30.mlp.experts.100.down_proj", + "model.layers.30.mlp.experts.100.gate_proj", + "model.layers.30.mlp.experts.100.up_proj", + "model.layers.30.mlp.experts.101.down_proj", + "model.layers.30.mlp.experts.101.gate_proj", + "model.layers.30.mlp.experts.101.up_proj", + "model.layers.30.mlp.experts.102.down_proj", + "model.layers.30.mlp.experts.102.gate_proj", + "model.layers.30.mlp.experts.102.up_proj", + "model.layers.30.mlp.experts.103.down_proj", + "model.layers.30.mlp.experts.103.gate_proj", + "model.layers.30.mlp.experts.103.up_proj", + "model.layers.30.mlp.experts.104.down_proj", + "model.layers.30.mlp.experts.104.gate_proj", + "model.layers.30.mlp.experts.104.up_proj", + "model.layers.30.mlp.experts.105.down_proj", + "model.layers.30.mlp.experts.105.gate_proj", + "model.layers.30.mlp.experts.105.up_proj", + "model.layers.30.mlp.experts.106.down_proj", + "model.layers.30.mlp.experts.106.gate_proj", + "model.layers.30.mlp.experts.106.up_proj", + "model.layers.30.mlp.experts.107.down_proj", + "model.layers.30.mlp.experts.107.gate_proj", + "model.layers.30.mlp.experts.107.up_proj", + "model.layers.30.mlp.experts.108.down_proj", + "model.layers.30.mlp.experts.108.gate_proj", + "model.layers.30.mlp.experts.108.up_proj", + "model.layers.30.mlp.experts.109.down_proj", + "model.layers.30.mlp.experts.109.gate_proj", + "model.layers.30.mlp.experts.109.up_proj", + "model.layers.30.mlp.experts.11.down_proj", + "model.layers.30.mlp.experts.11.gate_proj", + "model.layers.30.mlp.experts.11.up_proj", + "model.layers.30.mlp.experts.110.down_proj", + "model.layers.30.mlp.experts.110.gate_proj", + "model.layers.30.mlp.experts.110.up_proj", + "model.layers.30.mlp.experts.111.down_proj", + "model.layers.30.mlp.experts.111.gate_proj", + "model.layers.30.mlp.experts.111.up_proj", + "model.layers.30.mlp.experts.112.down_proj", + "model.layers.30.mlp.experts.112.gate_proj", + "model.layers.30.mlp.experts.112.up_proj", + "model.layers.30.mlp.experts.113.down_proj", + "model.layers.30.mlp.experts.113.gate_proj", + "model.layers.30.mlp.experts.113.up_proj", + "model.layers.30.mlp.experts.114.down_proj", + "model.layers.30.mlp.experts.114.gate_proj", + "model.layers.30.mlp.experts.114.up_proj", + "model.layers.30.mlp.experts.115.down_proj", + "model.layers.30.mlp.experts.115.gate_proj", + "model.layers.30.mlp.experts.115.up_proj", + "model.layers.30.mlp.experts.116.down_proj", + "model.layers.30.mlp.experts.116.gate_proj", + "model.layers.30.mlp.experts.116.up_proj", + "model.layers.30.mlp.experts.117.down_proj", + "model.layers.30.mlp.experts.117.gate_proj", + "model.layers.30.mlp.experts.117.up_proj", + "model.layers.30.mlp.experts.118.down_proj", + "model.layers.30.mlp.experts.118.gate_proj", + "model.layers.30.mlp.experts.118.up_proj", + "model.layers.30.mlp.experts.119.down_proj", + "model.layers.30.mlp.experts.119.gate_proj", + "model.layers.30.mlp.experts.119.up_proj", + "model.layers.30.mlp.experts.12.down_proj", + "model.layers.30.mlp.experts.12.gate_proj", + "model.layers.30.mlp.experts.12.up_proj", + "model.layers.30.mlp.experts.120.down_proj", + "model.layers.30.mlp.experts.120.gate_proj", + "model.layers.30.mlp.experts.120.up_proj", + "model.layers.30.mlp.experts.121.down_proj", + "model.layers.30.mlp.experts.121.gate_proj", + "model.layers.30.mlp.experts.121.up_proj", + "model.layers.30.mlp.experts.122.down_proj", + "model.layers.30.mlp.experts.122.gate_proj", + "model.layers.30.mlp.experts.122.up_proj", + "model.layers.30.mlp.experts.123.down_proj", + "model.layers.30.mlp.experts.123.gate_proj", + "model.layers.30.mlp.experts.123.up_proj", + "model.layers.30.mlp.experts.124.down_proj", + "model.layers.30.mlp.experts.124.gate_proj", + "model.layers.30.mlp.experts.124.up_proj", + "model.layers.30.mlp.experts.125.down_proj", + "model.layers.30.mlp.experts.125.gate_proj", + "model.layers.30.mlp.experts.125.up_proj", + "model.layers.30.mlp.experts.126.down_proj", + "model.layers.30.mlp.experts.126.gate_proj", + "model.layers.30.mlp.experts.126.up_proj", + "model.layers.30.mlp.experts.127.down_proj", + "model.layers.30.mlp.experts.127.gate_proj", + "model.layers.30.mlp.experts.127.up_proj", + "model.layers.30.mlp.experts.128.down_proj", + "model.layers.30.mlp.experts.128.gate_proj", + "model.layers.30.mlp.experts.128.up_proj", + "model.layers.30.mlp.experts.129.down_proj", + "model.layers.30.mlp.experts.129.gate_proj", + "model.layers.30.mlp.experts.129.up_proj", + "model.layers.30.mlp.experts.13.down_proj", + "model.layers.30.mlp.experts.13.gate_proj", + "model.layers.30.mlp.experts.13.up_proj", + "model.layers.30.mlp.experts.130.down_proj", + "model.layers.30.mlp.experts.130.gate_proj", + "model.layers.30.mlp.experts.130.up_proj", + "model.layers.30.mlp.experts.131.down_proj", + "model.layers.30.mlp.experts.131.gate_proj", + "model.layers.30.mlp.experts.131.up_proj", + "model.layers.30.mlp.experts.132.down_proj", + "model.layers.30.mlp.experts.132.gate_proj", + "model.layers.30.mlp.experts.132.up_proj", + "model.layers.30.mlp.experts.133.down_proj", + "model.layers.30.mlp.experts.133.gate_proj", + "model.layers.30.mlp.experts.133.up_proj", + "model.layers.30.mlp.experts.134.down_proj", + "model.layers.30.mlp.experts.134.gate_proj", + "model.layers.30.mlp.experts.134.up_proj", + "model.layers.30.mlp.experts.135.down_proj", + "model.layers.30.mlp.experts.135.gate_proj", + "model.layers.30.mlp.experts.135.up_proj", + "model.layers.30.mlp.experts.136.down_proj", + "model.layers.30.mlp.experts.136.gate_proj", + "model.layers.30.mlp.experts.136.up_proj", + "model.layers.30.mlp.experts.137.down_proj", + "model.layers.30.mlp.experts.137.gate_proj", + "model.layers.30.mlp.experts.137.up_proj", + "model.layers.30.mlp.experts.138.down_proj", + "model.layers.30.mlp.experts.138.gate_proj", + "model.layers.30.mlp.experts.138.up_proj", + "model.layers.30.mlp.experts.139.down_proj", + "model.layers.30.mlp.experts.139.gate_proj", + "model.layers.30.mlp.experts.139.up_proj", + "model.layers.30.mlp.experts.14.down_proj", + "model.layers.30.mlp.experts.14.gate_proj", + "model.layers.30.mlp.experts.14.up_proj", + "model.layers.30.mlp.experts.140.down_proj", + "model.layers.30.mlp.experts.140.gate_proj", + "model.layers.30.mlp.experts.140.up_proj", + "model.layers.30.mlp.experts.141.down_proj", + "model.layers.30.mlp.experts.141.gate_proj", + "model.layers.30.mlp.experts.141.up_proj", + "model.layers.30.mlp.experts.142.down_proj", + "model.layers.30.mlp.experts.142.gate_proj", + "model.layers.30.mlp.experts.142.up_proj", + "model.layers.30.mlp.experts.143.down_proj", + "model.layers.30.mlp.experts.143.gate_proj", + "model.layers.30.mlp.experts.143.up_proj", + "model.layers.30.mlp.experts.144.down_proj", + "model.layers.30.mlp.experts.144.gate_proj", + "model.layers.30.mlp.experts.144.up_proj", + "model.layers.30.mlp.experts.145.down_proj", + "model.layers.30.mlp.experts.145.gate_proj", + "model.layers.30.mlp.experts.145.up_proj", + "model.layers.30.mlp.experts.146.down_proj", + "model.layers.30.mlp.experts.146.gate_proj", + "model.layers.30.mlp.experts.146.up_proj", + "model.layers.30.mlp.experts.147.down_proj", + "model.layers.30.mlp.experts.147.gate_proj", + "model.layers.30.mlp.experts.147.up_proj", + "model.layers.30.mlp.experts.148.down_proj", + "model.layers.30.mlp.experts.148.gate_proj", + "model.layers.30.mlp.experts.148.up_proj", + "model.layers.30.mlp.experts.149.down_proj", + "model.layers.30.mlp.experts.149.gate_proj", + "model.layers.30.mlp.experts.149.up_proj", + "model.layers.30.mlp.experts.15.down_proj", + "model.layers.30.mlp.experts.15.gate_proj", + "model.layers.30.mlp.experts.15.up_proj", + "model.layers.30.mlp.experts.150.down_proj", + "model.layers.30.mlp.experts.150.gate_proj", + "model.layers.30.mlp.experts.150.up_proj", + "model.layers.30.mlp.experts.151.down_proj", + "model.layers.30.mlp.experts.151.gate_proj", + "model.layers.30.mlp.experts.151.up_proj", + "model.layers.30.mlp.experts.152.down_proj", + "model.layers.30.mlp.experts.152.gate_proj", + "model.layers.30.mlp.experts.152.up_proj", + "model.layers.30.mlp.experts.153.down_proj", + "model.layers.30.mlp.experts.153.gate_proj", + "model.layers.30.mlp.experts.153.up_proj", + "model.layers.30.mlp.experts.154.down_proj", + "model.layers.30.mlp.experts.154.gate_proj", + "model.layers.30.mlp.experts.154.up_proj", + "model.layers.30.mlp.experts.155.down_proj", + "model.layers.30.mlp.experts.155.gate_proj", + "model.layers.30.mlp.experts.155.up_proj", + "model.layers.30.mlp.experts.156.down_proj", + "model.layers.30.mlp.experts.156.gate_proj", + "model.layers.30.mlp.experts.156.up_proj", + "model.layers.30.mlp.experts.157.down_proj", + "model.layers.30.mlp.experts.157.gate_proj", + "model.layers.30.mlp.experts.157.up_proj", + "model.layers.30.mlp.experts.158.down_proj", + "model.layers.30.mlp.experts.158.gate_proj", + "model.layers.30.mlp.experts.158.up_proj", + "model.layers.30.mlp.experts.159.down_proj", + "model.layers.30.mlp.experts.159.gate_proj", + "model.layers.30.mlp.experts.159.up_proj", + "model.layers.30.mlp.experts.16.down_proj", + "model.layers.30.mlp.experts.16.gate_proj", + "model.layers.30.mlp.experts.16.up_proj", + "model.layers.30.mlp.experts.160.down_proj", + "model.layers.30.mlp.experts.160.gate_proj", + "model.layers.30.mlp.experts.160.up_proj", + "model.layers.30.mlp.experts.161.down_proj", + "model.layers.30.mlp.experts.161.gate_proj", + "model.layers.30.mlp.experts.161.up_proj", + "model.layers.30.mlp.experts.162.down_proj", + "model.layers.30.mlp.experts.162.gate_proj", + "model.layers.30.mlp.experts.162.up_proj", + "model.layers.30.mlp.experts.163.down_proj", + "model.layers.30.mlp.experts.163.gate_proj", + "model.layers.30.mlp.experts.163.up_proj", + "model.layers.30.mlp.experts.164.down_proj", + "model.layers.30.mlp.experts.164.gate_proj", + "model.layers.30.mlp.experts.164.up_proj", + "model.layers.30.mlp.experts.165.down_proj", + "model.layers.30.mlp.experts.165.gate_proj", + "model.layers.30.mlp.experts.165.up_proj", + "model.layers.30.mlp.experts.166.down_proj", + "model.layers.30.mlp.experts.166.gate_proj", + "model.layers.30.mlp.experts.166.up_proj", + "model.layers.30.mlp.experts.167.down_proj", + "model.layers.30.mlp.experts.167.gate_proj", + "model.layers.30.mlp.experts.167.up_proj", + "model.layers.30.mlp.experts.168.down_proj", + "model.layers.30.mlp.experts.168.gate_proj", + "model.layers.30.mlp.experts.168.up_proj", + "model.layers.30.mlp.experts.169.down_proj", + "model.layers.30.mlp.experts.169.gate_proj", + "model.layers.30.mlp.experts.169.up_proj", + "model.layers.30.mlp.experts.17.down_proj", + "model.layers.30.mlp.experts.17.gate_proj", + "model.layers.30.mlp.experts.17.up_proj", + "model.layers.30.mlp.experts.170.down_proj", + "model.layers.30.mlp.experts.170.gate_proj", + "model.layers.30.mlp.experts.170.up_proj", + "model.layers.30.mlp.experts.171.down_proj", + "model.layers.30.mlp.experts.171.gate_proj", + "model.layers.30.mlp.experts.171.up_proj", + "model.layers.30.mlp.experts.172.down_proj", + "model.layers.30.mlp.experts.172.gate_proj", + "model.layers.30.mlp.experts.172.up_proj", + "model.layers.30.mlp.experts.173.down_proj", + "model.layers.30.mlp.experts.173.gate_proj", + "model.layers.30.mlp.experts.173.up_proj", + "model.layers.30.mlp.experts.174.down_proj", + "model.layers.30.mlp.experts.174.gate_proj", + "model.layers.30.mlp.experts.174.up_proj", + "model.layers.30.mlp.experts.175.down_proj", + "model.layers.30.mlp.experts.175.gate_proj", + "model.layers.30.mlp.experts.175.up_proj", + "model.layers.30.mlp.experts.176.down_proj", + "model.layers.30.mlp.experts.176.gate_proj", + "model.layers.30.mlp.experts.176.up_proj", + "model.layers.30.mlp.experts.177.down_proj", + "model.layers.30.mlp.experts.177.gate_proj", + "model.layers.30.mlp.experts.177.up_proj", + "model.layers.30.mlp.experts.178.down_proj", + "model.layers.30.mlp.experts.178.gate_proj", + "model.layers.30.mlp.experts.178.up_proj", + "model.layers.30.mlp.experts.179.down_proj", + "model.layers.30.mlp.experts.179.gate_proj", + "model.layers.30.mlp.experts.179.up_proj", + "model.layers.30.mlp.experts.18.down_proj", + "model.layers.30.mlp.experts.18.gate_proj", + "model.layers.30.mlp.experts.18.up_proj", + "model.layers.30.mlp.experts.180.down_proj", + "model.layers.30.mlp.experts.180.gate_proj", + "model.layers.30.mlp.experts.180.up_proj", + "model.layers.30.mlp.experts.181.down_proj", + "model.layers.30.mlp.experts.181.gate_proj", + "model.layers.30.mlp.experts.181.up_proj", + "model.layers.30.mlp.experts.182.down_proj", + "model.layers.30.mlp.experts.182.gate_proj", + "model.layers.30.mlp.experts.182.up_proj", + "model.layers.30.mlp.experts.183.down_proj", + "model.layers.30.mlp.experts.183.gate_proj", + "model.layers.30.mlp.experts.183.up_proj", + "model.layers.30.mlp.experts.184.down_proj", + "model.layers.30.mlp.experts.184.gate_proj", + "model.layers.30.mlp.experts.184.up_proj", + "model.layers.30.mlp.experts.185.down_proj", + "model.layers.30.mlp.experts.185.gate_proj", + "model.layers.30.mlp.experts.185.up_proj", + "model.layers.30.mlp.experts.186.down_proj", + "model.layers.30.mlp.experts.186.gate_proj", + "model.layers.30.mlp.experts.186.up_proj", + "model.layers.30.mlp.experts.187.down_proj", + "model.layers.30.mlp.experts.187.gate_proj", + "model.layers.30.mlp.experts.187.up_proj", + "model.layers.30.mlp.experts.188.down_proj", + "model.layers.30.mlp.experts.188.gate_proj", + "model.layers.30.mlp.experts.188.up_proj", + "model.layers.30.mlp.experts.189.down_proj", + "model.layers.30.mlp.experts.189.gate_proj", + "model.layers.30.mlp.experts.189.up_proj", + "model.layers.30.mlp.experts.19.down_proj", + "model.layers.30.mlp.experts.19.gate_proj", + "model.layers.30.mlp.experts.19.up_proj", + "model.layers.30.mlp.experts.190.down_proj", + "model.layers.30.mlp.experts.190.gate_proj", + "model.layers.30.mlp.experts.190.up_proj", + "model.layers.30.mlp.experts.191.down_proj", + "model.layers.30.mlp.experts.191.gate_proj", + "model.layers.30.mlp.experts.191.up_proj", + "model.layers.30.mlp.experts.192.down_proj", + "model.layers.30.mlp.experts.192.gate_proj", + "model.layers.30.mlp.experts.192.up_proj", + "model.layers.30.mlp.experts.193.down_proj", + "model.layers.30.mlp.experts.193.gate_proj", + "model.layers.30.mlp.experts.193.up_proj", + "model.layers.30.mlp.experts.194.down_proj", + "model.layers.30.mlp.experts.194.gate_proj", + "model.layers.30.mlp.experts.194.up_proj", + "model.layers.30.mlp.experts.195.down_proj", + "model.layers.30.mlp.experts.195.gate_proj", + "model.layers.30.mlp.experts.195.up_proj", + "model.layers.30.mlp.experts.196.down_proj", + "model.layers.30.mlp.experts.196.gate_proj", + "model.layers.30.mlp.experts.196.up_proj", + "model.layers.30.mlp.experts.197.down_proj", + "model.layers.30.mlp.experts.197.gate_proj", + "model.layers.30.mlp.experts.197.up_proj", + "model.layers.30.mlp.experts.198.down_proj", + "model.layers.30.mlp.experts.198.gate_proj", + "model.layers.30.mlp.experts.198.up_proj", + "model.layers.30.mlp.experts.199.down_proj", + "model.layers.30.mlp.experts.199.gate_proj", + "model.layers.30.mlp.experts.199.up_proj", + "model.layers.30.mlp.experts.2.down_proj", + "model.layers.30.mlp.experts.2.gate_proj", + "model.layers.30.mlp.experts.2.up_proj", + "model.layers.30.mlp.experts.20.down_proj", + "model.layers.30.mlp.experts.20.gate_proj", + "model.layers.30.mlp.experts.20.up_proj", + "model.layers.30.mlp.experts.200.down_proj", + "model.layers.30.mlp.experts.200.gate_proj", + "model.layers.30.mlp.experts.200.up_proj", + "model.layers.30.mlp.experts.201.down_proj", + "model.layers.30.mlp.experts.201.gate_proj", + "model.layers.30.mlp.experts.201.up_proj", + "model.layers.30.mlp.experts.202.down_proj", + "model.layers.30.mlp.experts.202.gate_proj", + "model.layers.30.mlp.experts.202.up_proj", + "model.layers.30.mlp.experts.203.down_proj", + "model.layers.30.mlp.experts.203.gate_proj", + "model.layers.30.mlp.experts.203.up_proj", + "model.layers.30.mlp.experts.204.down_proj", + "model.layers.30.mlp.experts.204.gate_proj", + "model.layers.30.mlp.experts.204.up_proj", + "model.layers.30.mlp.experts.205.down_proj", + "model.layers.30.mlp.experts.205.gate_proj", + "model.layers.30.mlp.experts.205.up_proj", + "model.layers.30.mlp.experts.206.down_proj", + "model.layers.30.mlp.experts.206.gate_proj", + "model.layers.30.mlp.experts.206.up_proj", + "model.layers.30.mlp.experts.207.down_proj", + "model.layers.30.mlp.experts.207.gate_proj", + "model.layers.30.mlp.experts.207.up_proj", + "model.layers.30.mlp.experts.208.down_proj", + "model.layers.30.mlp.experts.208.gate_proj", + "model.layers.30.mlp.experts.208.up_proj", + "model.layers.30.mlp.experts.209.down_proj", + "model.layers.30.mlp.experts.209.gate_proj", + "model.layers.30.mlp.experts.209.up_proj", + "model.layers.30.mlp.experts.21.down_proj", + "model.layers.30.mlp.experts.21.gate_proj", + "model.layers.30.mlp.experts.21.up_proj", + "model.layers.30.mlp.experts.210.down_proj", + "model.layers.30.mlp.experts.210.gate_proj", + "model.layers.30.mlp.experts.210.up_proj", + "model.layers.30.mlp.experts.211.down_proj", + "model.layers.30.mlp.experts.211.gate_proj", + "model.layers.30.mlp.experts.211.up_proj", + "model.layers.30.mlp.experts.212.down_proj", + "model.layers.30.mlp.experts.212.gate_proj", + "model.layers.30.mlp.experts.212.up_proj", + "model.layers.30.mlp.experts.213.down_proj", + "model.layers.30.mlp.experts.213.gate_proj", + "model.layers.30.mlp.experts.213.up_proj", + "model.layers.30.mlp.experts.214.down_proj", + "model.layers.30.mlp.experts.214.gate_proj", + "model.layers.30.mlp.experts.214.up_proj", + "model.layers.30.mlp.experts.215.down_proj", + "model.layers.30.mlp.experts.215.gate_proj", + "model.layers.30.mlp.experts.215.up_proj", + "model.layers.30.mlp.experts.216.down_proj", + "model.layers.30.mlp.experts.216.gate_proj", + "model.layers.30.mlp.experts.216.up_proj", + "model.layers.30.mlp.experts.217.down_proj", + "model.layers.30.mlp.experts.217.gate_proj", + "model.layers.30.mlp.experts.217.up_proj", + "model.layers.30.mlp.experts.218.down_proj", + "model.layers.30.mlp.experts.218.gate_proj", + "model.layers.30.mlp.experts.218.up_proj", + "model.layers.30.mlp.experts.219.down_proj", + "model.layers.30.mlp.experts.219.gate_proj", + "model.layers.30.mlp.experts.219.up_proj", + "model.layers.30.mlp.experts.22.down_proj", + "model.layers.30.mlp.experts.22.gate_proj", + "model.layers.30.mlp.experts.22.up_proj", + "model.layers.30.mlp.experts.220.down_proj", + "model.layers.30.mlp.experts.220.gate_proj", + "model.layers.30.mlp.experts.220.up_proj", + "model.layers.30.mlp.experts.221.down_proj", + "model.layers.30.mlp.experts.221.gate_proj", + "model.layers.30.mlp.experts.221.up_proj", + "model.layers.30.mlp.experts.222.down_proj", + "model.layers.30.mlp.experts.222.gate_proj", + "model.layers.30.mlp.experts.222.up_proj", + "model.layers.30.mlp.experts.223.down_proj", + "model.layers.30.mlp.experts.223.gate_proj", + "model.layers.30.mlp.experts.223.up_proj", + "model.layers.30.mlp.experts.224.down_proj", + "model.layers.30.mlp.experts.224.gate_proj", + "model.layers.30.mlp.experts.224.up_proj", + "model.layers.30.mlp.experts.225.down_proj", + "model.layers.30.mlp.experts.225.gate_proj", + "model.layers.30.mlp.experts.225.up_proj", + "model.layers.30.mlp.experts.226.down_proj", + "model.layers.30.mlp.experts.226.gate_proj", + "model.layers.30.mlp.experts.226.up_proj", + "model.layers.30.mlp.experts.227.down_proj", + "model.layers.30.mlp.experts.227.gate_proj", + "model.layers.30.mlp.experts.227.up_proj", + "model.layers.30.mlp.experts.228.down_proj", + "model.layers.30.mlp.experts.228.gate_proj", + "model.layers.30.mlp.experts.228.up_proj", + "model.layers.30.mlp.experts.229.down_proj", + "model.layers.30.mlp.experts.229.gate_proj", + "model.layers.30.mlp.experts.229.up_proj", + "model.layers.30.mlp.experts.23.down_proj", + "model.layers.30.mlp.experts.23.gate_proj", + "model.layers.30.mlp.experts.23.up_proj", + "model.layers.30.mlp.experts.230.down_proj", + "model.layers.30.mlp.experts.230.gate_proj", + "model.layers.30.mlp.experts.230.up_proj", + "model.layers.30.mlp.experts.231.down_proj", + "model.layers.30.mlp.experts.231.gate_proj", + "model.layers.30.mlp.experts.231.up_proj", + "model.layers.30.mlp.experts.232.down_proj", + "model.layers.30.mlp.experts.232.gate_proj", + "model.layers.30.mlp.experts.232.up_proj", + "model.layers.30.mlp.experts.233.down_proj", + "model.layers.30.mlp.experts.233.gate_proj", + "model.layers.30.mlp.experts.233.up_proj", + "model.layers.30.mlp.experts.234.down_proj", + "model.layers.30.mlp.experts.234.gate_proj", + "model.layers.30.mlp.experts.234.up_proj", + "model.layers.30.mlp.experts.235.down_proj", + "model.layers.30.mlp.experts.235.gate_proj", + "model.layers.30.mlp.experts.235.up_proj", + "model.layers.30.mlp.experts.236.down_proj", + "model.layers.30.mlp.experts.236.gate_proj", + "model.layers.30.mlp.experts.236.up_proj", + "model.layers.30.mlp.experts.237.down_proj", + "model.layers.30.mlp.experts.237.gate_proj", + "model.layers.30.mlp.experts.237.up_proj", + "model.layers.30.mlp.experts.238.down_proj", + "model.layers.30.mlp.experts.238.gate_proj", + "model.layers.30.mlp.experts.238.up_proj", + "model.layers.30.mlp.experts.239.down_proj", + "model.layers.30.mlp.experts.239.gate_proj", + "model.layers.30.mlp.experts.239.up_proj", + "model.layers.30.mlp.experts.24.down_proj", + "model.layers.30.mlp.experts.24.gate_proj", + "model.layers.30.mlp.experts.24.up_proj", + "model.layers.30.mlp.experts.240.down_proj", + "model.layers.30.mlp.experts.240.gate_proj", + "model.layers.30.mlp.experts.240.up_proj", + "model.layers.30.mlp.experts.241.down_proj", + "model.layers.30.mlp.experts.241.gate_proj", + "model.layers.30.mlp.experts.241.up_proj", + "model.layers.30.mlp.experts.242.down_proj", + "model.layers.30.mlp.experts.242.gate_proj", + "model.layers.30.mlp.experts.242.up_proj", + "model.layers.30.mlp.experts.243.down_proj", + "model.layers.30.mlp.experts.243.gate_proj", + "model.layers.30.mlp.experts.243.up_proj", + "model.layers.30.mlp.experts.244.down_proj", + "model.layers.30.mlp.experts.244.gate_proj", + "model.layers.30.mlp.experts.244.up_proj", + "model.layers.30.mlp.experts.245.down_proj", + "model.layers.30.mlp.experts.245.gate_proj", + "model.layers.30.mlp.experts.245.up_proj", + "model.layers.30.mlp.experts.246.down_proj", + "model.layers.30.mlp.experts.246.gate_proj", + "model.layers.30.mlp.experts.246.up_proj", + "model.layers.30.mlp.experts.247.down_proj", + "model.layers.30.mlp.experts.247.gate_proj", + "model.layers.30.mlp.experts.247.up_proj", + "model.layers.30.mlp.experts.248.down_proj", + "model.layers.30.mlp.experts.248.gate_proj", + "model.layers.30.mlp.experts.248.up_proj", + "model.layers.30.mlp.experts.249.down_proj", + "model.layers.30.mlp.experts.249.gate_proj", + "model.layers.30.mlp.experts.249.up_proj", + "model.layers.30.mlp.experts.25.down_proj", + "model.layers.30.mlp.experts.25.gate_proj", + "model.layers.30.mlp.experts.25.up_proj", + "model.layers.30.mlp.experts.250.down_proj", + "model.layers.30.mlp.experts.250.gate_proj", + "model.layers.30.mlp.experts.250.up_proj", + "model.layers.30.mlp.experts.251.down_proj", + "model.layers.30.mlp.experts.251.gate_proj", + "model.layers.30.mlp.experts.251.up_proj", + "model.layers.30.mlp.experts.252.down_proj", + "model.layers.30.mlp.experts.252.gate_proj", + "model.layers.30.mlp.experts.252.up_proj", + "model.layers.30.mlp.experts.253.down_proj", + "model.layers.30.mlp.experts.253.gate_proj", + "model.layers.30.mlp.experts.253.up_proj", + "model.layers.30.mlp.experts.254.down_proj", + "model.layers.30.mlp.experts.254.gate_proj", + "model.layers.30.mlp.experts.254.up_proj", + "model.layers.30.mlp.experts.255.down_proj", + "model.layers.30.mlp.experts.255.gate_proj", + "model.layers.30.mlp.experts.255.up_proj", + "model.layers.30.mlp.experts.26.down_proj", + "model.layers.30.mlp.experts.26.gate_proj", + "model.layers.30.mlp.experts.26.up_proj", + "model.layers.30.mlp.experts.27.down_proj", + "model.layers.30.mlp.experts.27.gate_proj", + "model.layers.30.mlp.experts.27.up_proj", + "model.layers.30.mlp.experts.28.down_proj", + "model.layers.30.mlp.experts.28.gate_proj", + "model.layers.30.mlp.experts.28.up_proj", + "model.layers.30.mlp.experts.29.down_proj", + "model.layers.30.mlp.experts.29.gate_proj", + "model.layers.30.mlp.experts.29.up_proj", + "model.layers.30.mlp.experts.3.down_proj", + "model.layers.30.mlp.experts.3.gate_proj", + "model.layers.30.mlp.experts.3.up_proj", + "model.layers.30.mlp.experts.30.down_proj", + "model.layers.30.mlp.experts.30.gate_proj", + "model.layers.30.mlp.experts.30.up_proj", + "model.layers.30.mlp.experts.31.down_proj", + "model.layers.30.mlp.experts.31.gate_proj", + "model.layers.30.mlp.experts.31.up_proj", + "model.layers.30.mlp.experts.32.down_proj", + "model.layers.30.mlp.experts.32.gate_proj", + "model.layers.30.mlp.experts.32.up_proj", + "model.layers.30.mlp.experts.33.down_proj", + "model.layers.30.mlp.experts.33.gate_proj", + "model.layers.30.mlp.experts.33.up_proj", + "model.layers.30.mlp.experts.34.down_proj", + "model.layers.30.mlp.experts.34.gate_proj", + "model.layers.30.mlp.experts.34.up_proj", + "model.layers.30.mlp.experts.35.down_proj", + "model.layers.30.mlp.experts.35.gate_proj", + "model.layers.30.mlp.experts.35.up_proj", + "model.layers.30.mlp.experts.36.down_proj", + "model.layers.30.mlp.experts.36.gate_proj", + "model.layers.30.mlp.experts.36.up_proj", + "model.layers.30.mlp.experts.37.down_proj", + "model.layers.30.mlp.experts.37.gate_proj", + "model.layers.30.mlp.experts.37.up_proj", + "model.layers.30.mlp.experts.38.down_proj", + "model.layers.30.mlp.experts.38.gate_proj", + "model.layers.30.mlp.experts.38.up_proj", + "model.layers.30.mlp.experts.39.down_proj", + "model.layers.30.mlp.experts.39.gate_proj", + "model.layers.30.mlp.experts.39.up_proj", + "model.layers.30.mlp.experts.4.down_proj", + "model.layers.30.mlp.experts.4.gate_proj", + "model.layers.30.mlp.experts.4.up_proj", + "model.layers.30.mlp.experts.40.down_proj", + "model.layers.30.mlp.experts.40.gate_proj", + "model.layers.30.mlp.experts.40.up_proj", + "model.layers.30.mlp.experts.41.down_proj", + "model.layers.30.mlp.experts.41.gate_proj", + "model.layers.30.mlp.experts.41.up_proj", + "model.layers.30.mlp.experts.42.down_proj", + "model.layers.30.mlp.experts.42.gate_proj", + "model.layers.30.mlp.experts.42.up_proj", + "model.layers.30.mlp.experts.43.down_proj", + "model.layers.30.mlp.experts.43.gate_proj", + "model.layers.30.mlp.experts.43.up_proj", + "model.layers.30.mlp.experts.44.down_proj", + "model.layers.30.mlp.experts.44.gate_proj", + "model.layers.30.mlp.experts.44.up_proj", + "model.layers.30.mlp.experts.45.down_proj", + "model.layers.30.mlp.experts.45.gate_proj", + "model.layers.30.mlp.experts.45.up_proj", + "model.layers.30.mlp.experts.46.down_proj", + "model.layers.30.mlp.experts.46.gate_proj", + "model.layers.30.mlp.experts.46.up_proj", + "model.layers.30.mlp.experts.47.down_proj", + "model.layers.30.mlp.experts.47.gate_proj", + "model.layers.30.mlp.experts.47.up_proj", + "model.layers.30.mlp.experts.48.down_proj", + "model.layers.30.mlp.experts.48.gate_proj", + "model.layers.30.mlp.experts.48.up_proj", + "model.layers.30.mlp.experts.49.down_proj", + "model.layers.30.mlp.experts.49.gate_proj", + "model.layers.30.mlp.experts.49.up_proj", + "model.layers.30.mlp.experts.5.down_proj", + "model.layers.30.mlp.experts.5.gate_proj", + "model.layers.30.mlp.experts.5.up_proj", + "model.layers.30.mlp.experts.50.down_proj", + "model.layers.30.mlp.experts.50.gate_proj", + "model.layers.30.mlp.experts.50.up_proj", + "model.layers.30.mlp.experts.51.down_proj", + "model.layers.30.mlp.experts.51.gate_proj", + "model.layers.30.mlp.experts.51.up_proj", + "model.layers.30.mlp.experts.52.down_proj", + "model.layers.30.mlp.experts.52.gate_proj", + "model.layers.30.mlp.experts.52.up_proj", + "model.layers.30.mlp.experts.53.down_proj", + "model.layers.30.mlp.experts.53.gate_proj", + "model.layers.30.mlp.experts.53.up_proj", + "model.layers.30.mlp.experts.54.down_proj", + "model.layers.30.mlp.experts.54.gate_proj", + "model.layers.30.mlp.experts.54.up_proj", + "model.layers.30.mlp.experts.55.down_proj", + "model.layers.30.mlp.experts.55.gate_proj", + "model.layers.30.mlp.experts.55.up_proj", + "model.layers.30.mlp.experts.56.down_proj", + "model.layers.30.mlp.experts.56.gate_proj", + "model.layers.30.mlp.experts.56.up_proj", + "model.layers.30.mlp.experts.57.down_proj", + "model.layers.30.mlp.experts.57.gate_proj", + "model.layers.30.mlp.experts.57.up_proj", + "model.layers.30.mlp.experts.58.down_proj", + "model.layers.30.mlp.experts.58.gate_proj", + "model.layers.30.mlp.experts.58.up_proj", + "model.layers.30.mlp.experts.59.down_proj", + "model.layers.30.mlp.experts.59.gate_proj", + "model.layers.30.mlp.experts.59.up_proj", + "model.layers.30.mlp.experts.6.down_proj", + "model.layers.30.mlp.experts.6.gate_proj", + "model.layers.30.mlp.experts.6.up_proj", + "model.layers.30.mlp.experts.60.down_proj", + "model.layers.30.mlp.experts.60.gate_proj", + "model.layers.30.mlp.experts.60.up_proj", + "model.layers.30.mlp.experts.61.down_proj", + "model.layers.30.mlp.experts.61.gate_proj", + "model.layers.30.mlp.experts.61.up_proj", + "model.layers.30.mlp.experts.62.down_proj", + "model.layers.30.mlp.experts.62.gate_proj", + "model.layers.30.mlp.experts.62.up_proj", + "model.layers.30.mlp.experts.63.down_proj", + "model.layers.30.mlp.experts.63.gate_proj", + "model.layers.30.mlp.experts.63.up_proj", + "model.layers.30.mlp.experts.64.down_proj", + "model.layers.30.mlp.experts.64.gate_proj", + "model.layers.30.mlp.experts.64.up_proj", + "model.layers.30.mlp.experts.65.down_proj", + "model.layers.30.mlp.experts.65.gate_proj", + "model.layers.30.mlp.experts.65.up_proj", + "model.layers.30.mlp.experts.66.down_proj", + "model.layers.30.mlp.experts.66.gate_proj", + "model.layers.30.mlp.experts.66.up_proj", + "model.layers.30.mlp.experts.67.down_proj", + "model.layers.30.mlp.experts.67.gate_proj", + "model.layers.30.mlp.experts.67.up_proj", + "model.layers.30.mlp.experts.68.down_proj", + "model.layers.30.mlp.experts.68.gate_proj", + "model.layers.30.mlp.experts.68.up_proj", + "model.layers.30.mlp.experts.69.down_proj", + "model.layers.30.mlp.experts.69.gate_proj", + "model.layers.30.mlp.experts.69.up_proj", + "model.layers.30.mlp.experts.7.down_proj", + "model.layers.30.mlp.experts.7.gate_proj", + "model.layers.30.mlp.experts.7.up_proj", + "model.layers.30.mlp.experts.70.down_proj", + "model.layers.30.mlp.experts.70.gate_proj", + "model.layers.30.mlp.experts.70.up_proj", + "model.layers.30.mlp.experts.71.down_proj", + "model.layers.30.mlp.experts.71.gate_proj", + "model.layers.30.mlp.experts.71.up_proj", + "model.layers.30.mlp.experts.72.down_proj", + "model.layers.30.mlp.experts.72.gate_proj", + "model.layers.30.mlp.experts.72.up_proj", + "model.layers.30.mlp.experts.73.down_proj", + "model.layers.30.mlp.experts.73.gate_proj", + "model.layers.30.mlp.experts.73.up_proj", + "model.layers.30.mlp.experts.74.down_proj", + "model.layers.30.mlp.experts.74.gate_proj", + "model.layers.30.mlp.experts.74.up_proj", + "model.layers.30.mlp.experts.75.down_proj", + "model.layers.30.mlp.experts.75.gate_proj", + "model.layers.30.mlp.experts.75.up_proj", + "model.layers.30.mlp.experts.76.down_proj", + "model.layers.30.mlp.experts.76.gate_proj", + "model.layers.30.mlp.experts.76.up_proj", + "model.layers.30.mlp.experts.77.down_proj", + "model.layers.30.mlp.experts.77.gate_proj", + "model.layers.30.mlp.experts.77.up_proj", + "model.layers.30.mlp.experts.78.down_proj", + "model.layers.30.mlp.experts.78.gate_proj", + "model.layers.30.mlp.experts.78.up_proj", + "model.layers.30.mlp.experts.79.down_proj", + "model.layers.30.mlp.experts.79.gate_proj", + "model.layers.30.mlp.experts.79.up_proj", + "model.layers.30.mlp.experts.8.down_proj", + "model.layers.30.mlp.experts.8.gate_proj", + "model.layers.30.mlp.experts.8.up_proj", + "model.layers.30.mlp.experts.80.down_proj", + "model.layers.30.mlp.experts.80.gate_proj", + "model.layers.30.mlp.experts.80.up_proj", + "model.layers.30.mlp.experts.81.down_proj", + "model.layers.30.mlp.experts.81.gate_proj", + "model.layers.30.mlp.experts.81.up_proj", + "model.layers.30.mlp.experts.82.down_proj", + "model.layers.30.mlp.experts.82.gate_proj", + "model.layers.30.mlp.experts.82.up_proj", + "model.layers.30.mlp.experts.83.down_proj", + "model.layers.30.mlp.experts.83.gate_proj", + "model.layers.30.mlp.experts.83.up_proj", + "model.layers.30.mlp.experts.84.down_proj", + "model.layers.30.mlp.experts.84.gate_proj", + "model.layers.30.mlp.experts.84.up_proj", + "model.layers.30.mlp.experts.85.down_proj", + "model.layers.30.mlp.experts.85.gate_proj", + "model.layers.30.mlp.experts.85.up_proj", + "model.layers.30.mlp.experts.86.down_proj", + "model.layers.30.mlp.experts.86.gate_proj", + "model.layers.30.mlp.experts.86.up_proj", + "model.layers.30.mlp.experts.87.down_proj", + "model.layers.30.mlp.experts.87.gate_proj", + "model.layers.30.mlp.experts.87.up_proj", + "model.layers.30.mlp.experts.88.down_proj", + "model.layers.30.mlp.experts.88.gate_proj", + "model.layers.30.mlp.experts.88.up_proj", + "model.layers.30.mlp.experts.89.down_proj", + "model.layers.30.mlp.experts.89.gate_proj", + "model.layers.30.mlp.experts.89.up_proj", + "model.layers.30.mlp.experts.9.down_proj", + "model.layers.30.mlp.experts.9.gate_proj", + "model.layers.30.mlp.experts.9.up_proj", + "model.layers.30.mlp.experts.90.down_proj", + "model.layers.30.mlp.experts.90.gate_proj", + "model.layers.30.mlp.experts.90.up_proj", + "model.layers.30.mlp.experts.91.down_proj", + "model.layers.30.mlp.experts.91.gate_proj", + "model.layers.30.mlp.experts.91.up_proj", + "model.layers.30.mlp.experts.92.down_proj", + "model.layers.30.mlp.experts.92.gate_proj", + "model.layers.30.mlp.experts.92.up_proj", + "model.layers.30.mlp.experts.93.down_proj", + "model.layers.30.mlp.experts.93.gate_proj", + "model.layers.30.mlp.experts.93.up_proj", + "model.layers.30.mlp.experts.94.down_proj", + "model.layers.30.mlp.experts.94.gate_proj", + "model.layers.30.mlp.experts.94.up_proj", + "model.layers.30.mlp.experts.95.down_proj", + "model.layers.30.mlp.experts.95.gate_proj", + "model.layers.30.mlp.experts.95.up_proj", + "model.layers.30.mlp.experts.96.down_proj", + "model.layers.30.mlp.experts.96.gate_proj", + "model.layers.30.mlp.experts.96.up_proj", + "model.layers.30.mlp.experts.97.down_proj", + "model.layers.30.mlp.experts.97.gate_proj", + "model.layers.30.mlp.experts.97.up_proj", + "model.layers.30.mlp.experts.98.down_proj", + "model.layers.30.mlp.experts.98.gate_proj", + "model.layers.30.mlp.experts.98.up_proj", + "model.layers.30.mlp.experts.99.down_proj", + "model.layers.30.mlp.experts.99.gate_proj", + "model.layers.30.mlp.experts.99.up_proj", + "model.layers.31.mlp.experts.0.down_proj", + "model.layers.31.mlp.experts.0.gate_proj", + "model.layers.31.mlp.experts.0.up_proj", + "model.layers.31.mlp.experts.1.down_proj", + "model.layers.31.mlp.experts.1.gate_proj", + "model.layers.31.mlp.experts.1.up_proj", + "model.layers.31.mlp.experts.10.down_proj", + "model.layers.31.mlp.experts.10.gate_proj", + "model.layers.31.mlp.experts.10.up_proj", + "model.layers.31.mlp.experts.100.down_proj", + "model.layers.31.mlp.experts.100.gate_proj", + "model.layers.31.mlp.experts.100.up_proj", + "model.layers.31.mlp.experts.101.down_proj", + "model.layers.31.mlp.experts.101.gate_proj", + "model.layers.31.mlp.experts.101.up_proj", + "model.layers.31.mlp.experts.102.down_proj", + "model.layers.31.mlp.experts.102.gate_proj", + "model.layers.31.mlp.experts.102.up_proj", + "model.layers.31.mlp.experts.103.down_proj", + "model.layers.31.mlp.experts.103.gate_proj", + "model.layers.31.mlp.experts.103.up_proj", + "model.layers.31.mlp.experts.104.down_proj", + "model.layers.31.mlp.experts.104.gate_proj", + "model.layers.31.mlp.experts.104.up_proj", + "model.layers.31.mlp.experts.105.down_proj", + "model.layers.31.mlp.experts.105.gate_proj", + "model.layers.31.mlp.experts.105.up_proj", + "model.layers.31.mlp.experts.106.down_proj", + "model.layers.31.mlp.experts.106.gate_proj", + "model.layers.31.mlp.experts.106.up_proj", + "model.layers.31.mlp.experts.107.down_proj", + "model.layers.31.mlp.experts.107.gate_proj", + "model.layers.31.mlp.experts.107.up_proj", + "model.layers.31.mlp.experts.108.down_proj", + "model.layers.31.mlp.experts.108.gate_proj", + "model.layers.31.mlp.experts.108.up_proj", + "model.layers.31.mlp.experts.109.down_proj", + "model.layers.31.mlp.experts.109.gate_proj", + "model.layers.31.mlp.experts.109.up_proj", + "model.layers.31.mlp.experts.11.down_proj", + "model.layers.31.mlp.experts.11.gate_proj", + "model.layers.31.mlp.experts.11.up_proj", + "model.layers.31.mlp.experts.110.down_proj", + "model.layers.31.mlp.experts.110.gate_proj", + "model.layers.31.mlp.experts.110.up_proj", + "model.layers.31.mlp.experts.111.down_proj", + "model.layers.31.mlp.experts.111.gate_proj", + "model.layers.31.mlp.experts.111.up_proj", + "model.layers.31.mlp.experts.112.down_proj", + "model.layers.31.mlp.experts.112.gate_proj", + "model.layers.31.mlp.experts.112.up_proj", + "model.layers.31.mlp.experts.113.down_proj", + "model.layers.31.mlp.experts.113.gate_proj", + "model.layers.31.mlp.experts.113.up_proj", + "model.layers.31.mlp.experts.114.down_proj", + "model.layers.31.mlp.experts.114.gate_proj", + "model.layers.31.mlp.experts.114.up_proj", + "model.layers.31.mlp.experts.115.down_proj", + "model.layers.31.mlp.experts.115.gate_proj", + "model.layers.31.mlp.experts.115.up_proj", + "model.layers.31.mlp.experts.116.down_proj", + "model.layers.31.mlp.experts.116.gate_proj", + "model.layers.31.mlp.experts.116.up_proj", + "model.layers.31.mlp.experts.117.down_proj", + "model.layers.31.mlp.experts.117.gate_proj", + "model.layers.31.mlp.experts.117.up_proj", + "model.layers.31.mlp.experts.118.down_proj", + "model.layers.31.mlp.experts.118.gate_proj", + "model.layers.31.mlp.experts.118.up_proj", + "model.layers.31.mlp.experts.119.down_proj", + "model.layers.31.mlp.experts.119.gate_proj", + "model.layers.31.mlp.experts.119.up_proj", + "model.layers.31.mlp.experts.12.down_proj", + "model.layers.31.mlp.experts.12.gate_proj", + "model.layers.31.mlp.experts.12.up_proj", + "model.layers.31.mlp.experts.120.down_proj", + "model.layers.31.mlp.experts.120.gate_proj", + "model.layers.31.mlp.experts.120.up_proj", + "model.layers.31.mlp.experts.121.down_proj", + "model.layers.31.mlp.experts.121.gate_proj", + "model.layers.31.mlp.experts.121.up_proj", + "model.layers.31.mlp.experts.122.down_proj", + "model.layers.31.mlp.experts.122.gate_proj", + "model.layers.31.mlp.experts.122.up_proj", + "model.layers.31.mlp.experts.123.down_proj", + "model.layers.31.mlp.experts.123.gate_proj", + "model.layers.31.mlp.experts.123.up_proj", + "model.layers.31.mlp.experts.124.down_proj", + "model.layers.31.mlp.experts.124.gate_proj", + "model.layers.31.mlp.experts.124.up_proj", + "model.layers.31.mlp.experts.125.down_proj", + "model.layers.31.mlp.experts.125.gate_proj", + "model.layers.31.mlp.experts.125.up_proj", + "model.layers.31.mlp.experts.126.down_proj", + "model.layers.31.mlp.experts.126.gate_proj", + "model.layers.31.mlp.experts.126.up_proj", + "model.layers.31.mlp.experts.127.down_proj", + "model.layers.31.mlp.experts.127.gate_proj", + "model.layers.31.mlp.experts.127.up_proj", + "model.layers.31.mlp.experts.128.down_proj", + "model.layers.31.mlp.experts.128.gate_proj", + "model.layers.31.mlp.experts.128.up_proj", + "model.layers.31.mlp.experts.129.down_proj", + "model.layers.31.mlp.experts.129.gate_proj", + "model.layers.31.mlp.experts.129.up_proj", + "model.layers.31.mlp.experts.13.down_proj", + "model.layers.31.mlp.experts.13.gate_proj", + "model.layers.31.mlp.experts.13.up_proj", + "model.layers.31.mlp.experts.130.down_proj", + "model.layers.31.mlp.experts.130.gate_proj", + "model.layers.31.mlp.experts.130.up_proj", + "model.layers.31.mlp.experts.131.down_proj", + "model.layers.31.mlp.experts.131.gate_proj", + "model.layers.31.mlp.experts.131.up_proj", + "model.layers.31.mlp.experts.132.down_proj", + "model.layers.31.mlp.experts.132.gate_proj", + "model.layers.31.mlp.experts.132.up_proj", + "model.layers.31.mlp.experts.133.down_proj", + "model.layers.31.mlp.experts.133.gate_proj", + "model.layers.31.mlp.experts.133.up_proj", + "model.layers.31.mlp.experts.134.down_proj", + "model.layers.31.mlp.experts.134.gate_proj", + "model.layers.31.mlp.experts.134.up_proj", + "model.layers.31.mlp.experts.135.down_proj", + "model.layers.31.mlp.experts.135.gate_proj", + "model.layers.31.mlp.experts.135.up_proj", + "model.layers.31.mlp.experts.136.down_proj", + "model.layers.31.mlp.experts.136.gate_proj", + "model.layers.31.mlp.experts.136.up_proj", + "model.layers.31.mlp.experts.137.down_proj", + "model.layers.31.mlp.experts.137.gate_proj", + "model.layers.31.mlp.experts.137.up_proj", + "model.layers.31.mlp.experts.138.down_proj", + "model.layers.31.mlp.experts.138.gate_proj", + "model.layers.31.mlp.experts.138.up_proj", + "model.layers.31.mlp.experts.139.down_proj", + "model.layers.31.mlp.experts.139.gate_proj", + "model.layers.31.mlp.experts.139.up_proj", + "model.layers.31.mlp.experts.14.down_proj", + "model.layers.31.mlp.experts.14.gate_proj", + "model.layers.31.mlp.experts.14.up_proj", + "model.layers.31.mlp.experts.140.down_proj", + "model.layers.31.mlp.experts.140.gate_proj", + "model.layers.31.mlp.experts.140.up_proj", + "model.layers.31.mlp.experts.141.down_proj", + "model.layers.31.mlp.experts.141.gate_proj", + "model.layers.31.mlp.experts.141.up_proj", + "model.layers.31.mlp.experts.142.down_proj", + "model.layers.31.mlp.experts.142.gate_proj", + "model.layers.31.mlp.experts.142.up_proj", + "model.layers.31.mlp.experts.143.down_proj", + "model.layers.31.mlp.experts.143.gate_proj", + "model.layers.31.mlp.experts.143.up_proj", + "model.layers.31.mlp.experts.144.down_proj", + "model.layers.31.mlp.experts.144.gate_proj", + "model.layers.31.mlp.experts.144.up_proj", + "model.layers.31.mlp.experts.145.down_proj", + "model.layers.31.mlp.experts.145.gate_proj", + "model.layers.31.mlp.experts.145.up_proj", + "model.layers.31.mlp.experts.146.down_proj", + "model.layers.31.mlp.experts.146.gate_proj", + "model.layers.31.mlp.experts.146.up_proj", + "model.layers.31.mlp.experts.147.down_proj", + "model.layers.31.mlp.experts.147.gate_proj", + "model.layers.31.mlp.experts.147.up_proj", + "model.layers.31.mlp.experts.148.down_proj", + "model.layers.31.mlp.experts.148.gate_proj", + "model.layers.31.mlp.experts.148.up_proj", + "model.layers.31.mlp.experts.149.down_proj", + "model.layers.31.mlp.experts.149.gate_proj", + "model.layers.31.mlp.experts.149.up_proj", + "model.layers.31.mlp.experts.15.down_proj", + "model.layers.31.mlp.experts.15.gate_proj", + "model.layers.31.mlp.experts.15.up_proj", + "model.layers.31.mlp.experts.150.down_proj", + "model.layers.31.mlp.experts.150.gate_proj", + "model.layers.31.mlp.experts.150.up_proj", + "model.layers.31.mlp.experts.151.down_proj", + "model.layers.31.mlp.experts.151.gate_proj", + "model.layers.31.mlp.experts.151.up_proj", + "model.layers.31.mlp.experts.152.down_proj", + "model.layers.31.mlp.experts.152.gate_proj", + "model.layers.31.mlp.experts.152.up_proj", + "model.layers.31.mlp.experts.153.down_proj", + "model.layers.31.mlp.experts.153.gate_proj", + "model.layers.31.mlp.experts.153.up_proj", + "model.layers.31.mlp.experts.154.down_proj", + "model.layers.31.mlp.experts.154.gate_proj", + "model.layers.31.mlp.experts.154.up_proj", + "model.layers.31.mlp.experts.155.down_proj", + "model.layers.31.mlp.experts.155.gate_proj", + "model.layers.31.mlp.experts.155.up_proj", + "model.layers.31.mlp.experts.156.down_proj", + "model.layers.31.mlp.experts.156.gate_proj", + "model.layers.31.mlp.experts.156.up_proj", + "model.layers.31.mlp.experts.157.down_proj", + "model.layers.31.mlp.experts.157.gate_proj", + "model.layers.31.mlp.experts.157.up_proj", + "model.layers.31.mlp.experts.158.down_proj", + "model.layers.31.mlp.experts.158.gate_proj", + "model.layers.31.mlp.experts.158.up_proj", + "model.layers.31.mlp.experts.159.down_proj", + "model.layers.31.mlp.experts.159.gate_proj", + "model.layers.31.mlp.experts.159.up_proj", + "model.layers.31.mlp.experts.16.down_proj", + "model.layers.31.mlp.experts.16.gate_proj", + "model.layers.31.mlp.experts.16.up_proj", + "model.layers.31.mlp.experts.160.down_proj", + "model.layers.31.mlp.experts.160.gate_proj", + "model.layers.31.mlp.experts.160.up_proj", + "model.layers.31.mlp.experts.161.down_proj", + "model.layers.31.mlp.experts.161.gate_proj", + "model.layers.31.mlp.experts.161.up_proj", + "model.layers.31.mlp.experts.162.down_proj", + "model.layers.31.mlp.experts.162.gate_proj", + "model.layers.31.mlp.experts.162.up_proj", + "model.layers.31.mlp.experts.163.down_proj", + "model.layers.31.mlp.experts.163.gate_proj", + "model.layers.31.mlp.experts.163.up_proj", + "model.layers.31.mlp.experts.164.down_proj", + "model.layers.31.mlp.experts.164.gate_proj", + "model.layers.31.mlp.experts.164.up_proj", + "model.layers.31.mlp.experts.165.down_proj", + "model.layers.31.mlp.experts.165.gate_proj", + "model.layers.31.mlp.experts.165.up_proj", + "model.layers.31.mlp.experts.166.down_proj", + "model.layers.31.mlp.experts.166.gate_proj", + "model.layers.31.mlp.experts.166.up_proj", + "model.layers.31.mlp.experts.167.down_proj", + "model.layers.31.mlp.experts.167.gate_proj", + "model.layers.31.mlp.experts.167.up_proj", + "model.layers.31.mlp.experts.168.down_proj", + "model.layers.31.mlp.experts.168.gate_proj", + "model.layers.31.mlp.experts.168.up_proj", + "model.layers.31.mlp.experts.169.down_proj", + "model.layers.31.mlp.experts.169.gate_proj", + "model.layers.31.mlp.experts.169.up_proj", + "model.layers.31.mlp.experts.17.down_proj", + "model.layers.31.mlp.experts.17.gate_proj", + "model.layers.31.mlp.experts.17.up_proj", + "model.layers.31.mlp.experts.170.down_proj", + "model.layers.31.mlp.experts.170.gate_proj", + "model.layers.31.mlp.experts.170.up_proj", + "model.layers.31.mlp.experts.171.down_proj", + "model.layers.31.mlp.experts.171.gate_proj", + "model.layers.31.mlp.experts.171.up_proj", + "model.layers.31.mlp.experts.172.down_proj", + "model.layers.31.mlp.experts.172.gate_proj", + "model.layers.31.mlp.experts.172.up_proj", + "model.layers.31.mlp.experts.173.down_proj", + "model.layers.31.mlp.experts.173.gate_proj", + "model.layers.31.mlp.experts.173.up_proj", + "model.layers.31.mlp.experts.174.down_proj", + "model.layers.31.mlp.experts.174.gate_proj", + "model.layers.31.mlp.experts.174.up_proj", + "model.layers.31.mlp.experts.175.down_proj", + "model.layers.31.mlp.experts.175.gate_proj", + "model.layers.31.mlp.experts.175.up_proj", + "model.layers.31.mlp.experts.176.down_proj", + "model.layers.31.mlp.experts.176.gate_proj", + "model.layers.31.mlp.experts.176.up_proj", + "model.layers.31.mlp.experts.177.down_proj", + "model.layers.31.mlp.experts.177.gate_proj", + "model.layers.31.mlp.experts.177.up_proj", + "model.layers.31.mlp.experts.178.down_proj", + "model.layers.31.mlp.experts.178.gate_proj", + "model.layers.31.mlp.experts.178.up_proj", + "model.layers.31.mlp.experts.179.down_proj", + "model.layers.31.mlp.experts.179.gate_proj", + "model.layers.31.mlp.experts.179.up_proj", + "model.layers.31.mlp.experts.18.down_proj", + "model.layers.31.mlp.experts.18.gate_proj", + "model.layers.31.mlp.experts.18.up_proj", + "model.layers.31.mlp.experts.180.down_proj", + "model.layers.31.mlp.experts.180.gate_proj", + "model.layers.31.mlp.experts.180.up_proj", + "model.layers.31.mlp.experts.181.down_proj", + "model.layers.31.mlp.experts.181.gate_proj", + "model.layers.31.mlp.experts.181.up_proj", + "model.layers.31.mlp.experts.182.down_proj", + "model.layers.31.mlp.experts.182.gate_proj", + "model.layers.31.mlp.experts.182.up_proj", + "model.layers.31.mlp.experts.183.down_proj", + "model.layers.31.mlp.experts.183.gate_proj", + "model.layers.31.mlp.experts.183.up_proj", + "model.layers.31.mlp.experts.184.down_proj", + "model.layers.31.mlp.experts.184.gate_proj", + "model.layers.31.mlp.experts.184.up_proj", + "model.layers.31.mlp.experts.185.down_proj", + "model.layers.31.mlp.experts.185.gate_proj", + "model.layers.31.mlp.experts.185.up_proj", + "model.layers.31.mlp.experts.186.down_proj", + "model.layers.31.mlp.experts.186.gate_proj", + "model.layers.31.mlp.experts.186.up_proj", + "model.layers.31.mlp.experts.187.down_proj", + "model.layers.31.mlp.experts.187.gate_proj", + "model.layers.31.mlp.experts.187.up_proj", + "model.layers.31.mlp.experts.188.down_proj", + "model.layers.31.mlp.experts.188.gate_proj", + "model.layers.31.mlp.experts.188.up_proj", + "model.layers.31.mlp.experts.189.down_proj", + "model.layers.31.mlp.experts.189.gate_proj", + "model.layers.31.mlp.experts.189.up_proj", + "model.layers.31.mlp.experts.19.down_proj", + "model.layers.31.mlp.experts.19.gate_proj", + "model.layers.31.mlp.experts.19.up_proj", + "model.layers.31.mlp.experts.190.down_proj", + "model.layers.31.mlp.experts.190.gate_proj", + "model.layers.31.mlp.experts.190.up_proj", + "model.layers.31.mlp.experts.191.down_proj", + "model.layers.31.mlp.experts.191.gate_proj", + "model.layers.31.mlp.experts.191.up_proj", + "model.layers.31.mlp.experts.192.down_proj", + "model.layers.31.mlp.experts.192.gate_proj", + "model.layers.31.mlp.experts.192.up_proj", + "model.layers.31.mlp.experts.193.down_proj", + "model.layers.31.mlp.experts.193.gate_proj", + "model.layers.31.mlp.experts.193.up_proj", + "model.layers.31.mlp.experts.194.down_proj", + "model.layers.31.mlp.experts.194.gate_proj", + "model.layers.31.mlp.experts.194.up_proj", + "model.layers.31.mlp.experts.195.down_proj", + "model.layers.31.mlp.experts.195.gate_proj", + "model.layers.31.mlp.experts.195.up_proj", + "model.layers.31.mlp.experts.196.down_proj", + "model.layers.31.mlp.experts.196.gate_proj", + "model.layers.31.mlp.experts.196.up_proj", + "model.layers.31.mlp.experts.197.down_proj", + "model.layers.31.mlp.experts.197.gate_proj", + "model.layers.31.mlp.experts.197.up_proj", + "model.layers.31.mlp.experts.198.down_proj", + "model.layers.31.mlp.experts.198.gate_proj", + "model.layers.31.mlp.experts.198.up_proj", + "model.layers.31.mlp.experts.199.down_proj", + "model.layers.31.mlp.experts.199.gate_proj", + "model.layers.31.mlp.experts.199.up_proj", + "model.layers.31.mlp.experts.2.down_proj", + "model.layers.31.mlp.experts.2.gate_proj", + "model.layers.31.mlp.experts.2.up_proj", + "model.layers.31.mlp.experts.20.down_proj", + "model.layers.31.mlp.experts.20.gate_proj", + "model.layers.31.mlp.experts.20.up_proj", + "model.layers.31.mlp.experts.200.down_proj", + "model.layers.31.mlp.experts.200.gate_proj", + "model.layers.31.mlp.experts.200.up_proj", + "model.layers.31.mlp.experts.201.down_proj", + "model.layers.31.mlp.experts.201.gate_proj", + "model.layers.31.mlp.experts.201.up_proj", + "model.layers.31.mlp.experts.202.down_proj", + "model.layers.31.mlp.experts.202.gate_proj", + "model.layers.31.mlp.experts.202.up_proj", + "model.layers.31.mlp.experts.203.down_proj", + "model.layers.31.mlp.experts.203.gate_proj", + "model.layers.31.mlp.experts.203.up_proj", + "model.layers.31.mlp.experts.204.down_proj", + "model.layers.31.mlp.experts.204.gate_proj", + "model.layers.31.mlp.experts.204.up_proj", + "model.layers.31.mlp.experts.205.down_proj", + "model.layers.31.mlp.experts.205.gate_proj", + "model.layers.31.mlp.experts.205.up_proj", + "model.layers.31.mlp.experts.206.down_proj", + "model.layers.31.mlp.experts.206.gate_proj", + "model.layers.31.mlp.experts.206.up_proj", + "model.layers.31.mlp.experts.207.down_proj", + "model.layers.31.mlp.experts.207.gate_proj", + "model.layers.31.mlp.experts.207.up_proj", + "model.layers.31.mlp.experts.208.down_proj", + "model.layers.31.mlp.experts.208.gate_proj", + "model.layers.31.mlp.experts.208.up_proj", + "model.layers.31.mlp.experts.209.down_proj", + "model.layers.31.mlp.experts.209.gate_proj", + "model.layers.31.mlp.experts.209.up_proj", + "model.layers.31.mlp.experts.21.down_proj", + "model.layers.31.mlp.experts.21.gate_proj", + "model.layers.31.mlp.experts.21.up_proj", + "model.layers.31.mlp.experts.210.down_proj", + "model.layers.31.mlp.experts.210.gate_proj", + "model.layers.31.mlp.experts.210.up_proj", + "model.layers.31.mlp.experts.211.down_proj", + "model.layers.31.mlp.experts.211.gate_proj", + "model.layers.31.mlp.experts.211.up_proj", + "model.layers.31.mlp.experts.212.down_proj", + "model.layers.31.mlp.experts.212.gate_proj", + "model.layers.31.mlp.experts.212.up_proj", + "model.layers.31.mlp.experts.213.down_proj", + "model.layers.31.mlp.experts.213.gate_proj", + "model.layers.31.mlp.experts.213.up_proj", + "model.layers.31.mlp.experts.214.down_proj", + "model.layers.31.mlp.experts.214.gate_proj", + "model.layers.31.mlp.experts.214.up_proj", + "model.layers.31.mlp.experts.215.down_proj", + "model.layers.31.mlp.experts.215.gate_proj", + "model.layers.31.mlp.experts.215.up_proj", + "model.layers.31.mlp.experts.216.down_proj", + "model.layers.31.mlp.experts.216.gate_proj", + "model.layers.31.mlp.experts.216.up_proj", + "model.layers.31.mlp.experts.217.down_proj", + "model.layers.31.mlp.experts.217.gate_proj", + "model.layers.31.mlp.experts.217.up_proj", + "model.layers.31.mlp.experts.218.down_proj", + "model.layers.31.mlp.experts.218.gate_proj", + "model.layers.31.mlp.experts.218.up_proj", + "model.layers.31.mlp.experts.219.down_proj", + "model.layers.31.mlp.experts.219.gate_proj", + "model.layers.31.mlp.experts.219.up_proj", + "model.layers.31.mlp.experts.22.down_proj", + "model.layers.31.mlp.experts.22.gate_proj", + "model.layers.31.mlp.experts.22.up_proj", + "model.layers.31.mlp.experts.220.down_proj", + "model.layers.31.mlp.experts.220.gate_proj", + "model.layers.31.mlp.experts.220.up_proj", + "model.layers.31.mlp.experts.221.down_proj", + "model.layers.31.mlp.experts.221.gate_proj", + "model.layers.31.mlp.experts.221.up_proj", + "model.layers.31.mlp.experts.222.down_proj", + "model.layers.31.mlp.experts.222.gate_proj", + "model.layers.31.mlp.experts.222.up_proj", + "model.layers.31.mlp.experts.223.down_proj", + "model.layers.31.mlp.experts.223.gate_proj", + "model.layers.31.mlp.experts.223.up_proj", + "model.layers.31.mlp.experts.224.down_proj", + "model.layers.31.mlp.experts.224.gate_proj", + "model.layers.31.mlp.experts.224.up_proj", + "model.layers.31.mlp.experts.225.down_proj", + "model.layers.31.mlp.experts.225.gate_proj", + "model.layers.31.mlp.experts.225.up_proj", + "model.layers.31.mlp.experts.226.down_proj", + "model.layers.31.mlp.experts.226.gate_proj", + "model.layers.31.mlp.experts.226.up_proj", + "model.layers.31.mlp.experts.227.down_proj", + "model.layers.31.mlp.experts.227.gate_proj", + "model.layers.31.mlp.experts.227.up_proj", + "model.layers.31.mlp.experts.228.down_proj", + "model.layers.31.mlp.experts.228.gate_proj", + "model.layers.31.mlp.experts.228.up_proj", + "model.layers.31.mlp.experts.229.down_proj", + "model.layers.31.mlp.experts.229.gate_proj", + "model.layers.31.mlp.experts.229.up_proj", + "model.layers.31.mlp.experts.23.down_proj", + "model.layers.31.mlp.experts.23.gate_proj", + "model.layers.31.mlp.experts.23.up_proj", + "model.layers.31.mlp.experts.230.down_proj", + "model.layers.31.mlp.experts.230.gate_proj", + "model.layers.31.mlp.experts.230.up_proj", + "model.layers.31.mlp.experts.231.down_proj", + "model.layers.31.mlp.experts.231.gate_proj", + "model.layers.31.mlp.experts.231.up_proj", + "model.layers.31.mlp.experts.232.down_proj", + "model.layers.31.mlp.experts.232.gate_proj", + "model.layers.31.mlp.experts.232.up_proj", + "model.layers.31.mlp.experts.233.down_proj", + "model.layers.31.mlp.experts.233.gate_proj", + "model.layers.31.mlp.experts.233.up_proj", + "model.layers.31.mlp.experts.234.down_proj", + "model.layers.31.mlp.experts.234.gate_proj", + "model.layers.31.mlp.experts.234.up_proj", + "model.layers.31.mlp.experts.235.down_proj", + "model.layers.31.mlp.experts.235.gate_proj", + "model.layers.31.mlp.experts.235.up_proj", + "model.layers.31.mlp.experts.236.down_proj", + "model.layers.31.mlp.experts.236.gate_proj", + "model.layers.31.mlp.experts.236.up_proj", + "model.layers.31.mlp.experts.237.down_proj", + "model.layers.31.mlp.experts.237.gate_proj", + "model.layers.31.mlp.experts.237.up_proj", + "model.layers.31.mlp.experts.238.down_proj", + "model.layers.31.mlp.experts.238.gate_proj", + "model.layers.31.mlp.experts.238.up_proj", + "model.layers.31.mlp.experts.239.down_proj", + "model.layers.31.mlp.experts.239.gate_proj", + "model.layers.31.mlp.experts.239.up_proj", + "model.layers.31.mlp.experts.24.down_proj", + "model.layers.31.mlp.experts.24.gate_proj", + "model.layers.31.mlp.experts.24.up_proj", + "model.layers.31.mlp.experts.240.down_proj", + "model.layers.31.mlp.experts.240.gate_proj", + "model.layers.31.mlp.experts.240.up_proj", + "model.layers.31.mlp.experts.241.down_proj", + "model.layers.31.mlp.experts.241.gate_proj", + "model.layers.31.mlp.experts.241.up_proj", + "model.layers.31.mlp.experts.242.down_proj", + "model.layers.31.mlp.experts.242.gate_proj", + "model.layers.31.mlp.experts.242.up_proj", + "model.layers.31.mlp.experts.243.down_proj", + "model.layers.31.mlp.experts.243.gate_proj", + "model.layers.31.mlp.experts.243.up_proj", + "model.layers.31.mlp.experts.244.down_proj", + "model.layers.31.mlp.experts.244.gate_proj", + "model.layers.31.mlp.experts.244.up_proj", + "model.layers.31.mlp.experts.245.down_proj", + "model.layers.31.mlp.experts.245.gate_proj", + "model.layers.31.mlp.experts.245.up_proj", + "model.layers.31.mlp.experts.246.down_proj", + "model.layers.31.mlp.experts.246.gate_proj", + "model.layers.31.mlp.experts.246.up_proj", + "model.layers.31.mlp.experts.247.down_proj", + "model.layers.31.mlp.experts.247.gate_proj", + "model.layers.31.mlp.experts.247.up_proj", + "model.layers.31.mlp.experts.248.down_proj", + "model.layers.31.mlp.experts.248.gate_proj", + "model.layers.31.mlp.experts.248.up_proj", + "model.layers.31.mlp.experts.249.down_proj", + "model.layers.31.mlp.experts.249.gate_proj", + "model.layers.31.mlp.experts.249.up_proj", + "model.layers.31.mlp.experts.25.down_proj", + "model.layers.31.mlp.experts.25.gate_proj", + "model.layers.31.mlp.experts.25.up_proj", + "model.layers.31.mlp.experts.250.down_proj", + "model.layers.31.mlp.experts.250.gate_proj", + "model.layers.31.mlp.experts.250.up_proj", + "model.layers.31.mlp.experts.251.down_proj", + "model.layers.31.mlp.experts.251.gate_proj", + "model.layers.31.mlp.experts.251.up_proj", + "model.layers.31.mlp.experts.252.down_proj", + "model.layers.31.mlp.experts.252.gate_proj", + "model.layers.31.mlp.experts.252.up_proj", + "model.layers.31.mlp.experts.253.down_proj", + "model.layers.31.mlp.experts.253.gate_proj", + "model.layers.31.mlp.experts.253.up_proj", + "model.layers.31.mlp.experts.254.down_proj", + "model.layers.31.mlp.experts.254.gate_proj", + "model.layers.31.mlp.experts.254.up_proj", + "model.layers.31.mlp.experts.255.down_proj", + "model.layers.31.mlp.experts.255.gate_proj", + "model.layers.31.mlp.experts.255.up_proj", + "model.layers.31.mlp.experts.26.down_proj", + "model.layers.31.mlp.experts.26.gate_proj", + "model.layers.31.mlp.experts.26.up_proj", + "model.layers.31.mlp.experts.27.down_proj", + "model.layers.31.mlp.experts.27.gate_proj", + "model.layers.31.mlp.experts.27.up_proj", + "model.layers.31.mlp.experts.28.down_proj", + "model.layers.31.mlp.experts.28.gate_proj", + "model.layers.31.mlp.experts.28.up_proj", + "model.layers.31.mlp.experts.29.down_proj", + "model.layers.31.mlp.experts.29.gate_proj", + "model.layers.31.mlp.experts.29.up_proj", + "model.layers.31.mlp.experts.3.down_proj", + "model.layers.31.mlp.experts.3.gate_proj", + "model.layers.31.mlp.experts.3.up_proj", + "model.layers.31.mlp.experts.30.down_proj", + "model.layers.31.mlp.experts.30.gate_proj", + "model.layers.31.mlp.experts.30.up_proj", + "model.layers.31.mlp.experts.31.down_proj", + "model.layers.31.mlp.experts.31.gate_proj", + "model.layers.31.mlp.experts.31.up_proj", + "model.layers.31.mlp.experts.32.down_proj", + "model.layers.31.mlp.experts.32.gate_proj", + "model.layers.31.mlp.experts.32.up_proj", + "model.layers.31.mlp.experts.33.down_proj", + "model.layers.31.mlp.experts.33.gate_proj", + "model.layers.31.mlp.experts.33.up_proj", + "model.layers.31.mlp.experts.34.down_proj", + "model.layers.31.mlp.experts.34.gate_proj", + "model.layers.31.mlp.experts.34.up_proj", + "model.layers.31.mlp.experts.35.down_proj", + "model.layers.31.mlp.experts.35.gate_proj", + "model.layers.31.mlp.experts.35.up_proj", + "model.layers.31.mlp.experts.36.down_proj", + "model.layers.31.mlp.experts.36.gate_proj", + "model.layers.31.mlp.experts.36.up_proj", + "model.layers.31.mlp.experts.37.down_proj", + "model.layers.31.mlp.experts.37.gate_proj", + "model.layers.31.mlp.experts.37.up_proj", + "model.layers.31.mlp.experts.38.down_proj", + "model.layers.31.mlp.experts.38.gate_proj", + "model.layers.31.mlp.experts.38.up_proj", + "model.layers.31.mlp.experts.39.down_proj", + "model.layers.31.mlp.experts.39.gate_proj", + "model.layers.31.mlp.experts.39.up_proj", + "model.layers.31.mlp.experts.4.down_proj", + "model.layers.31.mlp.experts.4.gate_proj", + "model.layers.31.mlp.experts.4.up_proj", + "model.layers.31.mlp.experts.40.down_proj", + "model.layers.31.mlp.experts.40.gate_proj", + "model.layers.31.mlp.experts.40.up_proj", + "model.layers.31.mlp.experts.41.down_proj", + "model.layers.31.mlp.experts.41.gate_proj", + "model.layers.31.mlp.experts.41.up_proj", + "model.layers.31.mlp.experts.42.down_proj", + "model.layers.31.mlp.experts.42.gate_proj", + "model.layers.31.mlp.experts.42.up_proj", + "model.layers.31.mlp.experts.43.down_proj", + "model.layers.31.mlp.experts.43.gate_proj", + "model.layers.31.mlp.experts.43.up_proj", + "model.layers.31.mlp.experts.44.down_proj", + "model.layers.31.mlp.experts.44.gate_proj", + "model.layers.31.mlp.experts.44.up_proj", + "model.layers.31.mlp.experts.45.down_proj", + "model.layers.31.mlp.experts.45.gate_proj", + "model.layers.31.mlp.experts.45.up_proj", + "model.layers.31.mlp.experts.46.down_proj", + "model.layers.31.mlp.experts.46.gate_proj", + "model.layers.31.mlp.experts.46.up_proj", + "model.layers.31.mlp.experts.47.down_proj", + "model.layers.31.mlp.experts.47.gate_proj", + "model.layers.31.mlp.experts.47.up_proj", + "model.layers.31.mlp.experts.48.down_proj", + "model.layers.31.mlp.experts.48.gate_proj", + "model.layers.31.mlp.experts.48.up_proj", + "model.layers.31.mlp.experts.49.down_proj", + "model.layers.31.mlp.experts.49.gate_proj", + "model.layers.31.mlp.experts.49.up_proj", + "model.layers.31.mlp.experts.5.down_proj", + "model.layers.31.mlp.experts.5.gate_proj", + "model.layers.31.mlp.experts.5.up_proj", + "model.layers.31.mlp.experts.50.down_proj", + "model.layers.31.mlp.experts.50.gate_proj", + "model.layers.31.mlp.experts.50.up_proj", + "model.layers.31.mlp.experts.51.down_proj", + "model.layers.31.mlp.experts.51.gate_proj", + "model.layers.31.mlp.experts.51.up_proj", + "model.layers.31.mlp.experts.52.down_proj", + "model.layers.31.mlp.experts.52.gate_proj", + "model.layers.31.mlp.experts.52.up_proj", + "model.layers.31.mlp.experts.53.down_proj", + "model.layers.31.mlp.experts.53.gate_proj", + "model.layers.31.mlp.experts.53.up_proj", + "model.layers.31.mlp.experts.54.down_proj", + "model.layers.31.mlp.experts.54.gate_proj", + "model.layers.31.mlp.experts.54.up_proj", + "model.layers.31.mlp.experts.55.down_proj", + "model.layers.31.mlp.experts.55.gate_proj", + "model.layers.31.mlp.experts.55.up_proj", + "model.layers.31.mlp.experts.56.down_proj", + "model.layers.31.mlp.experts.56.gate_proj", + "model.layers.31.mlp.experts.56.up_proj", + "model.layers.31.mlp.experts.57.down_proj", + "model.layers.31.mlp.experts.57.gate_proj", + "model.layers.31.mlp.experts.57.up_proj", + "model.layers.31.mlp.experts.58.down_proj", + "model.layers.31.mlp.experts.58.gate_proj", + "model.layers.31.mlp.experts.58.up_proj", + "model.layers.31.mlp.experts.59.down_proj", + "model.layers.31.mlp.experts.59.gate_proj", + "model.layers.31.mlp.experts.59.up_proj", + "model.layers.31.mlp.experts.6.down_proj", + "model.layers.31.mlp.experts.6.gate_proj", + "model.layers.31.mlp.experts.6.up_proj", + "model.layers.31.mlp.experts.60.down_proj", + "model.layers.31.mlp.experts.60.gate_proj", + "model.layers.31.mlp.experts.60.up_proj", + "model.layers.31.mlp.experts.61.down_proj", + "model.layers.31.mlp.experts.61.gate_proj", + "model.layers.31.mlp.experts.61.up_proj", + "model.layers.31.mlp.experts.62.down_proj", + "model.layers.31.mlp.experts.62.gate_proj", + "model.layers.31.mlp.experts.62.up_proj", + "model.layers.31.mlp.experts.63.down_proj", + "model.layers.31.mlp.experts.63.gate_proj", + "model.layers.31.mlp.experts.63.up_proj", + "model.layers.31.mlp.experts.64.down_proj", + "model.layers.31.mlp.experts.64.gate_proj", + "model.layers.31.mlp.experts.64.up_proj", + "model.layers.31.mlp.experts.65.down_proj", + "model.layers.31.mlp.experts.65.gate_proj", + "model.layers.31.mlp.experts.65.up_proj", + "model.layers.31.mlp.experts.66.down_proj", + "model.layers.31.mlp.experts.66.gate_proj", + "model.layers.31.mlp.experts.66.up_proj", + "model.layers.31.mlp.experts.67.down_proj", + "model.layers.31.mlp.experts.67.gate_proj", + "model.layers.31.mlp.experts.67.up_proj", + "model.layers.31.mlp.experts.68.down_proj", + "model.layers.31.mlp.experts.68.gate_proj", + "model.layers.31.mlp.experts.68.up_proj", + "model.layers.31.mlp.experts.69.down_proj", + "model.layers.31.mlp.experts.69.gate_proj", + "model.layers.31.mlp.experts.69.up_proj", + "model.layers.31.mlp.experts.7.down_proj", + "model.layers.31.mlp.experts.7.gate_proj", + "model.layers.31.mlp.experts.7.up_proj", + "model.layers.31.mlp.experts.70.down_proj", + "model.layers.31.mlp.experts.70.gate_proj", + "model.layers.31.mlp.experts.70.up_proj", + "model.layers.31.mlp.experts.71.down_proj", + "model.layers.31.mlp.experts.71.gate_proj", + "model.layers.31.mlp.experts.71.up_proj", + "model.layers.31.mlp.experts.72.down_proj", + "model.layers.31.mlp.experts.72.gate_proj", + "model.layers.31.mlp.experts.72.up_proj", + "model.layers.31.mlp.experts.73.down_proj", + "model.layers.31.mlp.experts.73.gate_proj", + "model.layers.31.mlp.experts.73.up_proj", + "model.layers.31.mlp.experts.74.down_proj", + "model.layers.31.mlp.experts.74.gate_proj", + "model.layers.31.mlp.experts.74.up_proj", + "model.layers.31.mlp.experts.75.down_proj", + "model.layers.31.mlp.experts.75.gate_proj", + "model.layers.31.mlp.experts.75.up_proj", + "model.layers.31.mlp.experts.76.down_proj", + "model.layers.31.mlp.experts.76.gate_proj", + "model.layers.31.mlp.experts.76.up_proj", + "model.layers.31.mlp.experts.77.down_proj", + "model.layers.31.mlp.experts.77.gate_proj", + "model.layers.31.mlp.experts.77.up_proj", + "model.layers.31.mlp.experts.78.down_proj", + "model.layers.31.mlp.experts.78.gate_proj", + "model.layers.31.mlp.experts.78.up_proj", + "model.layers.31.mlp.experts.79.down_proj", + "model.layers.31.mlp.experts.79.gate_proj", + "model.layers.31.mlp.experts.79.up_proj", + "model.layers.31.mlp.experts.8.down_proj", + "model.layers.31.mlp.experts.8.gate_proj", + "model.layers.31.mlp.experts.8.up_proj", + "model.layers.31.mlp.experts.80.down_proj", + "model.layers.31.mlp.experts.80.gate_proj", + "model.layers.31.mlp.experts.80.up_proj", + "model.layers.31.mlp.experts.81.down_proj", + "model.layers.31.mlp.experts.81.gate_proj", + "model.layers.31.mlp.experts.81.up_proj", + "model.layers.31.mlp.experts.82.down_proj", + "model.layers.31.mlp.experts.82.gate_proj", + "model.layers.31.mlp.experts.82.up_proj", + "model.layers.31.mlp.experts.83.down_proj", + "model.layers.31.mlp.experts.83.gate_proj", + "model.layers.31.mlp.experts.83.up_proj", + "model.layers.31.mlp.experts.84.down_proj", + "model.layers.31.mlp.experts.84.gate_proj", + "model.layers.31.mlp.experts.84.up_proj", + "model.layers.31.mlp.experts.85.down_proj", + "model.layers.31.mlp.experts.85.gate_proj", + "model.layers.31.mlp.experts.85.up_proj", + "model.layers.31.mlp.experts.86.down_proj", + "model.layers.31.mlp.experts.86.gate_proj", + "model.layers.31.mlp.experts.86.up_proj", + "model.layers.31.mlp.experts.87.down_proj", + "model.layers.31.mlp.experts.87.gate_proj", + "model.layers.31.mlp.experts.87.up_proj", + "model.layers.31.mlp.experts.88.down_proj", + "model.layers.31.mlp.experts.88.gate_proj", + "model.layers.31.mlp.experts.88.up_proj", + "model.layers.31.mlp.experts.89.down_proj", + "model.layers.31.mlp.experts.89.gate_proj", + "model.layers.31.mlp.experts.89.up_proj", + "model.layers.31.mlp.experts.9.down_proj", + "model.layers.31.mlp.experts.9.gate_proj", + "model.layers.31.mlp.experts.9.up_proj", + "model.layers.31.mlp.experts.90.down_proj", + "model.layers.31.mlp.experts.90.gate_proj", + "model.layers.31.mlp.experts.90.up_proj", + "model.layers.31.mlp.experts.91.down_proj", + "model.layers.31.mlp.experts.91.gate_proj", + "model.layers.31.mlp.experts.91.up_proj", + "model.layers.31.mlp.experts.92.down_proj", + "model.layers.31.mlp.experts.92.gate_proj", + "model.layers.31.mlp.experts.92.up_proj", + "model.layers.31.mlp.experts.93.down_proj", + "model.layers.31.mlp.experts.93.gate_proj", + "model.layers.31.mlp.experts.93.up_proj", + "model.layers.31.mlp.experts.94.down_proj", + "model.layers.31.mlp.experts.94.gate_proj", + "model.layers.31.mlp.experts.94.up_proj", + "model.layers.31.mlp.experts.95.down_proj", + "model.layers.31.mlp.experts.95.gate_proj", + "model.layers.31.mlp.experts.95.up_proj", + "model.layers.31.mlp.experts.96.down_proj", + "model.layers.31.mlp.experts.96.gate_proj", + "model.layers.31.mlp.experts.96.up_proj", + "model.layers.31.mlp.experts.97.down_proj", + "model.layers.31.mlp.experts.97.gate_proj", + "model.layers.31.mlp.experts.97.up_proj", + "model.layers.31.mlp.experts.98.down_proj", + "model.layers.31.mlp.experts.98.gate_proj", + "model.layers.31.mlp.experts.98.up_proj", + "model.layers.31.mlp.experts.99.down_proj", + "model.layers.31.mlp.experts.99.gate_proj", + "model.layers.31.mlp.experts.99.up_proj", + "model.layers.32.mlp.experts.0.down_proj", + "model.layers.32.mlp.experts.0.gate_proj", + "model.layers.32.mlp.experts.0.up_proj", + "model.layers.32.mlp.experts.1.down_proj", + "model.layers.32.mlp.experts.1.gate_proj", + "model.layers.32.mlp.experts.1.up_proj", + "model.layers.32.mlp.experts.10.down_proj", + "model.layers.32.mlp.experts.10.gate_proj", + "model.layers.32.mlp.experts.10.up_proj", + "model.layers.32.mlp.experts.100.down_proj", + "model.layers.32.mlp.experts.100.gate_proj", + "model.layers.32.mlp.experts.100.up_proj", + "model.layers.32.mlp.experts.101.down_proj", + "model.layers.32.mlp.experts.101.gate_proj", + "model.layers.32.mlp.experts.101.up_proj", + "model.layers.32.mlp.experts.102.down_proj", + "model.layers.32.mlp.experts.102.gate_proj", + "model.layers.32.mlp.experts.102.up_proj", + "model.layers.32.mlp.experts.103.down_proj", + "model.layers.32.mlp.experts.103.gate_proj", + "model.layers.32.mlp.experts.103.up_proj", + "model.layers.32.mlp.experts.104.down_proj", + "model.layers.32.mlp.experts.104.gate_proj", + "model.layers.32.mlp.experts.104.up_proj", + "model.layers.32.mlp.experts.105.down_proj", + "model.layers.32.mlp.experts.105.gate_proj", + "model.layers.32.mlp.experts.105.up_proj", + "model.layers.32.mlp.experts.106.down_proj", + "model.layers.32.mlp.experts.106.gate_proj", + "model.layers.32.mlp.experts.106.up_proj", + "model.layers.32.mlp.experts.107.down_proj", + "model.layers.32.mlp.experts.107.gate_proj", + "model.layers.32.mlp.experts.107.up_proj", + "model.layers.32.mlp.experts.108.down_proj", + "model.layers.32.mlp.experts.108.gate_proj", + "model.layers.32.mlp.experts.108.up_proj", + "model.layers.32.mlp.experts.109.down_proj", + "model.layers.32.mlp.experts.109.gate_proj", + "model.layers.32.mlp.experts.109.up_proj", + "model.layers.32.mlp.experts.11.down_proj", + "model.layers.32.mlp.experts.11.gate_proj", + "model.layers.32.mlp.experts.11.up_proj", + "model.layers.32.mlp.experts.110.down_proj", + "model.layers.32.mlp.experts.110.gate_proj", + "model.layers.32.mlp.experts.110.up_proj", + "model.layers.32.mlp.experts.111.down_proj", + "model.layers.32.mlp.experts.111.gate_proj", + "model.layers.32.mlp.experts.111.up_proj", + "model.layers.32.mlp.experts.112.down_proj", + "model.layers.32.mlp.experts.112.gate_proj", + "model.layers.32.mlp.experts.112.up_proj", + "model.layers.32.mlp.experts.113.down_proj", + "model.layers.32.mlp.experts.113.gate_proj", + "model.layers.32.mlp.experts.113.up_proj", + "model.layers.32.mlp.experts.114.down_proj", + "model.layers.32.mlp.experts.114.gate_proj", + "model.layers.32.mlp.experts.114.up_proj", + "model.layers.32.mlp.experts.115.down_proj", + "model.layers.32.mlp.experts.115.gate_proj", + "model.layers.32.mlp.experts.115.up_proj", + "model.layers.32.mlp.experts.116.down_proj", + "model.layers.32.mlp.experts.116.gate_proj", + "model.layers.32.mlp.experts.116.up_proj", + "model.layers.32.mlp.experts.117.down_proj", + "model.layers.32.mlp.experts.117.gate_proj", + "model.layers.32.mlp.experts.117.up_proj", + "model.layers.32.mlp.experts.118.down_proj", + "model.layers.32.mlp.experts.118.gate_proj", + "model.layers.32.mlp.experts.118.up_proj", + "model.layers.32.mlp.experts.119.down_proj", + "model.layers.32.mlp.experts.119.gate_proj", + "model.layers.32.mlp.experts.119.up_proj", + "model.layers.32.mlp.experts.12.down_proj", + "model.layers.32.mlp.experts.12.gate_proj", + "model.layers.32.mlp.experts.12.up_proj", + "model.layers.32.mlp.experts.120.down_proj", + "model.layers.32.mlp.experts.120.gate_proj", + "model.layers.32.mlp.experts.120.up_proj", + "model.layers.32.mlp.experts.121.down_proj", + "model.layers.32.mlp.experts.121.gate_proj", + "model.layers.32.mlp.experts.121.up_proj", + "model.layers.32.mlp.experts.122.down_proj", + "model.layers.32.mlp.experts.122.gate_proj", + "model.layers.32.mlp.experts.122.up_proj", + "model.layers.32.mlp.experts.123.down_proj", + "model.layers.32.mlp.experts.123.gate_proj", + "model.layers.32.mlp.experts.123.up_proj", + "model.layers.32.mlp.experts.124.down_proj", + "model.layers.32.mlp.experts.124.gate_proj", + "model.layers.32.mlp.experts.124.up_proj", + "model.layers.32.mlp.experts.125.down_proj", + "model.layers.32.mlp.experts.125.gate_proj", + "model.layers.32.mlp.experts.125.up_proj", + "model.layers.32.mlp.experts.126.down_proj", + "model.layers.32.mlp.experts.126.gate_proj", + "model.layers.32.mlp.experts.126.up_proj", + "model.layers.32.mlp.experts.127.down_proj", + "model.layers.32.mlp.experts.127.gate_proj", + "model.layers.32.mlp.experts.127.up_proj", + "model.layers.32.mlp.experts.128.down_proj", + "model.layers.32.mlp.experts.128.gate_proj", + "model.layers.32.mlp.experts.128.up_proj", + "model.layers.32.mlp.experts.129.down_proj", + "model.layers.32.mlp.experts.129.gate_proj", + "model.layers.32.mlp.experts.129.up_proj", + "model.layers.32.mlp.experts.13.down_proj", + "model.layers.32.mlp.experts.13.gate_proj", + "model.layers.32.mlp.experts.13.up_proj", + "model.layers.32.mlp.experts.130.down_proj", + "model.layers.32.mlp.experts.130.gate_proj", + "model.layers.32.mlp.experts.130.up_proj", + "model.layers.32.mlp.experts.131.down_proj", + "model.layers.32.mlp.experts.131.gate_proj", + "model.layers.32.mlp.experts.131.up_proj", + "model.layers.32.mlp.experts.132.down_proj", + "model.layers.32.mlp.experts.132.gate_proj", + "model.layers.32.mlp.experts.132.up_proj", + "model.layers.32.mlp.experts.133.down_proj", + "model.layers.32.mlp.experts.133.gate_proj", + "model.layers.32.mlp.experts.133.up_proj", + "model.layers.32.mlp.experts.134.down_proj", + "model.layers.32.mlp.experts.134.gate_proj", + "model.layers.32.mlp.experts.134.up_proj", + "model.layers.32.mlp.experts.135.down_proj", + "model.layers.32.mlp.experts.135.gate_proj", + "model.layers.32.mlp.experts.135.up_proj", + "model.layers.32.mlp.experts.136.down_proj", + "model.layers.32.mlp.experts.136.gate_proj", + "model.layers.32.mlp.experts.136.up_proj", + "model.layers.32.mlp.experts.137.down_proj", + "model.layers.32.mlp.experts.137.gate_proj", + "model.layers.32.mlp.experts.137.up_proj", + "model.layers.32.mlp.experts.138.down_proj", + "model.layers.32.mlp.experts.138.gate_proj", + "model.layers.32.mlp.experts.138.up_proj", + "model.layers.32.mlp.experts.139.down_proj", + "model.layers.32.mlp.experts.139.gate_proj", + "model.layers.32.mlp.experts.139.up_proj", + "model.layers.32.mlp.experts.14.down_proj", + "model.layers.32.mlp.experts.14.gate_proj", + "model.layers.32.mlp.experts.14.up_proj", + "model.layers.32.mlp.experts.140.down_proj", + "model.layers.32.mlp.experts.140.gate_proj", + "model.layers.32.mlp.experts.140.up_proj", + "model.layers.32.mlp.experts.141.down_proj", + "model.layers.32.mlp.experts.141.gate_proj", + "model.layers.32.mlp.experts.141.up_proj", + "model.layers.32.mlp.experts.142.down_proj", + "model.layers.32.mlp.experts.142.gate_proj", + "model.layers.32.mlp.experts.142.up_proj", + "model.layers.32.mlp.experts.143.down_proj", + "model.layers.32.mlp.experts.143.gate_proj", + "model.layers.32.mlp.experts.143.up_proj", + "model.layers.32.mlp.experts.144.down_proj", + "model.layers.32.mlp.experts.144.gate_proj", + "model.layers.32.mlp.experts.144.up_proj", + "model.layers.32.mlp.experts.145.down_proj", + "model.layers.32.mlp.experts.145.gate_proj", + "model.layers.32.mlp.experts.145.up_proj", + "model.layers.32.mlp.experts.146.down_proj", + "model.layers.32.mlp.experts.146.gate_proj", + "model.layers.32.mlp.experts.146.up_proj", + "model.layers.32.mlp.experts.147.down_proj", + "model.layers.32.mlp.experts.147.gate_proj", + "model.layers.32.mlp.experts.147.up_proj", + "model.layers.32.mlp.experts.148.down_proj", + "model.layers.32.mlp.experts.148.gate_proj", + "model.layers.32.mlp.experts.148.up_proj", + "model.layers.32.mlp.experts.149.down_proj", + "model.layers.32.mlp.experts.149.gate_proj", + "model.layers.32.mlp.experts.149.up_proj", + "model.layers.32.mlp.experts.15.down_proj", + "model.layers.32.mlp.experts.15.gate_proj", + "model.layers.32.mlp.experts.15.up_proj", + "model.layers.32.mlp.experts.150.down_proj", + "model.layers.32.mlp.experts.150.gate_proj", + "model.layers.32.mlp.experts.150.up_proj", + "model.layers.32.mlp.experts.151.down_proj", + "model.layers.32.mlp.experts.151.gate_proj", + "model.layers.32.mlp.experts.151.up_proj", + "model.layers.32.mlp.experts.152.down_proj", + "model.layers.32.mlp.experts.152.gate_proj", + "model.layers.32.mlp.experts.152.up_proj", + "model.layers.32.mlp.experts.153.down_proj", + "model.layers.32.mlp.experts.153.gate_proj", + "model.layers.32.mlp.experts.153.up_proj", + "model.layers.32.mlp.experts.154.down_proj", + "model.layers.32.mlp.experts.154.gate_proj", + "model.layers.32.mlp.experts.154.up_proj", + "model.layers.32.mlp.experts.155.down_proj", + "model.layers.32.mlp.experts.155.gate_proj", + "model.layers.32.mlp.experts.155.up_proj", + "model.layers.32.mlp.experts.156.down_proj", + "model.layers.32.mlp.experts.156.gate_proj", + "model.layers.32.mlp.experts.156.up_proj", + "model.layers.32.mlp.experts.157.down_proj", + "model.layers.32.mlp.experts.157.gate_proj", + "model.layers.32.mlp.experts.157.up_proj", + "model.layers.32.mlp.experts.158.down_proj", + "model.layers.32.mlp.experts.158.gate_proj", + "model.layers.32.mlp.experts.158.up_proj", + "model.layers.32.mlp.experts.159.down_proj", + "model.layers.32.mlp.experts.159.gate_proj", + "model.layers.32.mlp.experts.159.up_proj", + "model.layers.32.mlp.experts.16.down_proj", + "model.layers.32.mlp.experts.16.gate_proj", + "model.layers.32.mlp.experts.16.up_proj", + "model.layers.32.mlp.experts.160.down_proj", + "model.layers.32.mlp.experts.160.gate_proj", + "model.layers.32.mlp.experts.160.up_proj", + "model.layers.32.mlp.experts.161.down_proj", + "model.layers.32.mlp.experts.161.gate_proj", + "model.layers.32.mlp.experts.161.up_proj", + "model.layers.32.mlp.experts.162.down_proj", + "model.layers.32.mlp.experts.162.gate_proj", + "model.layers.32.mlp.experts.162.up_proj", + "model.layers.32.mlp.experts.163.down_proj", + "model.layers.32.mlp.experts.163.gate_proj", + "model.layers.32.mlp.experts.163.up_proj", + "model.layers.32.mlp.experts.164.down_proj", + "model.layers.32.mlp.experts.164.gate_proj", + "model.layers.32.mlp.experts.164.up_proj", + "model.layers.32.mlp.experts.165.down_proj", + "model.layers.32.mlp.experts.165.gate_proj", + "model.layers.32.mlp.experts.165.up_proj", + "model.layers.32.mlp.experts.166.down_proj", + "model.layers.32.mlp.experts.166.gate_proj", + "model.layers.32.mlp.experts.166.up_proj", + "model.layers.32.mlp.experts.167.down_proj", + "model.layers.32.mlp.experts.167.gate_proj", + "model.layers.32.mlp.experts.167.up_proj", + "model.layers.32.mlp.experts.168.down_proj", + "model.layers.32.mlp.experts.168.gate_proj", + "model.layers.32.mlp.experts.168.up_proj", + "model.layers.32.mlp.experts.169.down_proj", + "model.layers.32.mlp.experts.169.gate_proj", + "model.layers.32.mlp.experts.169.up_proj", + "model.layers.32.mlp.experts.17.down_proj", + "model.layers.32.mlp.experts.17.gate_proj", + "model.layers.32.mlp.experts.17.up_proj", + "model.layers.32.mlp.experts.170.down_proj", + "model.layers.32.mlp.experts.170.gate_proj", + "model.layers.32.mlp.experts.170.up_proj", + "model.layers.32.mlp.experts.171.down_proj", + "model.layers.32.mlp.experts.171.gate_proj", + "model.layers.32.mlp.experts.171.up_proj", + "model.layers.32.mlp.experts.172.down_proj", + "model.layers.32.mlp.experts.172.gate_proj", + "model.layers.32.mlp.experts.172.up_proj", + "model.layers.32.mlp.experts.173.down_proj", + "model.layers.32.mlp.experts.173.gate_proj", + "model.layers.32.mlp.experts.173.up_proj", + "model.layers.32.mlp.experts.174.down_proj", + "model.layers.32.mlp.experts.174.gate_proj", + "model.layers.32.mlp.experts.174.up_proj", + "model.layers.32.mlp.experts.175.down_proj", + "model.layers.32.mlp.experts.175.gate_proj", + "model.layers.32.mlp.experts.175.up_proj", + "model.layers.32.mlp.experts.176.down_proj", + "model.layers.32.mlp.experts.176.gate_proj", + "model.layers.32.mlp.experts.176.up_proj", + "model.layers.32.mlp.experts.177.down_proj", + "model.layers.32.mlp.experts.177.gate_proj", + "model.layers.32.mlp.experts.177.up_proj", + "model.layers.32.mlp.experts.178.down_proj", + "model.layers.32.mlp.experts.178.gate_proj", + "model.layers.32.mlp.experts.178.up_proj", + "model.layers.32.mlp.experts.179.down_proj", + "model.layers.32.mlp.experts.179.gate_proj", + "model.layers.32.mlp.experts.179.up_proj", + "model.layers.32.mlp.experts.18.down_proj", + "model.layers.32.mlp.experts.18.gate_proj", + "model.layers.32.mlp.experts.18.up_proj", + "model.layers.32.mlp.experts.180.down_proj", + "model.layers.32.mlp.experts.180.gate_proj", + "model.layers.32.mlp.experts.180.up_proj", + "model.layers.32.mlp.experts.181.down_proj", + "model.layers.32.mlp.experts.181.gate_proj", + "model.layers.32.mlp.experts.181.up_proj", + "model.layers.32.mlp.experts.182.down_proj", + "model.layers.32.mlp.experts.182.gate_proj", + "model.layers.32.mlp.experts.182.up_proj", + "model.layers.32.mlp.experts.183.down_proj", + "model.layers.32.mlp.experts.183.gate_proj", + "model.layers.32.mlp.experts.183.up_proj", + "model.layers.32.mlp.experts.184.down_proj", + "model.layers.32.mlp.experts.184.gate_proj", + "model.layers.32.mlp.experts.184.up_proj", + "model.layers.32.mlp.experts.185.down_proj", + "model.layers.32.mlp.experts.185.gate_proj", + "model.layers.32.mlp.experts.185.up_proj", + "model.layers.32.mlp.experts.186.down_proj", + "model.layers.32.mlp.experts.186.gate_proj", + "model.layers.32.mlp.experts.186.up_proj", + "model.layers.32.mlp.experts.187.down_proj", + "model.layers.32.mlp.experts.187.gate_proj", + "model.layers.32.mlp.experts.187.up_proj", + "model.layers.32.mlp.experts.188.down_proj", + "model.layers.32.mlp.experts.188.gate_proj", + "model.layers.32.mlp.experts.188.up_proj", + "model.layers.32.mlp.experts.189.down_proj", + "model.layers.32.mlp.experts.189.gate_proj", + "model.layers.32.mlp.experts.189.up_proj", + "model.layers.32.mlp.experts.19.down_proj", + "model.layers.32.mlp.experts.19.gate_proj", + "model.layers.32.mlp.experts.19.up_proj", + "model.layers.32.mlp.experts.190.down_proj", + "model.layers.32.mlp.experts.190.gate_proj", + "model.layers.32.mlp.experts.190.up_proj", + "model.layers.32.mlp.experts.191.down_proj", + "model.layers.32.mlp.experts.191.gate_proj", + "model.layers.32.mlp.experts.191.up_proj", + "model.layers.32.mlp.experts.192.down_proj", + "model.layers.32.mlp.experts.192.gate_proj", + "model.layers.32.mlp.experts.192.up_proj", + "model.layers.32.mlp.experts.193.down_proj", + "model.layers.32.mlp.experts.193.gate_proj", + "model.layers.32.mlp.experts.193.up_proj", + "model.layers.32.mlp.experts.194.down_proj", + "model.layers.32.mlp.experts.194.gate_proj", + "model.layers.32.mlp.experts.194.up_proj", + "model.layers.32.mlp.experts.195.down_proj", + "model.layers.32.mlp.experts.195.gate_proj", + "model.layers.32.mlp.experts.195.up_proj", + "model.layers.32.mlp.experts.196.down_proj", + "model.layers.32.mlp.experts.196.gate_proj", + "model.layers.32.mlp.experts.196.up_proj", + "model.layers.32.mlp.experts.197.down_proj", + "model.layers.32.mlp.experts.197.gate_proj", + "model.layers.32.mlp.experts.197.up_proj", + "model.layers.32.mlp.experts.198.down_proj", + "model.layers.32.mlp.experts.198.gate_proj", + "model.layers.32.mlp.experts.198.up_proj", + "model.layers.32.mlp.experts.199.down_proj", + "model.layers.32.mlp.experts.199.gate_proj", + "model.layers.32.mlp.experts.199.up_proj", + "model.layers.32.mlp.experts.2.down_proj", + "model.layers.32.mlp.experts.2.gate_proj", + "model.layers.32.mlp.experts.2.up_proj", + "model.layers.32.mlp.experts.20.down_proj", + "model.layers.32.mlp.experts.20.gate_proj", + "model.layers.32.mlp.experts.20.up_proj", + "model.layers.32.mlp.experts.200.down_proj", + "model.layers.32.mlp.experts.200.gate_proj", + "model.layers.32.mlp.experts.200.up_proj", + "model.layers.32.mlp.experts.201.down_proj", + "model.layers.32.mlp.experts.201.gate_proj", + "model.layers.32.mlp.experts.201.up_proj", + "model.layers.32.mlp.experts.202.down_proj", + "model.layers.32.mlp.experts.202.gate_proj", + "model.layers.32.mlp.experts.202.up_proj", + "model.layers.32.mlp.experts.203.down_proj", + "model.layers.32.mlp.experts.203.gate_proj", + "model.layers.32.mlp.experts.203.up_proj", + "model.layers.32.mlp.experts.204.down_proj", + "model.layers.32.mlp.experts.204.gate_proj", + "model.layers.32.mlp.experts.204.up_proj", + "model.layers.32.mlp.experts.205.down_proj", + "model.layers.32.mlp.experts.205.gate_proj", + "model.layers.32.mlp.experts.205.up_proj", + "model.layers.32.mlp.experts.206.down_proj", + "model.layers.32.mlp.experts.206.gate_proj", + "model.layers.32.mlp.experts.206.up_proj", + "model.layers.32.mlp.experts.207.down_proj", + "model.layers.32.mlp.experts.207.gate_proj", + "model.layers.32.mlp.experts.207.up_proj", + "model.layers.32.mlp.experts.208.down_proj", + "model.layers.32.mlp.experts.208.gate_proj", + "model.layers.32.mlp.experts.208.up_proj", + "model.layers.32.mlp.experts.209.down_proj", + "model.layers.32.mlp.experts.209.gate_proj", + "model.layers.32.mlp.experts.209.up_proj", + "model.layers.32.mlp.experts.21.down_proj", + "model.layers.32.mlp.experts.21.gate_proj", + "model.layers.32.mlp.experts.21.up_proj", + "model.layers.32.mlp.experts.210.down_proj", + "model.layers.32.mlp.experts.210.gate_proj", + "model.layers.32.mlp.experts.210.up_proj", + "model.layers.32.mlp.experts.211.down_proj", + "model.layers.32.mlp.experts.211.gate_proj", + "model.layers.32.mlp.experts.211.up_proj", + "model.layers.32.mlp.experts.212.down_proj", + "model.layers.32.mlp.experts.212.gate_proj", + "model.layers.32.mlp.experts.212.up_proj", + "model.layers.32.mlp.experts.213.down_proj", + "model.layers.32.mlp.experts.213.gate_proj", + "model.layers.32.mlp.experts.213.up_proj", + "model.layers.32.mlp.experts.214.down_proj", + "model.layers.32.mlp.experts.214.gate_proj", + "model.layers.32.mlp.experts.214.up_proj", + "model.layers.32.mlp.experts.215.down_proj", + "model.layers.32.mlp.experts.215.gate_proj", + "model.layers.32.mlp.experts.215.up_proj", + "model.layers.32.mlp.experts.216.down_proj", + "model.layers.32.mlp.experts.216.gate_proj", + "model.layers.32.mlp.experts.216.up_proj", + "model.layers.32.mlp.experts.217.down_proj", + "model.layers.32.mlp.experts.217.gate_proj", + "model.layers.32.mlp.experts.217.up_proj", + "model.layers.32.mlp.experts.218.down_proj", + "model.layers.32.mlp.experts.218.gate_proj", + "model.layers.32.mlp.experts.218.up_proj", + "model.layers.32.mlp.experts.219.down_proj", + "model.layers.32.mlp.experts.219.gate_proj", + "model.layers.32.mlp.experts.219.up_proj", + "model.layers.32.mlp.experts.22.down_proj", + "model.layers.32.mlp.experts.22.gate_proj", + "model.layers.32.mlp.experts.22.up_proj", + "model.layers.32.mlp.experts.220.down_proj", + "model.layers.32.mlp.experts.220.gate_proj", + "model.layers.32.mlp.experts.220.up_proj", + "model.layers.32.mlp.experts.221.down_proj", + "model.layers.32.mlp.experts.221.gate_proj", + "model.layers.32.mlp.experts.221.up_proj", + "model.layers.32.mlp.experts.222.down_proj", + "model.layers.32.mlp.experts.222.gate_proj", + "model.layers.32.mlp.experts.222.up_proj", + "model.layers.32.mlp.experts.223.down_proj", + "model.layers.32.mlp.experts.223.gate_proj", + "model.layers.32.mlp.experts.223.up_proj", + "model.layers.32.mlp.experts.224.down_proj", + "model.layers.32.mlp.experts.224.gate_proj", + "model.layers.32.mlp.experts.224.up_proj", + "model.layers.32.mlp.experts.225.down_proj", + "model.layers.32.mlp.experts.225.gate_proj", + "model.layers.32.mlp.experts.225.up_proj", + "model.layers.32.mlp.experts.226.down_proj", + "model.layers.32.mlp.experts.226.gate_proj", + "model.layers.32.mlp.experts.226.up_proj", + "model.layers.32.mlp.experts.227.down_proj", + "model.layers.32.mlp.experts.227.gate_proj", + "model.layers.32.mlp.experts.227.up_proj", + "model.layers.32.mlp.experts.228.down_proj", + "model.layers.32.mlp.experts.228.gate_proj", + "model.layers.32.mlp.experts.228.up_proj", + "model.layers.32.mlp.experts.229.down_proj", + "model.layers.32.mlp.experts.229.gate_proj", + "model.layers.32.mlp.experts.229.up_proj", + "model.layers.32.mlp.experts.23.down_proj", + "model.layers.32.mlp.experts.23.gate_proj", + "model.layers.32.mlp.experts.23.up_proj", + "model.layers.32.mlp.experts.230.down_proj", + "model.layers.32.mlp.experts.230.gate_proj", + "model.layers.32.mlp.experts.230.up_proj", + "model.layers.32.mlp.experts.231.down_proj", + "model.layers.32.mlp.experts.231.gate_proj", + "model.layers.32.mlp.experts.231.up_proj", + "model.layers.32.mlp.experts.232.down_proj", + "model.layers.32.mlp.experts.232.gate_proj", + "model.layers.32.mlp.experts.232.up_proj", + "model.layers.32.mlp.experts.233.down_proj", + "model.layers.32.mlp.experts.233.gate_proj", + "model.layers.32.mlp.experts.233.up_proj", + "model.layers.32.mlp.experts.234.down_proj", + "model.layers.32.mlp.experts.234.gate_proj", + "model.layers.32.mlp.experts.234.up_proj", + "model.layers.32.mlp.experts.235.down_proj", + "model.layers.32.mlp.experts.235.gate_proj", + "model.layers.32.mlp.experts.235.up_proj", + "model.layers.32.mlp.experts.236.down_proj", + "model.layers.32.mlp.experts.236.gate_proj", + "model.layers.32.mlp.experts.236.up_proj", + "model.layers.32.mlp.experts.237.down_proj", + "model.layers.32.mlp.experts.237.gate_proj", + "model.layers.32.mlp.experts.237.up_proj", + "model.layers.32.mlp.experts.238.down_proj", + "model.layers.32.mlp.experts.238.gate_proj", + "model.layers.32.mlp.experts.238.up_proj", + "model.layers.32.mlp.experts.239.down_proj", + "model.layers.32.mlp.experts.239.gate_proj", + "model.layers.32.mlp.experts.239.up_proj", + "model.layers.32.mlp.experts.24.down_proj", + "model.layers.32.mlp.experts.24.gate_proj", + "model.layers.32.mlp.experts.24.up_proj", + "model.layers.32.mlp.experts.240.down_proj", + "model.layers.32.mlp.experts.240.gate_proj", + "model.layers.32.mlp.experts.240.up_proj", + "model.layers.32.mlp.experts.241.down_proj", + "model.layers.32.mlp.experts.241.gate_proj", + "model.layers.32.mlp.experts.241.up_proj", + "model.layers.32.mlp.experts.242.down_proj", + "model.layers.32.mlp.experts.242.gate_proj", + "model.layers.32.mlp.experts.242.up_proj", + "model.layers.32.mlp.experts.243.down_proj", + "model.layers.32.mlp.experts.243.gate_proj", + "model.layers.32.mlp.experts.243.up_proj", + "model.layers.32.mlp.experts.244.down_proj", + "model.layers.32.mlp.experts.244.gate_proj", + "model.layers.32.mlp.experts.244.up_proj", + "model.layers.32.mlp.experts.245.down_proj", + "model.layers.32.mlp.experts.245.gate_proj", + "model.layers.32.mlp.experts.245.up_proj", + "model.layers.32.mlp.experts.246.down_proj", + "model.layers.32.mlp.experts.246.gate_proj", + "model.layers.32.mlp.experts.246.up_proj", + "model.layers.32.mlp.experts.247.down_proj", + "model.layers.32.mlp.experts.247.gate_proj", + "model.layers.32.mlp.experts.247.up_proj", + "model.layers.32.mlp.experts.248.down_proj", + "model.layers.32.mlp.experts.248.gate_proj", + "model.layers.32.mlp.experts.248.up_proj", + "model.layers.32.mlp.experts.249.down_proj", + "model.layers.32.mlp.experts.249.gate_proj", + "model.layers.32.mlp.experts.249.up_proj", + "model.layers.32.mlp.experts.25.down_proj", + "model.layers.32.mlp.experts.25.gate_proj", + "model.layers.32.mlp.experts.25.up_proj", + "model.layers.32.mlp.experts.250.down_proj", + "model.layers.32.mlp.experts.250.gate_proj", + "model.layers.32.mlp.experts.250.up_proj", + "model.layers.32.mlp.experts.251.down_proj", + "model.layers.32.mlp.experts.251.gate_proj", + "model.layers.32.mlp.experts.251.up_proj", + "model.layers.32.mlp.experts.252.down_proj", + "model.layers.32.mlp.experts.252.gate_proj", + "model.layers.32.mlp.experts.252.up_proj", + "model.layers.32.mlp.experts.253.down_proj", + "model.layers.32.mlp.experts.253.gate_proj", + "model.layers.32.mlp.experts.253.up_proj", + "model.layers.32.mlp.experts.254.down_proj", + "model.layers.32.mlp.experts.254.gate_proj", + "model.layers.32.mlp.experts.254.up_proj", + "model.layers.32.mlp.experts.255.down_proj", + "model.layers.32.mlp.experts.255.gate_proj", + "model.layers.32.mlp.experts.255.up_proj", + "model.layers.32.mlp.experts.26.down_proj", + "model.layers.32.mlp.experts.26.gate_proj", + "model.layers.32.mlp.experts.26.up_proj", + "model.layers.32.mlp.experts.27.down_proj", + "model.layers.32.mlp.experts.27.gate_proj", + "model.layers.32.mlp.experts.27.up_proj", + "model.layers.32.mlp.experts.28.down_proj", + "model.layers.32.mlp.experts.28.gate_proj", + "model.layers.32.mlp.experts.28.up_proj", + "model.layers.32.mlp.experts.29.down_proj", + "model.layers.32.mlp.experts.29.gate_proj", + "model.layers.32.mlp.experts.29.up_proj", + "model.layers.32.mlp.experts.3.down_proj", + "model.layers.32.mlp.experts.3.gate_proj", + "model.layers.32.mlp.experts.3.up_proj", + "model.layers.32.mlp.experts.30.down_proj", + "model.layers.32.mlp.experts.30.gate_proj", + "model.layers.32.mlp.experts.30.up_proj", + "model.layers.32.mlp.experts.31.down_proj", + "model.layers.32.mlp.experts.31.gate_proj", + "model.layers.32.mlp.experts.31.up_proj", + "model.layers.32.mlp.experts.32.down_proj", + "model.layers.32.mlp.experts.32.gate_proj", + "model.layers.32.mlp.experts.32.up_proj", + "model.layers.32.mlp.experts.33.down_proj", + "model.layers.32.mlp.experts.33.gate_proj", + "model.layers.32.mlp.experts.33.up_proj", + "model.layers.32.mlp.experts.34.down_proj", + "model.layers.32.mlp.experts.34.gate_proj", + "model.layers.32.mlp.experts.34.up_proj", + "model.layers.32.mlp.experts.35.down_proj", + "model.layers.32.mlp.experts.35.gate_proj", + "model.layers.32.mlp.experts.35.up_proj", + "model.layers.32.mlp.experts.36.down_proj", + "model.layers.32.mlp.experts.36.gate_proj", + "model.layers.32.mlp.experts.36.up_proj", + "model.layers.32.mlp.experts.37.down_proj", + "model.layers.32.mlp.experts.37.gate_proj", + "model.layers.32.mlp.experts.37.up_proj", + "model.layers.32.mlp.experts.38.down_proj", + "model.layers.32.mlp.experts.38.gate_proj", + "model.layers.32.mlp.experts.38.up_proj", + "model.layers.32.mlp.experts.39.down_proj", + "model.layers.32.mlp.experts.39.gate_proj", + "model.layers.32.mlp.experts.39.up_proj", + "model.layers.32.mlp.experts.4.down_proj", + "model.layers.32.mlp.experts.4.gate_proj", + "model.layers.32.mlp.experts.4.up_proj", + "model.layers.32.mlp.experts.40.down_proj", + "model.layers.32.mlp.experts.40.gate_proj", + "model.layers.32.mlp.experts.40.up_proj", + "model.layers.32.mlp.experts.41.down_proj", + "model.layers.32.mlp.experts.41.gate_proj", + "model.layers.32.mlp.experts.41.up_proj", + "model.layers.32.mlp.experts.42.down_proj", + "model.layers.32.mlp.experts.42.gate_proj", + "model.layers.32.mlp.experts.42.up_proj", + "model.layers.32.mlp.experts.43.down_proj", + "model.layers.32.mlp.experts.43.gate_proj", + "model.layers.32.mlp.experts.43.up_proj", + "model.layers.32.mlp.experts.44.down_proj", + "model.layers.32.mlp.experts.44.gate_proj", + "model.layers.32.mlp.experts.44.up_proj", + "model.layers.32.mlp.experts.45.down_proj", + "model.layers.32.mlp.experts.45.gate_proj", + "model.layers.32.mlp.experts.45.up_proj", + "model.layers.32.mlp.experts.46.down_proj", + "model.layers.32.mlp.experts.46.gate_proj", + "model.layers.32.mlp.experts.46.up_proj", + "model.layers.32.mlp.experts.47.down_proj", + "model.layers.32.mlp.experts.47.gate_proj", + "model.layers.32.mlp.experts.47.up_proj", + "model.layers.32.mlp.experts.48.down_proj", + "model.layers.32.mlp.experts.48.gate_proj", + "model.layers.32.mlp.experts.48.up_proj", + "model.layers.32.mlp.experts.49.down_proj", + "model.layers.32.mlp.experts.49.gate_proj", + "model.layers.32.mlp.experts.49.up_proj", + "model.layers.32.mlp.experts.5.down_proj", + "model.layers.32.mlp.experts.5.gate_proj", + "model.layers.32.mlp.experts.5.up_proj", + "model.layers.32.mlp.experts.50.down_proj", + "model.layers.32.mlp.experts.50.gate_proj", + "model.layers.32.mlp.experts.50.up_proj", + "model.layers.32.mlp.experts.51.down_proj", + "model.layers.32.mlp.experts.51.gate_proj", + "model.layers.32.mlp.experts.51.up_proj", + "model.layers.32.mlp.experts.52.down_proj", + "model.layers.32.mlp.experts.52.gate_proj", + "model.layers.32.mlp.experts.52.up_proj", + "model.layers.32.mlp.experts.53.down_proj", + "model.layers.32.mlp.experts.53.gate_proj", + "model.layers.32.mlp.experts.53.up_proj", + "model.layers.32.mlp.experts.54.down_proj", + "model.layers.32.mlp.experts.54.gate_proj", + "model.layers.32.mlp.experts.54.up_proj", + "model.layers.32.mlp.experts.55.down_proj", + "model.layers.32.mlp.experts.55.gate_proj", + "model.layers.32.mlp.experts.55.up_proj", + "model.layers.32.mlp.experts.56.down_proj", + "model.layers.32.mlp.experts.56.gate_proj", + "model.layers.32.mlp.experts.56.up_proj", + "model.layers.32.mlp.experts.57.down_proj", + "model.layers.32.mlp.experts.57.gate_proj", + "model.layers.32.mlp.experts.57.up_proj", + "model.layers.32.mlp.experts.58.down_proj", + "model.layers.32.mlp.experts.58.gate_proj", + "model.layers.32.mlp.experts.58.up_proj", + "model.layers.32.mlp.experts.59.down_proj", + "model.layers.32.mlp.experts.59.gate_proj", + "model.layers.32.mlp.experts.59.up_proj", + "model.layers.32.mlp.experts.6.down_proj", + "model.layers.32.mlp.experts.6.gate_proj", + "model.layers.32.mlp.experts.6.up_proj", + "model.layers.32.mlp.experts.60.down_proj", + "model.layers.32.mlp.experts.60.gate_proj", + "model.layers.32.mlp.experts.60.up_proj", + "model.layers.32.mlp.experts.61.down_proj", + "model.layers.32.mlp.experts.61.gate_proj", + "model.layers.32.mlp.experts.61.up_proj", + "model.layers.32.mlp.experts.62.down_proj", + "model.layers.32.mlp.experts.62.gate_proj", + "model.layers.32.mlp.experts.62.up_proj", + "model.layers.32.mlp.experts.63.down_proj", + "model.layers.32.mlp.experts.63.gate_proj", + "model.layers.32.mlp.experts.63.up_proj", + "model.layers.32.mlp.experts.64.down_proj", + "model.layers.32.mlp.experts.64.gate_proj", + "model.layers.32.mlp.experts.64.up_proj", + "model.layers.32.mlp.experts.65.down_proj", + "model.layers.32.mlp.experts.65.gate_proj", + "model.layers.32.mlp.experts.65.up_proj", + "model.layers.32.mlp.experts.66.down_proj", + "model.layers.32.mlp.experts.66.gate_proj", + "model.layers.32.mlp.experts.66.up_proj", + "model.layers.32.mlp.experts.67.down_proj", + "model.layers.32.mlp.experts.67.gate_proj", + "model.layers.32.mlp.experts.67.up_proj", + "model.layers.32.mlp.experts.68.down_proj", + "model.layers.32.mlp.experts.68.gate_proj", + "model.layers.32.mlp.experts.68.up_proj", + "model.layers.32.mlp.experts.69.down_proj", + "model.layers.32.mlp.experts.69.gate_proj", + "model.layers.32.mlp.experts.69.up_proj", + "model.layers.32.mlp.experts.7.down_proj", + "model.layers.32.mlp.experts.7.gate_proj", + "model.layers.32.mlp.experts.7.up_proj", + "model.layers.32.mlp.experts.70.down_proj", + "model.layers.32.mlp.experts.70.gate_proj", + "model.layers.32.mlp.experts.70.up_proj", + "model.layers.32.mlp.experts.71.down_proj", + "model.layers.32.mlp.experts.71.gate_proj", + "model.layers.32.mlp.experts.71.up_proj", + "model.layers.32.mlp.experts.72.down_proj", + "model.layers.32.mlp.experts.72.gate_proj", + "model.layers.32.mlp.experts.72.up_proj", + "model.layers.32.mlp.experts.73.down_proj", + "model.layers.32.mlp.experts.73.gate_proj", + "model.layers.32.mlp.experts.73.up_proj", + "model.layers.32.mlp.experts.74.down_proj", + "model.layers.32.mlp.experts.74.gate_proj", + "model.layers.32.mlp.experts.74.up_proj", + "model.layers.32.mlp.experts.75.down_proj", + "model.layers.32.mlp.experts.75.gate_proj", + "model.layers.32.mlp.experts.75.up_proj", + "model.layers.32.mlp.experts.76.down_proj", + "model.layers.32.mlp.experts.76.gate_proj", + "model.layers.32.mlp.experts.76.up_proj", + "model.layers.32.mlp.experts.77.down_proj", + "model.layers.32.mlp.experts.77.gate_proj", + "model.layers.32.mlp.experts.77.up_proj", + "model.layers.32.mlp.experts.78.down_proj", + "model.layers.32.mlp.experts.78.gate_proj", + "model.layers.32.mlp.experts.78.up_proj", + "model.layers.32.mlp.experts.79.down_proj", + "model.layers.32.mlp.experts.79.gate_proj", + "model.layers.32.mlp.experts.79.up_proj", + "model.layers.32.mlp.experts.8.down_proj", + "model.layers.32.mlp.experts.8.gate_proj", + "model.layers.32.mlp.experts.8.up_proj", + "model.layers.32.mlp.experts.80.down_proj", + "model.layers.32.mlp.experts.80.gate_proj", + "model.layers.32.mlp.experts.80.up_proj", + "model.layers.32.mlp.experts.81.down_proj", + "model.layers.32.mlp.experts.81.gate_proj", + "model.layers.32.mlp.experts.81.up_proj", + "model.layers.32.mlp.experts.82.down_proj", + "model.layers.32.mlp.experts.82.gate_proj", + "model.layers.32.mlp.experts.82.up_proj", + "model.layers.32.mlp.experts.83.down_proj", + "model.layers.32.mlp.experts.83.gate_proj", + "model.layers.32.mlp.experts.83.up_proj", + "model.layers.32.mlp.experts.84.down_proj", + "model.layers.32.mlp.experts.84.gate_proj", + "model.layers.32.mlp.experts.84.up_proj", + "model.layers.32.mlp.experts.85.down_proj", + "model.layers.32.mlp.experts.85.gate_proj", + "model.layers.32.mlp.experts.85.up_proj", + "model.layers.32.mlp.experts.86.down_proj", + "model.layers.32.mlp.experts.86.gate_proj", + "model.layers.32.mlp.experts.86.up_proj", + "model.layers.32.mlp.experts.87.down_proj", + "model.layers.32.mlp.experts.87.gate_proj", + "model.layers.32.mlp.experts.87.up_proj", + "model.layers.32.mlp.experts.88.down_proj", + "model.layers.32.mlp.experts.88.gate_proj", + "model.layers.32.mlp.experts.88.up_proj", + "model.layers.32.mlp.experts.89.down_proj", + "model.layers.32.mlp.experts.89.gate_proj", + "model.layers.32.mlp.experts.89.up_proj", + "model.layers.32.mlp.experts.9.down_proj", + "model.layers.32.mlp.experts.9.gate_proj", + "model.layers.32.mlp.experts.9.up_proj", + "model.layers.32.mlp.experts.90.down_proj", + "model.layers.32.mlp.experts.90.gate_proj", + "model.layers.32.mlp.experts.90.up_proj", + "model.layers.32.mlp.experts.91.down_proj", + "model.layers.32.mlp.experts.91.gate_proj", + "model.layers.32.mlp.experts.91.up_proj", + "model.layers.32.mlp.experts.92.down_proj", + "model.layers.32.mlp.experts.92.gate_proj", + "model.layers.32.mlp.experts.92.up_proj", + "model.layers.32.mlp.experts.93.down_proj", + "model.layers.32.mlp.experts.93.gate_proj", + "model.layers.32.mlp.experts.93.up_proj", + "model.layers.32.mlp.experts.94.down_proj", + "model.layers.32.mlp.experts.94.gate_proj", + "model.layers.32.mlp.experts.94.up_proj", + "model.layers.32.mlp.experts.95.down_proj", + "model.layers.32.mlp.experts.95.gate_proj", + "model.layers.32.mlp.experts.95.up_proj", + "model.layers.32.mlp.experts.96.down_proj", + "model.layers.32.mlp.experts.96.gate_proj", + "model.layers.32.mlp.experts.96.up_proj", + "model.layers.32.mlp.experts.97.down_proj", + "model.layers.32.mlp.experts.97.gate_proj", + "model.layers.32.mlp.experts.97.up_proj", + "model.layers.32.mlp.experts.98.down_proj", + "model.layers.32.mlp.experts.98.gate_proj", + "model.layers.32.mlp.experts.98.up_proj", + "model.layers.32.mlp.experts.99.down_proj", + "model.layers.32.mlp.experts.99.gate_proj", + "model.layers.32.mlp.experts.99.up_proj", + "model.layers.33.mlp.experts.0.down_proj", + "model.layers.33.mlp.experts.0.gate_proj", + "model.layers.33.mlp.experts.0.up_proj", + "model.layers.33.mlp.experts.1.down_proj", + "model.layers.33.mlp.experts.1.gate_proj", + "model.layers.33.mlp.experts.1.up_proj", + "model.layers.33.mlp.experts.10.down_proj", + "model.layers.33.mlp.experts.10.gate_proj", + "model.layers.33.mlp.experts.10.up_proj", + "model.layers.33.mlp.experts.100.down_proj", + "model.layers.33.mlp.experts.100.gate_proj", + "model.layers.33.mlp.experts.100.up_proj", + "model.layers.33.mlp.experts.101.down_proj", + "model.layers.33.mlp.experts.101.gate_proj", + "model.layers.33.mlp.experts.101.up_proj", + "model.layers.33.mlp.experts.102.down_proj", + "model.layers.33.mlp.experts.102.gate_proj", + "model.layers.33.mlp.experts.102.up_proj", + "model.layers.33.mlp.experts.103.down_proj", + "model.layers.33.mlp.experts.103.gate_proj", + "model.layers.33.mlp.experts.103.up_proj", + "model.layers.33.mlp.experts.104.down_proj", + "model.layers.33.mlp.experts.104.gate_proj", + "model.layers.33.mlp.experts.104.up_proj", + "model.layers.33.mlp.experts.105.down_proj", + "model.layers.33.mlp.experts.105.gate_proj", + "model.layers.33.mlp.experts.105.up_proj", + "model.layers.33.mlp.experts.106.down_proj", + "model.layers.33.mlp.experts.106.gate_proj", + "model.layers.33.mlp.experts.106.up_proj", + "model.layers.33.mlp.experts.107.down_proj", + "model.layers.33.mlp.experts.107.gate_proj", + "model.layers.33.mlp.experts.107.up_proj", + "model.layers.33.mlp.experts.108.down_proj", + "model.layers.33.mlp.experts.108.gate_proj", + "model.layers.33.mlp.experts.108.up_proj", + "model.layers.33.mlp.experts.109.down_proj", + "model.layers.33.mlp.experts.109.gate_proj", + "model.layers.33.mlp.experts.109.up_proj", + "model.layers.33.mlp.experts.11.down_proj", + "model.layers.33.mlp.experts.11.gate_proj", + "model.layers.33.mlp.experts.11.up_proj", + "model.layers.33.mlp.experts.110.down_proj", + "model.layers.33.mlp.experts.110.gate_proj", + "model.layers.33.mlp.experts.110.up_proj", + "model.layers.33.mlp.experts.111.down_proj", + "model.layers.33.mlp.experts.111.gate_proj", + "model.layers.33.mlp.experts.111.up_proj", + "model.layers.33.mlp.experts.112.down_proj", + "model.layers.33.mlp.experts.112.gate_proj", + "model.layers.33.mlp.experts.112.up_proj", + "model.layers.33.mlp.experts.113.down_proj", + "model.layers.33.mlp.experts.113.gate_proj", + "model.layers.33.mlp.experts.113.up_proj", + "model.layers.33.mlp.experts.114.down_proj", + "model.layers.33.mlp.experts.114.gate_proj", + "model.layers.33.mlp.experts.114.up_proj", + "model.layers.33.mlp.experts.115.down_proj", + "model.layers.33.mlp.experts.115.gate_proj", + "model.layers.33.mlp.experts.115.up_proj", + "model.layers.33.mlp.experts.116.down_proj", + "model.layers.33.mlp.experts.116.gate_proj", + "model.layers.33.mlp.experts.116.up_proj", + "model.layers.33.mlp.experts.117.down_proj", + "model.layers.33.mlp.experts.117.gate_proj", + "model.layers.33.mlp.experts.117.up_proj", + "model.layers.33.mlp.experts.118.down_proj", + "model.layers.33.mlp.experts.118.gate_proj", + "model.layers.33.mlp.experts.118.up_proj", + "model.layers.33.mlp.experts.119.down_proj", + "model.layers.33.mlp.experts.119.gate_proj", + "model.layers.33.mlp.experts.119.up_proj", + "model.layers.33.mlp.experts.12.down_proj", + "model.layers.33.mlp.experts.12.gate_proj", + "model.layers.33.mlp.experts.12.up_proj", + "model.layers.33.mlp.experts.120.down_proj", + "model.layers.33.mlp.experts.120.gate_proj", + "model.layers.33.mlp.experts.120.up_proj", + "model.layers.33.mlp.experts.121.down_proj", + "model.layers.33.mlp.experts.121.gate_proj", + "model.layers.33.mlp.experts.121.up_proj", + "model.layers.33.mlp.experts.122.down_proj", + "model.layers.33.mlp.experts.122.gate_proj", + "model.layers.33.mlp.experts.122.up_proj", + "model.layers.33.mlp.experts.123.down_proj", + "model.layers.33.mlp.experts.123.gate_proj", + "model.layers.33.mlp.experts.123.up_proj", + "model.layers.33.mlp.experts.124.down_proj", + "model.layers.33.mlp.experts.124.gate_proj", + "model.layers.33.mlp.experts.124.up_proj", + "model.layers.33.mlp.experts.125.down_proj", + "model.layers.33.mlp.experts.125.gate_proj", + "model.layers.33.mlp.experts.125.up_proj", + "model.layers.33.mlp.experts.126.down_proj", + "model.layers.33.mlp.experts.126.gate_proj", + "model.layers.33.mlp.experts.126.up_proj", + "model.layers.33.mlp.experts.127.down_proj", + "model.layers.33.mlp.experts.127.gate_proj", + "model.layers.33.mlp.experts.127.up_proj", + "model.layers.33.mlp.experts.128.down_proj", + "model.layers.33.mlp.experts.128.gate_proj", + "model.layers.33.mlp.experts.128.up_proj", + "model.layers.33.mlp.experts.129.down_proj", + "model.layers.33.mlp.experts.129.gate_proj", + "model.layers.33.mlp.experts.129.up_proj", + "model.layers.33.mlp.experts.13.down_proj", + "model.layers.33.mlp.experts.13.gate_proj", + "model.layers.33.mlp.experts.13.up_proj", + "model.layers.33.mlp.experts.130.down_proj", + "model.layers.33.mlp.experts.130.gate_proj", + "model.layers.33.mlp.experts.130.up_proj", + "model.layers.33.mlp.experts.131.down_proj", + "model.layers.33.mlp.experts.131.gate_proj", + "model.layers.33.mlp.experts.131.up_proj", + "model.layers.33.mlp.experts.132.down_proj", + "model.layers.33.mlp.experts.132.gate_proj", + "model.layers.33.mlp.experts.132.up_proj", + "model.layers.33.mlp.experts.133.down_proj", + "model.layers.33.mlp.experts.133.gate_proj", + "model.layers.33.mlp.experts.133.up_proj", + "model.layers.33.mlp.experts.134.down_proj", + "model.layers.33.mlp.experts.134.gate_proj", + "model.layers.33.mlp.experts.134.up_proj", + "model.layers.33.mlp.experts.135.down_proj", + "model.layers.33.mlp.experts.135.gate_proj", + "model.layers.33.mlp.experts.135.up_proj", + "model.layers.33.mlp.experts.136.down_proj", + "model.layers.33.mlp.experts.136.gate_proj", + "model.layers.33.mlp.experts.136.up_proj", + "model.layers.33.mlp.experts.137.down_proj", + "model.layers.33.mlp.experts.137.gate_proj", + "model.layers.33.mlp.experts.137.up_proj", + "model.layers.33.mlp.experts.138.down_proj", + "model.layers.33.mlp.experts.138.gate_proj", + "model.layers.33.mlp.experts.138.up_proj", + "model.layers.33.mlp.experts.139.down_proj", + "model.layers.33.mlp.experts.139.gate_proj", + "model.layers.33.mlp.experts.139.up_proj", + "model.layers.33.mlp.experts.14.down_proj", + "model.layers.33.mlp.experts.14.gate_proj", + "model.layers.33.mlp.experts.14.up_proj", + "model.layers.33.mlp.experts.140.down_proj", + "model.layers.33.mlp.experts.140.gate_proj", + "model.layers.33.mlp.experts.140.up_proj", + "model.layers.33.mlp.experts.141.down_proj", + "model.layers.33.mlp.experts.141.gate_proj", + "model.layers.33.mlp.experts.141.up_proj", + "model.layers.33.mlp.experts.142.down_proj", + "model.layers.33.mlp.experts.142.gate_proj", + "model.layers.33.mlp.experts.142.up_proj", + "model.layers.33.mlp.experts.143.down_proj", + "model.layers.33.mlp.experts.143.gate_proj", + "model.layers.33.mlp.experts.143.up_proj", + "model.layers.33.mlp.experts.144.down_proj", + "model.layers.33.mlp.experts.144.gate_proj", + "model.layers.33.mlp.experts.144.up_proj", + "model.layers.33.mlp.experts.145.down_proj", + "model.layers.33.mlp.experts.145.gate_proj", + "model.layers.33.mlp.experts.145.up_proj", + "model.layers.33.mlp.experts.146.down_proj", + "model.layers.33.mlp.experts.146.gate_proj", + "model.layers.33.mlp.experts.146.up_proj", + "model.layers.33.mlp.experts.147.down_proj", + "model.layers.33.mlp.experts.147.gate_proj", + "model.layers.33.mlp.experts.147.up_proj", + "model.layers.33.mlp.experts.148.down_proj", + "model.layers.33.mlp.experts.148.gate_proj", + "model.layers.33.mlp.experts.148.up_proj", + "model.layers.33.mlp.experts.149.down_proj", + "model.layers.33.mlp.experts.149.gate_proj", + "model.layers.33.mlp.experts.149.up_proj", + "model.layers.33.mlp.experts.15.down_proj", + "model.layers.33.mlp.experts.15.gate_proj", + "model.layers.33.mlp.experts.15.up_proj", + "model.layers.33.mlp.experts.150.down_proj", + "model.layers.33.mlp.experts.150.gate_proj", + "model.layers.33.mlp.experts.150.up_proj", + "model.layers.33.mlp.experts.151.down_proj", + "model.layers.33.mlp.experts.151.gate_proj", + "model.layers.33.mlp.experts.151.up_proj", + "model.layers.33.mlp.experts.152.down_proj", + "model.layers.33.mlp.experts.152.gate_proj", + "model.layers.33.mlp.experts.152.up_proj", + "model.layers.33.mlp.experts.153.down_proj", + "model.layers.33.mlp.experts.153.gate_proj", + "model.layers.33.mlp.experts.153.up_proj", + "model.layers.33.mlp.experts.154.down_proj", + "model.layers.33.mlp.experts.154.gate_proj", + "model.layers.33.mlp.experts.154.up_proj", + "model.layers.33.mlp.experts.155.down_proj", + "model.layers.33.mlp.experts.155.gate_proj", + "model.layers.33.mlp.experts.155.up_proj", + "model.layers.33.mlp.experts.156.down_proj", + "model.layers.33.mlp.experts.156.gate_proj", + "model.layers.33.mlp.experts.156.up_proj", + "model.layers.33.mlp.experts.157.down_proj", + "model.layers.33.mlp.experts.157.gate_proj", + "model.layers.33.mlp.experts.157.up_proj", + "model.layers.33.mlp.experts.158.down_proj", + "model.layers.33.mlp.experts.158.gate_proj", + "model.layers.33.mlp.experts.158.up_proj", + "model.layers.33.mlp.experts.159.down_proj", + "model.layers.33.mlp.experts.159.gate_proj", + "model.layers.33.mlp.experts.159.up_proj", + "model.layers.33.mlp.experts.16.down_proj", + "model.layers.33.mlp.experts.16.gate_proj", + "model.layers.33.mlp.experts.16.up_proj", + "model.layers.33.mlp.experts.160.down_proj", + "model.layers.33.mlp.experts.160.gate_proj", + "model.layers.33.mlp.experts.160.up_proj", + "model.layers.33.mlp.experts.161.down_proj", + "model.layers.33.mlp.experts.161.gate_proj", + "model.layers.33.mlp.experts.161.up_proj", + "model.layers.33.mlp.experts.162.down_proj", + "model.layers.33.mlp.experts.162.gate_proj", + "model.layers.33.mlp.experts.162.up_proj", + "model.layers.33.mlp.experts.163.down_proj", + "model.layers.33.mlp.experts.163.gate_proj", + "model.layers.33.mlp.experts.163.up_proj", + "model.layers.33.mlp.experts.164.down_proj", + "model.layers.33.mlp.experts.164.gate_proj", + "model.layers.33.mlp.experts.164.up_proj", + "model.layers.33.mlp.experts.165.down_proj", + "model.layers.33.mlp.experts.165.gate_proj", + "model.layers.33.mlp.experts.165.up_proj", + "model.layers.33.mlp.experts.166.down_proj", + "model.layers.33.mlp.experts.166.gate_proj", + "model.layers.33.mlp.experts.166.up_proj", + "model.layers.33.mlp.experts.167.down_proj", + "model.layers.33.mlp.experts.167.gate_proj", + "model.layers.33.mlp.experts.167.up_proj", + "model.layers.33.mlp.experts.168.down_proj", + "model.layers.33.mlp.experts.168.gate_proj", + "model.layers.33.mlp.experts.168.up_proj", + "model.layers.33.mlp.experts.169.down_proj", + "model.layers.33.mlp.experts.169.gate_proj", + "model.layers.33.mlp.experts.169.up_proj", + "model.layers.33.mlp.experts.17.down_proj", + "model.layers.33.mlp.experts.17.gate_proj", + "model.layers.33.mlp.experts.17.up_proj", + "model.layers.33.mlp.experts.170.down_proj", + "model.layers.33.mlp.experts.170.gate_proj", + "model.layers.33.mlp.experts.170.up_proj", + "model.layers.33.mlp.experts.171.down_proj", + "model.layers.33.mlp.experts.171.gate_proj", + "model.layers.33.mlp.experts.171.up_proj", + "model.layers.33.mlp.experts.172.down_proj", + "model.layers.33.mlp.experts.172.gate_proj", + "model.layers.33.mlp.experts.172.up_proj", + "model.layers.33.mlp.experts.173.down_proj", + "model.layers.33.mlp.experts.173.gate_proj", + "model.layers.33.mlp.experts.173.up_proj", + "model.layers.33.mlp.experts.174.down_proj", + "model.layers.33.mlp.experts.174.gate_proj", + "model.layers.33.mlp.experts.174.up_proj", + "model.layers.33.mlp.experts.175.down_proj", + "model.layers.33.mlp.experts.175.gate_proj", + "model.layers.33.mlp.experts.175.up_proj", + "model.layers.33.mlp.experts.176.down_proj", + "model.layers.33.mlp.experts.176.gate_proj", + "model.layers.33.mlp.experts.176.up_proj", + "model.layers.33.mlp.experts.177.down_proj", + "model.layers.33.mlp.experts.177.gate_proj", + "model.layers.33.mlp.experts.177.up_proj", + "model.layers.33.mlp.experts.178.down_proj", + "model.layers.33.mlp.experts.178.gate_proj", + "model.layers.33.mlp.experts.178.up_proj", + "model.layers.33.mlp.experts.179.down_proj", + "model.layers.33.mlp.experts.179.gate_proj", + "model.layers.33.mlp.experts.179.up_proj", + "model.layers.33.mlp.experts.18.down_proj", + "model.layers.33.mlp.experts.18.gate_proj", + "model.layers.33.mlp.experts.18.up_proj", + "model.layers.33.mlp.experts.180.down_proj", + "model.layers.33.mlp.experts.180.gate_proj", + "model.layers.33.mlp.experts.180.up_proj", + "model.layers.33.mlp.experts.181.down_proj", + "model.layers.33.mlp.experts.181.gate_proj", + "model.layers.33.mlp.experts.181.up_proj", + "model.layers.33.mlp.experts.182.down_proj", + "model.layers.33.mlp.experts.182.gate_proj", + "model.layers.33.mlp.experts.182.up_proj", + "model.layers.33.mlp.experts.183.down_proj", + "model.layers.33.mlp.experts.183.gate_proj", + "model.layers.33.mlp.experts.183.up_proj", + "model.layers.33.mlp.experts.184.down_proj", + "model.layers.33.mlp.experts.184.gate_proj", + "model.layers.33.mlp.experts.184.up_proj", + "model.layers.33.mlp.experts.185.down_proj", + "model.layers.33.mlp.experts.185.gate_proj", + "model.layers.33.mlp.experts.185.up_proj", + "model.layers.33.mlp.experts.186.down_proj", + "model.layers.33.mlp.experts.186.gate_proj", + "model.layers.33.mlp.experts.186.up_proj", + "model.layers.33.mlp.experts.187.down_proj", + "model.layers.33.mlp.experts.187.gate_proj", + "model.layers.33.mlp.experts.187.up_proj", + "model.layers.33.mlp.experts.188.down_proj", + "model.layers.33.mlp.experts.188.gate_proj", + "model.layers.33.mlp.experts.188.up_proj", + "model.layers.33.mlp.experts.189.down_proj", + "model.layers.33.mlp.experts.189.gate_proj", + "model.layers.33.mlp.experts.189.up_proj", + "model.layers.33.mlp.experts.19.down_proj", + "model.layers.33.mlp.experts.19.gate_proj", + "model.layers.33.mlp.experts.19.up_proj", + "model.layers.33.mlp.experts.190.down_proj", + "model.layers.33.mlp.experts.190.gate_proj", + "model.layers.33.mlp.experts.190.up_proj", + "model.layers.33.mlp.experts.191.down_proj", + "model.layers.33.mlp.experts.191.gate_proj", + "model.layers.33.mlp.experts.191.up_proj", + "model.layers.33.mlp.experts.192.down_proj", + "model.layers.33.mlp.experts.192.gate_proj", + "model.layers.33.mlp.experts.192.up_proj", + "model.layers.33.mlp.experts.193.down_proj", + "model.layers.33.mlp.experts.193.gate_proj", + "model.layers.33.mlp.experts.193.up_proj", + "model.layers.33.mlp.experts.194.down_proj", + "model.layers.33.mlp.experts.194.gate_proj", + "model.layers.33.mlp.experts.194.up_proj", + "model.layers.33.mlp.experts.195.down_proj", + "model.layers.33.mlp.experts.195.gate_proj", + "model.layers.33.mlp.experts.195.up_proj", + "model.layers.33.mlp.experts.196.down_proj", + "model.layers.33.mlp.experts.196.gate_proj", + "model.layers.33.mlp.experts.196.up_proj", + "model.layers.33.mlp.experts.197.down_proj", + "model.layers.33.mlp.experts.197.gate_proj", + "model.layers.33.mlp.experts.197.up_proj", + "model.layers.33.mlp.experts.198.down_proj", + "model.layers.33.mlp.experts.198.gate_proj", + "model.layers.33.mlp.experts.198.up_proj", + "model.layers.33.mlp.experts.199.down_proj", + "model.layers.33.mlp.experts.199.gate_proj", + "model.layers.33.mlp.experts.199.up_proj", + "model.layers.33.mlp.experts.2.down_proj", + "model.layers.33.mlp.experts.2.gate_proj", + "model.layers.33.mlp.experts.2.up_proj", + "model.layers.33.mlp.experts.20.down_proj", + "model.layers.33.mlp.experts.20.gate_proj", + "model.layers.33.mlp.experts.20.up_proj", + "model.layers.33.mlp.experts.200.down_proj", + "model.layers.33.mlp.experts.200.gate_proj", + "model.layers.33.mlp.experts.200.up_proj", + "model.layers.33.mlp.experts.201.down_proj", + "model.layers.33.mlp.experts.201.gate_proj", + "model.layers.33.mlp.experts.201.up_proj", + "model.layers.33.mlp.experts.202.down_proj", + "model.layers.33.mlp.experts.202.gate_proj", + "model.layers.33.mlp.experts.202.up_proj", + "model.layers.33.mlp.experts.203.down_proj", + "model.layers.33.mlp.experts.203.gate_proj", + "model.layers.33.mlp.experts.203.up_proj", + "model.layers.33.mlp.experts.204.down_proj", + "model.layers.33.mlp.experts.204.gate_proj", + "model.layers.33.mlp.experts.204.up_proj", + "model.layers.33.mlp.experts.205.down_proj", + "model.layers.33.mlp.experts.205.gate_proj", + "model.layers.33.mlp.experts.205.up_proj", + "model.layers.33.mlp.experts.206.down_proj", + "model.layers.33.mlp.experts.206.gate_proj", + "model.layers.33.mlp.experts.206.up_proj", + "model.layers.33.mlp.experts.207.down_proj", + "model.layers.33.mlp.experts.207.gate_proj", + "model.layers.33.mlp.experts.207.up_proj", + "model.layers.33.mlp.experts.208.down_proj", + "model.layers.33.mlp.experts.208.gate_proj", + "model.layers.33.mlp.experts.208.up_proj", + "model.layers.33.mlp.experts.209.down_proj", + "model.layers.33.mlp.experts.209.gate_proj", + "model.layers.33.mlp.experts.209.up_proj", + "model.layers.33.mlp.experts.21.down_proj", + "model.layers.33.mlp.experts.21.gate_proj", + "model.layers.33.mlp.experts.21.up_proj", + "model.layers.33.mlp.experts.210.down_proj", + "model.layers.33.mlp.experts.210.gate_proj", + "model.layers.33.mlp.experts.210.up_proj", + "model.layers.33.mlp.experts.211.down_proj", + "model.layers.33.mlp.experts.211.gate_proj", + "model.layers.33.mlp.experts.211.up_proj", + "model.layers.33.mlp.experts.212.down_proj", + "model.layers.33.mlp.experts.212.gate_proj", + "model.layers.33.mlp.experts.212.up_proj", + "model.layers.33.mlp.experts.213.down_proj", + "model.layers.33.mlp.experts.213.gate_proj", + "model.layers.33.mlp.experts.213.up_proj", + "model.layers.33.mlp.experts.214.down_proj", + "model.layers.33.mlp.experts.214.gate_proj", + "model.layers.33.mlp.experts.214.up_proj", + "model.layers.33.mlp.experts.215.down_proj", + "model.layers.33.mlp.experts.215.gate_proj", + "model.layers.33.mlp.experts.215.up_proj", + "model.layers.33.mlp.experts.216.down_proj", + "model.layers.33.mlp.experts.216.gate_proj", + "model.layers.33.mlp.experts.216.up_proj", + "model.layers.33.mlp.experts.217.down_proj", + "model.layers.33.mlp.experts.217.gate_proj", + "model.layers.33.mlp.experts.217.up_proj", + "model.layers.33.mlp.experts.218.down_proj", + "model.layers.33.mlp.experts.218.gate_proj", + "model.layers.33.mlp.experts.218.up_proj", + "model.layers.33.mlp.experts.219.down_proj", + "model.layers.33.mlp.experts.219.gate_proj", + "model.layers.33.mlp.experts.219.up_proj", + "model.layers.33.mlp.experts.22.down_proj", + "model.layers.33.mlp.experts.22.gate_proj", + "model.layers.33.mlp.experts.22.up_proj", + "model.layers.33.mlp.experts.220.down_proj", + "model.layers.33.mlp.experts.220.gate_proj", + "model.layers.33.mlp.experts.220.up_proj", + "model.layers.33.mlp.experts.221.down_proj", + "model.layers.33.mlp.experts.221.gate_proj", + "model.layers.33.mlp.experts.221.up_proj", + "model.layers.33.mlp.experts.222.down_proj", + "model.layers.33.mlp.experts.222.gate_proj", + "model.layers.33.mlp.experts.222.up_proj", + "model.layers.33.mlp.experts.223.down_proj", + "model.layers.33.mlp.experts.223.gate_proj", + "model.layers.33.mlp.experts.223.up_proj", + "model.layers.33.mlp.experts.224.down_proj", + "model.layers.33.mlp.experts.224.gate_proj", + "model.layers.33.mlp.experts.224.up_proj", + "model.layers.33.mlp.experts.225.down_proj", + "model.layers.33.mlp.experts.225.gate_proj", + "model.layers.33.mlp.experts.225.up_proj", + "model.layers.33.mlp.experts.226.down_proj", + "model.layers.33.mlp.experts.226.gate_proj", + "model.layers.33.mlp.experts.226.up_proj", + "model.layers.33.mlp.experts.227.down_proj", + "model.layers.33.mlp.experts.227.gate_proj", + "model.layers.33.mlp.experts.227.up_proj", + "model.layers.33.mlp.experts.228.down_proj", + "model.layers.33.mlp.experts.228.gate_proj", + "model.layers.33.mlp.experts.228.up_proj", + "model.layers.33.mlp.experts.229.down_proj", + "model.layers.33.mlp.experts.229.gate_proj", + "model.layers.33.mlp.experts.229.up_proj", + "model.layers.33.mlp.experts.23.down_proj", + "model.layers.33.mlp.experts.23.gate_proj", + "model.layers.33.mlp.experts.23.up_proj", + "model.layers.33.mlp.experts.230.down_proj", + "model.layers.33.mlp.experts.230.gate_proj", + "model.layers.33.mlp.experts.230.up_proj", + "model.layers.33.mlp.experts.231.down_proj", + "model.layers.33.mlp.experts.231.gate_proj", + "model.layers.33.mlp.experts.231.up_proj", + "model.layers.33.mlp.experts.232.down_proj", + "model.layers.33.mlp.experts.232.gate_proj", + "model.layers.33.mlp.experts.232.up_proj", + "model.layers.33.mlp.experts.233.down_proj", + "model.layers.33.mlp.experts.233.gate_proj", + "model.layers.33.mlp.experts.233.up_proj", + "model.layers.33.mlp.experts.234.down_proj", + "model.layers.33.mlp.experts.234.gate_proj", + "model.layers.33.mlp.experts.234.up_proj", + "model.layers.33.mlp.experts.235.down_proj", + "model.layers.33.mlp.experts.235.gate_proj", + "model.layers.33.mlp.experts.235.up_proj", + "model.layers.33.mlp.experts.236.down_proj", + "model.layers.33.mlp.experts.236.gate_proj", + "model.layers.33.mlp.experts.236.up_proj", + "model.layers.33.mlp.experts.237.down_proj", + "model.layers.33.mlp.experts.237.gate_proj", + "model.layers.33.mlp.experts.237.up_proj", + "model.layers.33.mlp.experts.238.down_proj", + "model.layers.33.mlp.experts.238.gate_proj", + "model.layers.33.mlp.experts.238.up_proj", + "model.layers.33.mlp.experts.239.down_proj", + "model.layers.33.mlp.experts.239.gate_proj", + "model.layers.33.mlp.experts.239.up_proj", + "model.layers.33.mlp.experts.24.down_proj", + "model.layers.33.mlp.experts.24.gate_proj", + "model.layers.33.mlp.experts.24.up_proj", + "model.layers.33.mlp.experts.240.down_proj", + "model.layers.33.mlp.experts.240.gate_proj", + "model.layers.33.mlp.experts.240.up_proj", + "model.layers.33.mlp.experts.241.down_proj", + "model.layers.33.mlp.experts.241.gate_proj", + "model.layers.33.mlp.experts.241.up_proj", + "model.layers.33.mlp.experts.242.down_proj", + "model.layers.33.mlp.experts.242.gate_proj", + "model.layers.33.mlp.experts.242.up_proj", + "model.layers.33.mlp.experts.243.down_proj", + "model.layers.33.mlp.experts.243.gate_proj", + "model.layers.33.mlp.experts.243.up_proj", + "model.layers.33.mlp.experts.244.down_proj", + "model.layers.33.mlp.experts.244.gate_proj", + "model.layers.33.mlp.experts.244.up_proj", + "model.layers.33.mlp.experts.245.down_proj", + "model.layers.33.mlp.experts.245.gate_proj", + "model.layers.33.mlp.experts.245.up_proj", + "model.layers.33.mlp.experts.246.down_proj", + "model.layers.33.mlp.experts.246.gate_proj", + "model.layers.33.mlp.experts.246.up_proj", + "model.layers.33.mlp.experts.247.down_proj", + "model.layers.33.mlp.experts.247.gate_proj", + "model.layers.33.mlp.experts.247.up_proj", + "model.layers.33.mlp.experts.248.down_proj", + "model.layers.33.mlp.experts.248.gate_proj", + "model.layers.33.mlp.experts.248.up_proj", + "model.layers.33.mlp.experts.249.down_proj", + "model.layers.33.mlp.experts.249.gate_proj", + "model.layers.33.mlp.experts.249.up_proj", + "model.layers.33.mlp.experts.25.down_proj", + "model.layers.33.mlp.experts.25.gate_proj", + "model.layers.33.mlp.experts.25.up_proj", + "model.layers.33.mlp.experts.250.down_proj", + "model.layers.33.mlp.experts.250.gate_proj", + "model.layers.33.mlp.experts.250.up_proj", + "model.layers.33.mlp.experts.251.down_proj", + "model.layers.33.mlp.experts.251.gate_proj", + "model.layers.33.mlp.experts.251.up_proj", + "model.layers.33.mlp.experts.252.down_proj", + "model.layers.33.mlp.experts.252.gate_proj", + "model.layers.33.mlp.experts.252.up_proj", + "model.layers.33.mlp.experts.253.down_proj", + "model.layers.33.mlp.experts.253.gate_proj", + "model.layers.33.mlp.experts.253.up_proj", + "model.layers.33.mlp.experts.254.down_proj", + "model.layers.33.mlp.experts.254.gate_proj", + "model.layers.33.mlp.experts.254.up_proj", + "model.layers.33.mlp.experts.255.down_proj", + "model.layers.33.mlp.experts.255.gate_proj", + "model.layers.33.mlp.experts.255.up_proj", + "model.layers.33.mlp.experts.26.down_proj", + "model.layers.33.mlp.experts.26.gate_proj", + "model.layers.33.mlp.experts.26.up_proj", + "model.layers.33.mlp.experts.27.down_proj", + "model.layers.33.mlp.experts.27.gate_proj", + "model.layers.33.mlp.experts.27.up_proj", + "model.layers.33.mlp.experts.28.down_proj", + "model.layers.33.mlp.experts.28.gate_proj", + "model.layers.33.mlp.experts.28.up_proj", + "model.layers.33.mlp.experts.29.down_proj", + "model.layers.33.mlp.experts.29.gate_proj", + "model.layers.33.mlp.experts.29.up_proj", + "model.layers.33.mlp.experts.3.down_proj", + "model.layers.33.mlp.experts.3.gate_proj", + "model.layers.33.mlp.experts.3.up_proj", + "model.layers.33.mlp.experts.30.down_proj", + "model.layers.33.mlp.experts.30.gate_proj", + "model.layers.33.mlp.experts.30.up_proj", + "model.layers.33.mlp.experts.31.down_proj", + "model.layers.33.mlp.experts.31.gate_proj", + "model.layers.33.mlp.experts.31.up_proj", + "model.layers.33.mlp.experts.32.down_proj", + "model.layers.33.mlp.experts.32.gate_proj", + "model.layers.33.mlp.experts.32.up_proj", + "model.layers.33.mlp.experts.33.down_proj", + "model.layers.33.mlp.experts.33.gate_proj", + "model.layers.33.mlp.experts.33.up_proj", + "model.layers.33.mlp.experts.34.down_proj", + "model.layers.33.mlp.experts.34.gate_proj", + "model.layers.33.mlp.experts.34.up_proj", + "model.layers.33.mlp.experts.35.down_proj", + "model.layers.33.mlp.experts.35.gate_proj", + "model.layers.33.mlp.experts.35.up_proj", + "model.layers.33.mlp.experts.36.down_proj", + "model.layers.33.mlp.experts.36.gate_proj", + "model.layers.33.mlp.experts.36.up_proj", + "model.layers.33.mlp.experts.37.down_proj", + "model.layers.33.mlp.experts.37.gate_proj", + "model.layers.33.mlp.experts.37.up_proj", + "model.layers.33.mlp.experts.38.down_proj", + "model.layers.33.mlp.experts.38.gate_proj", + "model.layers.33.mlp.experts.38.up_proj", + "model.layers.33.mlp.experts.39.down_proj", + "model.layers.33.mlp.experts.39.gate_proj", + "model.layers.33.mlp.experts.39.up_proj", + "model.layers.33.mlp.experts.4.down_proj", + "model.layers.33.mlp.experts.4.gate_proj", + "model.layers.33.mlp.experts.4.up_proj", + "model.layers.33.mlp.experts.40.down_proj", + "model.layers.33.mlp.experts.40.gate_proj", + "model.layers.33.mlp.experts.40.up_proj", + "model.layers.33.mlp.experts.41.down_proj", + "model.layers.33.mlp.experts.41.gate_proj", + "model.layers.33.mlp.experts.41.up_proj", + "model.layers.33.mlp.experts.42.down_proj", + "model.layers.33.mlp.experts.42.gate_proj", + "model.layers.33.mlp.experts.42.up_proj", + "model.layers.33.mlp.experts.43.down_proj", + "model.layers.33.mlp.experts.43.gate_proj", + "model.layers.33.mlp.experts.43.up_proj", + "model.layers.33.mlp.experts.44.down_proj", + "model.layers.33.mlp.experts.44.gate_proj", + "model.layers.33.mlp.experts.44.up_proj", + "model.layers.33.mlp.experts.45.down_proj", + "model.layers.33.mlp.experts.45.gate_proj", + "model.layers.33.mlp.experts.45.up_proj", + "model.layers.33.mlp.experts.46.down_proj", + "model.layers.33.mlp.experts.46.gate_proj", + "model.layers.33.mlp.experts.46.up_proj", + "model.layers.33.mlp.experts.47.down_proj", + "model.layers.33.mlp.experts.47.gate_proj", + "model.layers.33.mlp.experts.47.up_proj", + "model.layers.33.mlp.experts.48.down_proj", + "model.layers.33.mlp.experts.48.gate_proj", + "model.layers.33.mlp.experts.48.up_proj", + "model.layers.33.mlp.experts.49.down_proj", + "model.layers.33.mlp.experts.49.gate_proj", + "model.layers.33.mlp.experts.49.up_proj", + "model.layers.33.mlp.experts.5.down_proj", + "model.layers.33.mlp.experts.5.gate_proj", + "model.layers.33.mlp.experts.5.up_proj", + "model.layers.33.mlp.experts.50.down_proj", + "model.layers.33.mlp.experts.50.gate_proj", + "model.layers.33.mlp.experts.50.up_proj", + "model.layers.33.mlp.experts.51.down_proj", + "model.layers.33.mlp.experts.51.gate_proj", + "model.layers.33.mlp.experts.51.up_proj", + "model.layers.33.mlp.experts.52.down_proj", + "model.layers.33.mlp.experts.52.gate_proj", + "model.layers.33.mlp.experts.52.up_proj", + "model.layers.33.mlp.experts.53.down_proj", + "model.layers.33.mlp.experts.53.gate_proj", + "model.layers.33.mlp.experts.53.up_proj", + "model.layers.33.mlp.experts.54.down_proj", + "model.layers.33.mlp.experts.54.gate_proj", + "model.layers.33.mlp.experts.54.up_proj", + "model.layers.33.mlp.experts.55.down_proj", + "model.layers.33.mlp.experts.55.gate_proj", + "model.layers.33.mlp.experts.55.up_proj", + "model.layers.33.mlp.experts.56.down_proj", + "model.layers.33.mlp.experts.56.gate_proj", + "model.layers.33.mlp.experts.56.up_proj", + "model.layers.33.mlp.experts.57.down_proj", + "model.layers.33.mlp.experts.57.gate_proj", + "model.layers.33.mlp.experts.57.up_proj", + "model.layers.33.mlp.experts.58.down_proj", + "model.layers.33.mlp.experts.58.gate_proj", + "model.layers.33.mlp.experts.58.up_proj", + "model.layers.33.mlp.experts.59.down_proj", + "model.layers.33.mlp.experts.59.gate_proj", + "model.layers.33.mlp.experts.59.up_proj", + "model.layers.33.mlp.experts.6.down_proj", + "model.layers.33.mlp.experts.6.gate_proj", + "model.layers.33.mlp.experts.6.up_proj", + "model.layers.33.mlp.experts.60.down_proj", + "model.layers.33.mlp.experts.60.gate_proj", + "model.layers.33.mlp.experts.60.up_proj", + "model.layers.33.mlp.experts.61.down_proj", + "model.layers.33.mlp.experts.61.gate_proj", + "model.layers.33.mlp.experts.61.up_proj", + "model.layers.33.mlp.experts.62.down_proj", + "model.layers.33.mlp.experts.62.gate_proj", + "model.layers.33.mlp.experts.62.up_proj", + "model.layers.33.mlp.experts.63.down_proj", + "model.layers.33.mlp.experts.63.gate_proj", + "model.layers.33.mlp.experts.63.up_proj", + "model.layers.33.mlp.experts.64.down_proj", + "model.layers.33.mlp.experts.64.gate_proj", + "model.layers.33.mlp.experts.64.up_proj", + "model.layers.33.mlp.experts.65.down_proj", + "model.layers.33.mlp.experts.65.gate_proj", + "model.layers.33.mlp.experts.65.up_proj", + "model.layers.33.mlp.experts.66.down_proj", + "model.layers.33.mlp.experts.66.gate_proj", + "model.layers.33.mlp.experts.66.up_proj", + "model.layers.33.mlp.experts.67.down_proj", + "model.layers.33.mlp.experts.67.gate_proj", + "model.layers.33.mlp.experts.67.up_proj", + "model.layers.33.mlp.experts.68.down_proj", + "model.layers.33.mlp.experts.68.gate_proj", + "model.layers.33.mlp.experts.68.up_proj", + "model.layers.33.mlp.experts.69.down_proj", + "model.layers.33.mlp.experts.69.gate_proj", + "model.layers.33.mlp.experts.69.up_proj", + "model.layers.33.mlp.experts.7.down_proj", + "model.layers.33.mlp.experts.7.gate_proj", + "model.layers.33.mlp.experts.7.up_proj", + "model.layers.33.mlp.experts.70.down_proj", + "model.layers.33.mlp.experts.70.gate_proj", + "model.layers.33.mlp.experts.70.up_proj", + "model.layers.33.mlp.experts.71.down_proj", + "model.layers.33.mlp.experts.71.gate_proj", + "model.layers.33.mlp.experts.71.up_proj", + "model.layers.33.mlp.experts.72.down_proj", + "model.layers.33.mlp.experts.72.gate_proj", + "model.layers.33.mlp.experts.72.up_proj", + "model.layers.33.mlp.experts.73.down_proj", + "model.layers.33.mlp.experts.73.gate_proj", + "model.layers.33.mlp.experts.73.up_proj", + "model.layers.33.mlp.experts.74.down_proj", + "model.layers.33.mlp.experts.74.gate_proj", + "model.layers.33.mlp.experts.74.up_proj", + "model.layers.33.mlp.experts.75.down_proj", + "model.layers.33.mlp.experts.75.gate_proj", + "model.layers.33.mlp.experts.75.up_proj", + "model.layers.33.mlp.experts.76.down_proj", + "model.layers.33.mlp.experts.76.gate_proj", + "model.layers.33.mlp.experts.76.up_proj", + "model.layers.33.mlp.experts.77.down_proj", + "model.layers.33.mlp.experts.77.gate_proj", + "model.layers.33.mlp.experts.77.up_proj", + "model.layers.33.mlp.experts.78.down_proj", + "model.layers.33.mlp.experts.78.gate_proj", + "model.layers.33.mlp.experts.78.up_proj", + "model.layers.33.mlp.experts.79.down_proj", + "model.layers.33.mlp.experts.79.gate_proj", + "model.layers.33.mlp.experts.79.up_proj", + "model.layers.33.mlp.experts.8.down_proj", + "model.layers.33.mlp.experts.8.gate_proj", + "model.layers.33.mlp.experts.8.up_proj", + "model.layers.33.mlp.experts.80.down_proj", + "model.layers.33.mlp.experts.80.gate_proj", + "model.layers.33.mlp.experts.80.up_proj", + "model.layers.33.mlp.experts.81.down_proj", + "model.layers.33.mlp.experts.81.gate_proj", + "model.layers.33.mlp.experts.81.up_proj", + "model.layers.33.mlp.experts.82.down_proj", + "model.layers.33.mlp.experts.82.gate_proj", + "model.layers.33.mlp.experts.82.up_proj", + "model.layers.33.mlp.experts.83.down_proj", + "model.layers.33.mlp.experts.83.gate_proj", + "model.layers.33.mlp.experts.83.up_proj", + "model.layers.33.mlp.experts.84.down_proj", + "model.layers.33.mlp.experts.84.gate_proj", + "model.layers.33.mlp.experts.84.up_proj", + "model.layers.33.mlp.experts.85.down_proj", + "model.layers.33.mlp.experts.85.gate_proj", + "model.layers.33.mlp.experts.85.up_proj", + "model.layers.33.mlp.experts.86.down_proj", + "model.layers.33.mlp.experts.86.gate_proj", + "model.layers.33.mlp.experts.86.up_proj", + "model.layers.33.mlp.experts.87.down_proj", + "model.layers.33.mlp.experts.87.gate_proj", + "model.layers.33.mlp.experts.87.up_proj", + "model.layers.33.mlp.experts.88.down_proj", + "model.layers.33.mlp.experts.88.gate_proj", + "model.layers.33.mlp.experts.88.up_proj", + "model.layers.33.mlp.experts.89.down_proj", + "model.layers.33.mlp.experts.89.gate_proj", + "model.layers.33.mlp.experts.89.up_proj", + "model.layers.33.mlp.experts.9.down_proj", + "model.layers.33.mlp.experts.9.gate_proj", + "model.layers.33.mlp.experts.9.up_proj", + "model.layers.33.mlp.experts.90.down_proj", + "model.layers.33.mlp.experts.90.gate_proj", + "model.layers.33.mlp.experts.90.up_proj", + "model.layers.33.mlp.experts.91.down_proj", + "model.layers.33.mlp.experts.91.gate_proj", + "model.layers.33.mlp.experts.91.up_proj", + "model.layers.33.mlp.experts.92.down_proj", + "model.layers.33.mlp.experts.92.gate_proj", + "model.layers.33.mlp.experts.92.up_proj", + "model.layers.33.mlp.experts.93.down_proj", + "model.layers.33.mlp.experts.93.gate_proj", + "model.layers.33.mlp.experts.93.up_proj", + "model.layers.33.mlp.experts.94.down_proj", + "model.layers.33.mlp.experts.94.gate_proj", + "model.layers.33.mlp.experts.94.up_proj", + "model.layers.33.mlp.experts.95.down_proj", + "model.layers.33.mlp.experts.95.gate_proj", + "model.layers.33.mlp.experts.95.up_proj", + "model.layers.33.mlp.experts.96.down_proj", + "model.layers.33.mlp.experts.96.gate_proj", + "model.layers.33.mlp.experts.96.up_proj", + "model.layers.33.mlp.experts.97.down_proj", + "model.layers.33.mlp.experts.97.gate_proj", + "model.layers.33.mlp.experts.97.up_proj", + "model.layers.33.mlp.experts.98.down_proj", + "model.layers.33.mlp.experts.98.gate_proj", + "model.layers.33.mlp.experts.98.up_proj", + "model.layers.33.mlp.experts.99.down_proj", + "model.layers.33.mlp.experts.99.gate_proj", + "model.layers.33.mlp.experts.99.up_proj", + "model.layers.34.mlp.experts.0.down_proj", + "model.layers.34.mlp.experts.0.gate_proj", + "model.layers.34.mlp.experts.0.up_proj", + "model.layers.34.mlp.experts.1.down_proj", + "model.layers.34.mlp.experts.1.gate_proj", + "model.layers.34.mlp.experts.1.up_proj", + "model.layers.34.mlp.experts.10.down_proj", + "model.layers.34.mlp.experts.10.gate_proj", + "model.layers.34.mlp.experts.10.up_proj", + "model.layers.34.mlp.experts.100.down_proj", + "model.layers.34.mlp.experts.100.gate_proj", + "model.layers.34.mlp.experts.100.up_proj", + "model.layers.34.mlp.experts.101.down_proj", + "model.layers.34.mlp.experts.101.gate_proj", + "model.layers.34.mlp.experts.101.up_proj", + "model.layers.34.mlp.experts.102.down_proj", + "model.layers.34.mlp.experts.102.gate_proj", + "model.layers.34.mlp.experts.102.up_proj", + "model.layers.34.mlp.experts.103.down_proj", + "model.layers.34.mlp.experts.103.gate_proj", + "model.layers.34.mlp.experts.103.up_proj", + "model.layers.34.mlp.experts.104.down_proj", + "model.layers.34.mlp.experts.104.gate_proj", + "model.layers.34.mlp.experts.104.up_proj", + "model.layers.34.mlp.experts.105.down_proj", + "model.layers.34.mlp.experts.105.gate_proj", + "model.layers.34.mlp.experts.105.up_proj", + "model.layers.34.mlp.experts.106.down_proj", + "model.layers.34.mlp.experts.106.gate_proj", + "model.layers.34.mlp.experts.106.up_proj", + "model.layers.34.mlp.experts.107.down_proj", + "model.layers.34.mlp.experts.107.gate_proj", + "model.layers.34.mlp.experts.107.up_proj", + "model.layers.34.mlp.experts.108.down_proj", + "model.layers.34.mlp.experts.108.gate_proj", + "model.layers.34.mlp.experts.108.up_proj", + "model.layers.34.mlp.experts.109.down_proj", + "model.layers.34.mlp.experts.109.gate_proj", + "model.layers.34.mlp.experts.109.up_proj", + "model.layers.34.mlp.experts.11.down_proj", + "model.layers.34.mlp.experts.11.gate_proj", + "model.layers.34.mlp.experts.11.up_proj", + "model.layers.34.mlp.experts.110.down_proj", + "model.layers.34.mlp.experts.110.gate_proj", + "model.layers.34.mlp.experts.110.up_proj", + "model.layers.34.mlp.experts.111.down_proj", + "model.layers.34.mlp.experts.111.gate_proj", + "model.layers.34.mlp.experts.111.up_proj", + "model.layers.34.mlp.experts.112.down_proj", + "model.layers.34.mlp.experts.112.gate_proj", + "model.layers.34.mlp.experts.112.up_proj", + "model.layers.34.mlp.experts.113.down_proj", + "model.layers.34.mlp.experts.113.gate_proj", + "model.layers.34.mlp.experts.113.up_proj", + "model.layers.34.mlp.experts.114.down_proj", + "model.layers.34.mlp.experts.114.gate_proj", + "model.layers.34.mlp.experts.114.up_proj", + "model.layers.34.mlp.experts.115.down_proj", + "model.layers.34.mlp.experts.115.gate_proj", + "model.layers.34.mlp.experts.115.up_proj", + "model.layers.34.mlp.experts.116.down_proj", + "model.layers.34.mlp.experts.116.gate_proj", + "model.layers.34.mlp.experts.116.up_proj", + "model.layers.34.mlp.experts.117.down_proj", + "model.layers.34.mlp.experts.117.gate_proj", + "model.layers.34.mlp.experts.117.up_proj", + "model.layers.34.mlp.experts.118.down_proj", + "model.layers.34.mlp.experts.118.gate_proj", + "model.layers.34.mlp.experts.118.up_proj", + "model.layers.34.mlp.experts.119.down_proj", + "model.layers.34.mlp.experts.119.gate_proj", + "model.layers.34.mlp.experts.119.up_proj", + "model.layers.34.mlp.experts.12.down_proj", + "model.layers.34.mlp.experts.12.gate_proj", + "model.layers.34.mlp.experts.12.up_proj", + "model.layers.34.mlp.experts.120.down_proj", + "model.layers.34.mlp.experts.120.gate_proj", + "model.layers.34.mlp.experts.120.up_proj", + "model.layers.34.mlp.experts.121.down_proj", + "model.layers.34.mlp.experts.121.gate_proj", + "model.layers.34.mlp.experts.121.up_proj", + "model.layers.34.mlp.experts.122.down_proj", + "model.layers.34.mlp.experts.122.gate_proj", + "model.layers.34.mlp.experts.122.up_proj", + "model.layers.34.mlp.experts.123.down_proj", + "model.layers.34.mlp.experts.123.gate_proj", + "model.layers.34.mlp.experts.123.up_proj", + "model.layers.34.mlp.experts.124.down_proj", + "model.layers.34.mlp.experts.124.gate_proj", + "model.layers.34.mlp.experts.124.up_proj", + "model.layers.34.mlp.experts.125.down_proj", + "model.layers.34.mlp.experts.125.gate_proj", + "model.layers.34.mlp.experts.125.up_proj", + "model.layers.34.mlp.experts.126.down_proj", + "model.layers.34.mlp.experts.126.gate_proj", + "model.layers.34.mlp.experts.126.up_proj", + "model.layers.34.mlp.experts.127.down_proj", + "model.layers.34.mlp.experts.127.gate_proj", + "model.layers.34.mlp.experts.127.up_proj", + "model.layers.34.mlp.experts.128.down_proj", + "model.layers.34.mlp.experts.128.gate_proj", + "model.layers.34.mlp.experts.128.up_proj", + "model.layers.34.mlp.experts.129.down_proj", + "model.layers.34.mlp.experts.129.gate_proj", + "model.layers.34.mlp.experts.129.up_proj", + "model.layers.34.mlp.experts.13.down_proj", + "model.layers.34.mlp.experts.13.gate_proj", + "model.layers.34.mlp.experts.13.up_proj", + "model.layers.34.mlp.experts.130.down_proj", + "model.layers.34.mlp.experts.130.gate_proj", + "model.layers.34.mlp.experts.130.up_proj", + "model.layers.34.mlp.experts.131.down_proj", + "model.layers.34.mlp.experts.131.gate_proj", + "model.layers.34.mlp.experts.131.up_proj", + "model.layers.34.mlp.experts.132.down_proj", + "model.layers.34.mlp.experts.132.gate_proj", + "model.layers.34.mlp.experts.132.up_proj", + "model.layers.34.mlp.experts.133.down_proj", + "model.layers.34.mlp.experts.133.gate_proj", + "model.layers.34.mlp.experts.133.up_proj", + "model.layers.34.mlp.experts.134.down_proj", + "model.layers.34.mlp.experts.134.gate_proj", + "model.layers.34.mlp.experts.134.up_proj", + "model.layers.34.mlp.experts.135.down_proj", + "model.layers.34.mlp.experts.135.gate_proj", + "model.layers.34.mlp.experts.135.up_proj", + "model.layers.34.mlp.experts.136.down_proj", + "model.layers.34.mlp.experts.136.gate_proj", + "model.layers.34.mlp.experts.136.up_proj", + "model.layers.34.mlp.experts.137.down_proj", + "model.layers.34.mlp.experts.137.gate_proj", + "model.layers.34.mlp.experts.137.up_proj", + "model.layers.34.mlp.experts.138.down_proj", + "model.layers.34.mlp.experts.138.gate_proj", + "model.layers.34.mlp.experts.138.up_proj", + "model.layers.34.mlp.experts.139.down_proj", + "model.layers.34.mlp.experts.139.gate_proj", + "model.layers.34.mlp.experts.139.up_proj", + "model.layers.34.mlp.experts.14.down_proj", + "model.layers.34.mlp.experts.14.gate_proj", + "model.layers.34.mlp.experts.14.up_proj", + "model.layers.34.mlp.experts.140.down_proj", + "model.layers.34.mlp.experts.140.gate_proj", + "model.layers.34.mlp.experts.140.up_proj", + "model.layers.34.mlp.experts.141.down_proj", + "model.layers.34.mlp.experts.141.gate_proj", + "model.layers.34.mlp.experts.141.up_proj", + "model.layers.34.mlp.experts.142.down_proj", + "model.layers.34.mlp.experts.142.gate_proj", + "model.layers.34.mlp.experts.142.up_proj", + "model.layers.34.mlp.experts.143.down_proj", + "model.layers.34.mlp.experts.143.gate_proj", + "model.layers.34.mlp.experts.143.up_proj", + "model.layers.34.mlp.experts.144.down_proj", + "model.layers.34.mlp.experts.144.gate_proj", + "model.layers.34.mlp.experts.144.up_proj", + "model.layers.34.mlp.experts.145.down_proj", + "model.layers.34.mlp.experts.145.gate_proj", + "model.layers.34.mlp.experts.145.up_proj", + "model.layers.34.mlp.experts.146.down_proj", + "model.layers.34.mlp.experts.146.gate_proj", + "model.layers.34.mlp.experts.146.up_proj", + "model.layers.34.mlp.experts.147.down_proj", + "model.layers.34.mlp.experts.147.gate_proj", + "model.layers.34.mlp.experts.147.up_proj", + "model.layers.34.mlp.experts.148.down_proj", + "model.layers.34.mlp.experts.148.gate_proj", + "model.layers.34.mlp.experts.148.up_proj", + "model.layers.34.mlp.experts.149.down_proj", + "model.layers.34.mlp.experts.149.gate_proj", + "model.layers.34.mlp.experts.149.up_proj", + "model.layers.34.mlp.experts.15.down_proj", + "model.layers.34.mlp.experts.15.gate_proj", + "model.layers.34.mlp.experts.15.up_proj", + "model.layers.34.mlp.experts.150.down_proj", + "model.layers.34.mlp.experts.150.gate_proj", + "model.layers.34.mlp.experts.150.up_proj", + "model.layers.34.mlp.experts.151.down_proj", + "model.layers.34.mlp.experts.151.gate_proj", + "model.layers.34.mlp.experts.151.up_proj", + "model.layers.34.mlp.experts.152.down_proj", + "model.layers.34.mlp.experts.152.gate_proj", + "model.layers.34.mlp.experts.152.up_proj", + "model.layers.34.mlp.experts.153.down_proj", + "model.layers.34.mlp.experts.153.gate_proj", + "model.layers.34.mlp.experts.153.up_proj", + "model.layers.34.mlp.experts.154.down_proj", + "model.layers.34.mlp.experts.154.gate_proj", + "model.layers.34.mlp.experts.154.up_proj", + "model.layers.34.mlp.experts.155.down_proj", + "model.layers.34.mlp.experts.155.gate_proj", + "model.layers.34.mlp.experts.155.up_proj", + "model.layers.34.mlp.experts.156.down_proj", + "model.layers.34.mlp.experts.156.gate_proj", + "model.layers.34.mlp.experts.156.up_proj", + "model.layers.34.mlp.experts.157.down_proj", + "model.layers.34.mlp.experts.157.gate_proj", + "model.layers.34.mlp.experts.157.up_proj", + "model.layers.34.mlp.experts.158.down_proj", + "model.layers.34.mlp.experts.158.gate_proj", + "model.layers.34.mlp.experts.158.up_proj", + "model.layers.34.mlp.experts.159.down_proj", + "model.layers.34.mlp.experts.159.gate_proj", + "model.layers.34.mlp.experts.159.up_proj", + "model.layers.34.mlp.experts.16.down_proj", + "model.layers.34.mlp.experts.16.gate_proj", + "model.layers.34.mlp.experts.16.up_proj", + "model.layers.34.mlp.experts.160.down_proj", + "model.layers.34.mlp.experts.160.gate_proj", + "model.layers.34.mlp.experts.160.up_proj", + "model.layers.34.mlp.experts.161.down_proj", + "model.layers.34.mlp.experts.161.gate_proj", + "model.layers.34.mlp.experts.161.up_proj", + "model.layers.34.mlp.experts.162.down_proj", + "model.layers.34.mlp.experts.162.gate_proj", + "model.layers.34.mlp.experts.162.up_proj", + "model.layers.34.mlp.experts.163.down_proj", + "model.layers.34.mlp.experts.163.gate_proj", + "model.layers.34.mlp.experts.163.up_proj", + "model.layers.34.mlp.experts.164.down_proj", + "model.layers.34.mlp.experts.164.gate_proj", + "model.layers.34.mlp.experts.164.up_proj", + "model.layers.34.mlp.experts.165.down_proj", + "model.layers.34.mlp.experts.165.gate_proj", + "model.layers.34.mlp.experts.165.up_proj", + "model.layers.34.mlp.experts.166.down_proj", + "model.layers.34.mlp.experts.166.gate_proj", + "model.layers.34.mlp.experts.166.up_proj", + "model.layers.34.mlp.experts.167.down_proj", + "model.layers.34.mlp.experts.167.gate_proj", + "model.layers.34.mlp.experts.167.up_proj", + "model.layers.34.mlp.experts.168.down_proj", + "model.layers.34.mlp.experts.168.gate_proj", + "model.layers.34.mlp.experts.168.up_proj", + "model.layers.34.mlp.experts.169.down_proj", + "model.layers.34.mlp.experts.169.gate_proj", + "model.layers.34.mlp.experts.169.up_proj", + "model.layers.34.mlp.experts.17.down_proj", + "model.layers.34.mlp.experts.17.gate_proj", + "model.layers.34.mlp.experts.17.up_proj", + "model.layers.34.mlp.experts.170.down_proj", + "model.layers.34.mlp.experts.170.gate_proj", + "model.layers.34.mlp.experts.170.up_proj", + "model.layers.34.mlp.experts.171.down_proj", + "model.layers.34.mlp.experts.171.gate_proj", + "model.layers.34.mlp.experts.171.up_proj", + "model.layers.34.mlp.experts.172.down_proj", + "model.layers.34.mlp.experts.172.gate_proj", + "model.layers.34.mlp.experts.172.up_proj", + "model.layers.34.mlp.experts.173.down_proj", + "model.layers.34.mlp.experts.173.gate_proj", + "model.layers.34.mlp.experts.173.up_proj", + "model.layers.34.mlp.experts.174.down_proj", + "model.layers.34.mlp.experts.174.gate_proj", + "model.layers.34.mlp.experts.174.up_proj", + "model.layers.34.mlp.experts.175.down_proj", + "model.layers.34.mlp.experts.175.gate_proj", + "model.layers.34.mlp.experts.175.up_proj", + "model.layers.34.mlp.experts.176.down_proj", + "model.layers.34.mlp.experts.176.gate_proj", + "model.layers.34.mlp.experts.176.up_proj", + "model.layers.34.mlp.experts.177.down_proj", + "model.layers.34.mlp.experts.177.gate_proj", + "model.layers.34.mlp.experts.177.up_proj", + "model.layers.34.mlp.experts.178.down_proj", + "model.layers.34.mlp.experts.178.gate_proj", + "model.layers.34.mlp.experts.178.up_proj", + "model.layers.34.mlp.experts.179.down_proj", + "model.layers.34.mlp.experts.179.gate_proj", + "model.layers.34.mlp.experts.179.up_proj", + "model.layers.34.mlp.experts.18.down_proj", + "model.layers.34.mlp.experts.18.gate_proj", + "model.layers.34.mlp.experts.18.up_proj", + "model.layers.34.mlp.experts.180.down_proj", + "model.layers.34.mlp.experts.180.gate_proj", + "model.layers.34.mlp.experts.180.up_proj", + "model.layers.34.mlp.experts.181.down_proj", + "model.layers.34.mlp.experts.181.gate_proj", + "model.layers.34.mlp.experts.181.up_proj", + "model.layers.34.mlp.experts.182.down_proj", + "model.layers.34.mlp.experts.182.gate_proj", + "model.layers.34.mlp.experts.182.up_proj", + "model.layers.34.mlp.experts.183.down_proj", + "model.layers.34.mlp.experts.183.gate_proj", + "model.layers.34.mlp.experts.183.up_proj", + "model.layers.34.mlp.experts.184.down_proj", + "model.layers.34.mlp.experts.184.gate_proj", + "model.layers.34.mlp.experts.184.up_proj", + "model.layers.34.mlp.experts.185.down_proj", + "model.layers.34.mlp.experts.185.gate_proj", + "model.layers.34.mlp.experts.185.up_proj", + "model.layers.34.mlp.experts.186.down_proj", + "model.layers.34.mlp.experts.186.gate_proj", + "model.layers.34.mlp.experts.186.up_proj", + "model.layers.34.mlp.experts.187.down_proj", + "model.layers.34.mlp.experts.187.gate_proj", + "model.layers.34.mlp.experts.187.up_proj", + "model.layers.34.mlp.experts.188.down_proj", + "model.layers.34.mlp.experts.188.gate_proj", + "model.layers.34.mlp.experts.188.up_proj", + "model.layers.34.mlp.experts.189.down_proj", + "model.layers.34.mlp.experts.189.gate_proj", + "model.layers.34.mlp.experts.189.up_proj", + "model.layers.34.mlp.experts.19.down_proj", + "model.layers.34.mlp.experts.19.gate_proj", + "model.layers.34.mlp.experts.19.up_proj", + "model.layers.34.mlp.experts.190.down_proj", + "model.layers.34.mlp.experts.190.gate_proj", + "model.layers.34.mlp.experts.190.up_proj", + "model.layers.34.mlp.experts.191.down_proj", + "model.layers.34.mlp.experts.191.gate_proj", + "model.layers.34.mlp.experts.191.up_proj", + "model.layers.34.mlp.experts.192.down_proj", + "model.layers.34.mlp.experts.192.gate_proj", + "model.layers.34.mlp.experts.192.up_proj", + "model.layers.34.mlp.experts.193.down_proj", + "model.layers.34.mlp.experts.193.gate_proj", + "model.layers.34.mlp.experts.193.up_proj", + "model.layers.34.mlp.experts.194.down_proj", + "model.layers.34.mlp.experts.194.gate_proj", + "model.layers.34.mlp.experts.194.up_proj", + "model.layers.34.mlp.experts.195.down_proj", + "model.layers.34.mlp.experts.195.gate_proj", + "model.layers.34.mlp.experts.195.up_proj", + "model.layers.34.mlp.experts.196.down_proj", + "model.layers.34.mlp.experts.196.gate_proj", + "model.layers.34.mlp.experts.196.up_proj", + "model.layers.34.mlp.experts.197.down_proj", + "model.layers.34.mlp.experts.197.gate_proj", + "model.layers.34.mlp.experts.197.up_proj", + "model.layers.34.mlp.experts.198.down_proj", + "model.layers.34.mlp.experts.198.gate_proj", + "model.layers.34.mlp.experts.198.up_proj", + "model.layers.34.mlp.experts.199.down_proj", + "model.layers.34.mlp.experts.199.gate_proj", + "model.layers.34.mlp.experts.199.up_proj", + "model.layers.34.mlp.experts.2.down_proj", + "model.layers.34.mlp.experts.2.gate_proj", + "model.layers.34.mlp.experts.2.up_proj", + "model.layers.34.mlp.experts.20.down_proj", + "model.layers.34.mlp.experts.20.gate_proj", + "model.layers.34.mlp.experts.20.up_proj", + "model.layers.34.mlp.experts.200.down_proj", + "model.layers.34.mlp.experts.200.gate_proj", + "model.layers.34.mlp.experts.200.up_proj", + "model.layers.34.mlp.experts.201.down_proj", + "model.layers.34.mlp.experts.201.gate_proj", + "model.layers.34.mlp.experts.201.up_proj", + "model.layers.34.mlp.experts.202.down_proj", + "model.layers.34.mlp.experts.202.gate_proj", + "model.layers.34.mlp.experts.202.up_proj", + "model.layers.34.mlp.experts.203.down_proj", + "model.layers.34.mlp.experts.203.gate_proj", + "model.layers.34.mlp.experts.203.up_proj", + "model.layers.34.mlp.experts.204.down_proj", + "model.layers.34.mlp.experts.204.gate_proj", + "model.layers.34.mlp.experts.204.up_proj", + "model.layers.34.mlp.experts.205.down_proj", + "model.layers.34.mlp.experts.205.gate_proj", + "model.layers.34.mlp.experts.205.up_proj", + "model.layers.34.mlp.experts.206.down_proj", + "model.layers.34.mlp.experts.206.gate_proj", + "model.layers.34.mlp.experts.206.up_proj", + "model.layers.34.mlp.experts.207.down_proj", + "model.layers.34.mlp.experts.207.gate_proj", + "model.layers.34.mlp.experts.207.up_proj", + "model.layers.34.mlp.experts.208.down_proj", + "model.layers.34.mlp.experts.208.gate_proj", + "model.layers.34.mlp.experts.208.up_proj", + "model.layers.34.mlp.experts.209.down_proj", + "model.layers.34.mlp.experts.209.gate_proj", + "model.layers.34.mlp.experts.209.up_proj", + "model.layers.34.mlp.experts.21.down_proj", + "model.layers.34.mlp.experts.21.gate_proj", + "model.layers.34.mlp.experts.21.up_proj", + "model.layers.34.mlp.experts.210.down_proj", + "model.layers.34.mlp.experts.210.gate_proj", + "model.layers.34.mlp.experts.210.up_proj", + "model.layers.34.mlp.experts.211.down_proj", + "model.layers.34.mlp.experts.211.gate_proj", + "model.layers.34.mlp.experts.211.up_proj", + "model.layers.34.mlp.experts.212.down_proj", + "model.layers.34.mlp.experts.212.gate_proj", + "model.layers.34.mlp.experts.212.up_proj", + "model.layers.34.mlp.experts.213.down_proj", + "model.layers.34.mlp.experts.213.gate_proj", + "model.layers.34.mlp.experts.213.up_proj", + "model.layers.34.mlp.experts.214.down_proj", + "model.layers.34.mlp.experts.214.gate_proj", + "model.layers.34.mlp.experts.214.up_proj", + "model.layers.34.mlp.experts.215.down_proj", + "model.layers.34.mlp.experts.215.gate_proj", + "model.layers.34.mlp.experts.215.up_proj", + "model.layers.34.mlp.experts.216.down_proj", + "model.layers.34.mlp.experts.216.gate_proj", + "model.layers.34.mlp.experts.216.up_proj", + "model.layers.34.mlp.experts.217.down_proj", + "model.layers.34.mlp.experts.217.gate_proj", + "model.layers.34.mlp.experts.217.up_proj", + "model.layers.34.mlp.experts.218.down_proj", + "model.layers.34.mlp.experts.218.gate_proj", + "model.layers.34.mlp.experts.218.up_proj", + "model.layers.34.mlp.experts.219.down_proj", + "model.layers.34.mlp.experts.219.gate_proj", + "model.layers.34.mlp.experts.219.up_proj", + "model.layers.34.mlp.experts.22.down_proj", + "model.layers.34.mlp.experts.22.gate_proj", + "model.layers.34.mlp.experts.22.up_proj", + "model.layers.34.mlp.experts.220.down_proj", + "model.layers.34.mlp.experts.220.gate_proj", + "model.layers.34.mlp.experts.220.up_proj", + "model.layers.34.mlp.experts.221.down_proj", + "model.layers.34.mlp.experts.221.gate_proj", + "model.layers.34.mlp.experts.221.up_proj", + "model.layers.34.mlp.experts.222.down_proj", + "model.layers.34.mlp.experts.222.gate_proj", + "model.layers.34.mlp.experts.222.up_proj", + "model.layers.34.mlp.experts.223.down_proj", + "model.layers.34.mlp.experts.223.gate_proj", + "model.layers.34.mlp.experts.223.up_proj", + "model.layers.34.mlp.experts.224.down_proj", + "model.layers.34.mlp.experts.224.gate_proj", + "model.layers.34.mlp.experts.224.up_proj", + "model.layers.34.mlp.experts.225.down_proj", + "model.layers.34.mlp.experts.225.gate_proj", + "model.layers.34.mlp.experts.225.up_proj", + "model.layers.34.mlp.experts.226.down_proj", + "model.layers.34.mlp.experts.226.gate_proj", + "model.layers.34.mlp.experts.226.up_proj", + "model.layers.34.mlp.experts.227.down_proj", + "model.layers.34.mlp.experts.227.gate_proj", + "model.layers.34.mlp.experts.227.up_proj", + "model.layers.34.mlp.experts.228.down_proj", + "model.layers.34.mlp.experts.228.gate_proj", + "model.layers.34.mlp.experts.228.up_proj", + "model.layers.34.mlp.experts.229.down_proj", + "model.layers.34.mlp.experts.229.gate_proj", + "model.layers.34.mlp.experts.229.up_proj", + "model.layers.34.mlp.experts.23.down_proj", + "model.layers.34.mlp.experts.23.gate_proj", + "model.layers.34.mlp.experts.23.up_proj", + "model.layers.34.mlp.experts.230.down_proj", + "model.layers.34.mlp.experts.230.gate_proj", + "model.layers.34.mlp.experts.230.up_proj", + "model.layers.34.mlp.experts.231.down_proj", + "model.layers.34.mlp.experts.231.gate_proj", + "model.layers.34.mlp.experts.231.up_proj", + "model.layers.34.mlp.experts.232.down_proj", + "model.layers.34.mlp.experts.232.gate_proj", + "model.layers.34.mlp.experts.232.up_proj", + "model.layers.34.mlp.experts.233.down_proj", + "model.layers.34.mlp.experts.233.gate_proj", + "model.layers.34.mlp.experts.233.up_proj", + "model.layers.34.mlp.experts.234.down_proj", + "model.layers.34.mlp.experts.234.gate_proj", + "model.layers.34.mlp.experts.234.up_proj", + "model.layers.34.mlp.experts.235.down_proj", + "model.layers.34.mlp.experts.235.gate_proj", + "model.layers.34.mlp.experts.235.up_proj", + "model.layers.34.mlp.experts.236.down_proj", + "model.layers.34.mlp.experts.236.gate_proj", + "model.layers.34.mlp.experts.236.up_proj", + "model.layers.34.mlp.experts.237.down_proj", + "model.layers.34.mlp.experts.237.gate_proj", + "model.layers.34.mlp.experts.237.up_proj", + "model.layers.34.mlp.experts.238.down_proj", + "model.layers.34.mlp.experts.238.gate_proj", + "model.layers.34.mlp.experts.238.up_proj", + "model.layers.34.mlp.experts.239.down_proj", + "model.layers.34.mlp.experts.239.gate_proj", + "model.layers.34.mlp.experts.239.up_proj", + "model.layers.34.mlp.experts.24.down_proj", + "model.layers.34.mlp.experts.24.gate_proj", + "model.layers.34.mlp.experts.24.up_proj", + "model.layers.34.mlp.experts.240.down_proj", + "model.layers.34.mlp.experts.240.gate_proj", + "model.layers.34.mlp.experts.240.up_proj", + "model.layers.34.mlp.experts.241.down_proj", + "model.layers.34.mlp.experts.241.gate_proj", + "model.layers.34.mlp.experts.241.up_proj", + "model.layers.34.mlp.experts.242.down_proj", + "model.layers.34.mlp.experts.242.gate_proj", + "model.layers.34.mlp.experts.242.up_proj", + "model.layers.34.mlp.experts.243.down_proj", + "model.layers.34.mlp.experts.243.gate_proj", + "model.layers.34.mlp.experts.243.up_proj", + "model.layers.34.mlp.experts.244.down_proj", + "model.layers.34.mlp.experts.244.gate_proj", + "model.layers.34.mlp.experts.244.up_proj", + "model.layers.34.mlp.experts.245.down_proj", + "model.layers.34.mlp.experts.245.gate_proj", + "model.layers.34.mlp.experts.245.up_proj", + "model.layers.34.mlp.experts.246.down_proj", + "model.layers.34.mlp.experts.246.gate_proj", + "model.layers.34.mlp.experts.246.up_proj", + "model.layers.34.mlp.experts.247.down_proj", + "model.layers.34.mlp.experts.247.gate_proj", + "model.layers.34.mlp.experts.247.up_proj", + "model.layers.34.mlp.experts.248.down_proj", + "model.layers.34.mlp.experts.248.gate_proj", + "model.layers.34.mlp.experts.248.up_proj", + "model.layers.34.mlp.experts.249.down_proj", + "model.layers.34.mlp.experts.249.gate_proj", + "model.layers.34.mlp.experts.249.up_proj", + "model.layers.34.mlp.experts.25.down_proj", + "model.layers.34.mlp.experts.25.gate_proj", + "model.layers.34.mlp.experts.25.up_proj", + "model.layers.34.mlp.experts.250.down_proj", + "model.layers.34.mlp.experts.250.gate_proj", + "model.layers.34.mlp.experts.250.up_proj", + "model.layers.34.mlp.experts.251.down_proj", + "model.layers.34.mlp.experts.251.gate_proj", + "model.layers.34.mlp.experts.251.up_proj", + "model.layers.34.mlp.experts.252.down_proj", + "model.layers.34.mlp.experts.252.gate_proj", + "model.layers.34.mlp.experts.252.up_proj", + "model.layers.34.mlp.experts.253.down_proj", + "model.layers.34.mlp.experts.253.gate_proj", + "model.layers.34.mlp.experts.253.up_proj", + "model.layers.34.mlp.experts.254.down_proj", + "model.layers.34.mlp.experts.254.gate_proj", + "model.layers.34.mlp.experts.254.up_proj", + "model.layers.34.mlp.experts.255.down_proj", + "model.layers.34.mlp.experts.255.gate_proj", + "model.layers.34.mlp.experts.255.up_proj", + "model.layers.34.mlp.experts.26.down_proj", + "model.layers.34.mlp.experts.26.gate_proj", + "model.layers.34.mlp.experts.26.up_proj", + "model.layers.34.mlp.experts.27.down_proj", + "model.layers.34.mlp.experts.27.gate_proj", + "model.layers.34.mlp.experts.27.up_proj", + "model.layers.34.mlp.experts.28.down_proj", + "model.layers.34.mlp.experts.28.gate_proj", + "model.layers.34.mlp.experts.28.up_proj", + "model.layers.34.mlp.experts.29.down_proj", + "model.layers.34.mlp.experts.29.gate_proj", + "model.layers.34.mlp.experts.29.up_proj", + "model.layers.34.mlp.experts.3.down_proj", + "model.layers.34.mlp.experts.3.gate_proj", + "model.layers.34.mlp.experts.3.up_proj", + "model.layers.34.mlp.experts.30.down_proj", + "model.layers.34.mlp.experts.30.gate_proj", + "model.layers.34.mlp.experts.30.up_proj", + "model.layers.34.mlp.experts.31.down_proj", + "model.layers.34.mlp.experts.31.gate_proj", + "model.layers.34.mlp.experts.31.up_proj", + "model.layers.34.mlp.experts.32.down_proj", + "model.layers.34.mlp.experts.32.gate_proj", + "model.layers.34.mlp.experts.32.up_proj", + "model.layers.34.mlp.experts.33.down_proj", + "model.layers.34.mlp.experts.33.gate_proj", + "model.layers.34.mlp.experts.33.up_proj", + "model.layers.34.mlp.experts.34.down_proj", + "model.layers.34.mlp.experts.34.gate_proj", + "model.layers.34.mlp.experts.34.up_proj", + "model.layers.34.mlp.experts.35.down_proj", + "model.layers.34.mlp.experts.35.gate_proj", + "model.layers.34.mlp.experts.35.up_proj", + "model.layers.34.mlp.experts.36.down_proj", + "model.layers.34.mlp.experts.36.gate_proj", + "model.layers.34.mlp.experts.36.up_proj", + "model.layers.34.mlp.experts.37.down_proj", + "model.layers.34.mlp.experts.37.gate_proj", + "model.layers.34.mlp.experts.37.up_proj", + "model.layers.34.mlp.experts.38.down_proj", + "model.layers.34.mlp.experts.38.gate_proj", + "model.layers.34.mlp.experts.38.up_proj", + "model.layers.34.mlp.experts.39.down_proj", + "model.layers.34.mlp.experts.39.gate_proj", + "model.layers.34.mlp.experts.39.up_proj", + "model.layers.34.mlp.experts.4.down_proj", + "model.layers.34.mlp.experts.4.gate_proj", + "model.layers.34.mlp.experts.4.up_proj", + "model.layers.34.mlp.experts.40.down_proj", + "model.layers.34.mlp.experts.40.gate_proj", + "model.layers.34.mlp.experts.40.up_proj", + "model.layers.34.mlp.experts.41.down_proj", + "model.layers.34.mlp.experts.41.gate_proj", + "model.layers.34.mlp.experts.41.up_proj", + "model.layers.34.mlp.experts.42.down_proj", + "model.layers.34.mlp.experts.42.gate_proj", + "model.layers.34.mlp.experts.42.up_proj", + "model.layers.34.mlp.experts.43.down_proj", + "model.layers.34.mlp.experts.43.gate_proj", + "model.layers.34.mlp.experts.43.up_proj", + "model.layers.34.mlp.experts.44.down_proj", + "model.layers.34.mlp.experts.44.gate_proj", + "model.layers.34.mlp.experts.44.up_proj", + "model.layers.34.mlp.experts.45.down_proj", + "model.layers.34.mlp.experts.45.gate_proj", + "model.layers.34.mlp.experts.45.up_proj", + "model.layers.34.mlp.experts.46.down_proj", + "model.layers.34.mlp.experts.46.gate_proj", + "model.layers.34.mlp.experts.46.up_proj", + "model.layers.34.mlp.experts.47.down_proj", + "model.layers.34.mlp.experts.47.gate_proj", + "model.layers.34.mlp.experts.47.up_proj", + "model.layers.34.mlp.experts.48.down_proj", + "model.layers.34.mlp.experts.48.gate_proj", + "model.layers.34.mlp.experts.48.up_proj", + "model.layers.34.mlp.experts.49.down_proj", + "model.layers.34.mlp.experts.49.gate_proj", + "model.layers.34.mlp.experts.49.up_proj", + "model.layers.34.mlp.experts.5.down_proj", + "model.layers.34.mlp.experts.5.gate_proj", + "model.layers.34.mlp.experts.5.up_proj", + "model.layers.34.mlp.experts.50.down_proj", + "model.layers.34.mlp.experts.50.gate_proj", + "model.layers.34.mlp.experts.50.up_proj", + "model.layers.34.mlp.experts.51.down_proj", + "model.layers.34.mlp.experts.51.gate_proj", + "model.layers.34.mlp.experts.51.up_proj", + "model.layers.34.mlp.experts.52.down_proj", + "model.layers.34.mlp.experts.52.gate_proj", + "model.layers.34.mlp.experts.52.up_proj", + "model.layers.34.mlp.experts.53.down_proj", + "model.layers.34.mlp.experts.53.gate_proj", + "model.layers.34.mlp.experts.53.up_proj", + "model.layers.34.mlp.experts.54.down_proj", + "model.layers.34.mlp.experts.54.gate_proj", + "model.layers.34.mlp.experts.54.up_proj", + "model.layers.34.mlp.experts.55.down_proj", + "model.layers.34.mlp.experts.55.gate_proj", + "model.layers.34.mlp.experts.55.up_proj", + "model.layers.34.mlp.experts.56.down_proj", + "model.layers.34.mlp.experts.56.gate_proj", + "model.layers.34.mlp.experts.56.up_proj", + "model.layers.34.mlp.experts.57.down_proj", + "model.layers.34.mlp.experts.57.gate_proj", + "model.layers.34.mlp.experts.57.up_proj", + "model.layers.34.mlp.experts.58.down_proj", + "model.layers.34.mlp.experts.58.gate_proj", + "model.layers.34.mlp.experts.58.up_proj", + "model.layers.34.mlp.experts.59.down_proj", + "model.layers.34.mlp.experts.59.gate_proj", + "model.layers.34.mlp.experts.59.up_proj", + "model.layers.34.mlp.experts.6.down_proj", + "model.layers.34.mlp.experts.6.gate_proj", + "model.layers.34.mlp.experts.6.up_proj", + "model.layers.34.mlp.experts.60.down_proj", + "model.layers.34.mlp.experts.60.gate_proj", + "model.layers.34.mlp.experts.60.up_proj", + "model.layers.34.mlp.experts.61.down_proj", + "model.layers.34.mlp.experts.61.gate_proj", + "model.layers.34.mlp.experts.61.up_proj", + "model.layers.34.mlp.experts.62.down_proj", + "model.layers.34.mlp.experts.62.gate_proj", + "model.layers.34.mlp.experts.62.up_proj", + "model.layers.34.mlp.experts.63.down_proj", + "model.layers.34.mlp.experts.63.gate_proj", + "model.layers.34.mlp.experts.63.up_proj", + "model.layers.34.mlp.experts.64.down_proj", + "model.layers.34.mlp.experts.64.gate_proj", + "model.layers.34.mlp.experts.64.up_proj", + "model.layers.34.mlp.experts.65.down_proj", + "model.layers.34.mlp.experts.65.gate_proj", + "model.layers.34.mlp.experts.65.up_proj", + "model.layers.34.mlp.experts.66.down_proj", + "model.layers.34.mlp.experts.66.gate_proj", + "model.layers.34.mlp.experts.66.up_proj", + "model.layers.34.mlp.experts.67.down_proj", + "model.layers.34.mlp.experts.67.gate_proj", + "model.layers.34.mlp.experts.67.up_proj", + "model.layers.34.mlp.experts.68.down_proj", + "model.layers.34.mlp.experts.68.gate_proj", + "model.layers.34.mlp.experts.68.up_proj", + "model.layers.34.mlp.experts.69.down_proj", + "model.layers.34.mlp.experts.69.gate_proj", + "model.layers.34.mlp.experts.69.up_proj", + "model.layers.34.mlp.experts.7.down_proj", + "model.layers.34.mlp.experts.7.gate_proj", + "model.layers.34.mlp.experts.7.up_proj", + "model.layers.34.mlp.experts.70.down_proj", + "model.layers.34.mlp.experts.70.gate_proj", + "model.layers.34.mlp.experts.70.up_proj", + "model.layers.34.mlp.experts.71.down_proj", + "model.layers.34.mlp.experts.71.gate_proj", + "model.layers.34.mlp.experts.71.up_proj", + "model.layers.34.mlp.experts.72.down_proj", + "model.layers.34.mlp.experts.72.gate_proj", + "model.layers.34.mlp.experts.72.up_proj", + "model.layers.34.mlp.experts.73.down_proj", + "model.layers.34.mlp.experts.73.gate_proj", + "model.layers.34.mlp.experts.73.up_proj", + "model.layers.34.mlp.experts.74.down_proj", + "model.layers.34.mlp.experts.74.gate_proj", + "model.layers.34.mlp.experts.74.up_proj", + "model.layers.34.mlp.experts.75.down_proj", + "model.layers.34.mlp.experts.75.gate_proj", + "model.layers.34.mlp.experts.75.up_proj", + "model.layers.34.mlp.experts.76.down_proj", + "model.layers.34.mlp.experts.76.gate_proj", + "model.layers.34.mlp.experts.76.up_proj", + "model.layers.34.mlp.experts.77.down_proj", + "model.layers.34.mlp.experts.77.gate_proj", + "model.layers.34.mlp.experts.77.up_proj", + "model.layers.34.mlp.experts.78.down_proj", + "model.layers.34.mlp.experts.78.gate_proj", + "model.layers.34.mlp.experts.78.up_proj", + "model.layers.34.mlp.experts.79.down_proj", + "model.layers.34.mlp.experts.79.gate_proj", + "model.layers.34.mlp.experts.79.up_proj", + "model.layers.34.mlp.experts.8.down_proj", + "model.layers.34.mlp.experts.8.gate_proj", + "model.layers.34.mlp.experts.8.up_proj", + "model.layers.34.mlp.experts.80.down_proj", + "model.layers.34.mlp.experts.80.gate_proj", + "model.layers.34.mlp.experts.80.up_proj", + "model.layers.34.mlp.experts.81.down_proj", + "model.layers.34.mlp.experts.81.gate_proj", + "model.layers.34.mlp.experts.81.up_proj", + "model.layers.34.mlp.experts.82.down_proj", + "model.layers.34.mlp.experts.82.gate_proj", + "model.layers.34.mlp.experts.82.up_proj", + "model.layers.34.mlp.experts.83.down_proj", + "model.layers.34.mlp.experts.83.gate_proj", + "model.layers.34.mlp.experts.83.up_proj", + "model.layers.34.mlp.experts.84.down_proj", + "model.layers.34.mlp.experts.84.gate_proj", + "model.layers.34.mlp.experts.84.up_proj", + "model.layers.34.mlp.experts.85.down_proj", + "model.layers.34.mlp.experts.85.gate_proj", + "model.layers.34.mlp.experts.85.up_proj", + "model.layers.34.mlp.experts.86.down_proj", + "model.layers.34.mlp.experts.86.gate_proj", + "model.layers.34.mlp.experts.86.up_proj", + "model.layers.34.mlp.experts.87.down_proj", + "model.layers.34.mlp.experts.87.gate_proj", + "model.layers.34.mlp.experts.87.up_proj", + "model.layers.34.mlp.experts.88.down_proj", + "model.layers.34.mlp.experts.88.gate_proj", + "model.layers.34.mlp.experts.88.up_proj", + "model.layers.34.mlp.experts.89.down_proj", + "model.layers.34.mlp.experts.89.gate_proj", + "model.layers.34.mlp.experts.89.up_proj", + "model.layers.34.mlp.experts.9.down_proj", + "model.layers.34.mlp.experts.9.gate_proj", + "model.layers.34.mlp.experts.9.up_proj", + "model.layers.34.mlp.experts.90.down_proj", + "model.layers.34.mlp.experts.90.gate_proj", + "model.layers.34.mlp.experts.90.up_proj", + "model.layers.34.mlp.experts.91.down_proj", + "model.layers.34.mlp.experts.91.gate_proj", + "model.layers.34.mlp.experts.91.up_proj", + "model.layers.34.mlp.experts.92.down_proj", + "model.layers.34.mlp.experts.92.gate_proj", + "model.layers.34.mlp.experts.92.up_proj", + "model.layers.34.mlp.experts.93.down_proj", + "model.layers.34.mlp.experts.93.gate_proj", + "model.layers.34.mlp.experts.93.up_proj", + "model.layers.34.mlp.experts.94.down_proj", + "model.layers.34.mlp.experts.94.gate_proj", + "model.layers.34.mlp.experts.94.up_proj", + "model.layers.34.mlp.experts.95.down_proj", + "model.layers.34.mlp.experts.95.gate_proj", + "model.layers.34.mlp.experts.95.up_proj", + "model.layers.34.mlp.experts.96.down_proj", + "model.layers.34.mlp.experts.96.gate_proj", + "model.layers.34.mlp.experts.96.up_proj", + "model.layers.34.mlp.experts.97.down_proj", + "model.layers.34.mlp.experts.97.gate_proj", + "model.layers.34.mlp.experts.97.up_proj", + "model.layers.34.mlp.experts.98.down_proj", + "model.layers.34.mlp.experts.98.gate_proj", + "model.layers.34.mlp.experts.98.up_proj", + "model.layers.34.mlp.experts.99.down_proj", + "model.layers.34.mlp.experts.99.gate_proj", + "model.layers.34.mlp.experts.99.up_proj", + "model.layers.35.mlp.experts.0.down_proj", + "model.layers.35.mlp.experts.0.gate_proj", + "model.layers.35.mlp.experts.0.up_proj", + "model.layers.35.mlp.experts.1.down_proj", + "model.layers.35.mlp.experts.1.gate_proj", + "model.layers.35.mlp.experts.1.up_proj", + "model.layers.35.mlp.experts.10.down_proj", + "model.layers.35.mlp.experts.10.gate_proj", + "model.layers.35.mlp.experts.10.up_proj", + "model.layers.35.mlp.experts.100.down_proj", + "model.layers.35.mlp.experts.100.gate_proj", + "model.layers.35.mlp.experts.100.up_proj", + "model.layers.35.mlp.experts.101.down_proj", + "model.layers.35.mlp.experts.101.gate_proj", + "model.layers.35.mlp.experts.101.up_proj", + "model.layers.35.mlp.experts.102.down_proj", + "model.layers.35.mlp.experts.102.gate_proj", + "model.layers.35.mlp.experts.102.up_proj", + "model.layers.35.mlp.experts.103.down_proj", + "model.layers.35.mlp.experts.103.gate_proj", + "model.layers.35.mlp.experts.103.up_proj", + "model.layers.35.mlp.experts.104.down_proj", + "model.layers.35.mlp.experts.104.gate_proj", + "model.layers.35.mlp.experts.104.up_proj", + "model.layers.35.mlp.experts.105.down_proj", + "model.layers.35.mlp.experts.105.gate_proj", + "model.layers.35.mlp.experts.105.up_proj", + "model.layers.35.mlp.experts.106.down_proj", + "model.layers.35.mlp.experts.106.gate_proj", + "model.layers.35.mlp.experts.106.up_proj", + "model.layers.35.mlp.experts.107.down_proj", + "model.layers.35.mlp.experts.107.gate_proj", + "model.layers.35.mlp.experts.107.up_proj", + "model.layers.35.mlp.experts.108.down_proj", + "model.layers.35.mlp.experts.108.gate_proj", + "model.layers.35.mlp.experts.108.up_proj", + "model.layers.35.mlp.experts.109.down_proj", + "model.layers.35.mlp.experts.109.gate_proj", + "model.layers.35.mlp.experts.109.up_proj", + "model.layers.35.mlp.experts.11.down_proj", + "model.layers.35.mlp.experts.11.gate_proj", + "model.layers.35.mlp.experts.11.up_proj", + "model.layers.35.mlp.experts.110.down_proj", + "model.layers.35.mlp.experts.110.gate_proj", + "model.layers.35.mlp.experts.110.up_proj", + "model.layers.35.mlp.experts.111.down_proj", + "model.layers.35.mlp.experts.111.gate_proj", + "model.layers.35.mlp.experts.111.up_proj", + "model.layers.35.mlp.experts.112.down_proj", + "model.layers.35.mlp.experts.112.gate_proj", + "model.layers.35.mlp.experts.112.up_proj", + "model.layers.35.mlp.experts.113.down_proj", + "model.layers.35.mlp.experts.113.gate_proj", + "model.layers.35.mlp.experts.113.up_proj", + "model.layers.35.mlp.experts.114.down_proj", + "model.layers.35.mlp.experts.114.gate_proj", + "model.layers.35.mlp.experts.114.up_proj", + "model.layers.35.mlp.experts.115.down_proj", + "model.layers.35.mlp.experts.115.gate_proj", + "model.layers.35.mlp.experts.115.up_proj", + "model.layers.35.mlp.experts.116.down_proj", + "model.layers.35.mlp.experts.116.gate_proj", + "model.layers.35.mlp.experts.116.up_proj", + "model.layers.35.mlp.experts.117.down_proj", + "model.layers.35.mlp.experts.117.gate_proj", + "model.layers.35.mlp.experts.117.up_proj", + "model.layers.35.mlp.experts.118.down_proj", + "model.layers.35.mlp.experts.118.gate_proj", + "model.layers.35.mlp.experts.118.up_proj", + "model.layers.35.mlp.experts.119.down_proj", + "model.layers.35.mlp.experts.119.gate_proj", + "model.layers.35.mlp.experts.119.up_proj", + "model.layers.35.mlp.experts.12.down_proj", + "model.layers.35.mlp.experts.12.gate_proj", + "model.layers.35.mlp.experts.12.up_proj", + "model.layers.35.mlp.experts.120.down_proj", + "model.layers.35.mlp.experts.120.gate_proj", + "model.layers.35.mlp.experts.120.up_proj", + "model.layers.35.mlp.experts.121.down_proj", + "model.layers.35.mlp.experts.121.gate_proj", + "model.layers.35.mlp.experts.121.up_proj", + "model.layers.35.mlp.experts.122.down_proj", + "model.layers.35.mlp.experts.122.gate_proj", + "model.layers.35.mlp.experts.122.up_proj", + "model.layers.35.mlp.experts.123.down_proj", + "model.layers.35.mlp.experts.123.gate_proj", + "model.layers.35.mlp.experts.123.up_proj", + "model.layers.35.mlp.experts.124.down_proj", + "model.layers.35.mlp.experts.124.gate_proj", + "model.layers.35.mlp.experts.124.up_proj", + "model.layers.35.mlp.experts.125.down_proj", + "model.layers.35.mlp.experts.125.gate_proj", + "model.layers.35.mlp.experts.125.up_proj", + "model.layers.35.mlp.experts.126.down_proj", + "model.layers.35.mlp.experts.126.gate_proj", + "model.layers.35.mlp.experts.126.up_proj", + "model.layers.35.mlp.experts.127.down_proj", + "model.layers.35.mlp.experts.127.gate_proj", + "model.layers.35.mlp.experts.127.up_proj", + "model.layers.35.mlp.experts.128.down_proj", + "model.layers.35.mlp.experts.128.gate_proj", + "model.layers.35.mlp.experts.128.up_proj", + "model.layers.35.mlp.experts.129.down_proj", + "model.layers.35.mlp.experts.129.gate_proj", + "model.layers.35.mlp.experts.129.up_proj", + "model.layers.35.mlp.experts.13.down_proj", + "model.layers.35.mlp.experts.13.gate_proj", + "model.layers.35.mlp.experts.13.up_proj", + "model.layers.35.mlp.experts.130.down_proj", + "model.layers.35.mlp.experts.130.gate_proj", + "model.layers.35.mlp.experts.130.up_proj", + "model.layers.35.mlp.experts.131.down_proj", + "model.layers.35.mlp.experts.131.gate_proj", + "model.layers.35.mlp.experts.131.up_proj", + "model.layers.35.mlp.experts.132.down_proj", + "model.layers.35.mlp.experts.132.gate_proj", + "model.layers.35.mlp.experts.132.up_proj", + "model.layers.35.mlp.experts.133.down_proj", + "model.layers.35.mlp.experts.133.gate_proj", + "model.layers.35.mlp.experts.133.up_proj", + "model.layers.35.mlp.experts.134.down_proj", + "model.layers.35.mlp.experts.134.gate_proj", + "model.layers.35.mlp.experts.134.up_proj", + "model.layers.35.mlp.experts.135.down_proj", + "model.layers.35.mlp.experts.135.gate_proj", + "model.layers.35.mlp.experts.135.up_proj", + "model.layers.35.mlp.experts.136.down_proj", + "model.layers.35.mlp.experts.136.gate_proj", + "model.layers.35.mlp.experts.136.up_proj", + "model.layers.35.mlp.experts.137.down_proj", + "model.layers.35.mlp.experts.137.gate_proj", + "model.layers.35.mlp.experts.137.up_proj", + "model.layers.35.mlp.experts.138.down_proj", + "model.layers.35.mlp.experts.138.gate_proj", + "model.layers.35.mlp.experts.138.up_proj", + "model.layers.35.mlp.experts.139.down_proj", + "model.layers.35.mlp.experts.139.gate_proj", + "model.layers.35.mlp.experts.139.up_proj", + "model.layers.35.mlp.experts.14.down_proj", + "model.layers.35.mlp.experts.14.gate_proj", + "model.layers.35.mlp.experts.14.up_proj", + "model.layers.35.mlp.experts.140.down_proj", + "model.layers.35.mlp.experts.140.gate_proj", + "model.layers.35.mlp.experts.140.up_proj", + "model.layers.35.mlp.experts.141.down_proj", + "model.layers.35.mlp.experts.141.gate_proj", + "model.layers.35.mlp.experts.141.up_proj", + "model.layers.35.mlp.experts.142.down_proj", + "model.layers.35.mlp.experts.142.gate_proj", + "model.layers.35.mlp.experts.142.up_proj", + "model.layers.35.mlp.experts.143.down_proj", + "model.layers.35.mlp.experts.143.gate_proj", + "model.layers.35.mlp.experts.143.up_proj", + "model.layers.35.mlp.experts.144.down_proj", + "model.layers.35.mlp.experts.144.gate_proj", + "model.layers.35.mlp.experts.144.up_proj", + "model.layers.35.mlp.experts.145.down_proj", + "model.layers.35.mlp.experts.145.gate_proj", + "model.layers.35.mlp.experts.145.up_proj", + "model.layers.35.mlp.experts.146.down_proj", + "model.layers.35.mlp.experts.146.gate_proj", + "model.layers.35.mlp.experts.146.up_proj", + "model.layers.35.mlp.experts.147.down_proj", + "model.layers.35.mlp.experts.147.gate_proj", + "model.layers.35.mlp.experts.147.up_proj", + "model.layers.35.mlp.experts.148.down_proj", + "model.layers.35.mlp.experts.148.gate_proj", + "model.layers.35.mlp.experts.148.up_proj", + "model.layers.35.mlp.experts.149.down_proj", + "model.layers.35.mlp.experts.149.gate_proj", + "model.layers.35.mlp.experts.149.up_proj", + "model.layers.35.mlp.experts.15.down_proj", + "model.layers.35.mlp.experts.15.gate_proj", + "model.layers.35.mlp.experts.15.up_proj", + "model.layers.35.mlp.experts.150.down_proj", + "model.layers.35.mlp.experts.150.gate_proj", + "model.layers.35.mlp.experts.150.up_proj", + "model.layers.35.mlp.experts.151.down_proj", + "model.layers.35.mlp.experts.151.gate_proj", + "model.layers.35.mlp.experts.151.up_proj", + "model.layers.35.mlp.experts.152.down_proj", + "model.layers.35.mlp.experts.152.gate_proj", + "model.layers.35.mlp.experts.152.up_proj", + "model.layers.35.mlp.experts.153.down_proj", + "model.layers.35.mlp.experts.153.gate_proj", + "model.layers.35.mlp.experts.153.up_proj", + "model.layers.35.mlp.experts.154.down_proj", + "model.layers.35.mlp.experts.154.gate_proj", + "model.layers.35.mlp.experts.154.up_proj", + "model.layers.35.mlp.experts.155.down_proj", + "model.layers.35.mlp.experts.155.gate_proj", + "model.layers.35.mlp.experts.155.up_proj", + "model.layers.35.mlp.experts.156.down_proj", + "model.layers.35.mlp.experts.156.gate_proj", + "model.layers.35.mlp.experts.156.up_proj", + "model.layers.35.mlp.experts.157.down_proj", + "model.layers.35.mlp.experts.157.gate_proj", + "model.layers.35.mlp.experts.157.up_proj", + "model.layers.35.mlp.experts.158.down_proj", + "model.layers.35.mlp.experts.158.gate_proj", + "model.layers.35.mlp.experts.158.up_proj", + "model.layers.35.mlp.experts.159.down_proj", + "model.layers.35.mlp.experts.159.gate_proj", + "model.layers.35.mlp.experts.159.up_proj", + "model.layers.35.mlp.experts.16.down_proj", + "model.layers.35.mlp.experts.16.gate_proj", + "model.layers.35.mlp.experts.16.up_proj", + "model.layers.35.mlp.experts.160.down_proj", + "model.layers.35.mlp.experts.160.gate_proj", + "model.layers.35.mlp.experts.160.up_proj", + "model.layers.35.mlp.experts.161.down_proj", + "model.layers.35.mlp.experts.161.gate_proj", + "model.layers.35.mlp.experts.161.up_proj", + "model.layers.35.mlp.experts.162.down_proj", + "model.layers.35.mlp.experts.162.gate_proj", + "model.layers.35.mlp.experts.162.up_proj", + "model.layers.35.mlp.experts.163.down_proj", + "model.layers.35.mlp.experts.163.gate_proj", + "model.layers.35.mlp.experts.163.up_proj", + "model.layers.35.mlp.experts.164.down_proj", + "model.layers.35.mlp.experts.164.gate_proj", + "model.layers.35.mlp.experts.164.up_proj", + "model.layers.35.mlp.experts.165.down_proj", + "model.layers.35.mlp.experts.165.gate_proj", + "model.layers.35.mlp.experts.165.up_proj", + "model.layers.35.mlp.experts.166.down_proj", + "model.layers.35.mlp.experts.166.gate_proj", + "model.layers.35.mlp.experts.166.up_proj", + "model.layers.35.mlp.experts.167.down_proj", + "model.layers.35.mlp.experts.167.gate_proj", + "model.layers.35.mlp.experts.167.up_proj", + "model.layers.35.mlp.experts.168.down_proj", + "model.layers.35.mlp.experts.168.gate_proj", + "model.layers.35.mlp.experts.168.up_proj", + "model.layers.35.mlp.experts.169.down_proj", + "model.layers.35.mlp.experts.169.gate_proj", + "model.layers.35.mlp.experts.169.up_proj", + "model.layers.35.mlp.experts.17.down_proj", + "model.layers.35.mlp.experts.17.gate_proj", + "model.layers.35.mlp.experts.17.up_proj", + "model.layers.35.mlp.experts.170.down_proj", + "model.layers.35.mlp.experts.170.gate_proj", + "model.layers.35.mlp.experts.170.up_proj", + "model.layers.35.mlp.experts.171.down_proj", + "model.layers.35.mlp.experts.171.gate_proj", + "model.layers.35.mlp.experts.171.up_proj", + "model.layers.35.mlp.experts.172.down_proj", + "model.layers.35.mlp.experts.172.gate_proj", + "model.layers.35.mlp.experts.172.up_proj", + "model.layers.35.mlp.experts.173.down_proj", + "model.layers.35.mlp.experts.173.gate_proj", + "model.layers.35.mlp.experts.173.up_proj", + "model.layers.35.mlp.experts.174.down_proj", + "model.layers.35.mlp.experts.174.gate_proj", + "model.layers.35.mlp.experts.174.up_proj", + "model.layers.35.mlp.experts.175.down_proj", + "model.layers.35.mlp.experts.175.gate_proj", + "model.layers.35.mlp.experts.175.up_proj", + "model.layers.35.mlp.experts.176.down_proj", + "model.layers.35.mlp.experts.176.gate_proj", + "model.layers.35.mlp.experts.176.up_proj", + "model.layers.35.mlp.experts.177.down_proj", + "model.layers.35.mlp.experts.177.gate_proj", + "model.layers.35.mlp.experts.177.up_proj", + "model.layers.35.mlp.experts.178.down_proj", + "model.layers.35.mlp.experts.178.gate_proj", + "model.layers.35.mlp.experts.178.up_proj", + "model.layers.35.mlp.experts.179.down_proj", + "model.layers.35.mlp.experts.179.gate_proj", + "model.layers.35.mlp.experts.179.up_proj", + "model.layers.35.mlp.experts.18.down_proj", + "model.layers.35.mlp.experts.18.gate_proj", + "model.layers.35.mlp.experts.18.up_proj", + "model.layers.35.mlp.experts.180.down_proj", + "model.layers.35.mlp.experts.180.gate_proj", + "model.layers.35.mlp.experts.180.up_proj", + "model.layers.35.mlp.experts.181.down_proj", + "model.layers.35.mlp.experts.181.gate_proj", + "model.layers.35.mlp.experts.181.up_proj", + "model.layers.35.mlp.experts.182.down_proj", + "model.layers.35.mlp.experts.182.gate_proj", + "model.layers.35.mlp.experts.182.up_proj", + "model.layers.35.mlp.experts.183.down_proj", + "model.layers.35.mlp.experts.183.gate_proj", + "model.layers.35.mlp.experts.183.up_proj", + "model.layers.35.mlp.experts.184.down_proj", + "model.layers.35.mlp.experts.184.gate_proj", + "model.layers.35.mlp.experts.184.up_proj", + "model.layers.35.mlp.experts.185.down_proj", + "model.layers.35.mlp.experts.185.gate_proj", + "model.layers.35.mlp.experts.185.up_proj", + "model.layers.35.mlp.experts.186.down_proj", + "model.layers.35.mlp.experts.186.gate_proj", + "model.layers.35.mlp.experts.186.up_proj", + "model.layers.35.mlp.experts.187.down_proj", + "model.layers.35.mlp.experts.187.gate_proj", + "model.layers.35.mlp.experts.187.up_proj", + "model.layers.35.mlp.experts.188.down_proj", + "model.layers.35.mlp.experts.188.gate_proj", + "model.layers.35.mlp.experts.188.up_proj", + "model.layers.35.mlp.experts.189.down_proj", + "model.layers.35.mlp.experts.189.gate_proj", + "model.layers.35.mlp.experts.189.up_proj", + "model.layers.35.mlp.experts.19.down_proj", + "model.layers.35.mlp.experts.19.gate_proj", + "model.layers.35.mlp.experts.19.up_proj", + "model.layers.35.mlp.experts.190.down_proj", + "model.layers.35.mlp.experts.190.gate_proj", + "model.layers.35.mlp.experts.190.up_proj", + "model.layers.35.mlp.experts.191.down_proj", + "model.layers.35.mlp.experts.191.gate_proj", + "model.layers.35.mlp.experts.191.up_proj", + "model.layers.35.mlp.experts.192.down_proj", + "model.layers.35.mlp.experts.192.gate_proj", + "model.layers.35.mlp.experts.192.up_proj", + "model.layers.35.mlp.experts.193.down_proj", + "model.layers.35.mlp.experts.193.gate_proj", + "model.layers.35.mlp.experts.193.up_proj", + "model.layers.35.mlp.experts.194.down_proj", + "model.layers.35.mlp.experts.194.gate_proj", + "model.layers.35.mlp.experts.194.up_proj", + "model.layers.35.mlp.experts.195.down_proj", + "model.layers.35.mlp.experts.195.gate_proj", + "model.layers.35.mlp.experts.195.up_proj", + "model.layers.35.mlp.experts.196.down_proj", + "model.layers.35.mlp.experts.196.gate_proj", + "model.layers.35.mlp.experts.196.up_proj", + "model.layers.35.mlp.experts.197.down_proj", + "model.layers.35.mlp.experts.197.gate_proj", + "model.layers.35.mlp.experts.197.up_proj", + "model.layers.35.mlp.experts.198.down_proj", + "model.layers.35.mlp.experts.198.gate_proj", + "model.layers.35.mlp.experts.198.up_proj", + "model.layers.35.mlp.experts.199.down_proj", + "model.layers.35.mlp.experts.199.gate_proj", + "model.layers.35.mlp.experts.199.up_proj", + "model.layers.35.mlp.experts.2.down_proj", + "model.layers.35.mlp.experts.2.gate_proj", + "model.layers.35.mlp.experts.2.up_proj", + "model.layers.35.mlp.experts.20.down_proj", + "model.layers.35.mlp.experts.20.gate_proj", + "model.layers.35.mlp.experts.20.up_proj", + "model.layers.35.mlp.experts.200.down_proj", + "model.layers.35.mlp.experts.200.gate_proj", + "model.layers.35.mlp.experts.200.up_proj", + "model.layers.35.mlp.experts.201.down_proj", + "model.layers.35.mlp.experts.201.gate_proj", + "model.layers.35.mlp.experts.201.up_proj", + "model.layers.35.mlp.experts.202.down_proj", + "model.layers.35.mlp.experts.202.gate_proj", + "model.layers.35.mlp.experts.202.up_proj", + "model.layers.35.mlp.experts.203.down_proj", + "model.layers.35.mlp.experts.203.gate_proj", + "model.layers.35.mlp.experts.203.up_proj", + "model.layers.35.mlp.experts.204.down_proj", + "model.layers.35.mlp.experts.204.gate_proj", + "model.layers.35.mlp.experts.204.up_proj", + "model.layers.35.mlp.experts.205.down_proj", + "model.layers.35.mlp.experts.205.gate_proj", + "model.layers.35.mlp.experts.205.up_proj", + "model.layers.35.mlp.experts.206.down_proj", + "model.layers.35.mlp.experts.206.gate_proj", + "model.layers.35.mlp.experts.206.up_proj", + "model.layers.35.mlp.experts.207.down_proj", + "model.layers.35.mlp.experts.207.gate_proj", + "model.layers.35.mlp.experts.207.up_proj", + "model.layers.35.mlp.experts.208.down_proj", + "model.layers.35.mlp.experts.208.gate_proj", + "model.layers.35.mlp.experts.208.up_proj", + "model.layers.35.mlp.experts.209.down_proj", + "model.layers.35.mlp.experts.209.gate_proj", + "model.layers.35.mlp.experts.209.up_proj", + "model.layers.35.mlp.experts.21.down_proj", + "model.layers.35.mlp.experts.21.gate_proj", + "model.layers.35.mlp.experts.21.up_proj", + "model.layers.35.mlp.experts.210.down_proj", + "model.layers.35.mlp.experts.210.gate_proj", + "model.layers.35.mlp.experts.210.up_proj", + "model.layers.35.mlp.experts.211.down_proj", + "model.layers.35.mlp.experts.211.gate_proj", + "model.layers.35.mlp.experts.211.up_proj", + "model.layers.35.mlp.experts.212.down_proj", + "model.layers.35.mlp.experts.212.gate_proj", + "model.layers.35.mlp.experts.212.up_proj", + "model.layers.35.mlp.experts.213.down_proj", + "model.layers.35.mlp.experts.213.gate_proj", + "model.layers.35.mlp.experts.213.up_proj", + "model.layers.35.mlp.experts.214.down_proj", + "model.layers.35.mlp.experts.214.gate_proj", + "model.layers.35.mlp.experts.214.up_proj", + "model.layers.35.mlp.experts.215.down_proj", + "model.layers.35.mlp.experts.215.gate_proj", + "model.layers.35.mlp.experts.215.up_proj", + "model.layers.35.mlp.experts.216.down_proj", + "model.layers.35.mlp.experts.216.gate_proj", + "model.layers.35.mlp.experts.216.up_proj", + "model.layers.35.mlp.experts.217.down_proj", + "model.layers.35.mlp.experts.217.gate_proj", + "model.layers.35.mlp.experts.217.up_proj", + "model.layers.35.mlp.experts.218.down_proj", + "model.layers.35.mlp.experts.218.gate_proj", + "model.layers.35.mlp.experts.218.up_proj", + "model.layers.35.mlp.experts.219.down_proj", + "model.layers.35.mlp.experts.219.gate_proj", + "model.layers.35.mlp.experts.219.up_proj", + "model.layers.35.mlp.experts.22.down_proj", + "model.layers.35.mlp.experts.22.gate_proj", + "model.layers.35.mlp.experts.22.up_proj", + "model.layers.35.mlp.experts.220.down_proj", + "model.layers.35.mlp.experts.220.gate_proj", + "model.layers.35.mlp.experts.220.up_proj", + "model.layers.35.mlp.experts.221.down_proj", + "model.layers.35.mlp.experts.221.gate_proj", + "model.layers.35.mlp.experts.221.up_proj", + "model.layers.35.mlp.experts.222.down_proj", + "model.layers.35.mlp.experts.222.gate_proj", + "model.layers.35.mlp.experts.222.up_proj", + "model.layers.35.mlp.experts.223.down_proj", + "model.layers.35.mlp.experts.223.gate_proj", + "model.layers.35.mlp.experts.223.up_proj", + "model.layers.35.mlp.experts.224.down_proj", + "model.layers.35.mlp.experts.224.gate_proj", + "model.layers.35.mlp.experts.224.up_proj", + "model.layers.35.mlp.experts.225.down_proj", + "model.layers.35.mlp.experts.225.gate_proj", + "model.layers.35.mlp.experts.225.up_proj", + "model.layers.35.mlp.experts.226.down_proj", + "model.layers.35.mlp.experts.226.gate_proj", + "model.layers.35.mlp.experts.226.up_proj", + "model.layers.35.mlp.experts.227.down_proj", + "model.layers.35.mlp.experts.227.gate_proj", + "model.layers.35.mlp.experts.227.up_proj", + "model.layers.35.mlp.experts.228.down_proj", + "model.layers.35.mlp.experts.228.gate_proj", + "model.layers.35.mlp.experts.228.up_proj", + "model.layers.35.mlp.experts.229.down_proj", + "model.layers.35.mlp.experts.229.gate_proj", + "model.layers.35.mlp.experts.229.up_proj", + "model.layers.35.mlp.experts.23.down_proj", + "model.layers.35.mlp.experts.23.gate_proj", + "model.layers.35.mlp.experts.23.up_proj", + "model.layers.35.mlp.experts.230.down_proj", + "model.layers.35.mlp.experts.230.gate_proj", + "model.layers.35.mlp.experts.230.up_proj", + "model.layers.35.mlp.experts.231.down_proj", + "model.layers.35.mlp.experts.231.gate_proj", + "model.layers.35.mlp.experts.231.up_proj", + "model.layers.35.mlp.experts.232.down_proj", + "model.layers.35.mlp.experts.232.gate_proj", + "model.layers.35.mlp.experts.232.up_proj", + "model.layers.35.mlp.experts.233.down_proj", + "model.layers.35.mlp.experts.233.gate_proj", + "model.layers.35.mlp.experts.233.up_proj", + "model.layers.35.mlp.experts.234.down_proj", + "model.layers.35.mlp.experts.234.gate_proj", + "model.layers.35.mlp.experts.234.up_proj", + "model.layers.35.mlp.experts.235.down_proj", + "model.layers.35.mlp.experts.235.gate_proj", + "model.layers.35.mlp.experts.235.up_proj", + "model.layers.35.mlp.experts.236.down_proj", + "model.layers.35.mlp.experts.236.gate_proj", + "model.layers.35.mlp.experts.236.up_proj", + "model.layers.35.mlp.experts.237.down_proj", + "model.layers.35.mlp.experts.237.gate_proj", + "model.layers.35.mlp.experts.237.up_proj", + "model.layers.35.mlp.experts.238.down_proj", + "model.layers.35.mlp.experts.238.gate_proj", + "model.layers.35.mlp.experts.238.up_proj", + "model.layers.35.mlp.experts.239.down_proj", + "model.layers.35.mlp.experts.239.gate_proj", + "model.layers.35.mlp.experts.239.up_proj", + "model.layers.35.mlp.experts.24.down_proj", + "model.layers.35.mlp.experts.24.gate_proj", + "model.layers.35.mlp.experts.24.up_proj", + "model.layers.35.mlp.experts.240.down_proj", + "model.layers.35.mlp.experts.240.gate_proj", + "model.layers.35.mlp.experts.240.up_proj", + "model.layers.35.mlp.experts.241.down_proj", + "model.layers.35.mlp.experts.241.gate_proj", + "model.layers.35.mlp.experts.241.up_proj", + "model.layers.35.mlp.experts.242.down_proj", + "model.layers.35.mlp.experts.242.gate_proj", + "model.layers.35.mlp.experts.242.up_proj", + "model.layers.35.mlp.experts.243.down_proj", + "model.layers.35.mlp.experts.243.gate_proj", + "model.layers.35.mlp.experts.243.up_proj", + "model.layers.35.mlp.experts.244.down_proj", + "model.layers.35.mlp.experts.244.gate_proj", + "model.layers.35.mlp.experts.244.up_proj", + "model.layers.35.mlp.experts.245.down_proj", + "model.layers.35.mlp.experts.245.gate_proj", + "model.layers.35.mlp.experts.245.up_proj", + "model.layers.35.mlp.experts.246.down_proj", + "model.layers.35.mlp.experts.246.gate_proj", + "model.layers.35.mlp.experts.246.up_proj", + "model.layers.35.mlp.experts.247.down_proj", + "model.layers.35.mlp.experts.247.gate_proj", + "model.layers.35.mlp.experts.247.up_proj", + "model.layers.35.mlp.experts.248.down_proj", + "model.layers.35.mlp.experts.248.gate_proj", + "model.layers.35.mlp.experts.248.up_proj", + "model.layers.35.mlp.experts.249.down_proj", + "model.layers.35.mlp.experts.249.gate_proj", + "model.layers.35.mlp.experts.249.up_proj", + "model.layers.35.mlp.experts.25.down_proj", + "model.layers.35.mlp.experts.25.gate_proj", + "model.layers.35.mlp.experts.25.up_proj", + "model.layers.35.mlp.experts.250.down_proj", + "model.layers.35.mlp.experts.250.gate_proj", + "model.layers.35.mlp.experts.250.up_proj", + "model.layers.35.mlp.experts.251.down_proj", + "model.layers.35.mlp.experts.251.gate_proj", + "model.layers.35.mlp.experts.251.up_proj", + "model.layers.35.mlp.experts.252.down_proj", + "model.layers.35.mlp.experts.252.gate_proj", + "model.layers.35.mlp.experts.252.up_proj", + "model.layers.35.mlp.experts.253.down_proj", + "model.layers.35.mlp.experts.253.gate_proj", + "model.layers.35.mlp.experts.253.up_proj", + "model.layers.35.mlp.experts.254.down_proj", + "model.layers.35.mlp.experts.254.gate_proj", + "model.layers.35.mlp.experts.254.up_proj", + "model.layers.35.mlp.experts.255.down_proj", + "model.layers.35.mlp.experts.255.gate_proj", + "model.layers.35.mlp.experts.255.up_proj", + "model.layers.35.mlp.experts.26.down_proj", + "model.layers.35.mlp.experts.26.gate_proj", + "model.layers.35.mlp.experts.26.up_proj", + "model.layers.35.mlp.experts.27.down_proj", + "model.layers.35.mlp.experts.27.gate_proj", + "model.layers.35.mlp.experts.27.up_proj", + "model.layers.35.mlp.experts.28.down_proj", + "model.layers.35.mlp.experts.28.gate_proj", + "model.layers.35.mlp.experts.28.up_proj", + "model.layers.35.mlp.experts.29.down_proj", + "model.layers.35.mlp.experts.29.gate_proj", + "model.layers.35.mlp.experts.29.up_proj", + "model.layers.35.mlp.experts.3.down_proj", + "model.layers.35.mlp.experts.3.gate_proj", + "model.layers.35.mlp.experts.3.up_proj", + "model.layers.35.mlp.experts.30.down_proj", + "model.layers.35.mlp.experts.30.gate_proj", + "model.layers.35.mlp.experts.30.up_proj", + "model.layers.35.mlp.experts.31.down_proj", + "model.layers.35.mlp.experts.31.gate_proj", + "model.layers.35.mlp.experts.31.up_proj", + "model.layers.35.mlp.experts.32.down_proj", + "model.layers.35.mlp.experts.32.gate_proj", + "model.layers.35.mlp.experts.32.up_proj", + "model.layers.35.mlp.experts.33.down_proj", + "model.layers.35.mlp.experts.33.gate_proj", + "model.layers.35.mlp.experts.33.up_proj", + "model.layers.35.mlp.experts.34.down_proj", + "model.layers.35.mlp.experts.34.gate_proj", + "model.layers.35.mlp.experts.34.up_proj", + "model.layers.35.mlp.experts.35.down_proj", + "model.layers.35.mlp.experts.35.gate_proj", + "model.layers.35.mlp.experts.35.up_proj", + "model.layers.35.mlp.experts.36.down_proj", + "model.layers.35.mlp.experts.36.gate_proj", + "model.layers.35.mlp.experts.36.up_proj", + "model.layers.35.mlp.experts.37.down_proj", + "model.layers.35.mlp.experts.37.gate_proj", + "model.layers.35.mlp.experts.37.up_proj", + "model.layers.35.mlp.experts.38.down_proj", + "model.layers.35.mlp.experts.38.gate_proj", + "model.layers.35.mlp.experts.38.up_proj", + "model.layers.35.mlp.experts.39.down_proj", + "model.layers.35.mlp.experts.39.gate_proj", + "model.layers.35.mlp.experts.39.up_proj", + "model.layers.35.mlp.experts.4.down_proj", + "model.layers.35.mlp.experts.4.gate_proj", + "model.layers.35.mlp.experts.4.up_proj", + "model.layers.35.mlp.experts.40.down_proj", + "model.layers.35.mlp.experts.40.gate_proj", + "model.layers.35.mlp.experts.40.up_proj", + "model.layers.35.mlp.experts.41.down_proj", + "model.layers.35.mlp.experts.41.gate_proj", + "model.layers.35.mlp.experts.41.up_proj", + "model.layers.35.mlp.experts.42.down_proj", + "model.layers.35.mlp.experts.42.gate_proj", + "model.layers.35.mlp.experts.42.up_proj", + "model.layers.35.mlp.experts.43.down_proj", + "model.layers.35.mlp.experts.43.gate_proj", + "model.layers.35.mlp.experts.43.up_proj", + "model.layers.35.mlp.experts.44.down_proj", + "model.layers.35.mlp.experts.44.gate_proj", + "model.layers.35.mlp.experts.44.up_proj", + "model.layers.35.mlp.experts.45.down_proj", + "model.layers.35.mlp.experts.45.gate_proj", + "model.layers.35.mlp.experts.45.up_proj", + "model.layers.35.mlp.experts.46.down_proj", + "model.layers.35.mlp.experts.46.gate_proj", + "model.layers.35.mlp.experts.46.up_proj", + "model.layers.35.mlp.experts.47.down_proj", + "model.layers.35.mlp.experts.47.gate_proj", + "model.layers.35.mlp.experts.47.up_proj", + "model.layers.35.mlp.experts.48.down_proj", + "model.layers.35.mlp.experts.48.gate_proj", + "model.layers.35.mlp.experts.48.up_proj", + "model.layers.35.mlp.experts.49.down_proj", + "model.layers.35.mlp.experts.49.gate_proj", + "model.layers.35.mlp.experts.49.up_proj", + "model.layers.35.mlp.experts.5.down_proj", + "model.layers.35.mlp.experts.5.gate_proj", + "model.layers.35.mlp.experts.5.up_proj", + "model.layers.35.mlp.experts.50.down_proj", + "model.layers.35.mlp.experts.50.gate_proj", + "model.layers.35.mlp.experts.50.up_proj", + "model.layers.35.mlp.experts.51.down_proj", + "model.layers.35.mlp.experts.51.gate_proj", + "model.layers.35.mlp.experts.51.up_proj", + "model.layers.35.mlp.experts.52.down_proj", + "model.layers.35.mlp.experts.52.gate_proj", + "model.layers.35.mlp.experts.52.up_proj", + "model.layers.35.mlp.experts.53.down_proj", + "model.layers.35.mlp.experts.53.gate_proj", + "model.layers.35.mlp.experts.53.up_proj", + "model.layers.35.mlp.experts.54.down_proj", + "model.layers.35.mlp.experts.54.gate_proj", + "model.layers.35.mlp.experts.54.up_proj", + "model.layers.35.mlp.experts.55.down_proj", + "model.layers.35.mlp.experts.55.gate_proj", + "model.layers.35.mlp.experts.55.up_proj", + "model.layers.35.mlp.experts.56.down_proj", + "model.layers.35.mlp.experts.56.gate_proj", + "model.layers.35.mlp.experts.56.up_proj", + "model.layers.35.mlp.experts.57.down_proj", + "model.layers.35.mlp.experts.57.gate_proj", + "model.layers.35.mlp.experts.57.up_proj", + "model.layers.35.mlp.experts.58.down_proj", + "model.layers.35.mlp.experts.58.gate_proj", + "model.layers.35.mlp.experts.58.up_proj", + "model.layers.35.mlp.experts.59.down_proj", + "model.layers.35.mlp.experts.59.gate_proj", + "model.layers.35.mlp.experts.59.up_proj", + "model.layers.35.mlp.experts.6.down_proj", + "model.layers.35.mlp.experts.6.gate_proj", + "model.layers.35.mlp.experts.6.up_proj", + "model.layers.35.mlp.experts.60.down_proj", + "model.layers.35.mlp.experts.60.gate_proj", + "model.layers.35.mlp.experts.60.up_proj", + "model.layers.35.mlp.experts.61.down_proj", + "model.layers.35.mlp.experts.61.gate_proj", + "model.layers.35.mlp.experts.61.up_proj", + "model.layers.35.mlp.experts.62.down_proj", + "model.layers.35.mlp.experts.62.gate_proj", + "model.layers.35.mlp.experts.62.up_proj", + "model.layers.35.mlp.experts.63.down_proj", + "model.layers.35.mlp.experts.63.gate_proj", + "model.layers.35.mlp.experts.63.up_proj", + "model.layers.35.mlp.experts.64.down_proj", + "model.layers.35.mlp.experts.64.gate_proj", + "model.layers.35.mlp.experts.64.up_proj", + "model.layers.35.mlp.experts.65.down_proj", + "model.layers.35.mlp.experts.65.gate_proj", + "model.layers.35.mlp.experts.65.up_proj", + "model.layers.35.mlp.experts.66.down_proj", + "model.layers.35.mlp.experts.66.gate_proj", + "model.layers.35.mlp.experts.66.up_proj", + "model.layers.35.mlp.experts.67.down_proj", + "model.layers.35.mlp.experts.67.gate_proj", + "model.layers.35.mlp.experts.67.up_proj", + "model.layers.35.mlp.experts.68.down_proj", + "model.layers.35.mlp.experts.68.gate_proj", + "model.layers.35.mlp.experts.68.up_proj", + "model.layers.35.mlp.experts.69.down_proj", + "model.layers.35.mlp.experts.69.gate_proj", + "model.layers.35.mlp.experts.69.up_proj", + "model.layers.35.mlp.experts.7.down_proj", + "model.layers.35.mlp.experts.7.gate_proj", + "model.layers.35.mlp.experts.7.up_proj", + "model.layers.35.mlp.experts.70.down_proj", + "model.layers.35.mlp.experts.70.gate_proj", + "model.layers.35.mlp.experts.70.up_proj", + "model.layers.35.mlp.experts.71.down_proj", + "model.layers.35.mlp.experts.71.gate_proj", + "model.layers.35.mlp.experts.71.up_proj", + "model.layers.35.mlp.experts.72.down_proj", + "model.layers.35.mlp.experts.72.gate_proj", + "model.layers.35.mlp.experts.72.up_proj", + "model.layers.35.mlp.experts.73.down_proj", + "model.layers.35.mlp.experts.73.gate_proj", + "model.layers.35.mlp.experts.73.up_proj", + "model.layers.35.mlp.experts.74.down_proj", + "model.layers.35.mlp.experts.74.gate_proj", + "model.layers.35.mlp.experts.74.up_proj", + "model.layers.35.mlp.experts.75.down_proj", + "model.layers.35.mlp.experts.75.gate_proj", + "model.layers.35.mlp.experts.75.up_proj", + "model.layers.35.mlp.experts.76.down_proj", + "model.layers.35.mlp.experts.76.gate_proj", + "model.layers.35.mlp.experts.76.up_proj", + "model.layers.35.mlp.experts.77.down_proj", + "model.layers.35.mlp.experts.77.gate_proj", + "model.layers.35.mlp.experts.77.up_proj", + "model.layers.35.mlp.experts.78.down_proj", + "model.layers.35.mlp.experts.78.gate_proj", + "model.layers.35.mlp.experts.78.up_proj", + "model.layers.35.mlp.experts.79.down_proj", + "model.layers.35.mlp.experts.79.gate_proj", + "model.layers.35.mlp.experts.79.up_proj", + "model.layers.35.mlp.experts.8.down_proj", + "model.layers.35.mlp.experts.8.gate_proj", + "model.layers.35.mlp.experts.8.up_proj", + "model.layers.35.mlp.experts.80.down_proj", + "model.layers.35.mlp.experts.80.gate_proj", + "model.layers.35.mlp.experts.80.up_proj", + "model.layers.35.mlp.experts.81.down_proj", + "model.layers.35.mlp.experts.81.gate_proj", + "model.layers.35.mlp.experts.81.up_proj", + "model.layers.35.mlp.experts.82.down_proj", + "model.layers.35.mlp.experts.82.gate_proj", + "model.layers.35.mlp.experts.82.up_proj", + "model.layers.35.mlp.experts.83.down_proj", + "model.layers.35.mlp.experts.83.gate_proj", + "model.layers.35.mlp.experts.83.up_proj", + "model.layers.35.mlp.experts.84.down_proj", + "model.layers.35.mlp.experts.84.gate_proj", + "model.layers.35.mlp.experts.84.up_proj", + "model.layers.35.mlp.experts.85.down_proj", + "model.layers.35.mlp.experts.85.gate_proj", + "model.layers.35.mlp.experts.85.up_proj", + "model.layers.35.mlp.experts.86.down_proj", + "model.layers.35.mlp.experts.86.gate_proj", + "model.layers.35.mlp.experts.86.up_proj", + "model.layers.35.mlp.experts.87.down_proj", + "model.layers.35.mlp.experts.87.gate_proj", + "model.layers.35.mlp.experts.87.up_proj", + "model.layers.35.mlp.experts.88.down_proj", + "model.layers.35.mlp.experts.88.gate_proj", + "model.layers.35.mlp.experts.88.up_proj", + "model.layers.35.mlp.experts.89.down_proj", + "model.layers.35.mlp.experts.89.gate_proj", + "model.layers.35.mlp.experts.89.up_proj", + "model.layers.35.mlp.experts.9.down_proj", + "model.layers.35.mlp.experts.9.gate_proj", + "model.layers.35.mlp.experts.9.up_proj", + "model.layers.35.mlp.experts.90.down_proj", + "model.layers.35.mlp.experts.90.gate_proj", + "model.layers.35.mlp.experts.90.up_proj", + "model.layers.35.mlp.experts.91.down_proj", + "model.layers.35.mlp.experts.91.gate_proj", + "model.layers.35.mlp.experts.91.up_proj", + "model.layers.35.mlp.experts.92.down_proj", + "model.layers.35.mlp.experts.92.gate_proj", + "model.layers.35.mlp.experts.92.up_proj", + "model.layers.35.mlp.experts.93.down_proj", + "model.layers.35.mlp.experts.93.gate_proj", + "model.layers.35.mlp.experts.93.up_proj", + "model.layers.35.mlp.experts.94.down_proj", + "model.layers.35.mlp.experts.94.gate_proj", + "model.layers.35.mlp.experts.94.up_proj", + "model.layers.35.mlp.experts.95.down_proj", + "model.layers.35.mlp.experts.95.gate_proj", + "model.layers.35.mlp.experts.95.up_proj", + "model.layers.35.mlp.experts.96.down_proj", + "model.layers.35.mlp.experts.96.gate_proj", + "model.layers.35.mlp.experts.96.up_proj", + "model.layers.35.mlp.experts.97.down_proj", + "model.layers.35.mlp.experts.97.gate_proj", + "model.layers.35.mlp.experts.97.up_proj", + "model.layers.35.mlp.experts.98.down_proj", + "model.layers.35.mlp.experts.98.gate_proj", + "model.layers.35.mlp.experts.98.up_proj", + "model.layers.35.mlp.experts.99.down_proj", + "model.layers.35.mlp.experts.99.gate_proj", + "model.layers.35.mlp.experts.99.up_proj", + "model.layers.36.mlp.experts.0.down_proj", + "model.layers.36.mlp.experts.0.gate_proj", + "model.layers.36.mlp.experts.0.up_proj", + "model.layers.36.mlp.experts.1.down_proj", + "model.layers.36.mlp.experts.1.gate_proj", + "model.layers.36.mlp.experts.1.up_proj", + "model.layers.36.mlp.experts.10.down_proj", + "model.layers.36.mlp.experts.10.gate_proj", + "model.layers.36.mlp.experts.10.up_proj", + "model.layers.36.mlp.experts.100.down_proj", + "model.layers.36.mlp.experts.100.gate_proj", + "model.layers.36.mlp.experts.100.up_proj", + "model.layers.36.mlp.experts.101.down_proj", + "model.layers.36.mlp.experts.101.gate_proj", + "model.layers.36.mlp.experts.101.up_proj", + "model.layers.36.mlp.experts.102.down_proj", + "model.layers.36.mlp.experts.102.gate_proj", + "model.layers.36.mlp.experts.102.up_proj", + "model.layers.36.mlp.experts.103.down_proj", + "model.layers.36.mlp.experts.103.gate_proj", + "model.layers.36.mlp.experts.103.up_proj", + "model.layers.36.mlp.experts.104.down_proj", + "model.layers.36.mlp.experts.104.gate_proj", + "model.layers.36.mlp.experts.104.up_proj", + "model.layers.36.mlp.experts.105.down_proj", + "model.layers.36.mlp.experts.105.gate_proj", + "model.layers.36.mlp.experts.105.up_proj", + "model.layers.36.mlp.experts.106.down_proj", + "model.layers.36.mlp.experts.106.gate_proj", + "model.layers.36.mlp.experts.106.up_proj", + "model.layers.36.mlp.experts.107.down_proj", + "model.layers.36.mlp.experts.107.gate_proj", + "model.layers.36.mlp.experts.107.up_proj", + "model.layers.36.mlp.experts.108.down_proj", + "model.layers.36.mlp.experts.108.gate_proj", + "model.layers.36.mlp.experts.108.up_proj", + "model.layers.36.mlp.experts.109.down_proj", + "model.layers.36.mlp.experts.109.gate_proj", + "model.layers.36.mlp.experts.109.up_proj", + "model.layers.36.mlp.experts.11.down_proj", + "model.layers.36.mlp.experts.11.gate_proj", + "model.layers.36.mlp.experts.11.up_proj", + "model.layers.36.mlp.experts.110.down_proj", + "model.layers.36.mlp.experts.110.gate_proj", + "model.layers.36.mlp.experts.110.up_proj", + "model.layers.36.mlp.experts.111.down_proj", + "model.layers.36.mlp.experts.111.gate_proj", + "model.layers.36.mlp.experts.111.up_proj", + "model.layers.36.mlp.experts.112.down_proj", + "model.layers.36.mlp.experts.112.gate_proj", + "model.layers.36.mlp.experts.112.up_proj", + "model.layers.36.mlp.experts.113.down_proj", + "model.layers.36.mlp.experts.113.gate_proj", + "model.layers.36.mlp.experts.113.up_proj", + "model.layers.36.mlp.experts.114.down_proj", + "model.layers.36.mlp.experts.114.gate_proj", + "model.layers.36.mlp.experts.114.up_proj", + "model.layers.36.mlp.experts.115.down_proj", + "model.layers.36.mlp.experts.115.gate_proj", + "model.layers.36.mlp.experts.115.up_proj", + "model.layers.36.mlp.experts.116.down_proj", + "model.layers.36.mlp.experts.116.gate_proj", + "model.layers.36.mlp.experts.116.up_proj", + "model.layers.36.mlp.experts.117.down_proj", + "model.layers.36.mlp.experts.117.gate_proj", + "model.layers.36.mlp.experts.117.up_proj", + "model.layers.36.mlp.experts.118.down_proj", + "model.layers.36.mlp.experts.118.gate_proj", + "model.layers.36.mlp.experts.118.up_proj", + "model.layers.36.mlp.experts.119.down_proj", + "model.layers.36.mlp.experts.119.gate_proj", + "model.layers.36.mlp.experts.119.up_proj", + "model.layers.36.mlp.experts.12.down_proj", + "model.layers.36.mlp.experts.12.gate_proj", + "model.layers.36.mlp.experts.12.up_proj", + "model.layers.36.mlp.experts.120.down_proj", + "model.layers.36.mlp.experts.120.gate_proj", + "model.layers.36.mlp.experts.120.up_proj", + "model.layers.36.mlp.experts.121.down_proj", + "model.layers.36.mlp.experts.121.gate_proj", + "model.layers.36.mlp.experts.121.up_proj", + "model.layers.36.mlp.experts.122.down_proj", + "model.layers.36.mlp.experts.122.gate_proj", + "model.layers.36.mlp.experts.122.up_proj", + "model.layers.36.mlp.experts.123.down_proj", + "model.layers.36.mlp.experts.123.gate_proj", + "model.layers.36.mlp.experts.123.up_proj", + "model.layers.36.mlp.experts.124.down_proj", + "model.layers.36.mlp.experts.124.gate_proj", + "model.layers.36.mlp.experts.124.up_proj", + "model.layers.36.mlp.experts.125.down_proj", + "model.layers.36.mlp.experts.125.gate_proj", + "model.layers.36.mlp.experts.125.up_proj", + "model.layers.36.mlp.experts.126.down_proj", + "model.layers.36.mlp.experts.126.gate_proj", + "model.layers.36.mlp.experts.126.up_proj", + "model.layers.36.mlp.experts.127.down_proj", + "model.layers.36.mlp.experts.127.gate_proj", + "model.layers.36.mlp.experts.127.up_proj", + "model.layers.36.mlp.experts.128.down_proj", + "model.layers.36.mlp.experts.128.gate_proj", + "model.layers.36.mlp.experts.128.up_proj", + "model.layers.36.mlp.experts.129.down_proj", + "model.layers.36.mlp.experts.129.gate_proj", + "model.layers.36.mlp.experts.129.up_proj", + "model.layers.36.mlp.experts.13.down_proj", + "model.layers.36.mlp.experts.13.gate_proj", + "model.layers.36.mlp.experts.13.up_proj", + "model.layers.36.mlp.experts.130.down_proj", + "model.layers.36.mlp.experts.130.gate_proj", + "model.layers.36.mlp.experts.130.up_proj", + "model.layers.36.mlp.experts.131.down_proj", + "model.layers.36.mlp.experts.131.gate_proj", + "model.layers.36.mlp.experts.131.up_proj", + "model.layers.36.mlp.experts.132.down_proj", + "model.layers.36.mlp.experts.132.gate_proj", + "model.layers.36.mlp.experts.132.up_proj", + "model.layers.36.mlp.experts.133.down_proj", + "model.layers.36.mlp.experts.133.gate_proj", + "model.layers.36.mlp.experts.133.up_proj", + "model.layers.36.mlp.experts.134.down_proj", + "model.layers.36.mlp.experts.134.gate_proj", + "model.layers.36.mlp.experts.134.up_proj", + "model.layers.36.mlp.experts.135.down_proj", + "model.layers.36.mlp.experts.135.gate_proj", + "model.layers.36.mlp.experts.135.up_proj", + "model.layers.36.mlp.experts.136.down_proj", + "model.layers.36.mlp.experts.136.gate_proj", + "model.layers.36.mlp.experts.136.up_proj", + "model.layers.36.mlp.experts.137.down_proj", + "model.layers.36.mlp.experts.137.gate_proj", + "model.layers.36.mlp.experts.137.up_proj", + "model.layers.36.mlp.experts.138.down_proj", + "model.layers.36.mlp.experts.138.gate_proj", + "model.layers.36.mlp.experts.138.up_proj", + "model.layers.36.mlp.experts.139.down_proj", + "model.layers.36.mlp.experts.139.gate_proj", + "model.layers.36.mlp.experts.139.up_proj", + "model.layers.36.mlp.experts.14.down_proj", + "model.layers.36.mlp.experts.14.gate_proj", + "model.layers.36.mlp.experts.14.up_proj", + "model.layers.36.mlp.experts.140.down_proj", + "model.layers.36.mlp.experts.140.gate_proj", + "model.layers.36.mlp.experts.140.up_proj", + "model.layers.36.mlp.experts.141.down_proj", + "model.layers.36.mlp.experts.141.gate_proj", + "model.layers.36.mlp.experts.141.up_proj", + "model.layers.36.mlp.experts.142.down_proj", + "model.layers.36.mlp.experts.142.gate_proj", + "model.layers.36.mlp.experts.142.up_proj", + "model.layers.36.mlp.experts.143.down_proj", + "model.layers.36.mlp.experts.143.gate_proj", + "model.layers.36.mlp.experts.143.up_proj", + "model.layers.36.mlp.experts.144.down_proj", + "model.layers.36.mlp.experts.144.gate_proj", + "model.layers.36.mlp.experts.144.up_proj", + "model.layers.36.mlp.experts.145.down_proj", + "model.layers.36.mlp.experts.145.gate_proj", + "model.layers.36.mlp.experts.145.up_proj", + "model.layers.36.mlp.experts.146.down_proj", + "model.layers.36.mlp.experts.146.gate_proj", + "model.layers.36.mlp.experts.146.up_proj", + "model.layers.36.mlp.experts.147.down_proj", + "model.layers.36.mlp.experts.147.gate_proj", + "model.layers.36.mlp.experts.147.up_proj", + "model.layers.36.mlp.experts.148.down_proj", + "model.layers.36.mlp.experts.148.gate_proj", + "model.layers.36.mlp.experts.148.up_proj", + "model.layers.36.mlp.experts.149.down_proj", + "model.layers.36.mlp.experts.149.gate_proj", + "model.layers.36.mlp.experts.149.up_proj", + "model.layers.36.mlp.experts.15.down_proj", + "model.layers.36.mlp.experts.15.gate_proj", + "model.layers.36.mlp.experts.15.up_proj", + "model.layers.36.mlp.experts.150.down_proj", + "model.layers.36.mlp.experts.150.gate_proj", + "model.layers.36.mlp.experts.150.up_proj", + "model.layers.36.mlp.experts.151.down_proj", + "model.layers.36.mlp.experts.151.gate_proj", + "model.layers.36.mlp.experts.151.up_proj", + "model.layers.36.mlp.experts.152.down_proj", + "model.layers.36.mlp.experts.152.gate_proj", + "model.layers.36.mlp.experts.152.up_proj", + "model.layers.36.mlp.experts.153.down_proj", + "model.layers.36.mlp.experts.153.gate_proj", + "model.layers.36.mlp.experts.153.up_proj", + "model.layers.36.mlp.experts.154.down_proj", + "model.layers.36.mlp.experts.154.gate_proj", + "model.layers.36.mlp.experts.154.up_proj", + "model.layers.36.mlp.experts.155.down_proj", + "model.layers.36.mlp.experts.155.gate_proj", + "model.layers.36.mlp.experts.155.up_proj", + "model.layers.36.mlp.experts.156.down_proj", + "model.layers.36.mlp.experts.156.gate_proj", + "model.layers.36.mlp.experts.156.up_proj", + "model.layers.36.mlp.experts.157.down_proj", + "model.layers.36.mlp.experts.157.gate_proj", + "model.layers.36.mlp.experts.157.up_proj", + "model.layers.36.mlp.experts.158.down_proj", + "model.layers.36.mlp.experts.158.gate_proj", + "model.layers.36.mlp.experts.158.up_proj", + "model.layers.36.mlp.experts.159.down_proj", + "model.layers.36.mlp.experts.159.gate_proj", + "model.layers.36.mlp.experts.159.up_proj", + "model.layers.36.mlp.experts.16.down_proj", + "model.layers.36.mlp.experts.16.gate_proj", + "model.layers.36.mlp.experts.16.up_proj", + "model.layers.36.mlp.experts.160.down_proj", + "model.layers.36.mlp.experts.160.gate_proj", + "model.layers.36.mlp.experts.160.up_proj", + "model.layers.36.mlp.experts.161.down_proj", + "model.layers.36.mlp.experts.161.gate_proj", + "model.layers.36.mlp.experts.161.up_proj", + "model.layers.36.mlp.experts.162.down_proj", + "model.layers.36.mlp.experts.162.gate_proj", + "model.layers.36.mlp.experts.162.up_proj", + "model.layers.36.mlp.experts.163.down_proj", + "model.layers.36.mlp.experts.163.gate_proj", + "model.layers.36.mlp.experts.163.up_proj", + "model.layers.36.mlp.experts.164.down_proj", + "model.layers.36.mlp.experts.164.gate_proj", + "model.layers.36.mlp.experts.164.up_proj", + "model.layers.36.mlp.experts.165.down_proj", + "model.layers.36.mlp.experts.165.gate_proj", + "model.layers.36.mlp.experts.165.up_proj", + "model.layers.36.mlp.experts.166.down_proj", + "model.layers.36.mlp.experts.166.gate_proj", + "model.layers.36.mlp.experts.166.up_proj", + "model.layers.36.mlp.experts.167.down_proj", + "model.layers.36.mlp.experts.167.gate_proj", + "model.layers.36.mlp.experts.167.up_proj", + "model.layers.36.mlp.experts.168.down_proj", + "model.layers.36.mlp.experts.168.gate_proj", + "model.layers.36.mlp.experts.168.up_proj", + "model.layers.36.mlp.experts.169.down_proj", + "model.layers.36.mlp.experts.169.gate_proj", + "model.layers.36.mlp.experts.169.up_proj", + "model.layers.36.mlp.experts.17.down_proj", + "model.layers.36.mlp.experts.17.gate_proj", + "model.layers.36.mlp.experts.17.up_proj", + "model.layers.36.mlp.experts.170.down_proj", + "model.layers.36.mlp.experts.170.gate_proj", + "model.layers.36.mlp.experts.170.up_proj", + "model.layers.36.mlp.experts.171.down_proj", + "model.layers.36.mlp.experts.171.gate_proj", + "model.layers.36.mlp.experts.171.up_proj", + "model.layers.36.mlp.experts.172.down_proj", + "model.layers.36.mlp.experts.172.gate_proj", + "model.layers.36.mlp.experts.172.up_proj", + "model.layers.36.mlp.experts.173.down_proj", + "model.layers.36.mlp.experts.173.gate_proj", + "model.layers.36.mlp.experts.173.up_proj", + "model.layers.36.mlp.experts.174.down_proj", + "model.layers.36.mlp.experts.174.gate_proj", + "model.layers.36.mlp.experts.174.up_proj", + "model.layers.36.mlp.experts.175.down_proj", + "model.layers.36.mlp.experts.175.gate_proj", + "model.layers.36.mlp.experts.175.up_proj", + "model.layers.36.mlp.experts.176.down_proj", + "model.layers.36.mlp.experts.176.gate_proj", + "model.layers.36.mlp.experts.176.up_proj", + "model.layers.36.mlp.experts.177.down_proj", + "model.layers.36.mlp.experts.177.gate_proj", + "model.layers.36.mlp.experts.177.up_proj", + "model.layers.36.mlp.experts.178.down_proj", + "model.layers.36.mlp.experts.178.gate_proj", + "model.layers.36.mlp.experts.178.up_proj", + "model.layers.36.mlp.experts.179.down_proj", + "model.layers.36.mlp.experts.179.gate_proj", + "model.layers.36.mlp.experts.179.up_proj", + "model.layers.36.mlp.experts.18.down_proj", + "model.layers.36.mlp.experts.18.gate_proj", + "model.layers.36.mlp.experts.18.up_proj", + "model.layers.36.mlp.experts.180.down_proj", + "model.layers.36.mlp.experts.180.gate_proj", + "model.layers.36.mlp.experts.180.up_proj", + "model.layers.36.mlp.experts.181.down_proj", + "model.layers.36.mlp.experts.181.gate_proj", + "model.layers.36.mlp.experts.181.up_proj", + "model.layers.36.mlp.experts.182.down_proj", + "model.layers.36.mlp.experts.182.gate_proj", + "model.layers.36.mlp.experts.182.up_proj", + "model.layers.36.mlp.experts.183.down_proj", + "model.layers.36.mlp.experts.183.gate_proj", + "model.layers.36.mlp.experts.183.up_proj", + "model.layers.36.mlp.experts.184.down_proj", + "model.layers.36.mlp.experts.184.gate_proj", + "model.layers.36.mlp.experts.184.up_proj", + "model.layers.36.mlp.experts.185.down_proj", + "model.layers.36.mlp.experts.185.gate_proj", + "model.layers.36.mlp.experts.185.up_proj", + "model.layers.36.mlp.experts.186.down_proj", + "model.layers.36.mlp.experts.186.gate_proj", + "model.layers.36.mlp.experts.186.up_proj", + "model.layers.36.mlp.experts.187.down_proj", + "model.layers.36.mlp.experts.187.gate_proj", + "model.layers.36.mlp.experts.187.up_proj", + "model.layers.36.mlp.experts.188.down_proj", + "model.layers.36.mlp.experts.188.gate_proj", + "model.layers.36.mlp.experts.188.up_proj", + "model.layers.36.mlp.experts.189.down_proj", + "model.layers.36.mlp.experts.189.gate_proj", + "model.layers.36.mlp.experts.189.up_proj", + "model.layers.36.mlp.experts.19.down_proj", + "model.layers.36.mlp.experts.19.gate_proj", + "model.layers.36.mlp.experts.19.up_proj", + "model.layers.36.mlp.experts.190.down_proj", + "model.layers.36.mlp.experts.190.gate_proj", + "model.layers.36.mlp.experts.190.up_proj", + "model.layers.36.mlp.experts.191.down_proj", + "model.layers.36.mlp.experts.191.gate_proj", + "model.layers.36.mlp.experts.191.up_proj", + "model.layers.36.mlp.experts.192.down_proj", + "model.layers.36.mlp.experts.192.gate_proj", + "model.layers.36.mlp.experts.192.up_proj", + "model.layers.36.mlp.experts.193.down_proj", + "model.layers.36.mlp.experts.193.gate_proj", + "model.layers.36.mlp.experts.193.up_proj", + "model.layers.36.mlp.experts.194.down_proj", + "model.layers.36.mlp.experts.194.gate_proj", + "model.layers.36.mlp.experts.194.up_proj", + "model.layers.36.mlp.experts.195.down_proj", + "model.layers.36.mlp.experts.195.gate_proj", + "model.layers.36.mlp.experts.195.up_proj", + "model.layers.36.mlp.experts.196.down_proj", + "model.layers.36.mlp.experts.196.gate_proj", + "model.layers.36.mlp.experts.196.up_proj", + "model.layers.36.mlp.experts.197.down_proj", + "model.layers.36.mlp.experts.197.gate_proj", + "model.layers.36.mlp.experts.197.up_proj", + "model.layers.36.mlp.experts.198.down_proj", + "model.layers.36.mlp.experts.198.gate_proj", + "model.layers.36.mlp.experts.198.up_proj", + "model.layers.36.mlp.experts.199.down_proj", + "model.layers.36.mlp.experts.199.gate_proj", + "model.layers.36.mlp.experts.199.up_proj", + "model.layers.36.mlp.experts.2.down_proj", + "model.layers.36.mlp.experts.2.gate_proj", + "model.layers.36.mlp.experts.2.up_proj", + "model.layers.36.mlp.experts.20.down_proj", + "model.layers.36.mlp.experts.20.gate_proj", + "model.layers.36.mlp.experts.20.up_proj", + "model.layers.36.mlp.experts.200.down_proj", + "model.layers.36.mlp.experts.200.gate_proj", + "model.layers.36.mlp.experts.200.up_proj", + "model.layers.36.mlp.experts.201.down_proj", + "model.layers.36.mlp.experts.201.gate_proj", + "model.layers.36.mlp.experts.201.up_proj", + "model.layers.36.mlp.experts.202.down_proj", + "model.layers.36.mlp.experts.202.gate_proj", + "model.layers.36.mlp.experts.202.up_proj", + "model.layers.36.mlp.experts.203.down_proj", + "model.layers.36.mlp.experts.203.gate_proj", + "model.layers.36.mlp.experts.203.up_proj", + "model.layers.36.mlp.experts.204.down_proj", + "model.layers.36.mlp.experts.204.gate_proj", + "model.layers.36.mlp.experts.204.up_proj", + "model.layers.36.mlp.experts.205.down_proj", + "model.layers.36.mlp.experts.205.gate_proj", + "model.layers.36.mlp.experts.205.up_proj", + "model.layers.36.mlp.experts.206.down_proj", + "model.layers.36.mlp.experts.206.gate_proj", + "model.layers.36.mlp.experts.206.up_proj", + "model.layers.36.mlp.experts.207.down_proj", + "model.layers.36.mlp.experts.207.gate_proj", + "model.layers.36.mlp.experts.207.up_proj", + "model.layers.36.mlp.experts.208.down_proj", + "model.layers.36.mlp.experts.208.gate_proj", + "model.layers.36.mlp.experts.208.up_proj", + "model.layers.36.mlp.experts.209.down_proj", + "model.layers.36.mlp.experts.209.gate_proj", + "model.layers.36.mlp.experts.209.up_proj", + "model.layers.36.mlp.experts.21.down_proj", + "model.layers.36.mlp.experts.21.gate_proj", + "model.layers.36.mlp.experts.21.up_proj", + "model.layers.36.mlp.experts.210.down_proj", + "model.layers.36.mlp.experts.210.gate_proj", + "model.layers.36.mlp.experts.210.up_proj", + "model.layers.36.mlp.experts.211.down_proj", + "model.layers.36.mlp.experts.211.gate_proj", + "model.layers.36.mlp.experts.211.up_proj", + "model.layers.36.mlp.experts.212.down_proj", + "model.layers.36.mlp.experts.212.gate_proj", + "model.layers.36.mlp.experts.212.up_proj", + "model.layers.36.mlp.experts.213.down_proj", + "model.layers.36.mlp.experts.213.gate_proj", + "model.layers.36.mlp.experts.213.up_proj", + "model.layers.36.mlp.experts.214.down_proj", + "model.layers.36.mlp.experts.214.gate_proj", + "model.layers.36.mlp.experts.214.up_proj", + "model.layers.36.mlp.experts.215.down_proj", + "model.layers.36.mlp.experts.215.gate_proj", + "model.layers.36.mlp.experts.215.up_proj", + "model.layers.36.mlp.experts.216.down_proj", + "model.layers.36.mlp.experts.216.gate_proj", + "model.layers.36.mlp.experts.216.up_proj", + "model.layers.36.mlp.experts.217.down_proj", + "model.layers.36.mlp.experts.217.gate_proj", + "model.layers.36.mlp.experts.217.up_proj", + "model.layers.36.mlp.experts.218.down_proj", + "model.layers.36.mlp.experts.218.gate_proj", + "model.layers.36.mlp.experts.218.up_proj", + "model.layers.36.mlp.experts.219.down_proj", + "model.layers.36.mlp.experts.219.gate_proj", + "model.layers.36.mlp.experts.219.up_proj", + "model.layers.36.mlp.experts.22.down_proj", + "model.layers.36.mlp.experts.22.gate_proj", + "model.layers.36.mlp.experts.22.up_proj", + "model.layers.36.mlp.experts.220.down_proj", + "model.layers.36.mlp.experts.220.gate_proj", + "model.layers.36.mlp.experts.220.up_proj", + "model.layers.36.mlp.experts.221.down_proj", + "model.layers.36.mlp.experts.221.gate_proj", + "model.layers.36.mlp.experts.221.up_proj", + "model.layers.36.mlp.experts.222.down_proj", + "model.layers.36.mlp.experts.222.gate_proj", + "model.layers.36.mlp.experts.222.up_proj", + "model.layers.36.mlp.experts.223.down_proj", + "model.layers.36.mlp.experts.223.gate_proj", + "model.layers.36.mlp.experts.223.up_proj", + "model.layers.36.mlp.experts.224.down_proj", + "model.layers.36.mlp.experts.224.gate_proj", + "model.layers.36.mlp.experts.224.up_proj", + "model.layers.36.mlp.experts.225.down_proj", + "model.layers.36.mlp.experts.225.gate_proj", + "model.layers.36.mlp.experts.225.up_proj", + "model.layers.36.mlp.experts.226.down_proj", + "model.layers.36.mlp.experts.226.gate_proj", + "model.layers.36.mlp.experts.226.up_proj", + "model.layers.36.mlp.experts.227.down_proj", + "model.layers.36.mlp.experts.227.gate_proj", + "model.layers.36.mlp.experts.227.up_proj", + "model.layers.36.mlp.experts.228.down_proj", + "model.layers.36.mlp.experts.228.gate_proj", + "model.layers.36.mlp.experts.228.up_proj", + "model.layers.36.mlp.experts.229.down_proj", + "model.layers.36.mlp.experts.229.gate_proj", + "model.layers.36.mlp.experts.229.up_proj", + "model.layers.36.mlp.experts.23.down_proj", + "model.layers.36.mlp.experts.23.gate_proj", + "model.layers.36.mlp.experts.23.up_proj", + "model.layers.36.mlp.experts.230.down_proj", + "model.layers.36.mlp.experts.230.gate_proj", + "model.layers.36.mlp.experts.230.up_proj", + "model.layers.36.mlp.experts.231.down_proj", + "model.layers.36.mlp.experts.231.gate_proj", + "model.layers.36.mlp.experts.231.up_proj", + "model.layers.36.mlp.experts.232.down_proj", + "model.layers.36.mlp.experts.232.gate_proj", + "model.layers.36.mlp.experts.232.up_proj", + "model.layers.36.mlp.experts.233.down_proj", + "model.layers.36.mlp.experts.233.gate_proj", + "model.layers.36.mlp.experts.233.up_proj", + "model.layers.36.mlp.experts.234.down_proj", + "model.layers.36.mlp.experts.234.gate_proj", + "model.layers.36.mlp.experts.234.up_proj", + "model.layers.36.mlp.experts.235.down_proj", + "model.layers.36.mlp.experts.235.gate_proj", + "model.layers.36.mlp.experts.235.up_proj", + "model.layers.36.mlp.experts.236.down_proj", + "model.layers.36.mlp.experts.236.gate_proj", + "model.layers.36.mlp.experts.236.up_proj", + "model.layers.36.mlp.experts.237.down_proj", + "model.layers.36.mlp.experts.237.gate_proj", + "model.layers.36.mlp.experts.237.up_proj", + "model.layers.36.mlp.experts.238.down_proj", + "model.layers.36.mlp.experts.238.gate_proj", + "model.layers.36.mlp.experts.238.up_proj", + "model.layers.36.mlp.experts.239.down_proj", + "model.layers.36.mlp.experts.239.gate_proj", + "model.layers.36.mlp.experts.239.up_proj", + "model.layers.36.mlp.experts.24.down_proj", + "model.layers.36.mlp.experts.24.gate_proj", + "model.layers.36.mlp.experts.24.up_proj", + "model.layers.36.mlp.experts.240.down_proj", + "model.layers.36.mlp.experts.240.gate_proj", + "model.layers.36.mlp.experts.240.up_proj", + "model.layers.36.mlp.experts.241.down_proj", + "model.layers.36.mlp.experts.241.gate_proj", + "model.layers.36.mlp.experts.241.up_proj", + "model.layers.36.mlp.experts.242.down_proj", + "model.layers.36.mlp.experts.242.gate_proj", + "model.layers.36.mlp.experts.242.up_proj", + "model.layers.36.mlp.experts.243.down_proj", + "model.layers.36.mlp.experts.243.gate_proj", + "model.layers.36.mlp.experts.243.up_proj", + "model.layers.36.mlp.experts.244.down_proj", + "model.layers.36.mlp.experts.244.gate_proj", + "model.layers.36.mlp.experts.244.up_proj", + "model.layers.36.mlp.experts.245.down_proj", + "model.layers.36.mlp.experts.245.gate_proj", + "model.layers.36.mlp.experts.245.up_proj", + "model.layers.36.mlp.experts.246.down_proj", + "model.layers.36.mlp.experts.246.gate_proj", + "model.layers.36.mlp.experts.246.up_proj", + "model.layers.36.mlp.experts.247.down_proj", + "model.layers.36.mlp.experts.247.gate_proj", + "model.layers.36.mlp.experts.247.up_proj", + "model.layers.36.mlp.experts.248.down_proj", + "model.layers.36.mlp.experts.248.gate_proj", + "model.layers.36.mlp.experts.248.up_proj", + "model.layers.36.mlp.experts.249.down_proj", + "model.layers.36.mlp.experts.249.gate_proj", + "model.layers.36.mlp.experts.249.up_proj", + "model.layers.36.mlp.experts.25.down_proj", + "model.layers.36.mlp.experts.25.gate_proj", + "model.layers.36.mlp.experts.25.up_proj", + "model.layers.36.mlp.experts.250.down_proj", + "model.layers.36.mlp.experts.250.gate_proj", + "model.layers.36.mlp.experts.250.up_proj", + "model.layers.36.mlp.experts.251.down_proj", + "model.layers.36.mlp.experts.251.gate_proj", + "model.layers.36.mlp.experts.251.up_proj", + "model.layers.36.mlp.experts.252.down_proj", + "model.layers.36.mlp.experts.252.gate_proj", + "model.layers.36.mlp.experts.252.up_proj", + "model.layers.36.mlp.experts.253.down_proj", + "model.layers.36.mlp.experts.253.gate_proj", + "model.layers.36.mlp.experts.253.up_proj", + "model.layers.36.mlp.experts.254.down_proj", + "model.layers.36.mlp.experts.254.gate_proj", + "model.layers.36.mlp.experts.254.up_proj", + "model.layers.36.mlp.experts.255.down_proj", + "model.layers.36.mlp.experts.255.gate_proj", + "model.layers.36.mlp.experts.255.up_proj", + "model.layers.36.mlp.experts.26.down_proj", + "model.layers.36.mlp.experts.26.gate_proj", + "model.layers.36.mlp.experts.26.up_proj", + "model.layers.36.mlp.experts.27.down_proj", + "model.layers.36.mlp.experts.27.gate_proj", + "model.layers.36.mlp.experts.27.up_proj", + "model.layers.36.mlp.experts.28.down_proj", + "model.layers.36.mlp.experts.28.gate_proj", + "model.layers.36.mlp.experts.28.up_proj", + "model.layers.36.mlp.experts.29.down_proj", + "model.layers.36.mlp.experts.29.gate_proj", + "model.layers.36.mlp.experts.29.up_proj", + "model.layers.36.mlp.experts.3.down_proj", + "model.layers.36.mlp.experts.3.gate_proj", + "model.layers.36.mlp.experts.3.up_proj", + "model.layers.36.mlp.experts.30.down_proj", + "model.layers.36.mlp.experts.30.gate_proj", + "model.layers.36.mlp.experts.30.up_proj", + "model.layers.36.mlp.experts.31.down_proj", + "model.layers.36.mlp.experts.31.gate_proj", + "model.layers.36.mlp.experts.31.up_proj", + "model.layers.36.mlp.experts.32.down_proj", + "model.layers.36.mlp.experts.32.gate_proj", + "model.layers.36.mlp.experts.32.up_proj", + "model.layers.36.mlp.experts.33.down_proj", + "model.layers.36.mlp.experts.33.gate_proj", + "model.layers.36.mlp.experts.33.up_proj", + "model.layers.36.mlp.experts.34.down_proj", + "model.layers.36.mlp.experts.34.gate_proj", + "model.layers.36.mlp.experts.34.up_proj", + "model.layers.36.mlp.experts.35.down_proj", + "model.layers.36.mlp.experts.35.gate_proj", + "model.layers.36.mlp.experts.35.up_proj", + "model.layers.36.mlp.experts.36.down_proj", + "model.layers.36.mlp.experts.36.gate_proj", + "model.layers.36.mlp.experts.36.up_proj", + "model.layers.36.mlp.experts.37.down_proj", + "model.layers.36.mlp.experts.37.gate_proj", + "model.layers.36.mlp.experts.37.up_proj", + "model.layers.36.mlp.experts.38.down_proj", + "model.layers.36.mlp.experts.38.gate_proj", + "model.layers.36.mlp.experts.38.up_proj", + "model.layers.36.mlp.experts.39.down_proj", + "model.layers.36.mlp.experts.39.gate_proj", + "model.layers.36.mlp.experts.39.up_proj", + "model.layers.36.mlp.experts.4.down_proj", + "model.layers.36.mlp.experts.4.gate_proj", + "model.layers.36.mlp.experts.4.up_proj", + "model.layers.36.mlp.experts.40.down_proj", + "model.layers.36.mlp.experts.40.gate_proj", + "model.layers.36.mlp.experts.40.up_proj", + "model.layers.36.mlp.experts.41.down_proj", + "model.layers.36.mlp.experts.41.gate_proj", + "model.layers.36.mlp.experts.41.up_proj", + "model.layers.36.mlp.experts.42.down_proj", + "model.layers.36.mlp.experts.42.gate_proj", + "model.layers.36.mlp.experts.42.up_proj", + "model.layers.36.mlp.experts.43.down_proj", + "model.layers.36.mlp.experts.43.gate_proj", + "model.layers.36.mlp.experts.43.up_proj", + "model.layers.36.mlp.experts.44.down_proj", + "model.layers.36.mlp.experts.44.gate_proj", + "model.layers.36.mlp.experts.44.up_proj", + "model.layers.36.mlp.experts.45.down_proj", + "model.layers.36.mlp.experts.45.gate_proj", + "model.layers.36.mlp.experts.45.up_proj", + "model.layers.36.mlp.experts.46.down_proj", + "model.layers.36.mlp.experts.46.gate_proj", + "model.layers.36.mlp.experts.46.up_proj", + "model.layers.36.mlp.experts.47.down_proj", + "model.layers.36.mlp.experts.47.gate_proj", + "model.layers.36.mlp.experts.47.up_proj", + "model.layers.36.mlp.experts.48.down_proj", + "model.layers.36.mlp.experts.48.gate_proj", + "model.layers.36.mlp.experts.48.up_proj", + "model.layers.36.mlp.experts.49.down_proj", + "model.layers.36.mlp.experts.49.gate_proj", + "model.layers.36.mlp.experts.49.up_proj", + "model.layers.36.mlp.experts.5.down_proj", + "model.layers.36.mlp.experts.5.gate_proj", + "model.layers.36.mlp.experts.5.up_proj", + "model.layers.36.mlp.experts.50.down_proj", + "model.layers.36.mlp.experts.50.gate_proj", + "model.layers.36.mlp.experts.50.up_proj", + "model.layers.36.mlp.experts.51.down_proj", + "model.layers.36.mlp.experts.51.gate_proj", + "model.layers.36.mlp.experts.51.up_proj", + "model.layers.36.mlp.experts.52.down_proj", + "model.layers.36.mlp.experts.52.gate_proj", + "model.layers.36.mlp.experts.52.up_proj", + "model.layers.36.mlp.experts.53.down_proj", + "model.layers.36.mlp.experts.53.gate_proj", + "model.layers.36.mlp.experts.53.up_proj", + "model.layers.36.mlp.experts.54.down_proj", + "model.layers.36.mlp.experts.54.gate_proj", + "model.layers.36.mlp.experts.54.up_proj", + "model.layers.36.mlp.experts.55.down_proj", + "model.layers.36.mlp.experts.55.gate_proj", + "model.layers.36.mlp.experts.55.up_proj", + "model.layers.36.mlp.experts.56.down_proj", + "model.layers.36.mlp.experts.56.gate_proj", + "model.layers.36.mlp.experts.56.up_proj", + "model.layers.36.mlp.experts.57.down_proj", + "model.layers.36.mlp.experts.57.gate_proj", + "model.layers.36.mlp.experts.57.up_proj", + "model.layers.36.mlp.experts.58.down_proj", + "model.layers.36.mlp.experts.58.gate_proj", + "model.layers.36.mlp.experts.58.up_proj", + "model.layers.36.mlp.experts.59.down_proj", + "model.layers.36.mlp.experts.59.gate_proj", + "model.layers.36.mlp.experts.59.up_proj", + "model.layers.36.mlp.experts.6.down_proj", + "model.layers.36.mlp.experts.6.gate_proj", + "model.layers.36.mlp.experts.6.up_proj", + "model.layers.36.mlp.experts.60.down_proj", + "model.layers.36.mlp.experts.60.gate_proj", + "model.layers.36.mlp.experts.60.up_proj", + "model.layers.36.mlp.experts.61.down_proj", + "model.layers.36.mlp.experts.61.gate_proj", + "model.layers.36.mlp.experts.61.up_proj", + "model.layers.36.mlp.experts.62.down_proj", + "model.layers.36.mlp.experts.62.gate_proj", + "model.layers.36.mlp.experts.62.up_proj", + "model.layers.36.mlp.experts.63.down_proj", + "model.layers.36.mlp.experts.63.gate_proj", + "model.layers.36.mlp.experts.63.up_proj", + "model.layers.36.mlp.experts.64.down_proj", + "model.layers.36.mlp.experts.64.gate_proj", + "model.layers.36.mlp.experts.64.up_proj", + "model.layers.36.mlp.experts.65.down_proj", + "model.layers.36.mlp.experts.65.gate_proj", + "model.layers.36.mlp.experts.65.up_proj", + "model.layers.36.mlp.experts.66.down_proj", + "model.layers.36.mlp.experts.66.gate_proj", + "model.layers.36.mlp.experts.66.up_proj", + "model.layers.36.mlp.experts.67.down_proj", + "model.layers.36.mlp.experts.67.gate_proj", + "model.layers.36.mlp.experts.67.up_proj", + "model.layers.36.mlp.experts.68.down_proj", + "model.layers.36.mlp.experts.68.gate_proj", + "model.layers.36.mlp.experts.68.up_proj", + "model.layers.36.mlp.experts.69.down_proj", + "model.layers.36.mlp.experts.69.gate_proj", + "model.layers.36.mlp.experts.69.up_proj", + "model.layers.36.mlp.experts.7.down_proj", + "model.layers.36.mlp.experts.7.gate_proj", + "model.layers.36.mlp.experts.7.up_proj", + "model.layers.36.mlp.experts.70.down_proj", + "model.layers.36.mlp.experts.70.gate_proj", + "model.layers.36.mlp.experts.70.up_proj", + "model.layers.36.mlp.experts.71.down_proj", + "model.layers.36.mlp.experts.71.gate_proj", + "model.layers.36.mlp.experts.71.up_proj", + "model.layers.36.mlp.experts.72.down_proj", + "model.layers.36.mlp.experts.72.gate_proj", + "model.layers.36.mlp.experts.72.up_proj", + "model.layers.36.mlp.experts.73.down_proj", + "model.layers.36.mlp.experts.73.gate_proj", + "model.layers.36.mlp.experts.73.up_proj", + "model.layers.36.mlp.experts.74.down_proj", + "model.layers.36.mlp.experts.74.gate_proj", + "model.layers.36.mlp.experts.74.up_proj", + "model.layers.36.mlp.experts.75.down_proj", + "model.layers.36.mlp.experts.75.gate_proj", + "model.layers.36.mlp.experts.75.up_proj", + "model.layers.36.mlp.experts.76.down_proj", + "model.layers.36.mlp.experts.76.gate_proj", + "model.layers.36.mlp.experts.76.up_proj", + "model.layers.36.mlp.experts.77.down_proj", + "model.layers.36.mlp.experts.77.gate_proj", + "model.layers.36.mlp.experts.77.up_proj", + "model.layers.36.mlp.experts.78.down_proj", + "model.layers.36.mlp.experts.78.gate_proj", + "model.layers.36.mlp.experts.78.up_proj", + "model.layers.36.mlp.experts.79.down_proj", + "model.layers.36.mlp.experts.79.gate_proj", + "model.layers.36.mlp.experts.79.up_proj", + "model.layers.36.mlp.experts.8.down_proj", + "model.layers.36.mlp.experts.8.gate_proj", + "model.layers.36.mlp.experts.8.up_proj", + "model.layers.36.mlp.experts.80.down_proj", + "model.layers.36.mlp.experts.80.gate_proj", + "model.layers.36.mlp.experts.80.up_proj", + "model.layers.36.mlp.experts.81.down_proj", + "model.layers.36.mlp.experts.81.gate_proj", + "model.layers.36.mlp.experts.81.up_proj", + "model.layers.36.mlp.experts.82.down_proj", + "model.layers.36.mlp.experts.82.gate_proj", + "model.layers.36.mlp.experts.82.up_proj", + "model.layers.36.mlp.experts.83.down_proj", + "model.layers.36.mlp.experts.83.gate_proj", + "model.layers.36.mlp.experts.83.up_proj", + "model.layers.36.mlp.experts.84.down_proj", + "model.layers.36.mlp.experts.84.gate_proj", + "model.layers.36.mlp.experts.84.up_proj", + "model.layers.36.mlp.experts.85.down_proj", + "model.layers.36.mlp.experts.85.gate_proj", + "model.layers.36.mlp.experts.85.up_proj", + "model.layers.36.mlp.experts.86.down_proj", + "model.layers.36.mlp.experts.86.gate_proj", + "model.layers.36.mlp.experts.86.up_proj", + "model.layers.36.mlp.experts.87.down_proj", + "model.layers.36.mlp.experts.87.gate_proj", + "model.layers.36.mlp.experts.87.up_proj", + "model.layers.36.mlp.experts.88.down_proj", + "model.layers.36.mlp.experts.88.gate_proj", + "model.layers.36.mlp.experts.88.up_proj", + "model.layers.36.mlp.experts.89.down_proj", + "model.layers.36.mlp.experts.89.gate_proj", + "model.layers.36.mlp.experts.89.up_proj", + "model.layers.36.mlp.experts.9.down_proj", + "model.layers.36.mlp.experts.9.gate_proj", + "model.layers.36.mlp.experts.9.up_proj", + "model.layers.36.mlp.experts.90.down_proj", + "model.layers.36.mlp.experts.90.gate_proj", + "model.layers.36.mlp.experts.90.up_proj", + "model.layers.36.mlp.experts.91.down_proj", + "model.layers.36.mlp.experts.91.gate_proj", + "model.layers.36.mlp.experts.91.up_proj", + "model.layers.36.mlp.experts.92.down_proj", + "model.layers.36.mlp.experts.92.gate_proj", + "model.layers.36.mlp.experts.92.up_proj", + "model.layers.36.mlp.experts.93.down_proj", + "model.layers.36.mlp.experts.93.gate_proj", + "model.layers.36.mlp.experts.93.up_proj", + "model.layers.36.mlp.experts.94.down_proj", + "model.layers.36.mlp.experts.94.gate_proj", + "model.layers.36.mlp.experts.94.up_proj", + "model.layers.36.mlp.experts.95.down_proj", + "model.layers.36.mlp.experts.95.gate_proj", + "model.layers.36.mlp.experts.95.up_proj", + "model.layers.36.mlp.experts.96.down_proj", + "model.layers.36.mlp.experts.96.gate_proj", + "model.layers.36.mlp.experts.96.up_proj", + "model.layers.36.mlp.experts.97.down_proj", + "model.layers.36.mlp.experts.97.gate_proj", + "model.layers.36.mlp.experts.97.up_proj", + "model.layers.36.mlp.experts.98.down_proj", + "model.layers.36.mlp.experts.98.gate_proj", + "model.layers.36.mlp.experts.98.up_proj", + "model.layers.36.mlp.experts.99.down_proj", + "model.layers.36.mlp.experts.99.gate_proj", + "model.layers.36.mlp.experts.99.up_proj", + "model.layers.37.mlp.experts.0.down_proj", + "model.layers.37.mlp.experts.0.gate_proj", + "model.layers.37.mlp.experts.0.up_proj", + "model.layers.37.mlp.experts.1.down_proj", + "model.layers.37.mlp.experts.1.gate_proj", + "model.layers.37.mlp.experts.1.up_proj", + "model.layers.37.mlp.experts.10.down_proj", + "model.layers.37.mlp.experts.10.gate_proj", + "model.layers.37.mlp.experts.10.up_proj", + "model.layers.37.mlp.experts.100.down_proj", + "model.layers.37.mlp.experts.100.gate_proj", + "model.layers.37.mlp.experts.100.up_proj", + "model.layers.37.mlp.experts.101.down_proj", + "model.layers.37.mlp.experts.101.gate_proj", + "model.layers.37.mlp.experts.101.up_proj", + "model.layers.37.mlp.experts.102.down_proj", + "model.layers.37.mlp.experts.102.gate_proj", + "model.layers.37.mlp.experts.102.up_proj", + "model.layers.37.mlp.experts.103.down_proj", + "model.layers.37.mlp.experts.103.gate_proj", + "model.layers.37.mlp.experts.103.up_proj", + "model.layers.37.mlp.experts.104.down_proj", + "model.layers.37.mlp.experts.104.gate_proj", + "model.layers.37.mlp.experts.104.up_proj", + "model.layers.37.mlp.experts.105.down_proj", + "model.layers.37.mlp.experts.105.gate_proj", + "model.layers.37.mlp.experts.105.up_proj", + "model.layers.37.mlp.experts.106.down_proj", + "model.layers.37.mlp.experts.106.gate_proj", + "model.layers.37.mlp.experts.106.up_proj", + "model.layers.37.mlp.experts.107.down_proj", + "model.layers.37.mlp.experts.107.gate_proj", + "model.layers.37.mlp.experts.107.up_proj", + "model.layers.37.mlp.experts.108.down_proj", + "model.layers.37.mlp.experts.108.gate_proj", + "model.layers.37.mlp.experts.108.up_proj", + "model.layers.37.mlp.experts.109.down_proj", + "model.layers.37.mlp.experts.109.gate_proj", + "model.layers.37.mlp.experts.109.up_proj", + "model.layers.37.mlp.experts.11.down_proj", + "model.layers.37.mlp.experts.11.gate_proj", + "model.layers.37.mlp.experts.11.up_proj", + "model.layers.37.mlp.experts.110.down_proj", + "model.layers.37.mlp.experts.110.gate_proj", + "model.layers.37.mlp.experts.110.up_proj", + "model.layers.37.mlp.experts.111.down_proj", + "model.layers.37.mlp.experts.111.gate_proj", + "model.layers.37.mlp.experts.111.up_proj", + "model.layers.37.mlp.experts.112.down_proj", + "model.layers.37.mlp.experts.112.gate_proj", + "model.layers.37.mlp.experts.112.up_proj", + "model.layers.37.mlp.experts.113.down_proj", + "model.layers.37.mlp.experts.113.gate_proj", + "model.layers.37.mlp.experts.113.up_proj", + "model.layers.37.mlp.experts.114.down_proj", + "model.layers.37.mlp.experts.114.gate_proj", + "model.layers.37.mlp.experts.114.up_proj", + "model.layers.37.mlp.experts.115.down_proj", + "model.layers.37.mlp.experts.115.gate_proj", + "model.layers.37.mlp.experts.115.up_proj", + "model.layers.37.mlp.experts.116.down_proj", + "model.layers.37.mlp.experts.116.gate_proj", + "model.layers.37.mlp.experts.116.up_proj", + "model.layers.37.mlp.experts.117.down_proj", + "model.layers.37.mlp.experts.117.gate_proj", + "model.layers.37.mlp.experts.117.up_proj", + "model.layers.37.mlp.experts.118.down_proj", + "model.layers.37.mlp.experts.118.gate_proj", + "model.layers.37.mlp.experts.118.up_proj", + "model.layers.37.mlp.experts.119.down_proj", + "model.layers.37.mlp.experts.119.gate_proj", + "model.layers.37.mlp.experts.119.up_proj", + "model.layers.37.mlp.experts.12.down_proj", + "model.layers.37.mlp.experts.12.gate_proj", + "model.layers.37.mlp.experts.12.up_proj", + "model.layers.37.mlp.experts.120.down_proj", + "model.layers.37.mlp.experts.120.gate_proj", + "model.layers.37.mlp.experts.120.up_proj", + "model.layers.37.mlp.experts.121.down_proj", + "model.layers.37.mlp.experts.121.gate_proj", + "model.layers.37.mlp.experts.121.up_proj", + "model.layers.37.mlp.experts.122.down_proj", + "model.layers.37.mlp.experts.122.gate_proj", + "model.layers.37.mlp.experts.122.up_proj", + "model.layers.37.mlp.experts.123.down_proj", + "model.layers.37.mlp.experts.123.gate_proj", + "model.layers.37.mlp.experts.123.up_proj", + "model.layers.37.mlp.experts.124.down_proj", + "model.layers.37.mlp.experts.124.gate_proj", + "model.layers.37.mlp.experts.124.up_proj", + "model.layers.37.mlp.experts.125.down_proj", + "model.layers.37.mlp.experts.125.gate_proj", + "model.layers.37.mlp.experts.125.up_proj", + "model.layers.37.mlp.experts.126.down_proj", + "model.layers.37.mlp.experts.126.gate_proj", + "model.layers.37.mlp.experts.126.up_proj", + "model.layers.37.mlp.experts.127.down_proj", + "model.layers.37.mlp.experts.127.gate_proj", + "model.layers.37.mlp.experts.127.up_proj", + "model.layers.37.mlp.experts.128.down_proj", + "model.layers.37.mlp.experts.128.gate_proj", + "model.layers.37.mlp.experts.128.up_proj", + "model.layers.37.mlp.experts.129.down_proj", + "model.layers.37.mlp.experts.129.gate_proj", + "model.layers.37.mlp.experts.129.up_proj", + "model.layers.37.mlp.experts.13.down_proj", + "model.layers.37.mlp.experts.13.gate_proj", + "model.layers.37.mlp.experts.13.up_proj", + "model.layers.37.mlp.experts.130.down_proj", + "model.layers.37.mlp.experts.130.gate_proj", + "model.layers.37.mlp.experts.130.up_proj", + "model.layers.37.mlp.experts.131.down_proj", + "model.layers.37.mlp.experts.131.gate_proj", + "model.layers.37.mlp.experts.131.up_proj", + "model.layers.37.mlp.experts.132.down_proj", + "model.layers.37.mlp.experts.132.gate_proj", + "model.layers.37.mlp.experts.132.up_proj", + "model.layers.37.mlp.experts.133.down_proj", + "model.layers.37.mlp.experts.133.gate_proj", + "model.layers.37.mlp.experts.133.up_proj", + "model.layers.37.mlp.experts.134.down_proj", + "model.layers.37.mlp.experts.134.gate_proj", + "model.layers.37.mlp.experts.134.up_proj", + "model.layers.37.mlp.experts.135.down_proj", + "model.layers.37.mlp.experts.135.gate_proj", + "model.layers.37.mlp.experts.135.up_proj", + "model.layers.37.mlp.experts.136.down_proj", + "model.layers.37.mlp.experts.136.gate_proj", + "model.layers.37.mlp.experts.136.up_proj", + "model.layers.37.mlp.experts.137.down_proj", + "model.layers.37.mlp.experts.137.gate_proj", + "model.layers.37.mlp.experts.137.up_proj", + "model.layers.37.mlp.experts.138.down_proj", + "model.layers.37.mlp.experts.138.gate_proj", + "model.layers.37.mlp.experts.138.up_proj", + "model.layers.37.mlp.experts.139.down_proj", + "model.layers.37.mlp.experts.139.gate_proj", + "model.layers.37.mlp.experts.139.up_proj", + "model.layers.37.mlp.experts.14.down_proj", + "model.layers.37.mlp.experts.14.gate_proj", + "model.layers.37.mlp.experts.14.up_proj", + "model.layers.37.mlp.experts.140.down_proj", + "model.layers.37.mlp.experts.140.gate_proj", + "model.layers.37.mlp.experts.140.up_proj", + "model.layers.37.mlp.experts.141.down_proj", + "model.layers.37.mlp.experts.141.gate_proj", + "model.layers.37.mlp.experts.141.up_proj", + "model.layers.37.mlp.experts.142.down_proj", + "model.layers.37.mlp.experts.142.gate_proj", + "model.layers.37.mlp.experts.142.up_proj", + "model.layers.37.mlp.experts.143.down_proj", + "model.layers.37.mlp.experts.143.gate_proj", + "model.layers.37.mlp.experts.143.up_proj", + "model.layers.37.mlp.experts.144.down_proj", + "model.layers.37.mlp.experts.144.gate_proj", + "model.layers.37.mlp.experts.144.up_proj", + "model.layers.37.mlp.experts.145.down_proj", + "model.layers.37.mlp.experts.145.gate_proj", + "model.layers.37.mlp.experts.145.up_proj", + "model.layers.37.mlp.experts.146.down_proj", + "model.layers.37.mlp.experts.146.gate_proj", + "model.layers.37.mlp.experts.146.up_proj", + "model.layers.37.mlp.experts.147.down_proj", + "model.layers.37.mlp.experts.147.gate_proj", + "model.layers.37.mlp.experts.147.up_proj", + "model.layers.37.mlp.experts.148.down_proj", + "model.layers.37.mlp.experts.148.gate_proj", + "model.layers.37.mlp.experts.148.up_proj", + "model.layers.37.mlp.experts.149.down_proj", + "model.layers.37.mlp.experts.149.gate_proj", + "model.layers.37.mlp.experts.149.up_proj", + "model.layers.37.mlp.experts.15.down_proj", + "model.layers.37.mlp.experts.15.gate_proj", + "model.layers.37.mlp.experts.15.up_proj", + "model.layers.37.mlp.experts.150.down_proj", + "model.layers.37.mlp.experts.150.gate_proj", + "model.layers.37.mlp.experts.150.up_proj", + "model.layers.37.mlp.experts.151.down_proj", + "model.layers.37.mlp.experts.151.gate_proj", + "model.layers.37.mlp.experts.151.up_proj", + "model.layers.37.mlp.experts.152.down_proj", + "model.layers.37.mlp.experts.152.gate_proj", + "model.layers.37.mlp.experts.152.up_proj", + "model.layers.37.mlp.experts.153.down_proj", + "model.layers.37.mlp.experts.153.gate_proj", + "model.layers.37.mlp.experts.153.up_proj", + "model.layers.37.mlp.experts.154.down_proj", + "model.layers.37.mlp.experts.154.gate_proj", + "model.layers.37.mlp.experts.154.up_proj", + "model.layers.37.mlp.experts.155.down_proj", + "model.layers.37.mlp.experts.155.gate_proj", + "model.layers.37.mlp.experts.155.up_proj", + "model.layers.37.mlp.experts.156.down_proj", + "model.layers.37.mlp.experts.156.gate_proj", + "model.layers.37.mlp.experts.156.up_proj", + "model.layers.37.mlp.experts.157.down_proj", + "model.layers.37.mlp.experts.157.gate_proj", + "model.layers.37.mlp.experts.157.up_proj", + "model.layers.37.mlp.experts.158.down_proj", + "model.layers.37.mlp.experts.158.gate_proj", + "model.layers.37.mlp.experts.158.up_proj", + "model.layers.37.mlp.experts.159.down_proj", + "model.layers.37.mlp.experts.159.gate_proj", + "model.layers.37.mlp.experts.159.up_proj", + "model.layers.37.mlp.experts.16.down_proj", + "model.layers.37.mlp.experts.16.gate_proj", + "model.layers.37.mlp.experts.16.up_proj", + "model.layers.37.mlp.experts.160.down_proj", + "model.layers.37.mlp.experts.160.gate_proj", + "model.layers.37.mlp.experts.160.up_proj", + "model.layers.37.mlp.experts.161.down_proj", + "model.layers.37.mlp.experts.161.gate_proj", + "model.layers.37.mlp.experts.161.up_proj", + "model.layers.37.mlp.experts.162.down_proj", + "model.layers.37.mlp.experts.162.gate_proj", + "model.layers.37.mlp.experts.162.up_proj", + "model.layers.37.mlp.experts.163.down_proj", + "model.layers.37.mlp.experts.163.gate_proj", + "model.layers.37.mlp.experts.163.up_proj", + "model.layers.37.mlp.experts.164.down_proj", + "model.layers.37.mlp.experts.164.gate_proj", + "model.layers.37.mlp.experts.164.up_proj", + "model.layers.37.mlp.experts.165.down_proj", + "model.layers.37.mlp.experts.165.gate_proj", + "model.layers.37.mlp.experts.165.up_proj", + "model.layers.37.mlp.experts.166.down_proj", + "model.layers.37.mlp.experts.166.gate_proj", + "model.layers.37.mlp.experts.166.up_proj", + "model.layers.37.mlp.experts.167.down_proj", + "model.layers.37.mlp.experts.167.gate_proj", + "model.layers.37.mlp.experts.167.up_proj", + "model.layers.37.mlp.experts.168.down_proj", + "model.layers.37.mlp.experts.168.gate_proj", + "model.layers.37.mlp.experts.168.up_proj", + "model.layers.37.mlp.experts.169.down_proj", + "model.layers.37.mlp.experts.169.gate_proj", + "model.layers.37.mlp.experts.169.up_proj", + "model.layers.37.mlp.experts.17.down_proj", + "model.layers.37.mlp.experts.17.gate_proj", + "model.layers.37.mlp.experts.17.up_proj", + "model.layers.37.mlp.experts.170.down_proj", + "model.layers.37.mlp.experts.170.gate_proj", + "model.layers.37.mlp.experts.170.up_proj", + "model.layers.37.mlp.experts.171.down_proj", + "model.layers.37.mlp.experts.171.gate_proj", + "model.layers.37.mlp.experts.171.up_proj", + "model.layers.37.mlp.experts.172.down_proj", + "model.layers.37.mlp.experts.172.gate_proj", + "model.layers.37.mlp.experts.172.up_proj", + "model.layers.37.mlp.experts.173.down_proj", + "model.layers.37.mlp.experts.173.gate_proj", + "model.layers.37.mlp.experts.173.up_proj", + "model.layers.37.mlp.experts.174.down_proj", + "model.layers.37.mlp.experts.174.gate_proj", + "model.layers.37.mlp.experts.174.up_proj", + "model.layers.37.mlp.experts.175.down_proj", + "model.layers.37.mlp.experts.175.gate_proj", + "model.layers.37.mlp.experts.175.up_proj", + "model.layers.37.mlp.experts.176.down_proj", + "model.layers.37.mlp.experts.176.gate_proj", + "model.layers.37.mlp.experts.176.up_proj", + "model.layers.37.mlp.experts.177.down_proj", + "model.layers.37.mlp.experts.177.gate_proj", + "model.layers.37.mlp.experts.177.up_proj", + "model.layers.37.mlp.experts.178.down_proj", + "model.layers.37.mlp.experts.178.gate_proj", + "model.layers.37.mlp.experts.178.up_proj", + "model.layers.37.mlp.experts.179.down_proj", + "model.layers.37.mlp.experts.179.gate_proj", + "model.layers.37.mlp.experts.179.up_proj", + "model.layers.37.mlp.experts.18.down_proj", + "model.layers.37.mlp.experts.18.gate_proj", + "model.layers.37.mlp.experts.18.up_proj", + "model.layers.37.mlp.experts.180.down_proj", + "model.layers.37.mlp.experts.180.gate_proj", + "model.layers.37.mlp.experts.180.up_proj", + "model.layers.37.mlp.experts.181.down_proj", + "model.layers.37.mlp.experts.181.gate_proj", + "model.layers.37.mlp.experts.181.up_proj", + "model.layers.37.mlp.experts.182.down_proj", + "model.layers.37.mlp.experts.182.gate_proj", + "model.layers.37.mlp.experts.182.up_proj", + "model.layers.37.mlp.experts.183.down_proj", + "model.layers.37.mlp.experts.183.gate_proj", + "model.layers.37.mlp.experts.183.up_proj", + "model.layers.37.mlp.experts.184.down_proj", + "model.layers.37.mlp.experts.184.gate_proj", + "model.layers.37.mlp.experts.184.up_proj", + "model.layers.37.mlp.experts.185.down_proj", + "model.layers.37.mlp.experts.185.gate_proj", + "model.layers.37.mlp.experts.185.up_proj", + "model.layers.37.mlp.experts.186.down_proj", + "model.layers.37.mlp.experts.186.gate_proj", + "model.layers.37.mlp.experts.186.up_proj", + "model.layers.37.mlp.experts.187.down_proj", + "model.layers.37.mlp.experts.187.gate_proj", + "model.layers.37.mlp.experts.187.up_proj", + "model.layers.37.mlp.experts.188.down_proj", + "model.layers.37.mlp.experts.188.gate_proj", + "model.layers.37.mlp.experts.188.up_proj", + "model.layers.37.mlp.experts.189.down_proj", + "model.layers.37.mlp.experts.189.gate_proj", + "model.layers.37.mlp.experts.189.up_proj", + "model.layers.37.mlp.experts.19.down_proj", + "model.layers.37.mlp.experts.19.gate_proj", + "model.layers.37.mlp.experts.19.up_proj", + "model.layers.37.mlp.experts.190.down_proj", + "model.layers.37.mlp.experts.190.gate_proj", + "model.layers.37.mlp.experts.190.up_proj", + "model.layers.37.mlp.experts.191.down_proj", + "model.layers.37.mlp.experts.191.gate_proj", + "model.layers.37.mlp.experts.191.up_proj", + "model.layers.37.mlp.experts.192.down_proj", + "model.layers.37.mlp.experts.192.gate_proj", + "model.layers.37.mlp.experts.192.up_proj", + "model.layers.37.mlp.experts.193.down_proj", + "model.layers.37.mlp.experts.193.gate_proj", + "model.layers.37.mlp.experts.193.up_proj", + "model.layers.37.mlp.experts.194.down_proj", + "model.layers.37.mlp.experts.194.gate_proj", + "model.layers.37.mlp.experts.194.up_proj", + "model.layers.37.mlp.experts.195.down_proj", + "model.layers.37.mlp.experts.195.gate_proj", + "model.layers.37.mlp.experts.195.up_proj", + "model.layers.37.mlp.experts.196.down_proj", + "model.layers.37.mlp.experts.196.gate_proj", + "model.layers.37.mlp.experts.196.up_proj", + "model.layers.37.mlp.experts.197.down_proj", + "model.layers.37.mlp.experts.197.gate_proj", + "model.layers.37.mlp.experts.197.up_proj", + "model.layers.37.mlp.experts.198.down_proj", + "model.layers.37.mlp.experts.198.gate_proj", + "model.layers.37.mlp.experts.198.up_proj", + "model.layers.37.mlp.experts.199.down_proj", + "model.layers.37.mlp.experts.199.gate_proj", + "model.layers.37.mlp.experts.199.up_proj", + "model.layers.37.mlp.experts.2.down_proj", + "model.layers.37.mlp.experts.2.gate_proj", + "model.layers.37.mlp.experts.2.up_proj", + "model.layers.37.mlp.experts.20.down_proj", + "model.layers.37.mlp.experts.20.gate_proj", + "model.layers.37.mlp.experts.20.up_proj", + "model.layers.37.mlp.experts.200.down_proj", + "model.layers.37.mlp.experts.200.gate_proj", + "model.layers.37.mlp.experts.200.up_proj", + "model.layers.37.mlp.experts.201.down_proj", + "model.layers.37.mlp.experts.201.gate_proj", + "model.layers.37.mlp.experts.201.up_proj", + "model.layers.37.mlp.experts.202.down_proj", + "model.layers.37.mlp.experts.202.gate_proj", + "model.layers.37.mlp.experts.202.up_proj", + "model.layers.37.mlp.experts.203.down_proj", + "model.layers.37.mlp.experts.203.gate_proj", + "model.layers.37.mlp.experts.203.up_proj", + "model.layers.37.mlp.experts.204.down_proj", + "model.layers.37.mlp.experts.204.gate_proj", + "model.layers.37.mlp.experts.204.up_proj", + "model.layers.37.mlp.experts.205.down_proj", + "model.layers.37.mlp.experts.205.gate_proj", + "model.layers.37.mlp.experts.205.up_proj", + "model.layers.37.mlp.experts.206.down_proj", + "model.layers.37.mlp.experts.206.gate_proj", + "model.layers.37.mlp.experts.206.up_proj", + "model.layers.37.mlp.experts.207.down_proj", + "model.layers.37.mlp.experts.207.gate_proj", + "model.layers.37.mlp.experts.207.up_proj", + "model.layers.37.mlp.experts.208.down_proj", + "model.layers.37.mlp.experts.208.gate_proj", + "model.layers.37.mlp.experts.208.up_proj", + "model.layers.37.mlp.experts.209.down_proj", + "model.layers.37.mlp.experts.209.gate_proj", + "model.layers.37.mlp.experts.209.up_proj", + "model.layers.37.mlp.experts.21.down_proj", + "model.layers.37.mlp.experts.21.gate_proj", + "model.layers.37.mlp.experts.21.up_proj", + "model.layers.37.mlp.experts.210.down_proj", + "model.layers.37.mlp.experts.210.gate_proj", + "model.layers.37.mlp.experts.210.up_proj", + "model.layers.37.mlp.experts.211.down_proj", + "model.layers.37.mlp.experts.211.gate_proj", + "model.layers.37.mlp.experts.211.up_proj", + "model.layers.37.mlp.experts.212.down_proj", + "model.layers.37.mlp.experts.212.gate_proj", + "model.layers.37.mlp.experts.212.up_proj", + "model.layers.37.mlp.experts.213.down_proj", + "model.layers.37.mlp.experts.213.gate_proj", + "model.layers.37.mlp.experts.213.up_proj", + "model.layers.37.mlp.experts.214.down_proj", + "model.layers.37.mlp.experts.214.gate_proj", + "model.layers.37.mlp.experts.214.up_proj", + "model.layers.37.mlp.experts.215.down_proj", + "model.layers.37.mlp.experts.215.gate_proj", + "model.layers.37.mlp.experts.215.up_proj", + "model.layers.37.mlp.experts.216.down_proj", + "model.layers.37.mlp.experts.216.gate_proj", + "model.layers.37.mlp.experts.216.up_proj", + "model.layers.37.mlp.experts.217.down_proj", + "model.layers.37.mlp.experts.217.gate_proj", + "model.layers.37.mlp.experts.217.up_proj", + "model.layers.37.mlp.experts.218.down_proj", + "model.layers.37.mlp.experts.218.gate_proj", + "model.layers.37.mlp.experts.218.up_proj", + "model.layers.37.mlp.experts.219.down_proj", + "model.layers.37.mlp.experts.219.gate_proj", + "model.layers.37.mlp.experts.219.up_proj", + "model.layers.37.mlp.experts.22.down_proj", + "model.layers.37.mlp.experts.22.gate_proj", + "model.layers.37.mlp.experts.22.up_proj", + "model.layers.37.mlp.experts.220.down_proj", + "model.layers.37.mlp.experts.220.gate_proj", + "model.layers.37.mlp.experts.220.up_proj", + "model.layers.37.mlp.experts.221.down_proj", + "model.layers.37.mlp.experts.221.gate_proj", + "model.layers.37.mlp.experts.221.up_proj", + "model.layers.37.mlp.experts.222.down_proj", + "model.layers.37.mlp.experts.222.gate_proj", + "model.layers.37.mlp.experts.222.up_proj", + "model.layers.37.mlp.experts.223.down_proj", + "model.layers.37.mlp.experts.223.gate_proj", + "model.layers.37.mlp.experts.223.up_proj", + "model.layers.37.mlp.experts.224.down_proj", + "model.layers.37.mlp.experts.224.gate_proj", + "model.layers.37.mlp.experts.224.up_proj", + "model.layers.37.mlp.experts.225.down_proj", + "model.layers.37.mlp.experts.225.gate_proj", + "model.layers.37.mlp.experts.225.up_proj", + "model.layers.37.mlp.experts.226.down_proj", + "model.layers.37.mlp.experts.226.gate_proj", + "model.layers.37.mlp.experts.226.up_proj", + "model.layers.37.mlp.experts.227.down_proj", + "model.layers.37.mlp.experts.227.gate_proj", + "model.layers.37.mlp.experts.227.up_proj", + "model.layers.37.mlp.experts.228.down_proj", + "model.layers.37.mlp.experts.228.gate_proj", + "model.layers.37.mlp.experts.228.up_proj", + "model.layers.37.mlp.experts.229.down_proj", + "model.layers.37.mlp.experts.229.gate_proj", + "model.layers.37.mlp.experts.229.up_proj", + "model.layers.37.mlp.experts.23.down_proj", + "model.layers.37.mlp.experts.23.gate_proj", + "model.layers.37.mlp.experts.23.up_proj", + "model.layers.37.mlp.experts.230.down_proj", + "model.layers.37.mlp.experts.230.gate_proj", + "model.layers.37.mlp.experts.230.up_proj", + "model.layers.37.mlp.experts.231.down_proj", + "model.layers.37.mlp.experts.231.gate_proj", + "model.layers.37.mlp.experts.231.up_proj", + "model.layers.37.mlp.experts.232.down_proj", + "model.layers.37.mlp.experts.232.gate_proj", + "model.layers.37.mlp.experts.232.up_proj", + "model.layers.37.mlp.experts.233.down_proj", + "model.layers.37.mlp.experts.233.gate_proj", + "model.layers.37.mlp.experts.233.up_proj", + "model.layers.37.mlp.experts.234.down_proj", + "model.layers.37.mlp.experts.234.gate_proj", + "model.layers.37.mlp.experts.234.up_proj", + "model.layers.37.mlp.experts.235.down_proj", + "model.layers.37.mlp.experts.235.gate_proj", + "model.layers.37.mlp.experts.235.up_proj", + "model.layers.37.mlp.experts.236.down_proj", + "model.layers.37.mlp.experts.236.gate_proj", + "model.layers.37.mlp.experts.236.up_proj", + "model.layers.37.mlp.experts.237.down_proj", + "model.layers.37.mlp.experts.237.gate_proj", + "model.layers.37.mlp.experts.237.up_proj", + "model.layers.37.mlp.experts.238.down_proj", + "model.layers.37.mlp.experts.238.gate_proj", + "model.layers.37.mlp.experts.238.up_proj", + "model.layers.37.mlp.experts.239.down_proj", + "model.layers.37.mlp.experts.239.gate_proj", + "model.layers.37.mlp.experts.239.up_proj", + "model.layers.37.mlp.experts.24.down_proj", + "model.layers.37.mlp.experts.24.gate_proj", + "model.layers.37.mlp.experts.24.up_proj", + "model.layers.37.mlp.experts.240.down_proj", + "model.layers.37.mlp.experts.240.gate_proj", + "model.layers.37.mlp.experts.240.up_proj", + "model.layers.37.mlp.experts.241.down_proj", + "model.layers.37.mlp.experts.241.gate_proj", + "model.layers.37.mlp.experts.241.up_proj", + "model.layers.37.mlp.experts.242.down_proj", + "model.layers.37.mlp.experts.242.gate_proj", + "model.layers.37.mlp.experts.242.up_proj", + "model.layers.37.mlp.experts.243.down_proj", + "model.layers.37.mlp.experts.243.gate_proj", + "model.layers.37.mlp.experts.243.up_proj", + "model.layers.37.mlp.experts.244.down_proj", + "model.layers.37.mlp.experts.244.gate_proj", + "model.layers.37.mlp.experts.244.up_proj", + "model.layers.37.mlp.experts.245.down_proj", + "model.layers.37.mlp.experts.245.gate_proj", + "model.layers.37.mlp.experts.245.up_proj", + "model.layers.37.mlp.experts.246.down_proj", + "model.layers.37.mlp.experts.246.gate_proj", + "model.layers.37.mlp.experts.246.up_proj", + "model.layers.37.mlp.experts.247.down_proj", + "model.layers.37.mlp.experts.247.gate_proj", + "model.layers.37.mlp.experts.247.up_proj", + "model.layers.37.mlp.experts.248.down_proj", + "model.layers.37.mlp.experts.248.gate_proj", + "model.layers.37.mlp.experts.248.up_proj", + "model.layers.37.mlp.experts.249.down_proj", + "model.layers.37.mlp.experts.249.gate_proj", + "model.layers.37.mlp.experts.249.up_proj", + "model.layers.37.mlp.experts.25.down_proj", + "model.layers.37.mlp.experts.25.gate_proj", + "model.layers.37.mlp.experts.25.up_proj", + "model.layers.37.mlp.experts.250.down_proj", + "model.layers.37.mlp.experts.250.gate_proj", + "model.layers.37.mlp.experts.250.up_proj", + "model.layers.37.mlp.experts.251.down_proj", + "model.layers.37.mlp.experts.251.gate_proj", + "model.layers.37.mlp.experts.251.up_proj", + "model.layers.37.mlp.experts.252.down_proj", + "model.layers.37.mlp.experts.252.gate_proj", + "model.layers.37.mlp.experts.252.up_proj", + "model.layers.37.mlp.experts.253.down_proj", + "model.layers.37.mlp.experts.253.gate_proj", + "model.layers.37.mlp.experts.253.up_proj", + "model.layers.37.mlp.experts.254.down_proj", + "model.layers.37.mlp.experts.254.gate_proj", + "model.layers.37.mlp.experts.254.up_proj", + "model.layers.37.mlp.experts.255.down_proj", + "model.layers.37.mlp.experts.255.gate_proj", + "model.layers.37.mlp.experts.255.up_proj", + "model.layers.37.mlp.experts.26.down_proj", + "model.layers.37.mlp.experts.26.gate_proj", + "model.layers.37.mlp.experts.26.up_proj", + "model.layers.37.mlp.experts.27.down_proj", + "model.layers.37.mlp.experts.27.gate_proj", + "model.layers.37.mlp.experts.27.up_proj", + "model.layers.37.mlp.experts.28.down_proj", + "model.layers.37.mlp.experts.28.gate_proj", + "model.layers.37.mlp.experts.28.up_proj", + "model.layers.37.mlp.experts.29.down_proj", + "model.layers.37.mlp.experts.29.gate_proj", + "model.layers.37.mlp.experts.29.up_proj", + "model.layers.37.mlp.experts.3.down_proj", + "model.layers.37.mlp.experts.3.gate_proj", + "model.layers.37.mlp.experts.3.up_proj", + "model.layers.37.mlp.experts.30.down_proj", + "model.layers.37.mlp.experts.30.gate_proj", + "model.layers.37.mlp.experts.30.up_proj", + "model.layers.37.mlp.experts.31.down_proj", + "model.layers.37.mlp.experts.31.gate_proj", + "model.layers.37.mlp.experts.31.up_proj", + "model.layers.37.mlp.experts.32.down_proj", + "model.layers.37.mlp.experts.32.gate_proj", + "model.layers.37.mlp.experts.32.up_proj", + "model.layers.37.mlp.experts.33.down_proj", + "model.layers.37.mlp.experts.33.gate_proj", + "model.layers.37.mlp.experts.33.up_proj", + "model.layers.37.mlp.experts.34.down_proj", + "model.layers.37.mlp.experts.34.gate_proj", + "model.layers.37.mlp.experts.34.up_proj", + "model.layers.37.mlp.experts.35.down_proj", + "model.layers.37.mlp.experts.35.gate_proj", + "model.layers.37.mlp.experts.35.up_proj", + "model.layers.37.mlp.experts.36.down_proj", + "model.layers.37.mlp.experts.36.gate_proj", + "model.layers.37.mlp.experts.36.up_proj", + "model.layers.37.mlp.experts.37.down_proj", + "model.layers.37.mlp.experts.37.gate_proj", + "model.layers.37.mlp.experts.37.up_proj", + "model.layers.37.mlp.experts.38.down_proj", + "model.layers.37.mlp.experts.38.gate_proj", + "model.layers.37.mlp.experts.38.up_proj", + "model.layers.37.mlp.experts.39.down_proj", + "model.layers.37.mlp.experts.39.gate_proj", + "model.layers.37.mlp.experts.39.up_proj", + "model.layers.37.mlp.experts.4.down_proj", + "model.layers.37.mlp.experts.4.gate_proj", + "model.layers.37.mlp.experts.4.up_proj", + "model.layers.37.mlp.experts.40.down_proj", + "model.layers.37.mlp.experts.40.gate_proj", + "model.layers.37.mlp.experts.40.up_proj", + "model.layers.37.mlp.experts.41.down_proj", + "model.layers.37.mlp.experts.41.gate_proj", + "model.layers.37.mlp.experts.41.up_proj", + "model.layers.37.mlp.experts.42.down_proj", + "model.layers.37.mlp.experts.42.gate_proj", + "model.layers.37.mlp.experts.42.up_proj", + "model.layers.37.mlp.experts.43.down_proj", + "model.layers.37.mlp.experts.43.gate_proj", + "model.layers.37.mlp.experts.43.up_proj", + "model.layers.37.mlp.experts.44.down_proj", + "model.layers.37.mlp.experts.44.gate_proj", + "model.layers.37.mlp.experts.44.up_proj", + "model.layers.37.mlp.experts.45.down_proj", + "model.layers.37.mlp.experts.45.gate_proj", + "model.layers.37.mlp.experts.45.up_proj", + "model.layers.37.mlp.experts.46.down_proj", + "model.layers.37.mlp.experts.46.gate_proj", + "model.layers.37.mlp.experts.46.up_proj", + "model.layers.37.mlp.experts.47.down_proj", + "model.layers.37.mlp.experts.47.gate_proj", + "model.layers.37.mlp.experts.47.up_proj", + "model.layers.37.mlp.experts.48.down_proj", + "model.layers.37.mlp.experts.48.gate_proj", + "model.layers.37.mlp.experts.48.up_proj", + "model.layers.37.mlp.experts.49.down_proj", + "model.layers.37.mlp.experts.49.gate_proj", + "model.layers.37.mlp.experts.49.up_proj", + "model.layers.37.mlp.experts.5.down_proj", + "model.layers.37.mlp.experts.5.gate_proj", + "model.layers.37.mlp.experts.5.up_proj", + "model.layers.37.mlp.experts.50.down_proj", + "model.layers.37.mlp.experts.50.gate_proj", + "model.layers.37.mlp.experts.50.up_proj", + "model.layers.37.mlp.experts.51.down_proj", + "model.layers.37.mlp.experts.51.gate_proj", + "model.layers.37.mlp.experts.51.up_proj", + "model.layers.37.mlp.experts.52.down_proj", + "model.layers.37.mlp.experts.52.gate_proj", + "model.layers.37.mlp.experts.52.up_proj", + "model.layers.37.mlp.experts.53.down_proj", + "model.layers.37.mlp.experts.53.gate_proj", + "model.layers.37.mlp.experts.53.up_proj", + "model.layers.37.mlp.experts.54.down_proj", + "model.layers.37.mlp.experts.54.gate_proj", + "model.layers.37.mlp.experts.54.up_proj", + "model.layers.37.mlp.experts.55.down_proj", + "model.layers.37.mlp.experts.55.gate_proj", + "model.layers.37.mlp.experts.55.up_proj", + "model.layers.37.mlp.experts.56.down_proj", + "model.layers.37.mlp.experts.56.gate_proj", + "model.layers.37.mlp.experts.56.up_proj", + "model.layers.37.mlp.experts.57.down_proj", + "model.layers.37.mlp.experts.57.gate_proj", + "model.layers.37.mlp.experts.57.up_proj", + "model.layers.37.mlp.experts.58.down_proj", + "model.layers.37.mlp.experts.58.gate_proj", + "model.layers.37.mlp.experts.58.up_proj", + "model.layers.37.mlp.experts.59.down_proj", + "model.layers.37.mlp.experts.59.gate_proj", + "model.layers.37.mlp.experts.59.up_proj", + "model.layers.37.mlp.experts.6.down_proj", + "model.layers.37.mlp.experts.6.gate_proj", + "model.layers.37.mlp.experts.6.up_proj", + "model.layers.37.mlp.experts.60.down_proj", + "model.layers.37.mlp.experts.60.gate_proj", + "model.layers.37.mlp.experts.60.up_proj", + "model.layers.37.mlp.experts.61.down_proj", + "model.layers.37.mlp.experts.61.gate_proj", + "model.layers.37.mlp.experts.61.up_proj", + "model.layers.37.mlp.experts.62.down_proj", + "model.layers.37.mlp.experts.62.gate_proj", + "model.layers.37.mlp.experts.62.up_proj", + "model.layers.37.mlp.experts.63.down_proj", + "model.layers.37.mlp.experts.63.gate_proj", + "model.layers.37.mlp.experts.63.up_proj", + "model.layers.37.mlp.experts.64.down_proj", + "model.layers.37.mlp.experts.64.gate_proj", + "model.layers.37.mlp.experts.64.up_proj", + "model.layers.37.mlp.experts.65.down_proj", + "model.layers.37.mlp.experts.65.gate_proj", + "model.layers.37.mlp.experts.65.up_proj", + "model.layers.37.mlp.experts.66.down_proj", + "model.layers.37.mlp.experts.66.gate_proj", + "model.layers.37.mlp.experts.66.up_proj", + "model.layers.37.mlp.experts.67.down_proj", + "model.layers.37.mlp.experts.67.gate_proj", + "model.layers.37.mlp.experts.67.up_proj", + "model.layers.37.mlp.experts.68.down_proj", + "model.layers.37.mlp.experts.68.gate_proj", + "model.layers.37.mlp.experts.68.up_proj", + "model.layers.37.mlp.experts.69.down_proj", + "model.layers.37.mlp.experts.69.gate_proj", + "model.layers.37.mlp.experts.69.up_proj", + "model.layers.37.mlp.experts.7.down_proj", + "model.layers.37.mlp.experts.7.gate_proj", + "model.layers.37.mlp.experts.7.up_proj", + "model.layers.37.mlp.experts.70.down_proj", + "model.layers.37.mlp.experts.70.gate_proj", + "model.layers.37.mlp.experts.70.up_proj", + "model.layers.37.mlp.experts.71.down_proj", + "model.layers.37.mlp.experts.71.gate_proj", + "model.layers.37.mlp.experts.71.up_proj", + "model.layers.37.mlp.experts.72.down_proj", + "model.layers.37.mlp.experts.72.gate_proj", + "model.layers.37.mlp.experts.72.up_proj", + "model.layers.37.mlp.experts.73.down_proj", + "model.layers.37.mlp.experts.73.gate_proj", + "model.layers.37.mlp.experts.73.up_proj", + "model.layers.37.mlp.experts.74.down_proj", + "model.layers.37.mlp.experts.74.gate_proj", + "model.layers.37.mlp.experts.74.up_proj", + "model.layers.37.mlp.experts.75.down_proj", + "model.layers.37.mlp.experts.75.gate_proj", + "model.layers.37.mlp.experts.75.up_proj", + "model.layers.37.mlp.experts.76.down_proj", + "model.layers.37.mlp.experts.76.gate_proj", + "model.layers.37.mlp.experts.76.up_proj", + "model.layers.37.mlp.experts.77.down_proj", + "model.layers.37.mlp.experts.77.gate_proj", + "model.layers.37.mlp.experts.77.up_proj", + "model.layers.37.mlp.experts.78.down_proj", + "model.layers.37.mlp.experts.78.gate_proj", + "model.layers.37.mlp.experts.78.up_proj", + "model.layers.37.mlp.experts.79.down_proj", + "model.layers.37.mlp.experts.79.gate_proj", + "model.layers.37.mlp.experts.79.up_proj", + "model.layers.37.mlp.experts.8.down_proj", + "model.layers.37.mlp.experts.8.gate_proj", + "model.layers.37.mlp.experts.8.up_proj", + "model.layers.37.mlp.experts.80.down_proj", + "model.layers.37.mlp.experts.80.gate_proj", + "model.layers.37.mlp.experts.80.up_proj", + "model.layers.37.mlp.experts.81.down_proj", + "model.layers.37.mlp.experts.81.gate_proj", + "model.layers.37.mlp.experts.81.up_proj", + "model.layers.37.mlp.experts.82.down_proj", + "model.layers.37.mlp.experts.82.gate_proj", + "model.layers.37.mlp.experts.82.up_proj", + "model.layers.37.mlp.experts.83.down_proj", + "model.layers.37.mlp.experts.83.gate_proj", + "model.layers.37.mlp.experts.83.up_proj", + "model.layers.37.mlp.experts.84.down_proj", + "model.layers.37.mlp.experts.84.gate_proj", + "model.layers.37.mlp.experts.84.up_proj", + "model.layers.37.mlp.experts.85.down_proj", + "model.layers.37.mlp.experts.85.gate_proj", + "model.layers.37.mlp.experts.85.up_proj", + "model.layers.37.mlp.experts.86.down_proj", + "model.layers.37.mlp.experts.86.gate_proj", + "model.layers.37.mlp.experts.86.up_proj", + "model.layers.37.mlp.experts.87.down_proj", + "model.layers.37.mlp.experts.87.gate_proj", + "model.layers.37.mlp.experts.87.up_proj", + "model.layers.37.mlp.experts.88.down_proj", + "model.layers.37.mlp.experts.88.gate_proj", + "model.layers.37.mlp.experts.88.up_proj", + "model.layers.37.mlp.experts.89.down_proj", + "model.layers.37.mlp.experts.89.gate_proj", + "model.layers.37.mlp.experts.89.up_proj", + "model.layers.37.mlp.experts.9.down_proj", + "model.layers.37.mlp.experts.9.gate_proj", + "model.layers.37.mlp.experts.9.up_proj", + "model.layers.37.mlp.experts.90.down_proj", + "model.layers.37.mlp.experts.90.gate_proj", + "model.layers.37.mlp.experts.90.up_proj", + "model.layers.37.mlp.experts.91.down_proj", + "model.layers.37.mlp.experts.91.gate_proj", + "model.layers.37.mlp.experts.91.up_proj", + "model.layers.37.mlp.experts.92.down_proj", + "model.layers.37.mlp.experts.92.gate_proj", + "model.layers.37.mlp.experts.92.up_proj", + "model.layers.37.mlp.experts.93.down_proj", + "model.layers.37.mlp.experts.93.gate_proj", + "model.layers.37.mlp.experts.93.up_proj", + "model.layers.37.mlp.experts.94.down_proj", + "model.layers.37.mlp.experts.94.gate_proj", + "model.layers.37.mlp.experts.94.up_proj", + "model.layers.37.mlp.experts.95.down_proj", + "model.layers.37.mlp.experts.95.gate_proj", + "model.layers.37.mlp.experts.95.up_proj", + "model.layers.37.mlp.experts.96.down_proj", + "model.layers.37.mlp.experts.96.gate_proj", + "model.layers.37.mlp.experts.96.up_proj", + "model.layers.37.mlp.experts.97.down_proj", + "model.layers.37.mlp.experts.97.gate_proj", + "model.layers.37.mlp.experts.97.up_proj", + "model.layers.37.mlp.experts.98.down_proj", + "model.layers.37.mlp.experts.98.gate_proj", + "model.layers.37.mlp.experts.98.up_proj", + "model.layers.37.mlp.experts.99.down_proj", + "model.layers.37.mlp.experts.99.gate_proj", + "model.layers.37.mlp.experts.99.up_proj", + "model.layers.38.mlp.experts.0.down_proj", + "model.layers.38.mlp.experts.0.gate_proj", + "model.layers.38.mlp.experts.0.up_proj", + "model.layers.38.mlp.experts.1.down_proj", + "model.layers.38.mlp.experts.1.gate_proj", + "model.layers.38.mlp.experts.1.up_proj", + "model.layers.38.mlp.experts.10.down_proj", + "model.layers.38.mlp.experts.10.gate_proj", + "model.layers.38.mlp.experts.10.up_proj", + "model.layers.38.mlp.experts.100.down_proj", + "model.layers.38.mlp.experts.100.gate_proj", + "model.layers.38.mlp.experts.100.up_proj", + "model.layers.38.mlp.experts.101.down_proj", + "model.layers.38.mlp.experts.101.gate_proj", + "model.layers.38.mlp.experts.101.up_proj", + "model.layers.38.mlp.experts.102.down_proj", + "model.layers.38.mlp.experts.102.gate_proj", + "model.layers.38.mlp.experts.102.up_proj", + "model.layers.38.mlp.experts.103.down_proj", + "model.layers.38.mlp.experts.103.gate_proj", + "model.layers.38.mlp.experts.103.up_proj", + "model.layers.38.mlp.experts.104.down_proj", + "model.layers.38.mlp.experts.104.gate_proj", + "model.layers.38.mlp.experts.104.up_proj", + "model.layers.38.mlp.experts.105.down_proj", + "model.layers.38.mlp.experts.105.gate_proj", + "model.layers.38.mlp.experts.105.up_proj", + "model.layers.38.mlp.experts.106.down_proj", + "model.layers.38.mlp.experts.106.gate_proj", + "model.layers.38.mlp.experts.106.up_proj", + "model.layers.38.mlp.experts.107.down_proj", + "model.layers.38.mlp.experts.107.gate_proj", + "model.layers.38.mlp.experts.107.up_proj", + "model.layers.38.mlp.experts.108.down_proj", + "model.layers.38.mlp.experts.108.gate_proj", + "model.layers.38.mlp.experts.108.up_proj", + "model.layers.38.mlp.experts.109.down_proj", + "model.layers.38.mlp.experts.109.gate_proj", + "model.layers.38.mlp.experts.109.up_proj", + "model.layers.38.mlp.experts.11.down_proj", + "model.layers.38.mlp.experts.11.gate_proj", + "model.layers.38.mlp.experts.11.up_proj", + "model.layers.38.mlp.experts.110.down_proj", + "model.layers.38.mlp.experts.110.gate_proj", + "model.layers.38.mlp.experts.110.up_proj", + "model.layers.38.mlp.experts.111.down_proj", + "model.layers.38.mlp.experts.111.gate_proj", + "model.layers.38.mlp.experts.111.up_proj", + "model.layers.38.mlp.experts.112.down_proj", + "model.layers.38.mlp.experts.112.gate_proj", + "model.layers.38.mlp.experts.112.up_proj", + "model.layers.38.mlp.experts.113.down_proj", + "model.layers.38.mlp.experts.113.gate_proj", + "model.layers.38.mlp.experts.113.up_proj", + "model.layers.38.mlp.experts.114.down_proj", + "model.layers.38.mlp.experts.114.gate_proj", + "model.layers.38.mlp.experts.114.up_proj", + "model.layers.38.mlp.experts.115.down_proj", + "model.layers.38.mlp.experts.115.gate_proj", + "model.layers.38.mlp.experts.115.up_proj", + "model.layers.38.mlp.experts.116.down_proj", + "model.layers.38.mlp.experts.116.gate_proj", + "model.layers.38.mlp.experts.116.up_proj", + "model.layers.38.mlp.experts.117.down_proj", + "model.layers.38.mlp.experts.117.gate_proj", + "model.layers.38.mlp.experts.117.up_proj", + "model.layers.38.mlp.experts.118.down_proj", + "model.layers.38.mlp.experts.118.gate_proj", + "model.layers.38.mlp.experts.118.up_proj", + "model.layers.38.mlp.experts.119.down_proj", + "model.layers.38.mlp.experts.119.gate_proj", + "model.layers.38.mlp.experts.119.up_proj", + "model.layers.38.mlp.experts.12.down_proj", + "model.layers.38.mlp.experts.12.gate_proj", + "model.layers.38.mlp.experts.12.up_proj", + "model.layers.38.mlp.experts.120.down_proj", + "model.layers.38.mlp.experts.120.gate_proj", + "model.layers.38.mlp.experts.120.up_proj", + "model.layers.38.mlp.experts.121.down_proj", + "model.layers.38.mlp.experts.121.gate_proj", + "model.layers.38.mlp.experts.121.up_proj", + "model.layers.38.mlp.experts.122.down_proj", + "model.layers.38.mlp.experts.122.gate_proj", + "model.layers.38.mlp.experts.122.up_proj", + "model.layers.38.mlp.experts.123.down_proj", + "model.layers.38.mlp.experts.123.gate_proj", + "model.layers.38.mlp.experts.123.up_proj", + "model.layers.38.mlp.experts.124.down_proj", + "model.layers.38.mlp.experts.124.gate_proj", + "model.layers.38.mlp.experts.124.up_proj", + "model.layers.38.mlp.experts.125.down_proj", + "model.layers.38.mlp.experts.125.gate_proj", + "model.layers.38.mlp.experts.125.up_proj", + "model.layers.38.mlp.experts.126.down_proj", + "model.layers.38.mlp.experts.126.gate_proj", + "model.layers.38.mlp.experts.126.up_proj", + "model.layers.38.mlp.experts.127.down_proj", + "model.layers.38.mlp.experts.127.gate_proj", + "model.layers.38.mlp.experts.127.up_proj", + "model.layers.38.mlp.experts.128.down_proj", + "model.layers.38.mlp.experts.128.gate_proj", + "model.layers.38.mlp.experts.128.up_proj", + "model.layers.38.mlp.experts.129.down_proj", + "model.layers.38.mlp.experts.129.gate_proj", + "model.layers.38.mlp.experts.129.up_proj", + "model.layers.38.mlp.experts.13.down_proj", + "model.layers.38.mlp.experts.13.gate_proj", + "model.layers.38.mlp.experts.13.up_proj", + "model.layers.38.mlp.experts.130.down_proj", + "model.layers.38.mlp.experts.130.gate_proj", + "model.layers.38.mlp.experts.130.up_proj", + "model.layers.38.mlp.experts.131.down_proj", + "model.layers.38.mlp.experts.131.gate_proj", + "model.layers.38.mlp.experts.131.up_proj", + "model.layers.38.mlp.experts.132.down_proj", + "model.layers.38.mlp.experts.132.gate_proj", + "model.layers.38.mlp.experts.132.up_proj", + "model.layers.38.mlp.experts.133.down_proj", + "model.layers.38.mlp.experts.133.gate_proj", + "model.layers.38.mlp.experts.133.up_proj", + "model.layers.38.mlp.experts.134.down_proj", + "model.layers.38.mlp.experts.134.gate_proj", + "model.layers.38.mlp.experts.134.up_proj", + "model.layers.38.mlp.experts.135.down_proj", + "model.layers.38.mlp.experts.135.gate_proj", + "model.layers.38.mlp.experts.135.up_proj", + "model.layers.38.mlp.experts.136.down_proj", + "model.layers.38.mlp.experts.136.gate_proj", + "model.layers.38.mlp.experts.136.up_proj", + "model.layers.38.mlp.experts.137.down_proj", + "model.layers.38.mlp.experts.137.gate_proj", + "model.layers.38.mlp.experts.137.up_proj", + "model.layers.38.mlp.experts.138.down_proj", + "model.layers.38.mlp.experts.138.gate_proj", + "model.layers.38.mlp.experts.138.up_proj", + "model.layers.38.mlp.experts.139.down_proj", + "model.layers.38.mlp.experts.139.gate_proj", + "model.layers.38.mlp.experts.139.up_proj", + "model.layers.38.mlp.experts.14.down_proj", + "model.layers.38.mlp.experts.14.gate_proj", + "model.layers.38.mlp.experts.14.up_proj", + "model.layers.38.mlp.experts.140.down_proj", + "model.layers.38.mlp.experts.140.gate_proj", + "model.layers.38.mlp.experts.140.up_proj", + "model.layers.38.mlp.experts.141.down_proj", + "model.layers.38.mlp.experts.141.gate_proj", + "model.layers.38.mlp.experts.141.up_proj", + "model.layers.38.mlp.experts.142.down_proj", + "model.layers.38.mlp.experts.142.gate_proj", + "model.layers.38.mlp.experts.142.up_proj", + "model.layers.38.mlp.experts.143.down_proj", + "model.layers.38.mlp.experts.143.gate_proj", + "model.layers.38.mlp.experts.143.up_proj", + "model.layers.38.mlp.experts.144.down_proj", + "model.layers.38.mlp.experts.144.gate_proj", + "model.layers.38.mlp.experts.144.up_proj", + "model.layers.38.mlp.experts.145.down_proj", + "model.layers.38.mlp.experts.145.gate_proj", + "model.layers.38.mlp.experts.145.up_proj", + "model.layers.38.mlp.experts.146.down_proj", + "model.layers.38.mlp.experts.146.gate_proj", + "model.layers.38.mlp.experts.146.up_proj", + "model.layers.38.mlp.experts.147.down_proj", + "model.layers.38.mlp.experts.147.gate_proj", + "model.layers.38.mlp.experts.147.up_proj", + "model.layers.38.mlp.experts.148.down_proj", + "model.layers.38.mlp.experts.148.gate_proj", + "model.layers.38.mlp.experts.148.up_proj", + "model.layers.38.mlp.experts.149.down_proj", + "model.layers.38.mlp.experts.149.gate_proj", + "model.layers.38.mlp.experts.149.up_proj", + "model.layers.38.mlp.experts.15.down_proj", + "model.layers.38.mlp.experts.15.gate_proj", + "model.layers.38.mlp.experts.15.up_proj", + "model.layers.38.mlp.experts.150.down_proj", + "model.layers.38.mlp.experts.150.gate_proj", + "model.layers.38.mlp.experts.150.up_proj", + "model.layers.38.mlp.experts.151.down_proj", + "model.layers.38.mlp.experts.151.gate_proj", + "model.layers.38.mlp.experts.151.up_proj", + "model.layers.38.mlp.experts.152.down_proj", + "model.layers.38.mlp.experts.152.gate_proj", + "model.layers.38.mlp.experts.152.up_proj", + "model.layers.38.mlp.experts.153.down_proj", + "model.layers.38.mlp.experts.153.gate_proj", + "model.layers.38.mlp.experts.153.up_proj", + "model.layers.38.mlp.experts.154.down_proj", + "model.layers.38.mlp.experts.154.gate_proj", + "model.layers.38.mlp.experts.154.up_proj", + "model.layers.38.mlp.experts.155.down_proj", + "model.layers.38.mlp.experts.155.gate_proj", + "model.layers.38.mlp.experts.155.up_proj", + "model.layers.38.mlp.experts.156.down_proj", + "model.layers.38.mlp.experts.156.gate_proj", + "model.layers.38.mlp.experts.156.up_proj", + "model.layers.38.mlp.experts.157.down_proj", + "model.layers.38.mlp.experts.157.gate_proj", + "model.layers.38.mlp.experts.157.up_proj", + "model.layers.38.mlp.experts.158.down_proj", + "model.layers.38.mlp.experts.158.gate_proj", + "model.layers.38.mlp.experts.158.up_proj", + "model.layers.38.mlp.experts.159.down_proj", + "model.layers.38.mlp.experts.159.gate_proj", + "model.layers.38.mlp.experts.159.up_proj", + "model.layers.38.mlp.experts.16.down_proj", + "model.layers.38.mlp.experts.16.gate_proj", + "model.layers.38.mlp.experts.16.up_proj", + "model.layers.38.mlp.experts.160.down_proj", + "model.layers.38.mlp.experts.160.gate_proj", + "model.layers.38.mlp.experts.160.up_proj", + "model.layers.38.mlp.experts.161.down_proj", + "model.layers.38.mlp.experts.161.gate_proj", + "model.layers.38.mlp.experts.161.up_proj", + "model.layers.38.mlp.experts.162.down_proj", + "model.layers.38.mlp.experts.162.gate_proj", + "model.layers.38.mlp.experts.162.up_proj", + "model.layers.38.mlp.experts.163.down_proj", + "model.layers.38.mlp.experts.163.gate_proj", + "model.layers.38.mlp.experts.163.up_proj", + "model.layers.38.mlp.experts.164.down_proj", + "model.layers.38.mlp.experts.164.gate_proj", + "model.layers.38.mlp.experts.164.up_proj", + "model.layers.38.mlp.experts.165.down_proj", + "model.layers.38.mlp.experts.165.gate_proj", + "model.layers.38.mlp.experts.165.up_proj", + "model.layers.38.mlp.experts.166.down_proj", + "model.layers.38.mlp.experts.166.gate_proj", + "model.layers.38.mlp.experts.166.up_proj", + "model.layers.38.mlp.experts.167.down_proj", + "model.layers.38.mlp.experts.167.gate_proj", + "model.layers.38.mlp.experts.167.up_proj", + "model.layers.38.mlp.experts.168.down_proj", + "model.layers.38.mlp.experts.168.gate_proj", + "model.layers.38.mlp.experts.168.up_proj", + "model.layers.38.mlp.experts.169.down_proj", + "model.layers.38.mlp.experts.169.gate_proj", + "model.layers.38.mlp.experts.169.up_proj", + "model.layers.38.mlp.experts.17.down_proj", + "model.layers.38.mlp.experts.17.gate_proj", + "model.layers.38.mlp.experts.17.up_proj", + "model.layers.38.mlp.experts.170.down_proj", + "model.layers.38.mlp.experts.170.gate_proj", + "model.layers.38.mlp.experts.170.up_proj", + "model.layers.38.mlp.experts.171.down_proj", + "model.layers.38.mlp.experts.171.gate_proj", + "model.layers.38.mlp.experts.171.up_proj", + "model.layers.38.mlp.experts.172.down_proj", + "model.layers.38.mlp.experts.172.gate_proj", + "model.layers.38.mlp.experts.172.up_proj", + "model.layers.38.mlp.experts.173.down_proj", + "model.layers.38.mlp.experts.173.gate_proj", + "model.layers.38.mlp.experts.173.up_proj", + "model.layers.38.mlp.experts.174.down_proj", + "model.layers.38.mlp.experts.174.gate_proj", + "model.layers.38.mlp.experts.174.up_proj", + "model.layers.38.mlp.experts.175.down_proj", + "model.layers.38.mlp.experts.175.gate_proj", + "model.layers.38.mlp.experts.175.up_proj", + "model.layers.38.mlp.experts.176.down_proj", + "model.layers.38.mlp.experts.176.gate_proj", + "model.layers.38.mlp.experts.176.up_proj", + "model.layers.38.mlp.experts.177.down_proj", + "model.layers.38.mlp.experts.177.gate_proj", + "model.layers.38.mlp.experts.177.up_proj", + "model.layers.38.mlp.experts.178.down_proj", + "model.layers.38.mlp.experts.178.gate_proj", + "model.layers.38.mlp.experts.178.up_proj", + "model.layers.38.mlp.experts.179.down_proj", + "model.layers.38.mlp.experts.179.gate_proj", + "model.layers.38.mlp.experts.179.up_proj", + "model.layers.38.mlp.experts.18.down_proj", + "model.layers.38.mlp.experts.18.gate_proj", + "model.layers.38.mlp.experts.18.up_proj", + "model.layers.38.mlp.experts.180.down_proj", + "model.layers.38.mlp.experts.180.gate_proj", + "model.layers.38.mlp.experts.180.up_proj", + "model.layers.38.mlp.experts.181.down_proj", + "model.layers.38.mlp.experts.181.gate_proj", + "model.layers.38.mlp.experts.181.up_proj", + "model.layers.38.mlp.experts.182.down_proj", + "model.layers.38.mlp.experts.182.gate_proj", + "model.layers.38.mlp.experts.182.up_proj", + "model.layers.38.mlp.experts.183.down_proj", + "model.layers.38.mlp.experts.183.gate_proj", + "model.layers.38.mlp.experts.183.up_proj", + "model.layers.38.mlp.experts.184.down_proj", + "model.layers.38.mlp.experts.184.gate_proj", + "model.layers.38.mlp.experts.184.up_proj", + "model.layers.38.mlp.experts.185.down_proj", + "model.layers.38.mlp.experts.185.gate_proj", + "model.layers.38.mlp.experts.185.up_proj", + "model.layers.38.mlp.experts.186.down_proj", + "model.layers.38.mlp.experts.186.gate_proj", + "model.layers.38.mlp.experts.186.up_proj", + "model.layers.38.mlp.experts.187.down_proj", + "model.layers.38.mlp.experts.187.gate_proj", + "model.layers.38.mlp.experts.187.up_proj", + "model.layers.38.mlp.experts.188.down_proj", + "model.layers.38.mlp.experts.188.gate_proj", + "model.layers.38.mlp.experts.188.up_proj", + "model.layers.38.mlp.experts.189.down_proj", + "model.layers.38.mlp.experts.189.gate_proj", + "model.layers.38.mlp.experts.189.up_proj", + "model.layers.38.mlp.experts.19.down_proj", + "model.layers.38.mlp.experts.19.gate_proj", + "model.layers.38.mlp.experts.19.up_proj", + "model.layers.38.mlp.experts.190.down_proj", + "model.layers.38.mlp.experts.190.gate_proj", + "model.layers.38.mlp.experts.190.up_proj", + "model.layers.38.mlp.experts.191.down_proj", + "model.layers.38.mlp.experts.191.gate_proj", + "model.layers.38.mlp.experts.191.up_proj", + "model.layers.38.mlp.experts.192.down_proj", + "model.layers.38.mlp.experts.192.gate_proj", + "model.layers.38.mlp.experts.192.up_proj", + "model.layers.38.mlp.experts.193.down_proj", + "model.layers.38.mlp.experts.193.gate_proj", + "model.layers.38.mlp.experts.193.up_proj", + "model.layers.38.mlp.experts.194.down_proj", + "model.layers.38.mlp.experts.194.gate_proj", + "model.layers.38.mlp.experts.194.up_proj", + "model.layers.38.mlp.experts.195.down_proj", + "model.layers.38.mlp.experts.195.gate_proj", + "model.layers.38.mlp.experts.195.up_proj", + "model.layers.38.mlp.experts.196.down_proj", + "model.layers.38.mlp.experts.196.gate_proj", + "model.layers.38.mlp.experts.196.up_proj", + "model.layers.38.mlp.experts.197.down_proj", + "model.layers.38.mlp.experts.197.gate_proj", + "model.layers.38.mlp.experts.197.up_proj", + "model.layers.38.mlp.experts.198.down_proj", + "model.layers.38.mlp.experts.198.gate_proj", + "model.layers.38.mlp.experts.198.up_proj", + "model.layers.38.mlp.experts.199.down_proj", + "model.layers.38.mlp.experts.199.gate_proj", + "model.layers.38.mlp.experts.199.up_proj", + "model.layers.38.mlp.experts.2.down_proj", + "model.layers.38.mlp.experts.2.gate_proj", + "model.layers.38.mlp.experts.2.up_proj", + "model.layers.38.mlp.experts.20.down_proj", + "model.layers.38.mlp.experts.20.gate_proj", + "model.layers.38.mlp.experts.20.up_proj", + "model.layers.38.mlp.experts.200.down_proj", + "model.layers.38.mlp.experts.200.gate_proj", + "model.layers.38.mlp.experts.200.up_proj", + "model.layers.38.mlp.experts.201.down_proj", + "model.layers.38.mlp.experts.201.gate_proj", + "model.layers.38.mlp.experts.201.up_proj", + "model.layers.38.mlp.experts.202.down_proj", + "model.layers.38.mlp.experts.202.gate_proj", + "model.layers.38.mlp.experts.202.up_proj", + "model.layers.38.mlp.experts.203.down_proj", + "model.layers.38.mlp.experts.203.gate_proj", + "model.layers.38.mlp.experts.203.up_proj", + "model.layers.38.mlp.experts.204.down_proj", + "model.layers.38.mlp.experts.204.gate_proj", + "model.layers.38.mlp.experts.204.up_proj", + "model.layers.38.mlp.experts.205.down_proj", + "model.layers.38.mlp.experts.205.gate_proj", + "model.layers.38.mlp.experts.205.up_proj", + "model.layers.38.mlp.experts.206.down_proj", + "model.layers.38.mlp.experts.206.gate_proj", + "model.layers.38.mlp.experts.206.up_proj", + "model.layers.38.mlp.experts.207.down_proj", + "model.layers.38.mlp.experts.207.gate_proj", + "model.layers.38.mlp.experts.207.up_proj", + "model.layers.38.mlp.experts.208.down_proj", + "model.layers.38.mlp.experts.208.gate_proj", + "model.layers.38.mlp.experts.208.up_proj", + "model.layers.38.mlp.experts.209.down_proj", + "model.layers.38.mlp.experts.209.gate_proj", + "model.layers.38.mlp.experts.209.up_proj", + "model.layers.38.mlp.experts.21.down_proj", + "model.layers.38.mlp.experts.21.gate_proj", + "model.layers.38.mlp.experts.21.up_proj", + "model.layers.38.mlp.experts.210.down_proj", + "model.layers.38.mlp.experts.210.gate_proj", + "model.layers.38.mlp.experts.210.up_proj", + "model.layers.38.mlp.experts.211.down_proj", + "model.layers.38.mlp.experts.211.gate_proj", + "model.layers.38.mlp.experts.211.up_proj", + "model.layers.38.mlp.experts.212.down_proj", + "model.layers.38.mlp.experts.212.gate_proj", + "model.layers.38.mlp.experts.212.up_proj", + "model.layers.38.mlp.experts.213.down_proj", + "model.layers.38.mlp.experts.213.gate_proj", + "model.layers.38.mlp.experts.213.up_proj", + "model.layers.38.mlp.experts.214.down_proj", + "model.layers.38.mlp.experts.214.gate_proj", + "model.layers.38.mlp.experts.214.up_proj", + "model.layers.38.mlp.experts.215.down_proj", + "model.layers.38.mlp.experts.215.gate_proj", + "model.layers.38.mlp.experts.215.up_proj", + "model.layers.38.mlp.experts.216.down_proj", + "model.layers.38.mlp.experts.216.gate_proj", + "model.layers.38.mlp.experts.216.up_proj", + "model.layers.38.mlp.experts.217.down_proj", + "model.layers.38.mlp.experts.217.gate_proj", + "model.layers.38.mlp.experts.217.up_proj", + "model.layers.38.mlp.experts.218.down_proj", + "model.layers.38.mlp.experts.218.gate_proj", + "model.layers.38.mlp.experts.218.up_proj", + "model.layers.38.mlp.experts.219.down_proj", + "model.layers.38.mlp.experts.219.gate_proj", + "model.layers.38.mlp.experts.219.up_proj", + "model.layers.38.mlp.experts.22.down_proj", + "model.layers.38.mlp.experts.22.gate_proj", + "model.layers.38.mlp.experts.22.up_proj", + "model.layers.38.mlp.experts.220.down_proj", + "model.layers.38.mlp.experts.220.gate_proj", + "model.layers.38.mlp.experts.220.up_proj", + "model.layers.38.mlp.experts.221.down_proj", + "model.layers.38.mlp.experts.221.gate_proj", + "model.layers.38.mlp.experts.221.up_proj", + "model.layers.38.mlp.experts.222.down_proj", + "model.layers.38.mlp.experts.222.gate_proj", + "model.layers.38.mlp.experts.222.up_proj", + "model.layers.38.mlp.experts.223.down_proj", + "model.layers.38.mlp.experts.223.gate_proj", + "model.layers.38.mlp.experts.223.up_proj", + "model.layers.38.mlp.experts.224.down_proj", + "model.layers.38.mlp.experts.224.gate_proj", + "model.layers.38.mlp.experts.224.up_proj", + "model.layers.38.mlp.experts.225.down_proj", + "model.layers.38.mlp.experts.225.gate_proj", + "model.layers.38.mlp.experts.225.up_proj", + "model.layers.38.mlp.experts.226.down_proj", + "model.layers.38.mlp.experts.226.gate_proj", + "model.layers.38.mlp.experts.226.up_proj", + "model.layers.38.mlp.experts.227.down_proj", + "model.layers.38.mlp.experts.227.gate_proj", + "model.layers.38.mlp.experts.227.up_proj", + "model.layers.38.mlp.experts.228.down_proj", + "model.layers.38.mlp.experts.228.gate_proj", + "model.layers.38.mlp.experts.228.up_proj", + "model.layers.38.mlp.experts.229.down_proj", + "model.layers.38.mlp.experts.229.gate_proj", + "model.layers.38.mlp.experts.229.up_proj", + "model.layers.38.mlp.experts.23.down_proj", + "model.layers.38.mlp.experts.23.gate_proj", + "model.layers.38.mlp.experts.23.up_proj", + "model.layers.38.mlp.experts.230.down_proj", + "model.layers.38.mlp.experts.230.gate_proj", + "model.layers.38.mlp.experts.230.up_proj", + "model.layers.38.mlp.experts.231.down_proj", + "model.layers.38.mlp.experts.231.gate_proj", + "model.layers.38.mlp.experts.231.up_proj", + "model.layers.38.mlp.experts.232.down_proj", + "model.layers.38.mlp.experts.232.gate_proj", + "model.layers.38.mlp.experts.232.up_proj", + "model.layers.38.mlp.experts.233.down_proj", + "model.layers.38.mlp.experts.233.gate_proj", + "model.layers.38.mlp.experts.233.up_proj", + "model.layers.38.mlp.experts.234.down_proj", + "model.layers.38.mlp.experts.234.gate_proj", + "model.layers.38.mlp.experts.234.up_proj", + "model.layers.38.mlp.experts.235.down_proj", + "model.layers.38.mlp.experts.235.gate_proj", + "model.layers.38.mlp.experts.235.up_proj", + "model.layers.38.mlp.experts.236.down_proj", + "model.layers.38.mlp.experts.236.gate_proj", + "model.layers.38.mlp.experts.236.up_proj", + "model.layers.38.mlp.experts.237.down_proj", + "model.layers.38.mlp.experts.237.gate_proj", + "model.layers.38.mlp.experts.237.up_proj", + "model.layers.38.mlp.experts.238.down_proj", + "model.layers.38.mlp.experts.238.gate_proj", + "model.layers.38.mlp.experts.238.up_proj", + "model.layers.38.mlp.experts.239.down_proj", + "model.layers.38.mlp.experts.239.gate_proj", + "model.layers.38.mlp.experts.239.up_proj", + "model.layers.38.mlp.experts.24.down_proj", + "model.layers.38.mlp.experts.24.gate_proj", + "model.layers.38.mlp.experts.24.up_proj", + "model.layers.38.mlp.experts.240.down_proj", + "model.layers.38.mlp.experts.240.gate_proj", + "model.layers.38.mlp.experts.240.up_proj", + "model.layers.38.mlp.experts.241.down_proj", + "model.layers.38.mlp.experts.241.gate_proj", + "model.layers.38.mlp.experts.241.up_proj", + "model.layers.38.mlp.experts.242.down_proj", + "model.layers.38.mlp.experts.242.gate_proj", + "model.layers.38.mlp.experts.242.up_proj", + "model.layers.38.mlp.experts.243.down_proj", + "model.layers.38.mlp.experts.243.gate_proj", + "model.layers.38.mlp.experts.243.up_proj", + "model.layers.38.mlp.experts.244.down_proj", + "model.layers.38.mlp.experts.244.gate_proj", + "model.layers.38.mlp.experts.244.up_proj", + "model.layers.38.mlp.experts.245.down_proj", + "model.layers.38.mlp.experts.245.gate_proj", + "model.layers.38.mlp.experts.245.up_proj", + "model.layers.38.mlp.experts.246.down_proj", + "model.layers.38.mlp.experts.246.gate_proj", + "model.layers.38.mlp.experts.246.up_proj", + "model.layers.38.mlp.experts.247.down_proj", + "model.layers.38.mlp.experts.247.gate_proj", + "model.layers.38.mlp.experts.247.up_proj", + "model.layers.38.mlp.experts.248.down_proj", + "model.layers.38.mlp.experts.248.gate_proj", + "model.layers.38.mlp.experts.248.up_proj", + "model.layers.38.mlp.experts.249.down_proj", + "model.layers.38.mlp.experts.249.gate_proj", + "model.layers.38.mlp.experts.249.up_proj", + "model.layers.38.mlp.experts.25.down_proj", + "model.layers.38.mlp.experts.25.gate_proj", + "model.layers.38.mlp.experts.25.up_proj", + "model.layers.38.mlp.experts.250.down_proj", + "model.layers.38.mlp.experts.250.gate_proj", + "model.layers.38.mlp.experts.250.up_proj", + "model.layers.38.mlp.experts.251.down_proj", + "model.layers.38.mlp.experts.251.gate_proj", + "model.layers.38.mlp.experts.251.up_proj", + "model.layers.38.mlp.experts.252.down_proj", + "model.layers.38.mlp.experts.252.gate_proj", + "model.layers.38.mlp.experts.252.up_proj", + "model.layers.38.mlp.experts.253.down_proj", + "model.layers.38.mlp.experts.253.gate_proj", + "model.layers.38.mlp.experts.253.up_proj", + "model.layers.38.mlp.experts.254.down_proj", + "model.layers.38.mlp.experts.254.gate_proj", + "model.layers.38.mlp.experts.254.up_proj", + "model.layers.38.mlp.experts.255.down_proj", + "model.layers.38.mlp.experts.255.gate_proj", + "model.layers.38.mlp.experts.255.up_proj", + "model.layers.38.mlp.experts.26.down_proj", + "model.layers.38.mlp.experts.26.gate_proj", + "model.layers.38.mlp.experts.26.up_proj", + "model.layers.38.mlp.experts.27.down_proj", + "model.layers.38.mlp.experts.27.gate_proj", + "model.layers.38.mlp.experts.27.up_proj", + "model.layers.38.mlp.experts.28.down_proj", + "model.layers.38.mlp.experts.28.gate_proj", + "model.layers.38.mlp.experts.28.up_proj", + "model.layers.38.mlp.experts.29.down_proj", + "model.layers.38.mlp.experts.29.gate_proj", + "model.layers.38.mlp.experts.29.up_proj", + "model.layers.38.mlp.experts.3.down_proj", + "model.layers.38.mlp.experts.3.gate_proj", + "model.layers.38.mlp.experts.3.up_proj", + "model.layers.38.mlp.experts.30.down_proj", + "model.layers.38.mlp.experts.30.gate_proj", + "model.layers.38.mlp.experts.30.up_proj", + "model.layers.38.mlp.experts.31.down_proj", + "model.layers.38.mlp.experts.31.gate_proj", + "model.layers.38.mlp.experts.31.up_proj", + "model.layers.38.mlp.experts.32.down_proj", + "model.layers.38.mlp.experts.32.gate_proj", + "model.layers.38.mlp.experts.32.up_proj", + "model.layers.38.mlp.experts.33.down_proj", + "model.layers.38.mlp.experts.33.gate_proj", + "model.layers.38.mlp.experts.33.up_proj", + "model.layers.38.mlp.experts.34.down_proj", + "model.layers.38.mlp.experts.34.gate_proj", + "model.layers.38.mlp.experts.34.up_proj", + "model.layers.38.mlp.experts.35.down_proj", + "model.layers.38.mlp.experts.35.gate_proj", + "model.layers.38.mlp.experts.35.up_proj", + "model.layers.38.mlp.experts.36.down_proj", + "model.layers.38.mlp.experts.36.gate_proj", + "model.layers.38.mlp.experts.36.up_proj", + "model.layers.38.mlp.experts.37.down_proj", + "model.layers.38.mlp.experts.37.gate_proj", + "model.layers.38.mlp.experts.37.up_proj", + "model.layers.38.mlp.experts.38.down_proj", + "model.layers.38.mlp.experts.38.gate_proj", + "model.layers.38.mlp.experts.38.up_proj", + "model.layers.38.mlp.experts.39.down_proj", + "model.layers.38.mlp.experts.39.gate_proj", + "model.layers.38.mlp.experts.39.up_proj", + "model.layers.38.mlp.experts.4.down_proj", + "model.layers.38.mlp.experts.4.gate_proj", + "model.layers.38.mlp.experts.4.up_proj", + "model.layers.38.mlp.experts.40.down_proj", + "model.layers.38.mlp.experts.40.gate_proj", + "model.layers.38.mlp.experts.40.up_proj", + "model.layers.38.mlp.experts.41.down_proj", + "model.layers.38.mlp.experts.41.gate_proj", + "model.layers.38.mlp.experts.41.up_proj", + "model.layers.38.mlp.experts.42.down_proj", + "model.layers.38.mlp.experts.42.gate_proj", + "model.layers.38.mlp.experts.42.up_proj", + "model.layers.38.mlp.experts.43.down_proj", + "model.layers.38.mlp.experts.43.gate_proj", + "model.layers.38.mlp.experts.43.up_proj", + "model.layers.38.mlp.experts.44.down_proj", + "model.layers.38.mlp.experts.44.gate_proj", + "model.layers.38.mlp.experts.44.up_proj", + "model.layers.38.mlp.experts.45.down_proj", + "model.layers.38.mlp.experts.45.gate_proj", + "model.layers.38.mlp.experts.45.up_proj", + "model.layers.38.mlp.experts.46.down_proj", + "model.layers.38.mlp.experts.46.gate_proj", + "model.layers.38.mlp.experts.46.up_proj", + "model.layers.38.mlp.experts.47.down_proj", + "model.layers.38.mlp.experts.47.gate_proj", + "model.layers.38.mlp.experts.47.up_proj", + "model.layers.38.mlp.experts.48.down_proj", + "model.layers.38.mlp.experts.48.gate_proj", + "model.layers.38.mlp.experts.48.up_proj", + "model.layers.38.mlp.experts.49.down_proj", + "model.layers.38.mlp.experts.49.gate_proj", + "model.layers.38.mlp.experts.49.up_proj", + "model.layers.38.mlp.experts.5.down_proj", + "model.layers.38.mlp.experts.5.gate_proj", + "model.layers.38.mlp.experts.5.up_proj", + "model.layers.38.mlp.experts.50.down_proj", + "model.layers.38.mlp.experts.50.gate_proj", + "model.layers.38.mlp.experts.50.up_proj", + "model.layers.38.mlp.experts.51.down_proj", + "model.layers.38.mlp.experts.51.gate_proj", + "model.layers.38.mlp.experts.51.up_proj", + "model.layers.38.mlp.experts.52.down_proj", + "model.layers.38.mlp.experts.52.gate_proj", + "model.layers.38.mlp.experts.52.up_proj", + "model.layers.38.mlp.experts.53.down_proj", + "model.layers.38.mlp.experts.53.gate_proj", + "model.layers.38.mlp.experts.53.up_proj", + "model.layers.38.mlp.experts.54.down_proj", + "model.layers.38.mlp.experts.54.gate_proj", + "model.layers.38.mlp.experts.54.up_proj", + "model.layers.38.mlp.experts.55.down_proj", + "model.layers.38.mlp.experts.55.gate_proj", + "model.layers.38.mlp.experts.55.up_proj", + "model.layers.38.mlp.experts.56.down_proj", + "model.layers.38.mlp.experts.56.gate_proj", + "model.layers.38.mlp.experts.56.up_proj", + "model.layers.38.mlp.experts.57.down_proj", + "model.layers.38.mlp.experts.57.gate_proj", + "model.layers.38.mlp.experts.57.up_proj", + "model.layers.38.mlp.experts.58.down_proj", + "model.layers.38.mlp.experts.58.gate_proj", + "model.layers.38.mlp.experts.58.up_proj", + "model.layers.38.mlp.experts.59.down_proj", + "model.layers.38.mlp.experts.59.gate_proj", + "model.layers.38.mlp.experts.59.up_proj", + "model.layers.38.mlp.experts.6.down_proj", + "model.layers.38.mlp.experts.6.gate_proj", + "model.layers.38.mlp.experts.6.up_proj", + "model.layers.38.mlp.experts.60.down_proj", + "model.layers.38.mlp.experts.60.gate_proj", + "model.layers.38.mlp.experts.60.up_proj", + "model.layers.38.mlp.experts.61.down_proj", + "model.layers.38.mlp.experts.61.gate_proj", + "model.layers.38.mlp.experts.61.up_proj", + "model.layers.38.mlp.experts.62.down_proj", + "model.layers.38.mlp.experts.62.gate_proj", + "model.layers.38.mlp.experts.62.up_proj", + "model.layers.38.mlp.experts.63.down_proj", + "model.layers.38.mlp.experts.63.gate_proj", + "model.layers.38.mlp.experts.63.up_proj", + "model.layers.38.mlp.experts.64.down_proj", + "model.layers.38.mlp.experts.64.gate_proj", + "model.layers.38.mlp.experts.64.up_proj", + "model.layers.38.mlp.experts.65.down_proj", + "model.layers.38.mlp.experts.65.gate_proj", + "model.layers.38.mlp.experts.65.up_proj", + "model.layers.38.mlp.experts.66.down_proj", + "model.layers.38.mlp.experts.66.gate_proj", + "model.layers.38.mlp.experts.66.up_proj", + "model.layers.38.mlp.experts.67.down_proj", + "model.layers.38.mlp.experts.67.gate_proj", + "model.layers.38.mlp.experts.67.up_proj", + "model.layers.38.mlp.experts.68.down_proj", + "model.layers.38.mlp.experts.68.gate_proj", + "model.layers.38.mlp.experts.68.up_proj", + "model.layers.38.mlp.experts.69.down_proj", + "model.layers.38.mlp.experts.69.gate_proj", + "model.layers.38.mlp.experts.69.up_proj", + "model.layers.38.mlp.experts.7.down_proj", + "model.layers.38.mlp.experts.7.gate_proj", + "model.layers.38.mlp.experts.7.up_proj", + "model.layers.38.mlp.experts.70.down_proj", + "model.layers.38.mlp.experts.70.gate_proj", + "model.layers.38.mlp.experts.70.up_proj", + "model.layers.38.mlp.experts.71.down_proj", + "model.layers.38.mlp.experts.71.gate_proj", + "model.layers.38.mlp.experts.71.up_proj", + "model.layers.38.mlp.experts.72.down_proj", + "model.layers.38.mlp.experts.72.gate_proj", + "model.layers.38.mlp.experts.72.up_proj", + "model.layers.38.mlp.experts.73.down_proj", + "model.layers.38.mlp.experts.73.gate_proj", + "model.layers.38.mlp.experts.73.up_proj", + "model.layers.38.mlp.experts.74.down_proj", + "model.layers.38.mlp.experts.74.gate_proj", + "model.layers.38.mlp.experts.74.up_proj", + "model.layers.38.mlp.experts.75.down_proj", + "model.layers.38.mlp.experts.75.gate_proj", + "model.layers.38.mlp.experts.75.up_proj", + "model.layers.38.mlp.experts.76.down_proj", + "model.layers.38.mlp.experts.76.gate_proj", + "model.layers.38.mlp.experts.76.up_proj", + "model.layers.38.mlp.experts.77.down_proj", + "model.layers.38.mlp.experts.77.gate_proj", + "model.layers.38.mlp.experts.77.up_proj", + "model.layers.38.mlp.experts.78.down_proj", + "model.layers.38.mlp.experts.78.gate_proj", + "model.layers.38.mlp.experts.78.up_proj", + "model.layers.38.mlp.experts.79.down_proj", + "model.layers.38.mlp.experts.79.gate_proj", + "model.layers.38.mlp.experts.79.up_proj", + "model.layers.38.mlp.experts.8.down_proj", + "model.layers.38.mlp.experts.8.gate_proj", + "model.layers.38.mlp.experts.8.up_proj", + "model.layers.38.mlp.experts.80.down_proj", + "model.layers.38.mlp.experts.80.gate_proj", + "model.layers.38.mlp.experts.80.up_proj", + "model.layers.38.mlp.experts.81.down_proj", + "model.layers.38.mlp.experts.81.gate_proj", + "model.layers.38.mlp.experts.81.up_proj", + "model.layers.38.mlp.experts.82.down_proj", + "model.layers.38.mlp.experts.82.gate_proj", + "model.layers.38.mlp.experts.82.up_proj", + "model.layers.38.mlp.experts.83.down_proj", + "model.layers.38.mlp.experts.83.gate_proj", + "model.layers.38.mlp.experts.83.up_proj", + "model.layers.38.mlp.experts.84.down_proj", + "model.layers.38.mlp.experts.84.gate_proj", + "model.layers.38.mlp.experts.84.up_proj", + "model.layers.38.mlp.experts.85.down_proj", + "model.layers.38.mlp.experts.85.gate_proj", + "model.layers.38.mlp.experts.85.up_proj", + "model.layers.38.mlp.experts.86.down_proj", + "model.layers.38.mlp.experts.86.gate_proj", + "model.layers.38.mlp.experts.86.up_proj", + "model.layers.38.mlp.experts.87.down_proj", + "model.layers.38.mlp.experts.87.gate_proj", + "model.layers.38.mlp.experts.87.up_proj", + "model.layers.38.mlp.experts.88.down_proj", + "model.layers.38.mlp.experts.88.gate_proj", + "model.layers.38.mlp.experts.88.up_proj", + "model.layers.38.mlp.experts.89.down_proj", + "model.layers.38.mlp.experts.89.gate_proj", + "model.layers.38.mlp.experts.89.up_proj", + "model.layers.38.mlp.experts.9.down_proj", + "model.layers.38.mlp.experts.9.gate_proj", + "model.layers.38.mlp.experts.9.up_proj", + "model.layers.38.mlp.experts.90.down_proj", + "model.layers.38.mlp.experts.90.gate_proj", + "model.layers.38.mlp.experts.90.up_proj", + "model.layers.38.mlp.experts.91.down_proj", + "model.layers.38.mlp.experts.91.gate_proj", + "model.layers.38.mlp.experts.91.up_proj", + "model.layers.38.mlp.experts.92.down_proj", + "model.layers.38.mlp.experts.92.gate_proj", + "model.layers.38.mlp.experts.92.up_proj", + "model.layers.38.mlp.experts.93.down_proj", + "model.layers.38.mlp.experts.93.gate_proj", + "model.layers.38.mlp.experts.93.up_proj", + "model.layers.38.mlp.experts.94.down_proj", + "model.layers.38.mlp.experts.94.gate_proj", + "model.layers.38.mlp.experts.94.up_proj", + "model.layers.38.mlp.experts.95.down_proj", + "model.layers.38.mlp.experts.95.gate_proj", + "model.layers.38.mlp.experts.95.up_proj", + "model.layers.38.mlp.experts.96.down_proj", + "model.layers.38.mlp.experts.96.gate_proj", + "model.layers.38.mlp.experts.96.up_proj", + "model.layers.38.mlp.experts.97.down_proj", + "model.layers.38.mlp.experts.97.gate_proj", + "model.layers.38.mlp.experts.97.up_proj", + "model.layers.38.mlp.experts.98.down_proj", + "model.layers.38.mlp.experts.98.gate_proj", + "model.layers.38.mlp.experts.98.up_proj", + "model.layers.38.mlp.experts.99.down_proj", + "model.layers.38.mlp.experts.99.gate_proj", + "model.layers.38.mlp.experts.99.up_proj", + "model.layers.39.mlp.experts.0.down_proj", + "model.layers.39.mlp.experts.0.gate_proj", + "model.layers.39.mlp.experts.0.up_proj", + "model.layers.39.mlp.experts.1.down_proj", + "model.layers.39.mlp.experts.1.gate_proj", + "model.layers.39.mlp.experts.1.up_proj", + "model.layers.39.mlp.experts.10.down_proj", + "model.layers.39.mlp.experts.10.gate_proj", + "model.layers.39.mlp.experts.10.up_proj", + "model.layers.39.mlp.experts.100.down_proj", + "model.layers.39.mlp.experts.100.gate_proj", + "model.layers.39.mlp.experts.100.up_proj", + "model.layers.39.mlp.experts.101.down_proj", + "model.layers.39.mlp.experts.101.gate_proj", + "model.layers.39.mlp.experts.101.up_proj", + "model.layers.39.mlp.experts.102.down_proj", + "model.layers.39.mlp.experts.102.gate_proj", + "model.layers.39.mlp.experts.102.up_proj", + "model.layers.39.mlp.experts.103.down_proj", + "model.layers.39.mlp.experts.103.gate_proj", + "model.layers.39.mlp.experts.103.up_proj", + "model.layers.39.mlp.experts.104.down_proj", + "model.layers.39.mlp.experts.104.gate_proj", + "model.layers.39.mlp.experts.104.up_proj", + "model.layers.39.mlp.experts.105.down_proj", + "model.layers.39.mlp.experts.105.gate_proj", + "model.layers.39.mlp.experts.105.up_proj", + "model.layers.39.mlp.experts.106.down_proj", + "model.layers.39.mlp.experts.106.gate_proj", + "model.layers.39.mlp.experts.106.up_proj", + "model.layers.39.mlp.experts.107.down_proj", + "model.layers.39.mlp.experts.107.gate_proj", + "model.layers.39.mlp.experts.107.up_proj", + "model.layers.39.mlp.experts.108.down_proj", + "model.layers.39.mlp.experts.108.gate_proj", + "model.layers.39.mlp.experts.108.up_proj", + "model.layers.39.mlp.experts.109.down_proj", + "model.layers.39.mlp.experts.109.gate_proj", + "model.layers.39.mlp.experts.109.up_proj", + "model.layers.39.mlp.experts.11.down_proj", + "model.layers.39.mlp.experts.11.gate_proj", + "model.layers.39.mlp.experts.11.up_proj", + "model.layers.39.mlp.experts.110.down_proj", + "model.layers.39.mlp.experts.110.gate_proj", + "model.layers.39.mlp.experts.110.up_proj", + "model.layers.39.mlp.experts.111.down_proj", + "model.layers.39.mlp.experts.111.gate_proj", + "model.layers.39.mlp.experts.111.up_proj", + "model.layers.39.mlp.experts.112.down_proj", + "model.layers.39.mlp.experts.112.gate_proj", + "model.layers.39.mlp.experts.112.up_proj", + "model.layers.39.mlp.experts.113.down_proj", + "model.layers.39.mlp.experts.113.gate_proj", + "model.layers.39.mlp.experts.113.up_proj", + "model.layers.39.mlp.experts.114.down_proj", + "model.layers.39.mlp.experts.114.gate_proj", + "model.layers.39.mlp.experts.114.up_proj", + "model.layers.39.mlp.experts.115.down_proj", + "model.layers.39.mlp.experts.115.gate_proj", + "model.layers.39.mlp.experts.115.up_proj", + "model.layers.39.mlp.experts.116.down_proj", + "model.layers.39.mlp.experts.116.gate_proj", + "model.layers.39.mlp.experts.116.up_proj", + "model.layers.39.mlp.experts.117.down_proj", + "model.layers.39.mlp.experts.117.gate_proj", + "model.layers.39.mlp.experts.117.up_proj", + "model.layers.39.mlp.experts.118.down_proj", + "model.layers.39.mlp.experts.118.gate_proj", + "model.layers.39.mlp.experts.118.up_proj", + "model.layers.39.mlp.experts.119.down_proj", + "model.layers.39.mlp.experts.119.gate_proj", + "model.layers.39.mlp.experts.119.up_proj", + "model.layers.39.mlp.experts.12.down_proj", + "model.layers.39.mlp.experts.12.gate_proj", + "model.layers.39.mlp.experts.12.up_proj", + "model.layers.39.mlp.experts.120.down_proj", + "model.layers.39.mlp.experts.120.gate_proj", + "model.layers.39.mlp.experts.120.up_proj", + "model.layers.39.mlp.experts.121.down_proj", + "model.layers.39.mlp.experts.121.gate_proj", + "model.layers.39.mlp.experts.121.up_proj", + "model.layers.39.mlp.experts.122.down_proj", + "model.layers.39.mlp.experts.122.gate_proj", + "model.layers.39.mlp.experts.122.up_proj", + "model.layers.39.mlp.experts.123.down_proj", + "model.layers.39.mlp.experts.123.gate_proj", + "model.layers.39.mlp.experts.123.up_proj", + "model.layers.39.mlp.experts.124.down_proj", + "model.layers.39.mlp.experts.124.gate_proj", + "model.layers.39.mlp.experts.124.up_proj", + "model.layers.39.mlp.experts.125.down_proj", + "model.layers.39.mlp.experts.125.gate_proj", + "model.layers.39.mlp.experts.125.up_proj", + "model.layers.39.mlp.experts.126.down_proj", + "model.layers.39.mlp.experts.126.gate_proj", + "model.layers.39.mlp.experts.126.up_proj", + "model.layers.39.mlp.experts.127.down_proj", + "model.layers.39.mlp.experts.127.gate_proj", + "model.layers.39.mlp.experts.127.up_proj", + "model.layers.39.mlp.experts.128.down_proj", + "model.layers.39.mlp.experts.128.gate_proj", + "model.layers.39.mlp.experts.128.up_proj", + "model.layers.39.mlp.experts.129.down_proj", + "model.layers.39.mlp.experts.129.gate_proj", + "model.layers.39.mlp.experts.129.up_proj", + "model.layers.39.mlp.experts.13.down_proj", + "model.layers.39.mlp.experts.13.gate_proj", + "model.layers.39.mlp.experts.13.up_proj", + "model.layers.39.mlp.experts.130.down_proj", + "model.layers.39.mlp.experts.130.gate_proj", + "model.layers.39.mlp.experts.130.up_proj", + "model.layers.39.mlp.experts.131.down_proj", + "model.layers.39.mlp.experts.131.gate_proj", + "model.layers.39.mlp.experts.131.up_proj", + "model.layers.39.mlp.experts.132.down_proj", + "model.layers.39.mlp.experts.132.gate_proj", + "model.layers.39.mlp.experts.132.up_proj", + "model.layers.39.mlp.experts.133.down_proj", + "model.layers.39.mlp.experts.133.gate_proj", + "model.layers.39.mlp.experts.133.up_proj", + "model.layers.39.mlp.experts.134.down_proj", + "model.layers.39.mlp.experts.134.gate_proj", + "model.layers.39.mlp.experts.134.up_proj", + "model.layers.39.mlp.experts.135.down_proj", + "model.layers.39.mlp.experts.135.gate_proj", + "model.layers.39.mlp.experts.135.up_proj", + "model.layers.39.mlp.experts.136.down_proj", + "model.layers.39.mlp.experts.136.gate_proj", + "model.layers.39.mlp.experts.136.up_proj", + "model.layers.39.mlp.experts.137.down_proj", + "model.layers.39.mlp.experts.137.gate_proj", + "model.layers.39.mlp.experts.137.up_proj", + "model.layers.39.mlp.experts.138.down_proj", + "model.layers.39.mlp.experts.138.gate_proj", + "model.layers.39.mlp.experts.138.up_proj", + "model.layers.39.mlp.experts.139.down_proj", + "model.layers.39.mlp.experts.139.gate_proj", + "model.layers.39.mlp.experts.139.up_proj", + "model.layers.39.mlp.experts.14.down_proj", + "model.layers.39.mlp.experts.14.gate_proj", + "model.layers.39.mlp.experts.14.up_proj", + "model.layers.39.mlp.experts.140.down_proj", + "model.layers.39.mlp.experts.140.gate_proj", + "model.layers.39.mlp.experts.140.up_proj", + "model.layers.39.mlp.experts.141.down_proj", + "model.layers.39.mlp.experts.141.gate_proj", + "model.layers.39.mlp.experts.141.up_proj", + "model.layers.39.mlp.experts.142.down_proj", + "model.layers.39.mlp.experts.142.gate_proj", + "model.layers.39.mlp.experts.142.up_proj", + "model.layers.39.mlp.experts.143.down_proj", + "model.layers.39.mlp.experts.143.gate_proj", + "model.layers.39.mlp.experts.143.up_proj", + "model.layers.39.mlp.experts.144.down_proj", + "model.layers.39.mlp.experts.144.gate_proj", + "model.layers.39.mlp.experts.144.up_proj", + "model.layers.39.mlp.experts.145.down_proj", + "model.layers.39.mlp.experts.145.gate_proj", + "model.layers.39.mlp.experts.145.up_proj", + "model.layers.39.mlp.experts.146.down_proj", + "model.layers.39.mlp.experts.146.gate_proj", + "model.layers.39.mlp.experts.146.up_proj", + "model.layers.39.mlp.experts.147.down_proj", + "model.layers.39.mlp.experts.147.gate_proj", + "model.layers.39.mlp.experts.147.up_proj", + "model.layers.39.mlp.experts.148.down_proj", + "model.layers.39.mlp.experts.148.gate_proj", + "model.layers.39.mlp.experts.148.up_proj", + "model.layers.39.mlp.experts.149.down_proj", + "model.layers.39.mlp.experts.149.gate_proj", + "model.layers.39.mlp.experts.149.up_proj", + "model.layers.39.mlp.experts.15.down_proj", + "model.layers.39.mlp.experts.15.gate_proj", + "model.layers.39.mlp.experts.15.up_proj", + "model.layers.39.mlp.experts.150.down_proj", + "model.layers.39.mlp.experts.150.gate_proj", + "model.layers.39.mlp.experts.150.up_proj", + "model.layers.39.mlp.experts.151.down_proj", + "model.layers.39.mlp.experts.151.gate_proj", + "model.layers.39.mlp.experts.151.up_proj", + "model.layers.39.mlp.experts.152.down_proj", + "model.layers.39.mlp.experts.152.gate_proj", + "model.layers.39.mlp.experts.152.up_proj", + "model.layers.39.mlp.experts.153.down_proj", + "model.layers.39.mlp.experts.153.gate_proj", + "model.layers.39.mlp.experts.153.up_proj", + "model.layers.39.mlp.experts.154.down_proj", + "model.layers.39.mlp.experts.154.gate_proj", + "model.layers.39.mlp.experts.154.up_proj", + "model.layers.39.mlp.experts.155.down_proj", + "model.layers.39.mlp.experts.155.gate_proj", + "model.layers.39.mlp.experts.155.up_proj", + "model.layers.39.mlp.experts.156.down_proj", + "model.layers.39.mlp.experts.156.gate_proj", + "model.layers.39.mlp.experts.156.up_proj", + "model.layers.39.mlp.experts.157.down_proj", + "model.layers.39.mlp.experts.157.gate_proj", + "model.layers.39.mlp.experts.157.up_proj", + "model.layers.39.mlp.experts.158.down_proj", + "model.layers.39.mlp.experts.158.gate_proj", + "model.layers.39.mlp.experts.158.up_proj", + "model.layers.39.mlp.experts.159.down_proj", + "model.layers.39.mlp.experts.159.gate_proj", + "model.layers.39.mlp.experts.159.up_proj", + "model.layers.39.mlp.experts.16.down_proj", + "model.layers.39.mlp.experts.16.gate_proj", + "model.layers.39.mlp.experts.16.up_proj", + "model.layers.39.mlp.experts.160.down_proj", + "model.layers.39.mlp.experts.160.gate_proj", + "model.layers.39.mlp.experts.160.up_proj", + "model.layers.39.mlp.experts.161.down_proj", + "model.layers.39.mlp.experts.161.gate_proj", + "model.layers.39.mlp.experts.161.up_proj", + "model.layers.39.mlp.experts.162.down_proj", + "model.layers.39.mlp.experts.162.gate_proj", + "model.layers.39.mlp.experts.162.up_proj", + "model.layers.39.mlp.experts.163.down_proj", + "model.layers.39.mlp.experts.163.gate_proj", + "model.layers.39.mlp.experts.163.up_proj", + "model.layers.39.mlp.experts.164.down_proj", + "model.layers.39.mlp.experts.164.gate_proj", + "model.layers.39.mlp.experts.164.up_proj", + "model.layers.39.mlp.experts.165.down_proj", + "model.layers.39.mlp.experts.165.gate_proj", + "model.layers.39.mlp.experts.165.up_proj", + "model.layers.39.mlp.experts.166.down_proj", + "model.layers.39.mlp.experts.166.gate_proj", + "model.layers.39.mlp.experts.166.up_proj", + "model.layers.39.mlp.experts.167.down_proj", + "model.layers.39.mlp.experts.167.gate_proj", + "model.layers.39.mlp.experts.167.up_proj", + "model.layers.39.mlp.experts.168.down_proj", + "model.layers.39.mlp.experts.168.gate_proj", + "model.layers.39.mlp.experts.168.up_proj", + "model.layers.39.mlp.experts.169.down_proj", + "model.layers.39.mlp.experts.169.gate_proj", + "model.layers.39.mlp.experts.169.up_proj", + "model.layers.39.mlp.experts.17.down_proj", + "model.layers.39.mlp.experts.17.gate_proj", + "model.layers.39.mlp.experts.17.up_proj", + "model.layers.39.mlp.experts.170.down_proj", + "model.layers.39.mlp.experts.170.gate_proj", + "model.layers.39.mlp.experts.170.up_proj", + "model.layers.39.mlp.experts.171.down_proj", + "model.layers.39.mlp.experts.171.gate_proj", + "model.layers.39.mlp.experts.171.up_proj", + "model.layers.39.mlp.experts.172.down_proj", + "model.layers.39.mlp.experts.172.gate_proj", + "model.layers.39.mlp.experts.172.up_proj", + "model.layers.39.mlp.experts.173.down_proj", + "model.layers.39.mlp.experts.173.gate_proj", + "model.layers.39.mlp.experts.173.up_proj", + "model.layers.39.mlp.experts.174.down_proj", + "model.layers.39.mlp.experts.174.gate_proj", + "model.layers.39.mlp.experts.174.up_proj", + "model.layers.39.mlp.experts.175.down_proj", + "model.layers.39.mlp.experts.175.gate_proj", + "model.layers.39.mlp.experts.175.up_proj", + "model.layers.39.mlp.experts.176.down_proj", + "model.layers.39.mlp.experts.176.gate_proj", + "model.layers.39.mlp.experts.176.up_proj", + "model.layers.39.mlp.experts.177.down_proj", + "model.layers.39.mlp.experts.177.gate_proj", + "model.layers.39.mlp.experts.177.up_proj", + "model.layers.39.mlp.experts.178.down_proj", + "model.layers.39.mlp.experts.178.gate_proj", + "model.layers.39.mlp.experts.178.up_proj", + "model.layers.39.mlp.experts.179.down_proj", + "model.layers.39.mlp.experts.179.gate_proj", + "model.layers.39.mlp.experts.179.up_proj", + "model.layers.39.mlp.experts.18.down_proj", + "model.layers.39.mlp.experts.18.gate_proj", + "model.layers.39.mlp.experts.18.up_proj", + "model.layers.39.mlp.experts.180.down_proj", + "model.layers.39.mlp.experts.180.gate_proj", + "model.layers.39.mlp.experts.180.up_proj", + "model.layers.39.mlp.experts.181.down_proj", + "model.layers.39.mlp.experts.181.gate_proj", + "model.layers.39.mlp.experts.181.up_proj", + "model.layers.39.mlp.experts.182.down_proj", + "model.layers.39.mlp.experts.182.gate_proj", + "model.layers.39.mlp.experts.182.up_proj", + "model.layers.39.mlp.experts.183.down_proj", + "model.layers.39.mlp.experts.183.gate_proj", + "model.layers.39.mlp.experts.183.up_proj", + "model.layers.39.mlp.experts.184.down_proj", + "model.layers.39.mlp.experts.184.gate_proj", + "model.layers.39.mlp.experts.184.up_proj", + "model.layers.39.mlp.experts.185.down_proj", + "model.layers.39.mlp.experts.185.gate_proj", + "model.layers.39.mlp.experts.185.up_proj", + "model.layers.39.mlp.experts.186.down_proj", + "model.layers.39.mlp.experts.186.gate_proj", + "model.layers.39.mlp.experts.186.up_proj", + "model.layers.39.mlp.experts.187.down_proj", + "model.layers.39.mlp.experts.187.gate_proj", + "model.layers.39.mlp.experts.187.up_proj", + "model.layers.39.mlp.experts.188.down_proj", + "model.layers.39.mlp.experts.188.gate_proj", + "model.layers.39.mlp.experts.188.up_proj", + "model.layers.39.mlp.experts.189.down_proj", + "model.layers.39.mlp.experts.189.gate_proj", + "model.layers.39.mlp.experts.189.up_proj", + "model.layers.39.mlp.experts.19.down_proj", + "model.layers.39.mlp.experts.19.gate_proj", + "model.layers.39.mlp.experts.19.up_proj", + "model.layers.39.mlp.experts.190.down_proj", + "model.layers.39.mlp.experts.190.gate_proj", + "model.layers.39.mlp.experts.190.up_proj", + "model.layers.39.mlp.experts.191.down_proj", + "model.layers.39.mlp.experts.191.gate_proj", + "model.layers.39.mlp.experts.191.up_proj", + "model.layers.39.mlp.experts.192.down_proj", + "model.layers.39.mlp.experts.192.gate_proj", + "model.layers.39.mlp.experts.192.up_proj", + "model.layers.39.mlp.experts.193.down_proj", + "model.layers.39.mlp.experts.193.gate_proj", + "model.layers.39.mlp.experts.193.up_proj", + "model.layers.39.mlp.experts.194.down_proj", + "model.layers.39.mlp.experts.194.gate_proj", + "model.layers.39.mlp.experts.194.up_proj", + "model.layers.39.mlp.experts.195.down_proj", + "model.layers.39.mlp.experts.195.gate_proj", + "model.layers.39.mlp.experts.195.up_proj", + "model.layers.39.mlp.experts.196.down_proj", + "model.layers.39.mlp.experts.196.gate_proj", + "model.layers.39.mlp.experts.196.up_proj", + "model.layers.39.mlp.experts.197.down_proj", + "model.layers.39.mlp.experts.197.gate_proj", + "model.layers.39.mlp.experts.197.up_proj", + "model.layers.39.mlp.experts.198.down_proj", + "model.layers.39.mlp.experts.198.gate_proj", + "model.layers.39.mlp.experts.198.up_proj", + "model.layers.39.mlp.experts.199.down_proj", + "model.layers.39.mlp.experts.199.gate_proj", + "model.layers.39.mlp.experts.199.up_proj", + "model.layers.39.mlp.experts.2.down_proj", + "model.layers.39.mlp.experts.2.gate_proj", + "model.layers.39.mlp.experts.2.up_proj", + "model.layers.39.mlp.experts.20.down_proj", + "model.layers.39.mlp.experts.20.gate_proj", + "model.layers.39.mlp.experts.20.up_proj", + "model.layers.39.mlp.experts.200.down_proj", + "model.layers.39.mlp.experts.200.gate_proj", + "model.layers.39.mlp.experts.200.up_proj", + "model.layers.39.mlp.experts.201.down_proj", + "model.layers.39.mlp.experts.201.gate_proj", + "model.layers.39.mlp.experts.201.up_proj", + "model.layers.39.mlp.experts.202.down_proj", + "model.layers.39.mlp.experts.202.gate_proj", + "model.layers.39.mlp.experts.202.up_proj", + "model.layers.39.mlp.experts.203.down_proj", + "model.layers.39.mlp.experts.203.gate_proj", + "model.layers.39.mlp.experts.203.up_proj", + "model.layers.39.mlp.experts.204.down_proj", + "model.layers.39.mlp.experts.204.gate_proj", + "model.layers.39.mlp.experts.204.up_proj", + "model.layers.39.mlp.experts.205.down_proj", + "model.layers.39.mlp.experts.205.gate_proj", + "model.layers.39.mlp.experts.205.up_proj", + "model.layers.39.mlp.experts.206.down_proj", + "model.layers.39.mlp.experts.206.gate_proj", + "model.layers.39.mlp.experts.206.up_proj", + "model.layers.39.mlp.experts.207.down_proj", + "model.layers.39.mlp.experts.207.gate_proj", + "model.layers.39.mlp.experts.207.up_proj", + "model.layers.39.mlp.experts.208.down_proj", + "model.layers.39.mlp.experts.208.gate_proj", + "model.layers.39.mlp.experts.208.up_proj", + "model.layers.39.mlp.experts.209.down_proj", + "model.layers.39.mlp.experts.209.gate_proj", + "model.layers.39.mlp.experts.209.up_proj", + "model.layers.39.mlp.experts.21.down_proj", + "model.layers.39.mlp.experts.21.gate_proj", + "model.layers.39.mlp.experts.21.up_proj", + "model.layers.39.mlp.experts.210.down_proj", + "model.layers.39.mlp.experts.210.gate_proj", + "model.layers.39.mlp.experts.210.up_proj", + "model.layers.39.mlp.experts.211.down_proj", + "model.layers.39.mlp.experts.211.gate_proj", + "model.layers.39.mlp.experts.211.up_proj", + "model.layers.39.mlp.experts.212.down_proj", + "model.layers.39.mlp.experts.212.gate_proj", + "model.layers.39.mlp.experts.212.up_proj", + "model.layers.39.mlp.experts.213.down_proj", + "model.layers.39.mlp.experts.213.gate_proj", + "model.layers.39.mlp.experts.213.up_proj", + "model.layers.39.mlp.experts.214.down_proj", + "model.layers.39.mlp.experts.214.gate_proj", + "model.layers.39.mlp.experts.214.up_proj", + "model.layers.39.mlp.experts.215.down_proj", + "model.layers.39.mlp.experts.215.gate_proj", + "model.layers.39.mlp.experts.215.up_proj", + "model.layers.39.mlp.experts.216.down_proj", + "model.layers.39.mlp.experts.216.gate_proj", + "model.layers.39.mlp.experts.216.up_proj", + "model.layers.39.mlp.experts.217.down_proj", + "model.layers.39.mlp.experts.217.gate_proj", + "model.layers.39.mlp.experts.217.up_proj", + "model.layers.39.mlp.experts.218.down_proj", + "model.layers.39.mlp.experts.218.gate_proj", + "model.layers.39.mlp.experts.218.up_proj", + "model.layers.39.mlp.experts.219.down_proj", + "model.layers.39.mlp.experts.219.gate_proj", + "model.layers.39.mlp.experts.219.up_proj", + "model.layers.39.mlp.experts.22.down_proj", + "model.layers.39.mlp.experts.22.gate_proj", + "model.layers.39.mlp.experts.22.up_proj", + "model.layers.39.mlp.experts.220.down_proj", + "model.layers.39.mlp.experts.220.gate_proj", + "model.layers.39.mlp.experts.220.up_proj", + "model.layers.39.mlp.experts.221.down_proj", + "model.layers.39.mlp.experts.221.gate_proj", + "model.layers.39.mlp.experts.221.up_proj", + "model.layers.39.mlp.experts.222.down_proj", + "model.layers.39.mlp.experts.222.gate_proj", + "model.layers.39.mlp.experts.222.up_proj", + "model.layers.39.mlp.experts.223.down_proj", + "model.layers.39.mlp.experts.223.gate_proj", + "model.layers.39.mlp.experts.223.up_proj", + "model.layers.39.mlp.experts.224.down_proj", + "model.layers.39.mlp.experts.224.gate_proj", + "model.layers.39.mlp.experts.224.up_proj", + "model.layers.39.mlp.experts.225.down_proj", + "model.layers.39.mlp.experts.225.gate_proj", + "model.layers.39.mlp.experts.225.up_proj", + "model.layers.39.mlp.experts.226.down_proj", + "model.layers.39.mlp.experts.226.gate_proj", + "model.layers.39.mlp.experts.226.up_proj", + "model.layers.39.mlp.experts.227.down_proj", + "model.layers.39.mlp.experts.227.gate_proj", + "model.layers.39.mlp.experts.227.up_proj", + "model.layers.39.mlp.experts.228.down_proj", + "model.layers.39.mlp.experts.228.gate_proj", + "model.layers.39.mlp.experts.228.up_proj", + "model.layers.39.mlp.experts.229.down_proj", + "model.layers.39.mlp.experts.229.gate_proj", + "model.layers.39.mlp.experts.229.up_proj", + "model.layers.39.mlp.experts.23.down_proj", + "model.layers.39.mlp.experts.23.gate_proj", + "model.layers.39.mlp.experts.23.up_proj", + "model.layers.39.mlp.experts.230.down_proj", + "model.layers.39.mlp.experts.230.gate_proj", + "model.layers.39.mlp.experts.230.up_proj", + "model.layers.39.mlp.experts.231.down_proj", + "model.layers.39.mlp.experts.231.gate_proj", + "model.layers.39.mlp.experts.231.up_proj", + "model.layers.39.mlp.experts.232.down_proj", + "model.layers.39.mlp.experts.232.gate_proj", + "model.layers.39.mlp.experts.232.up_proj", + "model.layers.39.mlp.experts.233.down_proj", + "model.layers.39.mlp.experts.233.gate_proj", + "model.layers.39.mlp.experts.233.up_proj", + "model.layers.39.mlp.experts.234.down_proj", + "model.layers.39.mlp.experts.234.gate_proj", + "model.layers.39.mlp.experts.234.up_proj", + "model.layers.39.mlp.experts.235.down_proj", + "model.layers.39.mlp.experts.235.gate_proj", + "model.layers.39.mlp.experts.235.up_proj", + "model.layers.39.mlp.experts.236.down_proj", + "model.layers.39.mlp.experts.236.gate_proj", + "model.layers.39.mlp.experts.236.up_proj", + "model.layers.39.mlp.experts.237.down_proj", + "model.layers.39.mlp.experts.237.gate_proj", + "model.layers.39.mlp.experts.237.up_proj", + "model.layers.39.mlp.experts.238.down_proj", + "model.layers.39.mlp.experts.238.gate_proj", + "model.layers.39.mlp.experts.238.up_proj", + "model.layers.39.mlp.experts.239.down_proj", + "model.layers.39.mlp.experts.239.gate_proj", + "model.layers.39.mlp.experts.239.up_proj", + "model.layers.39.mlp.experts.24.down_proj", + "model.layers.39.mlp.experts.24.gate_proj", + "model.layers.39.mlp.experts.24.up_proj", + "model.layers.39.mlp.experts.240.down_proj", + "model.layers.39.mlp.experts.240.gate_proj", + "model.layers.39.mlp.experts.240.up_proj", + "model.layers.39.mlp.experts.241.down_proj", + "model.layers.39.mlp.experts.241.gate_proj", + "model.layers.39.mlp.experts.241.up_proj", + "model.layers.39.mlp.experts.242.down_proj", + "model.layers.39.mlp.experts.242.gate_proj", + "model.layers.39.mlp.experts.242.up_proj", + "model.layers.39.mlp.experts.243.down_proj", + "model.layers.39.mlp.experts.243.gate_proj", + "model.layers.39.mlp.experts.243.up_proj", + "model.layers.39.mlp.experts.244.down_proj", + "model.layers.39.mlp.experts.244.gate_proj", + "model.layers.39.mlp.experts.244.up_proj", + "model.layers.39.mlp.experts.245.down_proj", + "model.layers.39.mlp.experts.245.gate_proj", + "model.layers.39.mlp.experts.245.up_proj", + "model.layers.39.mlp.experts.246.down_proj", + "model.layers.39.mlp.experts.246.gate_proj", + "model.layers.39.mlp.experts.246.up_proj", + "model.layers.39.mlp.experts.247.down_proj", + "model.layers.39.mlp.experts.247.gate_proj", + "model.layers.39.mlp.experts.247.up_proj", + "model.layers.39.mlp.experts.248.down_proj", + "model.layers.39.mlp.experts.248.gate_proj", + "model.layers.39.mlp.experts.248.up_proj", + "model.layers.39.mlp.experts.249.down_proj", + "model.layers.39.mlp.experts.249.gate_proj", + "model.layers.39.mlp.experts.249.up_proj", + "model.layers.39.mlp.experts.25.down_proj", + "model.layers.39.mlp.experts.25.gate_proj", + "model.layers.39.mlp.experts.25.up_proj", + "model.layers.39.mlp.experts.250.down_proj", + "model.layers.39.mlp.experts.250.gate_proj", + "model.layers.39.mlp.experts.250.up_proj", + "model.layers.39.mlp.experts.251.down_proj", + "model.layers.39.mlp.experts.251.gate_proj", + "model.layers.39.mlp.experts.251.up_proj", + "model.layers.39.mlp.experts.252.down_proj", + "model.layers.39.mlp.experts.252.gate_proj", + "model.layers.39.mlp.experts.252.up_proj", + "model.layers.39.mlp.experts.253.down_proj", + "model.layers.39.mlp.experts.253.gate_proj", + "model.layers.39.mlp.experts.253.up_proj", + "model.layers.39.mlp.experts.254.down_proj", + "model.layers.39.mlp.experts.254.gate_proj", + "model.layers.39.mlp.experts.254.up_proj", + "model.layers.39.mlp.experts.255.down_proj", + "model.layers.39.mlp.experts.255.gate_proj", + "model.layers.39.mlp.experts.255.up_proj", + "model.layers.39.mlp.experts.26.down_proj", + "model.layers.39.mlp.experts.26.gate_proj", + "model.layers.39.mlp.experts.26.up_proj", + "model.layers.39.mlp.experts.27.down_proj", + "model.layers.39.mlp.experts.27.gate_proj", + "model.layers.39.mlp.experts.27.up_proj", + "model.layers.39.mlp.experts.28.down_proj", + "model.layers.39.mlp.experts.28.gate_proj", + "model.layers.39.mlp.experts.28.up_proj", + "model.layers.39.mlp.experts.29.down_proj", + "model.layers.39.mlp.experts.29.gate_proj", + "model.layers.39.mlp.experts.29.up_proj", + "model.layers.39.mlp.experts.3.down_proj", + "model.layers.39.mlp.experts.3.gate_proj", + "model.layers.39.mlp.experts.3.up_proj", + "model.layers.39.mlp.experts.30.down_proj", + "model.layers.39.mlp.experts.30.gate_proj", + "model.layers.39.mlp.experts.30.up_proj", + "model.layers.39.mlp.experts.31.down_proj", + "model.layers.39.mlp.experts.31.gate_proj", + "model.layers.39.mlp.experts.31.up_proj", + "model.layers.39.mlp.experts.32.down_proj", + "model.layers.39.mlp.experts.32.gate_proj", + "model.layers.39.mlp.experts.32.up_proj", + "model.layers.39.mlp.experts.33.down_proj", + "model.layers.39.mlp.experts.33.gate_proj", + "model.layers.39.mlp.experts.33.up_proj", + "model.layers.39.mlp.experts.34.down_proj", + "model.layers.39.mlp.experts.34.gate_proj", + "model.layers.39.mlp.experts.34.up_proj", + "model.layers.39.mlp.experts.35.down_proj", + "model.layers.39.mlp.experts.35.gate_proj", + "model.layers.39.mlp.experts.35.up_proj", + "model.layers.39.mlp.experts.36.down_proj", + "model.layers.39.mlp.experts.36.gate_proj", + "model.layers.39.mlp.experts.36.up_proj", + "model.layers.39.mlp.experts.37.down_proj", + "model.layers.39.mlp.experts.37.gate_proj", + "model.layers.39.mlp.experts.37.up_proj", + "model.layers.39.mlp.experts.38.down_proj", + "model.layers.39.mlp.experts.38.gate_proj", + "model.layers.39.mlp.experts.38.up_proj", + "model.layers.39.mlp.experts.39.down_proj", + "model.layers.39.mlp.experts.39.gate_proj", + "model.layers.39.mlp.experts.39.up_proj", + "model.layers.39.mlp.experts.4.down_proj", + "model.layers.39.mlp.experts.4.gate_proj", + "model.layers.39.mlp.experts.4.up_proj", + "model.layers.39.mlp.experts.40.down_proj", + "model.layers.39.mlp.experts.40.gate_proj", + "model.layers.39.mlp.experts.40.up_proj", + "model.layers.39.mlp.experts.41.down_proj", + "model.layers.39.mlp.experts.41.gate_proj", + "model.layers.39.mlp.experts.41.up_proj", + "model.layers.39.mlp.experts.42.down_proj", + "model.layers.39.mlp.experts.42.gate_proj", + "model.layers.39.mlp.experts.42.up_proj", + "model.layers.39.mlp.experts.43.down_proj", + "model.layers.39.mlp.experts.43.gate_proj", + "model.layers.39.mlp.experts.43.up_proj", + "model.layers.39.mlp.experts.44.down_proj", + "model.layers.39.mlp.experts.44.gate_proj", + "model.layers.39.mlp.experts.44.up_proj", + "model.layers.39.mlp.experts.45.down_proj", + "model.layers.39.mlp.experts.45.gate_proj", + "model.layers.39.mlp.experts.45.up_proj", + "model.layers.39.mlp.experts.46.down_proj", + "model.layers.39.mlp.experts.46.gate_proj", + "model.layers.39.mlp.experts.46.up_proj", + "model.layers.39.mlp.experts.47.down_proj", + "model.layers.39.mlp.experts.47.gate_proj", + "model.layers.39.mlp.experts.47.up_proj", + "model.layers.39.mlp.experts.48.down_proj", + "model.layers.39.mlp.experts.48.gate_proj", + "model.layers.39.mlp.experts.48.up_proj", + "model.layers.39.mlp.experts.49.down_proj", + "model.layers.39.mlp.experts.49.gate_proj", + "model.layers.39.mlp.experts.49.up_proj", + "model.layers.39.mlp.experts.5.down_proj", + "model.layers.39.mlp.experts.5.gate_proj", + "model.layers.39.mlp.experts.5.up_proj", + "model.layers.39.mlp.experts.50.down_proj", + "model.layers.39.mlp.experts.50.gate_proj", + "model.layers.39.mlp.experts.50.up_proj", + "model.layers.39.mlp.experts.51.down_proj", + "model.layers.39.mlp.experts.51.gate_proj", + "model.layers.39.mlp.experts.51.up_proj", + "model.layers.39.mlp.experts.52.down_proj", + "model.layers.39.mlp.experts.52.gate_proj", + "model.layers.39.mlp.experts.52.up_proj", + "model.layers.39.mlp.experts.53.down_proj", + "model.layers.39.mlp.experts.53.gate_proj", + "model.layers.39.mlp.experts.53.up_proj", + "model.layers.39.mlp.experts.54.down_proj", + "model.layers.39.mlp.experts.54.gate_proj", + "model.layers.39.mlp.experts.54.up_proj", + "model.layers.39.mlp.experts.55.down_proj", + "model.layers.39.mlp.experts.55.gate_proj", + "model.layers.39.mlp.experts.55.up_proj", + "model.layers.39.mlp.experts.56.down_proj", + "model.layers.39.mlp.experts.56.gate_proj", + "model.layers.39.mlp.experts.56.up_proj", + "model.layers.39.mlp.experts.57.down_proj", + "model.layers.39.mlp.experts.57.gate_proj", + "model.layers.39.mlp.experts.57.up_proj", + "model.layers.39.mlp.experts.58.down_proj", + "model.layers.39.mlp.experts.58.gate_proj", + "model.layers.39.mlp.experts.58.up_proj", + "model.layers.39.mlp.experts.59.down_proj", + "model.layers.39.mlp.experts.59.gate_proj", + "model.layers.39.mlp.experts.59.up_proj", + "model.layers.39.mlp.experts.6.down_proj", + "model.layers.39.mlp.experts.6.gate_proj", + "model.layers.39.mlp.experts.6.up_proj", + "model.layers.39.mlp.experts.60.down_proj", + "model.layers.39.mlp.experts.60.gate_proj", + "model.layers.39.mlp.experts.60.up_proj", + "model.layers.39.mlp.experts.61.down_proj", + "model.layers.39.mlp.experts.61.gate_proj", + "model.layers.39.mlp.experts.61.up_proj", + "model.layers.39.mlp.experts.62.down_proj", + "model.layers.39.mlp.experts.62.gate_proj", + "model.layers.39.mlp.experts.62.up_proj", + "model.layers.39.mlp.experts.63.down_proj", + "model.layers.39.mlp.experts.63.gate_proj", + "model.layers.39.mlp.experts.63.up_proj", + "model.layers.39.mlp.experts.64.down_proj", + "model.layers.39.mlp.experts.64.gate_proj", + "model.layers.39.mlp.experts.64.up_proj", + "model.layers.39.mlp.experts.65.down_proj", + "model.layers.39.mlp.experts.65.gate_proj", + "model.layers.39.mlp.experts.65.up_proj", + "model.layers.39.mlp.experts.66.down_proj", + "model.layers.39.mlp.experts.66.gate_proj", + "model.layers.39.mlp.experts.66.up_proj", + "model.layers.39.mlp.experts.67.down_proj", + "model.layers.39.mlp.experts.67.gate_proj", + "model.layers.39.mlp.experts.67.up_proj", + "model.layers.39.mlp.experts.68.down_proj", + "model.layers.39.mlp.experts.68.gate_proj", + "model.layers.39.mlp.experts.68.up_proj", + "model.layers.39.mlp.experts.69.down_proj", + "model.layers.39.mlp.experts.69.gate_proj", + "model.layers.39.mlp.experts.69.up_proj", + "model.layers.39.mlp.experts.7.down_proj", + "model.layers.39.mlp.experts.7.gate_proj", + "model.layers.39.mlp.experts.7.up_proj", + "model.layers.39.mlp.experts.70.down_proj", + "model.layers.39.mlp.experts.70.gate_proj", + "model.layers.39.mlp.experts.70.up_proj", + "model.layers.39.mlp.experts.71.down_proj", + "model.layers.39.mlp.experts.71.gate_proj", + "model.layers.39.mlp.experts.71.up_proj", + "model.layers.39.mlp.experts.72.down_proj", + "model.layers.39.mlp.experts.72.gate_proj", + "model.layers.39.mlp.experts.72.up_proj", + "model.layers.39.mlp.experts.73.down_proj", + "model.layers.39.mlp.experts.73.gate_proj", + "model.layers.39.mlp.experts.73.up_proj", + "model.layers.39.mlp.experts.74.down_proj", + "model.layers.39.mlp.experts.74.gate_proj", + "model.layers.39.mlp.experts.74.up_proj", + "model.layers.39.mlp.experts.75.down_proj", + "model.layers.39.mlp.experts.75.gate_proj", + "model.layers.39.mlp.experts.75.up_proj", + "model.layers.39.mlp.experts.76.down_proj", + "model.layers.39.mlp.experts.76.gate_proj", + "model.layers.39.mlp.experts.76.up_proj", + "model.layers.39.mlp.experts.77.down_proj", + "model.layers.39.mlp.experts.77.gate_proj", + "model.layers.39.mlp.experts.77.up_proj", + "model.layers.39.mlp.experts.78.down_proj", + "model.layers.39.mlp.experts.78.gate_proj", + "model.layers.39.mlp.experts.78.up_proj", + "model.layers.39.mlp.experts.79.down_proj", + "model.layers.39.mlp.experts.79.gate_proj", + "model.layers.39.mlp.experts.79.up_proj", + "model.layers.39.mlp.experts.8.down_proj", + "model.layers.39.mlp.experts.8.gate_proj", + "model.layers.39.mlp.experts.8.up_proj", + "model.layers.39.mlp.experts.80.down_proj", + "model.layers.39.mlp.experts.80.gate_proj", + "model.layers.39.mlp.experts.80.up_proj", + "model.layers.39.mlp.experts.81.down_proj", + "model.layers.39.mlp.experts.81.gate_proj", + "model.layers.39.mlp.experts.81.up_proj", + "model.layers.39.mlp.experts.82.down_proj", + "model.layers.39.mlp.experts.82.gate_proj", + "model.layers.39.mlp.experts.82.up_proj", + "model.layers.39.mlp.experts.83.down_proj", + "model.layers.39.mlp.experts.83.gate_proj", + "model.layers.39.mlp.experts.83.up_proj", + "model.layers.39.mlp.experts.84.down_proj", + "model.layers.39.mlp.experts.84.gate_proj", + "model.layers.39.mlp.experts.84.up_proj", + "model.layers.39.mlp.experts.85.down_proj", + "model.layers.39.mlp.experts.85.gate_proj", + "model.layers.39.mlp.experts.85.up_proj", + "model.layers.39.mlp.experts.86.down_proj", + "model.layers.39.mlp.experts.86.gate_proj", + "model.layers.39.mlp.experts.86.up_proj", + "model.layers.39.mlp.experts.87.down_proj", + "model.layers.39.mlp.experts.87.gate_proj", + "model.layers.39.mlp.experts.87.up_proj", + "model.layers.39.mlp.experts.88.down_proj", + "model.layers.39.mlp.experts.88.gate_proj", + "model.layers.39.mlp.experts.88.up_proj", + "model.layers.39.mlp.experts.89.down_proj", + "model.layers.39.mlp.experts.89.gate_proj", + "model.layers.39.mlp.experts.89.up_proj", + "model.layers.39.mlp.experts.9.down_proj", + "model.layers.39.mlp.experts.9.gate_proj", + "model.layers.39.mlp.experts.9.up_proj", + "model.layers.39.mlp.experts.90.down_proj", + "model.layers.39.mlp.experts.90.gate_proj", + "model.layers.39.mlp.experts.90.up_proj", + "model.layers.39.mlp.experts.91.down_proj", + "model.layers.39.mlp.experts.91.gate_proj", + "model.layers.39.mlp.experts.91.up_proj", + "model.layers.39.mlp.experts.92.down_proj", + "model.layers.39.mlp.experts.92.gate_proj", + "model.layers.39.mlp.experts.92.up_proj", + "model.layers.39.mlp.experts.93.down_proj", + "model.layers.39.mlp.experts.93.gate_proj", + "model.layers.39.mlp.experts.93.up_proj", + "model.layers.39.mlp.experts.94.down_proj", + "model.layers.39.mlp.experts.94.gate_proj", + "model.layers.39.mlp.experts.94.up_proj", + "model.layers.39.mlp.experts.95.down_proj", + "model.layers.39.mlp.experts.95.gate_proj", + "model.layers.39.mlp.experts.95.up_proj", + "model.layers.39.mlp.experts.96.down_proj", + "model.layers.39.mlp.experts.96.gate_proj", + "model.layers.39.mlp.experts.96.up_proj", + "model.layers.39.mlp.experts.97.down_proj", + "model.layers.39.mlp.experts.97.gate_proj", + "model.layers.39.mlp.experts.97.up_proj", + "model.layers.39.mlp.experts.98.down_proj", + "model.layers.39.mlp.experts.98.gate_proj", + "model.layers.39.mlp.experts.98.up_proj", + "model.layers.39.mlp.experts.99.down_proj", + "model.layers.39.mlp.experts.99.gate_proj", + "model.layers.39.mlp.experts.99.up_proj", + "model.layers.4.mlp.experts.0.down_proj", + "model.layers.4.mlp.experts.0.gate_proj", + "model.layers.4.mlp.experts.0.up_proj", + "model.layers.4.mlp.experts.1.down_proj", + "model.layers.4.mlp.experts.1.gate_proj", + "model.layers.4.mlp.experts.1.up_proj", + "model.layers.4.mlp.experts.10.down_proj", + "model.layers.4.mlp.experts.10.gate_proj", + "model.layers.4.mlp.experts.10.up_proj", + "model.layers.4.mlp.experts.100.down_proj", + "model.layers.4.mlp.experts.100.gate_proj", + "model.layers.4.mlp.experts.100.up_proj", + "model.layers.4.mlp.experts.101.down_proj", + "model.layers.4.mlp.experts.101.gate_proj", + "model.layers.4.mlp.experts.101.up_proj", + "model.layers.4.mlp.experts.102.down_proj", + "model.layers.4.mlp.experts.102.gate_proj", + "model.layers.4.mlp.experts.102.up_proj", + "model.layers.4.mlp.experts.103.down_proj", + "model.layers.4.mlp.experts.103.gate_proj", + "model.layers.4.mlp.experts.103.up_proj", + "model.layers.4.mlp.experts.104.down_proj", + "model.layers.4.mlp.experts.104.gate_proj", + "model.layers.4.mlp.experts.104.up_proj", + "model.layers.4.mlp.experts.105.down_proj", + "model.layers.4.mlp.experts.105.gate_proj", + "model.layers.4.mlp.experts.105.up_proj", + "model.layers.4.mlp.experts.106.down_proj", + "model.layers.4.mlp.experts.106.gate_proj", + "model.layers.4.mlp.experts.106.up_proj", + "model.layers.4.mlp.experts.107.down_proj", + "model.layers.4.mlp.experts.107.gate_proj", + "model.layers.4.mlp.experts.107.up_proj", + "model.layers.4.mlp.experts.108.down_proj", + "model.layers.4.mlp.experts.108.gate_proj", + "model.layers.4.mlp.experts.108.up_proj", + "model.layers.4.mlp.experts.109.down_proj", + "model.layers.4.mlp.experts.109.gate_proj", + "model.layers.4.mlp.experts.109.up_proj", + "model.layers.4.mlp.experts.11.down_proj", + "model.layers.4.mlp.experts.11.gate_proj", + "model.layers.4.mlp.experts.11.up_proj", + "model.layers.4.mlp.experts.110.down_proj", + "model.layers.4.mlp.experts.110.gate_proj", + "model.layers.4.mlp.experts.110.up_proj", + "model.layers.4.mlp.experts.111.down_proj", + "model.layers.4.mlp.experts.111.gate_proj", + "model.layers.4.mlp.experts.111.up_proj", + "model.layers.4.mlp.experts.112.down_proj", + "model.layers.4.mlp.experts.112.gate_proj", + "model.layers.4.mlp.experts.112.up_proj", + "model.layers.4.mlp.experts.113.down_proj", + "model.layers.4.mlp.experts.113.gate_proj", + "model.layers.4.mlp.experts.113.up_proj", + "model.layers.4.mlp.experts.114.down_proj", + "model.layers.4.mlp.experts.114.gate_proj", + "model.layers.4.mlp.experts.114.up_proj", + "model.layers.4.mlp.experts.115.down_proj", + "model.layers.4.mlp.experts.115.gate_proj", + "model.layers.4.mlp.experts.115.up_proj", + "model.layers.4.mlp.experts.116.down_proj", + "model.layers.4.mlp.experts.116.gate_proj", + "model.layers.4.mlp.experts.116.up_proj", + "model.layers.4.mlp.experts.117.down_proj", + "model.layers.4.mlp.experts.117.gate_proj", + "model.layers.4.mlp.experts.117.up_proj", + "model.layers.4.mlp.experts.118.down_proj", + "model.layers.4.mlp.experts.118.gate_proj", + "model.layers.4.mlp.experts.118.up_proj", + "model.layers.4.mlp.experts.119.down_proj", + "model.layers.4.mlp.experts.119.gate_proj", + "model.layers.4.mlp.experts.119.up_proj", + "model.layers.4.mlp.experts.12.down_proj", + "model.layers.4.mlp.experts.12.gate_proj", + "model.layers.4.mlp.experts.12.up_proj", + "model.layers.4.mlp.experts.120.down_proj", + "model.layers.4.mlp.experts.120.gate_proj", + "model.layers.4.mlp.experts.120.up_proj", + "model.layers.4.mlp.experts.121.down_proj", + "model.layers.4.mlp.experts.121.gate_proj", + "model.layers.4.mlp.experts.121.up_proj", + "model.layers.4.mlp.experts.122.down_proj", + "model.layers.4.mlp.experts.122.gate_proj", + "model.layers.4.mlp.experts.122.up_proj", + "model.layers.4.mlp.experts.123.down_proj", + "model.layers.4.mlp.experts.123.gate_proj", + "model.layers.4.mlp.experts.123.up_proj", + "model.layers.4.mlp.experts.124.down_proj", + "model.layers.4.mlp.experts.124.gate_proj", + "model.layers.4.mlp.experts.124.up_proj", + "model.layers.4.mlp.experts.125.down_proj", + "model.layers.4.mlp.experts.125.gate_proj", + "model.layers.4.mlp.experts.125.up_proj", + "model.layers.4.mlp.experts.126.down_proj", + "model.layers.4.mlp.experts.126.gate_proj", + "model.layers.4.mlp.experts.126.up_proj", + "model.layers.4.mlp.experts.127.down_proj", + "model.layers.4.mlp.experts.127.gate_proj", + "model.layers.4.mlp.experts.127.up_proj", + "model.layers.4.mlp.experts.128.down_proj", + "model.layers.4.mlp.experts.128.gate_proj", + "model.layers.4.mlp.experts.128.up_proj", + "model.layers.4.mlp.experts.129.down_proj", + "model.layers.4.mlp.experts.129.gate_proj", + "model.layers.4.mlp.experts.129.up_proj", + "model.layers.4.mlp.experts.13.down_proj", + "model.layers.4.mlp.experts.13.gate_proj", + "model.layers.4.mlp.experts.13.up_proj", + "model.layers.4.mlp.experts.130.down_proj", + "model.layers.4.mlp.experts.130.gate_proj", + "model.layers.4.mlp.experts.130.up_proj", + "model.layers.4.mlp.experts.131.down_proj", + "model.layers.4.mlp.experts.131.gate_proj", + "model.layers.4.mlp.experts.131.up_proj", + "model.layers.4.mlp.experts.132.down_proj", + "model.layers.4.mlp.experts.132.gate_proj", + "model.layers.4.mlp.experts.132.up_proj", + "model.layers.4.mlp.experts.133.down_proj", + "model.layers.4.mlp.experts.133.gate_proj", + "model.layers.4.mlp.experts.133.up_proj", + "model.layers.4.mlp.experts.134.down_proj", + "model.layers.4.mlp.experts.134.gate_proj", + "model.layers.4.mlp.experts.134.up_proj", + "model.layers.4.mlp.experts.135.down_proj", + "model.layers.4.mlp.experts.135.gate_proj", + "model.layers.4.mlp.experts.135.up_proj", + "model.layers.4.mlp.experts.136.down_proj", + "model.layers.4.mlp.experts.136.gate_proj", + "model.layers.4.mlp.experts.136.up_proj", + "model.layers.4.mlp.experts.137.down_proj", + "model.layers.4.mlp.experts.137.gate_proj", + "model.layers.4.mlp.experts.137.up_proj", + "model.layers.4.mlp.experts.138.down_proj", + "model.layers.4.mlp.experts.138.gate_proj", + "model.layers.4.mlp.experts.138.up_proj", + "model.layers.4.mlp.experts.139.down_proj", + "model.layers.4.mlp.experts.139.gate_proj", + "model.layers.4.mlp.experts.139.up_proj", + "model.layers.4.mlp.experts.14.down_proj", + "model.layers.4.mlp.experts.14.gate_proj", + "model.layers.4.mlp.experts.14.up_proj", + "model.layers.4.mlp.experts.140.down_proj", + "model.layers.4.mlp.experts.140.gate_proj", + "model.layers.4.mlp.experts.140.up_proj", + "model.layers.4.mlp.experts.141.down_proj", + "model.layers.4.mlp.experts.141.gate_proj", + "model.layers.4.mlp.experts.141.up_proj", + "model.layers.4.mlp.experts.142.down_proj", + "model.layers.4.mlp.experts.142.gate_proj", + "model.layers.4.mlp.experts.142.up_proj", + "model.layers.4.mlp.experts.143.down_proj", + "model.layers.4.mlp.experts.143.gate_proj", + "model.layers.4.mlp.experts.143.up_proj", + "model.layers.4.mlp.experts.144.down_proj", + "model.layers.4.mlp.experts.144.gate_proj", + "model.layers.4.mlp.experts.144.up_proj", + "model.layers.4.mlp.experts.145.down_proj", + "model.layers.4.mlp.experts.145.gate_proj", + "model.layers.4.mlp.experts.145.up_proj", + "model.layers.4.mlp.experts.146.down_proj", + "model.layers.4.mlp.experts.146.gate_proj", + "model.layers.4.mlp.experts.146.up_proj", + "model.layers.4.mlp.experts.147.down_proj", + "model.layers.4.mlp.experts.147.gate_proj", + "model.layers.4.mlp.experts.147.up_proj", + "model.layers.4.mlp.experts.148.down_proj", + "model.layers.4.mlp.experts.148.gate_proj", + "model.layers.4.mlp.experts.148.up_proj", + "model.layers.4.mlp.experts.149.down_proj", + "model.layers.4.mlp.experts.149.gate_proj", + "model.layers.4.mlp.experts.149.up_proj", + "model.layers.4.mlp.experts.15.down_proj", + "model.layers.4.mlp.experts.15.gate_proj", + "model.layers.4.mlp.experts.15.up_proj", + "model.layers.4.mlp.experts.150.down_proj", + "model.layers.4.mlp.experts.150.gate_proj", + "model.layers.4.mlp.experts.150.up_proj", + "model.layers.4.mlp.experts.151.down_proj", + "model.layers.4.mlp.experts.151.gate_proj", + "model.layers.4.mlp.experts.151.up_proj", + "model.layers.4.mlp.experts.152.down_proj", + "model.layers.4.mlp.experts.152.gate_proj", + "model.layers.4.mlp.experts.152.up_proj", + "model.layers.4.mlp.experts.153.down_proj", + "model.layers.4.mlp.experts.153.gate_proj", + "model.layers.4.mlp.experts.153.up_proj", + "model.layers.4.mlp.experts.154.down_proj", + "model.layers.4.mlp.experts.154.gate_proj", + "model.layers.4.mlp.experts.154.up_proj", + "model.layers.4.mlp.experts.155.down_proj", + "model.layers.4.mlp.experts.155.gate_proj", + "model.layers.4.mlp.experts.155.up_proj", + "model.layers.4.mlp.experts.156.down_proj", + "model.layers.4.mlp.experts.156.gate_proj", + "model.layers.4.mlp.experts.156.up_proj", + "model.layers.4.mlp.experts.157.down_proj", + "model.layers.4.mlp.experts.157.gate_proj", + "model.layers.4.mlp.experts.157.up_proj", + "model.layers.4.mlp.experts.158.down_proj", + "model.layers.4.mlp.experts.158.gate_proj", + "model.layers.4.mlp.experts.158.up_proj", + "model.layers.4.mlp.experts.159.down_proj", + "model.layers.4.mlp.experts.159.gate_proj", + "model.layers.4.mlp.experts.159.up_proj", + "model.layers.4.mlp.experts.16.down_proj", + "model.layers.4.mlp.experts.16.gate_proj", + "model.layers.4.mlp.experts.16.up_proj", + "model.layers.4.mlp.experts.160.down_proj", + "model.layers.4.mlp.experts.160.gate_proj", + "model.layers.4.mlp.experts.160.up_proj", + "model.layers.4.mlp.experts.161.down_proj", + "model.layers.4.mlp.experts.161.gate_proj", + "model.layers.4.mlp.experts.161.up_proj", + "model.layers.4.mlp.experts.162.down_proj", + "model.layers.4.mlp.experts.162.gate_proj", + "model.layers.4.mlp.experts.162.up_proj", + "model.layers.4.mlp.experts.163.down_proj", + "model.layers.4.mlp.experts.163.gate_proj", + "model.layers.4.mlp.experts.163.up_proj", + "model.layers.4.mlp.experts.164.down_proj", + "model.layers.4.mlp.experts.164.gate_proj", + "model.layers.4.mlp.experts.164.up_proj", + "model.layers.4.mlp.experts.165.down_proj", + "model.layers.4.mlp.experts.165.gate_proj", + "model.layers.4.mlp.experts.165.up_proj", + "model.layers.4.mlp.experts.166.down_proj", + "model.layers.4.mlp.experts.166.gate_proj", + "model.layers.4.mlp.experts.166.up_proj", + "model.layers.4.mlp.experts.167.down_proj", + "model.layers.4.mlp.experts.167.gate_proj", + "model.layers.4.mlp.experts.167.up_proj", + "model.layers.4.mlp.experts.168.down_proj", + "model.layers.4.mlp.experts.168.gate_proj", + "model.layers.4.mlp.experts.168.up_proj", + "model.layers.4.mlp.experts.169.down_proj", + "model.layers.4.mlp.experts.169.gate_proj", + "model.layers.4.mlp.experts.169.up_proj", + "model.layers.4.mlp.experts.17.down_proj", + "model.layers.4.mlp.experts.17.gate_proj", + "model.layers.4.mlp.experts.17.up_proj", + "model.layers.4.mlp.experts.170.down_proj", + "model.layers.4.mlp.experts.170.gate_proj", + "model.layers.4.mlp.experts.170.up_proj", + "model.layers.4.mlp.experts.171.down_proj", + "model.layers.4.mlp.experts.171.gate_proj", + "model.layers.4.mlp.experts.171.up_proj", + "model.layers.4.mlp.experts.172.down_proj", + "model.layers.4.mlp.experts.172.gate_proj", + "model.layers.4.mlp.experts.172.up_proj", + "model.layers.4.mlp.experts.173.down_proj", + "model.layers.4.mlp.experts.173.gate_proj", + "model.layers.4.mlp.experts.173.up_proj", + "model.layers.4.mlp.experts.174.down_proj", + "model.layers.4.mlp.experts.174.gate_proj", + "model.layers.4.mlp.experts.174.up_proj", + "model.layers.4.mlp.experts.175.down_proj", + "model.layers.4.mlp.experts.175.gate_proj", + "model.layers.4.mlp.experts.175.up_proj", + "model.layers.4.mlp.experts.176.down_proj", + "model.layers.4.mlp.experts.176.gate_proj", + "model.layers.4.mlp.experts.176.up_proj", + "model.layers.4.mlp.experts.177.down_proj", + "model.layers.4.mlp.experts.177.gate_proj", + "model.layers.4.mlp.experts.177.up_proj", + "model.layers.4.mlp.experts.178.down_proj", + "model.layers.4.mlp.experts.178.gate_proj", + "model.layers.4.mlp.experts.178.up_proj", + "model.layers.4.mlp.experts.179.down_proj", + "model.layers.4.mlp.experts.179.gate_proj", + "model.layers.4.mlp.experts.179.up_proj", + "model.layers.4.mlp.experts.18.down_proj", + "model.layers.4.mlp.experts.18.gate_proj", + "model.layers.4.mlp.experts.18.up_proj", + "model.layers.4.mlp.experts.180.down_proj", + "model.layers.4.mlp.experts.180.gate_proj", + "model.layers.4.mlp.experts.180.up_proj", + "model.layers.4.mlp.experts.181.down_proj", + "model.layers.4.mlp.experts.181.gate_proj", + "model.layers.4.mlp.experts.181.up_proj", + "model.layers.4.mlp.experts.182.down_proj", + "model.layers.4.mlp.experts.182.gate_proj", + "model.layers.4.mlp.experts.182.up_proj", + "model.layers.4.mlp.experts.183.down_proj", + "model.layers.4.mlp.experts.183.gate_proj", + "model.layers.4.mlp.experts.183.up_proj", + "model.layers.4.mlp.experts.184.down_proj", + "model.layers.4.mlp.experts.184.gate_proj", + "model.layers.4.mlp.experts.184.up_proj", + "model.layers.4.mlp.experts.185.down_proj", + "model.layers.4.mlp.experts.185.gate_proj", + "model.layers.4.mlp.experts.185.up_proj", + "model.layers.4.mlp.experts.186.down_proj", + "model.layers.4.mlp.experts.186.gate_proj", + "model.layers.4.mlp.experts.186.up_proj", + "model.layers.4.mlp.experts.187.down_proj", + "model.layers.4.mlp.experts.187.gate_proj", + "model.layers.4.mlp.experts.187.up_proj", + "model.layers.4.mlp.experts.188.down_proj", + "model.layers.4.mlp.experts.188.gate_proj", + "model.layers.4.mlp.experts.188.up_proj", + "model.layers.4.mlp.experts.189.down_proj", + "model.layers.4.mlp.experts.189.gate_proj", + "model.layers.4.mlp.experts.189.up_proj", + "model.layers.4.mlp.experts.19.down_proj", + "model.layers.4.mlp.experts.19.gate_proj", + "model.layers.4.mlp.experts.19.up_proj", + "model.layers.4.mlp.experts.190.down_proj", + "model.layers.4.mlp.experts.190.gate_proj", + "model.layers.4.mlp.experts.190.up_proj", + "model.layers.4.mlp.experts.191.down_proj", + "model.layers.4.mlp.experts.191.gate_proj", + "model.layers.4.mlp.experts.191.up_proj", + "model.layers.4.mlp.experts.192.down_proj", + "model.layers.4.mlp.experts.192.gate_proj", + "model.layers.4.mlp.experts.192.up_proj", + "model.layers.4.mlp.experts.193.down_proj", + "model.layers.4.mlp.experts.193.gate_proj", + "model.layers.4.mlp.experts.193.up_proj", + "model.layers.4.mlp.experts.194.down_proj", + "model.layers.4.mlp.experts.194.gate_proj", + "model.layers.4.mlp.experts.194.up_proj", + "model.layers.4.mlp.experts.195.down_proj", + "model.layers.4.mlp.experts.195.gate_proj", + "model.layers.4.mlp.experts.195.up_proj", + "model.layers.4.mlp.experts.196.down_proj", + "model.layers.4.mlp.experts.196.gate_proj", + "model.layers.4.mlp.experts.196.up_proj", + "model.layers.4.mlp.experts.197.down_proj", + "model.layers.4.mlp.experts.197.gate_proj", + "model.layers.4.mlp.experts.197.up_proj", + "model.layers.4.mlp.experts.198.down_proj", + "model.layers.4.mlp.experts.198.gate_proj", + "model.layers.4.mlp.experts.198.up_proj", + "model.layers.4.mlp.experts.199.down_proj", + "model.layers.4.mlp.experts.199.gate_proj", + "model.layers.4.mlp.experts.199.up_proj", + "model.layers.4.mlp.experts.2.down_proj", + "model.layers.4.mlp.experts.2.gate_proj", + "model.layers.4.mlp.experts.2.up_proj", + "model.layers.4.mlp.experts.20.down_proj", + "model.layers.4.mlp.experts.20.gate_proj", + "model.layers.4.mlp.experts.20.up_proj", + "model.layers.4.mlp.experts.200.down_proj", + "model.layers.4.mlp.experts.200.gate_proj", + "model.layers.4.mlp.experts.200.up_proj", + "model.layers.4.mlp.experts.201.down_proj", + "model.layers.4.mlp.experts.201.gate_proj", + "model.layers.4.mlp.experts.201.up_proj", + "model.layers.4.mlp.experts.202.down_proj", + "model.layers.4.mlp.experts.202.gate_proj", + "model.layers.4.mlp.experts.202.up_proj", + "model.layers.4.mlp.experts.203.down_proj", + "model.layers.4.mlp.experts.203.gate_proj", + "model.layers.4.mlp.experts.203.up_proj", + "model.layers.4.mlp.experts.204.down_proj", + "model.layers.4.mlp.experts.204.gate_proj", + "model.layers.4.mlp.experts.204.up_proj", + "model.layers.4.mlp.experts.205.down_proj", + "model.layers.4.mlp.experts.205.gate_proj", + "model.layers.4.mlp.experts.205.up_proj", + "model.layers.4.mlp.experts.206.down_proj", + "model.layers.4.mlp.experts.206.gate_proj", + "model.layers.4.mlp.experts.206.up_proj", + "model.layers.4.mlp.experts.207.down_proj", + "model.layers.4.mlp.experts.207.gate_proj", + "model.layers.4.mlp.experts.207.up_proj", + "model.layers.4.mlp.experts.208.down_proj", + "model.layers.4.mlp.experts.208.gate_proj", + "model.layers.4.mlp.experts.208.up_proj", + "model.layers.4.mlp.experts.209.down_proj", + "model.layers.4.mlp.experts.209.gate_proj", + "model.layers.4.mlp.experts.209.up_proj", + "model.layers.4.mlp.experts.21.down_proj", + "model.layers.4.mlp.experts.21.gate_proj", + "model.layers.4.mlp.experts.21.up_proj", + "model.layers.4.mlp.experts.210.down_proj", + "model.layers.4.mlp.experts.210.gate_proj", + "model.layers.4.mlp.experts.210.up_proj", + "model.layers.4.mlp.experts.211.down_proj", + "model.layers.4.mlp.experts.211.gate_proj", + "model.layers.4.mlp.experts.211.up_proj", + "model.layers.4.mlp.experts.212.down_proj", + "model.layers.4.mlp.experts.212.gate_proj", + "model.layers.4.mlp.experts.212.up_proj", + "model.layers.4.mlp.experts.213.down_proj", + "model.layers.4.mlp.experts.213.gate_proj", + "model.layers.4.mlp.experts.213.up_proj", + "model.layers.4.mlp.experts.214.down_proj", + "model.layers.4.mlp.experts.214.gate_proj", + "model.layers.4.mlp.experts.214.up_proj", + "model.layers.4.mlp.experts.215.down_proj", + "model.layers.4.mlp.experts.215.gate_proj", + "model.layers.4.mlp.experts.215.up_proj", + "model.layers.4.mlp.experts.216.down_proj", + "model.layers.4.mlp.experts.216.gate_proj", + "model.layers.4.mlp.experts.216.up_proj", + "model.layers.4.mlp.experts.217.down_proj", + "model.layers.4.mlp.experts.217.gate_proj", + "model.layers.4.mlp.experts.217.up_proj", + "model.layers.4.mlp.experts.218.down_proj", + "model.layers.4.mlp.experts.218.gate_proj", + "model.layers.4.mlp.experts.218.up_proj", + "model.layers.4.mlp.experts.219.down_proj", + "model.layers.4.mlp.experts.219.gate_proj", + "model.layers.4.mlp.experts.219.up_proj", + "model.layers.4.mlp.experts.22.down_proj", + "model.layers.4.mlp.experts.22.gate_proj", + "model.layers.4.mlp.experts.22.up_proj", + "model.layers.4.mlp.experts.220.down_proj", + "model.layers.4.mlp.experts.220.gate_proj", + "model.layers.4.mlp.experts.220.up_proj", + "model.layers.4.mlp.experts.221.down_proj", + "model.layers.4.mlp.experts.221.gate_proj", + "model.layers.4.mlp.experts.221.up_proj", + "model.layers.4.mlp.experts.222.down_proj", + "model.layers.4.mlp.experts.222.gate_proj", + "model.layers.4.mlp.experts.222.up_proj", + "model.layers.4.mlp.experts.223.down_proj", + "model.layers.4.mlp.experts.223.gate_proj", + "model.layers.4.mlp.experts.223.up_proj", + "model.layers.4.mlp.experts.224.down_proj", + "model.layers.4.mlp.experts.224.gate_proj", + "model.layers.4.mlp.experts.224.up_proj", + "model.layers.4.mlp.experts.225.down_proj", + "model.layers.4.mlp.experts.225.gate_proj", + "model.layers.4.mlp.experts.225.up_proj", + "model.layers.4.mlp.experts.226.down_proj", + "model.layers.4.mlp.experts.226.gate_proj", + "model.layers.4.mlp.experts.226.up_proj", + "model.layers.4.mlp.experts.227.down_proj", + "model.layers.4.mlp.experts.227.gate_proj", + "model.layers.4.mlp.experts.227.up_proj", + "model.layers.4.mlp.experts.228.down_proj", + "model.layers.4.mlp.experts.228.gate_proj", + "model.layers.4.mlp.experts.228.up_proj", + "model.layers.4.mlp.experts.229.down_proj", + "model.layers.4.mlp.experts.229.gate_proj", + "model.layers.4.mlp.experts.229.up_proj", + "model.layers.4.mlp.experts.23.down_proj", + "model.layers.4.mlp.experts.23.gate_proj", + "model.layers.4.mlp.experts.23.up_proj", + "model.layers.4.mlp.experts.230.down_proj", + "model.layers.4.mlp.experts.230.gate_proj", + "model.layers.4.mlp.experts.230.up_proj", + "model.layers.4.mlp.experts.231.down_proj", + "model.layers.4.mlp.experts.231.gate_proj", + "model.layers.4.mlp.experts.231.up_proj", + "model.layers.4.mlp.experts.232.down_proj", + "model.layers.4.mlp.experts.232.gate_proj", + "model.layers.4.mlp.experts.232.up_proj", + "model.layers.4.mlp.experts.233.down_proj", + "model.layers.4.mlp.experts.233.gate_proj", + "model.layers.4.mlp.experts.233.up_proj", + "model.layers.4.mlp.experts.234.down_proj", + "model.layers.4.mlp.experts.234.gate_proj", + "model.layers.4.mlp.experts.234.up_proj", + "model.layers.4.mlp.experts.235.down_proj", + "model.layers.4.mlp.experts.235.gate_proj", + "model.layers.4.mlp.experts.235.up_proj", + "model.layers.4.mlp.experts.236.down_proj", + "model.layers.4.mlp.experts.236.gate_proj", + "model.layers.4.mlp.experts.236.up_proj", + "model.layers.4.mlp.experts.237.down_proj", + "model.layers.4.mlp.experts.237.gate_proj", + "model.layers.4.mlp.experts.237.up_proj", + "model.layers.4.mlp.experts.238.down_proj", + "model.layers.4.mlp.experts.238.gate_proj", + "model.layers.4.mlp.experts.238.up_proj", + "model.layers.4.mlp.experts.239.down_proj", + "model.layers.4.mlp.experts.239.gate_proj", + "model.layers.4.mlp.experts.239.up_proj", + "model.layers.4.mlp.experts.24.down_proj", + "model.layers.4.mlp.experts.24.gate_proj", + "model.layers.4.mlp.experts.24.up_proj", + "model.layers.4.mlp.experts.240.down_proj", + "model.layers.4.mlp.experts.240.gate_proj", + "model.layers.4.mlp.experts.240.up_proj", + "model.layers.4.mlp.experts.241.down_proj", + "model.layers.4.mlp.experts.241.gate_proj", + "model.layers.4.mlp.experts.241.up_proj", + "model.layers.4.mlp.experts.242.down_proj", + "model.layers.4.mlp.experts.242.gate_proj", + "model.layers.4.mlp.experts.242.up_proj", + "model.layers.4.mlp.experts.243.down_proj", + "model.layers.4.mlp.experts.243.gate_proj", + "model.layers.4.mlp.experts.243.up_proj", + "model.layers.4.mlp.experts.244.down_proj", + "model.layers.4.mlp.experts.244.gate_proj", + "model.layers.4.mlp.experts.244.up_proj", + "model.layers.4.mlp.experts.245.down_proj", + "model.layers.4.mlp.experts.245.gate_proj", + "model.layers.4.mlp.experts.245.up_proj", + "model.layers.4.mlp.experts.246.down_proj", + "model.layers.4.mlp.experts.246.gate_proj", + "model.layers.4.mlp.experts.246.up_proj", + "model.layers.4.mlp.experts.247.down_proj", + "model.layers.4.mlp.experts.247.gate_proj", + "model.layers.4.mlp.experts.247.up_proj", + "model.layers.4.mlp.experts.248.down_proj", + "model.layers.4.mlp.experts.248.gate_proj", + "model.layers.4.mlp.experts.248.up_proj", + "model.layers.4.mlp.experts.249.down_proj", + "model.layers.4.mlp.experts.249.gate_proj", + "model.layers.4.mlp.experts.249.up_proj", + "model.layers.4.mlp.experts.25.down_proj", + "model.layers.4.mlp.experts.25.gate_proj", + "model.layers.4.mlp.experts.25.up_proj", + "model.layers.4.mlp.experts.250.down_proj", + "model.layers.4.mlp.experts.250.gate_proj", + "model.layers.4.mlp.experts.250.up_proj", + "model.layers.4.mlp.experts.251.down_proj", + "model.layers.4.mlp.experts.251.gate_proj", + "model.layers.4.mlp.experts.251.up_proj", + "model.layers.4.mlp.experts.252.down_proj", + "model.layers.4.mlp.experts.252.gate_proj", + "model.layers.4.mlp.experts.252.up_proj", + "model.layers.4.mlp.experts.253.down_proj", + "model.layers.4.mlp.experts.253.gate_proj", + "model.layers.4.mlp.experts.253.up_proj", + "model.layers.4.mlp.experts.254.down_proj", + "model.layers.4.mlp.experts.254.gate_proj", + "model.layers.4.mlp.experts.254.up_proj", + "model.layers.4.mlp.experts.255.down_proj", + "model.layers.4.mlp.experts.255.gate_proj", + "model.layers.4.mlp.experts.255.up_proj", + "model.layers.4.mlp.experts.26.down_proj", + "model.layers.4.mlp.experts.26.gate_proj", + "model.layers.4.mlp.experts.26.up_proj", + "model.layers.4.mlp.experts.27.down_proj", + "model.layers.4.mlp.experts.27.gate_proj", + "model.layers.4.mlp.experts.27.up_proj", + "model.layers.4.mlp.experts.28.down_proj", + "model.layers.4.mlp.experts.28.gate_proj", + "model.layers.4.mlp.experts.28.up_proj", + "model.layers.4.mlp.experts.29.down_proj", + "model.layers.4.mlp.experts.29.gate_proj", + "model.layers.4.mlp.experts.29.up_proj", + "model.layers.4.mlp.experts.3.down_proj", + "model.layers.4.mlp.experts.3.gate_proj", + "model.layers.4.mlp.experts.3.up_proj", + "model.layers.4.mlp.experts.30.down_proj", + "model.layers.4.mlp.experts.30.gate_proj", + "model.layers.4.mlp.experts.30.up_proj", + "model.layers.4.mlp.experts.31.down_proj", + "model.layers.4.mlp.experts.31.gate_proj", + "model.layers.4.mlp.experts.31.up_proj", + "model.layers.4.mlp.experts.32.down_proj", + "model.layers.4.mlp.experts.32.gate_proj", + "model.layers.4.mlp.experts.32.up_proj", + "model.layers.4.mlp.experts.33.down_proj", + "model.layers.4.mlp.experts.33.gate_proj", + "model.layers.4.mlp.experts.33.up_proj", + "model.layers.4.mlp.experts.34.down_proj", + "model.layers.4.mlp.experts.34.gate_proj", + "model.layers.4.mlp.experts.34.up_proj", + "model.layers.4.mlp.experts.35.down_proj", + "model.layers.4.mlp.experts.35.gate_proj", + "model.layers.4.mlp.experts.35.up_proj", + "model.layers.4.mlp.experts.36.down_proj", + "model.layers.4.mlp.experts.36.gate_proj", + "model.layers.4.mlp.experts.36.up_proj", + "model.layers.4.mlp.experts.37.down_proj", + "model.layers.4.mlp.experts.37.gate_proj", + "model.layers.4.mlp.experts.37.up_proj", + "model.layers.4.mlp.experts.38.down_proj", + "model.layers.4.mlp.experts.38.gate_proj", + "model.layers.4.mlp.experts.38.up_proj", + "model.layers.4.mlp.experts.39.down_proj", + "model.layers.4.mlp.experts.39.gate_proj", + "model.layers.4.mlp.experts.39.up_proj", + "model.layers.4.mlp.experts.4.down_proj", + "model.layers.4.mlp.experts.4.gate_proj", + "model.layers.4.mlp.experts.4.up_proj", + "model.layers.4.mlp.experts.40.down_proj", + "model.layers.4.mlp.experts.40.gate_proj", + "model.layers.4.mlp.experts.40.up_proj", + "model.layers.4.mlp.experts.41.down_proj", + "model.layers.4.mlp.experts.41.gate_proj", + "model.layers.4.mlp.experts.41.up_proj", + "model.layers.4.mlp.experts.42.down_proj", + "model.layers.4.mlp.experts.42.gate_proj", + "model.layers.4.mlp.experts.42.up_proj", + "model.layers.4.mlp.experts.43.down_proj", + "model.layers.4.mlp.experts.43.gate_proj", + "model.layers.4.mlp.experts.43.up_proj", + "model.layers.4.mlp.experts.44.down_proj", + "model.layers.4.mlp.experts.44.gate_proj", + "model.layers.4.mlp.experts.44.up_proj", + "model.layers.4.mlp.experts.45.down_proj", + "model.layers.4.mlp.experts.45.gate_proj", + "model.layers.4.mlp.experts.45.up_proj", + "model.layers.4.mlp.experts.46.down_proj", + "model.layers.4.mlp.experts.46.gate_proj", + "model.layers.4.mlp.experts.46.up_proj", + "model.layers.4.mlp.experts.47.down_proj", + "model.layers.4.mlp.experts.47.gate_proj", + "model.layers.4.mlp.experts.47.up_proj", + "model.layers.4.mlp.experts.48.down_proj", + "model.layers.4.mlp.experts.48.gate_proj", + "model.layers.4.mlp.experts.48.up_proj", + "model.layers.4.mlp.experts.49.down_proj", + "model.layers.4.mlp.experts.49.gate_proj", + "model.layers.4.mlp.experts.49.up_proj", + "model.layers.4.mlp.experts.5.down_proj", + "model.layers.4.mlp.experts.5.gate_proj", + "model.layers.4.mlp.experts.5.up_proj", + "model.layers.4.mlp.experts.50.down_proj", + "model.layers.4.mlp.experts.50.gate_proj", + "model.layers.4.mlp.experts.50.up_proj", + "model.layers.4.mlp.experts.51.down_proj", + "model.layers.4.mlp.experts.51.gate_proj", + "model.layers.4.mlp.experts.51.up_proj", + "model.layers.4.mlp.experts.52.down_proj", + "model.layers.4.mlp.experts.52.gate_proj", + "model.layers.4.mlp.experts.52.up_proj", + "model.layers.4.mlp.experts.53.down_proj", + "model.layers.4.mlp.experts.53.gate_proj", + "model.layers.4.mlp.experts.53.up_proj", + "model.layers.4.mlp.experts.54.down_proj", + "model.layers.4.mlp.experts.54.gate_proj", + "model.layers.4.mlp.experts.54.up_proj", + "model.layers.4.mlp.experts.55.down_proj", + "model.layers.4.mlp.experts.55.gate_proj", + "model.layers.4.mlp.experts.55.up_proj", + "model.layers.4.mlp.experts.56.down_proj", + "model.layers.4.mlp.experts.56.gate_proj", + "model.layers.4.mlp.experts.56.up_proj", + "model.layers.4.mlp.experts.57.down_proj", + "model.layers.4.mlp.experts.57.gate_proj", + "model.layers.4.mlp.experts.57.up_proj", + "model.layers.4.mlp.experts.58.down_proj", + "model.layers.4.mlp.experts.58.gate_proj", + "model.layers.4.mlp.experts.58.up_proj", + "model.layers.4.mlp.experts.59.down_proj", + "model.layers.4.mlp.experts.59.gate_proj", + "model.layers.4.mlp.experts.59.up_proj", + "model.layers.4.mlp.experts.6.down_proj", + "model.layers.4.mlp.experts.6.gate_proj", + "model.layers.4.mlp.experts.6.up_proj", + "model.layers.4.mlp.experts.60.down_proj", + "model.layers.4.mlp.experts.60.gate_proj", + "model.layers.4.mlp.experts.60.up_proj", + "model.layers.4.mlp.experts.61.down_proj", + "model.layers.4.mlp.experts.61.gate_proj", + "model.layers.4.mlp.experts.61.up_proj", + "model.layers.4.mlp.experts.62.down_proj", + "model.layers.4.mlp.experts.62.gate_proj", + "model.layers.4.mlp.experts.62.up_proj", + "model.layers.4.mlp.experts.63.down_proj", + "model.layers.4.mlp.experts.63.gate_proj", + "model.layers.4.mlp.experts.63.up_proj", + "model.layers.4.mlp.experts.64.down_proj", + "model.layers.4.mlp.experts.64.gate_proj", + "model.layers.4.mlp.experts.64.up_proj", + "model.layers.4.mlp.experts.65.down_proj", + "model.layers.4.mlp.experts.65.gate_proj", + "model.layers.4.mlp.experts.65.up_proj", + "model.layers.4.mlp.experts.66.down_proj", + "model.layers.4.mlp.experts.66.gate_proj", + "model.layers.4.mlp.experts.66.up_proj", + "model.layers.4.mlp.experts.67.down_proj", + "model.layers.4.mlp.experts.67.gate_proj", + "model.layers.4.mlp.experts.67.up_proj", + "model.layers.4.mlp.experts.68.down_proj", + "model.layers.4.mlp.experts.68.gate_proj", + "model.layers.4.mlp.experts.68.up_proj", + "model.layers.4.mlp.experts.69.down_proj", + "model.layers.4.mlp.experts.69.gate_proj", + "model.layers.4.mlp.experts.69.up_proj", + "model.layers.4.mlp.experts.7.down_proj", + "model.layers.4.mlp.experts.7.gate_proj", + "model.layers.4.mlp.experts.7.up_proj", + "model.layers.4.mlp.experts.70.down_proj", + "model.layers.4.mlp.experts.70.gate_proj", + "model.layers.4.mlp.experts.70.up_proj", + "model.layers.4.mlp.experts.71.down_proj", + "model.layers.4.mlp.experts.71.gate_proj", + "model.layers.4.mlp.experts.71.up_proj", + "model.layers.4.mlp.experts.72.down_proj", + "model.layers.4.mlp.experts.72.gate_proj", + "model.layers.4.mlp.experts.72.up_proj", + "model.layers.4.mlp.experts.73.down_proj", + "model.layers.4.mlp.experts.73.gate_proj", + "model.layers.4.mlp.experts.73.up_proj", + "model.layers.4.mlp.experts.74.down_proj", + "model.layers.4.mlp.experts.74.gate_proj", + "model.layers.4.mlp.experts.74.up_proj", + "model.layers.4.mlp.experts.75.down_proj", + "model.layers.4.mlp.experts.75.gate_proj", + "model.layers.4.mlp.experts.75.up_proj", + "model.layers.4.mlp.experts.76.down_proj", + "model.layers.4.mlp.experts.76.gate_proj", + "model.layers.4.mlp.experts.76.up_proj", + "model.layers.4.mlp.experts.77.down_proj", + "model.layers.4.mlp.experts.77.gate_proj", + "model.layers.4.mlp.experts.77.up_proj", + "model.layers.4.mlp.experts.78.down_proj", + "model.layers.4.mlp.experts.78.gate_proj", + "model.layers.4.mlp.experts.78.up_proj", + "model.layers.4.mlp.experts.79.down_proj", + "model.layers.4.mlp.experts.79.gate_proj", + "model.layers.4.mlp.experts.79.up_proj", + "model.layers.4.mlp.experts.8.down_proj", + "model.layers.4.mlp.experts.8.gate_proj", + "model.layers.4.mlp.experts.8.up_proj", + "model.layers.4.mlp.experts.80.down_proj", + "model.layers.4.mlp.experts.80.gate_proj", + "model.layers.4.mlp.experts.80.up_proj", + "model.layers.4.mlp.experts.81.down_proj", + "model.layers.4.mlp.experts.81.gate_proj", + "model.layers.4.mlp.experts.81.up_proj", + "model.layers.4.mlp.experts.82.down_proj", + "model.layers.4.mlp.experts.82.gate_proj", + "model.layers.4.mlp.experts.82.up_proj", + "model.layers.4.mlp.experts.83.down_proj", + "model.layers.4.mlp.experts.83.gate_proj", + "model.layers.4.mlp.experts.83.up_proj", + "model.layers.4.mlp.experts.84.down_proj", + "model.layers.4.mlp.experts.84.gate_proj", + "model.layers.4.mlp.experts.84.up_proj", + "model.layers.4.mlp.experts.85.down_proj", + "model.layers.4.mlp.experts.85.gate_proj", + "model.layers.4.mlp.experts.85.up_proj", + "model.layers.4.mlp.experts.86.down_proj", + "model.layers.4.mlp.experts.86.gate_proj", + "model.layers.4.mlp.experts.86.up_proj", + "model.layers.4.mlp.experts.87.down_proj", + "model.layers.4.mlp.experts.87.gate_proj", + "model.layers.4.mlp.experts.87.up_proj", + "model.layers.4.mlp.experts.88.down_proj", + "model.layers.4.mlp.experts.88.gate_proj", + "model.layers.4.mlp.experts.88.up_proj", + "model.layers.4.mlp.experts.89.down_proj", + "model.layers.4.mlp.experts.89.gate_proj", + "model.layers.4.mlp.experts.89.up_proj", + "model.layers.4.mlp.experts.9.down_proj", + "model.layers.4.mlp.experts.9.gate_proj", + "model.layers.4.mlp.experts.9.up_proj", + "model.layers.4.mlp.experts.90.down_proj", + "model.layers.4.mlp.experts.90.gate_proj", + "model.layers.4.mlp.experts.90.up_proj", + "model.layers.4.mlp.experts.91.down_proj", + "model.layers.4.mlp.experts.91.gate_proj", + "model.layers.4.mlp.experts.91.up_proj", + "model.layers.4.mlp.experts.92.down_proj", + "model.layers.4.mlp.experts.92.gate_proj", + "model.layers.4.mlp.experts.92.up_proj", + "model.layers.4.mlp.experts.93.down_proj", + "model.layers.4.mlp.experts.93.gate_proj", + "model.layers.4.mlp.experts.93.up_proj", + "model.layers.4.mlp.experts.94.down_proj", + "model.layers.4.mlp.experts.94.gate_proj", + "model.layers.4.mlp.experts.94.up_proj", + "model.layers.4.mlp.experts.95.down_proj", + "model.layers.4.mlp.experts.95.gate_proj", + "model.layers.4.mlp.experts.95.up_proj", + "model.layers.4.mlp.experts.96.down_proj", + "model.layers.4.mlp.experts.96.gate_proj", + "model.layers.4.mlp.experts.96.up_proj", + "model.layers.4.mlp.experts.97.down_proj", + "model.layers.4.mlp.experts.97.gate_proj", + "model.layers.4.mlp.experts.97.up_proj", + "model.layers.4.mlp.experts.98.down_proj", + "model.layers.4.mlp.experts.98.gate_proj", + "model.layers.4.mlp.experts.98.up_proj", + "model.layers.4.mlp.experts.99.down_proj", + "model.layers.4.mlp.experts.99.gate_proj", + "model.layers.4.mlp.experts.99.up_proj", + "model.layers.40.mlp.experts.0.down_proj", + "model.layers.40.mlp.experts.0.gate_proj", + "model.layers.40.mlp.experts.0.up_proj", + "model.layers.40.mlp.experts.1.down_proj", + "model.layers.40.mlp.experts.1.gate_proj", + "model.layers.40.mlp.experts.1.up_proj", + "model.layers.40.mlp.experts.10.down_proj", + "model.layers.40.mlp.experts.10.gate_proj", + "model.layers.40.mlp.experts.10.up_proj", + "model.layers.40.mlp.experts.100.down_proj", + "model.layers.40.mlp.experts.100.gate_proj", + "model.layers.40.mlp.experts.100.up_proj", + "model.layers.40.mlp.experts.101.down_proj", + "model.layers.40.mlp.experts.101.gate_proj", + "model.layers.40.mlp.experts.101.up_proj", + "model.layers.40.mlp.experts.102.down_proj", + "model.layers.40.mlp.experts.102.gate_proj", + "model.layers.40.mlp.experts.102.up_proj", + "model.layers.40.mlp.experts.103.down_proj", + "model.layers.40.mlp.experts.103.gate_proj", + "model.layers.40.mlp.experts.103.up_proj", + "model.layers.40.mlp.experts.104.down_proj", + "model.layers.40.mlp.experts.104.gate_proj", + "model.layers.40.mlp.experts.104.up_proj", + "model.layers.40.mlp.experts.105.down_proj", + "model.layers.40.mlp.experts.105.gate_proj", + "model.layers.40.mlp.experts.105.up_proj", + "model.layers.40.mlp.experts.106.down_proj", + "model.layers.40.mlp.experts.106.gate_proj", + "model.layers.40.mlp.experts.106.up_proj", + "model.layers.40.mlp.experts.107.down_proj", + "model.layers.40.mlp.experts.107.gate_proj", + "model.layers.40.mlp.experts.107.up_proj", + "model.layers.40.mlp.experts.108.down_proj", + "model.layers.40.mlp.experts.108.gate_proj", + "model.layers.40.mlp.experts.108.up_proj", + "model.layers.40.mlp.experts.109.down_proj", + "model.layers.40.mlp.experts.109.gate_proj", + "model.layers.40.mlp.experts.109.up_proj", + "model.layers.40.mlp.experts.11.down_proj", + "model.layers.40.mlp.experts.11.gate_proj", + "model.layers.40.mlp.experts.11.up_proj", + "model.layers.40.mlp.experts.110.down_proj", + "model.layers.40.mlp.experts.110.gate_proj", + "model.layers.40.mlp.experts.110.up_proj", + "model.layers.40.mlp.experts.111.down_proj", + "model.layers.40.mlp.experts.111.gate_proj", + "model.layers.40.mlp.experts.111.up_proj", + "model.layers.40.mlp.experts.112.down_proj", + "model.layers.40.mlp.experts.112.gate_proj", + "model.layers.40.mlp.experts.112.up_proj", + "model.layers.40.mlp.experts.113.down_proj", + "model.layers.40.mlp.experts.113.gate_proj", + "model.layers.40.mlp.experts.113.up_proj", + "model.layers.40.mlp.experts.114.down_proj", + "model.layers.40.mlp.experts.114.gate_proj", + "model.layers.40.mlp.experts.114.up_proj", + "model.layers.40.mlp.experts.115.down_proj", + "model.layers.40.mlp.experts.115.gate_proj", + "model.layers.40.mlp.experts.115.up_proj", + "model.layers.40.mlp.experts.116.down_proj", + "model.layers.40.mlp.experts.116.gate_proj", + "model.layers.40.mlp.experts.116.up_proj", + "model.layers.40.mlp.experts.117.down_proj", + "model.layers.40.mlp.experts.117.gate_proj", + "model.layers.40.mlp.experts.117.up_proj", + "model.layers.40.mlp.experts.118.down_proj", + "model.layers.40.mlp.experts.118.gate_proj", + "model.layers.40.mlp.experts.118.up_proj", + "model.layers.40.mlp.experts.119.down_proj", + "model.layers.40.mlp.experts.119.gate_proj", + "model.layers.40.mlp.experts.119.up_proj", + "model.layers.40.mlp.experts.12.down_proj", + "model.layers.40.mlp.experts.12.gate_proj", + "model.layers.40.mlp.experts.12.up_proj", + "model.layers.40.mlp.experts.120.down_proj", + "model.layers.40.mlp.experts.120.gate_proj", + "model.layers.40.mlp.experts.120.up_proj", + "model.layers.40.mlp.experts.121.down_proj", + "model.layers.40.mlp.experts.121.gate_proj", + "model.layers.40.mlp.experts.121.up_proj", + "model.layers.40.mlp.experts.122.down_proj", + "model.layers.40.mlp.experts.122.gate_proj", + "model.layers.40.mlp.experts.122.up_proj", + "model.layers.40.mlp.experts.123.down_proj", + "model.layers.40.mlp.experts.123.gate_proj", + "model.layers.40.mlp.experts.123.up_proj", + "model.layers.40.mlp.experts.124.down_proj", + "model.layers.40.mlp.experts.124.gate_proj", + "model.layers.40.mlp.experts.124.up_proj", + "model.layers.40.mlp.experts.125.down_proj", + "model.layers.40.mlp.experts.125.gate_proj", + "model.layers.40.mlp.experts.125.up_proj", + "model.layers.40.mlp.experts.126.down_proj", + "model.layers.40.mlp.experts.126.gate_proj", + "model.layers.40.mlp.experts.126.up_proj", + "model.layers.40.mlp.experts.127.down_proj", + "model.layers.40.mlp.experts.127.gate_proj", + "model.layers.40.mlp.experts.127.up_proj", + "model.layers.40.mlp.experts.128.down_proj", + "model.layers.40.mlp.experts.128.gate_proj", + "model.layers.40.mlp.experts.128.up_proj", + "model.layers.40.mlp.experts.129.down_proj", + "model.layers.40.mlp.experts.129.gate_proj", + "model.layers.40.mlp.experts.129.up_proj", + "model.layers.40.mlp.experts.13.down_proj", + "model.layers.40.mlp.experts.13.gate_proj", + "model.layers.40.mlp.experts.13.up_proj", + "model.layers.40.mlp.experts.130.down_proj", + "model.layers.40.mlp.experts.130.gate_proj", + "model.layers.40.mlp.experts.130.up_proj", + "model.layers.40.mlp.experts.131.down_proj", + "model.layers.40.mlp.experts.131.gate_proj", + "model.layers.40.mlp.experts.131.up_proj", + "model.layers.40.mlp.experts.132.down_proj", + "model.layers.40.mlp.experts.132.gate_proj", + "model.layers.40.mlp.experts.132.up_proj", + "model.layers.40.mlp.experts.133.down_proj", + "model.layers.40.mlp.experts.133.gate_proj", + "model.layers.40.mlp.experts.133.up_proj", + "model.layers.40.mlp.experts.134.down_proj", + "model.layers.40.mlp.experts.134.gate_proj", + "model.layers.40.mlp.experts.134.up_proj", + "model.layers.40.mlp.experts.135.down_proj", + "model.layers.40.mlp.experts.135.gate_proj", + "model.layers.40.mlp.experts.135.up_proj", + "model.layers.40.mlp.experts.136.down_proj", + "model.layers.40.mlp.experts.136.gate_proj", + "model.layers.40.mlp.experts.136.up_proj", + "model.layers.40.mlp.experts.137.down_proj", + "model.layers.40.mlp.experts.137.gate_proj", + "model.layers.40.mlp.experts.137.up_proj", + "model.layers.40.mlp.experts.138.down_proj", + "model.layers.40.mlp.experts.138.gate_proj", + "model.layers.40.mlp.experts.138.up_proj", + "model.layers.40.mlp.experts.139.down_proj", + "model.layers.40.mlp.experts.139.gate_proj", + "model.layers.40.mlp.experts.139.up_proj", + "model.layers.40.mlp.experts.14.down_proj", + "model.layers.40.mlp.experts.14.gate_proj", + "model.layers.40.mlp.experts.14.up_proj", + "model.layers.40.mlp.experts.140.down_proj", + "model.layers.40.mlp.experts.140.gate_proj", + "model.layers.40.mlp.experts.140.up_proj", + "model.layers.40.mlp.experts.141.down_proj", + "model.layers.40.mlp.experts.141.gate_proj", + "model.layers.40.mlp.experts.141.up_proj", + "model.layers.40.mlp.experts.142.down_proj", + "model.layers.40.mlp.experts.142.gate_proj", + "model.layers.40.mlp.experts.142.up_proj", + "model.layers.40.mlp.experts.143.down_proj", + "model.layers.40.mlp.experts.143.gate_proj", + "model.layers.40.mlp.experts.143.up_proj", + "model.layers.40.mlp.experts.144.down_proj", + "model.layers.40.mlp.experts.144.gate_proj", + "model.layers.40.mlp.experts.144.up_proj", + "model.layers.40.mlp.experts.145.down_proj", + "model.layers.40.mlp.experts.145.gate_proj", + "model.layers.40.mlp.experts.145.up_proj", + "model.layers.40.mlp.experts.146.down_proj", + "model.layers.40.mlp.experts.146.gate_proj", + "model.layers.40.mlp.experts.146.up_proj", + "model.layers.40.mlp.experts.147.down_proj", + "model.layers.40.mlp.experts.147.gate_proj", + "model.layers.40.mlp.experts.147.up_proj", + "model.layers.40.mlp.experts.148.down_proj", + "model.layers.40.mlp.experts.148.gate_proj", + "model.layers.40.mlp.experts.148.up_proj", + "model.layers.40.mlp.experts.149.down_proj", + "model.layers.40.mlp.experts.149.gate_proj", + "model.layers.40.mlp.experts.149.up_proj", + "model.layers.40.mlp.experts.15.down_proj", + "model.layers.40.mlp.experts.15.gate_proj", + "model.layers.40.mlp.experts.15.up_proj", + "model.layers.40.mlp.experts.150.down_proj", + "model.layers.40.mlp.experts.150.gate_proj", + "model.layers.40.mlp.experts.150.up_proj", + "model.layers.40.mlp.experts.151.down_proj", + "model.layers.40.mlp.experts.151.gate_proj", + "model.layers.40.mlp.experts.151.up_proj", + "model.layers.40.mlp.experts.152.down_proj", + "model.layers.40.mlp.experts.152.gate_proj", + "model.layers.40.mlp.experts.152.up_proj", + "model.layers.40.mlp.experts.153.down_proj", + "model.layers.40.mlp.experts.153.gate_proj", + "model.layers.40.mlp.experts.153.up_proj", + "model.layers.40.mlp.experts.154.down_proj", + "model.layers.40.mlp.experts.154.gate_proj", + "model.layers.40.mlp.experts.154.up_proj", + "model.layers.40.mlp.experts.155.down_proj", + "model.layers.40.mlp.experts.155.gate_proj", + "model.layers.40.mlp.experts.155.up_proj", + "model.layers.40.mlp.experts.156.down_proj", + "model.layers.40.mlp.experts.156.gate_proj", + "model.layers.40.mlp.experts.156.up_proj", + "model.layers.40.mlp.experts.157.down_proj", + "model.layers.40.mlp.experts.157.gate_proj", + "model.layers.40.mlp.experts.157.up_proj", + "model.layers.40.mlp.experts.158.down_proj", + "model.layers.40.mlp.experts.158.gate_proj", + "model.layers.40.mlp.experts.158.up_proj", + "model.layers.40.mlp.experts.159.down_proj", + "model.layers.40.mlp.experts.159.gate_proj", + "model.layers.40.mlp.experts.159.up_proj", + "model.layers.40.mlp.experts.16.down_proj", + "model.layers.40.mlp.experts.16.gate_proj", + "model.layers.40.mlp.experts.16.up_proj", + "model.layers.40.mlp.experts.160.down_proj", + "model.layers.40.mlp.experts.160.gate_proj", + "model.layers.40.mlp.experts.160.up_proj", + "model.layers.40.mlp.experts.161.down_proj", + "model.layers.40.mlp.experts.161.gate_proj", + "model.layers.40.mlp.experts.161.up_proj", + "model.layers.40.mlp.experts.162.down_proj", + "model.layers.40.mlp.experts.162.gate_proj", + "model.layers.40.mlp.experts.162.up_proj", + "model.layers.40.mlp.experts.163.down_proj", + "model.layers.40.mlp.experts.163.gate_proj", + "model.layers.40.mlp.experts.163.up_proj", + "model.layers.40.mlp.experts.164.down_proj", + "model.layers.40.mlp.experts.164.gate_proj", + "model.layers.40.mlp.experts.164.up_proj", + "model.layers.40.mlp.experts.165.down_proj", + "model.layers.40.mlp.experts.165.gate_proj", + "model.layers.40.mlp.experts.165.up_proj", + "model.layers.40.mlp.experts.166.down_proj", + "model.layers.40.mlp.experts.166.gate_proj", + "model.layers.40.mlp.experts.166.up_proj", + "model.layers.40.mlp.experts.167.down_proj", + "model.layers.40.mlp.experts.167.gate_proj", + "model.layers.40.mlp.experts.167.up_proj", + "model.layers.40.mlp.experts.168.down_proj", + "model.layers.40.mlp.experts.168.gate_proj", + "model.layers.40.mlp.experts.168.up_proj", + "model.layers.40.mlp.experts.169.down_proj", + "model.layers.40.mlp.experts.169.gate_proj", + "model.layers.40.mlp.experts.169.up_proj", + "model.layers.40.mlp.experts.17.down_proj", + "model.layers.40.mlp.experts.17.gate_proj", + "model.layers.40.mlp.experts.17.up_proj", + "model.layers.40.mlp.experts.170.down_proj", + "model.layers.40.mlp.experts.170.gate_proj", + "model.layers.40.mlp.experts.170.up_proj", + "model.layers.40.mlp.experts.171.down_proj", + "model.layers.40.mlp.experts.171.gate_proj", + "model.layers.40.mlp.experts.171.up_proj", + "model.layers.40.mlp.experts.172.down_proj", + "model.layers.40.mlp.experts.172.gate_proj", + "model.layers.40.mlp.experts.172.up_proj", + "model.layers.40.mlp.experts.173.down_proj", + "model.layers.40.mlp.experts.173.gate_proj", + "model.layers.40.mlp.experts.173.up_proj", + "model.layers.40.mlp.experts.174.down_proj", + "model.layers.40.mlp.experts.174.gate_proj", + "model.layers.40.mlp.experts.174.up_proj", + "model.layers.40.mlp.experts.175.down_proj", + "model.layers.40.mlp.experts.175.gate_proj", + "model.layers.40.mlp.experts.175.up_proj", + "model.layers.40.mlp.experts.176.down_proj", + "model.layers.40.mlp.experts.176.gate_proj", + "model.layers.40.mlp.experts.176.up_proj", + "model.layers.40.mlp.experts.177.down_proj", + "model.layers.40.mlp.experts.177.gate_proj", + "model.layers.40.mlp.experts.177.up_proj", + "model.layers.40.mlp.experts.178.down_proj", + "model.layers.40.mlp.experts.178.gate_proj", + "model.layers.40.mlp.experts.178.up_proj", + "model.layers.40.mlp.experts.179.down_proj", + "model.layers.40.mlp.experts.179.gate_proj", + "model.layers.40.mlp.experts.179.up_proj", + "model.layers.40.mlp.experts.18.down_proj", + "model.layers.40.mlp.experts.18.gate_proj", + "model.layers.40.mlp.experts.18.up_proj", + "model.layers.40.mlp.experts.180.down_proj", + "model.layers.40.mlp.experts.180.gate_proj", + "model.layers.40.mlp.experts.180.up_proj", + "model.layers.40.mlp.experts.181.down_proj", + "model.layers.40.mlp.experts.181.gate_proj", + "model.layers.40.mlp.experts.181.up_proj", + "model.layers.40.mlp.experts.182.down_proj", + "model.layers.40.mlp.experts.182.gate_proj", + "model.layers.40.mlp.experts.182.up_proj", + "model.layers.40.mlp.experts.183.down_proj", + "model.layers.40.mlp.experts.183.gate_proj", + "model.layers.40.mlp.experts.183.up_proj", + "model.layers.40.mlp.experts.184.down_proj", + "model.layers.40.mlp.experts.184.gate_proj", + "model.layers.40.mlp.experts.184.up_proj", + "model.layers.40.mlp.experts.185.down_proj", + "model.layers.40.mlp.experts.185.gate_proj", + "model.layers.40.mlp.experts.185.up_proj", + "model.layers.40.mlp.experts.186.down_proj", + "model.layers.40.mlp.experts.186.gate_proj", + "model.layers.40.mlp.experts.186.up_proj", + "model.layers.40.mlp.experts.187.down_proj", + "model.layers.40.mlp.experts.187.gate_proj", + "model.layers.40.mlp.experts.187.up_proj", + "model.layers.40.mlp.experts.188.down_proj", + "model.layers.40.mlp.experts.188.gate_proj", + "model.layers.40.mlp.experts.188.up_proj", + "model.layers.40.mlp.experts.189.down_proj", + "model.layers.40.mlp.experts.189.gate_proj", + "model.layers.40.mlp.experts.189.up_proj", + "model.layers.40.mlp.experts.19.down_proj", + "model.layers.40.mlp.experts.19.gate_proj", + "model.layers.40.mlp.experts.19.up_proj", + "model.layers.40.mlp.experts.190.down_proj", + "model.layers.40.mlp.experts.190.gate_proj", + "model.layers.40.mlp.experts.190.up_proj", + "model.layers.40.mlp.experts.191.down_proj", + "model.layers.40.mlp.experts.191.gate_proj", + "model.layers.40.mlp.experts.191.up_proj", + "model.layers.40.mlp.experts.192.down_proj", + "model.layers.40.mlp.experts.192.gate_proj", + "model.layers.40.mlp.experts.192.up_proj", + "model.layers.40.mlp.experts.193.down_proj", + "model.layers.40.mlp.experts.193.gate_proj", + "model.layers.40.mlp.experts.193.up_proj", + "model.layers.40.mlp.experts.194.down_proj", + "model.layers.40.mlp.experts.194.gate_proj", + "model.layers.40.mlp.experts.194.up_proj", + "model.layers.40.mlp.experts.195.down_proj", + "model.layers.40.mlp.experts.195.gate_proj", + "model.layers.40.mlp.experts.195.up_proj", + "model.layers.40.mlp.experts.196.down_proj", + "model.layers.40.mlp.experts.196.gate_proj", + "model.layers.40.mlp.experts.196.up_proj", + "model.layers.40.mlp.experts.197.down_proj", + "model.layers.40.mlp.experts.197.gate_proj", + "model.layers.40.mlp.experts.197.up_proj", + "model.layers.40.mlp.experts.198.down_proj", + "model.layers.40.mlp.experts.198.gate_proj", + "model.layers.40.mlp.experts.198.up_proj", + "model.layers.40.mlp.experts.199.down_proj", + "model.layers.40.mlp.experts.199.gate_proj", + "model.layers.40.mlp.experts.199.up_proj", + "model.layers.40.mlp.experts.2.down_proj", + "model.layers.40.mlp.experts.2.gate_proj", + "model.layers.40.mlp.experts.2.up_proj", + "model.layers.40.mlp.experts.20.down_proj", + "model.layers.40.mlp.experts.20.gate_proj", + "model.layers.40.mlp.experts.20.up_proj", + "model.layers.40.mlp.experts.200.down_proj", + "model.layers.40.mlp.experts.200.gate_proj", + "model.layers.40.mlp.experts.200.up_proj", + "model.layers.40.mlp.experts.201.down_proj", + "model.layers.40.mlp.experts.201.gate_proj", + "model.layers.40.mlp.experts.201.up_proj", + "model.layers.40.mlp.experts.202.down_proj", + "model.layers.40.mlp.experts.202.gate_proj", + "model.layers.40.mlp.experts.202.up_proj", + "model.layers.40.mlp.experts.203.down_proj", + "model.layers.40.mlp.experts.203.gate_proj", + "model.layers.40.mlp.experts.203.up_proj", + "model.layers.40.mlp.experts.204.down_proj", + "model.layers.40.mlp.experts.204.gate_proj", + "model.layers.40.mlp.experts.204.up_proj", + "model.layers.40.mlp.experts.205.down_proj", + "model.layers.40.mlp.experts.205.gate_proj", + "model.layers.40.mlp.experts.205.up_proj", + "model.layers.40.mlp.experts.206.down_proj", + "model.layers.40.mlp.experts.206.gate_proj", + "model.layers.40.mlp.experts.206.up_proj", + "model.layers.40.mlp.experts.207.down_proj", + "model.layers.40.mlp.experts.207.gate_proj", + "model.layers.40.mlp.experts.207.up_proj", + "model.layers.40.mlp.experts.208.down_proj", + "model.layers.40.mlp.experts.208.gate_proj", + "model.layers.40.mlp.experts.208.up_proj", + "model.layers.40.mlp.experts.209.down_proj", + "model.layers.40.mlp.experts.209.gate_proj", + "model.layers.40.mlp.experts.209.up_proj", + "model.layers.40.mlp.experts.21.down_proj", + "model.layers.40.mlp.experts.21.gate_proj", + "model.layers.40.mlp.experts.21.up_proj", + "model.layers.40.mlp.experts.210.down_proj", + "model.layers.40.mlp.experts.210.gate_proj", + "model.layers.40.mlp.experts.210.up_proj", + "model.layers.40.mlp.experts.211.down_proj", + "model.layers.40.mlp.experts.211.gate_proj", + "model.layers.40.mlp.experts.211.up_proj", + "model.layers.40.mlp.experts.212.down_proj", + "model.layers.40.mlp.experts.212.gate_proj", + "model.layers.40.mlp.experts.212.up_proj", + "model.layers.40.mlp.experts.213.down_proj", + "model.layers.40.mlp.experts.213.gate_proj", + "model.layers.40.mlp.experts.213.up_proj", + "model.layers.40.mlp.experts.214.down_proj", + "model.layers.40.mlp.experts.214.gate_proj", + "model.layers.40.mlp.experts.214.up_proj", + "model.layers.40.mlp.experts.215.down_proj", + "model.layers.40.mlp.experts.215.gate_proj", + "model.layers.40.mlp.experts.215.up_proj", + "model.layers.40.mlp.experts.216.down_proj", + "model.layers.40.mlp.experts.216.gate_proj", + "model.layers.40.mlp.experts.216.up_proj", + "model.layers.40.mlp.experts.217.down_proj", + "model.layers.40.mlp.experts.217.gate_proj", + "model.layers.40.mlp.experts.217.up_proj", + "model.layers.40.mlp.experts.218.down_proj", + "model.layers.40.mlp.experts.218.gate_proj", + "model.layers.40.mlp.experts.218.up_proj", + "model.layers.40.mlp.experts.219.down_proj", + "model.layers.40.mlp.experts.219.gate_proj", + "model.layers.40.mlp.experts.219.up_proj", + "model.layers.40.mlp.experts.22.down_proj", + "model.layers.40.mlp.experts.22.gate_proj", + "model.layers.40.mlp.experts.22.up_proj", + "model.layers.40.mlp.experts.220.down_proj", + "model.layers.40.mlp.experts.220.gate_proj", + "model.layers.40.mlp.experts.220.up_proj", + "model.layers.40.mlp.experts.221.down_proj", + "model.layers.40.mlp.experts.221.gate_proj", + "model.layers.40.mlp.experts.221.up_proj", + "model.layers.40.mlp.experts.222.down_proj", + "model.layers.40.mlp.experts.222.gate_proj", + "model.layers.40.mlp.experts.222.up_proj", + "model.layers.40.mlp.experts.223.down_proj", + "model.layers.40.mlp.experts.223.gate_proj", + "model.layers.40.mlp.experts.223.up_proj", + "model.layers.40.mlp.experts.224.down_proj", + "model.layers.40.mlp.experts.224.gate_proj", + "model.layers.40.mlp.experts.224.up_proj", + "model.layers.40.mlp.experts.225.down_proj", + "model.layers.40.mlp.experts.225.gate_proj", + "model.layers.40.mlp.experts.225.up_proj", + "model.layers.40.mlp.experts.226.down_proj", + "model.layers.40.mlp.experts.226.gate_proj", + "model.layers.40.mlp.experts.226.up_proj", + "model.layers.40.mlp.experts.227.down_proj", + "model.layers.40.mlp.experts.227.gate_proj", + "model.layers.40.mlp.experts.227.up_proj", + "model.layers.40.mlp.experts.228.down_proj", + "model.layers.40.mlp.experts.228.gate_proj", + "model.layers.40.mlp.experts.228.up_proj", + "model.layers.40.mlp.experts.229.down_proj", + "model.layers.40.mlp.experts.229.gate_proj", + "model.layers.40.mlp.experts.229.up_proj", + "model.layers.40.mlp.experts.23.down_proj", + "model.layers.40.mlp.experts.23.gate_proj", + "model.layers.40.mlp.experts.23.up_proj", + "model.layers.40.mlp.experts.230.down_proj", + "model.layers.40.mlp.experts.230.gate_proj", + "model.layers.40.mlp.experts.230.up_proj", + "model.layers.40.mlp.experts.231.down_proj", + "model.layers.40.mlp.experts.231.gate_proj", + "model.layers.40.mlp.experts.231.up_proj", + "model.layers.40.mlp.experts.232.down_proj", + "model.layers.40.mlp.experts.232.gate_proj", + "model.layers.40.mlp.experts.232.up_proj", + "model.layers.40.mlp.experts.233.down_proj", + "model.layers.40.mlp.experts.233.gate_proj", + "model.layers.40.mlp.experts.233.up_proj", + "model.layers.40.mlp.experts.234.down_proj", + "model.layers.40.mlp.experts.234.gate_proj", + "model.layers.40.mlp.experts.234.up_proj", + "model.layers.40.mlp.experts.235.down_proj", + "model.layers.40.mlp.experts.235.gate_proj", + "model.layers.40.mlp.experts.235.up_proj", + "model.layers.40.mlp.experts.236.down_proj", + "model.layers.40.mlp.experts.236.gate_proj", + "model.layers.40.mlp.experts.236.up_proj", + "model.layers.40.mlp.experts.237.down_proj", + "model.layers.40.mlp.experts.237.gate_proj", + "model.layers.40.mlp.experts.237.up_proj", + "model.layers.40.mlp.experts.238.down_proj", + "model.layers.40.mlp.experts.238.gate_proj", + "model.layers.40.mlp.experts.238.up_proj", + "model.layers.40.mlp.experts.239.down_proj", + "model.layers.40.mlp.experts.239.gate_proj", + "model.layers.40.mlp.experts.239.up_proj", + "model.layers.40.mlp.experts.24.down_proj", + "model.layers.40.mlp.experts.24.gate_proj", + "model.layers.40.mlp.experts.24.up_proj", + "model.layers.40.mlp.experts.240.down_proj", + "model.layers.40.mlp.experts.240.gate_proj", + "model.layers.40.mlp.experts.240.up_proj", + "model.layers.40.mlp.experts.241.down_proj", + "model.layers.40.mlp.experts.241.gate_proj", + "model.layers.40.mlp.experts.241.up_proj", + "model.layers.40.mlp.experts.242.down_proj", + "model.layers.40.mlp.experts.242.gate_proj", + "model.layers.40.mlp.experts.242.up_proj", + "model.layers.40.mlp.experts.243.down_proj", + "model.layers.40.mlp.experts.243.gate_proj", + "model.layers.40.mlp.experts.243.up_proj", + "model.layers.40.mlp.experts.244.down_proj", + "model.layers.40.mlp.experts.244.gate_proj", + "model.layers.40.mlp.experts.244.up_proj", + "model.layers.40.mlp.experts.245.down_proj", + "model.layers.40.mlp.experts.245.gate_proj", + "model.layers.40.mlp.experts.245.up_proj", + "model.layers.40.mlp.experts.246.down_proj", + "model.layers.40.mlp.experts.246.gate_proj", + "model.layers.40.mlp.experts.246.up_proj", + "model.layers.40.mlp.experts.247.down_proj", + "model.layers.40.mlp.experts.247.gate_proj", + "model.layers.40.mlp.experts.247.up_proj", + "model.layers.40.mlp.experts.248.down_proj", + "model.layers.40.mlp.experts.248.gate_proj", + "model.layers.40.mlp.experts.248.up_proj", + "model.layers.40.mlp.experts.249.down_proj", + "model.layers.40.mlp.experts.249.gate_proj", + "model.layers.40.mlp.experts.249.up_proj", + "model.layers.40.mlp.experts.25.down_proj", + "model.layers.40.mlp.experts.25.gate_proj", + "model.layers.40.mlp.experts.25.up_proj", + "model.layers.40.mlp.experts.250.down_proj", + "model.layers.40.mlp.experts.250.gate_proj", + "model.layers.40.mlp.experts.250.up_proj", + "model.layers.40.mlp.experts.251.down_proj", + "model.layers.40.mlp.experts.251.gate_proj", + "model.layers.40.mlp.experts.251.up_proj", + "model.layers.40.mlp.experts.252.down_proj", + "model.layers.40.mlp.experts.252.gate_proj", + "model.layers.40.mlp.experts.252.up_proj", + "model.layers.40.mlp.experts.253.down_proj", + "model.layers.40.mlp.experts.253.gate_proj", + "model.layers.40.mlp.experts.253.up_proj", + "model.layers.40.mlp.experts.254.down_proj", + "model.layers.40.mlp.experts.254.gate_proj", + "model.layers.40.mlp.experts.254.up_proj", + "model.layers.40.mlp.experts.255.down_proj", + "model.layers.40.mlp.experts.255.gate_proj", + "model.layers.40.mlp.experts.255.up_proj", + "model.layers.40.mlp.experts.26.down_proj", + "model.layers.40.mlp.experts.26.gate_proj", + "model.layers.40.mlp.experts.26.up_proj", + "model.layers.40.mlp.experts.27.down_proj", + "model.layers.40.mlp.experts.27.gate_proj", + "model.layers.40.mlp.experts.27.up_proj", + "model.layers.40.mlp.experts.28.down_proj", + "model.layers.40.mlp.experts.28.gate_proj", + "model.layers.40.mlp.experts.28.up_proj", + "model.layers.40.mlp.experts.29.down_proj", + "model.layers.40.mlp.experts.29.gate_proj", + "model.layers.40.mlp.experts.29.up_proj", + "model.layers.40.mlp.experts.3.down_proj", + "model.layers.40.mlp.experts.3.gate_proj", + "model.layers.40.mlp.experts.3.up_proj", + "model.layers.40.mlp.experts.30.down_proj", + "model.layers.40.mlp.experts.30.gate_proj", + "model.layers.40.mlp.experts.30.up_proj", + "model.layers.40.mlp.experts.31.down_proj", + "model.layers.40.mlp.experts.31.gate_proj", + "model.layers.40.mlp.experts.31.up_proj", + "model.layers.40.mlp.experts.32.down_proj", + "model.layers.40.mlp.experts.32.gate_proj", + "model.layers.40.mlp.experts.32.up_proj", + "model.layers.40.mlp.experts.33.down_proj", + "model.layers.40.mlp.experts.33.gate_proj", + "model.layers.40.mlp.experts.33.up_proj", + "model.layers.40.mlp.experts.34.down_proj", + "model.layers.40.mlp.experts.34.gate_proj", + "model.layers.40.mlp.experts.34.up_proj", + "model.layers.40.mlp.experts.35.down_proj", + "model.layers.40.mlp.experts.35.gate_proj", + "model.layers.40.mlp.experts.35.up_proj", + "model.layers.40.mlp.experts.36.down_proj", + "model.layers.40.mlp.experts.36.gate_proj", + "model.layers.40.mlp.experts.36.up_proj", + "model.layers.40.mlp.experts.37.down_proj", + "model.layers.40.mlp.experts.37.gate_proj", + "model.layers.40.mlp.experts.37.up_proj", + "model.layers.40.mlp.experts.38.down_proj", + "model.layers.40.mlp.experts.38.gate_proj", + "model.layers.40.mlp.experts.38.up_proj", + "model.layers.40.mlp.experts.39.down_proj", + "model.layers.40.mlp.experts.39.gate_proj", + "model.layers.40.mlp.experts.39.up_proj", + "model.layers.40.mlp.experts.4.down_proj", + "model.layers.40.mlp.experts.4.gate_proj", + "model.layers.40.mlp.experts.4.up_proj", + "model.layers.40.mlp.experts.40.down_proj", + "model.layers.40.mlp.experts.40.gate_proj", + "model.layers.40.mlp.experts.40.up_proj", + "model.layers.40.mlp.experts.41.down_proj", + "model.layers.40.mlp.experts.41.gate_proj", + "model.layers.40.mlp.experts.41.up_proj", + "model.layers.40.mlp.experts.42.down_proj", + "model.layers.40.mlp.experts.42.gate_proj", + "model.layers.40.mlp.experts.42.up_proj", + "model.layers.40.mlp.experts.43.down_proj", + "model.layers.40.mlp.experts.43.gate_proj", + "model.layers.40.mlp.experts.43.up_proj", + "model.layers.40.mlp.experts.44.down_proj", + "model.layers.40.mlp.experts.44.gate_proj", + "model.layers.40.mlp.experts.44.up_proj", + "model.layers.40.mlp.experts.45.down_proj", + "model.layers.40.mlp.experts.45.gate_proj", + "model.layers.40.mlp.experts.45.up_proj", + "model.layers.40.mlp.experts.46.down_proj", + "model.layers.40.mlp.experts.46.gate_proj", + "model.layers.40.mlp.experts.46.up_proj", + "model.layers.40.mlp.experts.47.down_proj", + "model.layers.40.mlp.experts.47.gate_proj", + "model.layers.40.mlp.experts.47.up_proj", + "model.layers.40.mlp.experts.48.down_proj", + "model.layers.40.mlp.experts.48.gate_proj", + "model.layers.40.mlp.experts.48.up_proj", + "model.layers.40.mlp.experts.49.down_proj", + "model.layers.40.mlp.experts.49.gate_proj", + "model.layers.40.mlp.experts.49.up_proj", + "model.layers.40.mlp.experts.5.down_proj", + "model.layers.40.mlp.experts.5.gate_proj", + "model.layers.40.mlp.experts.5.up_proj", + "model.layers.40.mlp.experts.50.down_proj", + "model.layers.40.mlp.experts.50.gate_proj", + "model.layers.40.mlp.experts.50.up_proj", + "model.layers.40.mlp.experts.51.down_proj", + "model.layers.40.mlp.experts.51.gate_proj", + "model.layers.40.mlp.experts.51.up_proj", + "model.layers.40.mlp.experts.52.down_proj", + "model.layers.40.mlp.experts.52.gate_proj", + "model.layers.40.mlp.experts.52.up_proj", + "model.layers.40.mlp.experts.53.down_proj", + "model.layers.40.mlp.experts.53.gate_proj", + "model.layers.40.mlp.experts.53.up_proj", + "model.layers.40.mlp.experts.54.down_proj", + "model.layers.40.mlp.experts.54.gate_proj", + "model.layers.40.mlp.experts.54.up_proj", + "model.layers.40.mlp.experts.55.down_proj", + "model.layers.40.mlp.experts.55.gate_proj", + "model.layers.40.mlp.experts.55.up_proj", + "model.layers.40.mlp.experts.56.down_proj", + "model.layers.40.mlp.experts.56.gate_proj", + "model.layers.40.mlp.experts.56.up_proj", + "model.layers.40.mlp.experts.57.down_proj", + "model.layers.40.mlp.experts.57.gate_proj", + "model.layers.40.mlp.experts.57.up_proj", + "model.layers.40.mlp.experts.58.down_proj", + "model.layers.40.mlp.experts.58.gate_proj", + "model.layers.40.mlp.experts.58.up_proj", + "model.layers.40.mlp.experts.59.down_proj", + "model.layers.40.mlp.experts.59.gate_proj", + "model.layers.40.mlp.experts.59.up_proj", + "model.layers.40.mlp.experts.6.down_proj", + "model.layers.40.mlp.experts.6.gate_proj", + "model.layers.40.mlp.experts.6.up_proj", + "model.layers.40.mlp.experts.60.down_proj", + "model.layers.40.mlp.experts.60.gate_proj", + "model.layers.40.mlp.experts.60.up_proj", + "model.layers.40.mlp.experts.61.down_proj", + "model.layers.40.mlp.experts.61.gate_proj", + "model.layers.40.mlp.experts.61.up_proj", + "model.layers.40.mlp.experts.62.down_proj", + "model.layers.40.mlp.experts.62.gate_proj", + "model.layers.40.mlp.experts.62.up_proj", + "model.layers.40.mlp.experts.63.down_proj", + "model.layers.40.mlp.experts.63.gate_proj", + "model.layers.40.mlp.experts.63.up_proj", + "model.layers.40.mlp.experts.64.down_proj", + "model.layers.40.mlp.experts.64.gate_proj", + "model.layers.40.mlp.experts.64.up_proj", + "model.layers.40.mlp.experts.65.down_proj", + "model.layers.40.mlp.experts.65.gate_proj", + "model.layers.40.mlp.experts.65.up_proj", + "model.layers.40.mlp.experts.66.down_proj", + "model.layers.40.mlp.experts.66.gate_proj", + "model.layers.40.mlp.experts.66.up_proj", + "model.layers.40.mlp.experts.67.down_proj", + "model.layers.40.mlp.experts.67.gate_proj", + "model.layers.40.mlp.experts.67.up_proj", + "model.layers.40.mlp.experts.68.down_proj", + "model.layers.40.mlp.experts.68.gate_proj", + "model.layers.40.mlp.experts.68.up_proj", + "model.layers.40.mlp.experts.69.down_proj", + "model.layers.40.mlp.experts.69.gate_proj", + "model.layers.40.mlp.experts.69.up_proj", + "model.layers.40.mlp.experts.7.down_proj", + "model.layers.40.mlp.experts.7.gate_proj", + "model.layers.40.mlp.experts.7.up_proj", + "model.layers.40.mlp.experts.70.down_proj", + "model.layers.40.mlp.experts.70.gate_proj", + "model.layers.40.mlp.experts.70.up_proj", + "model.layers.40.mlp.experts.71.down_proj", + "model.layers.40.mlp.experts.71.gate_proj", + "model.layers.40.mlp.experts.71.up_proj", + "model.layers.40.mlp.experts.72.down_proj", + "model.layers.40.mlp.experts.72.gate_proj", + "model.layers.40.mlp.experts.72.up_proj", + "model.layers.40.mlp.experts.73.down_proj", + "model.layers.40.mlp.experts.73.gate_proj", + "model.layers.40.mlp.experts.73.up_proj", + "model.layers.40.mlp.experts.74.down_proj", + "model.layers.40.mlp.experts.74.gate_proj", + "model.layers.40.mlp.experts.74.up_proj", + "model.layers.40.mlp.experts.75.down_proj", + "model.layers.40.mlp.experts.75.gate_proj", + "model.layers.40.mlp.experts.75.up_proj", + "model.layers.40.mlp.experts.76.down_proj", + "model.layers.40.mlp.experts.76.gate_proj", + "model.layers.40.mlp.experts.76.up_proj", + "model.layers.40.mlp.experts.77.down_proj", + "model.layers.40.mlp.experts.77.gate_proj", + "model.layers.40.mlp.experts.77.up_proj", + "model.layers.40.mlp.experts.78.down_proj", + "model.layers.40.mlp.experts.78.gate_proj", + "model.layers.40.mlp.experts.78.up_proj", + "model.layers.40.mlp.experts.79.down_proj", + "model.layers.40.mlp.experts.79.gate_proj", + "model.layers.40.mlp.experts.79.up_proj", + "model.layers.40.mlp.experts.8.down_proj", + "model.layers.40.mlp.experts.8.gate_proj", + "model.layers.40.mlp.experts.8.up_proj", + "model.layers.40.mlp.experts.80.down_proj", + "model.layers.40.mlp.experts.80.gate_proj", + "model.layers.40.mlp.experts.80.up_proj", + "model.layers.40.mlp.experts.81.down_proj", + "model.layers.40.mlp.experts.81.gate_proj", + "model.layers.40.mlp.experts.81.up_proj", + "model.layers.40.mlp.experts.82.down_proj", + "model.layers.40.mlp.experts.82.gate_proj", + "model.layers.40.mlp.experts.82.up_proj", + "model.layers.40.mlp.experts.83.down_proj", + "model.layers.40.mlp.experts.83.gate_proj", + "model.layers.40.mlp.experts.83.up_proj", + "model.layers.40.mlp.experts.84.down_proj", + "model.layers.40.mlp.experts.84.gate_proj", + "model.layers.40.mlp.experts.84.up_proj", + "model.layers.40.mlp.experts.85.down_proj", + "model.layers.40.mlp.experts.85.gate_proj", + "model.layers.40.mlp.experts.85.up_proj", + "model.layers.40.mlp.experts.86.down_proj", + "model.layers.40.mlp.experts.86.gate_proj", + "model.layers.40.mlp.experts.86.up_proj", + "model.layers.40.mlp.experts.87.down_proj", + "model.layers.40.mlp.experts.87.gate_proj", + "model.layers.40.mlp.experts.87.up_proj", + "model.layers.40.mlp.experts.88.down_proj", + "model.layers.40.mlp.experts.88.gate_proj", + "model.layers.40.mlp.experts.88.up_proj", + "model.layers.40.mlp.experts.89.down_proj", + "model.layers.40.mlp.experts.89.gate_proj", + "model.layers.40.mlp.experts.89.up_proj", + "model.layers.40.mlp.experts.9.down_proj", + "model.layers.40.mlp.experts.9.gate_proj", + "model.layers.40.mlp.experts.9.up_proj", + "model.layers.40.mlp.experts.90.down_proj", + "model.layers.40.mlp.experts.90.gate_proj", + "model.layers.40.mlp.experts.90.up_proj", + "model.layers.40.mlp.experts.91.down_proj", + "model.layers.40.mlp.experts.91.gate_proj", + "model.layers.40.mlp.experts.91.up_proj", + "model.layers.40.mlp.experts.92.down_proj", + "model.layers.40.mlp.experts.92.gate_proj", + "model.layers.40.mlp.experts.92.up_proj", + "model.layers.40.mlp.experts.93.down_proj", + "model.layers.40.mlp.experts.93.gate_proj", + "model.layers.40.mlp.experts.93.up_proj", + "model.layers.40.mlp.experts.94.down_proj", + "model.layers.40.mlp.experts.94.gate_proj", + "model.layers.40.mlp.experts.94.up_proj", + "model.layers.40.mlp.experts.95.down_proj", + "model.layers.40.mlp.experts.95.gate_proj", + "model.layers.40.mlp.experts.95.up_proj", + "model.layers.40.mlp.experts.96.down_proj", + "model.layers.40.mlp.experts.96.gate_proj", + "model.layers.40.mlp.experts.96.up_proj", + "model.layers.40.mlp.experts.97.down_proj", + "model.layers.40.mlp.experts.97.gate_proj", + "model.layers.40.mlp.experts.97.up_proj", + "model.layers.40.mlp.experts.98.down_proj", + "model.layers.40.mlp.experts.98.gate_proj", + "model.layers.40.mlp.experts.98.up_proj", + "model.layers.40.mlp.experts.99.down_proj", + "model.layers.40.mlp.experts.99.gate_proj", + "model.layers.40.mlp.experts.99.up_proj", + "model.layers.41.mlp.experts.0.down_proj", + "model.layers.41.mlp.experts.0.gate_proj", + "model.layers.41.mlp.experts.0.up_proj", + "model.layers.41.mlp.experts.1.down_proj", + "model.layers.41.mlp.experts.1.gate_proj", + "model.layers.41.mlp.experts.1.up_proj", + "model.layers.41.mlp.experts.10.down_proj", + "model.layers.41.mlp.experts.10.gate_proj", + "model.layers.41.mlp.experts.10.up_proj", + "model.layers.41.mlp.experts.100.down_proj", + "model.layers.41.mlp.experts.100.gate_proj", + "model.layers.41.mlp.experts.100.up_proj", + "model.layers.41.mlp.experts.101.down_proj", + "model.layers.41.mlp.experts.101.gate_proj", + "model.layers.41.mlp.experts.101.up_proj", + "model.layers.41.mlp.experts.102.down_proj", + "model.layers.41.mlp.experts.102.gate_proj", + "model.layers.41.mlp.experts.102.up_proj", + "model.layers.41.mlp.experts.103.down_proj", + "model.layers.41.mlp.experts.103.gate_proj", + "model.layers.41.mlp.experts.103.up_proj", + "model.layers.41.mlp.experts.104.down_proj", + "model.layers.41.mlp.experts.104.gate_proj", + "model.layers.41.mlp.experts.104.up_proj", + "model.layers.41.mlp.experts.105.down_proj", + "model.layers.41.mlp.experts.105.gate_proj", + "model.layers.41.mlp.experts.105.up_proj", + "model.layers.41.mlp.experts.106.down_proj", + "model.layers.41.mlp.experts.106.gate_proj", + "model.layers.41.mlp.experts.106.up_proj", + "model.layers.41.mlp.experts.107.down_proj", + "model.layers.41.mlp.experts.107.gate_proj", + "model.layers.41.mlp.experts.107.up_proj", + "model.layers.41.mlp.experts.108.down_proj", + "model.layers.41.mlp.experts.108.gate_proj", + "model.layers.41.mlp.experts.108.up_proj", + "model.layers.41.mlp.experts.109.down_proj", + "model.layers.41.mlp.experts.109.gate_proj", + "model.layers.41.mlp.experts.109.up_proj", + "model.layers.41.mlp.experts.11.down_proj", + "model.layers.41.mlp.experts.11.gate_proj", + "model.layers.41.mlp.experts.11.up_proj", + "model.layers.41.mlp.experts.110.down_proj", + "model.layers.41.mlp.experts.110.gate_proj", + "model.layers.41.mlp.experts.110.up_proj", + "model.layers.41.mlp.experts.111.down_proj", + "model.layers.41.mlp.experts.111.gate_proj", + "model.layers.41.mlp.experts.111.up_proj", + "model.layers.41.mlp.experts.112.down_proj", + "model.layers.41.mlp.experts.112.gate_proj", + "model.layers.41.mlp.experts.112.up_proj", + "model.layers.41.mlp.experts.113.down_proj", + "model.layers.41.mlp.experts.113.gate_proj", + "model.layers.41.mlp.experts.113.up_proj", + "model.layers.41.mlp.experts.114.down_proj", + "model.layers.41.mlp.experts.114.gate_proj", + "model.layers.41.mlp.experts.114.up_proj", + "model.layers.41.mlp.experts.115.down_proj", + "model.layers.41.mlp.experts.115.gate_proj", + "model.layers.41.mlp.experts.115.up_proj", + "model.layers.41.mlp.experts.116.down_proj", + "model.layers.41.mlp.experts.116.gate_proj", + "model.layers.41.mlp.experts.116.up_proj", + "model.layers.41.mlp.experts.117.down_proj", + "model.layers.41.mlp.experts.117.gate_proj", + "model.layers.41.mlp.experts.117.up_proj", + "model.layers.41.mlp.experts.118.down_proj", + "model.layers.41.mlp.experts.118.gate_proj", + "model.layers.41.mlp.experts.118.up_proj", + "model.layers.41.mlp.experts.119.down_proj", + "model.layers.41.mlp.experts.119.gate_proj", + "model.layers.41.mlp.experts.119.up_proj", + "model.layers.41.mlp.experts.12.down_proj", + "model.layers.41.mlp.experts.12.gate_proj", + "model.layers.41.mlp.experts.12.up_proj", + "model.layers.41.mlp.experts.120.down_proj", + "model.layers.41.mlp.experts.120.gate_proj", + "model.layers.41.mlp.experts.120.up_proj", + "model.layers.41.mlp.experts.121.down_proj", + "model.layers.41.mlp.experts.121.gate_proj", + "model.layers.41.mlp.experts.121.up_proj", + "model.layers.41.mlp.experts.122.down_proj", + "model.layers.41.mlp.experts.122.gate_proj", + "model.layers.41.mlp.experts.122.up_proj", + "model.layers.41.mlp.experts.123.down_proj", + "model.layers.41.mlp.experts.123.gate_proj", + "model.layers.41.mlp.experts.123.up_proj", + "model.layers.41.mlp.experts.124.down_proj", + "model.layers.41.mlp.experts.124.gate_proj", + "model.layers.41.mlp.experts.124.up_proj", + "model.layers.41.mlp.experts.125.down_proj", + "model.layers.41.mlp.experts.125.gate_proj", + "model.layers.41.mlp.experts.125.up_proj", + "model.layers.41.mlp.experts.126.down_proj", + "model.layers.41.mlp.experts.126.gate_proj", + "model.layers.41.mlp.experts.126.up_proj", + "model.layers.41.mlp.experts.127.down_proj", + "model.layers.41.mlp.experts.127.gate_proj", + "model.layers.41.mlp.experts.127.up_proj", + "model.layers.41.mlp.experts.128.down_proj", + "model.layers.41.mlp.experts.128.gate_proj", + "model.layers.41.mlp.experts.128.up_proj", + "model.layers.41.mlp.experts.129.down_proj", + "model.layers.41.mlp.experts.129.gate_proj", + "model.layers.41.mlp.experts.129.up_proj", + "model.layers.41.mlp.experts.13.down_proj", + "model.layers.41.mlp.experts.13.gate_proj", + "model.layers.41.mlp.experts.13.up_proj", + "model.layers.41.mlp.experts.130.down_proj", + "model.layers.41.mlp.experts.130.gate_proj", + "model.layers.41.mlp.experts.130.up_proj", + "model.layers.41.mlp.experts.131.down_proj", + "model.layers.41.mlp.experts.131.gate_proj", + "model.layers.41.mlp.experts.131.up_proj", + "model.layers.41.mlp.experts.132.down_proj", + "model.layers.41.mlp.experts.132.gate_proj", + "model.layers.41.mlp.experts.132.up_proj", + "model.layers.41.mlp.experts.133.down_proj", + "model.layers.41.mlp.experts.133.gate_proj", + "model.layers.41.mlp.experts.133.up_proj", + "model.layers.41.mlp.experts.134.down_proj", + "model.layers.41.mlp.experts.134.gate_proj", + "model.layers.41.mlp.experts.134.up_proj", + "model.layers.41.mlp.experts.135.down_proj", + "model.layers.41.mlp.experts.135.gate_proj", + "model.layers.41.mlp.experts.135.up_proj", + "model.layers.41.mlp.experts.136.down_proj", + "model.layers.41.mlp.experts.136.gate_proj", + "model.layers.41.mlp.experts.136.up_proj", + "model.layers.41.mlp.experts.137.down_proj", + "model.layers.41.mlp.experts.137.gate_proj", + "model.layers.41.mlp.experts.137.up_proj", + "model.layers.41.mlp.experts.138.down_proj", + "model.layers.41.mlp.experts.138.gate_proj", + "model.layers.41.mlp.experts.138.up_proj", + "model.layers.41.mlp.experts.139.down_proj", + "model.layers.41.mlp.experts.139.gate_proj", + "model.layers.41.mlp.experts.139.up_proj", + "model.layers.41.mlp.experts.14.down_proj", + "model.layers.41.mlp.experts.14.gate_proj", + "model.layers.41.mlp.experts.14.up_proj", + "model.layers.41.mlp.experts.140.down_proj", + "model.layers.41.mlp.experts.140.gate_proj", + "model.layers.41.mlp.experts.140.up_proj", + "model.layers.41.mlp.experts.141.down_proj", + "model.layers.41.mlp.experts.141.gate_proj", + "model.layers.41.mlp.experts.141.up_proj", + "model.layers.41.mlp.experts.142.down_proj", + "model.layers.41.mlp.experts.142.gate_proj", + "model.layers.41.mlp.experts.142.up_proj", + "model.layers.41.mlp.experts.143.down_proj", + "model.layers.41.mlp.experts.143.gate_proj", + "model.layers.41.mlp.experts.143.up_proj", + "model.layers.41.mlp.experts.144.down_proj", + "model.layers.41.mlp.experts.144.gate_proj", + "model.layers.41.mlp.experts.144.up_proj", + "model.layers.41.mlp.experts.145.down_proj", + "model.layers.41.mlp.experts.145.gate_proj", + "model.layers.41.mlp.experts.145.up_proj", + "model.layers.41.mlp.experts.146.down_proj", + "model.layers.41.mlp.experts.146.gate_proj", + "model.layers.41.mlp.experts.146.up_proj", + "model.layers.41.mlp.experts.147.down_proj", + "model.layers.41.mlp.experts.147.gate_proj", + "model.layers.41.mlp.experts.147.up_proj", + "model.layers.41.mlp.experts.148.down_proj", + "model.layers.41.mlp.experts.148.gate_proj", + "model.layers.41.mlp.experts.148.up_proj", + "model.layers.41.mlp.experts.149.down_proj", + "model.layers.41.mlp.experts.149.gate_proj", + "model.layers.41.mlp.experts.149.up_proj", + "model.layers.41.mlp.experts.15.down_proj", + "model.layers.41.mlp.experts.15.gate_proj", + "model.layers.41.mlp.experts.15.up_proj", + "model.layers.41.mlp.experts.150.down_proj", + "model.layers.41.mlp.experts.150.gate_proj", + "model.layers.41.mlp.experts.150.up_proj", + "model.layers.41.mlp.experts.151.down_proj", + "model.layers.41.mlp.experts.151.gate_proj", + "model.layers.41.mlp.experts.151.up_proj", + "model.layers.41.mlp.experts.152.down_proj", + "model.layers.41.mlp.experts.152.gate_proj", + "model.layers.41.mlp.experts.152.up_proj", + "model.layers.41.mlp.experts.153.down_proj", + "model.layers.41.mlp.experts.153.gate_proj", + "model.layers.41.mlp.experts.153.up_proj", + "model.layers.41.mlp.experts.154.down_proj", + "model.layers.41.mlp.experts.154.gate_proj", + "model.layers.41.mlp.experts.154.up_proj", + "model.layers.41.mlp.experts.155.down_proj", + "model.layers.41.mlp.experts.155.gate_proj", + "model.layers.41.mlp.experts.155.up_proj", + "model.layers.41.mlp.experts.156.down_proj", + "model.layers.41.mlp.experts.156.gate_proj", + "model.layers.41.mlp.experts.156.up_proj", + "model.layers.41.mlp.experts.157.down_proj", + "model.layers.41.mlp.experts.157.gate_proj", + "model.layers.41.mlp.experts.157.up_proj", + "model.layers.41.mlp.experts.158.down_proj", + "model.layers.41.mlp.experts.158.gate_proj", + "model.layers.41.mlp.experts.158.up_proj", + "model.layers.41.mlp.experts.159.down_proj", + "model.layers.41.mlp.experts.159.gate_proj", + "model.layers.41.mlp.experts.159.up_proj", + "model.layers.41.mlp.experts.16.down_proj", + "model.layers.41.mlp.experts.16.gate_proj", + "model.layers.41.mlp.experts.16.up_proj", + "model.layers.41.mlp.experts.160.down_proj", + "model.layers.41.mlp.experts.160.gate_proj", + "model.layers.41.mlp.experts.160.up_proj", + "model.layers.41.mlp.experts.161.down_proj", + "model.layers.41.mlp.experts.161.gate_proj", + "model.layers.41.mlp.experts.161.up_proj", + "model.layers.41.mlp.experts.162.down_proj", + "model.layers.41.mlp.experts.162.gate_proj", + "model.layers.41.mlp.experts.162.up_proj", + "model.layers.41.mlp.experts.163.down_proj", + "model.layers.41.mlp.experts.163.gate_proj", + "model.layers.41.mlp.experts.163.up_proj", + "model.layers.41.mlp.experts.164.down_proj", + "model.layers.41.mlp.experts.164.gate_proj", + "model.layers.41.mlp.experts.164.up_proj", + "model.layers.41.mlp.experts.165.down_proj", + "model.layers.41.mlp.experts.165.gate_proj", + "model.layers.41.mlp.experts.165.up_proj", + "model.layers.41.mlp.experts.166.down_proj", + "model.layers.41.mlp.experts.166.gate_proj", + "model.layers.41.mlp.experts.166.up_proj", + "model.layers.41.mlp.experts.167.down_proj", + "model.layers.41.mlp.experts.167.gate_proj", + "model.layers.41.mlp.experts.167.up_proj", + "model.layers.41.mlp.experts.168.down_proj", + "model.layers.41.mlp.experts.168.gate_proj", + "model.layers.41.mlp.experts.168.up_proj", + "model.layers.41.mlp.experts.169.down_proj", + "model.layers.41.mlp.experts.169.gate_proj", + "model.layers.41.mlp.experts.169.up_proj", + "model.layers.41.mlp.experts.17.down_proj", + "model.layers.41.mlp.experts.17.gate_proj", + "model.layers.41.mlp.experts.17.up_proj", + "model.layers.41.mlp.experts.170.down_proj", + "model.layers.41.mlp.experts.170.gate_proj", + "model.layers.41.mlp.experts.170.up_proj", + "model.layers.41.mlp.experts.171.down_proj", + "model.layers.41.mlp.experts.171.gate_proj", + "model.layers.41.mlp.experts.171.up_proj", + "model.layers.41.mlp.experts.172.down_proj", + "model.layers.41.mlp.experts.172.gate_proj", + "model.layers.41.mlp.experts.172.up_proj", + "model.layers.41.mlp.experts.173.down_proj", + "model.layers.41.mlp.experts.173.gate_proj", + "model.layers.41.mlp.experts.173.up_proj", + "model.layers.41.mlp.experts.174.down_proj", + "model.layers.41.mlp.experts.174.gate_proj", + "model.layers.41.mlp.experts.174.up_proj", + "model.layers.41.mlp.experts.175.down_proj", + "model.layers.41.mlp.experts.175.gate_proj", + "model.layers.41.mlp.experts.175.up_proj", + "model.layers.41.mlp.experts.176.down_proj", + "model.layers.41.mlp.experts.176.gate_proj", + "model.layers.41.mlp.experts.176.up_proj", + "model.layers.41.mlp.experts.177.down_proj", + "model.layers.41.mlp.experts.177.gate_proj", + "model.layers.41.mlp.experts.177.up_proj", + "model.layers.41.mlp.experts.178.down_proj", + "model.layers.41.mlp.experts.178.gate_proj", + "model.layers.41.mlp.experts.178.up_proj", + "model.layers.41.mlp.experts.179.down_proj", + "model.layers.41.mlp.experts.179.gate_proj", + "model.layers.41.mlp.experts.179.up_proj", + "model.layers.41.mlp.experts.18.down_proj", + "model.layers.41.mlp.experts.18.gate_proj", + "model.layers.41.mlp.experts.18.up_proj", + "model.layers.41.mlp.experts.180.down_proj", + "model.layers.41.mlp.experts.180.gate_proj", + "model.layers.41.mlp.experts.180.up_proj", + "model.layers.41.mlp.experts.181.down_proj", + "model.layers.41.mlp.experts.181.gate_proj", + "model.layers.41.mlp.experts.181.up_proj", + "model.layers.41.mlp.experts.182.down_proj", + "model.layers.41.mlp.experts.182.gate_proj", + "model.layers.41.mlp.experts.182.up_proj", + "model.layers.41.mlp.experts.183.down_proj", + "model.layers.41.mlp.experts.183.gate_proj", + "model.layers.41.mlp.experts.183.up_proj", + "model.layers.41.mlp.experts.184.down_proj", + "model.layers.41.mlp.experts.184.gate_proj", + "model.layers.41.mlp.experts.184.up_proj", + "model.layers.41.mlp.experts.185.down_proj", + "model.layers.41.mlp.experts.185.gate_proj", + "model.layers.41.mlp.experts.185.up_proj", + "model.layers.41.mlp.experts.186.down_proj", + "model.layers.41.mlp.experts.186.gate_proj", + "model.layers.41.mlp.experts.186.up_proj", + "model.layers.41.mlp.experts.187.down_proj", + "model.layers.41.mlp.experts.187.gate_proj", + "model.layers.41.mlp.experts.187.up_proj", + "model.layers.41.mlp.experts.188.down_proj", + "model.layers.41.mlp.experts.188.gate_proj", + "model.layers.41.mlp.experts.188.up_proj", + "model.layers.41.mlp.experts.189.down_proj", + "model.layers.41.mlp.experts.189.gate_proj", + "model.layers.41.mlp.experts.189.up_proj", + "model.layers.41.mlp.experts.19.down_proj", + "model.layers.41.mlp.experts.19.gate_proj", + "model.layers.41.mlp.experts.19.up_proj", + "model.layers.41.mlp.experts.190.down_proj", + "model.layers.41.mlp.experts.190.gate_proj", + "model.layers.41.mlp.experts.190.up_proj", + "model.layers.41.mlp.experts.191.down_proj", + "model.layers.41.mlp.experts.191.gate_proj", + "model.layers.41.mlp.experts.191.up_proj", + "model.layers.41.mlp.experts.192.down_proj", + "model.layers.41.mlp.experts.192.gate_proj", + "model.layers.41.mlp.experts.192.up_proj", + "model.layers.41.mlp.experts.193.down_proj", + "model.layers.41.mlp.experts.193.gate_proj", + "model.layers.41.mlp.experts.193.up_proj", + "model.layers.41.mlp.experts.194.down_proj", + "model.layers.41.mlp.experts.194.gate_proj", + "model.layers.41.mlp.experts.194.up_proj", + "model.layers.41.mlp.experts.195.down_proj", + "model.layers.41.mlp.experts.195.gate_proj", + "model.layers.41.mlp.experts.195.up_proj", + "model.layers.41.mlp.experts.196.down_proj", + "model.layers.41.mlp.experts.196.gate_proj", + "model.layers.41.mlp.experts.196.up_proj", + "model.layers.41.mlp.experts.197.down_proj", + "model.layers.41.mlp.experts.197.gate_proj", + "model.layers.41.mlp.experts.197.up_proj", + "model.layers.41.mlp.experts.198.down_proj", + "model.layers.41.mlp.experts.198.gate_proj", + "model.layers.41.mlp.experts.198.up_proj", + "model.layers.41.mlp.experts.199.down_proj", + "model.layers.41.mlp.experts.199.gate_proj", + "model.layers.41.mlp.experts.199.up_proj", + "model.layers.41.mlp.experts.2.down_proj", + "model.layers.41.mlp.experts.2.gate_proj", + "model.layers.41.mlp.experts.2.up_proj", + "model.layers.41.mlp.experts.20.down_proj", + "model.layers.41.mlp.experts.20.gate_proj", + "model.layers.41.mlp.experts.20.up_proj", + "model.layers.41.mlp.experts.200.down_proj", + "model.layers.41.mlp.experts.200.gate_proj", + "model.layers.41.mlp.experts.200.up_proj", + "model.layers.41.mlp.experts.201.down_proj", + "model.layers.41.mlp.experts.201.gate_proj", + "model.layers.41.mlp.experts.201.up_proj", + "model.layers.41.mlp.experts.202.down_proj", + "model.layers.41.mlp.experts.202.gate_proj", + "model.layers.41.mlp.experts.202.up_proj", + "model.layers.41.mlp.experts.203.down_proj", + "model.layers.41.mlp.experts.203.gate_proj", + "model.layers.41.mlp.experts.203.up_proj", + "model.layers.41.mlp.experts.204.down_proj", + "model.layers.41.mlp.experts.204.gate_proj", + "model.layers.41.mlp.experts.204.up_proj", + "model.layers.41.mlp.experts.205.down_proj", + "model.layers.41.mlp.experts.205.gate_proj", + "model.layers.41.mlp.experts.205.up_proj", + "model.layers.41.mlp.experts.206.down_proj", + "model.layers.41.mlp.experts.206.gate_proj", + "model.layers.41.mlp.experts.206.up_proj", + "model.layers.41.mlp.experts.207.down_proj", + "model.layers.41.mlp.experts.207.gate_proj", + "model.layers.41.mlp.experts.207.up_proj", + "model.layers.41.mlp.experts.208.down_proj", + "model.layers.41.mlp.experts.208.gate_proj", + "model.layers.41.mlp.experts.208.up_proj", + "model.layers.41.mlp.experts.209.down_proj", + "model.layers.41.mlp.experts.209.gate_proj", + "model.layers.41.mlp.experts.209.up_proj", + "model.layers.41.mlp.experts.21.down_proj", + "model.layers.41.mlp.experts.21.gate_proj", + "model.layers.41.mlp.experts.21.up_proj", + "model.layers.41.mlp.experts.210.down_proj", + "model.layers.41.mlp.experts.210.gate_proj", + "model.layers.41.mlp.experts.210.up_proj", + "model.layers.41.mlp.experts.211.down_proj", + "model.layers.41.mlp.experts.211.gate_proj", + "model.layers.41.mlp.experts.211.up_proj", + "model.layers.41.mlp.experts.212.down_proj", + "model.layers.41.mlp.experts.212.gate_proj", + "model.layers.41.mlp.experts.212.up_proj", + "model.layers.41.mlp.experts.213.down_proj", + "model.layers.41.mlp.experts.213.gate_proj", + "model.layers.41.mlp.experts.213.up_proj", + "model.layers.41.mlp.experts.214.down_proj", + "model.layers.41.mlp.experts.214.gate_proj", + "model.layers.41.mlp.experts.214.up_proj", + "model.layers.41.mlp.experts.215.down_proj", + "model.layers.41.mlp.experts.215.gate_proj", + "model.layers.41.mlp.experts.215.up_proj", + "model.layers.41.mlp.experts.216.down_proj", + "model.layers.41.mlp.experts.216.gate_proj", + "model.layers.41.mlp.experts.216.up_proj", + "model.layers.41.mlp.experts.217.down_proj", + "model.layers.41.mlp.experts.217.gate_proj", + "model.layers.41.mlp.experts.217.up_proj", + "model.layers.41.mlp.experts.218.down_proj", + "model.layers.41.mlp.experts.218.gate_proj", + "model.layers.41.mlp.experts.218.up_proj", + "model.layers.41.mlp.experts.219.down_proj", + "model.layers.41.mlp.experts.219.gate_proj", + "model.layers.41.mlp.experts.219.up_proj", + "model.layers.41.mlp.experts.22.down_proj", + "model.layers.41.mlp.experts.22.gate_proj", + "model.layers.41.mlp.experts.22.up_proj", + "model.layers.41.mlp.experts.220.down_proj", + "model.layers.41.mlp.experts.220.gate_proj", + "model.layers.41.mlp.experts.220.up_proj", + "model.layers.41.mlp.experts.221.down_proj", + "model.layers.41.mlp.experts.221.gate_proj", + "model.layers.41.mlp.experts.221.up_proj", + "model.layers.41.mlp.experts.222.down_proj", + "model.layers.41.mlp.experts.222.gate_proj", + "model.layers.41.mlp.experts.222.up_proj", + "model.layers.41.mlp.experts.223.down_proj", + "model.layers.41.mlp.experts.223.gate_proj", + "model.layers.41.mlp.experts.223.up_proj", + "model.layers.41.mlp.experts.224.down_proj", + "model.layers.41.mlp.experts.224.gate_proj", + "model.layers.41.mlp.experts.224.up_proj", + "model.layers.41.mlp.experts.225.down_proj", + "model.layers.41.mlp.experts.225.gate_proj", + "model.layers.41.mlp.experts.225.up_proj", + "model.layers.41.mlp.experts.226.down_proj", + "model.layers.41.mlp.experts.226.gate_proj", + "model.layers.41.mlp.experts.226.up_proj", + "model.layers.41.mlp.experts.227.down_proj", + "model.layers.41.mlp.experts.227.gate_proj", + "model.layers.41.mlp.experts.227.up_proj", + "model.layers.41.mlp.experts.228.down_proj", + "model.layers.41.mlp.experts.228.gate_proj", + "model.layers.41.mlp.experts.228.up_proj", + "model.layers.41.mlp.experts.229.down_proj", + "model.layers.41.mlp.experts.229.gate_proj", + "model.layers.41.mlp.experts.229.up_proj", + "model.layers.41.mlp.experts.23.down_proj", + "model.layers.41.mlp.experts.23.gate_proj", + "model.layers.41.mlp.experts.23.up_proj", + "model.layers.41.mlp.experts.230.down_proj", + "model.layers.41.mlp.experts.230.gate_proj", + "model.layers.41.mlp.experts.230.up_proj", + "model.layers.41.mlp.experts.231.down_proj", + "model.layers.41.mlp.experts.231.gate_proj", + "model.layers.41.mlp.experts.231.up_proj", + "model.layers.41.mlp.experts.232.down_proj", + "model.layers.41.mlp.experts.232.gate_proj", + "model.layers.41.mlp.experts.232.up_proj", + "model.layers.41.mlp.experts.233.down_proj", + "model.layers.41.mlp.experts.233.gate_proj", + "model.layers.41.mlp.experts.233.up_proj", + "model.layers.41.mlp.experts.234.down_proj", + "model.layers.41.mlp.experts.234.gate_proj", + "model.layers.41.mlp.experts.234.up_proj", + "model.layers.41.mlp.experts.235.down_proj", + "model.layers.41.mlp.experts.235.gate_proj", + "model.layers.41.mlp.experts.235.up_proj", + "model.layers.41.mlp.experts.236.down_proj", + "model.layers.41.mlp.experts.236.gate_proj", + "model.layers.41.mlp.experts.236.up_proj", + "model.layers.41.mlp.experts.237.down_proj", + "model.layers.41.mlp.experts.237.gate_proj", + "model.layers.41.mlp.experts.237.up_proj", + "model.layers.41.mlp.experts.238.down_proj", + "model.layers.41.mlp.experts.238.gate_proj", + "model.layers.41.mlp.experts.238.up_proj", + "model.layers.41.mlp.experts.239.down_proj", + "model.layers.41.mlp.experts.239.gate_proj", + "model.layers.41.mlp.experts.239.up_proj", + "model.layers.41.mlp.experts.24.down_proj", + "model.layers.41.mlp.experts.24.gate_proj", + "model.layers.41.mlp.experts.24.up_proj", + "model.layers.41.mlp.experts.240.down_proj", + "model.layers.41.mlp.experts.240.gate_proj", + "model.layers.41.mlp.experts.240.up_proj", + "model.layers.41.mlp.experts.241.down_proj", + "model.layers.41.mlp.experts.241.gate_proj", + "model.layers.41.mlp.experts.241.up_proj", + "model.layers.41.mlp.experts.242.down_proj", + "model.layers.41.mlp.experts.242.gate_proj", + "model.layers.41.mlp.experts.242.up_proj", + "model.layers.41.mlp.experts.243.down_proj", + "model.layers.41.mlp.experts.243.gate_proj", + "model.layers.41.mlp.experts.243.up_proj", + "model.layers.41.mlp.experts.244.down_proj", + "model.layers.41.mlp.experts.244.gate_proj", + "model.layers.41.mlp.experts.244.up_proj", + "model.layers.41.mlp.experts.245.down_proj", + "model.layers.41.mlp.experts.245.gate_proj", + "model.layers.41.mlp.experts.245.up_proj", + "model.layers.41.mlp.experts.246.down_proj", + "model.layers.41.mlp.experts.246.gate_proj", + "model.layers.41.mlp.experts.246.up_proj", + "model.layers.41.mlp.experts.247.down_proj", + "model.layers.41.mlp.experts.247.gate_proj", + "model.layers.41.mlp.experts.247.up_proj", + "model.layers.41.mlp.experts.248.down_proj", + "model.layers.41.mlp.experts.248.gate_proj", + "model.layers.41.mlp.experts.248.up_proj", + "model.layers.41.mlp.experts.249.down_proj", + "model.layers.41.mlp.experts.249.gate_proj", + "model.layers.41.mlp.experts.249.up_proj", + "model.layers.41.mlp.experts.25.down_proj", + "model.layers.41.mlp.experts.25.gate_proj", + "model.layers.41.mlp.experts.25.up_proj", + "model.layers.41.mlp.experts.250.down_proj", + "model.layers.41.mlp.experts.250.gate_proj", + "model.layers.41.mlp.experts.250.up_proj", + "model.layers.41.mlp.experts.251.down_proj", + "model.layers.41.mlp.experts.251.gate_proj", + "model.layers.41.mlp.experts.251.up_proj", + "model.layers.41.mlp.experts.252.down_proj", + "model.layers.41.mlp.experts.252.gate_proj", + "model.layers.41.mlp.experts.252.up_proj", + "model.layers.41.mlp.experts.253.down_proj", + "model.layers.41.mlp.experts.253.gate_proj", + "model.layers.41.mlp.experts.253.up_proj", + "model.layers.41.mlp.experts.254.down_proj", + "model.layers.41.mlp.experts.254.gate_proj", + "model.layers.41.mlp.experts.254.up_proj", + "model.layers.41.mlp.experts.255.down_proj", + "model.layers.41.mlp.experts.255.gate_proj", + "model.layers.41.mlp.experts.255.up_proj", + "model.layers.41.mlp.experts.26.down_proj", + "model.layers.41.mlp.experts.26.gate_proj", + "model.layers.41.mlp.experts.26.up_proj", + "model.layers.41.mlp.experts.27.down_proj", + "model.layers.41.mlp.experts.27.gate_proj", + "model.layers.41.mlp.experts.27.up_proj", + "model.layers.41.mlp.experts.28.down_proj", + "model.layers.41.mlp.experts.28.gate_proj", + "model.layers.41.mlp.experts.28.up_proj", + "model.layers.41.mlp.experts.29.down_proj", + "model.layers.41.mlp.experts.29.gate_proj", + "model.layers.41.mlp.experts.29.up_proj", + "model.layers.41.mlp.experts.3.down_proj", + "model.layers.41.mlp.experts.3.gate_proj", + "model.layers.41.mlp.experts.3.up_proj", + "model.layers.41.mlp.experts.30.down_proj", + "model.layers.41.mlp.experts.30.gate_proj", + "model.layers.41.mlp.experts.30.up_proj", + "model.layers.41.mlp.experts.31.down_proj", + "model.layers.41.mlp.experts.31.gate_proj", + "model.layers.41.mlp.experts.31.up_proj", + "model.layers.41.mlp.experts.32.down_proj", + "model.layers.41.mlp.experts.32.gate_proj", + "model.layers.41.mlp.experts.32.up_proj", + "model.layers.41.mlp.experts.33.down_proj", + "model.layers.41.mlp.experts.33.gate_proj", + "model.layers.41.mlp.experts.33.up_proj", + "model.layers.41.mlp.experts.34.down_proj", + "model.layers.41.mlp.experts.34.gate_proj", + "model.layers.41.mlp.experts.34.up_proj", + "model.layers.41.mlp.experts.35.down_proj", + "model.layers.41.mlp.experts.35.gate_proj", + "model.layers.41.mlp.experts.35.up_proj", + "model.layers.41.mlp.experts.36.down_proj", + "model.layers.41.mlp.experts.36.gate_proj", + "model.layers.41.mlp.experts.36.up_proj", + "model.layers.41.mlp.experts.37.down_proj", + "model.layers.41.mlp.experts.37.gate_proj", + "model.layers.41.mlp.experts.37.up_proj", + "model.layers.41.mlp.experts.38.down_proj", + "model.layers.41.mlp.experts.38.gate_proj", + "model.layers.41.mlp.experts.38.up_proj", + "model.layers.41.mlp.experts.39.down_proj", + "model.layers.41.mlp.experts.39.gate_proj", + "model.layers.41.mlp.experts.39.up_proj", + "model.layers.41.mlp.experts.4.down_proj", + "model.layers.41.mlp.experts.4.gate_proj", + "model.layers.41.mlp.experts.4.up_proj", + "model.layers.41.mlp.experts.40.down_proj", + "model.layers.41.mlp.experts.40.gate_proj", + "model.layers.41.mlp.experts.40.up_proj", + "model.layers.41.mlp.experts.41.down_proj", + "model.layers.41.mlp.experts.41.gate_proj", + "model.layers.41.mlp.experts.41.up_proj", + "model.layers.41.mlp.experts.42.down_proj", + "model.layers.41.mlp.experts.42.gate_proj", + "model.layers.41.mlp.experts.42.up_proj", + "model.layers.41.mlp.experts.43.down_proj", + "model.layers.41.mlp.experts.43.gate_proj", + "model.layers.41.mlp.experts.43.up_proj", + "model.layers.41.mlp.experts.44.down_proj", + "model.layers.41.mlp.experts.44.gate_proj", + "model.layers.41.mlp.experts.44.up_proj", + "model.layers.41.mlp.experts.45.down_proj", + "model.layers.41.mlp.experts.45.gate_proj", + "model.layers.41.mlp.experts.45.up_proj", + "model.layers.41.mlp.experts.46.down_proj", + "model.layers.41.mlp.experts.46.gate_proj", + "model.layers.41.mlp.experts.46.up_proj", + "model.layers.41.mlp.experts.47.down_proj", + "model.layers.41.mlp.experts.47.gate_proj", + "model.layers.41.mlp.experts.47.up_proj", + "model.layers.41.mlp.experts.48.down_proj", + "model.layers.41.mlp.experts.48.gate_proj", + "model.layers.41.mlp.experts.48.up_proj", + "model.layers.41.mlp.experts.49.down_proj", + "model.layers.41.mlp.experts.49.gate_proj", + "model.layers.41.mlp.experts.49.up_proj", + "model.layers.41.mlp.experts.5.down_proj", + "model.layers.41.mlp.experts.5.gate_proj", + "model.layers.41.mlp.experts.5.up_proj", + "model.layers.41.mlp.experts.50.down_proj", + "model.layers.41.mlp.experts.50.gate_proj", + "model.layers.41.mlp.experts.50.up_proj", + "model.layers.41.mlp.experts.51.down_proj", + "model.layers.41.mlp.experts.51.gate_proj", + "model.layers.41.mlp.experts.51.up_proj", + "model.layers.41.mlp.experts.52.down_proj", + "model.layers.41.mlp.experts.52.gate_proj", + "model.layers.41.mlp.experts.52.up_proj", + "model.layers.41.mlp.experts.53.down_proj", + "model.layers.41.mlp.experts.53.gate_proj", + "model.layers.41.mlp.experts.53.up_proj", + "model.layers.41.mlp.experts.54.down_proj", + "model.layers.41.mlp.experts.54.gate_proj", + "model.layers.41.mlp.experts.54.up_proj", + "model.layers.41.mlp.experts.55.down_proj", + "model.layers.41.mlp.experts.55.gate_proj", + "model.layers.41.mlp.experts.55.up_proj", + "model.layers.41.mlp.experts.56.down_proj", + "model.layers.41.mlp.experts.56.gate_proj", + "model.layers.41.mlp.experts.56.up_proj", + "model.layers.41.mlp.experts.57.down_proj", + "model.layers.41.mlp.experts.57.gate_proj", + "model.layers.41.mlp.experts.57.up_proj", + "model.layers.41.mlp.experts.58.down_proj", + "model.layers.41.mlp.experts.58.gate_proj", + "model.layers.41.mlp.experts.58.up_proj", + "model.layers.41.mlp.experts.59.down_proj", + "model.layers.41.mlp.experts.59.gate_proj", + "model.layers.41.mlp.experts.59.up_proj", + "model.layers.41.mlp.experts.6.down_proj", + "model.layers.41.mlp.experts.6.gate_proj", + "model.layers.41.mlp.experts.6.up_proj", + "model.layers.41.mlp.experts.60.down_proj", + "model.layers.41.mlp.experts.60.gate_proj", + "model.layers.41.mlp.experts.60.up_proj", + "model.layers.41.mlp.experts.61.down_proj", + "model.layers.41.mlp.experts.61.gate_proj", + "model.layers.41.mlp.experts.61.up_proj", + "model.layers.41.mlp.experts.62.down_proj", + "model.layers.41.mlp.experts.62.gate_proj", + "model.layers.41.mlp.experts.62.up_proj", + "model.layers.41.mlp.experts.63.down_proj", + "model.layers.41.mlp.experts.63.gate_proj", + "model.layers.41.mlp.experts.63.up_proj", + "model.layers.41.mlp.experts.64.down_proj", + "model.layers.41.mlp.experts.64.gate_proj", + "model.layers.41.mlp.experts.64.up_proj", + "model.layers.41.mlp.experts.65.down_proj", + "model.layers.41.mlp.experts.65.gate_proj", + "model.layers.41.mlp.experts.65.up_proj", + "model.layers.41.mlp.experts.66.down_proj", + "model.layers.41.mlp.experts.66.gate_proj", + "model.layers.41.mlp.experts.66.up_proj", + "model.layers.41.mlp.experts.67.down_proj", + "model.layers.41.mlp.experts.67.gate_proj", + "model.layers.41.mlp.experts.67.up_proj", + "model.layers.41.mlp.experts.68.down_proj", + "model.layers.41.mlp.experts.68.gate_proj", + "model.layers.41.mlp.experts.68.up_proj", + "model.layers.41.mlp.experts.69.down_proj", + "model.layers.41.mlp.experts.69.gate_proj", + "model.layers.41.mlp.experts.69.up_proj", + "model.layers.41.mlp.experts.7.down_proj", + "model.layers.41.mlp.experts.7.gate_proj", + "model.layers.41.mlp.experts.7.up_proj", + "model.layers.41.mlp.experts.70.down_proj", + "model.layers.41.mlp.experts.70.gate_proj", + "model.layers.41.mlp.experts.70.up_proj", + "model.layers.41.mlp.experts.71.down_proj", + "model.layers.41.mlp.experts.71.gate_proj", + "model.layers.41.mlp.experts.71.up_proj", + "model.layers.41.mlp.experts.72.down_proj", + "model.layers.41.mlp.experts.72.gate_proj", + "model.layers.41.mlp.experts.72.up_proj", + "model.layers.41.mlp.experts.73.down_proj", + "model.layers.41.mlp.experts.73.gate_proj", + "model.layers.41.mlp.experts.73.up_proj", + "model.layers.41.mlp.experts.74.down_proj", + "model.layers.41.mlp.experts.74.gate_proj", + "model.layers.41.mlp.experts.74.up_proj", + "model.layers.41.mlp.experts.75.down_proj", + "model.layers.41.mlp.experts.75.gate_proj", + "model.layers.41.mlp.experts.75.up_proj", + "model.layers.41.mlp.experts.76.down_proj", + "model.layers.41.mlp.experts.76.gate_proj", + "model.layers.41.mlp.experts.76.up_proj", + "model.layers.41.mlp.experts.77.down_proj", + "model.layers.41.mlp.experts.77.gate_proj", + "model.layers.41.mlp.experts.77.up_proj", + "model.layers.41.mlp.experts.78.down_proj", + "model.layers.41.mlp.experts.78.gate_proj", + "model.layers.41.mlp.experts.78.up_proj", + "model.layers.41.mlp.experts.79.down_proj", + "model.layers.41.mlp.experts.79.gate_proj", + "model.layers.41.mlp.experts.79.up_proj", + "model.layers.41.mlp.experts.8.down_proj", + "model.layers.41.mlp.experts.8.gate_proj", + "model.layers.41.mlp.experts.8.up_proj", + "model.layers.41.mlp.experts.80.down_proj", + "model.layers.41.mlp.experts.80.gate_proj", + "model.layers.41.mlp.experts.80.up_proj", + "model.layers.41.mlp.experts.81.down_proj", + "model.layers.41.mlp.experts.81.gate_proj", + "model.layers.41.mlp.experts.81.up_proj", + "model.layers.41.mlp.experts.82.down_proj", + "model.layers.41.mlp.experts.82.gate_proj", + "model.layers.41.mlp.experts.82.up_proj", + "model.layers.41.mlp.experts.83.down_proj", + "model.layers.41.mlp.experts.83.gate_proj", + "model.layers.41.mlp.experts.83.up_proj", + "model.layers.41.mlp.experts.84.down_proj", + "model.layers.41.mlp.experts.84.gate_proj", + "model.layers.41.mlp.experts.84.up_proj", + "model.layers.41.mlp.experts.85.down_proj", + "model.layers.41.mlp.experts.85.gate_proj", + "model.layers.41.mlp.experts.85.up_proj", + "model.layers.41.mlp.experts.86.down_proj", + "model.layers.41.mlp.experts.86.gate_proj", + "model.layers.41.mlp.experts.86.up_proj", + "model.layers.41.mlp.experts.87.down_proj", + "model.layers.41.mlp.experts.87.gate_proj", + "model.layers.41.mlp.experts.87.up_proj", + "model.layers.41.mlp.experts.88.down_proj", + "model.layers.41.mlp.experts.88.gate_proj", + "model.layers.41.mlp.experts.88.up_proj", + "model.layers.41.mlp.experts.89.down_proj", + "model.layers.41.mlp.experts.89.gate_proj", + "model.layers.41.mlp.experts.89.up_proj", + "model.layers.41.mlp.experts.9.down_proj", + "model.layers.41.mlp.experts.9.gate_proj", + "model.layers.41.mlp.experts.9.up_proj", + "model.layers.41.mlp.experts.90.down_proj", + "model.layers.41.mlp.experts.90.gate_proj", + "model.layers.41.mlp.experts.90.up_proj", + "model.layers.41.mlp.experts.91.down_proj", + "model.layers.41.mlp.experts.91.gate_proj", + "model.layers.41.mlp.experts.91.up_proj", + "model.layers.41.mlp.experts.92.down_proj", + "model.layers.41.mlp.experts.92.gate_proj", + "model.layers.41.mlp.experts.92.up_proj", + "model.layers.41.mlp.experts.93.down_proj", + "model.layers.41.mlp.experts.93.gate_proj", + "model.layers.41.mlp.experts.93.up_proj", + "model.layers.41.mlp.experts.94.down_proj", + "model.layers.41.mlp.experts.94.gate_proj", + "model.layers.41.mlp.experts.94.up_proj", + "model.layers.41.mlp.experts.95.down_proj", + "model.layers.41.mlp.experts.95.gate_proj", + "model.layers.41.mlp.experts.95.up_proj", + "model.layers.41.mlp.experts.96.down_proj", + "model.layers.41.mlp.experts.96.gate_proj", + "model.layers.41.mlp.experts.96.up_proj", + "model.layers.41.mlp.experts.97.down_proj", + "model.layers.41.mlp.experts.97.gate_proj", + "model.layers.41.mlp.experts.97.up_proj", + "model.layers.41.mlp.experts.98.down_proj", + "model.layers.41.mlp.experts.98.gate_proj", + "model.layers.41.mlp.experts.98.up_proj", + "model.layers.41.mlp.experts.99.down_proj", + "model.layers.41.mlp.experts.99.gate_proj", + "model.layers.41.mlp.experts.99.up_proj", + "model.layers.42.mlp.experts.0.down_proj", + "model.layers.42.mlp.experts.0.gate_proj", + "model.layers.42.mlp.experts.0.up_proj", + "model.layers.42.mlp.experts.1.down_proj", + "model.layers.42.mlp.experts.1.gate_proj", + "model.layers.42.mlp.experts.1.up_proj", + "model.layers.42.mlp.experts.10.down_proj", + "model.layers.42.mlp.experts.10.gate_proj", + "model.layers.42.mlp.experts.10.up_proj", + "model.layers.42.mlp.experts.100.down_proj", + "model.layers.42.mlp.experts.100.gate_proj", + "model.layers.42.mlp.experts.100.up_proj", + "model.layers.42.mlp.experts.101.down_proj", + "model.layers.42.mlp.experts.101.gate_proj", + "model.layers.42.mlp.experts.101.up_proj", + "model.layers.42.mlp.experts.102.down_proj", + "model.layers.42.mlp.experts.102.gate_proj", + "model.layers.42.mlp.experts.102.up_proj", + "model.layers.42.mlp.experts.103.down_proj", + "model.layers.42.mlp.experts.103.gate_proj", + "model.layers.42.mlp.experts.103.up_proj", + "model.layers.42.mlp.experts.104.down_proj", + "model.layers.42.mlp.experts.104.gate_proj", + "model.layers.42.mlp.experts.104.up_proj", + "model.layers.42.mlp.experts.105.down_proj", + "model.layers.42.mlp.experts.105.gate_proj", + "model.layers.42.mlp.experts.105.up_proj", + "model.layers.42.mlp.experts.106.down_proj", + "model.layers.42.mlp.experts.106.gate_proj", + "model.layers.42.mlp.experts.106.up_proj", + "model.layers.42.mlp.experts.107.down_proj", + "model.layers.42.mlp.experts.107.gate_proj", + "model.layers.42.mlp.experts.107.up_proj", + "model.layers.42.mlp.experts.108.down_proj", + "model.layers.42.mlp.experts.108.gate_proj", + "model.layers.42.mlp.experts.108.up_proj", + "model.layers.42.mlp.experts.109.down_proj", + "model.layers.42.mlp.experts.109.gate_proj", + "model.layers.42.mlp.experts.109.up_proj", + "model.layers.42.mlp.experts.11.down_proj", + "model.layers.42.mlp.experts.11.gate_proj", + "model.layers.42.mlp.experts.11.up_proj", + "model.layers.42.mlp.experts.110.down_proj", + "model.layers.42.mlp.experts.110.gate_proj", + "model.layers.42.mlp.experts.110.up_proj", + "model.layers.42.mlp.experts.111.down_proj", + "model.layers.42.mlp.experts.111.gate_proj", + "model.layers.42.mlp.experts.111.up_proj", + "model.layers.42.mlp.experts.112.down_proj", + "model.layers.42.mlp.experts.112.gate_proj", + "model.layers.42.mlp.experts.112.up_proj", + "model.layers.42.mlp.experts.113.down_proj", + "model.layers.42.mlp.experts.113.gate_proj", + "model.layers.42.mlp.experts.113.up_proj", + "model.layers.42.mlp.experts.114.down_proj", + "model.layers.42.mlp.experts.114.gate_proj", + "model.layers.42.mlp.experts.114.up_proj", + "model.layers.42.mlp.experts.115.down_proj", + "model.layers.42.mlp.experts.115.gate_proj", + "model.layers.42.mlp.experts.115.up_proj", + "model.layers.42.mlp.experts.116.down_proj", + "model.layers.42.mlp.experts.116.gate_proj", + "model.layers.42.mlp.experts.116.up_proj", + "model.layers.42.mlp.experts.117.down_proj", + "model.layers.42.mlp.experts.117.gate_proj", + "model.layers.42.mlp.experts.117.up_proj", + "model.layers.42.mlp.experts.118.down_proj", + "model.layers.42.mlp.experts.118.gate_proj", + "model.layers.42.mlp.experts.118.up_proj", + "model.layers.42.mlp.experts.119.down_proj", + "model.layers.42.mlp.experts.119.gate_proj", + "model.layers.42.mlp.experts.119.up_proj", + "model.layers.42.mlp.experts.12.down_proj", + "model.layers.42.mlp.experts.12.gate_proj", + "model.layers.42.mlp.experts.12.up_proj", + "model.layers.42.mlp.experts.120.down_proj", + "model.layers.42.mlp.experts.120.gate_proj", + "model.layers.42.mlp.experts.120.up_proj", + "model.layers.42.mlp.experts.121.down_proj", + "model.layers.42.mlp.experts.121.gate_proj", + "model.layers.42.mlp.experts.121.up_proj", + "model.layers.42.mlp.experts.122.down_proj", + "model.layers.42.mlp.experts.122.gate_proj", + "model.layers.42.mlp.experts.122.up_proj", + "model.layers.42.mlp.experts.123.down_proj", + "model.layers.42.mlp.experts.123.gate_proj", + "model.layers.42.mlp.experts.123.up_proj", + "model.layers.42.mlp.experts.124.down_proj", + "model.layers.42.mlp.experts.124.gate_proj", + "model.layers.42.mlp.experts.124.up_proj", + "model.layers.42.mlp.experts.125.down_proj", + "model.layers.42.mlp.experts.125.gate_proj", + "model.layers.42.mlp.experts.125.up_proj", + "model.layers.42.mlp.experts.126.down_proj", + "model.layers.42.mlp.experts.126.gate_proj", + "model.layers.42.mlp.experts.126.up_proj", + "model.layers.42.mlp.experts.127.down_proj", + "model.layers.42.mlp.experts.127.gate_proj", + "model.layers.42.mlp.experts.127.up_proj", + "model.layers.42.mlp.experts.128.down_proj", + "model.layers.42.mlp.experts.128.gate_proj", + "model.layers.42.mlp.experts.128.up_proj", + "model.layers.42.mlp.experts.129.down_proj", + "model.layers.42.mlp.experts.129.gate_proj", + "model.layers.42.mlp.experts.129.up_proj", + "model.layers.42.mlp.experts.13.down_proj", + "model.layers.42.mlp.experts.13.gate_proj", + "model.layers.42.mlp.experts.13.up_proj", + "model.layers.42.mlp.experts.130.down_proj", + "model.layers.42.mlp.experts.130.gate_proj", + "model.layers.42.mlp.experts.130.up_proj", + "model.layers.42.mlp.experts.131.down_proj", + "model.layers.42.mlp.experts.131.gate_proj", + "model.layers.42.mlp.experts.131.up_proj", + "model.layers.42.mlp.experts.132.down_proj", + "model.layers.42.mlp.experts.132.gate_proj", + "model.layers.42.mlp.experts.132.up_proj", + "model.layers.42.mlp.experts.133.down_proj", + "model.layers.42.mlp.experts.133.gate_proj", + "model.layers.42.mlp.experts.133.up_proj", + "model.layers.42.mlp.experts.134.down_proj", + "model.layers.42.mlp.experts.134.gate_proj", + "model.layers.42.mlp.experts.134.up_proj", + "model.layers.42.mlp.experts.135.down_proj", + "model.layers.42.mlp.experts.135.gate_proj", + "model.layers.42.mlp.experts.135.up_proj", + "model.layers.42.mlp.experts.136.down_proj", + "model.layers.42.mlp.experts.136.gate_proj", + "model.layers.42.mlp.experts.136.up_proj", + "model.layers.42.mlp.experts.137.down_proj", + "model.layers.42.mlp.experts.137.gate_proj", + "model.layers.42.mlp.experts.137.up_proj", + "model.layers.42.mlp.experts.138.down_proj", + "model.layers.42.mlp.experts.138.gate_proj", + "model.layers.42.mlp.experts.138.up_proj", + "model.layers.42.mlp.experts.139.down_proj", + "model.layers.42.mlp.experts.139.gate_proj", + "model.layers.42.mlp.experts.139.up_proj", + "model.layers.42.mlp.experts.14.down_proj", + "model.layers.42.mlp.experts.14.gate_proj", + "model.layers.42.mlp.experts.14.up_proj", + "model.layers.42.mlp.experts.140.down_proj", + "model.layers.42.mlp.experts.140.gate_proj", + "model.layers.42.mlp.experts.140.up_proj", + "model.layers.42.mlp.experts.141.down_proj", + "model.layers.42.mlp.experts.141.gate_proj", + "model.layers.42.mlp.experts.141.up_proj", + "model.layers.42.mlp.experts.142.down_proj", + "model.layers.42.mlp.experts.142.gate_proj", + "model.layers.42.mlp.experts.142.up_proj", + "model.layers.42.mlp.experts.143.down_proj", + "model.layers.42.mlp.experts.143.gate_proj", + "model.layers.42.mlp.experts.143.up_proj", + "model.layers.42.mlp.experts.144.down_proj", + "model.layers.42.mlp.experts.144.gate_proj", + "model.layers.42.mlp.experts.144.up_proj", + "model.layers.42.mlp.experts.145.down_proj", + "model.layers.42.mlp.experts.145.gate_proj", + "model.layers.42.mlp.experts.145.up_proj", + "model.layers.42.mlp.experts.146.down_proj", + "model.layers.42.mlp.experts.146.gate_proj", + "model.layers.42.mlp.experts.146.up_proj", + "model.layers.42.mlp.experts.147.down_proj", + "model.layers.42.mlp.experts.147.gate_proj", + "model.layers.42.mlp.experts.147.up_proj", + "model.layers.42.mlp.experts.148.down_proj", + "model.layers.42.mlp.experts.148.gate_proj", + "model.layers.42.mlp.experts.148.up_proj", + "model.layers.42.mlp.experts.149.down_proj", + "model.layers.42.mlp.experts.149.gate_proj", + "model.layers.42.mlp.experts.149.up_proj", + "model.layers.42.mlp.experts.15.down_proj", + "model.layers.42.mlp.experts.15.gate_proj", + "model.layers.42.mlp.experts.15.up_proj", + "model.layers.42.mlp.experts.150.down_proj", + "model.layers.42.mlp.experts.150.gate_proj", + "model.layers.42.mlp.experts.150.up_proj", + "model.layers.42.mlp.experts.151.down_proj", + "model.layers.42.mlp.experts.151.gate_proj", + "model.layers.42.mlp.experts.151.up_proj", + "model.layers.42.mlp.experts.152.down_proj", + "model.layers.42.mlp.experts.152.gate_proj", + "model.layers.42.mlp.experts.152.up_proj", + "model.layers.42.mlp.experts.153.down_proj", + "model.layers.42.mlp.experts.153.gate_proj", + "model.layers.42.mlp.experts.153.up_proj", + "model.layers.42.mlp.experts.154.down_proj", + "model.layers.42.mlp.experts.154.gate_proj", + "model.layers.42.mlp.experts.154.up_proj", + "model.layers.42.mlp.experts.155.down_proj", + "model.layers.42.mlp.experts.155.gate_proj", + "model.layers.42.mlp.experts.155.up_proj", + "model.layers.42.mlp.experts.156.down_proj", + "model.layers.42.mlp.experts.156.gate_proj", + "model.layers.42.mlp.experts.156.up_proj", + "model.layers.42.mlp.experts.157.down_proj", + "model.layers.42.mlp.experts.157.gate_proj", + "model.layers.42.mlp.experts.157.up_proj", + "model.layers.42.mlp.experts.158.down_proj", + "model.layers.42.mlp.experts.158.gate_proj", + "model.layers.42.mlp.experts.158.up_proj", + "model.layers.42.mlp.experts.159.down_proj", + "model.layers.42.mlp.experts.159.gate_proj", + "model.layers.42.mlp.experts.159.up_proj", + "model.layers.42.mlp.experts.16.down_proj", + "model.layers.42.mlp.experts.16.gate_proj", + "model.layers.42.mlp.experts.16.up_proj", + "model.layers.42.mlp.experts.160.down_proj", + "model.layers.42.mlp.experts.160.gate_proj", + "model.layers.42.mlp.experts.160.up_proj", + "model.layers.42.mlp.experts.161.down_proj", + "model.layers.42.mlp.experts.161.gate_proj", + "model.layers.42.mlp.experts.161.up_proj", + "model.layers.42.mlp.experts.162.down_proj", + "model.layers.42.mlp.experts.162.gate_proj", + "model.layers.42.mlp.experts.162.up_proj", + "model.layers.42.mlp.experts.163.down_proj", + "model.layers.42.mlp.experts.163.gate_proj", + "model.layers.42.mlp.experts.163.up_proj", + "model.layers.42.mlp.experts.164.down_proj", + "model.layers.42.mlp.experts.164.gate_proj", + "model.layers.42.mlp.experts.164.up_proj", + "model.layers.42.mlp.experts.165.down_proj", + "model.layers.42.mlp.experts.165.gate_proj", + "model.layers.42.mlp.experts.165.up_proj", + "model.layers.42.mlp.experts.166.down_proj", + "model.layers.42.mlp.experts.166.gate_proj", + "model.layers.42.mlp.experts.166.up_proj", + "model.layers.42.mlp.experts.167.down_proj", + "model.layers.42.mlp.experts.167.gate_proj", + "model.layers.42.mlp.experts.167.up_proj", + "model.layers.42.mlp.experts.168.down_proj", + "model.layers.42.mlp.experts.168.gate_proj", + "model.layers.42.mlp.experts.168.up_proj", + "model.layers.42.mlp.experts.169.down_proj", + "model.layers.42.mlp.experts.169.gate_proj", + "model.layers.42.mlp.experts.169.up_proj", + "model.layers.42.mlp.experts.17.down_proj", + "model.layers.42.mlp.experts.17.gate_proj", + "model.layers.42.mlp.experts.17.up_proj", + "model.layers.42.mlp.experts.170.down_proj", + "model.layers.42.mlp.experts.170.gate_proj", + "model.layers.42.mlp.experts.170.up_proj", + "model.layers.42.mlp.experts.171.down_proj", + "model.layers.42.mlp.experts.171.gate_proj", + "model.layers.42.mlp.experts.171.up_proj", + "model.layers.42.mlp.experts.172.down_proj", + "model.layers.42.mlp.experts.172.gate_proj", + "model.layers.42.mlp.experts.172.up_proj", + "model.layers.42.mlp.experts.173.down_proj", + "model.layers.42.mlp.experts.173.gate_proj", + "model.layers.42.mlp.experts.173.up_proj", + "model.layers.42.mlp.experts.174.down_proj", + "model.layers.42.mlp.experts.174.gate_proj", + "model.layers.42.mlp.experts.174.up_proj", + "model.layers.42.mlp.experts.175.down_proj", + "model.layers.42.mlp.experts.175.gate_proj", + "model.layers.42.mlp.experts.175.up_proj", + "model.layers.42.mlp.experts.176.down_proj", + "model.layers.42.mlp.experts.176.gate_proj", + "model.layers.42.mlp.experts.176.up_proj", + "model.layers.42.mlp.experts.177.down_proj", + "model.layers.42.mlp.experts.177.gate_proj", + "model.layers.42.mlp.experts.177.up_proj", + "model.layers.42.mlp.experts.178.down_proj", + "model.layers.42.mlp.experts.178.gate_proj", + "model.layers.42.mlp.experts.178.up_proj", + "model.layers.42.mlp.experts.179.down_proj", + "model.layers.42.mlp.experts.179.gate_proj", + "model.layers.42.mlp.experts.179.up_proj", + "model.layers.42.mlp.experts.18.down_proj", + "model.layers.42.mlp.experts.18.gate_proj", + "model.layers.42.mlp.experts.18.up_proj", + "model.layers.42.mlp.experts.180.down_proj", + "model.layers.42.mlp.experts.180.gate_proj", + "model.layers.42.mlp.experts.180.up_proj", + "model.layers.42.mlp.experts.181.down_proj", + "model.layers.42.mlp.experts.181.gate_proj", + "model.layers.42.mlp.experts.181.up_proj", + "model.layers.42.mlp.experts.182.down_proj", + "model.layers.42.mlp.experts.182.gate_proj", + "model.layers.42.mlp.experts.182.up_proj", + "model.layers.42.mlp.experts.183.down_proj", + "model.layers.42.mlp.experts.183.gate_proj", + "model.layers.42.mlp.experts.183.up_proj", + "model.layers.42.mlp.experts.184.down_proj", + "model.layers.42.mlp.experts.184.gate_proj", + "model.layers.42.mlp.experts.184.up_proj", + "model.layers.42.mlp.experts.185.down_proj", + "model.layers.42.mlp.experts.185.gate_proj", + "model.layers.42.mlp.experts.185.up_proj", + "model.layers.42.mlp.experts.186.down_proj", + "model.layers.42.mlp.experts.186.gate_proj", + "model.layers.42.mlp.experts.186.up_proj", + "model.layers.42.mlp.experts.187.down_proj", + "model.layers.42.mlp.experts.187.gate_proj", + "model.layers.42.mlp.experts.187.up_proj", + "model.layers.42.mlp.experts.188.down_proj", + "model.layers.42.mlp.experts.188.gate_proj", + "model.layers.42.mlp.experts.188.up_proj", + "model.layers.42.mlp.experts.189.down_proj", + "model.layers.42.mlp.experts.189.gate_proj", + "model.layers.42.mlp.experts.189.up_proj", + "model.layers.42.mlp.experts.19.down_proj", + "model.layers.42.mlp.experts.19.gate_proj", + "model.layers.42.mlp.experts.19.up_proj", + "model.layers.42.mlp.experts.190.down_proj", + "model.layers.42.mlp.experts.190.gate_proj", + "model.layers.42.mlp.experts.190.up_proj", + "model.layers.42.mlp.experts.191.down_proj", + "model.layers.42.mlp.experts.191.gate_proj", + "model.layers.42.mlp.experts.191.up_proj", + "model.layers.42.mlp.experts.192.down_proj", + "model.layers.42.mlp.experts.192.gate_proj", + "model.layers.42.mlp.experts.192.up_proj", + "model.layers.42.mlp.experts.193.down_proj", + "model.layers.42.mlp.experts.193.gate_proj", + "model.layers.42.mlp.experts.193.up_proj", + "model.layers.42.mlp.experts.194.down_proj", + "model.layers.42.mlp.experts.194.gate_proj", + "model.layers.42.mlp.experts.194.up_proj", + "model.layers.42.mlp.experts.195.down_proj", + "model.layers.42.mlp.experts.195.gate_proj", + "model.layers.42.mlp.experts.195.up_proj", + "model.layers.42.mlp.experts.196.down_proj", + "model.layers.42.mlp.experts.196.gate_proj", + "model.layers.42.mlp.experts.196.up_proj", + "model.layers.42.mlp.experts.197.down_proj", + "model.layers.42.mlp.experts.197.gate_proj", + "model.layers.42.mlp.experts.197.up_proj", + "model.layers.42.mlp.experts.198.down_proj", + "model.layers.42.mlp.experts.198.gate_proj", + "model.layers.42.mlp.experts.198.up_proj", + "model.layers.42.mlp.experts.199.down_proj", + "model.layers.42.mlp.experts.199.gate_proj", + "model.layers.42.mlp.experts.199.up_proj", + "model.layers.42.mlp.experts.2.down_proj", + "model.layers.42.mlp.experts.2.gate_proj", + "model.layers.42.mlp.experts.2.up_proj", + "model.layers.42.mlp.experts.20.down_proj", + "model.layers.42.mlp.experts.20.gate_proj", + "model.layers.42.mlp.experts.20.up_proj", + "model.layers.42.mlp.experts.200.down_proj", + "model.layers.42.mlp.experts.200.gate_proj", + "model.layers.42.mlp.experts.200.up_proj", + "model.layers.42.mlp.experts.201.down_proj", + "model.layers.42.mlp.experts.201.gate_proj", + "model.layers.42.mlp.experts.201.up_proj", + "model.layers.42.mlp.experts.202.down_proj", + "model.layers.42.mlp.experts.202.gate_proj", + "model.layers.42.mlp.experts.202.up_proj", + "model.layers.42.mlp.experts.203.down_proj", + "model.layers.42.mlp.experts.203.gate_proj", + "model.layers.42.mlp.experts.203.up_proj", + "model.layers.42.mlp.experts.204.down_proj", + "model.layers.42.mlp.experts.204.gate_proj", + "model.layers.42.mlp.experts.204.up_proj", + "model.layers.42.mlp.experts.205.down_proj", + "model.layers.42.mlp.experts.205.gate_proj", + "model.layers.42.mlp.experts.205.up_proj", + "model.layers.42.mlp.experts.206.down_proj", + "model.layers.42.mlp.experts.206.gate_proj", + "model.layers.42.mlp.experts.206.up_proj", + "model.layers.42.mlp.experts.207.down_proj", + "model.layers.42.mlp.experts.207.gate_proj", + "model.layers.42.mlp.experts.207.up_proj", + "model.layers.42.mlp.experts.208.down_proj", + "model.layers.42.mlp.experts.208.gate_proj", + "model.layers.42.mlp.experts.208.up_proj", + "model.layers.42.mlp.experts.209.down_proj", + "model.layers.42.mlp.experts.209.gate_proj", + "model.layers.42.mlp.experts.209.up_proj", + "model.layers.42.mlp.experts.21.down_proj", + "model.layers.42.mlp.experts.21.gate_proj", + "model.layers.42.mlp.experts.21.up_proj", + "model.layers.42.mlp.experts.210.down_proj", + "model.layers.42.mlp.experts.210.gate_proj", + "model.layers.42.mlp.experts.210.up_proj", + "model.layers.42.mlp.experts.211.down_proj", + "model.layers.42.mlp.experts.211.gate_proj", + "model.layers.42.mlp.experts.211.up_proj", + "model.layers.42.mlp.experts.212.down_proj", + "model.layers.42.mlp.experts.212.gate_proj", + "model.layers.42.mlp.experts.212.up_proj", + "model.layers.42.mlp.experts.213.down_proj", + "model.layers.42.mlp.experts.213.gate_proj", + "model.layers.42.mlp.experts.213.up_proj", + "model.layers.42.mlp.experts.214.down_proj", + "model.layers.42.mlp.experts.214.gate_proj", + "model.layers.42.mlp.experts.214.up_proj", + "model.layers.42.mlp.experts.215.down_proj", + "model.layers.42.mlp.experts.215.gate_proj", + "model.layers.42.mlp.experts.215.up_proj", + "model.layers.42.mlp.experts.216.down_proj", + "model.layers.42.mlp.experts.216.gate_proj", + "model.layers.42.mlp.experts.216.up_proj", + "model.layers.42.mlp.experts.217.down_proj", + "model.layers.42.mlp.experts.217.gate_proj", + "model.layers.42.mlp.experts.217.up_proj", + "model.layers.42.mlp.experts.218.down_proj", + "model.layers.42.mlp.experts.218.gate_proj", + "model.layers.42.mlp.experts.218.up_proj", + "model.layers.42.mlp.experts.219.down_proj", + "model.layers.42.mlp.experts.219.gate_proj", + "model.layers.42.mlp.experts.219.up_proj", + "model.layers.42.mlp.experts.22.down_proj", + "model.layers.42.mlp.experts.22.gate_proj", + "model.layers.42.mlp.experts.22.up_proj", + "model.layers.42.mlp.experts.220.down_proj", + "model.layers.42.mlp.experts.220.gate_proj", + "model.layers.42.mlp.experts.220.up_proj", + "model.layers.42.mlp.experts.221.down_proj", + "model.layers.42.mlp.experts.221.gate_proj", + "model.layers.42.mlp.experts.221.up_proj", + "model.layers.42.mlp.experts.222.down_proj", + "model.layers.42.mlp.experts.222.gate_proj", + "model.layers.42.mlp.experts.222.up_proj", + "model.layers.42.mlp.experts.223.down_proj", + "model.layers.42.mlp.experts.223.gate_proj", + "model.layers.42.mlp.experts.223.up_proj", + "model.layers.42.mlp.experts.224.down_proj", + "model.layers.42.mlp.experts.224.gate_proj", + "model.layers.42.mlp.experts.224.up_proj", + "model.layers.42.mlp.experts.225.down_proj", + "model.layers.42.mlp.experts.225.gate_proj", + "model.layers.42.mlp.experts.225.up_proj", + "model.layers.42.mlp.experts.226.down_proj", + "model.layers.42.mlp.experts.226.gate_proj", + "model.layers.42.mlp.experts.226.up_proj", + "model.layers.42.mlp.experts.227.down_proj", + "model.layers.42.mlp.experts.227.gate_proj", + "model.layers.42.mlp.experts.227.up_proj", + "model.layers.42.mlp.experts.228.down_proj", + "model.layers.42.mlp.experts.228.gate_proj", + "model.layers.42.mlp.experts.228.up_proj", + "model.layers.42.mlp.experts.229.down_proj", + "model.layers.42.mlp.experts.229.gate_proj", + "model.layers.42.mlp.experts.229.up_proj", + "model.layers.42.mlp.experts.23.down_proj", + "model.layers.42.mlp.experts.23.gate_proj", + "model.layers.42.mlp.experts.23.up_proj", + "model.layers.42.mlp.experts.230.down_proj", + "model.layers.42.mlp.experts.230.gate_proj", + "model.layers.42.mlp.experts.230.up_proj", + "model.layers.42.mlp.experts.231.down_proj", + "model.layers.42.mlp.experts.231.gate_proj", + "model.layers.42.mlp.experts.231.up_proj", + "model.layers.42.mlp.experts.232.down_proj", + "model.layers.42.mlp.experts.232.gate_proj", + "model.layers.42.mlp.experts.232.up_proj", + "model.layers.42.mlp.experts.233.down_proj", + "model.layers.42.mlp.experts.233.gate_proj", + "model.layers.42.mlp.experts.233.up_proj", + "model.layers.42.mlp.experts.234.down_proj", + "model.layers.42.mlp.experts.234.gate_proj", + "model.layers.42.mlp.experts.234.up_proj", + "model.layers.42.mlp.experts.235.down_proj", + "model.layers.42.mlp.experts.235.gate_proj", + "model.layers.42.mlp.experts.235.up_proj", + "model.layers.42.mlp.experts.236.down_proj", + "model.layers.42.mlp.experts.236.gate_proj", + "model.layers.42.mlp.experts.236.up_proj", + "model.layers.42.mlp.experts.237.down_proj", + "model.layers.42.mlp.experts.237.gate_proj", + "model.layers.42.mlp.experts.237.up_proj", + "model.layers.42.mlp.experts.238.down_proj", + "model.layers.42.mlp.experts.238.gate_proj", + "model.layers.42.mlp.experts.238.up_proj", + "model.layers.42.mlp.experts.239.down_proj", + "model.layers.42.mlp.experts.239.gate_proj", + "model.layers.42.mlp.experts.239.up_proj", + "model.layers.42.mlp.experts.24.down_proj", + "model.layers.42.mlp.experts.24.gate_proj", + "model.layers.42.mlp.experts.24.up_proj", + "model.layers.42.mlp.experts.240.down_proj", + "model.layers.42.mlp.experts.240.gate_proj", + "model.layers.42.mlp.experts.240.up_proj", + "model.layers.42.mlp.experts.241.down_proj", + "model.layers.42.mlp.experts.241.gate_proj", + "model.layers.42.mlp.experts.241.up_proj", + "model.layers.42.mlp.experts.242.down_proj", + "model.layers.42.mlp.experts.242.gate_proj", + "model.layers.42.mlp.experts.242.up_proj", + "model.layers.42.mlp.experts.243.down_proj", + "model.layers.42.mlp.experts.243.gate_proj", + "model.layers.42.mlp.experts.243.up_proj", + "model.layers.42.mlp.experts.244.down_proj", + "model.layers.42.mlp.experts.244.gate_proj", + "model.layers.42.mlp.experts.244.up_proj", + "model.layers.42.mlp.experts.245.down_proj", + "model.layers.42.mlp.experts.245.gate_proj", + "model.layers.42.mlp.experts.245.up_proj", + "model.layers.42.mlp.experts.246.down_proj", + "model.layers.42.mlp.experts.246.gate_proj", + "model.layers.42.mlp.experts.246.up_proj", + "model.layers.42.mlp.experts.247.down_proj", + "model.layers.42.mlp.experts.247.gate_proj", + "model.layers.42.mlp.experts.247.up_proj", + "model.layers.42.mlp.experts.248.down_proj", + "model.layers.42.mlp.experts.248.gate_proj", + "model.layers.42.mlp.experts.248.up_proj", + "model.layers.42.mlp.experts.249.down_proj", + "model.layers.42.mlp.experts.249.gate_proj", + "model.layers.42.mlp.experts.249.up_proj", + "model.layers.42.mlp.experts.25.down_proj", + "model.layers.42.mlp.experts.25.gate_proj", + "model.layers.42.mlp.experts.25.up_proj", + "model.layers.42.mlp.experts.250.down_proj", + "model.layers.42.mlp.experts.250.gate_proj", + "model.layers.42.mlp.experts.250.up_proj", + "model.layers.42.mlp.experts.251.down_proj", + "model.layers.42.mlp.experts.251.gate_proj", + "model.layers.42.mlp.experts.251.up_proj", + "model.layers.42.mlp.experts.252.down_proj", + "model.layers.42.mlp.experts.252.gate_proj", + "model.layers.42.mlp.experts.252.up_proj", + "model.layers.42.mlp.experts.253.down_proj", + "model.layers.42.mlp.experts.253.gate_proj", + "model.layers.42.mlp.experts.253.up_proj", + "model.layers.42.mlp.experts.254.down_proj", + "model.layers.42.mlp.experts.254.gate_proj", + "model.layers.42.mlp.experts.254.up_proj", + "model.layers.42.mlp.experts.255.down_proj", + "model.layers.42.mlp.experts.255.gate_proj", + "model.layers.42.mlp.experts.255.up_proj", + "model.layers.42.mlp.experts.26.down_proj", + "model.layers.42.mlp.experts.26.gate_proj", + "model.layers.42.mlp.experts.26.up_proj", + "model.layers.42.mlp.experts.27.down_proj", + "model.layers.42.mlp.experts.27.gate_proj", + "model.layers.42.mlp.experts.27.up_proj", + "model.layers.42.mlp.experts.28.down_proj", + "model.layers.42.mlp.experts.28.gate_proj", + "model.layers.42.mlp.experts.28.up_proj", + "model.layers.42.mlp.experts.29.down_proj", + "model.layers.42.mlp.experts.29.gate_proj", + "model.layers.42.mlp.experts.29.up_proj", + "model.layers.42.mlp.experts.3.down_proj", + "model.layers.42.mlp.experts.3.gate_proj", + "model.layers.42.mlp.experts.3.up_proj", + "model.layers.42.mlp.experts.30.down_proj", + "model.layers.42.mlp.experts.30.gate_proj", + "model.layers.42.mlp.experts.30.up_proj", + "model.layers.42.mlp.experts.31.down_proj", + "model.layers.42.mlp.experts.31.gate_proj", + "model.layers.42.mlp.experts.31.up_proj", + "model.layers.42.mlp.experts.32.down_proj", + "model.layers.42.mlp.experts.32.gate_proj", + "model.layers.42.mlp.experts.32.up_proj", + "model.layers.42.mlp.experts.33.down_proj", + "model.layers.42.mlp.experts.33.gate_proj", + "model.layers.42.mlp.experts.33.up_proj", + "model.layers.42.mlp.experts.34.down_proj", + "model.layers.42.mlp.experts.34.gate_proj", + "model.layers.42.mlp.experts.34.up_proj", + "model.layers.42.mlp.experts.35.down_proj", + "model.layers.42.mlp.experts.35.gate_proj", + "model.layers.42.mlp.experts.35.up_proj", + "model.layers.42.mlp.experts.36.down_proj", + "model.layers.42.mlp.experts.36.gate_proj", + "model.layers.42.mlp.experts.36.up_proj", + "model.layers.42.mlp.experts.37.down_proj", + "model.layers.42.mlp.experts.37.gate_proj", + "model.layers.42.mlp.experts.37.up_proj", + "model.layers.42.mlp.experts.38.down_proj", + "model.layers.42.mlp.experts.38.gate_proj", + "model.layers.42.mlp.experts.38.up_proj", + "model.layers.42.mlp.experts.39.down_proj", + "model.layers.42.mlp.experts.39.gate_proj", + "model.layers.42.mlp.experts.39.up_proj", + "model.layers.42.mlp.experts.4.down_proj", + "model.layers.42.mlp.experts.4.gate_proj", + "model.layers.42.mlp.experts.4.up_proj", + "model.layers.42.mlp.experts.40.down_proj", + "model.layers.42.mlp.experts.40.gate_proj", + "model.layers.42.mlp.experts.40.up_proj", + "model.layers.42.mlp.experts.41.down_proj", + "model.layers.42.mlp.experts.41.gate_proj", + "model.layers.42.mlp.experts.41.up_proj", + "model.layers.42.mlp.experts.42.down_proj", + "model.layers.42.mlp.experts.42.gate_proj", + "model.layers.42.mlp.experts.42.up_proj", + "model.layers.42.mlp.experts.43.down_proj", + "model.layers.42.mlp.experts.43.gate_proj", + "model.layers.42.mlp.experts.43.up_proj", + "model.layers.42.mlp.experts.44.down_proj", + "model.layers.42.mlp.experts.44.gate_proj", + "model.layers.42.mlp.experts.44.up_proj", + "model.layers.42.mlp.experts.45.down_proj", + "model.layers.42.mlp.experts.45.gate_proj", + "model.layers.42.mlp.experts.45.up_proj", + "model.layers.42.mlp.experts.46.down_proj", + "model.layers.42.mlp.experts.46.gate_proj", + "model.layers.42.mlp.experts.46.up_proj", + "model.layers.42.mlp.experts.47.down_proj", + "model.layers.42.mlp.experts.47.gate_proj", + "model.layers.42.mlp.experts.47.up_proj", + "model.layers.42.mlp.experts.48.down_proj", + "model.layers.42.mlp.experts.48.gate_proj", + "model.layers.42.mlp.experts.48.up_proj", + "model.layers.42.mlp.experts.49.down_proj", + "model.layers.42.mlp.experts.49.gate_proj", + "model.layers.42.mlp.experts.49.up_proj", + "model.layers.42.mlp.experts.5.down_proj", + "model.layers.42.mlp.experts.5.gate_proj", + "model.layers.42.mlp.experts.5.up_proj", + "model.layers.42.mlp.experts.50.down_proj", + "model.layers.42.mlp.experts.50.gate_proj", + "model.layers.42.mlp.experts.50.up_proj", + "model.layers.42.mlp.experts.51.down_proj", + "model.layers.42.mlp.experts.51.gate_proj", + "model.layers.42.mlp.experts.51.up_proj", + "model.layers.42.mlp.experts.52.down_proj", + "model.layers.42.mlp.experts.52.gate_proj", + "model.layers.42.mlp.experts.52.up_proj", + "model.layers.42.mlp.experts.53.down_proj", + "model.layers.42.mlp.experts.53.gate_proj", + "model.layers.42.mlp.experts.53.up_proj", + "model.layers.42.mlp.experts.54.down_proj", + "model.layers.42.mlp.experts.54.gate_proj", + "model.layers.42.mlp.experts.54.up_proj", + "model.layers.42.mlp.experts.55.down_proj", + "model.layers.42.mlp.experts.55.gate_proj", + "model.layers.42.mlp.experts.55.up_proj", + "model.layers.42.mlp.experts.56.down_proj", + "model.layers.42.mlp.experts.56.gate_proj", + "model.layers.42.mlp.experts.56.up_proj", + "model.layers.42.mlp.experts.57.down_proj", + "model.layers.42.mlp.experts.57.gate_proj", + "model.layers.42.mlp.experts.57.up_proj", + "model.layers.42.mlp.experts.58.down_proj", + "model.layers.42.mlp.experts.58.gate_proj", + "model.layers.42.mlp.experts.58.up_proj", + "model.layers.42.mlp.experts.59.down_proj", + "model.layers.42.mlp.experts.59.gate_proj", + "model.layers.42.mlp.experts.59.up_proj", + "model.layers.42.mlp.experts.6.down_proj", + "model.layers.42.mlp.experts.6.gate_proj", + "model.layers.42.mlp.experts.6.up_proj", + "model.layers.42.mlp.experts.60.down_proj", + "model.layers.42.mlp.experts.60.gate_proj", + "model.layers.42.mlp.experts.60.up_proj", + "model.layers.42.mlp.experts.61.down_proj", + "model.layers.42.mlp.experts.61.gate_proj", + "model.layers.42.mlp.experts.61.up_proj", + "model.layers.42.mlp.experts.62.down_proj", + "model.layers.42.mlp.experts.62.gate_proj", + "model.layers.42.mlp.experts.62.up_proj", + "model.layers.42.mlp.experts.63.down_proj", + "model.layers.42.mlp.experts.63.gate_proj", + "model.layers.42.mlp.experts.63.up_proj", + "model.layers.42.mlp.experts.64.down_proj", + "model.layers.42.mlp.experts.64.gate_proj", + "model.layers.42.mlp.experts.64.up_proj", + "model.layers.42.mlp.experts.65.down_proj", + "model.layers.42.mlp.experts.65.gate_proj", + "model.layers.42.mlp.experts.65.up_proj", + "model.layers.42.mlp.experts.66.down_proj", + "model.layers.42.mlp.experts.66.gate_proj", + "model.layers.42.mlp.experts.66.up_proj", + "model.layers.42.mlp.experts.67.down_proj", + "model.layers.42.mlp.experts.67.gate_proj", + "model.layers.42.mlp.experts.67.up_proj", + "model.layers.42.mlp.experts.68.down_proj", + "model.layers.42.mlp.experts.68.gate_proj", + "model.layers.42.mlp.experts.68.up_proj", + "model.layers.42.mlp.experts.69.down_proj", + "model.layers.42.mlp.experts.69.gate_proj", + "model.layers.42.mlp.experts.69.up_proj", + "model.layers.42.mlp.experts.7.down_proj", + "model.layers.42.mlp.experts.7.gate_proj", + "model.layers.42.mlp.experts.7.up_proj", + "model.layers.42.mlp.experts.70.down_proj", + "model.layers.42.mlp.experts.70.gate_proj", + "model.layers.42.mlp.experts.70.up_proj", + "model.layers.42.mlp.experts.71.down_proj", + "model.layers.42.mlp.experts.71.gate_proj", + "model.layers.42.mlp.experts.71.up_proj", + "model.layers.42.mlp.experts.72.down_proj", + "model.layers.42.mlp.experts.72.gate_proj", + "model.layers.42.mlp.experts.72.up_proj", + "model.layers.42.mlp.experts.73.down_proj", + "model.layers.42.mlp.experts.73.gate_proj", + "model.layers.42.mlp.experts.73.up_proj", + "model.layers.42.mlp.experts.74.down_proj", + "model.layers.42.mlp.experts.74.gate_proj", + "model.layers.42.mlp.experts.74.up_proj", + "model.layers.42.mlp.experts.75.down_proj", + "model.layers.42.mlp.experts.75.gate_proj", + "model.layers.42.mlp.experts.75.up_proj", + "model.layers.42.mlp.experts.76.down_proj", + "model.layers.42.mlp.experts.76.gate_proj", + "model.layers.42.mlp.experts.76.up_proj", + "model.layers.42.mlp.experts.77.down_proj", + "model.layers.42.mlp.experts.77.gate_proj", + "model.layers.42.mlp.experts.77.up_proj", + "model.layers.42.mlp.experts.78.down_proj", + "model.layers.42.mlp.experts.78.gate_proj", + "model.layers.42.mlp.experts.78.up_proj", + "model.layers.42.mlp.experts.79.down_proj", + "model.layers.42.mlp.experts.79.gate_proj", + "model.layers.42.mlp.experts.79.up_proj", + "model.layers.42.mlp.experts.8.down_proj", + "model.layers.42.mlp.experts.8.gate_proj", + "model.layers.42.mlp.experts.8.up_proj", + "model.layers.42.mlp.experts.80.down_proj", + "model.layers.42.mlp.experts.80.gate_proj", + "model.layers.42.mlp.experts.80.up_proj", + "model.layers.42.mlp.experts.81.down_proj", + "model.layers.42.mlp.experts.81.gate_proj", + "model.layers.42.mlp.experts.81.up_proj", + "model.layers.42.mlp.experts.82.down_proj", + "model.layers.42.mlp.experts.82.gate_proj", + "model.layers.42.mlp.experts.82.up_proj", + "model.layers.42.mlp.experts.83.down_proj", + "model.layers.42.mlp.experts.83.gate_proj", + "model.layers.42.mlp.experts.83.up_proj", + "model.layers.42.mlp.experts.84.down_proj", + "model.layers.42.mlp.experts.84.gate_proj", + "model.layers.42.mlp.experts.84.up_proj", + "model.layers.42.mlp.experts.85.down_proj", + "model.layers.42.mlp.experts.85.gate_proj", + "model.layers.42.mlp.experts.85.up_proj", + "model.layers.42.mlp.experts.86.down_proj", + "model.layers.42.mlp.experts.86.gate_proj", + "model.layers.42.mlp.experts.86.up_proj", + "model.layers.42.mlp.experts.87.down_proj", + "model.layers.42.mlp.experts.87.gate_proj", + "model.layers.42.mlp.experts.87.up_proj", + "model.layers.42.mlp.experts.88.down_proj", + "model.layers.42.mlp.experts.88.gate_proj", + "model.layers.42.mlp.experts.88.up_proj", + "model.layers.42.mlp.experts.89.down_proj", + "model.layers.42.mlp.experts.89.gate_proj", + "model.layers.42.mlp.experts.89.up_proj", + "model.layers.42.mlp.experts.9.down_proj", + "model.layers.42.mlp.experts.9.gate_proj", + "model.layers.42.mlp.experts.9.up_proj", + "model.layers.42.mlp.experts.90.down_proj", + "model.layers.42.mlp.experts.90.gate_proj", + "model.layers.42.mlp.experts.90.up_proj", + "model.layers.42.mlp.experts.91.down_proj", + "model.layers.42.mlp.experts.91.gate_proj", + "model.layers.42.mlp.experts.91.up_proj", + "model.layers.42.mlp.experts.92.down_proj", + "model.layers.42.mlp.experts.92.gate_proj", + "model.layers.42.mlp.experts.92.up_proj", + "model.layers.42.mlp.experts.93.down_proj", + "model.layers.42.mlp.experts.93.gate_proj", + "model.layers.42.mlp.experts.93.up_proj", + "model.layers.42.mlp.experts.94.down_proj", + "model.layers.42.mlp.experts.94.gate_proj", + "model.layers.42.mlp.experts.94.up_proj", + "model.layers.42.mlp.experts.95.down_proj", + "model.layers.42.mlp.experts.95.gate_proj", + "model.layers.42.mlp.experts.95.up_proj", + "model.layers.42.mlp.experts.96.down_proj", + "model.layers.42.mlp.experts.96.gate_proj", + "model.layers.42.mlp.experts.96.up_proj", + "model.layers.42.mlp.experts.97.down_proj", + "model.layers.42.mlp.experts.97.gate_proj", + "model.layers.42.mlp.experts.97.up_proj", + "model.layers.42.mlp.experts.98.down_proj", + "model.layers.42.mlp.experts.98.gate_proj", + "model.layers.42.mlp.experts.98.up_proj", + "model.layers.42.mlp.experts.99.down_proj", + "model.layers.42.mlp.experts.99.gate_proj", + "model.layers.42.mlp.experts.99.up_proj", + "model.layers.43.mlp.experts.0.down_proj", + "model.layers.43.mlp.experts.0.gate_proj", + "model.layers.43.mlp.experts.0.up_proj", + "model.layers.43.mlp.experts.1.down_proj", + "model.layers.43.mlp.experts.1.gate_proj", + "model.layers.43.mlp.experts.1.up_proj", + "model.layers.43.mlp.experts.10.down_proj", + "model.layers.43.mlp.experts.10.gate_proj", + "model.layers.43.mlp.experts.10.up_proj", + "model.layers.43.mlp.experts.100.down_proj", + "model.layers.43.mlp.experts.100.gate_proj", + "model.layers.43.mlp.experts.100.up_proj", + "model.layers.43.mlp.experts.101.down_proj", + "model.layers.43.mlp.experts.101.gate_proj", + "model.layers.43.mlp.experts.101.up_proj", + "model.layers.43.mlp.experts.102.down_proj", + "model.layers.43.mlp.experts.102.gate_proj", + "model.layers.43.mlp.experts.102.up_proj", + "model.layers.43.mlp.experts.103.down_proj", + "model.layers.43.mlp.experts.103.gate_proj", + "model.layers.43.mlp.experts.103.up_proj", + "model.layers.43.mlp.experts.104.down_proj", + "model.layers.43.mlp.experts.104.gate_proj", + "model.layers.43.mlp.experts.104.up_proj", + "model.layers.43.mlp.experts.105.down_proj", + "model.layers.43.mlp.experts.105.gate_proj", + "model.layers.43.mlp.experts.105.up_proj", + "model.layers.43.mlp.experts.106.down_proj", + "model.layers.43.mlp.experts.106.gate_proj", + "model.layers.43.mlp.experts.106.up_proj", + "model.layers.43.mlp.experts.107.down_proj", + "model.layers.43.mlp.experts.107.gate_proj", + "model.layers.43.mlp.experts.107.up_proj", + "model.layers.43.mlp.experts.108.down_proj", + "model.layers.43.mlp.experts.108.gate_proj", + "model.layers.43.mlp.experts.108.up_proj", + "model.layers.43.mlp.experts.109.down_proj", + "model.layers.43.mlp.experts.109.gate_proj", + "model.layers.43.mlp.experts.109.up_proj", + "model.layers.43.mlp.experts.11.down_proj", + "model.layers.43.mlp.experts.11.gate_proj", + "model.layers.43.mlp.experts.11.up_proj", + "model.layers.43.mlp.experts.110.down_proj", + "model.layers.43.mlp.experts.110.gate_proj", + "model.layers.43.mlp.experts.110.up_proj", + "model.layers.43.mlp.experts.111.down_proj", + "model.layers.43.mlp.experts.111.gate_proj", + "model.layers.43.mlp.experts.111.up_proj", + "model.layers.43.mlp.experts.112.down_proj", + "model.layers.43.mlp.experts.112.gate_proj", + "model.layers.43.mlp.experts.112.up_proj", + "model.layers.43.mlp.experts.113.down_proj", + "model.layers.43.mlp.experts.113.gate_proj", + "model.layers.43.mlp.experts.113.up_proj", + "model.layers.43.mlp.experts.114.down_proj", + "model.layers.43.mlp.experts.114.gate_proj", + "model.layers.43.mlp.experts.114.up_proj", + "model.layers.43.mlp.experts.115.down_proj", + "model.layers.43.mlp.experts.115.gate_proj", + "model.layers.43.mlp.experts.115.up_proj", + "model.layers.43.mlp.experts.116.down_proj", + "model.layers.43.mlp.experts.116.gate_proj", + "model.layers.43.mlp.experts.116.up_proj", + "model.layers.43.mlp.experts.117.down_proj", + "model.layers.43.mlp.experts.117.gate_proj", + "model.layers.43.mlp.experts.117.up_proj", + "model.layers.43.mlp.experts.118.down_proj", + "model.layers.43.mlp.experts.118.gate_proj", + "model.layers.43.mlp.experts.118.up_proj", + "model.layers.43.mlp.experts.119.down_proj", + "model.layers.43.mlp.experts.119.gate_proj", + "model.layers.43.mlp.experts.119.up_proj", + "model.layers.43.mlp.experts.12.down_proj", + "model.layers.43.mlp.experts.12.gate_proj", + "model.layers.43.mlp.experts.12.up_proj", + "model.layers.43.mlp.experts.120.down_proj", + "model.layers.43.mlp.experts.120.gate_proj", + "model.layers.43.mlp.experts.120.up_proj", + "model.layers.43.mlp.experts.121.down_proj", + "model.layers.43.mlp.experts.121.gate_proj", + "model.layers.43.mlp.experts.121.up_proj", + "model.layers.43.mlp.experts.122.down_proj", + "model.layers.43.mlp.experts.122.gate_proj", + "model.layers.43.mlp.experts.122.up_proj", + "model.layers.43.mlp.experts.123.down_proj", + "model.layers.43.mlp.experts.123.gate_proj", + "model.layers.43.mlp.experts.123.up_proj", + "model.layers.43.mlp.experts.124.down_proj", + "model.layers.43.mlp.experts.124.gate_proj", + "model.layers.43.mlp.experts.124.up_proj", + "model.layers.43.mlp.experts.125.down_proj", + "model.layers.43.mlp.experts.125.gate_proj", + "model.layers.43.mlp.experts.125.up_proj", + "model.layers.43.mlp.experts.126.down_proj", + "model.layers.43.mlp.experts.126.gate_proj", + "model.layers.43.mlp.experts.126.up_proj", + "model.layers.43.mlp.experts.127.down_proj", + "model.layers.43.mlp.experts.127.gate_proj", + "model.layers.43.mlp.experts.127.up_proj", + "model.layers.43.mlp.experts.128.down_proj", + "model.layers.43.mlp.experts.128.gate_proj", + "model.layers.43.mlp.experts.128.up_proj", + "model.layers.43.mlp.experts.129.down_proj", + "model.layers.43.mlp.experts.129.gate_proj", + "model.layers.43.mlp.experts.129.up_proj", + "model.layers.43.mlp.experts.13.down_proj", + "model.layers.43.mlp.experts.13.gate_proj", + "model.layers.43.mlp.experts.13.up_proj", + "model.layers.43.mlp.experts.130.down_proj", + "model.layers.43.mlp.experts.130.gate_proj", + "model.layers.43.mlp.experts.130.up_proj", + "model.layers.43.mlp.experts.131.down_proj", + "model.layers.43.mlp.experts.131.gate_proj", + "model.layers.43.mlp.experts.131.up_proj", + "model.layers.43.mlp.experts.132.down_proj", + "model.layers.43.mlp.experts.132.gate_proj", + "model.layers.43.mlp.experts.132.up_proj", + "model.layers.43.mlp.experts.133.down_proj", + "model.layers.43.mlp.experts.133.gate_proj", + "model.layers.43.mlp.experts.133.up_proj", + "model.layers.43.mlp.experts.134.down_proj", + "model.layers.43.mlp.experts.134.gate_proj", + "model.layers.43.mlp.experts.134.up_proj", + "model.layers.43.mlp.experts.135.down_proj", + "model.layers.43.mlp.experts.135.gate_proj", + "model.layers.43.mlp.experts.135.up_proj", + "model.layers.43.mlp.experts.136.down_proj", + "model.layers.43.mlp.experts.136.gate_proj", + "model.layers.43.mlp.experts.136.up_proj", + "model.layers.43.mlp.experts.137.down_proj", + "model.layers.43.mlp.experts.137.gate_proj", + "model.layers.43.mlp.experts.137.up_proj", + "model.layers.43.mlp.experts.138.down_proj", + "model.layers.43.mlp.experts.138.gate_proj", + "model.layers.43.mlp.experts.138.up_proj", + "model.layers.43.mlp.experts.139.down_proj", + "model.layers.43.mlp.experts.139.gate_proj", + "model.layers.43.mlp.experts.139.up_proj", + "model.layers.43.mlp.experts.14.down_proj", + "model.layers.43.mlp.experts.14.gate_proj", + "model.layers.43.mlp.experts.14.up_proj", + "model.layers.43.mlp.experts.140.down_proj", + "model.layers.43.mlp.experts.140.gate_proj", + "model.layers.43.mlp.experts.140.up_proj", + "model.layers.43.mlp.experts.141.down_proj", + "model.layers.43.mlp.experts.141.gate_proj", + "model.layers.43.mlp.experts.141.up_proj", + "model.layers.43.mlp.experts.142.down_proj", + "model.layers.43.mlp.experts.142.gate_proj", + "model.layers.43.mlp.experts.142.up_proj", + "model.layers.43.mlp.experts.143.down_proj", + "model.layers.43.mlp.experts.143.gate_proj", + "model.layers.43.mlp.experts.143.up_proj", + "model.layers.43.mlp.experts.144.down_proj", + "model.layers.43.mlp.experts.144.gate_proj", + "model.layers.43.mlp.experts.144.up_proj", + "model.layers.43.mlp.experts.145.down_proj", + "model.layers.43.mlp.experts.145.gate_proj", + "model.layers.43.mlp.experts.145.up_proj", + "model.layers.43.mlp.experts.146.down_proj", + "model.layers.43.mlp.experts.146.gate_proj", + "model.layers.43.mlp.experts.146.up_proj", + "model.layers.43.mlp.experts.147.down_proj", + "model.layers.43.mlp.experts.147.gate_proj", + "model.layers.43.mlp.experts.147.up_proj", + "model.layers.43.mlp.experts.148.down_proj", + "model.layers.43.mlp.experts.148.gate_proj", + "model.layers.43.mlp.experts.148.up_proj", + "model.layers.43.mlp.experts.149.down_proj", + "model.layers.43.mlp.experts.149.gate_proj", + "model.layers.43.mlp.experts.149.up_proj", + "model.layers.43.mlp.experts.15.down_proj", + "model.layers.43.mlp.experts.15.gate_proj", + "model.layers.43.mlp.experts.15.up_proj", + "model.layers.43.mlp.experts.150.down_proj", + "model.layers.43.mlp.experts.150.gate_proj", + "model.layers.43.mlp.experts.150.up_proj", + "model.layers.43.mlp.experts.151.down_proj", + "model.layers.43.mlp.experts.151.gate_proj", + "model.layers.43.mlp.experts.151.up_proj", + "model.layers.43.mlp.experts.152.down_proj", + "model.layers.43.mlp.experts.152.gate_proj", + "model.layers.43.mlp.experts.152.up_proj", + "model.layers.43.mlp.experts.153.down_proj", + "model.layers.43.mlp.experts.153.gate_proj", + "model.layers.43.mlp.experts.153.up_proj", + "model.layers.43.mlp.experts.154.down_proj", + "model.layers.43.mlp.experts.154.gate_proj", + "model.layers.43.mlp.experts.154.up_proj", + "model.layers.43.mlp.experts.155.down_proj", + "model.layers.43.mlp.experts.155.gate_proj", + "model.layers.43.mlp.experts.155.up_proj", + "model.layers.43.mlp.experts.156.down_proj", + "model.layers.43.mlp.experts.156.gate_proj", + "model.layers.43.mlp.experts.156.up_proj", + "model.layers.43.mlp.experts.157.down_proj", + "model.layers.43.mlp.experts.157.gate_proj", + "model.layers.43.mlp.experts.157.up_proj", + "model.layers.43.mlp.experts.158.down_proj", + "model.layers.43.mlp.experts.158.gate_proj", + "model.layers.43.mlp.experts.158.up_proj", + "model.layers.43.mlp.experts.159.down_proj", + "model.layers.43.mlp.experts.159.gate_proj", + "model.layers.43.mlp.experts.159.up_proj", + "model.layers.43.mlp.experts.16.down_proj", + "model.layers.43.mlp.experts.16.gate_proj", + "model.layers.43.mlp.experts.16.up_proj", + "model.layers.43.mlp.experts.160.down_proj", + "model.layers.43.mlp.experts.160.gate_proj", + "model.layers.43.mlp.experts.160.up_proj", + "model.layers.43.mlp.experts.161.down_proj", + "model.layers.43.mlp.experts.161.gate_proj", + "model.layers.43.mlp.experts.161.up_proj", + "model.layers.43.mlp.experts.162.down_proj", + "model.layers.43.mlp.experts.162.gate_proj", + "model.layers.43.mlp.experts.162.up_proj", + "model.layers.43.mlp.experts.163.down_proj", + "model.layers.43.mlp.experts.163.gate_proj", + "model.layers.43.mlp.experts.163.up_proj", + "model.layers.43.mlp.experts.164.down_proj", + "model.layers.43.mlp.experts.164.gate_proj", + "model.layers.43.mlp.experts.164.up_proj", + "model.layers.43.mlp.experts.165.down_proj", + "model.layers.43.mlp.experts.165.gate_proj", + "model.layers.43.mlp.experts.165.up_proj", + "model.layers.43.mlp.experts.166.down_proj", + "model.layers.43.mlp.experts.166.gate_proj", + "model.layers.43.mlp.experts.166.up_proj", + "model.layers.43.mlp.experts.167.down_proj", + "model.layers.43.mlp.experts.167.gate_proj", + "model.layers.43.mlp.experts.167.up_proj", + "model.layers.43.mlp.experts.168.down_proj", + "model.layers.43.mlp.experts.168.gate_proj", + "model.layers.43.mlp.experts.168.up_proj", + "model.layers.43.mlp.experts.169.down_proj", + "model.layers.43.mlp.experts.169.gate_proj", + "model.layers.43.mlp.experts.169.up_proj", + "model.layers.43.mlp.experts.17.down_proj", + "model.layers.43.mlp.experts.17.gate_proj", + "model.layers.43.mlp.experts.17.up_proj", + "model.layers.43.mlp.experts.170.down_proj", + "model.layers.43.mlp.experts.170.gate_proj", + "model.layers.43.mlp.experts.170.up_proj", + "model.layers.43.mlp.experts.171.down_proj", + "model.layers.43.mlp.experts.171.gate_proj", + "model.layers.43.mlp.experts.171.up_proj", + "model.layers.43.mlp.experts.172.down_proj", + "model.layers.43.mlp.experts.172.gate_proj", + "model.layers.43.mlp.experts.172.up_proj", + "model.layers.43.mlp.experts.173.down_proj", + "model.layers.43.mlp.experts.173.gate_proj", + "model.layers.43.mlp.experts.173.up_proj", + "model.layers.43.mlp.experts.174.down_proj", + "model.layers.43.mlp.experts.174.gate_proj", + "model.layers.43.mlp.experts.174.up_proj", + "model.layers.43.mlp.experts.175.down_proj", + "model.layers.43.mlp.experts.175.gate_proj", + "model.layers.43.mlp.experts.175.up_proj", + "model.layers.43.mlp.experts.176.down_proj", + "model.layers.43.mlp.experts.176.gate_proj", + "model.layers.43.mlp.experts.176.up_proj", + "model.layers.43.mlp.experts.177.down_proj", + "model.layers.43.mlp.experts.177.gate_proj", + "model.layers.43.mlp.experts.177.up_proj", + "model.layers.43.mlp.experts.178.down_proj", + "model.layers.43.mlp.experts.178.gate_proj", + "model.layers.43.mlp.experts.178.up_proj", + "model.layers.43.mlp.experts.179.down_proj", + "model.layers.43.mlp.experts.179.gate_proj", + "model.layers.43.mlp.experts.179.up_proj", + "model.layers.43.mlp.experts.18.down_proj", + "model.layers.43.mlp.experts.18.gate_proj", + "model.layers.43.mlp.experts.18.up_proj", + "model.layers.43.mlp.experts.180.down_proj", + "model.layers.43.mlp.experts.180.gate_proj", + "model.layers.43.mlp.experts.180.up_proj", + "model.layers.43.mlp.experts.181.down_proj", + "model.layers.43.mlp.experts.181.gate_proj", + "model.layers.43.mlp.experts.181.up_proj", + "model.layers.43.mlp.experts.182.down_proj", + "model.layers.43.mlp.experts.182.gate_proj", + "model.layers.43.mlp.experts.182.up_proj", + "model.layers.43.mlp.experts.183.down_proj", + "model.layers.43.mlp.experts.183.gate_proj", + "model.layers.43.mlp.experts.183.up_proj", + "model.layers.43.mlp.experts.184.down_proj", + "model.layers.43.mlp.experts.184.gate_proj", + "model.layers.43.mlp.experts.184.up_proj", + "model.layers.43.mlp.experts.185.down_proj", + "model.layers.43.mlp.experts.185.gate_proj", + "model.layers.43.mlp.experts.185.up_proj", + "model.layers.43.mlp.experts.186.down_proj", + "model.layers.43.mlp.experts.186.gate_proj", + "model.layers.43.mlp.experts.186.up_proj", + "model.layers.43.mlp.experts.187.down_proj", + "model.layers.43.mlp.experts.187.gate_proj", + "model.layers.43.mlp.experts.187.up_proj", + "model.layers.43.mlp.experts.188.down_proj", + "model.layers.43.mlp.experts.188.gate_proj", + "model.layers.43.mlp.experts.188.up_proj", + "model.layers.43.mlp.experts.189.down_proj", + "model.layers.43.mlp.experts.189.gate_proj", + "model.layers.43.mlp.experts.189.up_proj", + "model.layers.43.mlp.experts.19.down_proj", + "model.layers.43.mlp.experts.19.gate_proj", + "model.layers.43.mlp.experts.19.up_proj", + "model.layers.43.mlp.experts.190.down_proj", + "model.layers.43.mlp.experts.190.gate_proj", + "model.layers.43.mlp.experts.190.up_proj", + "model.layers.43.mlp.experts.191.down_proj", + "model.layers.43.mlp.experts.191.gate_proj", + "model.layers.43.mlp.experts.191.up_proj", + "model.layers.43.mlp.experts.192.down_proj", + "model.layers.43.mlp.experts.192.gate_proj", + "model.layers.43.mlp.experts.192.up_proj", + "model.layers.43.mlp.experts.193.down_proj", + "model.layers.43.mlp.experts.193.gate_proj", + "model.layers.43.mlp.experts.193.up_proj", + "model.layers.43.mlp.experts.194.down_proj", + "model.layers.43.mlp.experts.194.gate_proj", + "model.layers.43.mlp.experts.194.up_proj", + "model.layers.43.mlp.experts.195.down_proj", + "model.layers.43.mlp.experts.195.gate_proj", + "model.layers.43.mlp.experts.195.up_proj", + "model.layers.43.mlp.experts.196.down_proj", + "model.layers.43.mlp.experts.196.gate_proj", + "model.layers.43.mlp.experts.196.up_proj", + "model.layers.43.mlp.experts.197.down_proj", + "model.layers.43.mlp.experts.197.gate_proj", + "model.layers.43.mlp.experts.197.up_proj", + "model.layers.43.mlp.experts.198.down_proj", + "model.layers.43.mlp.experts.198.gate_proj", + "model.layers.43.mlp.experts.198.up_proj", + "model.layers.43.mlp.experts.199.down_proj", + "model.layers.43.mlp.experts.199.gate_proj", + "model.layers.43.mlp.experts.199.up_proj", + "model.layers.43.mlp.experts.2.down_proj", + "model.layers.43.mlp.experts.2.gate_proj", + "model.layers.43.mlp.experts.2.up_proj", + "model.layers.43.mlp.experts.20.down_proj", + "model.layers.43.mlp.experts.20.gate_proj", + "model.layers.43.mlp.experts.20.up_proj", + "model.layers.43.mlp.experts.200.down_proj", + "model.layers.43.mlp.experts.200.gate_proj", + "model.layers.43.mlp.experts.200.up_proj", + "model.layers.43.mlp.experts.201.down_proj", + "model.layers.43.mlp.experts.201.gate_proj", + "model.layers.43.mlp.experts.201.up_proj", + "model.layers.43.mlp.experts.202.down_proj", + "model.layers.43.mlp.experts.202.gate_proj", + "model.layers.43.mlp.experts.202.up_proj", + "model.layers.43.mlp.experts.203.down_proj", + "model.layers.43.mlp.experts.203.gate_proj", + "model.layers.43.mlp.experts.203.up_proj", + "model.layers.43.mlp.experts.204.down_proj", + "model.layers.43.mlp.experts.204.gate_proj", + "model.layers.43.mlp.experts.204.up_proj", + "model.layers.43.mlp.experts.205.down_proj", + "model.layers.43.mlp.experts.205.gate_proj", + "model.layers.43.mlp.experts.205.up_proj", + "model.layers.43.mlp.experts.206.down_proj", + "model.layers.43.mlp.experts.206.gate_proj", + "model.layers.43.mlp.experts.206.up_proj", + "model.layers.43.mlp.experts.207.down_proj", + "model.layers.43.mlp.experts.207.gate_proj", + "model.layers.43.mlp.experts.207.up_proj", + "model.layers.43.mlp.experts.208.down_proj", + "model.layers.43.mlp.experts.208.gate_proj", + "model.layers.43.mlp.experts.208.up_proj", + "model.layers.43.mlp.experts.209.down_proj", + "model.layers.43.mlp.experts.209.gate_proj", + "model.layers.43.mlp.experts.209.up_proj", + "model.layers.43.mlp.experts.21.down_proj", + "model.layers.43.mlp.experts.21.gate_proj", + "model.layers.43.mlp.experts.21.up_proj", + "model.layers.43.mlp.experts.210.down_proj", + "model.layers.43.mlp.experts.210.gate_proj", + "model.layers.43.mlp.experts.210.up_proj", + "model.layers.43.mlp.experts.211.down_proj", + "model.layers.43.mlp.experts.211.gate_proj", + "model.layers.43.mlp.experts.211.up_proj", + "model.layers.43.mlp.experts.212.down_proj", + "model.layers.43.mlp.experts.212.gate_proj", + "model.layers.43.mlp.experts.212.up_proj", + "model.layers.43.mlp.experts.213.down_proj", + "model.layers.43.mlp.experts.213.gate_proj", + "model.layers.43.mlp.experts.213.up_proj", + "model.layers.43.mlp.experts.214.down_proj", + "model.layers.43.mlp.experts.214.gate_proj", + "model.layers.43.mlp.experts.214.up_proj", + "model.layers.43.mlp.experts.215.down_proj", + "model.layers.43.mlp.experts.215.gate_proj", + "model.layers.43.mlp.experts.215.up_proj", + "model.layers.43.mlp.experts.216.down_proj", + "model.layers.43.mlp.experts.216.gate_proj", + "model.layers.43.mlp.experts.216.up_proj", + "model.layers.43.mlp.experts.217.down_proj", + "model.layers.43.mlp.experts.217.gate_proj", + "model.layers.43.mlp.experts.217.up_proj", + "model.layers.43.mlp.experts.218.down_proj", + "model.layers.43.mlp.experts.218.gate_proj", + "model.layers.43.mlp.experts.218.up_proj", + "model.layers.43.mlp.experts.219.down_proj", + "model.layers.43.mlp.experts.219.gate_proj", + "model.layers.43.mlp.experts.219.up_proj", + "model.layers.43.mlp.experts.22.down_proj", + "model.layers.43.mlp.experts.22.gate_proj", + "model.layers.43.mlp.experts.22.up_proj", + "model.layers.43.mlp.experts.220.down_proj", + "model.layers.43.mlp.experts.220.gate_proj", + "model.layers.43.mlp.experts.220.up_proj", + "model.layers.43.mlp.experts.221.down_proj", + "model.layers.43.mlp.experts.221.gate_proj", + "model.layers.43.mlp.experts.221.up_proj", + "model.layers.43.mlp.experts.222.down_proj", + "model.layers.43.mlp.experts.222.gate_proj", + "model.layers.43.mlp.experts.222.up_proj", + "model.layers.43.mlp.experts.223.down_proj", + "model.layers.43.mlp.experts.223.gate_proj", + "model.layers.43.mlp.experts.223.up_proj", + "model.layers.43.mlp.experts.224.down_proj", + "model.layers.43.mlp.experts.224.gate_proj", + "model.layers.43.mlp.experts.224.up_proj", + "model.layers.43.mlp.experts.225.down_proj", + "model.layers.43.mlp.experts.225.gate_proj", + "model.layers.43.mlp.experts.225.up_proj", + "model.layers.43.mlp.experts.226.down_proj", + "model.layers.43.mlp.experts.226.gate_proj", + "model.layers.43.mlp.experts.226.up_proj", + "model.layers.43.mlp.experts.227.down_proj", + "model.layers.43.mlp.experts.227.gate_proj", + "model.layers.43.mlp.experts.227.up_proj", + "model.layers.43.mlp.experts.228.down_proj", + "model.layers.43.mlp.experts.228.gate_proj", + "model.layers.43.mlp.experts.228.up_proj", + "model.layers.43.mlp.experts.229.down_proj", + "model.layers.43.mlp.experts.229.gate_proj", + "model.layers.43.mlp.experts.229.up_proj", + "model.layers.43.mlp.experts.23.down_proj", + "model.layers.43.mlp.experts.23.gate_proj", + "model.layers.43.mlp.experts.23.up_proj", + "model.layers.43.mlp.experts.230.down_proj", + "model.layers.43.mlp.experts.230.gate_proj", + "model.layers.43.mlp.experts.230.up_proj", + "model.layers.43.mlp.experts.231.down_proj", + "model.layers.43.mlp.experts.231.gate_proj", + "model.layers.43.mlp.experts.231.up_proj", + "model.layers.43.mlp.experts.232.down_proj", + "model.layers.43.mlp.experts.232.gate_proj", + "model.layers.43.mlp.experts.232.up_proj", + "model.layers.43.mlp.experts.233.down_proj", + "model.layers.43.mlp.experts.233.gate_proj", + "model.layers.43.mlp.experts.233.up_proj", + "model.layers.43.mlp.experts.234.down_proj", + "model.layers.43.mlp.experts.234.gate_proj", + "model.layers.43.mlp.experts.234.up_proj", + "model.layers.43.mlp.experts.235.down_proj", + "model.layers.43.mlp.experts.235.gate_proj", + "model.layers.43.mlp.experts.235.up_proj", + "model.layers.43.mlp.experts.236.down_proj", + "model.layers.43.mlp.experts.236.gate_proj", + "model.layers.43.mlp.experts.236.up_proj", + "model.layers.43.mlp.experts.237.down_proj", + "model.layers.43.mlp.experts.237.gate_proj", + "model.layers.43.mlp.experts.237.up_proj", + "model.layers.43.mlp.experts.238.down_proj", + "model.layers.43.mlp.experts.238.gate_proj", + "model.layers.43.mlp.experts.238.up_proj", + "model.layers.43.mlp.experts.239.down_proj", + "model.layers.43.mlp.experts.239.gate_proj", + "model.layers.43.mlp.experts.239.up_proj", + "model.layers.43.mlp.experts.24.down_proj", + "model.layers.43.mlp.experts.24.gate_proj", + "model.layers.43.mlp.experts.24.up_proj", + "model.layers.43.mlp.experts.240.down_proj", + "model.layers.43.mlp.experts.240.gate_proj", + "model.layers.43.mlp.experts.240.up_proj", + "model.layers.43.mlp.experts.241.down_proj", + "model.layers.43.mlp.experts.241.gate_proj", + "model.layers.43.mlp.experts.241.up_proj", + "model.layers.43.mlp.experts.242.down_proj", + "model.layers.43.mlp.experts.242.gate_proj", + "model.layers.43.mlp.experts.242.up_proj", + "model.layers.43.mlp.experts.243.down_proj", + "model.layers.43.mlp.experts.243.gate_proj", + "model.layers.43.mlp.experts.243.up_proj", + "model.layers.43.mlp.experts.244.down_proj", + "model.layers.43.mlp.experts.244.gate_proj", + "model.layers.43.mlp.experts.244.up_proj", + "model.layers.43.mlp.experts.245.down_proj", + "model.layers.43.mlp.experts.245.gate_proj", + "model.layers.43.mlp.experts.245.up_proj", + "model.layers.43.mlp.experts.246.down_proj", + "model.layers.43.mlp.experts.246.gate_proj", + "model.layers.43.mlp.experts.246.up_proj", + "model.layers.43.mlp.experts.247.down_proj", + "model.layers.43.mlp.experts.247.gate_proj", + "model.layers.43.mlp.experts.247.up_proj", + "model.layers.43.mlp.experts.248.down_proj", + "model.layers.43.mlp.experts.248.gate_proj", + "model.layers.43.mlp.experts.248.up_proj", + "model.layers.43.mlp.experts.249.down_proj", + "model.layers.43.mlp.experts.249.gate_proj", + "model.layers.43.mlp.experts.249.up_proj", + "model.layers.43.mlp.experts.25.down_proj", + "model.layers.43.mlp.experts.25.gate_proj", + "model.layers.43.mlp.experts.25.up_proj", + "model.layers.43.mlp.experts.250.down_proj", + "model.layers.43.mlp.experts.250.gate_proj", + "model.layers.43.mlp.experts.250.up_proj", + "model.layers.43.mlp.experts.251.down_proj", + "model.layers.43.mlp.experts.251.gate_proj", + "model.layers.43.mlp.experts.251.up_proj", + "model.layers.43.mlp.experts.252.down_proj", + "model.layers.43.mlp.experts.252.gate_proj", + "model.layers.43.mlp.experts.252.up_proj", + "model.layers.43.mlp.experts.253.down_proj", + "model.layers.43.mlp.experts.253.gate_proj", + "model.layers.43.mlp.experts.253.up_proj", + "model.layers.43.mlp.experts.254.down_proj", + "model.layers.43.mlp.experts.254.gate_proj", + "model.layers.43.mlp.experts.254.up_proj", + "model.layers.43.mlp.experts.255.down_proj", + "model.layers.43.mlp.experts.255.gate_proj", + "model.layers.43.mlp.experts.255.up_proj", + "model.layers.43.mlp.experts.26.down_proj", + "model.layers.43.mlp.experts.26.gate_proj", + "model.layers.43.mlp.experts.26.up_proj", + "model.layers.43.mlp.experts.27.down_proj", + "model.layers.43.mlp.experts.27.gate_proj", + "model.layers.43.mlp.experts.27.up_proj", + "model.layers.43.mlp.experts.28.down_proj", + "model.layers.43.mlp.experts.28.gate_proj", + "model.layers.43.mlp.experts.28.up_proj", + "model.layers.43.mlp.experts.29.down_proj", + "model.layers.43.mlp.experts.29.gate_proj", + "model.layers.43.mlp.experts.29.up_proj", + "model.layers.43.mlp.experts.3.down_proj", + "model.layers.43.mlp.experts.3.gate_proj", + "model.layers.43.mlp.experts.3.up_proj", + "model.layers.43.mlp.experts.30.down_proj", + "model.layers.43.mlp.experts.30.gate_proj", + "model.layers.43.mlp.experts.30.up_proj", + "model.layers.43.mlp.experts.31.down_proj", + "model.layers.43.mlp.experts.31.gate_proj", + "model.layers.43.mlp.experts.31.up_proj", + "model.layers.43.mlp.experts.32.down_proj", + "model.layers.43.mlp.experts.32.gate_proj", + "model.layers.43.mlp.experts.32.up_proj", + "model.layers.43.mlp.experts.33.down_proj", + "model.layers.43.mlp.experts.33.gate_proj", + "model.layers.43.mlp.experts.33.up_proj", + "model.layers.43.mlp.experts.34.down_proj", + "model.layers.43.mlp.experts.34.gate_proj", + "model.layers.43.mlp.experts.34.up_proj", + "model.layers.43.mlp.experts.35.down_proj", + "model.layers.43.mlp.experts.35.gate_proj", + "model.layers.43.mlp.experts.35.up_proj", + "model.layers.43.mlp.experts.36.down_proj", + "model.layers.43.mlp.experts.36.gate_proj", + "model.layers.43.mlp.experts.36.up_proj", + "model.layers.43.mlp.experts.37.down_proj", + "model.layers.43.mlp.experts.37.gate_proj", + "model.layers.43.mlp.experts.37.up_proj", + "model.layers.43.mlp.experts.38.down_proj", + "model.layers.43.mlp.experts.38.gate_proj", + "model.layers.43.mlp.experts.38.up_proj", + "model.layers.43.mlp.experts.39.down_proj", + "model.layers.43.mlp.experts.39.gate_proj", + "model.layers.43.mlp.experts.39.up_proj", + "model.layers.43.mlp.experts.4.down_proj", + "model.layers.43.mlp.experts.4.gate_proj", + "model.layers.43.mlp.experts.4.up_proj", + "model.layers.43.mlp.experts.40.down_proj", + "model.layers.43.mlp.experts.40.gate_proj", + "model.layers.43.mlp.experts.40.up_proj", + "model.layers.43.mlp.experts.41.down_proj", + "model.layers.43.mlp.experts.41.gate_proj", + "model.layers.43.mlp.experts.41.up_proj", + "model.layers.43.mlp.experts.42.down_proj", + "model.layers.43.mlp.experts.42.gate_proj", + "model.layers.43.mlp.experts.42.up_proj", + "model.layers.43.mlp.experts.43.down_proj", + "model.layers.43.mlp.experts.43.gate_proj", + "model.layers.43.mlp.experts.43.up_proj", + "model.layers.43.mlp.experts.44.down_proj", + "model.layers.43.mlp.experts.44.gate_proj", + "model.layers.43.mlp.experts.44.up_proj", + "model.layers.43.mlp.experts.45.down_proj", + "model.layers.43.mlp.experts.45.gate_proj", + "model.layers.43.mlp.experts.45.up_proj", + "model.layers.43.mlp.experts.46.down_proj", + "model.layers.43.mlp.experts.46.gate_proj", + "model.layers.43.mlp.experts.46.up_proj", + "model.layers.43.mlp.experts.47.down_proj", + "model.layers.43.mlp.experts.47.gate_proj", + "model.layers.43.mlp.experts.47.up_proj", + "model.layers.43.mlp.experts.48.down_proj", + "model.layers.43.mlp.experts.48.gate_proj", + "model.layers.43.mlp.experts.48.up_proj", + "model.layers.43.mlp.experts.49.down_proj", + "model.layers.43.mlp.experts.49.gate_proj", + "model.layers.43.mlp.experts.49.up_proj", + "model.layers.43.mlp.experts.5.down_proj", + "model.layers.43.mlp.experts.5.gate_proj", + "model.layers.43.mlp.experts.5.up_proj", + "model.layers.43.mlp.experts.50.down_proj", + "model.layers.43.mlp.experts.50.gate_proj", + "model.layers.43.mlp.experts.50.up_proj", + "model.layers.43.mlp.experts.51.down_proj", + "model.layers.43.mlp.experts.51.gate_proj", + "model.layers.43.mlp.experts.51.up_proj", + "model.layers.43.mlp.experts.52.down_proj", + "model.layers.43.mlp.experts.52.gate_proj", + "model.layers.43.mlp.experts.52.up_proj", + "model.layers.43.mlp.experts.53.down_proj", + "model.layers.43.mlp.experts.53.gate_proj", + "model.layers.43.mlp.experts.53.up_proj", + "model.layers.43.mlp.experts.54.down_proj", + "model.layers.43.mlp.experts.54.gate_proj", + "model.layers.43.mlp.experts.54.up_proj", + "model.layers.43.mlp.experts.55.down_proj", + "model.layers.43.mlp.experts.55.gate_proj", + "model.layers.43.mlp.experts.55.up_proj", + "model.layers.43.mlp.experts.56.down_proj", + "model.layers.43.mlp.experts.56.gate_proj", + "model.layers.43.mlp.experts.56.up_proj", + "model.layers.43.mlp.experts.57.down_proj", + "model.layers.43.mlp.experts.57.gate_proj", + "model.layers.43.mlp.experts.57.up_proj", + "model.layers.43.mlp.experts.58.down_proj", + "model.layers.43.mlp.experts.58.gate_proj", + "model.layers.43.mlp.experts.58.up_proj", + "model.layers.43.mlp.experts.59.down_proj", + "model.layers.43.mlp.experts.59.gate_proj", + "model.layers.43.mlp.experts.59.up_proj", + "model.layers.43.mlp.experts.6.down_proj", + "model.layers.43.mlp.experts.6.gate_proj", + "model.layers.43.mlp.experts.6.up_proj", + "model.layers.43.mlp.experts.60.down_proj", + "model.layers.43.mlp.experts.60.gate_proj", + "model.layers.43.mlp.experts.60.up_proj", + "model.layers.43.mlp.experts.61.down_proj", + "model.layers.43.mlp.experts.61.gate_proj", + "model.layers.43.mlp.experts.61.up_proj", + "model.layers.43.mlp.experts.62.down_proj", + "model.layers.43.mlp.experts.62.gate_proj", + "model.layers.43.mlp.experts.62.up_proj", + "model.layers.43.mlp.experts.63.down_proj", + "model.layers.43.mlp.experts.63.gate_proj", + "model.layers.43.mlp.experts.63.up_proj", + "model.layers.43.mlp.experts.64.down_proj", + "model.layers.43.mlp.experts.64.gate_proj", + "model.layers.43.mlp.experts.64.up_proj", + "model.layers.43.mlp.experts.65.down_proj", + "model.layers.43.mlp.experts.65.gate_proj", + "model.layers.43.mlp.experts.65.up_proj", + "model.layers.43.mlp.experts.66.down_proj", + "model.layers.43.mlp.experts.66.gate_proj", + "model.layers.43.mlp.experts.66.up_proj", + "model.layers.43.mlp.experts.67.down_proj", + "model.layers.43.mlp.experts.67.gate_proj", + "model.layers.43.mlp.experts.67.up_proj", + "model.layers.43.mlp.experts.68.down_proj", + "model.layers.43.mlp.experts.68.gate_proj", + "model.layers.43.mlp.experts.68.up_proj", + "model.layers.43.mlp.experts.69.down_proj", + "model.layers.43.mlp.experts.69.gate_proj", + "model.layers.43.mlp.experts.69.up_proj", + "model.layers.43.mlp.experts.7.down_proj", + "model.layers.43.mlp.experts.7.gate_proj", + "model.layers.43.mlp.experts.7.up_proj", + "model.layers.43.mlp.experts.70.down_proj", + "model.layers.43.mlp.experts.70.gate_proj", + "model.layers.43.mlp.experts.70.up_proj", + "model.layers.43.mlp.experts.71.down_proj", + "model.layers.43.mlp.experts.71.gate_proj", + "model.layers.43.mlp.experts.71.up_proj", + "model.layers.43.mlp.experts.72.down_proj", + "model.layers.43.mlp.experts.72.gate_proj", + "model.layers.43.mlp.experts.72.up_proj", + "model.layers.43.mlp.experts.73.down_proj", + "model.layers.43.mlp.experts.73.gate_proj", + "model.layers.43.mlp.experts.73.up_proj", + "model.layers.43.mlp.experts.74.down_proj", + "model.layers.43.mlp.experts.74.gate_proj", + "model.layers.43.mlp.experts.74.up_proj", + "model.layers.43.mlp.experts.75.down_proj", + "model.layers.43.mlp.experts.75.gate_proj", + "model.layers.43.mlp.experts.75.up_proj", + "model.layers.43.mlp.experts.76.down_proj", + "model.layers.43.mlp.experts.76.gate_proj", + "model.layers.43.mlp.experts.76.up_proj", + "model.layers.43.mlp.experts.77.down_proj", + "model.layers.43.mlp.experts.77.gate_proj", + "model.layers.43.mlp.experts.77.up_proj", + "model.layers.43.mlp.experts.78.down_proj", + "model.layers.43.mlp.experts.78.gate_proj", + "model.layers.43.mlp.experts.78.up_proj", + "model.layers.43.mlp.experts.79.down_proj", + "model.layers.43.mlp.experts.79.gate_proj", + "model.layers.43.mlp.experts.79.up_proj", + "model.layers.43.mlp.experts.8.down_proj", + "model.layers.43.mlp.experts.8.gate_proj", + "model.layers.43.mlp.experts.8.up_proj", + "model.layers.43.mlp.experts.80.down_proj", + "model.layers.43.mlp.experts.80.gate_proj", + "model.layers.43.mlp.experts.80.up_proj", + "model.layers.43.mlp.experts.81.down_proj", + "model.layers.43.mlp.experts.81.gate_proj", + "model.layers.43.mlp.experts.81.up_proj", + "model.layers.43.mlp.experts.82.down_proj", + "model.layers.43.mlp.experts.82.gate_proj", + "model.layers.43.mlp.experts.82.up_proj", + "model.layers.43.mlp.experts.83.down_proj", + "model.layers.43.mlp.experts.83.gate_proj", + "model.layers.43.mlp.experts.83.up_proj", + "model.layers.43.mlp.experts.84.down_proj", + "model.layers.43.mlp.experts.84.gate_proj", + "model.layers.43.mlp.experts.84.up_proj", + "model.layers.43.mlp.experts.85.down_proj", + "model.layers.43.mlp.experts.85.gate_proj", + "model.layers.43.mlp.experts.85.up_proj", + "model.layers.43.mlp.experts.86.down_proj", + "model.layers.43.mlp.experts.86.gate_proj", + "model.layers.43.mlp.experts.86.up_proj", + "model.layers.43.mlp.experts.87.down_proj", + "model.layers.43.mlp.experts.87.gate_proj", + "model.layers.43.mlp.experts.87.up_proj", + "model.layers.43.mlp.experts.88.down_proj", + "model.layers.43.mlp.experts.88.gate_proj", + "model.layers.43.mlp.experts.88.up_proj", + "model.layers.43.mlp.experts.89.down_proj", + "model.layers.43.mlp.experts.89.gate_proj", + "model.layers.43.mlp.experts.89.up_proj", + "model.layers.43.mlp.experts.9.down_proj", + "model.layers.43.mlp.experts.9.gate_proj", + "model.layers.43.mlp.experts.9.up_proj", + "model.layers.43.mlp.experts.90.down_proj", + "model.layers.43.mlp.experts.90.gate_proj", + "model.layers.43.mlp.experts.90.up_proj", + "model.layers.43.mlp.experts.91.down_proj", + "model.layers.43.mlp.experts.91.gate_proj", + "model.layers.43.mlp.experts.91.up_proj", + "model.layers.43.mlp.experts.92.down_proj", + "model.layers.43.mlp.experts.92.gate_proj", + "model.layers.43.mlp.experts.92.up_proj", + "model.layers.43.mlp.experts.93.down_proj", + "model.layers.43.mlp.experts.93.gate_proj", + "model.layers.43.mlp.experts.93.up_proj", + "model.layers.43.mlp.experts.94.down_proj", + "model.layers.43.mlp.experts.94.gate_proj", + "model.layers.43.mlp.experts.94.up_proj", + "model.layers.43.mlp.experts.95.down_proj", + "model.layers.43.mlp.experts.95.gate_proj", + "model.layers.43.mlp.experts.95.up_proj", + "model.layers.43.mlp.experts.96.down_proj", + "model.layers.43.mlp.experts.96.gate_proj", + "model.layers.43.mlp.experts.96.up_proj", + "model.layers.43.mlp.experts.97.down_proj", + "model.layers.43.mlp.experts.97.gate_proj", + "model.layers.43.mlp.experts.97.up_proj", + "model.layers.43.mlp.experts.98.down_proj", + "model.layers.43.mlp.experts.98.gate_proj", + "model.layers.43.mlp.experts.98.up_proj", + "model.layers.43.mlp.experts.99.down_proj", + "model.layers.43.mlp.experts.99.gate_proj", + "model.layers.43.mlp.experts.99.up_proj", + "model.layers.44.mlp.experts.0.down_proj", + "model.layers.44.mlp.experts.0.gate_proj", + "model.layers.44.mlp.experts.0.up_proj", + "model.layers.44.mlp.experts.1.down_proj", + "model.layers.44.mlp.experts.1.gate_proj", + "model.layers.44.mlp.experts.1.up_proj", + "model.layers.44.mlp.experts.10.down_proj", + "model.layers.44.mlp.experts.10.gate_proj", + "model.layers.44.mlp.experts.10.up_proj", + "model.layers.44.mlp.experts.100.down_proj", + "model.layers.44.mlp.experts.100.gate_proj", + "model.layers.44.mlp.experts.100.up_proj", + "model.layers.44.mlp.experts.101.down_proj", + "model.layers.44.mlp.experts.101.gate_proj", + "model.layers.44.mlp.experts.101.up_proj", + "model.layers.44.mlp.experts.102.down_proj", + "model.layers.44.mlp.experts.102.gate_proj", + "model.layers.44.mlp.experts.102.up_proj", + "model.layers.44.mlp.experts.103.down_proj", + "model.layers.44.mlp.experts.103.gate_proj", + "model.layers.44.mlp.experts.103.up_proj", + "model.layers.44.mlp.experts.104.down_proj", + "model.layers.44.mlp.experts.104.gate_proj", + "model.layers.44.mlp.experts.104.up_proj", + "model.layers.44.mlp.experts.105.down_proj", + "model.layers.44.mlp.experts.105.gate_proj", + "model.layers.44.mlp.experts.105.up_proj", + "model.layers.44.mlp.experts.106.down_proj", + "model.layers.44.mlp.experts.106.gate_proj", + "model.layers.44.mlp.experts.106.up_proj", + "model.layers.44.mlp.experts.107.down_proj", + "model.layers.44.mlp.experts.107.gate_proj", + "model.layers.44.mlp.experts.107.up_proj", + "model.layers.44.mlp.experts.108.down_proj", + "model.layers.44.mlp.experts.108.gate_proj", + "model.layers.44.mlp.experts.108.up_proj", + "model.layers.44.mlp.experts.109.down_proj", + "model.layers.44.mlp.experts.109.gate_proj", + "model.layers.44.mlp.experts.109.up_proj", + "model.layers.44.mlp.experts.11.down_proj", + "model.layers.44.mlp.experts.11.gate_proj", + "model.layers.44.mlp.experts.11.up_proj", + "model.layers.44.mlp.experts.110.down_proj", + "model.layers.44.mlp.experts.110.gate_proj", + "model.layers.44.mlp.experts.110.up_proj", + "model.layers.44.mlp.experts.111.down_proj", + "model.layers.44.mlp.experts.111.gate_proj", + "model.layers.44.mlp.experts.111.up_proj", + "model.layers.44.mlp.experts.112.down_proj", + "model.layers.44.mlp.experts.112.gate_proj", + "model.layers.44.mlp.experts.112.up_proj", + "model.layers.44.mlp.experts.113.down_proj", + "model.layers.44.mlp.experts.113.gate_proj", + "model.layers.44.mlp.experts.113.up_proj", + "model.layers.44.mlp.experts.114.down_proj", + "model.layers.44.mlp.experts.114.gate_proj", + "model.layers.44.mlp.experts.114.up_proj", + "model.layers.44.mlp.experts.115.down_proj", + "model.layers.44.mlp.experts.115.gate_proj", + "model.layers.44.mlp.experts.115.up_proj", + "model.layers.44.mlp.experts.116.down_proj", + "model.layers.44.mlp.experts.116.gate_proj", + "model.layers.44.mlp.experts.116.up_proj", + "model.layers.44.mlp.experts.117.down_proj", + "model.layers.44.mlp.experts.117.gate_proj", + "model.layers.44.mlp.experts.117.up_proj", + "model.layers.44.mlp.experts.118.down_proj", + "model.layers.44.mlp.experts.118.gate_proj", + "model.layers.44.mlp.experts.118.up_proj", + "model.layers.44.mlp.experts.119.down_proj", + "model.layers.44.mlp.experts.119.gate_proj", + "model.layers.44.mlp.experts.119.up_proj", + "model.layers.44.mlp.experts.12.down_proj", + "model.layers.44.mlp.experts.12.gate_proj", + "model.layers.44.mlp.experts.12.up_proj", + "model.layers.44.mlp.experts.120.down_proj", + "model.layers.44.mlp.experts.120.gate_proj", + "model.layers.44.mlp.experts.120.up_proj", + "model.layers.44.mlp.experts.121.down_proj", + "model.layers.44.mlp.experts.121.gate_proj", + "model.layers.44.mlp.experts.121.up_proj", + "model.layers.44.mlp.experts.122.down_proj", + "model.layers.44.mlp.experts.122.gate_proj", + "model.layers.44.mlp.experts.122.up_proj", + "model.layers.44.mlp.experts.123.down_proj", + "model.layers.44.mlp.experts.123.gate_proj", + "model.layers.44.mlp.experts.123.up_proj", + "model.layers.44.mlp.experts.124.down_proj", + "model.layers.44.mlp.experts.124.gate_proj", + "model.layers.44.mlp.experts.124.up_proj", + "model.layers.44.mlp.experts.125.down_proj", + "model.layers.44.mlp.experts.125.gate_proj", + "model.layers.44.mlp.experts.125.up_proj", + "model.layers.44.mlp.experts.126.down_proj", + "model.layers.44.mlp.experts.126.gate_proj", + "model.layers.44.mlp.experts.126.up_proj", + "model.layers.44.mlp.experts.127.down_proj", + "model.layers.44.mlp.experts.127.gate_proj", + "model.layers.44.mlp.experts.127.up_proj", + "model.layers.44.mlp.experts.128.down_proj", + "model.layers.44.mlp.experts.128.gate_proj", + "model.layers.44.mlp.experts.128.up_proj", + "model.layers.44.mlp.experts.129.down_proj", + "model.layers.44.mlp.experts.129.gate_proj", + "model.layers.44.mlp.experts.129.up_proj", + "model.layers.44.mlp.experts.13.down_proj", + "model.layers.44.mlp.experts.13.gate_proj", + "model.layers.44.mlp.experts.13.up_proj", + "model.layers.44.mlp.experts.130.down_proj", + "model.layers.44.mlp.experts.130.gate_proj", + "model.layers.44.mlp.experts.130.up_proj", + "model.layers.44.mlp.experts.131.down_proj", + "model.layers.44.mlp.experts.131.gate_proj", + "model.layers.44.mlp.experts.131.up_proj", + "model.layers.44.mlp.experts.132.down_proj", + "model.layers.44.mlp.experts.132.gate_proj", + "model.layers.44.mlp.experts.132.up_proj", + "model.layers.44.mlp.experts.133.down_proj", + "model.layers.44.mlp.experts.133.gate_proj", + "model.layers.44.mlp.experts.133.up_proj", + "model.layers.44.mlp.experts.134.down_proj", + "model.layers.44.mlp.experts.134.gate_proj", + "model.layers.44.mlp.experts.134.up_proj", + "model.layers.44.mlp.experts.135.down_proj", + "model.layers.44.mlp.experts.135.gate_proj", + "model.layers.44.mlp.experts.135.up_proj", + "model.layers.44.mlp.experts.136.down_proj", + "model.layers.44.mlp.experts.136.gate_proj", + "model.layers.44.mlp.experts.136.up_proj", + "model.layers.44.mlp.experts.137.down_proj", + "model.layers.44.mlp.experts.137.gate_proj", + "model.layers.44.mlp.experts.137.up_proj", + "model.layers.44.mlp.experts.138.down_proj", + "model.layers.44.mlp.experts.138.gate_proj", + "model.layers.44.mlp.experts.138.up_proj", + "model.layers.44.mlp.experts.139.down_proj", + "model.layers.44.mlp.experts.139.gate_proj", + "model.layers.44.mlp.experts.139.up_proj", + "model.layers.44.mlp.experts.14.down_proj", + "model.layers.44.mlp.experts.14.gate_proj", + "model.layers.44.mlp.experts.14.up_proj", + "model.layers.44.mlp.experts.140.down_proj", + "model.layers.44.mlp.experts.140.gate_proj", + "model.layers.44.mlp.experts.140.up_proj", + "model.layers.44.mlp.experts.141.down_proj", + "model.layers.44.mlp.experts.141.gate_proj", + "model.layers.44.mlp.experts.141.up_proj", + "model.layers.44.mlp.experts.142.down_proj", + "model.layers.44.mlp.experts.142.gate_proj", + "model.layers.44.mlp.experts.142.up_proj", + "model.layers.44.mlp.experts.143.down_proj", + "model.layers.44.mlp.experts.143.gate_proj", + "model.layers.44.mlp.experts.143.up_proj", + "model.layers.44.mlp.experts.144.down_proj", + "model.layers.44.mlp.experts.144.gate_proj", + "model.layers.44.mlp.experts.144.up_proj", + "model.layers.44.mlp.experts.145.down_proj", + "model.layers.44.mlp.experts.145.gate_proj", + "model.layers.44.mlp.experts.145.up_proj", + "model.layers.44.mlp.experts.146.down_proj", + "model.layers.44.mlp.experts.146.gate_proj", + "model.layers.44.mlp.experts.146.up_proj", + "model.layers.44.mlp.experts.147.down_proj", + "model.layers.44.mlp.experts.147.gate_proj", + "model.layers.44.mlp.experts.147.up_proj", + "model.layers.44.mlp.experts.148.down_proj", + "model.layers.44.mlp.experts.148.gate_proj", + "model.layers.44.mlp.experts.148.up_proj", + "model.layers.44.mlp.experts.149.down_proj", + "model.layers.44.mlp.experts.149.gate_proj", + "model.layers.44.mlp.experts.149.up_proj", + "model.layers.44.mlp.experts.15.down_proj", + "model.layers.44.mlp.experts.15.gate_proj", + "model.layers.44.mlp.experts.15.up_proj", + "model.layers.44.mlp.experts.150.down_proj", + "model.layers.44.mlp.experts.150.gate_proj", + "model.layers.44.mlp.experts.150.up_proj", + "model.layers.44.mlp.experts.151.down_proj", + "model.layers.44.mlp.experts.151.gate_proj", + "model.layers.44.mlp.experts.151.up_proj", + "model.layers.44.mlp.experts.152.down_proj", + "model.layers.44.mlp.experts.152.gate_proj", + "model.layers.44.mlp.experts.152.up_proj", + "model.layers.44.mlp.experts.153.down_proj", + "model.layers.44.mlp.experts.153.gate_proj", + "model.layers.44.mlp.experts.153.up_proj", + "model.layers.44.mlp.experts.154.down_proj", + "model.layers.44.mlp.experts.154.gate_proj", + "model.layers.44.mlp.experts.154.up_proj", + "model.layers.44.mlp.experts.155.down_proj", + "model.layers.44.mlp.experts.155.gate_proj", + "model.layers.44.mlp.experts.155.up_proj", + "model.layers.44.mlp.experts.156.down_proj", + "model.layers.44.mlp.experts.156.gate_proj", + "model.layers.44.mlp.experts.156.up_proj", + "model.layers.44.mlp.experts.157.down_proj", + "model.layers.44.mlp.experts.157.gate_proj", + "model.layers.44.mlp.experts.157.up_proj", + "model.layers.44.mlp.experts.158.down_proj", + "model.layers.44.mlp.experts.158.gate_proj", + "model.layers.44.mlp.experts.158.up_proj", + "model.layers.44.mlp.experts.159.down_proj", + "model.layers.44.mlp.experts.159.gate_proj", + "model.layers.44.mlp.experts.159.up_proj", + "model.layers.44.mlp.experts.16.down_proj", + "model.layers.44.mlp.experts.16.gate_proj", + "model.layers.44.mlp.experts.16.up_proj", + "model.layers.44.mlp.experts.160.down_proj", + "model.layers.44.mlp.experts.160.gate_proj", + "model.layers.44.mlp.experts.160.up_proj", + "model.layers.44.mlp.experts.161.down_proj", + "model.layers.44.mlp.experts.161.gate_proj", + "model.layers.44.mlp.experts.161.up_proj", + "model.layers.44.mlp.experts.162.down_proj", + "model.layers.44.mlp.experts.162.gate_proj", + "model.layers.44.mlp.experts.162.up_proj", + "model.layers.44.mlp.experts.163.down_proj", + "model.layers.44.mlp.experts.163.gate_proj", + "model.layers.44.mlp.experts.163.up_proj", + "model.layers.44.mlp.experts.164.down_proj", + "model.layers.44.mlp.experts.164.gate_proj", + "model.layers.44.mlp.experts.164.up_proj", + "model.layers.44.mlp.experts.165.down_proj", + "model.layers.44.mlp.experts.165.gate_proj", + "model.layers.44.mlp.experts.165.up_proj", + "model.layers.44.mlp.experts.166.down_proj", + "model.layers.44.mlp.experts.166.gate_proj", + "model.layers.44.mlp.experts.166.up_proj", + "model.layers.44.mlp.experts.167.down_proj", + "model.layers.44.mlp.experts.167.gate_proj", + "model.layers.44.mlp.experts.167.up_proj", + "model.layers.44.mlp.experts.168.down_proj", + "model.layers.44.mlp.experts.168.gate_proj", + "model.layers.44.mlp.experts.168.up_proj", + "model.layers.44.mlp.experts.169.down_proj", + "model.layers.44.mlp.experts.169.gate_proj", + "model.layers.44.mlp.experts.169.up_proj", + "model.layers.44.mlp.experts.17.down_proj", + "model.layers.44.mlp.experts.17.gate_proj", + "model.layers.44.mlp.experts.17.up_proj", + "model.layers.44.mlp.experts.170.down_proj", + "model.layers.44.mlp.experts.170.gate_proj", + "model.layers.44.mlp.experts.170.up_proj", + "model.layers.44.mlp.experts.171.down_proj", + "model.layers.44.mlp.experts.171.gate_proj", + "model.layers.44.mlp.experts.171.up_proj", + "model.layers.44.mlp.experts.172.down_proj", + "model.layers.44.mlp.experts.172.gate_proj", + "model.layers.44.mlp.experts.172.up_proj", + "model.layers.44.mlp.experts.173.down_proj", + "model.layers.44.mlp.experts.173.gate_proj", + "model.layers.44.mlp.experts.173.up_proj", + "model.layers.44.mlp.experts.174.down_proj", + "model.layers.44.mlp.experts.174.gate_proj", + "model.layers.44.mlp.experts.174.up_proj", + "model.layers.44.mlp.experts.175.down_proj", + "model.layers.44.mlp.experts.175.gate_proj", + "model.layers.44.mlp.experts.175.up_proj", + "model.layers.44.mlp.experts.176.down_proj", + "model.layers.44.mlp.experts.176.gate_proj", + "model.layers.44.mlp.experts.176.up_proj", + "model.layers.44.mlp.experts.177.down_proj", + "model.layers.44.mlp.experts.177.gate_proj", + "model.layers.44.mlp.experts.177.up_proj", + "model.layers.44.mlp.experts.178.down_proj", + "model.layers.44.mlp.experts.178.gate_proj", + "model.layers.44.mlp.experts.178.up_proj", + "model.layers.44.mlp.experts.179.down_proj", + "model.layers.44.mlp.experts.179.gate_proj", + "model.layers.44.mlp.experts.179.up_proj", + "model.layers.44.mlp.experts.18.down_proj", + "model.layers.44.mlp.experts.18.gate_proj", + "model.layers.44.mlp.experts.18.up_proj", + "model.layers.44.mlp.experts.180.down_proj", + "model.layers.44.mlp.experts.180.gate_proj", + "model.layers.44.mlp.experts.180.up_proj", + "model.layers.44.mlp.experts.181.down_proj", + "model.layers.44.mlp.experts.181.gate_proj", + "model.layers.44.mlp.experts.181.up_proj", + "model.layers.44.mlp.experts.182.down_proj", + "model.layers.44.mlp.experts.182.gate_proj", + "model.layers.44.mlp.experts.182.up_proj", + "model.layers.44.mlp.experts.183.down_proj", + "model.layers.44.mlp.experts.183.gate_proj", + "model.layers.44.mlp.experts.183.up_proj", + "model.layers.44.mlp.experts.184.down_proj", + "model.layers.44.mlp.experts.184.gate_proj", + "model.layers.44.mlp.experts.184.up_proj", + "model.layers.44.mlp.experts.185.down_proj", + "model.layers.44.mlp.experts.185.gate_proj", + "model.layers.44.mlp.experts.185.up_proj", + "model.layers.44.mlp.experts.186.down_proj", + "model.layers.44.mlp.experts.186.gate_proj", + "model.layers.44.mlp.experts.186.up_proj", + "model.layers.44.mlp.experts.187.down_proj", + "model.layers.44.mlp.experts.187.gate_proj", + "model.layers.44.mlp.experts.187.up_proj", + "model.layers.44.mlp.experts.188.down_proj", + "model.layers.44.mlp.experts.188.gate_proj", + "model.layers.44.mlp.experts.188.up_proj", + "model.layers.44.mlp.experts.189.down_proj", + "model.layers.44.mlp.experts.189.gate_proj", + "model.layers.44.mlp.experts.189.up_proj", + "model.layers.44.mlp.experts.19.down_proj", + "model.layers.44.mlp.experts.19.gate_proj", + "model.layers.44.mlp.experts.19.up_proj", + "model.layers.44.mlp.experts.190.down_proj", + "model.layers.44.mlp.experts.190.gate_proj", + "model.layers.44.mlp.experts.190.up_proj", + "model.layers.44.mlp.experts.191.down_proj", + "model.layers.44.mlp.experts.191.gate_proj", + "model.layers.44.mlp.experts.191.up_proj", + "model.layers.44.mlp.experts.192.down_proj", + "model.layers.44.mlp.experts.192.gate_proj", + "model.layers.44.mlp.experts.192.up_proj", + "model.layers.44.mlp.experts.193.down_proj", + "model.layers.44.mlp.experts.193.gate_proj", + "model.layers.44.mlp.experts.193.up_proj", + "model.layers.44.mlp.experts.194.down_proj", + "model.layers.44.mlp.experts.194.gate_proj", + "model.layers.44.mlp.experts.194.up_proj", + "model.layers.44.mlp.experts.195.down_proj", + "model.layers.44.mlp.experts.195.gate_proj", + "model.layers.44.mlp.experts.195.up_proj", + "model.layers.44.mlp.experts.196.down_proj", + "model.layers.44.mlp.experts.196.gate_proj", + "model.layers.44.mlp.experts.196.up_proj", + "model.layers.44.mlp.experts.197.down_proj", + "model.layers.44.mlp.experts.197.gate_proj", + "model.layers.44.mlp.experts.197.up_proj", + "model.layers.44.mlp.experts.198.down_proj", + "model.layers.44.mlp.experts.198.gate_proj", + "model.layers.44.mlp.experts.198.up_proj", + "model.layers.44.mlp.experts.199.down_proj", + "model.layers.44.mlp.experts.199.gate_proj", + "model.layers.44.mlp.experts.199.up_proj", + "model.layers.44.mlp.experts.2.down_proj", + "model.layers.44.mlp.experts.2.gate_proj", + "model.layers.44.mlp.experts.2.up_proj", + "model.layers.44.mlp.experts.20.down_proj", + "model.layers.44.mlp.experts.20.gate_proj", + "model.layers.44.mlp.experts.20.up_proj", + "model.layers.44.mlp.experts.200.down_proj", + "model.layers.44.mlp.experts.200.gate_proj", + "model.layers.44.mlp.experts.200.up_proj", + "model.layers.44.mlp.experts.201.down_proj", + "model.layers.44.mlp.experts.201.gate_proj", + "model.layers.44.mlp.experts.201.up_proj", + "model.layers.44.mlp.experts.202.down_proj", + "model.layers.44.mlp.experts.202.gate_proj", + "model.layers.44.mlp.experts.202.up_proj", + "model.layers.44.mlp.experts.203.down_proj", + "model.layers.44.mlp.experts.203.gate_proj", + "model.layers.44.mlp.experts.203.up_proj", + "model.layers.44.mlp.experts.204.down_proj", + "model.layers.44.mlp.experts.204.gate_proj", + "model.layers.44.mlp.experts.204.up_proj", + "model.layers.44.mlp.experts.205.down_proj", + "model.layers.44.mlp.experts.205.gate_proj", + "model.layers.44.mlp.experts.205.up_proj", + "model.layers.44.mlp.experts.206.down_proj", + "model.layers.44.mlp.experts.206.gate_proj", + "model.layers.44.mlp.experts.206.up_proj", + "model.layers.44.mlp.experts.207.down_proj", + "model.layers.44.mlp.experts.207.gate_proj", + "model.layers.44.mlp.experts.207.up_proj", + "model.layers.44.mlp.experts.208.down_proj", + "model.layers.44.mlp.experts.208.gate_proj", + "model.layers.44.mlp.experts.208.up_proj", + "model.layers.44.mlp.experts.209.down_proj", + "model.layers.44.mlp.experts.209.gate_proj", + "model.layers.44.mlp.experts.209.up_proj", + "model.layers.44.mlp.experts.21.down_proj", + "model.layers.44.mlp.experts.21.gate_proj", + "model.layers.44.mlp.experts.21.up_proj", + "model.layers.44.mlp.experts.210.down_proj", + "model.layers.44.mlp.experts.210.gate_proj", + "model.layers.44.mlp.experts.210.up_proj", + "model.layers.44.mlp.experts.211.down_proj", + "model.layers.44.mlp.experts.211.gate_proj", + "model.layers.44.mlp.experts.211.up_proj", + "model.layers.44.mlp.experts.212.down_proj", + "model.layers.44.mlp.experts.212.gate_proj", + "model.layers.44.mlp.experts.212.up_proj", + "model.layers.44.mlp.experts.213.down_proj", + "model.layers.44.mlp.experts.213.gate_proj", + "model.layers.44.mlp.experts.213.up_proj", + "model.layers.44.mlp.experts.214.down_proj", + "model.layers.44.mlp.experts.214.gate_proj", + "model.layers.44.mlp.experts.214.up_proj", + "model.layers.44.mlp.experts.215.down_proj", + "model.layers.44.mlp.experts.215.gate_proj", + "model.layers.44.mlp.experts.215.up_proj", + "model.layers.44.mlp.experts.216.down_proj", + "model.layers.44.mlp.experts.216.gate_proj", + "model.layers.44.mlp.experts.216.up_proj", + "model.layers.44.mlp.experts.217.down_proj", + "model.layers.44.mlp.experts.217.gate_proj", + "model.layers.44.mlp.experts.217.up_proj", + "model.layers.44.mlp.experts.218.down_proj", + "model.layers.44.mlp.experts.218.gate_proj", + "model.layers.44.mlp.experts.218.up_proj", + "model.layers.44.mlp.experts.219.down_proj", + "model.layers.44.mlp.experts.219.gate_proj", + "model.layers.44.mlp.experts.219.up_proj", + "model.layers.44.mlp.experts.22.down_proj", + "model.layers.44.mlp.experts.22.gate_proj", + "model.layers.44.mlp.experts.22.up_proj", + "model.layers.44.mlp.experts.220.down_proj", + "model.layers.44.mlp.experts.220.gate_proj", + "model.layers.44.mlp.experts.220.up_proj", + "model.layers.44.mlp.experts.221.down_proj", + "model.layers.44.mlp.experts.221.gate_proj", + "model.layers.44.mlp.experts.221.up_proj", + "model.layers.44.mlp.experts.222.down_proj", + "model.layers.44.mlp.experts.222.gate_proj", + "model.layers.44.mlp.experts.222.up_proj", + "model.layers.44.mlp.experts.223.down_proj", + "model.layers.44.mlp.experts.223.gate_proj", + "model.layers.44.mlp.experts.223.up_proj", + "model.layers.44.mlp.experts.224.down_proj", + "model.layers.44.mlp.experts.224.gate_proj", + "model.layers.44.mlp.experts.224.up_proj", + "model.layers.44.mlp.experts.225.down_proj", + "model.layers.44.mlp.experts.225.gate_proj", + "model.layers.44.mlp.experts.225.up_proj", + "model.layers.44.mlp.experts.226.down_proj", + "model.layers.44.mlp.experts.226.gate_proj", + "model.layers.44.mlp.experts.226.up_proj", + "model.layers.44.mlp.experts.227.down_proj", + "model.layers.44.mlp.experts.227.gate_proj", + "model.layers.44.mlp.experts.227.up_proj", + "model.layers.44.mlp.experts.228.down_proj", + "model.layers.44.mlp.experts.228.gate_proj", + "model.layers.44.mlp.experts.228.up_proj", + "model.layers.44.mlp.experts.229.down_proj", + "model.layers.44.mlp.experts.229.gate_proj", + "model.layers.44.mlp.experts.229.up_proj", + "model.layers.44.mlp.experts.23.down_proj", + "model.layers.44.mlp.experts.23.gate_proj", + "model.layers.44.mlp.experts.23.up_proj", + "model.layers.44.mlp.experts.230.down_proj", + "model.layers.44.mlp.experts.230.gate_proj", + "model.layers.44.mlp.experts.230.up_proj", + "model.layers.44.mlp.experts.231.down_proj", + "model.layers.44.mlp.experts.231.gate_proj", + "model.layers.44.mlp.experts.231.up_proj", + "model.layers.44.mlp.experts.232.down_proj", + "model.layers.44.mlp.experts.232.gate_proj", + "model.layers.44.mlp.experts.232.up_proj", + "model.layers.44.mlp.experts.233.down_proj", + "model.layers.44.mlp.experts.233.gate_proj", + "model.layers.44.mlp.experts.233.up_proj", + "model.layers.44.mlp.experts.234.down_proj", + "model.layers.44.mlp.experts.234.gate_proj", + "model.layers.44.mlp.experts.234.up_proj", + "model.layers.44.mlp.experts.235.down_proj", + "model.layers.44.mlp.experts.235.gate_proj", + "model.layers.44.mlp.experts.235.up_proj", + "model.layers.44.mlp.experts.236.down_proj", + "model.layers.44.mlp.experts.236.gate_proj", + "model.layers.44.mlp.experts.236.up_proj", + "model.layers.44.mlp.experts.237.down_proj", + "model.layers.44.mlp.experts.237.gate_proj", + "model.layers.44.mlp.experts.237.up_proj", + "model.layers.44.mlp.experts.238.down_proj", + "model.layers.44.mlp.experts.238.gate_proj", + "model.layers.44.mlp.experts.238.up_proj", + "model.layers.44.mlp.experts.239.down_proj", + "model.layers.44.mlp.experts.239.gate_proj", + "model.layers.44.mlp.experts.239.up_proj", + "model.layers.44.mlp.experts.24.down_proj", + "model.layers.44.mlp.experts.24.gate_proj", + "model.layers.44.mlp.experts.24.up_proj", + "model.layers.44.mlp.experts.240.down_proj", + "model.layers.44.mlp.experts.240.gate_proj", + "model.layers.44.mlp.experts.240.up_proj", + "model.layers.44.mlp.experts.241.down_proj", + "model.layers.44.mlp.experts.241.gate_proj", + "model.layers.44.mlp.experts.241.up_proj", + "model.layers.44.mlp.experts.242.down_proj", + "model.layers.44.mlp.experts.242.gate_proj", + "model.layers.44.mlp.experts.242.up_proj", + "model.layers.44.mlp.experts.243.down_proj", + "model.layers.44.mlp.experts.243.gate_proj", + "model.layers.44.mlp.experts.243.up_proj", + "model.layers.44.mlp.experts.244.down_proj", + "model.layers.44.mlp.experts.244.gate_proj", + "model.layers.44.mlp.experts.244.up_proj", + "model.layers.44.mlp.experts.245.down_proj", + "model.layers.44.mlp.experts.245.gate_proj", + "model.layers.44.mlp.experts.245.up_proj", + "model.layers.44.mlp.experts.246.down_proj", + "model.layers.44.mlp.experts.246.gate_proj", + "model.layers.44.mlp.experts.246.up_proj", + "model.layers.44.mlp.experts.247.down_proj", + "model.layers.44.mlp.experts.247.gate_proj", + "model.layers.44.mlp.experts.247.up_proj", + "model.layers.44.mlp.experts.248.down_proj", + "model.layers.44.mlp.experts.248.gate_proj", + "model.layers.44.mlp.experts.248.up_proj", + "model.layers.44.mlp.experts.249.down_proj", + "model.layers.44.mlp.experts.249.gate_proj", + "model.layers.44.mlp.experts.249.up_proj", + "model.layers.44.mlp.experts.25.down_proj", + "model.layers.44.mlp.experts.25.gate_proj", + "model.layers.44.mlp.experts.25.up_proj", + "model.layers.44.mlp.experts.250.down_proj", + "model.layers.44.mlp.experts.250.gate_proj", + "model.layers.44.mlp.experts.250.up_proj", + "model.layers.44.mlp.experts.251.down_proj", + "model.layers.44.mlp.experts.251.gate_proj", + "model.layers.44.mlp.experts.251.up_proj", + "model.layers.44.mlp.experts.252.down_proj", + "model.layers.44.mlp.experts.252.gate_proj", + "model.layers.44.mlp.experts.252.up_proj", + "model.layers.44.mlp.experts.253.down_proj", + "model.layers.44.mlp.experts.253.gate_proj", + "model.layers.44.mlp.experts.253.up_proj", + "model.layers.44.mlp.experts.254.down_proj", + "model.layers.44.mlp.experts.254.gate_proj", + "model.layers.44.mlp.experts.254.up_proj", + "model.layers.44.mlp.experts.255.down_proj", + "model.layers.44.mlp.experts.255.gate_proj", + "model.layers.44.mlp.experts.255.up_proj", + "model.layers.44.mlp.experts.26.down_proj", + "model.layers.44.mlp.experts.26.gate_proj", + "model.layers.44.mlp.experts.26.up_proj", + "model.layers.44.mlp.experts.27.down_proj", + "model.layers.44.mlp.experts.27.gate_proj", + "model.layers.44.mlp.experts.27.up_proj", + "model.layers.44.mlp.experts.28.down_proj", + "model.layers.44.mlp.experts.28.gate_proj", + "model.layers.44.mlp.experts.28.up_proj", + "model.layers.44.mlp.experts.29.down_proj", + "model.layers.44.mlp.experts.29.gate_proj", + "model.layers.44.mlp.experts.29.up_proj", + "model.layers.44.mlp.experts.3.down_proj", + "model.layers.44.mlp.experts.3.gate_proj", + "model.layers.44.mlp.experts.3.up_proj", + "model.layers.44.mlp.experts.30.down_proj", + "model.layers.44.mlp.experts.30.gate_proj", + "model.layers.44.mlp.experts.30.up_proj", + "model.layers.44.mlp.experts.31.down_proj", + "model.layers.44.mlp.experts.31.gate_proj", + "model.layers.44.mlp.experts.31.up_proj", + "model.layers.44.mlp.experts.32.down_proj", + "model.layers.44.mlp.experts.32.gate_proj", + "model.layers.44.mlp.experts.32.up_proj", + "model.layers.44.mlp.experts.33.down_proj", + "model.layers.44.mlp.experts.33.gate_proj", + "model.layers.44.mlp.experts.33.up_proj", + "model.layers.44.mlp.experts.34.down_proj", + "model.layers.44.mlp.experts.34.gate_proj", + "model.layers.44.mlp.experts.34.up_proj", + "model.layers.44.mlp.experts.35.down_proj", + "model.layers.44.mlp.experts.35.gate_proj", + "model.layers.44.mlp.experts.35.up_proj", + "model.layers.44.mlp.experts.36.down_proj", + "model.layers.44.mlp.experts.36.gate_proj", + "model.layers.44.mlp.experts.36.up_proj", + "model.layers.44.mlp.experts.37.down_proj", + "model.layers.44.mlp.experts.37.gate_proj", + "model.layers.44.mlp.experts.37.up_proj", + "model.layers.44.mlp.experts.38.down_proj", + "model.layers.44.mlp.experts.38.gate_proj", + "model.layers.44.mlp.experts.38.up_proj", + "model.layers.44.mlp.experts.39.down_proj", + "model.layers.44.mlp.experts.39.gate_proj", + "model.layers.44.mlp.experts.39.up_proj", + "model.layers.44.mlp.experts.4.down_proj", + "model.layers.44.mlp.experts.4.gate_proj", + "model.layers.44.mlp.experts.4.up_proj", + "model.layers.44.mlp.experts.40.down_proj", + "model.layers.44.mlp.experts.40.gate_proj", + "model.layers.44.mlp.experts.40.up_proj", + "model.layers.44.mlp.experts.41.down_proj", + "model.layers.44.mlp.experts.41.gate_proj", + "model.layers.44.mlp.experts.41.up_proj", + "model.layers.44.mlp.experts.42.down_proj", + "model.layers.44.mlp.experts.42.gate_proj", + "model.layers.44.mlp.experts.42.up_proj", + "model.layers.44.mlp.experts.43.down_proj", + "model.layers.44.mlp.experts.43.gate_proj", + "model.layers.44.mlp.experts.43.up_proj", + "model.layers.44.mlp.experts.44.down_proj", + "model.layers.44.mlp.experts.44.gate_proj", + "model.layers.44.mlp.experts.44.up_proj", + "model.layers.44.mlp.experts.45.down_proj", + "model.layers.44.mlp.experts.45.gate_proj", + "model.layers.44.mlp.experts.45.up_proj", + "model.layers.44.mlp.experts.46.down_proj", + "model.layers.44.mlp.experts.46.gate_proj", + "model.layers.44.mlp.experts.46.up_proj", + "model.layers.44.mlp.experts.47.down_proj", + "model.layers.44.mlp.experts.47.gate_proj", + "model.layers.44.mlp.experts.47.up_proj", + "model.layers.44.mlp.experts.48.down_proj", + "model.layers.44.mlp.experts.48.gate_proj", + "model.layers.44.mlp.experts.48.up_proj", + "model.layers.44.mlp.experts.49.down_proj", + "model.layers.44.mlp.experts.49.gate_proj", + "model.layers.44.mlp.experts.49.up_proj", + "model.layers.44.mlp.experts.5.down_proj", + "model.layers.44.mlp.experts.5.gate_proj", + "model.layers.44.mlp.experts.5.up_proj", + "model.layers.44.mlp.experts.50.down_proj", + "model.layers.44.mlp.experts.50.gate_proj", + "model.layers.44.mlp.experts.50.up_proj", + "model.layers.44.mlp.experts.51.down_proj", + "model.layers.44.mlp.experts.51.gate_proj", + "model.layers.44.mlp.experts.51.up_proj", + "model.layers.44.mlp.experts.52.down_proj", + "model.layers.44.mlp.experts.52.gate_proj", + "model.layers.44.mlp.experts.52.up_proj", + "model.layers.44.mlp.experts.53.down_proj", + "model.layers.44.mlp.experts.53.gate_proj", + "model.layers.44.mlp.experts.53.up_proj", + "model.layers.44.mlp.experts.54.down_proj", + "model.layers.44.mlp.experts.54.gate_proj", + "model.layers.44.mlp.experts.54.up_proj", + "model.layers.44.mlp.experts.55.down_proj", + "model.layers.44.mlp.experts.55.gate_proj", + "model.layers.44.mlp.experts.55.up_proj", + "model.layers.44.mlp.experts.56.down_proj", + "model.layers.44.mlp.experts.56.gate_proj", + "model.layers.44.mlp.experts.56.up_proj", + "model.layers.44.mlp.experts.57.down_proj", + "model.layers.44.mlp.experts.57.gate_proj", + "model.layers.44.mlp.experts.57.up_proj", + "model.layers.44.mlp.experts.58.down_proj", + "model.layers.44.mlp.experts.58.gate_proj", + "model.layers.44.mlp.experts.58.up_proj", + "model.layers.44.mlp.experts.59.down_proj", + "model.layers.44.mlp.experts.59.gate_proj", + "model.layers.44.mlp.experts.59.up_proj", + "model.layers.44.mlp.experts.6.down_proj", + "model.layers.44.mlp.experts.6.gate_proj", + "model.layers.44.mlp.experts.6.up_proj", + "model.layers.44.mlp.experts.60.down_proj", + "model.layers.44.mlp.experts.60.gate_proj", + "model.layers.44.mlp.experts.60.up_proj", + "model.layers.44.mlp.experts.61.down_proj", + "model.layers.44.mlp.experts.61.gate_proj", + "model.layers.44.mlp.experts.61.up_proj", + "model.layers.44.mlp.experts.62.down_proj", + "model.layers.44.mlp.experts.62.gate_proj", + "model.layers.44.mlp.experts.62.up_proj", + "model.layers.44.mlp.experts.63.down_proj", + "model.layers.44.mlp.experts.63.gate_proj", + "model.layers.44.mlp.experts.63.up_proj", + "model.layers.44.mlp.experts.64.down_proj", + "model.layers.44.mlp.experts.64.gate_proj", + "model.layers.44.mlp.experts.64.up_proj", + "model.layers.44.mlp.experts.65.down_proj", + "model.layers.44.mlp.experts.65.gate_proj", + "model.layers.44.mlp.experts.65.up_proj", + "model.layers.44.mlp.experts.66.down_proj", + "model.layers.44.mlp.experts.66.gate_proj", + "model.layers.44.mlp.experts.66.up_proj", + "model.layers.44.mlp.experts.67.down_proj", + "model.layers.44.mlp.experts.67.gate_proj", + "model.layers.44.mlp.experts.67.up_proj", + "model.layers.44.mlp.experts.68.down_proj", + "model.layers.44.mlp.experts.68.gate_proj", + "model.layers.44.mlp.experts.68.up_proj", + "model.layers.44.mlp.experts.69.down_proj", + "model.layers.44.mlp.experts.69.gate_proj", + "model.layers.44.mlp.experts.69.up_proj", + "model.layers.44.mlp.experts.7.down_proj", + "model.layers.44.mlp.experts.7.gate_proj", + "model.layers.44.mlp.experts.7.up_proj", + "model.layers.44.mlp.experts.70.down_proj", + "model.layers.44.mlp.experts.70.gate_proj", + "model.layers.44.mlp.experts.70.up_proj", + "model.layers.44.mlp.experts.71.down_proj", + "model.layers.44.mlp.experts.71.gate_proj", + "model.layers.44.mlp.experts.71.up_proj", + "model.layers.44.mlp.experts.72.down_proj", + "model.layers.44.mlp.experts.72.gate_proj", + "model.layers.44.mlp.experts.72.up_proj", + "model.layers.44.mlp.experts.73.down_proj", + "model.layers.44.mlp.experts.73.gate_proj", + "model.layers.44.mlp.experts.73.up_proj", + "model.layers.44.mlp.experts.74.down_proj", + "model.layers.44.mlp.experts.74.gate_proj", + "model.layers.44.mlp.experts.74.up_proj", + "model.layers.44.mlp.experts.75.down_proj", + "model.layers.44.mlp.experts.75.gate_proj", + "model.layers.44.mlp.experts.75.up_proj", + "model.layers.44.mlp.experts.76.down_proj", + "model.layers.44.mlp.experts.76.gate_proj", + "model.layers.44.mlp.experts.76.up_proj", + "model.layers.44.mlp.experts.77.down_proj", + "model.layers.44.mlp.experts.77.gate_proj", + "model.layers.44.mlp.experts.77.up_proj", + "model.layers.44.mlp.experts.78.down_proj", + "model.layers.44.mlp.experts.78.gate_proj", + "model.layers.44.mlp.experts.78.up_proj", + "model.layers.44.mlp.experts.79.down_proj", + "model.layers.44.mlp.experts.79.gate_proj", + "model.layers.44.mlp.experts.79.up_proj", + "model.layers.44.mlp.experts.8.down_proj", + "model.layers.44.mlp.experts.8.gate_proj", + "model.layers.44.mlp.experts.8.up_proj", + "model.layers.44.mlp.experts.80.down_proj", + "model.layers.44.mlp.experts.80.gate_proj", + "model.layers.44.mlp.experts.80.up_proj", + "model.layers.44.mlp.experts.81.down_proj", + "model.layers.44.mlp.experts.81.gate_proj", + "model.layers.44.mlp.experts.81.up_proj", + "model.layers.44.mlp.experts.82.down_proj", + "model.layers.44.mlp.experts.82.gate_proj", + "model.layers.44.mlp.experts.82.up_proj", + "model.layers.44.mlp.experts.83.down_proj", + "model.layers.44.mlp.experts.83.gate_proj", + "model.layers.44.mlp.experts.83.up_proj", + "model.layers.44.mlp.experts.84.down_proj", + "model.layers.44.mlp.experts.84.gate_proj", + "model.layers.44.mlp.experts.84.up_proj", + "model.layers.44.mlp.experts.85.down_proj", + "model.layers.44.mlp.experts.85.gate_proj", + "model.layers.44.mlp.experts.85.up_proj", + "model.layers.44.mlp.experts.86.down_proj", + "model.layers.44.mlp.experts.86.gate_proj", + "model.layers.44.mlp.experts.86.up_proj", + "model.layers.44.mlp.experts.87.down_proj", + "model.layers.44.mlp.experts.87.gate_proj", + "model.layers.44.mlp.experts.87.up_proj", + "model.layers.44.mlp.experts.88.down_proj", + "model.layers.44.mlp.experts.88.gate_proj", + "model.layers.44.mlp.experts.88.up_proj", + "model.layers.44.mlp.experts.89.down_proj", + "model.layers.44.mlp.experts.89.gate_proj", + "model.layers.44.mlp.experts.89.up_proj", + "model.layers.44.mlp.experts.9.down_proj", + "model.layers.44.mlp.experts.9.gate_proj", + "model.layers.44.mlp.experts.9.up_proj", + "model.layers.44.mlp.experts.90.down_proj", + "model.layers.44.mlp.experts.90.gate_proj", + "model.layers.44.mlp.experts.90.up_proj", + "model.layers.44.mlp.experts.91.down_proj", + "model.layers.44.mlp.experts.91.gate_proj", + "model.layers.44.mlp.experts.91.up_proj", + "model.layers.44.mlp.experts.92.down_proj", + "model.layers.44.mlp.experts.92.gate_proj", + "model.layers.44.mlp.experts.92.up_proj", + "model.layers.44.mlp.experts.93.down_proj", + "model.layers.44.mlp.experts.93.gate_proj", + "model.layers.44.mlp.experts.93.up_proj", + "model.layers.44.mlp.experts.94.down_proj", + "model.layers.44.mlp.experts.94.gate_proj", + "model.layers.44.mlp.experts.94.up_proj", + "model.layers.44.mlp.experts.95.down_proj", + "model.layers.44.mlp.experts.95.gate_proj", + "model.layers.44.mlp.experts.95.up_proj", + "model.layers.44.mlp.experts.96.down_proj", + "model.layers.44.mlp.experts.96.gate_proj", + "model.layers.44.mlp.experts.96.up_proj", + "model.layers.44.mlp.experts.97.down_proj", + "model.layers.44.mlp.experts.97.gate_proj", + "model.layers.44.mlp.experts.97.up_proj", + "model.layers.44.mlp.experts.98.down_proj", + "model.layers.44.mlp.experts.98.gate_proj", + "model.layers.44.mlp.experts.98.up_proj", + "model.layers.44.mlp.experts.99.down_proj", + "model.layers.44.mlp.experts.99.gate_proj", + "model.layers.44.mlp.experts.99.up_proj", + "model.layers.45.mlp.experts.0.down_proj", + "model.layers.45.mlp.experts.0.gate_proj", + "model.layers.45.mlp.experts.0.up_proj", + "model.layers.45.mlp.experts.1.down_proj", + "model.layers.45.mlp.experts.1.gate_proj", + "model.layers.45.mlp.experts.1.up_proj", + "model.layers.45.mlp.experts.10.down_proj", + "model.layers.45.mlp.experts.10.gate_proj", + "model.layers.45.mlp.experts.10.up_proj", + "model.layers.45.mlp.experts.100.down_proj", + "model.layers.45.mlp.experts.100.gate_proj", + "model.layers.45.mlp.experts.100.up_proj", + "model.layers.45.mlp.experts.101.down_proj", + "model.layers.45.mlp.experts.101.gate_proj", + "model.layers.45.mlp.experts.101.up_proj", + "model.layers.45.mlp.experts.102.down_proj", + "model.layers.45.mlp.experts.102.gate_proj", + "model.layers.45.mlp.experts.102.up_proj", + "model.layers.45.mlp.experts.103.down_proj", + "model.layers.45.mlp.experts.103.gate_proj", + "model.layers.45.mlp.experts.103.up_proj", + "model.layers.45.mlp.experts.104.down_proj", + "model.layers.45.mlp.experts.104.gate_proj", + "model.layers.45.mlp.experts.104.up_proj", + "model.layers.45.mlp.experts.105.down_proj", + "model.layers.45.mlp.experts.105.gate_proj", + "model.layers.45.mlp.experts.105.up_proj", + "model.layers.45.mlp.experts.106.down_proj", + "model.layers.45.mlp.experts.106.gate_proj", + "model.layers.45.mlp.experts.106.up_proj", + "model.layers.45.mlp.experts.107.down_proj", + "model.layers.45.mlp.experts.107.gate_proj", + "model.layers.45.mlp.experts.107.up_proj", + "model.layers.45.mlp.experts.108.down_proj", + "model.layers.45.mlp.experts.108.gate_proj", + "model.layers.45.mlp.experts.108.up_proj", + "model.layers.45.mlp.experts.109.down_proj", + "model.layers.45.mlp.experts.109.gate_proj", + "model.layers.45.mlp.experts.109.up_proj", + "model.layers.45.mlp.experts.11.down_proj", + "model.layers.45.mlp.experts.11.gate_proj", + "model.layers.45.mlp.experts.11.up_proj", + "model.layers.45.mlp.experts.110.down_proj", + "model.layers.45.mlp.experts.110.gate_proj", + "model.layers.45.mlp.experts.110.up_proj", + "model.layers.45.mlp.experts.111.down_proj", + "model.layers.45.mlp.experts.111.gate_proj", + "model.layers.45.mlp.experts.111.up_proj", + "model.layers.45.mlp.experts.112.down_proj", + "model.layers.45.mlp.experts.112.gate_proj", + "model.layers.45.mlp.experts.112.up_proj", + "model.layers.45.mlp.experts.113.down_proj", + "model.layers.45.mlp.experts.113.gate_proj", + "model.layers.45.mlp.experts.113.up_proj", + "model.layers.45.mlp.experts.114.down_proj", + "model.layers.45.mlp.experts.114.gate_proj", + "model.layers.45.mlp.experts.114.up_proj", + "model.layers.45.mlp.experts.115.down_proj", + "model.layers.45.mlp.experts.115.gate_proj", + "model.layers.45.mlp.experts.115.up_proj", + "model.layers.45.mlp.experts.116.down_proj", + "model.layers.45.mlp.experts.116.gate_proj", + "model.layers.45.mlp.experts.116.up_proj", + "model.layers.45.mlp.experts.117.down_proj", + "model.layers.45.mlp.experts.117.gate_proj", + "model.layers.45.mlp.experts.117.up_proj", + "model.layers.45.mlp.experts.118.down_proj", + "model.layers.45.mlp.experts.118.gate_proj", + "model.layers.45.mlp.experts.118.up_proj", + "model.layers.45.mlp.experts.119.down_proj", + "model.layers.45.mlp.experts.119.gate_proj", + "model.layers.45.mlp.experts.119.up_proj", + "model.layers.45.mlp.experts.12.down_proj", + "model.layers.45.mlp.experts.12.gate_proj", + "model.layers.45.mlp.experts.12.up_proj", + "model.layers.45.mlp.experts.120.down_proj", + "model.layers.45.mlp.experts.120.gate_proj", + "model.layers.45.mlp.experts.120.up_proj", + "model.layers.45.mlp.experts.121.down_proj", + "model.layers.45.mlp.experts.121.gate_proj", + "model.layers.45.mlp.experts.121.up_proj", + "model.layers.45.mlp.experts.122.down_proj", + "model.layers.45.mlp.experts.122.gate_proj", + "model.layers.45.mlp.experts.122.up_proj", + "model.layers.45.mlp.experts.123.down_proj", + "model.layers.45.mlp.experts.123.gate_proj", + "model.layers.45.mlp.experts.123.up_proj", + "model.layers.45.mlp.experts.124.down_proj", + "model.layers.45.mlp.experts.124.gate_proj", + "model.layers.45.mlp.experts.124.up_proj", + "model.layers.45.mlp.experts.125.down_proj", + "model.layers.45.mlp.experts.125.gate_proj", + "model.layers.45.mlp.experts.125.up_proj", + "model.layers.45.mlp.experts.126.down_proj", + "model.layers.45.mlp.experts.126.gate_proj", + "model.layers.45.mlp.experts.126.up_proj", + "model.layers.45.mlp.experts.127.down_proj", + "model.layers.45.mlp.experts.127.gate_proj", + "model.layers.45.mlp.experts.127.up_proj", + "model.layers.45.mlp.experts.128.down_proj", + "model.layers.45.mlp.experts.128.gate_proj", + "model.layers.45.mlp.experts.128.up_proj", + "model.layers.45.mlp.experts.129.down_proj", + "model.layers.45.mlp.experts.129.gate_proj", + "model.layers.45.mlp.experts.129.up_proj", + "model.layers.45.mlp.experts.13.down_proj", + "model.layers.45.mlp.experts.13.gate_proj", + "model.layers.45.mlp.experts.13.up_proj", + "model.layers.45.mlp.experts.130.down_proj", + "model.layers.45.mlp.experts.130.gate_proj", + "model.layers.45.mlp.experts.130.up_proj", + "model.layers.45.mlp.experts.131.down_proj", + "model.layers.45.mlp.experts.131.gate_proj", + "model.layers.45.mlp.experts.131.up_proj", + "model.layers.45.mlp.experts.132.down_proj", + "model.layers.45.mlp.experts.132.gate_proj", + "model.layers.45.mlp.experts.132.up_proj", + "model.layers.45.mlp.experts.133.down_proj", + "model.layers.45.mlp.experts.133.gate_proj", + "model.layers.45.mlp.experts.133.up_proj", + "model.layers.45.mlp.experts.134.down_proj", + "model.layers.45.mlp.experts.134.gate_proj", + "model.layers.45.mlp.experts.134.up_proj", + "model.layers.45.mlp.experts.135.down_proj", + "model.layers.45.mlp.experts.135.gate_proj", + "model.layers.45.mlp.experts.135.up_proj", + "model.layers.45.mlp.experts.136.down_proj", + "model.layers.45.mlp.experts.136.gate_proj", + "model.layers.45.mlp.experts.136.up_proj", + "model.layers.45.mlp.experts.137.down_proj", + "model.layers.45.mlp.experts.137.gate_proj", + "model.layers.45.mlp.experts.137.up_proj", + "model.layers.45.mlp.experts.138.down_proj", + "model.layers.45.mlp.experts.138.gate_proj", + "model.layers.45.mlp.experts.138.up_proj", + "model.layers.45.mlp.experts.139.down_proj", + "model.layers.45.mlp.experts.139.gate_proj", + "model.layers.45.mlp.experts.139.up_proj", + "model.layers.45.mlp.experts.14.down_proj", + "model.layers.45.mlp.experts.14.gate_proj", + "model.layers.45.mlp.experts.14.up_proj", + "model.layers.45.mlp.experts.140.down_proj", + "model.layers.45.mlp.experts.140.gate_proj", + "model.layers.45.mlp.experts.140.up_proj", + "model.layers.45.mlp.experts.141.down_proj", + "model.layers.45.mlp.experts.141.gate_proj", + "model.layers.45.mlp.experts.141.up_proj", + "model.layers.45.mlp.experts.142.down_proj", + "model.layers.45.mlp.experts.142.gate_proj", + "model.layers.45.mlp.experts.142.up_proj", + "model.layers.45.mlp.experts.143.down_proj", + "model.layers.45.mlp.experts.143.gate_proj", + "model.layers.45.mlp.experts.143.up_proj", + "model.layers.45.mlp.experts.144.down_proj", + "model.layers.45.mlp.experts.144.gate_proj", + "model.layers.45.mlp.experts.144.up_proj", + "model.layers.45.mlp.experts.145.down_proj", + "model.layers.45.mlp.experts.145.gate_proj", + "model.layers.45.mlp.experts.145.up_proj", + "model.layers.45.mlp.experts.146.down_proj", + "model.layers.45.mlp.experts.146.gate_proj", + "model.layers.45.mlp.experts.146.up_proj", + "model.layers.45.mlp.experts.147.down_proj", + "model.layers.45.mlp.experts.147.gate_proj", + "model.layers.45.mlp.experts.147.up_proj", + "model.layers.45.mlp.experts.148.down_proj", + "model.layers.45.mlp.experts.148.gate_proj", + "model.layers.45.mlp.experts.148.up_proj", + "model.layers.45.mlp.experts.149.down_proj", + "model.layers.45.mlp.experts.149.gate_proj", + "model.layers.45.mlp.experts.149.up_proj", + "model.layers.45.mlp.experts.15.down_proj", + "model.layers.45.mlp.experts.15.gate_proj", + "model.layers.45.mlp.experts.15.up_proj", + "model.layers.45.mlp.experts.150.down_proj", + "model.layers.45.mlp.experts.150.gate_proj", + "model.layers.45.mlp.experts.150.up_proj", + "model.layers.45.mlp.experts.151.down_proj", + "model.layers.45.mlp.experts.151.gate_proj", + "model.layers.45.mlp.experts.151.up_proj", + "model.layers.45.mlp.experts.152.down_proj", + "model.layers.45.mlp.experts.152.gate_proj", + "model.layers.45.mlp.experts.152.up_proj", + "model.layers.45.mlp.experts.153.down_proj", + "model.layers.45.mlp.experts.153.gate_proj", + "model.layers.45.mlp.experts.153.up_proj", + "model.layers.45.mlp.experts.154.down_proj", + "model.layers.45.mlp.experts.154.gate_proj", + "model.layers.45.mlp.experts.154.up_proj", + "model.layers.45.mlp.experts.155.down_proj", + "model.layers.45.mlp.experts.155.gate_proj", + "model.layers.45.mlp.experts.155.up_proj", + "model.layers.45.mlp.experts.156.down_proj", + "model.layers.45.mlp.experts.156.gate_proj", + "model.layers.45.mlp.experts.156.up_proj", + "model.layers.45.mlp.experts.157.down_proj", + "model.layers.45.mlp.experts.157.gate_proj", + "model.layers.45.mlp.experts.157.up_proj", + "model.layers.45.mlp.experts.158.down_proj", + "model.layers.45.mlp.experts.158.gate_proj", + "model.layers.45.mlp.experts.158.up_proj", + "model.layers.45.mlp.experts.159.down_proj", + "model.layers.45.mlp.experts.159.gate_proj", + "model.layers.45.mlp.experts.159.up_proj", + "model.layers.45.mlp.experts.16.down_proj", + "model.layers.45.mlp.experts.16.gate_proj", + "model.layers.45.mlp.experts.16.up_proj", + "model.layers.45.mlp.experts.160.down_proj", + "model.layers.45.mlp.experts.160.gate_proj", + "model.layers.45.mlp.experts.160.up_proj", + "model.layers.45.mlp.experts.161.down_proj", + "model.layers.45.mlp.experts.161.gate_proj", + "model.layers.45.mlp.experts.161.up_proj", + "model.layers.45.mlp.experts.162.down_proj", + "model.layers.45.mlp.experts.162.gate_proj", + "model.layers.45.mlp.experts.162.up_proj", + "model.layers.45.mlp.experts.163.down_proj", + "model.layers.45.mlp.experts.163.gate_proj", + "model.layers.45.mlp.experts.163.up_proj", + "model.layers.45.mlp.experts.164.down_proj", + "model.layers.45.mlp.experts.164.gate_proj", + "model.layers.45.mlp.experts.164.up_proj", + "model.layers.45.mlp.experts.165.down_proj", + "model.layers.45.mlp.experts.165.gate_proj", + "model.layers.45.mlp.experts.165.up_proj", + "model.layers.45.mlp.experts.166.down_proj", + "model.layers.45.mlp.experts.166.gate_proj", + "model.layers.45.mlp.experts.166.up_proj", + "model.layers.45.mlp.experts.167.down_proj", + "model.layers.45.mlp.experts.167.gate_proj", + "model.layers.45.mlp.experts.167.up_proj", + "model.layers.45.mlp.experts.168.down_proj", + "model.layers.45.mlp.experts.168.gate_proj", + "model.layers.45.mlp.experts.168.up_proj", + "model.layers.45.mlp.experts.169.down_proj", + "model.layers.45.mlp.experts.169.gate_proj", + "model.layers.45.mlp.experts.169.up_proj", + "model.layers.45.mlp.experts.17.down_proj", + "model.layers.45.mlp.experts.17.gate_proj", + "model.layers.45.mlp.experts.17.up_proj", + "model.layers.45.mlp.experts.170.down_proj", + "model.layers.45.mlp.experts.170.gate_proj", + "model.layers.45.mlp.experts.170.up_proj", + "model.layers.45.mlp.experts.171.down_proj", + "model.layers.45.mlp.experts.171.gate_proj", + "model.layers.45.mlp.experts.171.up_proj", + "model.layers.45.mlp.experts.172.down_proj", + "model.layers.45.mlp.experts.172.gate_proj", + "model.layers.45.mlp.experts.172.up_proj", + "model.layers.45.mlp.experts.173.down_proj", + "model.layers.45.mlp.experts.173.gate_proj", + "model.layers.45.mlp.experts.173.up_proj", + "model.layers.45.mlp.experts.174.down_proj", + "model.layers.45.mlp.experts.174.gate_proj", + "model.layers.45.mlp.experts.174.up_proj", + "model.layers.45.mlp.experts.175.down_proj", + "model.layers.45.mlp.experts.175.gate_proj", + "model.layers.45.mlp.experts.175.up_proj", + "model.layers.45.mlp.experts.176.down_proj", + "model.layers.45.mlp.experts.176.gate_proj", + "model.layers.45.mlp.experts.176.up_proj", + "model.layers.45.mlp.experts.177.down_proj", + "model.layers.45.mlp.experts.177.gate_proj", + "model.layers.45.mlp.experts.177.up_proj", + "model.layers.45.mlp.experts.178.down_proj", + "model.layers.45.mlp.experts.178.gate_proj", + "model.layers.45.mlp.experts.178.up_proj", + "model.layers.45.mlp.experts.179.down_proj", + "model.layers.45.mlp.experts.179.gate_proj", + "model.layers.45.mlp.experts.179.up_proj", + "model.layers.45.mlp.experts.18.down_proj", + "model.layers.45.mlp.experts.18.gate_proj", + "model.layers.45.mlp.experts.18.up_proj", + "model.layers.45.mlp.experts.180.down_proj", + "model.layers.45.mlp.experts.180.gate_proj", + "model.layers.45.mlp.experts.180.up_proj", + "model.layers.45.mlp.experts.181.down_proj", + "model.layers.45.mlp.experts.181.gate_proj", + "model.layers.45.mlp.experts.181.up_proj", + "model.layers.45.mlp.experts.182.down_proj", + "model.layers.45.mlp.experts.182.gate_proj", + "model.layers.45.mlp.experts.182.up_proj", + "model.layers.45.mlp.experts.183.down_proj", + "model.layers.45.mlp.experts.183.gate_proj", + "model.layers.45.mlp.experts.183.up_proj", + "model.layers.45.mlp.experts.184.down_proj", + "model.layers.45.mlp.experts.184.gate_proj", + "model.layers.45.mlp.experts.184.up_proj", + "model.layers.45.mlp.experts.185.down_proj", + "model.layers.45.mlp.experts.185.gate_proj", + "model.layers.45.mlp.experts.185.up_proj", + "model.layers.45.mlp.experts.186.down_proj", + "model.layers.45.mlp.experts.186.gate_proj", + "model.layers.45.mlp.experts.186.up_proj", + "model.layers.45.mlp.experts.187.down_proj", + "model.layers.45.mlp.experts.187.gate_proj", + "model.layers.45.mlp.experts.187.up_proj", + "model.layers.45.mlp.experts.188.down_proj", + "model.layers.45.mlp.experts.188.gate_proj", + "model.layers.45.mlp.experts.188.up_proj", + "model.layers.45.mlp.experts.189.down_proj", + "model.layers.45.mlp.experts.189.gate_proj", + "model.layers.45.mlp.experts.189.up_proj", + "model.layers.45.mlp.experts.19.down_proj", + "model.layers.45.mlp.experts.19.gate_proj", + "model.layers.45.mlp.experts.19.up_proj", + "model.layers.45.mlp.experts.190.down_proj", + "model.layers.45.mlp.experts.190.gate_proj", + "model.layers.45.mlp.experts.190.up_proj", + "model.layers.45.mlp.experts.191.down_proj", + "model.layers.45.mlp.experts.191.gate_proj", + "model.layers.45.mlp.experts.191.up_proj", + "model.layers.45.mlp.experts.192.down_proj", + "model.layers.45.mlp.experts.192.gate_proj", + "model.layers.45.mlp.experts.192.up_proj", + "model.layers.45.mlp.experts.193.down_proj", + "model.layers.45.mlp.experts.193.gate_proj", + "model.layers.45.mlp.experts.193.up_proj", + "model.layers.45.mlp.experts.194.down_proj", + "model.layers.45.mlp.experts.194.gate_proj", + "model.layers.45.mlp.experts.194.up_proj", + "model.layers.45.mlp.experts.195.down_proj", + "model.layers.45.mlp.experts.195.gate_proj", + "model.layers.45.mlp.experts.195.up_proj", + "model.layers.45.mlp.experts.196.down_proj", + "model.layers.45.mlp.experts.196.gate_proj", + "model.layers.45.mlp.experts.196.up_proj", + "model.layers.45.mlp.experts.197.down_proj", + "model.layers.45.mlp.experts.197.gate_proj", + "model.layers.45.mlp.experts.197.up_proj", + "model.layers.45.mlp.experts.198.down_proj", + "model.layers.45.mlp.experts.198.gate_proj", + "model.layers.45.mlp.experts.198.up_proj", + "model.layers.45.mlp.experts.199.down_proj", + "model.layers.45.mlp.experts.199.gate_proj", + "model.layers.45.mlp.experts.199.up_proj", + "model.layers.45.mlp.experts.2.down_proj", + "model.layers.45.mlp.experts.2.gate_proj", + "model.layers.45.mlp.experts.2.up_proj", + "model.layers.45.mlp.experts.20.down_proj", + "model.layers.45.mlp.experts.20.gate_proj", + "model.layers.45.mlp.experts.20.up_proj", + "model.layers.45.mlp.experts.200.down_proj", + "model.layers.45.mlp.experts.200.gate_proj", + "model.layers.45.mlp.experts.200.up_proj", + "model.layers.45.mlp.experts.201.down_proj", + "model.layers.45.mlp.experts.201.gate_proj", + "model.layers.45.mlp.experts.201.up_proj", + "model.layers.45.mlp.experts.202.down_proj", + "model.layers.45.mlp.experts.202.gate_proj", + "model.layers.45.mlp.experts.202.up_proj", + "model.layers.45.mlp.experts.203.down_proj", + "model.layers.45.mlp.experts.203.gate_proj", + "model.layers.45.mlp.experts.203.up_proj", + "model.layers.45.mlp.experts.204.down_proj", + "model.layers.45.mlp.experts.204.gate_proj", + "model.layers.45.mlp.experts.204.up_proj", + "model.layers.45.mlp.experts.205.down_proj", + "model.layers.45.mlp.experts.205.gate_proj", + "model.layers.45.mlp.experts.205.up_proj", + "model.layers.45.mlp.experts.206.down_proj", + "model.layers.45.mlp.experts.206.gate_proj", + "model.layers.45.mlp.experts.206.up_proj", + "model.layers.45.mlp.experts.207.down_proj", + "model.layers.45.mlp.experts.207.gate_proj", + "model.layers.45.mlp.experts.207.up_proj", + "model.layers.45.mlp.experts.208.down_proj", + "model.layers.45.mlp.experts.208.gate_proj", + "model.layers.45.mlp.experts.208.up_proj", + "model.layers.45.mlp.experts.209.down_proj", + "model.layers.45.mlp.experts.209.gate_proj", + "model.layers.45.mlp.experts.209.up_proj", + "model.layers.45.mlp.experts.21.down_proj", + "model.layers.45.mlp.experts.21.gate_proj", + "model.layers.45.mlp.experts.21.up_proj", + "model.layers.45.mlp.experts.210.down_proj", + "model.layers.45.mlp.experts.210.gate_proj", + "model.layers.45.mlp.experts.210.up_proj", + "model.layers.45.mlp.experts.211.down_proj", + "model.layers.45.mlp.experts.211.gate_proj", + "model.layers.45.mlp.experts.211.up_proj", + "model.layers.45.mlp.experts.212.down_proj", + "model.layers.45.mlp.experts.212.gate_proj", + "model.layers.45.mlp.experts.212.up_proj", + "model.layers.45.mlp.experts.213.down_proj", + "model.layers.45.mlp.experts.213.gate_proj", + "model.layers.45.mlp.experts.213.up_proj", + "model.layers.45.mlp.experts.214.down_proj", + "model.layers.45.mlp.experts.214.gate_proj", + "model.layers.45.mlp.experts.214.up_proj", + "model.layers.45.mlp.experts.215.down_proj", + "model.layers.45.mlp.experts.215.gate_proj", + "model.layers.45.mlp.experts.215.up_proj", + "model.layers.45.mlp.experts.216.down_proj", + "model.layers.45.mlp.experts.216.gate_proj", + "model.layers.45.mlp.experts.216.up_proj", + "model.layers.45.mlp.experts.217.down_proj", + "model.layers.45.mlp.experts.217.gate_proj", + "model.layers.45.mlp.experts.217.up_proj", + "model.layers.45.mlp.experts.218.down_proj", + "model.layers.45.mlp.experts.218.gate_proj", + "model.layers.45.mlp.experts.218.up_proj", + "model.layers.45.mlp.experts.219.down_proj", + "model.layers.45.mlp.experts.219.gate_proj", + "model.layers.45.mlp.experts.219.up_proj", + "model.layers.45.mlp.experts.22.down_proj", + "model.layers.45.mlp.experts.22.gate_proj", + "model.layers.45.mlp.experts.22.up_proj", + "model.layers.45.mlp.experts.220.down_proj", + "model.layers.45.mlp.experts.220.gate_proj", + "model.layers.45.mlp.experts.220.up_proj", + "model.layers.45.mlp.experts.221.down_proj", + "model.layers.45.mlp.experts.221.gate_proj", + "model.layers.45.mlp.experts.221.up_proj", + "model.layers.45.mlp.experts.222.down_proj", + "model.layers.45.mlp.experts.222.gate_proj", + "model.layers.45.mlp.experts.222.up_proj", + "model.layers.45.mlp.experts.223.down_proj", + "model.layers.45.mlp.experts.223.gate_proj", + "model.layers.45.mlp.experts.223.up_proj", + "model.layers.45.mlp.experts.224.down_proj", + "model.layers.45.mlp.experts.224.gate_proj", + "model.layers.45.mlp.experts.224.up_proj", + "model.layers.45.mlp.experts.225.down_proj", + "model.layers.45.mlp.experts.225.gate_proj", + "model.layers.45.mlp.experts.225.up_proj", + "model.layers.45.mlp.experts.226.down_proj", + "model.layers.45.mlp.experts.226.gate_proj", + "model.layers.45.mlp.experts.226.up_proj", + "model.layers.45.mlp.experts.227.down_proj", + "model.layers.45.mlp.experts.227.gate_proj", + "model.layers.45.mlp.experts.227.up_proj", + "model.layers.45.mlp.experts.228.down_proj", + "model.layers.45.mlp.experts.228.gate_proj", + "model.layers.45.mlp.experts.228.up_proj", + "model.layers.45.mlp.experts.229.down_proj", + "model.layers.45.mlp.experts.229.gate_proj", + "model.layers.45.mlp.experts.229.up_proj", + "model.layers.45.mlp.experts.23.down_proj", + "model.layers.45.mlp.experts.23.gate_proj", + "model.layers.45.mlp.experts.23.up_proj", + "model.layers.45.mlp.experts.230.down_proj", + "model.layers.45.mlp.experts.230.gate_proj", + "model.layers.45.mlp.experts.230.up_proj", + "model.layers.45.mlp.experts.231.down_proj", + "model.layers.45.mlp.experts.231.gate_proj", + "model.layers.45.mlp.experts.231.up_proj", + "model.layers.45.mlp.experts.232.down_proj", + "model.layers.45.mlp.experts.232.gate_proj", + "model.layers.45.mlp.experts.232.up_proj", + "model.layers.45.mlp.experts.233.down_proj", + "model.layers.45.mlp.experts.233.gate_proj", + "model.layers.45.mlp.experts.233.up_proj", + "model.layers.45.mlp.experts.234.down_proj", + "model.layers.45.mlp.experts.234.gate_proj", + "model.layers.45.mlp.experts.234.up_proj", + "model.layers.45.mlp.experts.235.down_proj", + "model.layers.45.mlp.experts.235.gate_proj", + "model.layers.45.mlp.experts.235.up_proj", + "model.layers.45.mlp.experts.236.down_proj", + "model.layers.45.mlp.experts.236.gate_proj", + "model.layers.45.mlp.experts.236.up_proj", + "model.layers.45.mlp.experts.237.down_proj", + "model.layers.45.mlp.experts.237.gate_proj", + "model.layers.45.mlp.experts.237.up_proj", + "model.layers.45.mlp.experts.238.down_proj", + "model.layers.45.mlp.experts.238.gate_proj", + "model.layers.45.mlp.experts.238.up_proj", + "model.layers.45.mlp.experts.239.down_proj", + "model.layers.45.mlp.experts.239.gate_proj", + "model.layers.45.mlp.experts.239.up_proj", + "model.layers.45.mlp.experts.24.down_proj", + "model.layers.45.mlp.experts.24.gate_proj", + "model.layers.45.mlp.experts.24.up_proj", + "model.layers.45.mlp.experts.240.down_proj", + "model.layers.45.mlp.experts.240.gate_proj", + "model.layers.45.mlp.experts.240.up_proj", + "model.layers.45.mlp.experts.241.down_proj", + "model.layers.45.mlp.experts.241.gate_proj", + "model.layers.45.mlp.experts.241.up_proj", + "model.layers.45.mlp.experts.242.down_proj", + "model.layers.45.mlp.experts.242.gate_proj", + "model.layers.45.mlp.experts.242.up_proj", + "model.layers.45.mlp.experts.243.down_proj", + "model.layers.45.mlp.experts.243.gate_proj", + "model.layers.45.mlp.experts.243.up_proj", + "model.layers.45.mlp.experts.244.down_proj", + "model.layers.45.mlp.experts.244.gate_proj", + "model.layers.45.mlp.experts.244.up_proj", + "model.layers.45.mlp.experts.245.down_proj", + "model.layers.45.mlp.experts.245.gate_proj", + "model.layers.45.mlp.experts.245.up_proj", + "model.layers.45.mlp.experts.246.down_proj", + "model.layers.45.mlp.experts.246.gate_proj", + "model.layers.45.mlp.experts.246.up_proj", + "model.layers.45.mlp.experts.247.down_proj", + "model.layers.45.mlp.experts.247.gate_proj", + "model.layers.45.mlp.experts.247.up_proj", + "model.layers.45.mlp.experts.248.down_proj", + "model.layers.45.mlp.experts.248.gate_proj", + "model.layers.45.mlp.experts.248.up_proj", + "model.layers.45.mlp.experts.249.down_proj", + "model.layers.45.mlp.experts.249.gate_proj", + "model.layers.45.mlp.experts.249.up_proj", + "model.layers.45.mlp.experts.25.down_proj", + "model.layers.45.mlp.experts.25.gate_proj", + "model.layers.45.mlp.experts.25.up_proj", + "model.layers.45.mlp.experts.250.down_proj", + "model.layers.45.mlp.experts.250.gate_proj", + "model.layers.45.mlp.experts.250.up_proj", + "model.layers.45.mlp.experts.251.down_proj", + "model.layers.45.mlp.experts.251.gate_proj", + "model.layers.45.mlp.experts.251.up_proj", + "model.layers.45.mlp.experts.252.down_proj", + "model.layers.45.mlp.experts.252.gate_proj", + "model.layers.45.mlp.experts.252.up_proj", + "model.layers.45.mlp.experts.253.down_proj", + "model.layers.45.mlp.experts.253.gate_proj", + "model.layers.45.mlp.experts.253.up_proj", + "model.layers.45.mlp.experts.254.down_proj", + "model.layers.45.mlp.experts.254.gate_proj", + "model.layers.45.mlp.experts.254.up_proj", + "model.layers.45.mlp.experts.255.down_proj", + "model.layers.45.mlp.experts.255.gate_proj", + "model.layers.45.mlp.experts.255.up_proj", + "model.layers.45.mlp.experts.26.down_proj", + "model.layers.45.mlp.experts.26.gate_proj", + "model.layers.45.mlp.experts.26.up_proj", + "model.layers.45.mlp.experts.27.down_proj", + "model.layers.45.mlp.experts.27.gate_proj", + "model.layers.45.mlp.experts.27.up_proj", + "model.layers.45.mlp.experts.28.down_proj", + "model.layers.45.mlp.experts.28.gate_proj", + "model.layers.45.mlp.experts.28.up_proj", + "model.layers.45.mlp.experts.29.down_proj", + "model.layers.45.mlp.experts.29.gate_proj", + "model.layers.45.mlp.experts.29.up_proj", + "model.layers.45.mlp.experts.3.down_proj", + "model.layers.45.mlp.experts.3.gate_proj", + "model.layers.45.mlp.experts.3.up_proj", + "model.layers.45.mlp.experts.30.down_proj", + "model.layers.45.mlp.experts.30.gate_proj", + "model.layers.45.mlp.experts.30.up_proj", + "model.layers.45.mlp.experts.31.down_proj", + "model.layers.45.mlp.experts.31.gate_proj", + "model.layers.45.mlp.experts.31.up_proj", + "model.layers.45.mlp.experts.32.down_proj", + "model.layers.45.mlp.experts.32.gate_proj", + "model.layers.45.mlp.experts.32.up_proj", + "model.layers.45.mlp.experts.33.down_proj", + "model.layers.45.mlp.experts.33.gate_proj", + "model.layers.45.mlp.experts.33.up_proj", + "model.layers.45.mlp.experts.34.down_proj", + "model.layers.45.mlp.experts.34.gate_proj", + "model.layers.45.mlp.experts.34.up_proj", + "model.layers.45.mlp.experts.35.down_proj", + "model.layers.45.mlp.experts.35.gate_proj", + "model.layers.45.mlp.experts.35.up_proj", + "model.layers.45.mlp.experts.36.down_proj", + "model.layers.45.mlp.experts.36.gate_proj", + "model.layers.45.mlp.experts.36.up_proj", + "model.layers.45.mlp.experts.37.down_proj", + "model.layers.45.mlp.experts.37.gate_proj", + "model.layers.45.mlp.experts.37.up_proj", + "model.layers.45.mlp.experts.38.down_proj", + "model.layers.45.mlp.experts.38.gate_proj", + "model.layers.45.mlp.experts.38.up_proj", + "model.layers.45.mlp.experts.39.down_proj", + "model.layers.45.mlp.experts.39.gate_proj", + "model.layers.45.mlp.experts.39.up_proj", + "model.layers.45.mlp.experts.4.down_proj", + "model.layers.45.mlp.experts.4.gate_proj", + "model.layers.45.mlp.experts.4.up_proj", + "model.layers.45.mlp.experts.40.down_proj", + "model.layers.45.mlp.experts.40.gate_proj", + "model.layers.45.mlp.experts.40.up_proj", + "model.layers.45.mlp.experts.41.down_proj", + "model.layers.45.mlp.experts.41.gate_proj", + "model.layers.45.mlp.experts.41.up_proj", + "model.layers.45.mlp.experts.42.down_proj", + "model.layers.45.mlp.experts.42.gate_proj", + "model.layers.45.mlp.experts.42.up_proj", + "model.layers.45.mlp.experts.43.down_proj", + "model.layers.45.mlp.experts.43.gate_proj", + "model.layers.45.mlp.experts.43.up_proj", + "model.layers.45.mlp.experts.44.down_proj", + "model.layers.45.mlp.experts.44.gate_proj", + "model.layers.45.mlp.experts.44.up_proj", + "model.layers.45.mlp.experts.45.down_proj", + "model.layers.45.mlp.experts.45.gate_proj", + "model.layers.45.mlp.experts.45.up_proj", + "model.layers.45.mlp.experts.46.down_proj", + "model.layers.45.mlp.experts.46.gate_proj", + "model.layers.45.mlp.experts.46.up_proj", + "model.layers.45.mlp.experts.47.down_proj", + "model.layers.45.mlp.experts.47.gate_proj", + "model.layers.45.mlp.experts.47.up_proj", + "model.layers.45.mlp.experts.48.down_proj", + "model.layers.45.mlp.experts.48.gate_proj", + "model.layers.45.mlp.experts.48.up_proj", + "model.layers.45.mlp.experts.49.down_proj", + "model.layers.45.mlp.experts.49.gate_proj", + "model.layers.45.mlp.experts.49.up_proj", + "model.layers.45.mlp.experts.5.down_proj", + "model.layers.45.mlp.experts.5.gate_proj", + "model.layers.45.mlp.experts.5.up_proj", + "model.layers.45.mlp.experts.50.down_proj", + "model.layers.45.mlp.experts.50.gate_proj", + "model.layers.45.mlp.experts.50.up_proj", + "model.layers.45.mlp.experts.51.down_proj", + "model.layers.45.mlp.experts.51.gate_proj", + "model.layers.45.mlp.experts.51.up_proj", + "model.layers.45.mlp.experts.52.down_proj", + "model.layers.45.mlp.experts.52.gate_proj", + "model.layers.45.mlp.experts.52.up_proj", + "model.layers.45.mlp.experts.53.down_proj", + "model.layers.45.mlp.experts.53.gate_proj", + "model.layers.45.mlp.experts.53.up_proj", + "model.layers.45.mlp.experts.54.down_proj", + "model.layers.45.mlp.experts.54.gate_proj", + "model.layers.45.mlp.experts.54.up_proj", + "model.layers.45.mlp.experts.55.down_proj", + "model.layers.45.mlp.experts.55.gate_proj", + "model.layers.45.mlp.experts.55.up_proj", + "model.layers.45.mlp.experts.56.down_proj", + "model.layers.45.mlp.experts.56.gate_proj", + "model.layers.45.mlp.experts.56.up_proj", + "model.layers.45.mlp.experts.57.down_proj", + "model.layers.45.mlp.experts.57.gate_proj", + "model.layers.45.mlp.experts.57.up_proj", + "model.layers.45.mlp.experts.58.down_proj", + "model.layers.45.mlp.experts.58.gate_proj", + "model.layers.45.mlp.experts.58.up_proj", + "model.layers.45.mlp.experts.59.down_proj", + "model.layers.45.mlp.experts.59.gate_proj", + "model.layers.45.mlp.experts.59.up_proj", + "model.layers.45.mlp.experts.6.down_proj", + "model.layers.45.mlp.experts.6.gate_proj", + "model.layers.45.mlp.experts.6.up_proj", + "model.layers.45.mlp.experts.60.down_proj", + "model.layers.45.mlp.experts.60.gate_proj", + "model.layers.45.mlp.experts.60.up_proj", + "model.layers.45.mlp.experts.61.down_proj", + "model.layers.45.mlp.experts.61.gate_proj", + "model.layers.45.mlp.experts.61.up_proj", + "model.layers.45.mlp.experts.62.down_proj", + "model.layers.45.mlp.experts.62.gate_proj", + "model.layers.45.mlp.experts.62.up_proj", + "model.layers.45.mlp.experts.63.down_proj", + "model.layers.45.mlp.experts.63.gate_proj", + "model.layers.45.mlp.experts.63.up_proj", + "model.layers.45.mlp.experts.64.down_proj", + "model.layers.45.mlp.experts.64.gate_proj", + "model.layers.45.mlp.experts.64.up_proj", + "model.layers.45.mlp.experts.65.down_proj", + "model.layers.45.mlp.experts.65.gate_proj", + "model.layers.45.mlp.experts.65.up_proj", + "model.layers.45.mlp.experts.66.down_proj", + "model.layers.45.mlp.experts.66.gate_proj", + "model.layers.45.mlp.experts.66.up_proj", + "model.layers.45.mlp.experts.67.down_proj", + "model.layers.45.mlp.experts.67.gate_proj", + "model.layers.45.mlp.experts.67.up_proj", + "model.layers.45.mlp.experts.68.down_proj", + "model.layers.45.mlp.experts.68.gate_proj", + "model.layers.45.mlp.experts.68.up_proj", + "model.layers.45.mlp.experts.69.down_proj", + "model.layers.45.mlp.experts.69.gate_proj", + "model.layers.45.mlp.experts.69.up_proj", + "model.layers.45.mlp.experts.7.down_proj", + "model.layers.45.mlp.experts.7.gate_proj", + "model.layers.45.mlp.experts.7.up_proj", + "model.layers.45.mlp.experts.70.down_proj", + "model.layers.45.mlp.experts.70.gate_proj", + "model.layers.45.mlp.experts.70.up_proj", + "model.layers.45.mlp.experts.71.down_proj", + "model.layers.45.mlp.experts.71.gate_proj", + "model.layers.45.mlp.experts.71.up_proj", + "model.layers.45.mlp.experts.72.down_proj", + "model.layers.45.mlp.experts.72.gate_proj", + "model.layers.45.mlp.experts.72.up_proj", + "model.layers.45.mlp.experts.73.down_proj", + "model.layers.45.mlp.experts.73.gate_proj", + "model.layers.45.mlp.experts.73.up_proj", + "model.layers.45.mlp.experts.74.down_proj", + "model.layers.45.mlp.experts.74.gate_proj", + "model.layers.45.mlp.experts.74.up_proj", + "model.layers.45.mlp.experts.75.down_proj", + "model.layers.45.mlp.experts.75.gate_proj", + "model.layers.45.mlp.experts.75.up_proj", + "model.layers.45.mlp.experts.76.down_proj", + "model.layers.45.mlp.experts.76.gate_proj", + "model.layers.45.mlp.experts.76.up_proj", + "model.layers.45.mlp.experts.77.down_proj", + "model.layers.45.mlp.experts.77.gate_proj", + "model.layers.45.mlp.experts.77.up_proj", + "model.layers.45.mlp.experts.78.down_proj", + "model.layers.45.mlp.experts.78.gate_proj", + "model.layers.45.mlp.experts.78.up_proj", + "model.layers.45.mlp.experts.79.down_proj", + "model.layers.45.mlp.experts.79.gate_proj", + "model.layers.45.mlp.experts.79.up_proj", + "model.layers.45.mlp.experts.8.down_proj", + "model.layers.45.mlp.experts.8.gate_proj", + "model.layers.45.mlp.experts.8.up_proj", + "model.layers.45.mlp.experts.80.down_proj", + "model.layers.45.mlp.experts.80.gate_proj", + "model.layers.45.mlp.experts.80.up_proj", + "model.layers.45.mlp.experts.81.down_proj", + "model.layers.45.mlp.experts.81.gate_proj", + "model.layers.45.mlp.experts.81.up_proj", + "model.layers.45.mlp.experts.82.down_proj", + "model.layers.45.mlp.experts.82.gate_proj", + "model.layers.45.mlp.experts.82.up_proj", + "model.layers.45.mlp.experts.83.down_proj", + "model.layers.45.mlp.experts.83.gate_proj", + "model.layers.45.mlp.experts.83.up_proj", + "model.layers.45.mlp.experts.84.down_proj", + "model.layers.45.mlp.experts.84.gate_proj", + "model.layers.45.mlp.experts.84.up_proj", + "model.layers.45.mlp.experts.85.down_proj", + "model.layers.45.mlp.experts.85.gate_proj", + "model.layers.45.mlp.experts.85.up_proj", + "model.layers.45.mlp.experts.86.down_proj", + "model.layers.45.mlp.experts.86.gate_proj", + "model.layers.45.mlp.experts.86.up_proj", + "model.layers.45.mlp.experts.87.down_proj", + "model.layers.45.mlp.experts.87.gate_proj", + "model.layers.45.mlp.experts.87.up_proj", + "model.layers.45.mlp.experts.88.down_proj", + "model.layers.45.mlp.experts.88.gate_proj", + "model.layers.45.mlp.experts.88.up_proj", + "model.layers.45.mlp.experts.89.down_proj", + "model.layers.45.mlp.experts.89.gate_proj", + "model.layers.45.mlp.experts.89.up_proj", + "model.layers.45.mlp.experts.9.down_proj", + "model.layers.45.mlp.experts.9.gate_proj", + "model.layers.45.mlp.experts.9.up_proj", + "model.layers.45.mlp.experts.90.down_proj", + "model.layers.45.mlp.experts.90.gate_proj", + "model.layers.45.mlp.experts.90.up_proj", + "model.layers.45.mlp.experts.91.down_proj", + "model.layers.45.mlp.experts.91.gate_proj", + "model.layers.45.mlp.experts.91.up_proj", + "model.layers.45.mlp.experts.92.down_proj", + "model.layers.45.mlp.experts.92.gate_proj", + "model.layers.45.mlp.experts.92.up_proj", + "model.layers.45.mlp.experts.93.down_proj", + "model.layers.45.mlp.experts.93.gate_proj", + "model.layers.45.mlp.experts.93.up_proj", + "model.layers.45.mlp.experts.94.down_proj", + "model.layers.45.mlp.experts.94.gate_proj", + "model.layers.45.mlp.experts.94.up_proj", + "model.layers.45.mlp.experts.95.down_proj", + "model.layers.45.mlp.experts.95.gate_proj", + "model.layers.45.mlp.experts.95.up_proj", + "model.layers.45.mlp.experts.96.down_proj", + "model.layers.45.mlp.experts.96.gate_proj", + "model.layers.45.mlp.experts.96.up_proj", + "model.layers.45.mlp.experts.97.down_proj", + "model.layers.45.mlp.experts.97.gate_proj", + "model.layers.45.mlp.experts.97.up_proj", + "model.layers.45.mlp.experts.98.down_proj", + "model.layers.45.mlp.experts.98.gate_proj", + "model.layers.45.mlp.experts.98.up_proj", + "model.layers.45.mlp.experts.99.down_proj", + "model.layers.45.mlp.experts.99.gate_proj", + "model.layers.45.mlp.experts.99.up_proj", + "model.layers.46.mlp.experts.0.down_proj", + "model.layers.46.mlp.experts.0.gate_proj", + "model.layers.46.mlp.experts.0.up_proj", + "model.layers.46.mlp.experts.1.down_proj", + "model.layers.46.mlp.experts.1.gate_proj", + "model.layers.46.mlp.experts.1.up_proj", + "model.layers.46.mlp.experts.10.down_proj", + "model.layers.46.mlp.experts.10.gate_proj", + "model.layers.46.mlp.experts.10.up_proj", + "model.layers.46.mlp.experts.100.down_proj", + "model.layers.46.mlp.experts.100.gate_proj", + "model.layers.46.mlp.experts.100.up_proj", + "model.layers.46.mlp.experts.101.down_proj", + "model.layers.46.mlp.experts.101.gate_proj", + "model.layers.46.mlp.experts.101.up_proj", + "model.layers.46.mlp.experts.102.down_proj", + "model.layers.46.mlp.experts.102.gate_proj", + "model.layers.46.mlp.experts.102.up_proj", + "model.layers.46.mlp.experts.103.down_proj", + "model.layers.46.mlp.experts.103.gate_proj", + "model.layers.46.mlp.experts.103.up_proj", + "model.layers.46.mlp.experts.104.down_proj", + "model.layers.46.mlp.experts.104.gate_proj", + "model.layers.46.mlp.experts.104.up_proj", + "model.layers.46.mlp.experts.105.down_proj", + "model.layers.46.mlp.experts.105.gate_proj", + "model.layers.46.mlp.experts.105.up_proj", + "model.layers.46.mlp.experts.106.down_proj", + "model.layers.46.mlp.experts.106.gate_proj", + "model.layers.46.mlp.experts.106.up_proj", + "model.layers.46.mlp.experts.107.down_proj", + "model.layers.46.mlp.experts.107.gate_proj", + "model.layers.46.mlp.experts.107.up_proj", + "model.layers.46.mlp.experts.108.down_proj", + "model.layers.46.mlp.experts.108.gate_proj", + "model.layers.46.mlp.experts.108.up_proj", + "model.layers.46.mlp.experts.109.down_proj", + "model.layers.46.mlp.experts.109.gate_proj", + "model.layers.46.mlp.experts.109.up_proj", + "model.layers.46.mlp.experts.11.down_proj", + "model.layers.46.mlp.experts.11.gate_proj", + "model.layers.46.mlp.experts.11.up_proj", + "model.layers.46.mlp.experts.110.down_proj", + "model.layers.46.mlp.experts.110.gate_proj", + "model.layers.46.mlp.experts.110.up_proj", + "model.layers.46.mlp.experts.111.down_proj", + "model.layers.46.mlp.experts.111.gate_proj", + "model.layers.46.mlp.experts.111.up_proj", + "model.layers.46.mlp.experts.112.down_proj", + "model.layers.46.mlp.experts.112.gate_proj", + "model.layers.46.mlp.experts.112.up_proj", + "model.layers.46.mlp.experts.113.down_proj", + "model.layers.46.mlp.experts.113.gate_proj", + "model.layers.46.mlp.experts.113.up_proj", + "model.layers.46.mlp.experts.114.down_proj", + "model.layers.46.mlp.experts.114.gate_proj", + "model.layers.46.mlp.experts.114.up_proj", + "model.layers.46.mlp.experts.115.down_proj", + "model.layers.46.mlp.experts.115.gate_proj", + "model.layers.46.mlp.experts.115.up_proj", + "model.layers.46.mlp.experts.116.down_proj", + "model.layers.46.mlp.experts.116.gate_proj", + "model.layers.46.mlp.experts.116.up_proj", + "model.layers.46.mlp.experts.117.down_proj", + "model.layers.46.mlp.experts.117.gate_proj", + "model.layers.46.mlp.experts.117.up_proj", + "model.layers.46.mlp.experts.118.down_proj", + "model.layers.46.mlp.experts.118.gate_proj", + "model.layers.46.mlp.experts.118.up_proj", + "model.layers.46.mlp.experts.119.down_proj", + "model.layers.46.mlp.experts.119.gate_proj", + "model.layers.46.mlp.experts.119.up_proj", + "model.layers.46.mlp.experts.12.down_proj", + "model.layers.46.mlp.experts.12.gate_proj", + "model.layers.46.mlp.experts.12.up_proj", + "model.layers.46.mlp.experts.120.down_proj", + "model.layers.46.mlp.experts.120.gate_proj", + "model.layers.46.mlp.experts.120.up_proj", + "model.layers.46.mlp.experts.121.down_proj", + "model.layers.46.mlp.experts.121.gate_proj", + "model.layers.46.mlp.experts.121.up_proj", + "model.layers.46.mlp.experts.122.down_proj", + "model.layers.46.mlp.experts.122.gate_proj", + "model.layers.46.mlp.experts.122.up_proj", + "model.layers.46.mlp.experts.123.down_proj", + "model.layers.46.mlp.experts.123.gate_proj", + "model.layers.46.mlp.experts.123.up_proj", + "model.layers.46.mlp.experts.124.down_proj", + "model.layers.46.mlp.experts.124.gate_proj", + "model.layers.46.mlp.experts.124.up_proj", + "model.layers.46.mlp.experts.125.down_proj", + "model.layers.46.mlp.experts.125.gate_proj", + "model.layers.46.mlp.experts.125.up_proj", + "model.layers.46.mlp.experts.126.down_proj", + "model.layers.46.mlp.experts.126.gate_proj", + "model.layers.46.mlp.experts.126.up_proj", + "model.layers.46.mlp.experts.127.down_proj", + "model.layers.46.mlp.experts.127.gate_proj", + "model.layers.46.mlp.experts.127.up_proj", + "model.layers.46.mlp.experts.128.down_proj", + "model.layers.46.mlp.experts.128.gate_proj", + "model.layers.46.mlp.experts.128.up_proj", + "model.layers.46.mlp.experts.129.down_proj", + "model.layers.46.mlp.experts.129.gate_proj", + "model.layers.46.mlp.experts.129.up_proj", + "model.layers.46.mlp.experts.13.down_proj", + "model.layers.46.mlp.experts.13.gate_proj", + "model.layers.46.mlp.experts.13.up_proj", + "model.layers.46.mlp.experts.130.down_proj", + "model.layers.46.mlp.experts.130.gate_proj", + "model.layers.46.mlp.experts.130.up_proj", + "model.layers.46.mlp.experts.131.down_proj", + "model.layers.46.mlp.experts.131.gate_proj", + "model.layers.46.mlp.experts.131.up_proj", + "model.layers.46.mlp.experts.132.down_proj", + "model.layers.46.mlp.experts.132.gate_proj", + "model.layers.46.mlp.experts.132.up_proj", + "model.layers.46.mlp.experts.133.down_proj", + "model.layers.46.mlp.experts.133.gate_proj", + "model.layers.46.mlp.experts.133.up_proj", + "model.layers.46.mlp.experts.134.down_proj", + "model.layers.46.mlp.experts.134.gate_proj", + "model.layers.46.mlp.experts.134.up_proj", + "model.layers.46.mlp.experts.135.down_proj", + "model.layers.46.mlp.experts.135.gate_proj", + "model.layers.46.mlp.experts.135.up_proj", + "model.layers.46.mlp.experts.136.down_proj", + "model.layers.46.mlp.experts.136.gate_proj", + "model.layers.46.mlp.experts.136.up_proj", + "model.layers.46.mlp.experts.137.down_proj", + "model.layers.46.mlp.experts.137.gate_proj", + "model.layers.46.mlp.experts.137.up_proj", + "model.layers.46.mlp.experts.138.down_proj", + "model.layers.46.mlp.experts.138.gate_proj", + "model.layers.46.mlp.experts.138.up_proj", + "model.layers.46.mlp.experts.139.down_proj", + "model.layers.46.mlp.experts.139.gate_proj", + "model.layers.46.mlp.experts.139.up_proj", + "model.layers.46.mlp.experts.14.down_proj", + "model.layers.46.mlp.experts.14.gate_proj", + "model.layers.46.mlp.experts.14.up_proj", + "model.layers.46.mlp.experts.140.down_proj", + "model.layers.46.mlp.experts.140.gate_proj", + "model.layers.46.mlp.experts.140.up_proj", + "model.layers.46.mlp.experts.141.down_proj", + "model.layers.46.mlp.experts.141.gate_proj", + "model.layers.46.mlp.experts.141.up_proj", + "model.layers.46.mlp.experts.142.down_proj", + "model.layers.46.mlp.experts.142.gate_proj", + "model.layers.46.mlp.experts.142.up_proj", + "model.layers.46.mlp.experts.143.down_proj", + "model.layers.46.mlp.experts.143.gate_proj", + "model.layers.46.mlp.experts.143.up_proj", + "model.layers.46.mlp.experts.144.down_proj", + "model.layers.46.mlp.experts.144.gate_proj", + "model.layers.46.mlp.experts.144.up_proj", + "model.layers.46.mlp.experts.145.down_proj", + "model.layers.46.mlp.experts.145.gate_proj", + "model.layers.46.mlp.experts.145.up_proj", + "model.layers.46.mlp.experts.146.down_proj", + "model.layers.46.mlp.experts.146.gate_proj", + "model.layers.46.mlp.experts.146.up_proj", + "model.layers.46.mlp.experts.147.down_proj", + "model.layers.46.mlp.experts.147.gate_proj", + "model.layers.46.mlp.experts.147.up_proj", + "model.layers.46.mlp.experts.148.down_proj", + "model.layers.46.mlp.experts.148.gate_proj", + "model.layers.46.mlp.experts.148.up_proj", + "model.layers.46.mlp.experts.149.down_proj", + "model.layers.46.mlp.experts.149.gate_proj", + "model.layers.46.mlp.experts.149.up_proj", + "model.layers.46.mlp.experts.15.down_proj", + "model.layers.46.mlp.experts.15.gate_proj", + "model.layers.46.mlp.experts.15.up_proj", + "model.layers.46.mlp.experts.150.down_proj", + "model.layers.46.mlp.experts.150.gate_proj", + "model.layers.46.mlp.experts.150.up_proj", + "model.layers.46.mlp.experts.151.down_proj", + "model.layers.46.mlp.experts.151.gate_proj", + "model.layers.46.mlp.experts.151.up_proj", + "model.layers.46.mlp.experts.152.down_proj", + "model.layers.46.mlp.experts.152.gate_proj", + "model.layers.46.mlp.experts.152.up_proj", + "model.layers.46.mlp.experts.153.down_proj", + "model.layers.46.mlp.experts.153.gate_proj", + "model.layers.46.mlp.experts.153.up_proj", + "model.layers.46.mlp.experts.154.down_proj", + "model.layers.46.mlp.experts.154.gate_proj", + "model.layers.46.mlp.experts.154.up_proj", + "model.layers.46.mlp.experts.155.down_proj", + "model.layers.46.mlp.experts.155.gate_proj", + "model.layers.46.mlp.experts.155.up_proj", + "model.layers.46.mlp.experts.156.down_proj", + "model.layers.46.mlp.experts.156.gate_proj", + "model.layers.46.mlp.experts.156.up_proj", + "model.layers.46.mlp.experts.157.down_proj", + "model.layers.46.mlp.experts.157.gate_proj", + "model.layers.46.mlp.experts.157.up_proj", + "model.layers.46.mlp.experts.158.down_proj", + "model.layers.46.mlp.experts.158.gate_proj", + "model.layers.46.mlp.experts.158.up_proj", + "model.layers.46.mlp.experts.159.down_proj", + "model.layers.46.mlp.experts.159.gate_proj", + "model.layers.46.mlp.experts.159.up_proj", + "model.layers.46.mlp.experts.16.down_proj", + "model.layers.46.mlp.experts.16.gate_proj", + "model.layers.46.mlp.experts.16.up_proj", + "model.layers.46.mlp.experts.160.down_proj", + "model.layers.46.mlp.experts.160.gate_proj", + "model.layers.46.mlp.experts.160.up_proj", + "model.layers.46.mlp.experts.161.down_proj", + "model.layers.46.mlp.experts.161.gate_proj", + "model.layers.46.mlp.experts.161.up_proj", + "model.layers.46.mlp.experts.162.down_proj", + "model.layers.46.mlp.experts.162.gate_proj", + "model.layers.46.mlp.experts.162.up_proj", + "model.layers.46.mlp.experts.163.down_proj", + "model.layers.46.mlp.experts.163.gate_proj", + "model.layers.46.mlp.experts.163.up_proj", + "model.layers.46.mlp.experts.164.down_proj", + "model.layers.46.mlp.experts.164.gate_proj", + "model.layers.46.mlp.experts.164.up_proj", + "model.layers.46.mlp.experts.165.down_proj", + "model.layers.46.mlp.experts.165.gate_proj", + "model.layers.46.mlp.experts.165.up_proj", + "model.layers.46.mlp.experts.166.down_proj", + "model.layers.46.mlp.experts.166.gate_proj", + "model.layers.46.mlp.experts.166.up_proj", + "model.layers.46.mlp.experts.167.down_proj", + "model.layers.46.mlp.experts.167.gate_proj", + "model.layers.46.mlp.experts.167.up_proj", + "model.layers.46.mlp.experts.168.down_proj", + "model.layers.46.mlp.experts.168.gate_proj", + "model.layers.46.mlp.experts.168.up_proj", + "model.layers.46.mlp.experts.169.down_proj", + "model.layers.46.mlp.experts.169.gate_proj", + "model.layers.46.mlp.experts.169.up_proj", + "model.layers.46.mlp.experts.17.down_proj", + "model.layers.46.mlp.experts.17.gate_proj", + "model.layers.46.mlp.experts.17.up_proj", + "model.layers.46.mlp.experts.170.down_proj", + "model.layers.46.mlp.experts.170.gate_proj", + "model.layers.46.mlp.experts.170.up_proj", + "model.layers.46.mlp.experts.171.down_proj", + "model.layers.46.mlp.experts.171.gate_proj", + "model.layers.46.mlp.experts.171.up_proj", + "model.layers.46.mlp.experts.172.down_proj", + "model.layers.46.mlp.experts.172.gate_proj", + "model.layers.46.mlp.experts.172.up_proj", + "model.layers.46.mlp.experts.173.down_proj", + "model.layers.46.mlp.experts.173.gate_proj", + "model.layers.46.mlp.experts.173.up_proj", + "model.layers.46.mlp.experts.174.down_proj", + "model.layers.46.mlp.experts.174.gate_proj", + "model.layers.46.mlp.experts.174.up_proj", + "model.layers.46.mlp.experts.175.down_proj", + "model.layers.46.mlp.experts.175.gate_proj", + "model.layers.46.mlp.experts.175.up_proj", + "model.layers.46.mlp.experts.176.down_proj", + "model.layers.46.mlp.experts.176.gate_proj", + "model.layers.46.mlp.experts.176.up_proj", + "model.layers.46.mlp.experts.177.down_proj", + "model.layers.46.mlp.experts.177.gate_proj", + "model.layers.46.mlp.experts.177.up_proj", + "model.layers.46.mlp.experts.178.down_proj", + "model.layers.46.mlp.experts.178.gate_proj", + "model.layers.46.mlp.experts.178.up_proj", + "model.layers.46.mlp.experts.179.down_proj", + "model.layers.46.mlp.experts.179.gate_proj", + "model.layers.46.mlp.experts.179.up_proj", + "model.layers.46.mlp.experts.18.down_proj", + "model.layers.46.mlp.experts.18.gate_proj", + "model.layers.46.mlp.experts.18.up_proj", + "model.layers.46.mlp.experts.180.down_proj", + "model.layers.46.mlp.experts.180.gate_proj", + "model.layers.46.mlp.experts.180.up_proj", + "model.layers.46.mlp.experts.181.down_proj", + "model.layers.46.mlp.experts.181.gate_proj", + "model.layers.46.mlp.experts.181.up_proj", + "model.layers.46.mlp.experts.182.down_proj", + "model.layers.46.mlp.experts.182.gate_proj", + "model.layers.46.mlp.experts.182.up_proj", + "model.layers.46.mlp.experts.183.down_proj", + "model.layers.46.mlp.experts.183.gate_proj", + "model.layers.46.mlp.experts.183.up_proj", + "model.layers.46.mlp.experts.184.down_proj", + "model.layers.46.mlp.experts.184.gate_proj", + "model.layers.46.mlp.experts.184.up_proj", + "model.layers.46.mlp.experts.185.down_proj", + "model.layers.46.mlp.experts.185.gate_proj", + "model.layers.46.mlp.experts.185.up_proj", + "model.layers.46.mlp.experts.186.down_proj", + "model.layers.46.mlp.experts.186.gate_proj", + "model.layers.46.mlp.experts.186.up_proj", + "model.layers.46.mlp.experts.187.down_proj", + "model.layers.46.mlp.experts.187.gate_proj", + "model.layers.46.mlp.experts.187.up_proj", + "model.layers.46.mlp.experts.188.down_proj", + "model.layers.46.mlp.experts.188.gate_proj", + "model.layers.46.mlp.experts.188.up_proj", + "model.layers.46.mlp.experts.189.down_proj", + "model.layers.46.mlp.experts.189.gate_proj", + "model.layers.46.mlp.experts.189.up_proj", + "model.layers.46.mlp.experts.19.down_proj", + "model.layers.46.mlp.experts.19.gate_proj", + "model.layers.46.mlp.experts.19.up_proj", + "model.layers.46.mlp.experts.190.down_proj", + "model.layers.46.mlp.experts.190.gate_proj", + "model.layers.46.mlp.experts.190.up_proj", + "model.layers.46.mlp.experts.191.down_proj", + "model.layers.46.mlp.experts.191.gate_proj", + "model.layers.46.mlp.experts.191.up_proj", + "model.layers.46.mlp.experts.192.down_proj", + "model.layers.46.mlp.experts.192.gate_proj", + "model.layers.46.mlp.experts.192.up_proj", + "model.layers.46.mlp.experts.193.down_proj", + "model.layers.46.mlp.experts.193.gate_proj", + "model.layers.46.mlp.experts.193.up_proj", + "model.layers.46.mlp.experts.194.down_proj", + "model.layers.46.mlp.experts.194.gate_proj", + "model.layers.46.mlp.experts.194.up_proj", + "model.layers.46.mlp.experts.195.down_proj", + "model.layers.46.mlp.experts.195.gate_proj", + "model.layers.46.mlp.experts.195.up_proj", + "model.layers.46.mlp.experts.196.down_proj", + "model.layers.46.mlp.experts.196.gate_proj", + "model.layers.46.mlp.experts.196.up_proj", + "model.layers.46.mlp.experts.197.down_proj", + "model.layers.46.mlp.experts.197.gate_proj", + "model.layers.46.mlp.experts.197.up_proj", + "model.layers.46.mlp.experts.198.down_proj", + "model.layers.46.mlp.experts.198.gate_proj", + "model.layers.46.mlp.experts.198.up_proj", + "model.layers.46.mlp.experts.199.down_proj", + "model.layers.46.mlp.experts.199.gate_proj", + "model.layers.46.mlp.experts.199.up_proj", + "model.layers.46.mlp.experts.2.down_proj", + "model.layers.46.mlp.experts.2.gate_proj", + "model.layers.46.mlp.experts.2.up_proj", + "model.layers.46.mlp.experts.20.down_proj", + "model.layers.46.mlp.experts.20.gate_proj", + "model.layers.46.mlp.experts.20.up_proj", + "model.layers.46.mlp.experts.200.down_proj", + "model.layers.46.mlp.experts.200.gate_proj", + "model.layers.46.mlp.experts.200.up_proj", + "model.layers.46.mlp.experts.201.down_proj", + "model.layers.46.mlp.experts.201.gate_proj", + "model.layers.46.mlp.experts.201.up_proj", + "model.layers.46.mlp.experts.202.down_proj", + "model.layers.46.mlp.experts.202.gate_proj", + "model.layers.46.mlp.experts.202.up_proj", + "model.layers.46.mlp.experts.203.down_proj", + "model.layers.46.mlp.experts.203.gate_proj", + "model.layers.46.mlp.experts.203.up_proj", + "model.layers.46.mlp.experts.204.down_proj", + "model.layers.46.mlp.experts.204.gate_proj", + "model.layers.46.mlp.experts.204.up_proj", + "model.layers.46.mlp.experts.205.down_proj", + "model.layers.46.mlp.experts.205.gate_proj", + "model.layers.46.mlp.experts.205.up_proj", + "model.layers.46.mlp.experts.206.down_proj", + "model.layers.46.mlp.experts.206.gate_proj", + "model.layers.46.mlp.experts.206.up_proj", + "model.layers.46.mlp.experts.207.down_proj", + "model.layers.46.mlp.experts.207.gate_proj", + "model.layers.46.mlp.experts.207.up_proj", + "model.layers.46.mlp.experts.208.down_proj", + "model.layers.46.mlp.experts.208.gate_proj", + "model.layers.46.mlp.experts.208.up_proj", + "model.layers.46.mlp.experts.209.down_proj", + "model.layers.46.mlp.experts.209.gate_proj", + "model.layers.46.mlp.experts.209.up_proj", + "model.layers.46.mlp.experts.21.down_proj", + "model.layers.46.mlp.experts.21.gate_proj", + "model.layers.46.mlp.experts.21.up_proj", + "model.layers.46.mlp.experts.210.down_proj", + "model.layers.46.mlp.experts.210.gate_proj", + "model.layers.46.mlp.experts.210.up_proj", + "model.layers.46.mlp.experts.211.down_proj", + "model.layers.46.mlp.experts.211.gate_proj", + "model.layers.46.mlp.experts.211.up_proj", + "model.layers.46.mlp.experts.212.down_proj", + "model.layers.46.mlp.experts.212.gate_proj", + "model.layers.46.mlp.experts.212.up_proj", + "model.layers.46.mlp.experts.213.down_proj", + "model.layers.46.mlp.experts.213.gate_proj", + "model.layers.46.mlp.experts.213.up_proj", + "model.layers.46.mlp.experts.214.down_proj", + "model.layers.46.mlp.experts.214.gate_proj", + "model.layers.46.mlp.experts.214.up_proj", + "model.layers.46.mlp.experts.215.down_proj", + "model.layers.46.mlp.experts.215.gate_proj", + "model.layers.46.mlp.experts.215.up_proj", + "model.layers.46.mlp.experts.216.down_proj", + "model.layers.46.mlp.experts.216.gate_proj", + "model.layers.46.mlp.experts.216.up_proj", + "model.layers.46.mlp.experts.217.down_proj", + "model.layers.46.mlp.experts.217.gate_proj", + "model.layers.46.mlp.experts.217.up_proj", + "model.layers.46.mlp.experts.218.down_proj", + "model.layers.46.mlp.experts.218.gate_proj", + "model.layers.46.mlp.experts.218.up_proj", + "model.layers.46.mlp.experts.219.down_proj", + "model.layers.46.mlp.experts.219.gate_proj", + "model.layers.46.mlp.experts.219.up_proj", + "model.layers.46.mlp.experts.22.down_proj", + "model.layers.46.mlp.experts.22.gate_proj", + "model.layers.46.mlp.experts.22.up_proj", + "model.layers.46.mlp.experts.220.down_proj", + "model.layers.46.mlp.experts.220.gate_proj", + "model.layers.46.mlp.experts.220.up_proj", + "model.layers.46.mlp.experts.221.down_proj", + "model.layers.46.mlp.experts.221.gate_proj", + "model.layers.46.mlp.experts.221.up_proj", + "model.layers.46.mlp.experts.222.down_proj", + "model.layers.46.mlp.experts.222.gate_proj", + "model.layers.46.mlp.experts.222.up_proj", + "model.layers.46.mlp.experts.223.down_proj", + "model.layers.46.mlp.experts.223.gate_proj", + "model.layers.46.mlp.experts.223.up_proj", + "model.layers.46.mlp.experts.224.down_proj", + "model.layers.46.mlp.experts.224.gate_proj", + "model.layers.46.mlp.experts.224.up_proj", + "model.layers.46.mlp.experts.225.down_proj", + "model.layers.46.mlp.experts.225.gate_proj", + "model.layers.46.mlp.experts.225.up_proj", + "model.layers.46.mlp.experts.226.down_proj", + "model.layers.46.mlp.experts.226.gate_proj", + "model.layers.46.mlp.experts.226.up_proj", + "model.layers.46.mlp.experts.227.down_proj", + "model.layers.46.mlp.experts.227.gate_proj", + "model.layers.46.mlp.experts.227.up_proj", + "model.layers.46.mlp.experts.228.down_proj", + "model.layers.46.mlp.experts.228.gate_proj", + "model.layers.46.mlp.experts.228.up_proj", + "model.layers.46.mlp.experts.229.down_proj", + "model.layers.46.mlp.experts.229.gate_proj", + "model.layers.46.mlp.experts.229.up_proj", + "model.layers.46.mlp.experts.23.down_proj", + "model.layers.46.mlp.experts.23.gate_proj", + "model.layers.46.mlp.experts.23.up_proj", + "model.layers.46.mlp.experts.230.down_proj", + "model.layers.46.mlp.experts.230.gate_proj", + "model.layers.46.mlp.experts.230.up_proj", + "model.layers.46.mlp.experts.231.down_proj", + "model.layers.46.mlp.experts.231.gate_proj", + "model.layers.46.mlp.experts.231.up_proj", + "model.layers.46.mlp.experts.232.down_proj", + "model.layers.46.mlp.experts.232.gate_proj", + "model.layers.46.mlp.experts.232.up_proj", + "model.layers.46.mlp.experts.233.down_proj", + "model.layers.46.mlp.experts.233.gate_proj", + "model.layers.46.mlp.experts.233.up_proj", + "model.layers.46.mlp.experts.234.down_proj", + "model.layers.46.mlp.experts.234.gate_proj", + "model.layers.46.mlp.experts.234.up_proj", + "model.layers.46.mlp.experts.235.down_proj", + "model.layers.46.mlp.experts.235.gate_proj", + "model.layers.46.mlp.experts.235.up_proj", + "model.layers.46.mlp.experts.236.down_proj", + "model.layers.46.mlp.experts.236.gate_proj", + "model.layers.46.mlp.experts.236.up_proj", + "model.layers.46.mlp.experts.237.down_proj", + "model.layers.46.mlp.experts.237.gate_proj", + "model.layers.46.mlp.experts.237.up_proj", + "model.layers.46.mlp.experts.238.down_proj", + "model.layers.46.mlp.experts.238.gate_proj", + "model.layers.46.mlp.experts.238.up_proj", + "model.layers.46.mlp.experts.239.down_proj", + "model.layers.46.mlp.experts.239.gate_proj", + "model.layers.46.mlp.experts.239.up_proj", + "model.layers.46.mlp.experts.24.down_proj", + "model.layers.46.mlp.experts.24.gate_proj", + "model.layers.46.mlp.experts.24.up_proj", + "model.layers.46.mlp.experts.240.down_proj", + "model.layers.46.mlp.experts.240.gate_proj", + "model.layers.46.mlp.experts.240.up_proj", + "model.layers.46.mlp.experts.241.down_proj", + "model.layers.46.mlp.experts.241.gate_proj", + "model.layers.46.mlp.experts.241.up_proj", + "model.layers.46.mlp.experts.242.down_proj", + "model.layers.46.mlp.experts.242.gate_proj", + "model.layers.46.mlp.experts.242.up_proj", + "model.layers.46.mlp.experts.243.down_proj", + "model.layers.46.mlp.experts.243.gate_proj", + "model.layers.46.mlp.experts.243.up_proj", + "model.layers.46.mlp.experts.244.down_proj", + "model.layers.46.mlp.experts.244.gate_proj", + "model.layers.46.mlp.experts.244.up_proj", + "model.layers.46.mlp.experts.245.down_proj", + "model.layers.46.mlp.experts.245.gate_proj", + "model.layers.46.mlp.experts.245.up_proj", + "model.layers.46.mlp.experts.246.down_proj", + "model.layers.46.mlp.experts.246.gate_proj", + "model.layers.46.mlp.experts.246.up_proj", + "model.layers.46.mlp.experts.247.down_proj", + "model.layers.46.mlp.experts.247.gate_proj", + "model.layers.46.mlp.experts.247.up_proj", + "model.layers.46.mlp.experts.248.down_proj", + "model.layers.46.mlp.experts.248.gate_proj", + "model.layers.46.mlp.experts.248.up_proj", + "model.layers.46.mlp.experts.249.down_proj", + "model.layers.46.mlp.experts.249.gate_proj", + "model.layers.46.mlp.experts.249.up_proj", + "model.layers.46.mlp.experts.25.down_proj", + "model.layers.46.mlp.experts.25.gate_proj", + "model.layers.46.mlp.experts.25.up_proj", + "model.layers.46.mlp.experts.250.down_proj", + "model.layers.46.mlp.experts.250.gate_proj", + "model.layers.46.mlp.experts.250.up_proj", + "model.layers.46.mlp.experts.251.down_proj", + "model.layers.46.mlp.experts.251.gate_proj", + "model.layers.46.mlp.experts.251.up_proj", + "model.layers.46.mlp.experts.252.down_proj", + "model.layers.46.mlp.experts.252.gate_proj", + "model.layers.46.mlp.experts.252.up_proj", + "model.layers.46.mlp.experts.253.down_proj", + "model.layers.46.mlp.experts.253.gate_proj", + "model.layers.46.mlp.experts.253.up_proj", + "model.layers.46.mlp.experts.254.down_proj", + "model.layers.46.mlp.experts.254.gate_proj", + "model.layers.46.mlp.experts.254.up_proj", + "model.layers.46.mlp.experts.255.down_proj", + "model.layers.46.mlp.experts.255.gate_proj", + "model.layers.46.mlp.experts.255.up_proj", + "model.layers.46.mlp.experts.26.down_proj", + "model.layers.46.mlp.experts.26.gate_proj", + "model.layers.46.mlp.experts.26.up_proj", + "model.layers.46.mlp.experts.27.down_proj", + "model.layers.46.mlp.experts.27.gate_proj", + "model.layers.46.mlp.experts.27.up_proj", + "model.layers.46.mlp.experts.28.down_proj", + "model.layers.46.mlp.experts.28.gate_proj", + "model.layers.46.mlp.experts.28.up_proj", + "model.layers.46.mlp.experts.29.down_proj", + "model.layers.46.mlp.experts.29.gate_proj", + "model.layers.46.mlp.experts.29.up_proj", + "model.layers.46.mlp.experts.3.down_proj", + "model.layers.46.mlp.experts.3.gate_proj", + "model.layers.46.mlp.experts.3.up_proj", + "model.layers.46.mlp.experts.30.down_proj", + "model.layers.46.mlp.experts.30.gate_proj", + "model.layers.46.mlp.experts.30.up_proj", + "model.layers.46.mlp.experts.31.down_proj", + "model.layers.46.mlp.experts.31.gate_proj", + "model.layers.46.mlp.experts.31.up_proj", + "model.layers.46.mlp.experts.32.down_proj", + "model.layers.46.mlp.experts.32.gate_proj", + "model.layers.46.mlp.experts.32.up_proj", + "model.layers.46.mlp.experts.33.down_proj", + "model.layers.46.mlp.experts.33.gate_proj", + "model.layers.46.mlp.experts.33.up_proj", + "model.layers.46.mlp.experts.34.down_proj", + "model.layers.46.mlp.experts.34.gate_proj", + "model.layers.46.mlp.experts.34.up_proj", + "model.layers.46.mlp.experts.35.down_proj", + "model.layers.46.mlp.experts.35.gate_proj", + "model.layers.46.mlp.experts.35.up_proj", + "model.layers.46.mlp.experts.36.down_proj", + "model.layers.46.mlp.experts.36.gate_proj", + "model.layers.46.mlp.experts.36.up_proj", + "model.layers.46.mlp.experts.37.down_proj", + "model.layers.46.mlp.experts.37.gate_proj", + "model.layers.46.mlp.experts.37.up_proj", + "model.layers.46.mlp.experts.38.down_proj", + "model.layers.46.mlp.experts.38.gate_proj", + "model.layers.46.mlp.experts.38.up_proj", + "model.layers.46.mlp.experts.39.down_proj", + "model.layers.46.mlp.experts.39.gate_proj", + "model.layers.46.mlp.experts.39.up_proj", + "model.layers.46.mlp.experts.4.down_proj", + "model.layers.46.mlp.experts.4.gate_proj", + "model.layers.46.mlp.experts.4.up_proj", + "model.layers.46.mlp.experts.40.down_proj", + "model.layers.46.mlp.experts.40.gate_proj", + "model.layers.46.mlp.experts.40.up_proj", + "model.layers.46.mlp.experts.41.down_proj", + "model.layers.46.mlp.experts.41.gate_proj", + "model.layers.46.mlp.experts.41.up_proj", + "model.layers.46.mlp.experts.42.down_proj", + "model.layers.46.mlp.experts.42.gate_proj", + "model.layers.46.mlp.experts.42.up_proj", + "model.layers.46.mlp.experts.43.down_proj", + "model.layers.46.mlp.experts.43.gate_proj", + "model.layers.46.mlp.experts.43.up_proj", + "model.layers.46.mlp.experts.44.down_proj", + "model.layers.46.mlp.experts.44.gate_proj", + "model.layers.46.mlp.experts.44.up_proj", + "model.layers.46.mlp.experts.45.down_proj", + "model.layers.46.mlp.experts.45.gate_proj", + "model.layers.46.mlp.experts.45.up_proj", + "model.layers.46.mlp.experts.46.down_proj", + "model.layers.46.mlp.experts.46.gate_proj", + "model.layers.46.mlp.experts.46.up_proj", + "model.layers.46.mlp.experts.47.down_proj", + "model.layers.46.mlp.experts.47.gate_proj", + "model.layers.46.mlp.experts.47.up_proj", + "model.layers.46.mlp.experts.48.down_proj", + "model.layers.46.mlp.experts.48.gate_proj", + "model.layers.46.mlp.experts.48.up_proj", + "model.layers.46.mlp.experts.49.down_proj", + "model.layers.46.mlp.experts.49.gate_proj", + "model.layers.46.mlp.experts.49.up_proj", + "model.layers.46.mlp.experts.5.down_proj", + "model.layers.46.mlp.experts.5.gate_proj", + "model.layers.46.mlp.experts.5.up_proj", + "model.layers.46.mlp.experts.50.down_proj", + "model.layers.46.mlp.experts.50.gate_proj", + "model.layers.46.mlp.experts.50.up_proj", + "model.layers.46.mlp.experts.51.down_proj", + "model.layers.46.mlp.experts.51.gate_proj", + "model.layers.46.mlp.experts.51.up_proj", + "model.layers.46.mlp.experts.52.down_proj", + "model.layers.46.mlp.experts.52.gate_proj", + "model.layers.46.mlp.experts.52.up_proj", + "model.layers.46.mlp.experts.53.down_proj", + "model.layers.46.mlp.experts.53.gate_proj", + "model.layers.46.mlp.experts.53.up_proj", + "model.layers.46.mlp.experts.54.down_proj", + "model.layers.46.mlp.experts.54.gate_proj", + "model.layers.46.mlp.experts.54.up_proj", + "model.layers.46.mlp.experts.55.down_proj", + "model.layers.46.mlp.experts.55.gate_proj", + "model.layers.46.mlp.experts.55.up_proj", + "model.layers.46.mlp.experts.56.down_proj", + "model.layers.46.mlp.experts.56.gate_proj", + "model.layers.46.mlp.experts.56.up_proj", + "model.layers.46.mlp.experts.57.down_proj", + "model.layers.46.mlp.experts.57.gate_proj", + "model.layers.46.mlp.experts.57.up_proj", + "model.layers.46.mlp.experts.58.down_proj", + "model.layers.46.mlp.experts.58.gate_proj", + "model.layers.46.mlp.experts.58.up_proj", + "model.layers.46.mlp.experts.59.down_proj", + "model.layers.46.mlp.experts.59.gate_proj", + "model.layers.46.mlp.experts.59.up_proj", + "model.layers.46.mlp.experts.6.down_proj", + "model.layers.46.mlp.experts.6.gate_proj", + "model.layers.46.mlp.experts.6.up_proj", + "model.layers.46.mlp.experts.60.down_proj", + "model.layers.46.mlp.experts.60.gate_proj", + "model.layers.46.mlp.experts.60.up_proj", + "model.layers.46.mlp.experts.61.down_proj", + "model.layers.46.mlp.experts.61.gate_proj", + "model.layers.46.mlp.experts.61.up_proj", + "model.layers.46.mlp.experts.62.down_proj", + "model.layers.46.mlp.experts.62.gate_proj", + "model.layers.46.mlp.experts.62.up_proj", + "model.layers.46.mlp.experts.63.down_proj", + "model.layers.46.mlp.experts.63.gate_proj", + "model.layers.46.mlp.experts.63.up_proj", + "model.layers.46.mlp.experts.64.down_proj", + "model.layers.46.mlp.experts.64.gate_proj", + "model.layers.46.mlp.experts.64.up_proj", + "model.layers.46.mlp.experts.65.down_proj", + "model.layers.46.mlp.experts.65.gate_proj", + "model.layers.46.mlp.experts.65.up_proj", + "model.layers.46.mlp.experts.66.down_proj", + "model.layers.46.mlp.experts.66.gate_proj", + "model.layers.46.mlp.experts.66.up_proj", + "model.layers.46.mlp.experts.67.down_proj", + "model.layers.46.mlp.experts.67.gate_proj", + "model.layers.46.mlp.experts.67.up_proj", + "model.layers.46.mlp.experts.68.down_proj", + "model.layers.46.mlp.experts.68.gate_proj", + "model.layers.46.mlp.experts.68.up_proj", + "model.layers.46.mlp.experts.69.down_proj", + "model.layers.46.mlp.experts.69.gate_proj", + "model.layers.46.mlp.experts.69.up_proj", + "model.layers.46.mlp.experts.7.down_proj", + "model.layers.46.mlp.experts.7.gate_proj", + "model.layers.46.mlp.experts.7.up_proj", + "model.layers.46.mlp.experts.70.down_proj", + "model.layers.46.mlp.experts.70.gate_proj", + "model.layers.46.mlp.experts.70.up_proj", + "model.layers.46.mlp.experts.71.down_proj", + "model.layers.46.mlp.experts.71.gate_proj", + "model.layers.46.mlp.experts.71.up_proj", + "model.layers.46.mlp.experts.72.down_proj", + "model.layers.46.mlp.experts.72.gate_proj", + "model.layers.46.mlp.experts.72.up_proj", + "model.layers.46.mlp.experts.73.down_proj", + "model.layers.46.mlp.experts.73.gate_proj", + "model.layers.46.mlp.experts.73.up_proj", + "model.layers.46.mlp.experts.74.down_proj", + "model.layers.46.mlp.experts.74.gate_proj", + "model.layers.46.mlp.experts.74.up_proj", + "model.layers.46.mlp.experts.75.down_proj", + "model.layers.46.mlp.experts.75.gate_proj", + "model.layers.46.mlp.experts.75.up_proj", + "model.layers.46.mlp.experts.76.down_proj", + "model.layers.46.mlp.experts.76.gate_proj", + "model.layers.46.mlp.experts.76.up_proj", + "model.layers.46.mlp.experts.77.down_proj", + "model.layers.46.mlp.experts.77.gate_proj", + "model.layers.46.mlp.experts.77.up_proj", + "model.layers.46.mlp.experts.78.down_proj", + "model.layers.46.mlp.experts.78.gate_proj", + "model.layers.46.mlp.experts.78.up_proj", + "model.layers.46.mlp.experts.79.down_proj", + "model.layers.46.mlp.experts.79.gate_proj", + "model.layers.46.mlp.experts.79.up_proj", + "model.layers.46.mlp.experts.8.down_proj", + "model.layers.46.mlp.experts.8.gate_proj", + "model.layers.46.mlp.experts.8.up_proj", + "model.layers.46.mlp.experts.80.down_proj", + "model.layers.46.mlp.experts.80.gate_proj", + "model.layers.46.mlp.experts.80.up_proj", + "model.layers.46.mlp.experts.81.down_proj", + "model.layers.46.mlp.experts.81.gate_proj", + "model.layers.46.mlp.experts.81.up_proj", + "model.layers.46.mlp.experts.82.down_proj", + "model.layers.46.mlp.experts.82.gate_proj", + "model.layers.46.mlp.experts.82.up_proj", + "model.layers.46.mlp.experts.83.down_proj", + "model.layers.46.mlp.experts.83.gate_proj", + "model.layers.46.mlp.experts.83.up_proj", + "model.layers.46.mlp.experts.84.down_proj", + "model.layers.46.mlp.experts.84.gate_proj", + "model.layers.46.mlp.experts.84.up_proj", + "model.layers.46.mlp.experts.85.down_proj", + "model.layers.46.mlp.experts.85.gate_proj", + "model.layers.46.mlp.experts.85.up_proj", + "model.layers.46.mlp.experts.86.down_proj", + "model.layers.46.mlp.experts.86.gate_proj", + "model.layers.46.mlp.experts.86.up_proj", + "model.layers.46.mlp.experts.87.down_proj", + "model.layers.46.mlp.experts.87.gate_proj", + "model.layers.46.mlp.experts.87.up_proj", + "model.layers.46.mlp.experts.88.down_proj", + "model.layers.46.mlp.experts.88.gate_proj", + "model.layers.46.mlp.experts.88.up_proj", + "model.layers.46.mlp.experts.89.down_proj", + "model.layers.46.mlp.experts.89.gate_proj", + "model.layers.46.mlp.experts.89.up_proj", + "model.layers.46.mlp.experts.9.down_proj", + "model.layers.46.mlp.experts.9.gate_proj", + "model.layers.46.mlp.experts.9.up_proj", + "model.layers.46.mlp.experts.90.down_proj", + "model.layers.46.mlp.experts.90.gate_proj", + "model.layers.46.mlp.experts.90.up_proj", + "model.layers.46.mlp.experts.91.down_proj", + "model.layers.46.mlp.experts.91.gate_proj", + "model.layers.46.mlp.experts.91.up_proj", + "model.layers.46.mlp.experts.92.down_proj", + "model.layers.46.mlp.experts.92.gate_proj", + "model.layers.46.mlp.experts.92.up_proj", + "model.layers.46.mlp.experts.93.down_proj", + "model.layers.46.mlp.experts.93.gate_proj", + "model.layers.46.mlp.experts.93.up_proj", + "model.layers.46.mlp.experts.94.down_proj", + "model.layers.46.mlp.experts.94.gate_proj", + "model.layers.46.mlp.experts.94.up_proj", + "model.layers.46.mlp.experts.95.down_proj", + "model.layers.46.mlp.experts.95.gate_proj", + "model.layers.46.mlp.experts.95.up_proj", + "model.layers.46.mlp.experts.96.down_proj", + "model.layers.46.mlp.experts.96.gate_proj", + "model.layers.46.mlp.experts.96.up_proj", + "model.layers.46.mlp.experts.97.down_proj", + "model.layers.46.mlp.experts.97.gate_proj", + "model.layers.46.mlp.experts.97.up_proj", + "model.layers.46.mlp.experts.98.down_proj", + "model.layers.46.mlp.experts.98.gate_proj", + "model.layers.46.mlp.experts.98.up_proj", + "model.layers.46.mlp.experts.99.down_proj", + "model.layers.46.mlp.experts.99.gate_proj", + "model.layers.46.mlp.experts.99.up_proj", + "model.layers.47.mlp.experts.0.down_proj", + "model.layers.47.mlp.experts.0.gate_proj", + "model.layers.47.mlp.experts.0.up_proj", + "model.layers.47.mlp.experts.1.down_proj", + "model.layers.47.mlp.experts.1.gate_proj", + "model.layers.47.mlp.experts.1.up_proj", + "model.layers.47.mlp.experts.10.down_proj", + "model.layers.47.mlp.experts.10.gate_proj", + "model.layers.47.mlp.experts.10.up_proj", + "model.layers.47.mlp.experts.100.down_proj", + "model.layers.47.mlp.experts.100.gate_proj", + "model.layers.47.mlp.experts.100.up_proj", + "model.layers.47.mlp.experts.101.down_proj", + "model.layers.47.mlp.experts.101.gate_proj", + "model.layers.47.mlp.experts.101.up_proj", + "model.layers.47.mlp.experts.102.down_proj", + "model.layers.47.mlp.experts.102.gate_proj", + "model.layers.47.mlp.experts.102.up_proj", + "model.layers.47.mlp.experts.103.down_proj", + "model.layers.47.mlp.experts.103.gate_proj", + "model.layers.47.mlp.experts.103.up_proj", + "model.layers.47.mlp.experts.104.down_proj", + "model.layers.47.mlp.experts.104.gate_proj", + "model.layers.47.mlp.experts.104.up_proj", + "model.layers.47.mlp.experts.105.down_proj", + "model.layers.47.mlp.experts.105.gate_proj", + "model.layers.47.mlp.experts.105.up_proj", + "model.layers.47.mlp.experts.106.down_proj", + "model.layers.47.mlp.experts.106.gate_proj", + "model.layers.47.mlp.experts.106.up_proj", + "model.layers.47.mlp.experts.107.down_proj", + "model.layers.47.mlp.experts.107.gate_proj", + "model.layers.47.mlp.experts.107.up_proj", + "model.layers.47.mlp.experts.108.down_proj", + "model.layers.47.mlp.experts.108.gate_proj", + "model.layers.47.mlp.experts.108.up_proj", + "model.layers.47.mlp.experts.109.down_proj", + "model.layers.47.mlp.experts.109.gate_proj", + "model.layers.47.mlp.experts.109.up_proj", + "model.layers.47.mlp.experts.11.down_proj", + "model.layers.47.mlp.experts.11.gate_proj", + "model.layers.47.mlp.experts.11.up_proj", + "model.layers.47.mlp.experts.110.down_proj", + "model.layers.47.mlp.experts.110.gate_proj", + "model.layers.47.mlp.experts.110.up_proj", + "model.layers.47.mlp.experts.111.down_proj", + "model.layers.47.mlp.experts.111.gate_proj", + "model.layers.47.mlp.experts.111.up_proj", + "model.layers.47.mlp.experts.112.down_proj", + "model.layers.47.mlp.experts.112.gate_proj", + "model.layers.47.mlp.experts.112.up_proj", + "model.layers.47.mlp.experts.113.down_proj", + "model.layers.47.mlp.experts.113.gate_proj", + "model.layers.47.mlp.experts.113.up_proj", + "model.layers.47.mlp.experts.114.down_proj", + "model.layers.47.mlp.experts.114.gate_proj", + "model.layers.47.mlp.experts.114.up_proj", + "model.layers.47.mlp.experts.115.down_proj", + "model.layers.47.mlp.experts.115.gate_proj", + "model.layers.47.mlp.experts.115.up_proj", + "model.layers.47.mlp.experts.116.down_proj", + "model.layers.47.mlp.experts.116.gate_proj", + "model.layers.47.mlp.experts.116.up_proj", + "model.layers.47.mlp.experts.117.down_proj", + "model.layers.47.mlp.experts.117.gate_proj", + "model.layers.47.mlp.experts.117.up_proj", + "model.layers.47.mlp.experts.118.down_proj", + "model.layers.47.mlp.experts.118.gate_proj", + "model.layers.47.mlp.experts.118.up_proj", + "model.layers.47.mlp.experts.119.down_proj", + "model.layers.47.mlp.experts.119.gate_proj", + "model.layers.47.mlp.experts.119.up_proj", + "model.layers.47.mlp.experts.12.down_proj", + "model.layers.47.mlp.experts.12.gate_proj", + "model.layers.47.mlp.experts.12.up_proj", + "model.layers.47.mlp.experts.120.down_proj", + "model.layers.47.mlp.experts.120.gate_proj", + "model.layers.47.mlp.experts.120.up_proj", + "model.layers.47.mlp.experts.121.down_proj", + "model.layers.47.mlp.experts.121.gate_proj", + "model.layers.47.mlp.experts.121.up_proj", + "model.layers.47.mlp.experts.122.down_proj", + "model.layers.47.mlp.experts.122.gate_proj", + "model.layers.47.mlp.experts.122.up_proj", + "model.layers.47.mlp.experts.123.down_proj", + "model.layers.47.mlp.experts.123.gate_proj", + "model.layers.47.mlp.experts.123.up_proj", + "model.layers.47.mlp.experts.124.down_proj", + "model.layers.47.mlp.experts.124.gate_proj", + "model.layers.47.mlp.experts.124.up_proj", + "model.layers.47.mlp.experts.125.down_proj", + "model.layers.47.mlp.experts.125.gate_proj", + "model.layers.47.mlp.experts.125.up_proj", + "model.layers.47.mlp.experts.126.down_proj", + "model.layers.47.mlp.experts.126.gate_proj", + "model.layers.47.mlp.experts.126.up_proj", + "model.layers.47.mlp.experts.127.down_proj", + "model.layers.47.mlp.experts.127.gate_proj", + "model.layers.47.mlp.experts.127.up_proj", + "model.layers.47.mlp.experts.128.down_proj", + "model.layers.47.mlp.experts.128.gate_proj", + "model.layers.47.mlp.experts.128.up_proj", + "model.layers.47.mlp.experts.129.down_proj", + "model.layers.47.mlp.experts.129.gate_proj", + "model.layers.47.mlp.experts.129.up_proj", + "model.layers.47.mlp.experts.13.down_proj", + "model.layers.47.mlp.experts.13.gate_proj", + "model.layers.47.mlp.experts.13.up_proj", + "model.layers.47.mlp.experts.130.down_proj", + "model.layers.47.mlp.experts.130.gate_proj", + "model.layers.47.mlp.experts.130.up_proj", + "model.layers.47.mlp.experts.131.down_proj", + "model.layers.47.mlp.experts.131.gate_proj", + "model.layers.47.mlp.experts.131.up_proj", + "model.layers.47.mlp.experts.132.down_proj", + "model.layers.47.mlp.experts.132.gate_proj", + "model.layers.47.mlp.experts.132.up_proj", + "model.layers.47.mlp.experts.133.down_proj", + "model.layers.47.mlp.experts.133.gate_proj", + "model.layers.47.mlp.experts.133.up_proj", + "model.layers.47.mlp.experts.134.down_proj", + "model.layers.47.mlp.experts.134.gate_proj", + "model.layers.47.mlp.experts.134.up_proj", + "model.layers.47.mlp.experts.135.down_proj", + "model.layers.47.mlp.experts.135.gate_proj", + "model.layers.47.mlp.experts.135.up_proj", + "model.layers.47.mlp.experts.136.down_proj", + "model.layers.47.mlp.experts.136.gate_proj", + "model.layers.47.mlp.experts.136.up_proj", + "model.layers.47.mlp.experts.137.down_proj", + "model.layers.47.mlp.experts.137.gate_proj", + "model.layers.47.mlp.experts.137.up_proj", + "model.layers.47.mlp.experts.138.down_proj", + "model.layers.47.mlp.experts.138.gate_proj", + "model.layers.47.mlp.experts.138.up_proj", + "model.layers.47.mlp.experts.139.down_proj", + "model.layers.47.mlp.experts.139.gate_proj", + "model.layers.47.mlp.experts.139.up_proj", + "model.layers.47.mlp.experts.14.down_proj", + "model.layers.47.mlp.experts.14.gate_proj", + "model.layers.47.mlp.experts.14.up_proj", + "model.layers.47.mlp.experts.140.down_proj", + "model.layers.47.mlp.experts.140.gate_proj", + "model.layers.47.mlp.experts.140.up_proj", + "model.layers.47.mlp.experts.141.down_proj", + "model.layers.47.mlp.experts.141.gate_proj", + "model.layers.47.mlp.experts.141.up_proj", + "model.layers.47.mlp.experts.142.down_proj", + "model.layers.47.mlp.experts.142.gate_proj", + "model.layers.47.mlp.experts.142.up_proj", + "model.layers.47.mlp.experts.143.down_proj", + "model.layers.47.mlp.experts.143.gate_proj", + "model.layers.47.mlp.experts.143.up_proj", + "model.layers.47.mlp.experts.144.down_proj", + "model.layers.47.mlp.experts.144.gate_proj", + "model.layers.47.mlp.experts.144.up_proj", + "model.layers.47.mlp.experts.145.down_proj", + "model.layers.47.mlp.experts.145.gate_proj", + "model.layers.47.mlp.experts.145.up_proj", + "model.layers.47.mlp.experts.146.down_proj", + "model.layers.47.mlp.experts.146.gate_proj", + "model.layers.47.mlp.experts.146.up_proj", + "model.layers.47.mlp.experts.147.down_proj", + "model.layers.47.mlp.experts.147.gate_proj", + "model.layers.47.mlp.experts.147.up_proj", + "model.layers.47.mlp.experts.148.down_proj", + "model.layers.47.mlp.experts.148.gate_proj", + "model.layers.47.mlp.experts.148.up_proj", + "model.layers.47.mlp.experts.149.down_proj", + "model.layers.47.mlp.experts.149.gate_proj", + "model.layers.47.mlp.experts.149.up_proj", + "model.layers.47.mlp.experts.15.down_proj", + "model.layers.47.mlp.experts.15.gate_proj", + "model.layers.47.mlp.experts.15.up_proj", + "model.layers.47.mlp.experts.150.down_proj", + "model.layers.47.mlp.experts.150.gate_proj", + "model.layers.47.mlp.experts.150.up_proj", + "model.layers.47.mlp.experts.151.down_proj", + "model.layers.47.mlp.experts.151.gate_proj", + "model.layers.47.mlp.experts.151.up_proj", + "model.layers.47.mlp.experts.152.down_proj", + "model.layers.47.mlp.experts.152.gate_proj", + "model.layers.47.mlp.experts.152.up_proj", + "model.layers.47.mlp.experts.153.down_proj", + "model.layers.47.mlp.experts.153.gate_proj", + "model.layers.47.mlp.experts.153.up_proj", + "model.layers.47.mlp.experts.154.down_proj", + "model.layers.47.mlp.experts.154.gate_proj", + "model.layers.47.mlp.experts.154.up_proj", + "model.layers.47.mlp.experts.155.down_proj", + "model.layers.47.mlp.experts.155.gate_proj", + "model.layers.47.mlp.experts.155.up_proj", + "model.layers.47.mlp.experts.156.down_proj", + "model.layers.47.mlp.experts.156.gate_proj", + "model.layers.47.mlp.experts.156.up_proj", + "model.layers.47.mlp.experts.157.down_proj", + "model.layers.47.mlp.experts.157.gate_proj", + "model.layers.47.mlp.experts.157.up_proj", + "model.layers.47.mlp.experts.158.down_proj", + "model.layers.47.mlp.experts.158.gate_proj", + "model.layers.47.mlp.experts.158.up_proj", + "model.layers.47.mlp.experts.159.down_proj", + "model.layers.47.mlp.experts.159.gate_proj", + "model.layers.47.mlp.experts.159.up_proj", + "model.layers.47.mlp.experts.16.down_proj", + "model.layers.47.mlp.experts.16.gate_proj", + "model.layers.47.mlp.experts.16.up_proj", + "model.layers.47.mlp.experts.160.down_proj", + "model.layers.47.mlp.experts.160.gate_proj", + "model.layers.47.mlp.experts.160.up_proj", + "model.layers.47.mlp.experts.161.down_proj", + "model.layers.47.mlp.experts.161.gate_proj", + "model.layers.47.mlp.experts.161.up_proj", + "model.layers.47.mlp.experts.162.down_proj", + "model.layers.47.mlp.experts.162.gate_proj", + "model.layers.47.mlp.experts.162.up_proj", + "model.layers.47.mlp.experts.163.down_proj", + "model.layers.47.mlp.experts.163.gate_proj", + "model.layers.47.mlp.experts.163.up_proj", + "model.layers.47.mlp.experts.164.down_proj", + "model.layers.47.mlp.experts.164.gate_proj", + "model.layers.47.mlp.experts.164.up_proj", + "model.layers.47.mlp.experts.165.down_proj", + "model.layers.47.mlp.experts.165.gate_proj", + "model.layers.47.mlp.experts.165.up_proj", + "model.layers.47.mlp.experts.166.down_proj", + "model.layers.47.mlp.experts.166.gate_proj", + "model.layers.47.mlp.experts.166.up_proj", + "model.layers.47.mlp.experts.167.down_proj", + "model.layers.47.mlp.experts.167.gate_proj", + "model.layers.47.mlp.experts.167.up_proj", + "model.layers.47.mlp.experts.168.down_proj", + "model.layers.47.mlp.experts.168.gate_proj", + "model.layers.47.mlp.experts.168.up_proj", + "model.layers.47.mlp.experts.169.down_proj", + "model.layers.47.mlp.experts.169.gate_proj", + "model.layers.47.mlp.experts.169.up_proj", + "model.layers.47.mlp.experts.17.down_proj", + "model.layers.47.mlp.experts.17.gate_proj", + "model.layers.47.mlp.experts.17.up_proj", + "model.layers.47.mlp.experts.170.down_proj", + "model.layers.47.mlp.experts.170.gate_proj", + "model.layers.47.mlp.experts.170.up_proj", + "model.layers.47.mlp.experts.171.down_proj", + "model.layers.47.mlp.experts.171.gate_proj", + "model.layers.47.mlp.experts.171.up_proj", + "model.layers.47.mlp.experts.172.down_proj", + "model.layers.47.mlp.experts.172.gate_proj", + "model.layers.47.mlp.experts.172.up_proj", + "model.layers.47.mlp.experts.173.down_proj", + "model.layers.47.mlp.experts.173.gate_proj", + "model.layers.47.mlp.experts.173.up_proj", + "model.layers.47.mlp.experts.174.down_proj", + "model.layers.47.mlp.experts.174.gate_proj", + "model.layers.47.mlp.experts.174.up_proj", + "model.layers.47.mlp.experts.175.down_proj", + "model.layers.47.mlp.experts.175.gate_proj", + "model.layers.47.mlp.experts.175.up_proj", + "model.layers.47.mlp.experts.176.down_proj", + "model.layers.47.mlp.experts.176.gate_proj", + "model.layers.47.mlp.experts.176.up_proj", + "model.layers.47.mlp.experts.177.down_proj", + "model.layers.47.mlp.experts.177.gate_proj", + "model.layers.47.mlp.experts.177.up_proj", + "model.layers.47.mlp.experts.178.down_proj", + "model.layers.47.mlp.experts.178.gate_proj", + "model.layers.47.mlp.experts.178.up_proj", + "model.layers.47.mlp.experts.179.down_proj", + "model.layers.47.mlp.experts.179.gate_proj", + "model.layers.47.mlp.experts.179.up_proj", + "model.layers.47.mlp.experts.18.down_proj", + "model.layers.47.mlp.experts.18.gate_proj", + "model.layers.47.mlp.experts.18.up_proj", + "model.layers.47.mlp.experts.180.down_proj", + "model.layers.47.mlp.experts.180.gate_proj", + "model.layers.47.mlp.experts.180.up_proj", + "model.layers.47.mlp.experts.181.down_proj", + "model.layers.47.mlp.experts.181.gate_proj", + "model.layers.47.mlp.experts.181.up_proj", + "model.layers.47.mlp.experts.182.down_proj", + "model.layers.47.mlp.experts.182.gate_proj", + "model.layers.47.mlp.experts.182.up_proj", + "model.layers.47.mlp.experts.183.down_proj", + "model.layers.47.mlp.experts.183.gate_proj", + "model.layers.47.mlp.experts.183.up_proj", + "model.layers.47.mlp.experts.184.down_proj", + "model.layers.47.mlp.experts.184.gate_proj", + "model.layers.47.mlp.experts.184.up_proj", + "model.layers.47.mlp.experts.185.down_proj", + "model.layers.47.mlp.experts.185.gate_proj", + "model.layers.47.mlp.experts.185.up_proj", + "model.layers.47.mlp.experts.186.down_proj", + "model.layers.47.mlp.experts.186.gate_proj", + "model.layers.47.mlp.experts.186.up_proj", + "model.layers.47.mlp.experts.187.down_proj", + "model.layers.47.mlp.experts.187.gate_proj", + "model.layers.47.mlp.experts.187.up_proj", + "model.layers.47.mlp.experts.188.down_proj", + "model.layers.47.mlp.experts.188.gate_proj", + "model.layers.47.mlp.experts.188.up_proj", + "model.layers.47.mlp.experts.189.down_proj", + "model.layers.47.mlp.experts.189.gate_proj", + "model.layers.47.mlp.experts.189.up_proj", + "model.layers.47.mlp.experts.19.down_proj", + "model.layers.47.mlp.experts.19.gate_proj", + "model.layers.47.mlp.experts.19.up_proj", + "model.layers.47.mlp.experts.190.down_proj", + "model.layers.47.mlp.experts.190.gate_proj", + "model.layers.47.mlp.experts.190.up_proj", + "model.layers.47.mlp.experts.191.down_proj", + "model.layers.47.mlp.experts.191.gate_proj", + "model.layers.47.mlp.experts.191.up_proj", + "model.layers.47.mlp.experts.192.down_proj", + "model.layers.47.mlp.experts.192.gate_proj", + "model.layers.47.mlp.experts.192.up_proj", + "model.layers.47.mlp.experts.193.down_proj", + "model.layers.47.mlp.experts.193.gate_proj", + "model.layers.47.mlp.experts.193.up_proj", + "model.layers.47.mlp.experts.194.down_proj", + "model.layers.47.mlp.experts.194.gate_proj", + "model.layers.47.mlp.experts.194.up_proj", + "model.layers.47.mlp.experts.195.down_proj", + "model.layers.47.mlp.experts.195.gate_proj", + "model.layers.47.mlp.experts.195.up_proj", + "model.layers.47.mlp.experts.196.down_proj", + "model.layers.47.mlp.experts.196.gate_proj", + "model.layers.47.mlp.experts.196.up_proj", + "model.layers.47.mlp.experts.197.down_proj", + "model.layers.47.mlp.experts.197.gate_proj", + "model.layers.47.mlp.experts.197.up_proj", + "model.layers.47.mlp.experts.198.down_proj", + "model.layers.47.mlp.experts.198.gate_proj", + "model.layers.47.mlp.experts.198.up_proj", + "model.layers.47.mlp.experts.199.down_proj", + "model.layers.47.mlp.experts.199.gate_proj", + "model.layers.47.mlp.experts.199.up_proj", + "model.layers.47.mlp.experts.2.down_proj", + "model.layers.47.mlp.experts.2.gate_proj", + "model.layers.47.mlp.experts.2.up_proj", + "model.layers.47.mlp.experts.20.down_proj", + "model.layers.47.mlp.experts.20.gate_proj", + "model.layers.47.mlp.experts.20.up_proj", + "model.layers.47.mlp.experts.200.down_proj", + "model.layers.47.mlp.experts.200.gate_proj", + "model.layers.47.mlp.experts.200.up_proj", + "model.layers.47.mlp.experts.201.down_proj", + "model.layers.47.mlp.experts.201.gate_proj", + "model.layers.47.mlp.experts.201.up_proj", + "model.layers.47.mlp.experts.202.down_proj", + "model.layers.47.mlp.experts.202.gate_proj", + "model.layers.47.mlp.experts.202.up_proj", + "model.layers.47.mlp.experts.203.down_proj", + "model.layers.47.mlp.experts.203.gate_proj", + "model.layers.47.mlp.experts.203.up_proj", + "model.layers.47.mlp.experts.204.down_proj", + "model.layers.47.mlp.experts.204.gate_proj", + "model.layers.47.mlp.experts.204.up_proj", + "model.layers.47.mlp.experts.205.down_proj", + "model.layers.47.mlp.experts.205.gate_proj", + "model.layers.47.mlp.experts.205.up_proj", + "model.layers.47.mlp.experts.206.down_proj", + "model.layers.47.mlp.experts.206.gate_proj", + "model.layers.47.mlp.experts.206.up_proj", + "model.layers.47.mlp.experts.207.down_proj", + "model.layers.47.mlp.experts.207.gate_proj", + "model.layers.47.mlp.experts.207.up_proj", + "model.layers.47.mlp.experts.208.down_proj", + "model.layers.47.mlp.experts.208.gate_proj", + "model.layers.47.mlp.experts.208.up_proj", + "model.layers.47.mlp.experts.209.down_proj", + "model.layers.47.mlp.experts.209.gate_proj", + "model.layers.47.mlp.experts.209.up_proj", + "model.layers.47.mlp.experts.21.down_proj", + "model.layers.47.mlp.experts.21.gate_proj", + "model.layers.47.mlp.experts.21.up_proj", + "model.layers.47.mlp.experts.210.down_proj", + "model.layers.47.mlp.experts.210.gate_proj", + "model.layers.47.mlp.experts.210.up_proj", + "model.layers.47.mlp.experts.211.down_proj", + "model.layers.47.mlp.experts.211.gate_proj", + "model.layers.47.mlp.experts.211.up_proj", + "model.layers.47.mlp.experts.212.down_proj", + "model.layers.47.mlp.experts.212.gate_proj", + "model.layers.47.mlp.experts.212.up_proj", + "model.layers.47.mlp.experts.213.down_proj", + "model.layers.47.mlp.experts.213.gate_proj", + "model.layers.47.mlp.experts.213.up_proj", + "model.layers.47.mlp.experts.214.down_proj", + "model.layers.47.mlp.experts.214.gate_proj", + "model.layers.47.mlp.experts.214.up_proj", + "model.layers.47.mlp.experts.215.down_proj", + "model.layers.47.mlp.experts.215.gate_proj", + "model.layers.47.mlp.experts.215.up_proj", + "model.layers.47.mlp.experts.216.down_proj", + "model.layers.47.mlp.experts.216.gate_proj", + "model.layers.47.mlp.experts.216.up_proj", + "model.layers.47.mlp.experts.217.down_proj", + "model.layers.47.mlp.experts.217.gate_proj", + "model.layers.47.mlp.experts.217.up_proj", + "model.layers.47.mlp.experts.218.down_proj", + "model.layers.47.mlp.experts.218.gate_proj", + "model.layers.47.mlp.experts.218.up_proj", + "model.layers.47.mlp.experts.219.down_proj", + "model.layers.47.mlp.experts.219.gate_proj", + "model.layers.47.mlp.experts.219.up_proj", + "model.layers.47.mlp.experts.22.down_proj", + "model.layers.47.mlp.experts.22.gate_proj", + "model.layers.47.mlp.experts.22.up_proj", + "model.layers.47.mlp.experts.220.down_proj", + "model.layers.47.mlp.experts.220.gate_proj", + "model.layers.47.mlp.experts.220.up_proj", + "model.layers.47.mlp.experts.221.down_proj", + "model.layers.47.mlp.experts.221.gate_proj", + "model.layers.47.mlp.experts.221.up_proj", + "model.layers.47.mlp.experts.222.down_proj", + "model.layers.47.mlp.experts.222.gate_proj", + "model.layers.47.mlp.experts.222.up_proj", + "model.layers.47.mlp.experts.223.down_proj", + "model.layers.47.mlp.experts.223.gate_proj", + "model.layers.47.mlp.experts.223.up_proj", + "model.layers.47.mlp.experts.224.down_proj", + "model.layers.47.mlp.experts.224.gate_proj", + "model.layers.47.mlp.experts.224.up_proj", + "model.layers.47.mlp.experts.225.down_proj", + "model.layers.47.mlp.experts.225.gate_proj", + "model.layers.47.mlp.experts.225.up_proj", + "model.layers.47.mlp.experts.226.down_proj", + "model.layers.47.mlp.experts.226.gate_proj", + "model.layers.47.mlp.experts.226.up_proj", + "model.layers.47.mlp.experts.227.down_proj", + "model.layers.47.mlp.experts.227.gate_proj", + "model.layers.47.mlp.experts.227.up_proj", + "model.layers.47.mlp.experts.228.down_proj", + "model.layers.47.mlp.experts.228.gate_proj", + "model.layers.47.mlp.experts.228.up_proj", + "model.layers.47.mlp.experts.229.down_proj", + "model.layers.47.mlp.experts.229.gate_proj", + "model.layers.47.mlp.experts.229.up_proj", + "model.layers.47.mlp.experts.23.down_proj", + "model.layers.47.mlp.experts.23.gate_proj", + "model.layers.47.mlp.experts.23.up_proj", + "model.layers.47.mlp.experts.230.down_proj", + "model.layers.47.mlp.experts.230.gate_proj", + "model.layers.47.mlp.experts.230.up_proj", + "model.layers.47.mlp.experts.231.down_proj", + "model.layers.47.mlp.experts.231.gate_proj", + "model.layers.47.mlp.experts.231.up_proj", + "model.layers.47.mlp.experts.232.down_proj", + "model.layers.47.mlp.experts.232.gate_proj", + "model.layers.47.mlp.experts.232.up_proj", + "model.layers.47.mlp.experts.233.down_proj", + "model.layers.47.mlp.experts.233.gate_proj", + "model.layers.47.mlp.experts.233.up_proj", + "model.layers.47.mlp.experts.234.down_proj", + "model.layers.47.mlp.experts.234.gate_proj", + "model.layers.47.mlp.experts.234.up_proj", + "model.layers.47.mlp.experts.235.down_proj", + "model.layers.47.mlp.experts.235.gate_proj", + "model.layers.47.mlp.experts.235.up_proj", + "model.layers.47.mlp.experts.236.down_proj", + "model.layers.47.mlp.experts.236.gate_proj", + "model.layers.47.mlp.experts.236.up_proj", + "model.layers.47.mlp.experts.237.down_proj", + "model.layers.47.mlp.experts.237.gate_proj", + "model.layers.47.mlp.experts.237.up_proj", + "model.layers.47.mlp.experts.238.down_proj", + "model.layers.47.mlp.experts.238.gate_proj", + "model.layers.47.mlp.experts.238.up_proj", + "model.layers.47.mlp.experts.239.down_proj", + "model.layers.47.mlp.experts.239.gate_proj", + "model.layers.47.mlp.experts.239.up_proj", + "model.layers.47.mlp.experts.24.down_proj", + "model.layers.47.mlp.experts.24.gate_proj", + "model.layers.47.mlp.experts.24.up_proj", + "model.layers.47.mlp.experts.240.down_proj", + "model.layers.47.mlp.experts.240.gate_proj", + "model.layers.47.mlp.experts.240.up_proj", + "model.layers.47.mlp.experts.241.down_proj", + "model.layers.47.mlp.experts.241.gate_proj", + "model.layers.47.mlp.experts.241.up_proj", + "model.layers.47.mlp.experts.242.down_proj", + "model.layers.47.mlp.experts.242.gate_proj", + "model.layers.47.mlp.experts.242.up_proj", + "model.layers.47.mlp.experts.243.down_proj", + "model.layers.47.mlp.experts.243.gate_proj", + "model.layers.47.mlp.experts.243.up_proj", + "model.layers.47.mlp.experts.244.down_proj", + "model.layers.47.mlp.experts.244.gate_proj", + "model.layers.47.mlp.experts.244.up_proj", + "model.layers.47.mlp.experts.245.down_proj", + "model.layers.47.mlp.experts.245.gate_proj", + "model.layers.47.mlp.experts.245.up_proj", + "model.layers.47.mlp.experts.246.down_proj", + "model.layers.47.mlp.experts.246.gate_proj", + "model.layers.47.mlp.experts.246.up_proj", + "model.layers.47.mlp.experts.247.down_proj", + "model.layers.47.mlp.experts.247.gate_proj", + "model.layers.47.mlp.experts.247.up_proj", + "model.layers.47.mlp.experts.248.down_proj", + "model.layers.47.mlp.experts.248.gate_proj", + "model.layers.47.mlp.experts.248.up_proj", + "model.layers.47.mlp.experts.249.down_proj", + "model.layers.47.mlp.experts.249.gate_proj", + "model.layers.47.mlp.experts.249.up_proj", + "model.layers.47.mlp.experts.25.down_proj", + "model.layers.47.mlp.experts.25.gate_proj", + "model.layers.47.mlp.experts.25.up_proj", + "model.layers.47.mlp.experts.250.down_proj", + "model.layers.47.mlp.experts.250.gate_proj", + "model.layers.47.mlp.experts.250.up_proj", + "model.layers.47.mlp.experts.251.down_proj", + "model.layers.47.mlp.experts.251.gate_proj", + "model.layers.47.mlp.experts.251.up_proj", + "model.layers.47.mlp.experts.252.down_proj", + "model.layers.47.mlp.experts.252.gate_proj", + "model.layers.47.mlp.experts.252.up_proj", + "model.layers.47.mlp.experts.253.down_proj", + "model.layers.47.mlp.experts.253.gate_proj", + "model.layers.47.mlp.experts.253.up_proj", + "model.layers.47.mlp.experts.254.down_proj", + "model.layers.47.mlp.experts.254.gate_proj", + "model.layers.47.mlp.experts.254.up_proj", + "model.layers.47.mlp.experts.255.down_proj", + "model.layers.47.mlp.experts.255.gate_proj", + "model.layers.47.mlp.experts.255.up_proj", + "model.layers.47.mlp.experts.26.down_proj", + "model.layers.47.mlp.experts.26.gate_proj", + "model.layers.47.mlp.experts.26.up_proj", + "model.layers.47.mlp.experts.27.down_proj", + "model.layers.47.mlp.experts.27.gate_proj", + "model.layers.47.mlp.experts.27.up_proj", + "model.layers.47.mlp.experts.28.down_proj", + "model.layers.47.mlp.experts.28.gate_proj", + "model.layers.47.mlp.experts.28.up_proj", + "model.layers.47.mlp.experts.29.down_proj", + "model.layers.47.mlp.experts.29.gate_proj", + "model.layers.47.mlp.experts.29.up_proj", + "model.layers.47.mlp.experts.3.down_proj", + "model.layers.47.mlp.experts.3.gate_proj", + "model.layers.47.mlp.experts.3.up_proj", + "model.layers.47.mlp.experts.30.down_proj", + "model.layers.47.mlp.experts.30.gate_proj", + "model.layers.47.mlp.experts.30.up_proj", + "model.layers.47.mlp.experts.31.down_proj", + "model.layers.47.mlp.experts.31.gate_proj", + "model.layers.47.mlp.experts.31.up_proj", + "model.layers.47.mlp.experts.32.down_proj", + "model.layers.47.mlp.experts.32.gate_proj", + "model.layers.47.mlp.experts.32.up_proj", + "model.layers.47.mlp.experts.33.down_proj", + "model.layers.47.mlp.experts.33.gate_proj", + "model.layers.47.mlp.experts.33.up_proj", + "model.layers.47.mlp.experts.34.down_proj", + "model.layers.47.mlp.experts.34.gate_proj", + "model.layers.47.mlp.experts.34.up_proj", + "model.layers.47.mlp.experts.35.down_proj", + "model.layers.47.mlp.experts.35.gate_proj", + "model.layers.47.mlp.experts.35.up_proj", + "model.layers.47.mlp.experts.36.down_proj", + "model.layers.47.mlp.experts.36.gate_proj", + "model.layers.47.mlp.experts.36.up_proj", + "model.layers.47.mlp.experts.37.down_proj", + "model.layers.47.mlp.experts.37.gate_proj", + "model.layers.47.mlp.experts.37.up_proj", + "model.layers.47.mlp.experts.38.down_proj", + "model.layers.47.mlp.experts.38.gate_proj", + "model.layers.47.mlp.experts.38.up_proj", + "model.layers.47.mlp.experts.39.down_proj", + "model.layers.47.mlp.experts.39.gate_proj", + "model.layers.47.mlp.experts.39.up_proj", + "model.layers.47.mlp.experts.4.down_proj", + "model.layers.47.mlp.experts.4.gate_proj", + "model.layers.47.mlp.experts.4.up_proj", + "model.layers.47.mlp.experts.40.down_proj", + "model.layers.47.mlp.experts.40.gate_proj", + "model.layers.47.mlp.experts.40.up_proj", + "model.layers.47.mlp.experts.41.down_proj", + "model.layers.47.mlp.experts.41.gate_proj", + "model.layers.47.mlp.experts.41.up_proj", + "model.layers.47.mlp.experts.42.down_proj", + "model.layers.47.mlp.experts.42.gate_proj", + "model.layers.47.mlp.experts.42.up_proj", + "model.layers.47.mlp.experts.43.down_proj", + "model.layers.47.mlp.experts.43.gate_proj", + "model.layers.47.mlp.experts.43.up_proj", + "model.layers.47.mlp.experts.44.down_proj", + "model.layers.47.mlp.experts.44.gate_proj", + "model.layers.47.mlp.experts.44.up_proj", + "model.layers.47.mlp.experts.45.down_proj", + "model.layers.47.mlp.experts.45.gate_proj", + "model.layers.47.mlp.experts.45.up_proj", + "model.layers.47.mlp.experts.46.down_proj", + "model.layers.47.mlp.experts.46.gate_proj", + "model.layers.47.mlp.experts.46.up_proj", + "model.layers.47.mlp.experts.47.down_proj", + "model.layers.47.mlp.experts.47.gate_proj", + "model.layers.47.mlp.experts.47.up_proj", + "model.layers.47.mlp.experts.48.down_proj", + "model.layers.47.mlp.experts.48.gate_proj", + "model.layers.47.mlp.experts.48.up_proj", + "model.layers.47.mlp.experts.49.down_proj", + "model.layers.47.mlp.experts.49.gate_proj", + "model.layers.47.mlp.experts.49.up_proj", + "model.layers.47.mlp.experts.5.down_proj", + "model.layers.47.mlp.experts.5.gate_proj", + "model.layers.47.mlp.experts.5.up_proj", + "model.layers.47.mlp.experts.50.down_proj", + "model.layers.47.mlp.experts.50.gate_proj", + "model.layers.47.mlp.experts.50.up_proj", + "model.layers.47.mlp.experts.51.down_proj", + "model.layers.47.mlp.experts.51.gate_proj", + "model.layers.47.mlp.experts.51.up_proj", + "model.layers.47.mlp.experts.52.down_proj", + "model.layers.47.mlp.experts.52.gate_proj", + "model.layers.47.mlp.experts.52.up_proj", + "model.layers.47.mlp.experts.53.down_proj", + "model.layers.47.mlp.experts.53.gate_proj", + "model.layers.47.mlp.experts.53.up_proj", + "model.layers.47.mlp.experts.54.down_proj", + "model.layers.47.mlp.experts.54.gate_proj", + "model.layers.47.mlp.experts.54.up_proj", + "model.layers.47.mlp.experts.55.down_proj", + "model.layers.47.mlp.experts.55.gate_proj", + "model.layers.47.mlp.experts.55.up_proj", + "model.layers.47.mlp.experts.56.down_proj", + "model.layers.47.mlp.experts.56.gate_proj", + "model.layers.47.mlp.experts.56.up_proj", + "model.layers.47.mlp.experts.57.down_proj", + "model.layers.47.mlp.experts.57.gate_proj", + "model.layers.47.mlp.experts.57.up_proj", + "model.layers.47.mlp.experts.58.down_proj", + "model.layers.47.mlp.experts.58.gate_proj", + "model.layers.47.mlp.experts.58.up_proj", + "model.layers.47.mlp.experts.59.down_proj", + "model.layers.47.mlp.experts.59.gate_proj", + "model.layers.47.mlp.experts.59.up_proj", + "model.layers.47.mlp.experts.6.down_proj", + "model.layers.47.mlp.experts.6.gate_proj", + "model.layers.47.mlp.experts.6.up_proj", + "model.layers.47.mlp.experts.60.down_proj", + "model.layers.47.mlp.experts.60.gate_proj", + "model.layers.47.mlp.experts.60.up_proj", + "model.layers.47.mlp.experts.61.down_proj", + "model.layers.47.mlp.experts.61.gate_proj", + "model.layers.47.mlp.experts.61.up_proj", + "model.layers.47.mlp.experts.62.down_proj", + "model.layers.47.mlp.experts.62.gate_proj", + "model.layers.47.mlp.experts.62.up_proj", + "model.layers.47.mlp.experts.63.down_proj", + "model.layers.47.mlp.experts.63.gate_proj", + "model.layers.47.mlp.experts.63.up_proj", + "model.layers.47.mlp.experts.64.down_proj", + "model.layers.47.mlp.experts.64.gate_proj", + "model.layers.47.mlp.experts.64.up_proj", + "model.layers.47.mlp.experts.65.down_proj", + "model.layers.47.mlp.experts.65.gate_proj", + "model.layers.47.mlp.experts.65.up_proj", + "model.layers.47.mlp.experts.66.down_proj", + "model.layers.47.mlp.experts.66.gate_proj", + "model.layers.47.mlp.experts.66.up_proj", + "model.layers.47.mlp.experts.67.down_proj", + "model.layers.47.mlp.experts.67.gate_proj", + "model.layers.47.mlp.experts.67.up_proj", + "model.layers.47.mlp.experts.68.down_proj", + "model.layers.47.mlp.experts.68.gate_proj", + "model.layers.47.mlp.experts.68.up_proj", + "model.layers.47.mlp.experts.69.down_proj", + "model.layers.47.mlp.experts.69.gate_proj", + "model.layers.47.mlp.experts.69.up_proj", + "model.layers.47.mlp.experts.7.down_proj", + "model.layers.47.mlp.experts.7.gate_proj", + "model.layers.47.mlp.experts.7.up_proj", + "model.layers.47.mlp.experts.70.down_proj", + "model.layers.47.mlp.experts.70.gate_proj", + "model.layers.47.mlp.experts.70.up_proj", + "model.layers.47.mlp.experts.71.down_proj", + "model.layers.47.mlp.experts.71.gate_proj", + "model.layers.47.mlp.experts.71.up_proj", + "model.layers.47.mlp.experts.72.down_proj", + "model.layers.47.mlp.experts.72.gate_proj", + "model.layers.47.mlp.experts.72.up_proj", + "model.layers.47.mlp.experts.73.down_proj", + "model.layers.47.mlp.experts.73.gate_proj", + "model.layers.47.mlp.experts.73.up_proj", + "model.layers.47.mlp.experts.74.down_proj", + "model.layers.47.mlp.experts.74.gate_proj", + "model.layers.47.mlp.experts.74.up_proj", + "model.layers.47.mlp.experts.75.down_proj", + "model.layers.47.mlp.experts.75.gate_proj", + "model.layers.47.mlp.experts.75.up_proj", + "model.layers.47.mlp.experts.76.down_proj", + "model.layers.47.mlp.experts.76.gate_proj", + "model.layers.47.mlp.experts.76.up_proj", + "model.layers.47.mlp.experts.77.down_proj", + "model.layers.47.mlp.experts.77.gate_proj", + "model.layers.47.mlp.experts.77.up_proj", + "model.layers.47.mlp.experts.78.down_proj", + "model.layers.47.mlp.experts.78.gate_proj", + "model.layers.47.mlp.experts.78.up_proj", + "model.layers.47.mlp.experts.79.down_proj", + "model.layers.47.mlp.experts.79.gate_proj", + "model.layers.47.mlp.experts.79.up_proj", + "model.layers.47.mlp.experts.8.down_proj", + "model.layers.47.mlp.experts.8.gate_proj", + "model.layers.47.mlp.experts.8.up_proj", + "model.layers.47.mlp.experts.80.down_proj", + "model.layers.47.mlp.experts.80.gate_proj", + "model.layers.47.mlp.experts.80.up_proj", + "model.layers.47.mlp.experts.81.down_proj", + "model.layers.47.mlp.experts.81.gate_proj", + "model.layers.47.mlp.experts.81.up_proj", + "model.layers.47.mlp.experts.82.down_proj", + "model.layers.47.mlp.experts.82.gate_proj", + "model.layers.47.mlp.experts.82.up_proj", + "model.layers.47.mlp.experts.83.down_proj", + "model.layers.47.mlp.experts.83.gate_proj", + "model.layers.47.mlp.experts.83.up_proj", + "model.layers.47.mlp.experts.84.down_proj", + "model.layers.47.mlp.experts.84.gate_proj", + "model.layers.47.mlp.experts.84.up_proj", + "model.layers.47.mlp.experts.85.down_proj", + "model.layers.47.mlp.experts.85.gate_proj", + "model.layers.47.mlp.experts.85.up_proj", + "model.layers.47.mlp.experts.86.down_proj", + "model.layers.47.mlp.experts.86.gate_proj", + "model.layers.47.mlp.experts.86.up_proj", + "model.layers.47.mlp.experts.87.down_proj", + "model.layers.47.mlp.experts.87.gate_proj", + "model.layers.47.mlp.experts.87.up_proj", + "model.layers.47.mlp.experts.88.down_proj", + "model.layers.47.mlp.experts.88.gate_proj", + "model.layers.47.mlp.experts.88.up_proj", + "model.layers.47.mlp.experts.89.down_proj", + "model.layers.47.mlp.experts.89.gate_proj", + "model.layers.47.mlp.experts.89.up_proj", + "model.layers.47.mlp.experts.9.down_proj", + "model.layers.47.mlp.experts.9.gate_proj", + "model.layers.47.mlp.experts.9.up_proj", + "model.layers.47.mlp.experts.90.down_proj", + "model.layers.47.mlp.experts.90.gate_proj", + "model.layers.47.mlp.experts.90.up_proj", + "model.layers.47.mlp.experts.91.down_proj", + "model.layers.47.mlp.experts.91.gate_proj", + "model.layers.47.mlp.experts.91.up_proj", + "model.layers.47.mlp.experts.92.down_proj", + "model.layers.47.mlp.experts.92.gate_proj", + "model.layers.47.mlp.experts.92.up_proj", + "model.layers.47.mlp.experts.93.down_proj", + "model.layers.47.mlp.experts.93.gate_proj", + "model.layers.47.mlp.experts.93.up_proj", + "model.layers.47.mlp.experts.94.down_proj", + "model.layers.47.mlp.experts.94.gate_proj", + "model.layers.47.mlp.experts.94.up_proj", + "model.layers.47.mlp.experts.95.down_proj", + "model.layers.47.mlp.experts.95.gate_proj", + "model.layers.47.mlp.experts.95.up_proj", + "model.layers.47.mlp.experts.96.down_proj", + "model.layers.47.mlp.experts.96.gate_proj", + "model.layers.47.mlp.experts.96.up_proj", + "model.layers.47.mlp.experts.97.down_proj", + "model.layers.47.mlp.experts.97.gate_proj", + "model.layers.47.mlp.experts.97.up_proj", + "model.layers.47.mlp.experts.98.down_proj", + "model.layers.47.mlp.experts.98.gate_proj", + "model.layers.47.mlp.experts.98.up_proj", + "model.layers.47.mlp.experts.99.down_proj", + "model.layers.47.mlp.experts.99.gate_proj", + "model.layers.47.mlp.experts.99.up_proj", + "model.layers.48.mlp.experts.0.down_proj", + "model.layers.48.mlp.experts.0.gate_proj", + "model.layers.48.mlp.experts.0.up_proj", + "model.layers.48.mlp.experts.1.down_proj", + "model.layers.48.mlp.experts.1.gate_proj", + "model.layers.48.mlp.experts.1.up_proj", + "model.layers.48.mlp.experts.10.down_proj", + "model.layers.48.mlp.experts.10.gate_proj", + "model.layers.48.mlp.experts.10.up_proj", + "model.layers.48.mlp.experts.100.down_proj", + "model.layers.48.mlp.experts.100.gate_proj", + "model.layers.48.mlp.experts.100.up_proj", + "model.layers.48.mlp.experts.101.down_proj", + "model.layers.48.mlp.experts.101.gate_proj", + "model.layers.48.mlp.experts.101.up_proj", + "model.layers.48.mlp.experts.102.down_proj", + "model.layers.48.mlp.experts.102.gate_proj", + "model.layers.48.mlp.experts.102.up_proj", + "model.layers.48.mlp.experts.103.down_proj", + "model.layers.48.mlp.experts.103.gate_proj", + "model.layers.48.mlp.experts.103.up_proj", + "model.layers.48.mlp.experts.104.down_proj", + "model.layers.48.mlp.experts.104.gate_proj", + "model.layers.48.mlp.experts.104.up_proj", + "model.layers.48.mlp.experts.105.down_proj", + "model.layers.48.mlp.experts.105.gate_proj", + "model.layers.48.mlp.experts.105.up_proj", + "model.layers.48.mlp.experts.106.down_proj", + "model.layers.48.mlp.experts.106.gate_proj", + "model.layers.48.mlp.experts.106.up_proj", + "model.layers.48.mlp.experts.107.down_proj", + "model.layers.48.mlp.experts.107.gate_proj", + "model.layers.48.mlp.experts.107.up_proj", + "model.layers.48.mlp.experts.108.down_proj", + "model.layers.48.mlp.experts.108.gate_proj", + "model.layers.48.mlp.experts.108.up_proj", + "model.layers.48.mlp.experts.109.down_proj", + "model.layers.48.mlp.experts.109.gate_proj", + "model.layers.48.mlp.experts.109.up_proj", + "model.layers.48.mlp.experts.11.down_proj", + "model.layers.48.mlp.experts.11.gate_proj", + "model.layers.48.mlp.experts.11.up_proj", + "model.layers.48.mlp.experts.110.down_proj", + "model.layers.48.mlp.experts.110.gate_proj", + "model.layers.48.mlp.experts.110.up_proj", + "model.layers.48.mlp.experts.111.down_proj", + "model.layers.48.mlp.experts.111.gate_proj", + "model.layers.48.mlp.experts.111.up_proj", + "model.layers.48.mlp.experts.112.down_proj", + "model.layers.48.mlp.experts.112.gate_proj", + "model.layers.48.mlp.experts.112.up_proj", + "model.layers.48.mlp.experts.113.down_proj", + "model.layers.48.mlp.experts.113.gate_proj", + "model.layers.48.mlp.experts.113.up_proj", + "model.layers.48.mlp.experts.114.down_proj", + "model.layers.48.mlp.experts.114.gate_proj", + "model.layers.48.mlp.experts.114.up_proj", + "model.layers.48.mlp.experts.115.down_proj", + "model.layers.48.mlp.experts.115.gate_proj", + "model.layers.48.mlp.experts.115.up_proj", + "model.layers.48.mlp.experts.116.down_proj", + "model.layers.48.mlp.experts.116.gate_proj", + "model.layers.48.mlp.experts.116.up_proj", + "model.layers.48.mlp.experts.117.down_proj", + "model.layers.48.mlp.experts.117.gate_proj", + "model.layers.48.mlp.experts.117.up_proj", + "model.layers.48.mlp.experts.118.down_proj", + "model.layers.48.mlp.experts.118.gate_proj", + "model.layers.48.mlp.experts.118.up_proj", + "model.layers.48.mlp.experts.119.down_proj", + "model.layers.48.mlp.experts.119.gate_proj", + "model.layers.48.mlp.experts.119.up_proj", + "model.layers.48.mlp.experts.12.down_proj", + "model.layers.48.mlp.experts.12.gate_proj", + "model.layers.48.mlp.experts.12.up_proj", + "model.layers.48.mlp.experts.120.down_proj", + "model.layers.48.mlp.experts.120.gate_proj", + "model.layers.48.mlp.experts.120.up_proj", + "model.layers.48.mlp.experts.121.down_proj", + "model.layers.48.mlp.experts.121.gate_proj", + "model.layers.48.mlp.experts.121.up_proj", + "model.layers.48.mlp.experts.122.down_proj", + "model.layers.48.mlp.experts.122.gate_proj", + "model.layers.48.mlp.experts.122.up_proj", + "model.layers.48.mlp.experts.123.down_proj", + "model.layers.48.mlp.experts.123.gate_proj", + "model.layers.48.mlp.experts.123.up_proj", + "model.layers.48.mlp.experts.124.down_proj", + "model.layers.48.mlp.experts.124.gate_proj", + "model.layers.48.mlp.experts.124.up_proj", + "model.layers.48.mlp.experts.125.down_proj", + "model.layers.48.mlp.experts.125.gate_proj", + "model.layers.48.mlp.experts.125.up_proj", + "model.layers.48.mlp.experts.126.down_proj", + "model.layers.48.mlp.experts.126.gate_proj", + "model.layers.48.mlp.experts.126.up_proj", + "model.layers.48.mlp.experts.127.down_proj", + "model.layers.48.mlp.experts.127.gate_proj", + "model.layers.48.mlp.experts.127.up_proj", + "model.layers.48.mlp.experts.128.down_proj", + "model.layers.48.mlp.experts.128.gate_proj", + "model.layers.48.mlp.experts.128.up_proj", + "model.layers.48.mlp.experts.129.down_proj", + "model.layers.48.mlp.experts.129.gate_proj", + "model.layers.48.mlp.experts.129.up_proj", + "model.layers.48.mlp.experts.13.down_proj", + "model.layers.48.mlp.experts.13.gate_proj", + "model.layers.48.mlp.experts.13.up_proj", + "model.layers.48.mlp.experts.130.down_proj", + "model.layers.48.mlp.experts.130.gate_proj", + "model.layers.48.mlp.experts.130.up_proj", + "model.layers.48.mlp.experts.131.down_proj", + "model.layers.48.mlp.experts.131.gate_proj", + "model.layers.48.mlp.experts.131.up_proj", + "model.layers.48.mlp.experts.132.down_proj", + "model.layers.48.mlp.experts.132.gate_proj", + "model.layers.48.mlp.experts.132.up_proj", + "model.layers.48.mlp.experts.133.down_proj", + "model.layers.48.mlp.experts.133.gate_proj", + "model.layers.48.mlp.experts.133.up_proj", + "model.layers.48.mlp.experts.134.down_proj", + "model.layers.48.mlp.experts.134.gate_proj", + "model.layers.48.mlp.experts.134.up_proj", + "model.layers.48.mlp.experts.135.down_proj", + "model.layers.48.mlp.experts.135.gate_proj", + "model.layers.48.mlp.experts.135.up_proj", + "model.layers.48.mlp.experts.136.down_proj", + "model.layers.48.mlp.experts.136.gate_proj", + "model.layers.48.mlp.experts.136.up_proj", + "model.layers.48.mlp.experts.137.down_proj", + "model.layers.48.mlp.experts.137.gate_proj", + "model.layers.48.mlp.experts.137.up_proj", + "model.layers.48.mlp.experts.138.down_proj", + "model.layers.48.mlp.experts.138.gate_proj", + "model.layers.48.mlp.experts.138.up_proj", + "model.layers.48.mlp.experts.139.down_proj", + "model.layers.48.mlp.experts.139.gate_proj", + "model.layers.48.mlp.experts.139.up_proj", + "model.layers.48.mlp.experts.14.down_proj", + "model.layers.48.mlp.experts.14.gate_proj", + "model.layers.48.mlp.experts.14.up_proj", + "model.layers.48.mlp.experts.140.down_proj", + "model.layers.48.mlp.experts.140.gate_proj", + "model.layers.48.mlp.experts.140.up_proj", + "model.layers.48.mlp.experts.141.down_proj", + "model.layers.48.mlp.experts.141.gate_proj", + "model.layers.48.mlp.experts.141.up_proj", + "model.layers.48.mlp.experts.142.down_proj", + "model.layers.48.mlp.experts.142.gate_proj", + "model.layers.48.mlp.experts.142.up_proj", + "model.layers.48.mlp.experts.143.down_proj", + "model.layers.48.mlp.experts.143.gate_proj", + "model.layers.48.mlp.experts.143.up_proj", + "model.layers.48.mlp.experts.144.down_proj", + "model.layers.48.mlp.experts.144.gate_proj", + "model.layers.48.mlp.experts.144.up_proj", + "model.layers.48.mlp.experts.145.down_proj", + "model.layers.48.mlp.experts.145.gate_proj", + "model.layers.48.mlp.experts.145.up_proj", + "model.layers.48.mlp.experts.146.down_proj", + "model.layers.48.mlp.experts.146.gate_proj", + "model.layers.48.mlp.experts.146.up_proj", + "model.layers.48.mlp.experts.147.down_proj", + "model.layers.48.mlp.experts.147.gate_proj", + "model.layers.48.mlp.experts.147.up_proj", + "model.layers.48.mlp.experts.148.down_proj", + "model.layers.48.mlp.experts.148.gate_proj", + "model.layers.48.mlp.experts.148.up_proj", + "model.layers.48.mlp.experts.149.down_proj", + "model.layers.48.mlp.experts.149.gate_proj", + "model.layers.48.mlp.experts.149.up_proj", + "model.layers.48.mlp.experts.15.down_proj", + "model.layers.48.mlp.experts.15.gate_proj", + "model.layers.48.mlp.experts.15.up_proj", + "model.layers.48.mlp.experts.150.down_proj", + "model.layers.48.mlp.experts.150.gate_proj", + "model.layers.48.mlp.experts.150.up_proj", + "model.layers.48.mlp.experts.151.down_proj", + "model.layers.48.mlp.experts.151.gate_proj", + "model.layers.48.mlp.experts.151.up_proj", + "model.layers.48.mlp.experts.152.down_proj", + "model.layers.48.mlp.experts.152.gate_proj", + "model.layers.48.mlp.experts.152.up_proj", + "model.layers.48.mlp.experts.153.down_proj", + "model.layers.48.mlp.experts.153.gate_proj", + "model.layers.48.mlp.experts.153.up_proj", + "model.layers.48.mlp.experts.154.down_proj", + "model.layers.48.mlp.experts.154.gate_proj", + "model.layers.48.mlp.experts.154.up_proj", + "model.layers.48.mlp.experts.155.down_proj", + "model.layers.48.mlp.experts.155.gate_proj", + "model.layers.48.mlp.experts.155.up_proj", + "model.layers.48.mlp.experts.156.down_proj", + "model.layers.48.mlp.experts.156.gate_proj", + "model.layers.48.mlp.experts.156.up_proj", + "model.layers.48.mlp.experts.157.down_proj", + "model.layers.48.mlp.experts.157.gate_proj", + "model.layers.48.mlp.experts.157.up_proj", + "model.layers.48.mlp.experts.158.down_proj", + "model.layers.48.mlp.experts.158.gate_proj", + "model.layers.48.mlp.experts.158.up_proj", + "model.layers.48.mlp.experts.159.down_proj", + "model.layers.48.mlp.experts.159.gate_proj", + "model.layers.48.mlp.experts.159.up_proj", + "model.layers.48.mlp.experts.16.down_proj", + "model.layers.48.mlp.experts.16.gate_proj", + "model.layers.48.mlp.experts.16.up_proj", + "model.layers.48.mlp.experts.160.down_proj", + "model.layers.48.mlp.experts.160.gate_proj", + "model.layers.48.mlp.experts.160.up_proj", + "model.layers.48.mlp.experts.161.down_proj", + "model.layers.48.mlp.experts.161.gate_proj", + "model.layers.48.mlp.experts.161.up_proj", + "model.layers.48.mlp.experts.162.down_proj", + "model.layers.48.mlp.experts.162.gate_proj", + "model.layers.48.mlp.experts.162.up_proj", + "model.layers.48.mlp.experts.163.down_proj", + "model.layers.48.mlp.experts.163.gate_proj", + "model.layers.48.mlp.experts.163.up_proj", + "model.layers.48.mlp.experts.164.down_proj", + "model.layers.48.mlp.experts.164.gate_proj", + "model.layers.48.mlp.experts.164.up_proj", + "model.layers.48.mlp.experts.165.down_proj", + "model.layers.48.mlp.experts.165.gate_proj", + "model.layers.48.mlp.experts.165.up_proj", + "model.layers.48.mlp.experts.166.down_proj", + "model.layers.48.mlp.experts.166.gate_proj", + "model.layers.48.mlp.experts.166.up_proj", + "model.layers.48.mlp.experts.167.down_proj", + "model.layers.48.mlp.experts.167.gate_proj", + "model.layers.48.mlp.experts.167.up_proj", + "model.layers.48.mlp.experts.168.down_proj", + "model.layers.48.mlp.experts.168.gate_proj", + "model.layers.48.mlp.experts.168.up_proj", + "model.layers.48.mlp.experts.169.down_proj", + "model.layers.48.mlp.experts.169.gate_proj", + "model.layers.48.mlp.experts.169.up_proj", + "model.layers.48.mlp.experts.17.down_proj", + "model.layers.48.mlp.experts.17.gate_proj", + "model.layers.48.mlp.experts.17.up_proj", + "model.layers.48.mlp.experts.170.down_proj", + "model.layers.48.mlp.experts.170.gate_proj", + "model.layers.48.mlp.experts.170.up_proj", + "model.layers.48.mlp.experts.171.down_proj", + "model.layers.48.mlp.experts.171.gate_proj", + "model.layers.48.mlp.experts.171.up_proj", + "model.layers.48.mlp.experts.172.down_proj", + "model.layers.48.mlp.experts.172.gate_proj", + "model.layers.48.mlp.experts.172.up_proj", + "model.layers.48.mlp.experts.173.down_proj", + "model.layers.48.mlp.experts.173.gate_proj", + "model.layers.48.mlp.experts.173.up_proj", + "model.layers.48.mlp.experts.174.down_proj", + "model.layers.48.mlp.experts.174.gate_proj", + "model.layers.48.mlp.experts.174.up_proj", + "model.layers.48.mlp.experts.175.down_proj", + "model.layers.48.mlp.experts.175.gate_proj", + "model.layers.48.mlp.experts.175.up_proj", + "model.layers.48.mlp.experts.176.down_proj", + "model.layers.48.mlp.experts.176.gate_proj", + "model.layers.48.mlp.experts.176.up_proj", + "model.layers.48.mlp.experts.177.down_proj", + "model.layers.48.mlp.experts.177.gate_proj", + "model.layers.48.mlp.experts.177.up_proj", + "model.layers.48.mlp.experts.178.down_proj", + "model.layers.48.mlp.experts.178.gate_proj", + "model.layers.48.mlp.experts.178.up_proj", + "model.layers.48.mlp.experts.179.down_proj", + "model.layers.48.mlp.experts.179.gate_proj", + "model.layers.48.mlp.experts.179.up_proj", + "model.layers.48.mlp.experts.18.down_proj", + "model.layers.48.mlp.experts.18.gate_proj", + "model.layers.48.mlp.experts.18.up_proj", + "model.layers.48.mlp.experts.180.down_proj", + "model.layers.48.mlp.experts.180.gate_proj", + "model.layers.48.mlp.experts.180.up_proj", + "model.layers.48.mlp.experts.181.down_proj", + "model.layers.48.mlp.experts.181.gate_proj", + "model.layers.48.mlp.experts.181.up_proj", + "model.layers.48.mlp.experts.182.down_proj", + "model.layers.48.mlp.experts.182.gate_proj", + "model.layers.48.mlp.experts.182.up_proj", + "model.layers.48.mlp.experts.183.down_proj", + "model.layers.48.mlp.experts.183.gate_proj", + "model.layers.48.mlp.experts.183.up_proj", + "model.layers.48.mlp.experts.184.down_proj", + "model.layers.48.mlp.experts.184.gate_proj", + "model.layers.48.mlp.experts.184.up_proj", + "model.layers.48.mlp.experts.185.down_proj", + "model.layers.48.mlp.experts.185.gate_proj", + "model.layers.48.mlp.experts.185.up_proj", + "model.layers.48.mlp.experts.186.down_proj", + "model.layers.48.mlp.experts.186.gate_proj", + "model.layers.48.mlp.experts.186.up_proj", + "model.layers.48.mlp.experts.187.down_proj", + "model.layers.48.mlp.experts.187.gate_proj", + "model.layers.48.mlp.experts.187.up_proj", + "model.layers.48.mlp.experts.188.down_proj", + "model.layers.48.mlp.experts.188.gate_proj", + "model.layers.48.mlp.experts.188.up_proj", + "model.layers.48.mlp.experts.189.down_proj", + "model.layers.48.mlp.experts.189.gate_proj", + "model.layers.48.mlp.experts.189.up_proj", + "model.layers.48.mlp.experts.19.down_proj", + "model.layers.48.mlp.experts.19.gate_proj", + "model.layers.48.mlp.experts.19.up_proj", + "model.layers.48.mlp.experts.190.down_proj", + "model.layers.48.mlp.experts.190.gate_proj", + "model.layers.48.mlp.experts.190.up_proj", + "model.layers.48.mlp.experts.191.down_proj", + "model.layers.48.mlp.experts.191.gate_proj", + "model.layers.48.mlp.experts.191.up_proj", + "model.layers.48.mlp.experts.192.down_proj", + "model.layers.48.mlp.experts.192.gate_proj", + "model.layers.48.mlp.experts.192.up_proj", + "model.layers.48.mlp.experts.193.down_proj", + "model.layers.48.mlp.experts.193.gate_proj", + "model.layers.48.mlp.experts.193.up_proj", + "model.layers.48.mlp.experts.194.down_proj", + "model.layers.48.mlp.experts.194.gate_proj", + "model.layers.48.mlp.experts.194.up_proj", + "model.layers.48.mlp.experts.195.down_proj", + "model.layers.48.mlp.experts.195.gate_proj", + "model.layers.48.mlp.experts.195.up_proj", + "model.layers.48.mlp.experts.196.down_proj", + "model.layers.48.mlp.experts.196.gate_proj", + "model.layers.48.mlp.experts.196.up_proj", + "model.layers.48.mlp.experts.197.down_proj", + "model.layers.48.mlp.experts.197.gate_proj", + "model.layers.48.mlp.experts.197.up_proj", + "model.layers.48.mlp.experts.198.down_proj", + "model.layers.48.mlp.experts.198.gate_proj", + "model.layers.48.mlp.experts.198.up_proj", + "model.layers.48.mlp.experts.199.down_proj", + "model.layers.48.mlp.experts.199.gate_proj", + "model.layers.48.mlp.experts.199.up_proj", + "model.layers.48.mlp.experts.2.down_proj", + "model.layers.48.mlp.experts.2.gate_proj", + "model.layers.48.mlp.experts.2.up_proj", + "model.layers.48.mlp.experts.20.down_proj", + "model.layers.48.mlp.experts.20.gate_proj", + "model.layers.48.mlp.experts.20.up_proj", + "model.layers.48.mlp.experts.200.down_proj", + "model.layers.48.mlp.experts.200.gate_proj", + "model.layers.48.mlp.experts.200.up_proj", + "model.layers.48.mlp.experts.201.down_proj", + "model.layers.48.mlp.experts.201.gate_proj", + "model.layers.48.mlp.experts.201.up_proj", + "model.layers.48.mlp.experts.202.down_proj", + "model.layers.48.mlp.experts.202.gate_proj", + "model.layers.48.mlp.experts.202.up_proj", + "model.layers.48.mlp.experts.203.down_proj", + "model.layers.48.mlp.experts.203.gate_proj", + "model.layers.48.mlp.experts.203.up_proj", + "model.layers.48.mlp.experts.204.down_proj", + "model.layers.48.mlp.experts.204.gate_proj", + "model.layers.48.mlp.experts.204.up_proj", + "model.layers.48.mlp.experts.205.down_proj", + "model.layers.48.mlp.experts.205.gate_proj", + "model.layers.48.mlp.experts.205.up_proj", + "model.layers.48.mlp.experts.206.down_proj", + "model.layers.48.mlp.experts.206.gate_proj", + "model.layers.48.mlp.experts.206.up_proj", + "model.layers.48.mlp.experts.207.down_proj", + "model.layers.48.mlp.experts.207.gate_proj", + "model.layers.48.mlp.experts.207.up_proj", + "model.layers.48.mlp.experts.208.down_proj", + "model.layers.48.mlp.experts.208.gate_proj", + "model.layers.48.mlp.experts.208.up_proj", + "model.layers.48.mlp.experts.209.down_proj", + "model.layers.48.mlp.experts.209.gate_proj", + "model.layers.48.mlp.experts.209.up_proj", + "model.layers.48.mlp.experts.21.down_proj", + "model.layers.48.mlp.experts.21.gate_proj", + "model.layers.48.mlp.experts.21.up_proj", + "model.layers.48.mlp.experts.210.down_proj", + "model.layers.48.mlp.experts.210.gate_proj", + "model.layers.48.mlp.experts.210.up_proj", + "model.layers.48.mlp.experts.211.down_proj", + "model.layers.48.mlp.experts.211.gate_proj", + "model.layers.48.mlp.experts.211.up_proj", + "model.layers.48.mlp.experts.212.down_proj", + "model.layers.48.mlp.experts.212.gate_proj", + "model.layers.48.mlp.experts.212.up_proj", + "model.layers.48.mlp.experts.213.down_proj", + "model.layers.48.mlp.experts.213.gate_proj", + "model.layers.48.mlp.experts.213.up_proj", + "model.layers.48.mlp.experts.214.down_proj", + "model.layers.48.mlp.experts.214.gate_proj", + "model.layers.48.mlp.experts.214.up_proj", + "model.layers.48.mlp.experts.215.down_proj", + "model.layers.48.mlp.experts.215.gate_proj", + "model.layers.48.mlp.experts.215.up_proj", + "model.layers.48.mlp.experts.216.down_proj", + "model.layers.48.mlp.experts.216.gate_proj", + "model.layers.48.mlp.experts.216.up_proj", + "model.layers.48.mlp.experts.217.down_proj", + "model.layers.48.mlp.experts.217.gate_proj", + "model.layers.48.mlp.experts.217.up_proj", + "model.layers.48.mlp.experts.218.down_proj", + "model.layers.48.mlp.experts.218.gate_proj", + "model.layers.48.mlp.experts.218.up_proj", + "model.layers.48.mlp.experts.219.down_proj", + "model.layers.48.mlp.experts.219.gate_proj", + "model.layers.48.mlp.experts.219.up_proj", + "model.layers.48.mlp.experts.22.down_proj", + "model.layers.48.mlp.experts.22.gate_proj", + "model.layers.48.mlp.experts.22.up_proj", + "model.layers.48.mlp.experts.220.down_proj", + "model.layers.48.mlp.experts.220.gate_proj", + "model.layers.48.mlp.experts.220.up_proj", + "model.layers.48.mlp.experts.221.down_proj", + "model.layers.48.mlp.experts.221.gate_proj", + "model.layers.48.mlp.experts.221.up_proj", + "model.layers.48.mlp.experts.222.down_proj", + "model.layers.48.mlp.experts.222.gate_proj", + "model.layers.48.mlp.experts.222.up_proj", + "model.layers.48.mlp.experts.223.down_proj", + "model.layers.48.mlp.experts.223.gate_proj", + "model.layers.48.mlp.experts.223.up_proj", + "model.layers.48.mlp.experts.224.down_proj", + "model.layers.48.mlp.experts.224.gate_proj", + "model.layers.48.mlp.experts.224.up_proj", + "model.layers.48.mlp.experts.225.down_proj", + "model.layers.48.mlp.experts.225.gate_proj", + "model.layers.48.mlp.experts.225.up_proj", + "model.layers.48.mlp.experts.226.down_proj", + "model.layers.48.mlp.experts.226.gate_proj", + "model.layers.48.mlp.experts.226.up_proj", + "model.layers.48.mlp.experts.227.down_proj", + "model.layers.48.mlp.experts.227.gate_proj", + "model.layers.48.mlp.experts.227.up_proj", + "model.layers.48.mlp.experts.228.down_proj", + "model.layers.48.mlp.experts.228.gate_proj", + "model.layers.48.mlp.experts.228.up_proj", + "model.layers.48.mlp.experts.229.down_proj", + "model.layers.48.mlp.experts.229.gate_proj", + "model.layers.48.mlp.experts.229.up_proj", + "model.layers.48.mlp.experts.23.down_proj", + "model.layers.48.mlp.experts.23.gate_proj", + "model.layers.48.mlp.experts.23.up_proj", + "model.layers.48.mlp.experts.230.down_proj", + "model.layers.48.mlp.experts.230.gate_proj", + "model.layers.48.mlp.experts.230.up_proj", + "model.layers.48.mlp.experts.231.down_proj", + "model.layers.48.mlp.experts.231.gate_proj", + "model.layers.48.mlp.experts.231.up_proj", + "model.layers.48.mlp.experts.232.down_proj", + "model.layers.48.mlp.experts.232.gate_proj", + "model.layers.48.mlp.experts.232.up_proj", + "model.layers.48.mlp.experts.233.down_proj", + "model.layers.48.mlp.experts.233.gate_proj", + "model.layers.48.mlp.experts.233.up_proj", + "model.layers.48.mlp.experts.234.down_proj", + "model.layers.48.mlp.experts.234.gate_proj", + "model.layers.48.mlp.experts.234.up_proj", + "model.layers.48.mlp.experts.235.down_proj", + "model.layers.48.mlp.experts.235.gate_proj", + "model.layers.48.mlp.experts.235.up_proj", + "model.layers.48.mlp.experts.236.down_proj", + "model.layers.48.mlp.experts.236.gate_proj", + "model.layers.48.mlp.experts.236.up_proj", + "model.layers.48.mlp.experts.237.down_proj", + "model.layers.48.mlp.experts.237.gate_proj", + "model.layers.48.mlp.experts.237.up_proj", + "model.layers.48.mlp.experts.238.down_proj", + "model.layers.48.mlp.experts.238.gate_proj", + "model.layers.48.mlp.experts.238.up_proj", + "model.layers.48.mlp.experts.239.down_proj", + "model.layers.48.mlp.experts.239.gate_proj", + "model.layers.48.mlp.experts.239.up_proj", + "model.layers.48.mlp.experts.24.down_proj", + "model.layers.48.mlp.experts.24.gate_proj", + "model.layers.48.mlp.experts.24.up_proj", + "model.layers.48.mlp.experts.240.down_proj", + "model.layers.48.mlp.experts.240.gate_proj", + "model.layers.48.mlp.experts.240.up_proj", + "model.layers.48.mlp.experts.241.down_proj", + "model.layers.48.mlp.experts.241.gate_proj", + "model.layers.48.mlp.experts.241.up_proj", + "model.layers.48.mlp.experts.242.down_proj", + "model.layers.48.mlp.experts.242.gate_proj", + "model.layers.48.mlp.experts.242.up_proj", + "model.layers.48.mlp.experts.243.down_proj", + "model.layers.48.mlp.experts.243.gate_proj", + "model.layers.48.mlp.experts.243.up_proj", + "model.layers.48.mlp.experts.244.down_proj", + "model.layers.48.mlp.experts.244.gate_proj", + "model.layers.48.mlp.experts.244.up_proj", + "model.layers.48.mlp.experts.245.down_proj", + "model.layers.48.mlp.experts.245.gate_proj", + "model.layers.48.mlp.experts.245.up_proj", + "model.layers.48.mlp.experts.246.down_proj", + "model.layers.48.mlp.experts.246.gate_proj", + "model.layers.48.mlp.experts.246.up_proj", + "model.layers.48.mlp.experts.247.down_proj", + "model.layers.48.mlp.experts.247.gate_proj", + "model.layers.48.mlp.experts.247.up_proj", + "model.layers.48.mlp.experts.248.down_proj", + "model.layers.48.mlp.experts.248.gate_proj", + "model.layers.48.mlp.experts.248.up_proj", + "model.layers.48.mlp.experts.249.down_proj", + "model.layers.48.mlp.experts.249.gate_proj", + "model.layers.48.mlp.experts.249.up_proj", + "model.layers.48.mlp.experts.25.down_proj", + "model.layers.48.mlp.experts.25.gate_proj", + "model.layers.48.mlp.experts.25.up_proj", + "model.layers.48.mlp.experts.250.down_proj", + "model.layers.48.mlp.experts.250.gate_proj", + "model.layers.48.mlp.experts.250.up_proj", + "model.layers.48.mlp.experts.251.down_proj", + "model.layers.48.mlp.experts.251.gate_proj", + "model.layers.48.mlp.experts.251.up_proj", + "model.layers.48.mlp.experts.252.down_proj", + "model.layers.48.mlp.experts.252.gate_proj", + "model.layers.48.mlp.experts.252.up_proj", + "model.layers.48.mlp.experts.253.down_proj", + "model.layers.48.mlp.experts.253.gate_proj", + "model.layers.48.mlp.experts.253.up_proj", + "model.layers.48.mlp.experts.254.down_proj", + "model.layers.48.mlp.experts.254.gate_proj", + "model.layers.48.mlp.experts.254.up_proj", + "model.layers.48.mlp.experts.255.down_proj", + "model.layers.48.mlp.experts.255.gate_proj", + "model.layers.48.mlp.experts.255.up_proj", + "model.layers.48.mlp.experts.26.down_proj", + "model.layers.48.mlp.experts.26.gate_proj", + "model.layers.48.mlp.experts.26.up_proj", + "model.layers.48.mlp.experts.27.down_proj", + "model.layers.48.mlp.experts.27.gate_proj", + "model.layers.48.mlp.experts.27.up_proj", + "model.layers.48.mlp.experts.28.down_proj", + "model.layers.48.mlp.experts.28.gate_proj", + "model.layers.48.mlp.experts.28.up_proj", + "model.layers.48.mlp.experts.29.down_proj", + "model.layers.48.mlp.experts.29.gate_proj", + "model.layers.48.mlp.experts.29.up_proj", + "model.layers.48.mlp.experts.3.down_proj", + "model.layers.48.mlp.experts.3.gate_proj", + "model.layers.48.mlp.experts.3.up_proj", + "model.layers.48.mlp.experts.30.down_proj", + "model.layers.48.mlp.experts.30.gate_proj", + "model.layers.48.mlp.experts.30.up_proj", + "model.layers.48.mlp.experts.31.down_proj", + "model.layers.48.mlp.experts.31.gate_proj", + "model.layers.48.mlp.experts.31.up_proj", + "model.layers.48.mlp.experts.32.down_proj", + "model.layers.48.mlp.experts.32.gate_proj", + "model.layers.48.mlp.experts.32.up_proj", + "model.layers.48.mlp.experts.33.down_proj", + "model.layers.48.mlp.experts.33.gate_proj", + "model.layers.48.mlp.experts.33.up_proj", + "model.layers.48.mlp.experts.34.down_proj", + "model.layers.48.mlp.experts.34.gate_proj", + "model.layers.48.mlp.experts.34.up_proj", + "model.layers.48.mlp.experts.35.down_proj", + "model.layers.48.mlp.experts.35.gate_proj", + "model.layers.48.mlp.experts.35.up_proj", + "model.layers.48.mlp.experts.36.down_proj", + "model.layers.48.mlp.experts.36.gate_proj", + "model.layers.48.mlp.experts.36.up_proj", + "model.layers.48.mlp.experts.37.down_proj", + "model.layers.48.mlp.experts.37.gate_proj", + "model.layers.48.mlp.experts.37.up_proj", + "model.layers.48.mlp.experts.38.down_proj", + "model.layers.48.mlp.experts.38.gate_proj", + "model.layers.48.mlp.experts.38.up_proj", + "model.layers.48.mlp.experts.39.down_proj", + "model.layers.48.mlp.experts.39.gate_proj", + "model.layers.48.mlp.experts.39.up_proj", + "model.layers.48.mlp.experts.4.down_proj", + "model.layers.48.mlp.experts.4.gate_proj", + "model.layers.48.mlp.experts.4.up_proj", + "model.layers.48.mlp.experts.40.down_proj", + "model.layers.48.mlp.experts.40.gate_proj", + "model.layers.48.mlp.experts.40.up_proj", + "model.layers.48.mlp.experts.41.down_proj", + "model.layers.48.mlp.experts.41.gate_proj", + "model.layers.48.mlp.experts.41.up_proj", + "model.layers.48.mlp.experts.42.down_proj", + "model.layers.48.mlp.experts.42.gate_proj", + "model.layers.48.mlp.experts.42.up_proj", + "model.layers.48.mlp.experts.43.down_proj", + "model.layers.48.mlp.experts.43.gate_proj", + "model.layers.48.mlp.experts.43.up_proj", + "model.layers.48.mlp.experts.44.down_proj", + "model.layers.48.mlp.experts.44.gate_proj", + "model.layers.48.mlp.experts.44.up_proj", + "model.layers.48.mlp.experts.45.down_proj", + "model.layers.48.mlp.experts.45.gate_proj", + "model.layers.48.mlp.experts.45.up_proj", + "model.layers.48.mlp.experts.46.down_proj", + "model.layers.48.mlp.experts.46.gate_proj", + "model.layers.48.mlp.experts.46.up_proj", + "model.layers.48.mlp.experts.47.down_proj", + "model.layers.48.mlp.experts.47.gate_proj", + "model.layers.48.mlp.experts.47.up_proj", + "model.layers.48.mlp.experts.48.down_proj", + "model.layers.48.mlp.experts.48.gate_proj", + "model.layers.48.mlp.experts.48.up_proj", + "model.layers.48.mlp.experts.49.down_proj", + "model.layers.48.mlp.experts.49.gate_proj", + "model.layers.48.mlp.experts.49.up_proj", + "model.layers.48.mlp.experts.5.down_proj", + "model.layers.48.mlp.experts.5.gate_proj", + "model.layers.48.mlp.experts.5.up_proj", + "model.layers.48.mlp.experts.50.down_proj", + "model.layers.48.mlp.experts.50.gate_proj", + "model.layers.48.mlp.experts.50.up_proj", + "model.layers.48.mlp.experts.51.down_proj", + "model.layers.48.mlp.experts.51.gate_proj", + "model.layers.48.mlp.experts.51.up_proj", + "model.layers.48.mlp.experts.52.down_proj", + "model.layers.48.mlp.experts.52.gate_proj", + "model.layers.48.mlp.experts.52.up_proj", + "model.layers.48.mlp.experts.53.down_proj", + "model.layers.48.mlp.experts.53.gate_proj", + "model.layers.48.mlp.experts.53.up_proj", + "model.layers.48.mlp.experts.54.down_proj", + "model.layers.48.mlp.experts.54.gate_proj", + "model.layers.48.mlp.experts.54.up_proj", + "model.layers.48.mlp.experts.55.down_proj", + "model.layers.48.mlp.experts.55.gate_proj", + "model.layers.48.mlp.experts.55.up_proj", + "model.layers.48.mlp.experts.56.down_proj", + "model.layers.48.mlp.experts.56.gate_proj", + "model.layers.48.mlp.experts.56.up_proj", + "model.layers.48.mlp.experts.57.down_proj", + "model.layers.48.mlp.experts.57.gate_proj", + "model.layers.48.mlp.experts.57.up_proj", + "model.layers.48.mlp.experts.58.down_proj", + "model.layers.48.mlp.experts.58.gate_proj", + "model.layers.48.mlp.experts.58.up_proj", + "model.layers.48.mlp.experts.59.down_proj", + "model.layers.48.mlp.experts.59.gate_proj", + "model.layers.48.mlp.experts.59.up_proj", + "model.layers.48.mlp.experts.6.down_proj", + "model.layers.48.mlp.experts.6.gate_proj", + "model.layers.48.mlp.experts.6.up_proj", + "model.layers.48.mlp.experts.60.down_proj", + "model.layers.48.mlp.experts.60.gate_proj", + "model.layers.48.mlp.experts.60.up_proj", + "model.layers.48.mlp.experts.61.down_proj", + "model.layers.48.mlp.experts.61.gate_proj", + "model.layers.48.mlp.experts.61.up_proj", + "model.layers.48.mlp.experts.62.down_proj", + "model.layers.48.mlp.experts.62.gate_proj", + "model.layers.48.mlp.experts.62.up_proj", + "model.layers.48.mlp.experts.63.down_proj", + "model.layers.48.mlp.experts.63.gate_proj", + "model.layers.48.mlp.experts.63.up_proj", + "model.layers.48.mlp.experts.64.down_proj", + "model.layers.48.mlp.experts.64.gate_proj", + "model.layers.48.mlp.experts.64.up_proj", + "model.layers.48.mlp.experts.65.down_proj", + "model.layers.48.mlp.experts.65.gate_proj", + "model.layers.48.mlp.experts.65.up_proj", + "model.layers.48.mlp.experts.66.down_proj", + "model.layers.48.mlp.experts.66.gate_proj", + "model.layers.48.mlp.experts.66.up_proj", + "model.layers.48.mlp.experts.67.down_proj", + "model.layers.48.mlp.experts.67.gate_proj", + "model.layers.48.mlp.experts.67.up_proj", + "model.layers.48.mlp.experts.68.down_proj", + "model.layers.48.mlp.experts.68.gate_proj", + "model.layers.48.mlp.experts.68.up_proj", + "model.layers.48.mlp.experts.69.down_proj", + "model.layers.48.mlp.experts.69.gate_proj", + "model.layers.48.mlp.experts.69.up_proj", + "model.layers.48.mlp.experts.7.down_proj", + "model.layers.48.mlp.experts.7.gate_proj", + "model.layers.48.mlp.experts.7.up_proj", + "model.layers.48.mlp.experts.70.down_proj", + "model.layers.48.mlp.experts.70.gate_proj", + "model.layers.48.mlp.experts.70.up_proj", + "model.layers.48.mlp.experts.71.down_proj", + "model.layers.48.mlp.experts.71.gate_proj", + "model.layers.48.mlp.experts.71.up_proj", + "model.layers.48.mlp.experts.72.down_proj", + "model.layers.48.mlp.experts.72.gate_proj", + "model.layers.48.mlp.experts.72.up_proj", + "model.layers.48.mlp.experts.73.down_proj", + "model.layers.48.mlp.experts.73.gate_proj", + "model.layers.48.mlp.experts.73.up_proj", + "model.layers.48.mlp.experts.74.down_proj", + "model.layers.48.mlp.experts.74.gate_proj", + "model.layers.48.mlp.experts.74.up_proj", + "model.layers.48.mlp.experts.75.down_proj", + "model.layers.48.mlp.experts.75.gate_proj", + "model.layers.48.mlp.experts.75.up_proj", + "model.layers.48.mlp.experts.76.down_proj", + "model.layers.48.mlp.experts.76.gate_proj", + "model.layers.48.mlp.experts.76.up_proj", + "model.layers.48.mlp.experts.77.down_proj", + "model.layers.48.mlp.experts.77.gate_proj", + "model.layers.48.mlp.experts.77.up_proj", + "model.layers.48.mlp.experts.78.down_proj", + "model.layers.48.mlp.experts.78.gate_proj", + "model.layers.48.mlp.experts.78.up_proj", + "model.layers.48.mlp.experts.79.down_proj", + "model.layers.48.mlp.experts.79.gate_proj", + "model.layers.48.mlp.experts.79.up_proj", + "model.layers.48.mlp.experts.8.down_proj", + "model.layers.48.mlp.experts.8.gate_proj", + "model.layers.48.mlp.experts.8.up_proj", + "model.layers.48.mlp.experts.80.down_proj", + "model.layers.48.mlp.experts.80.gate_proj", + "model.layers.48.mlp.experts.80.up_proj", + "model.layers.48.mlp.experts.81.down_proj", + "model.layers.48.mlp.experts.81.gate_proj", + "model.layers.48.mlp.experts.81.up_proj", + "model.layers.48.mlp.experts.82.down_proj", + "model.layers.48.mlp.experts.82.gate_proj", + "model.layers.48.mlp.experts.82.up_proj", + "model.layers.48.mlp.experts.83.down_proj", + "model.layers.48.mlp.experts.83.gate_proj", + "model.layers.48.mlp.experts.83.up_proj", + "model.layers.48.mlp.experts.84.down_proj", + "model.layers.48.mlp.experts.84.gate_proj", + "model.layers.48.mlp.experts.84.up_proj", + "model.layers.48.mlp.experts.85.down_proj", + "model.layers.48.mlp.experts.85.gate_proj", + "model.layers.48.mlp.experts.85.up_proj", + "model.layers.48.mlp.experts.86.down_proj", + "model.layers.48.mlp.experts.86.gate_proj", + "model.layers.48.mlp.experts.86.up_proj", + "model.layers.48.mlp.experts.87.down_proj", + "model.layers.48.mlp.experts.87.gate_proj", + "model.layers.48.mlp.experts.87.up_proj", + "model.layers.48.mlp.experts.88.down_proj", + "model.layers.48.mlp.experts.88.gate_proj", + "model.layers.48.mlp.experts.88.up_proj", + "model.layers.48.mlp.experts.89.down_proj", + "model.layers.48.mlp.experts.89.gate_proj", + "model.layers.48.mlp.experts.89.up_proj", + "model.layers.48.mlp.experts.9.down_proj", + "model.layers.48.mlp.experts.9.gate_proj", + "model.layers.48.mlp.experts.9.up_proj", + "model.layers.48.mlp.experts.90.down_proj", + "model.layers.48.mlp.experts.90.gate_proj", + "model.layers.48.mlp.experts.90.up_proj", + "model.layers.48.mlp.experts.91.down_proj", + "model.layers.48.mlp.experts.91.gate_proj", + "model.layers.48.mlp.experts.91.up_proj", + "model.layers.48.mlp.experts.92.down_proj", + "model.layers.48.mlp.experts.92.gate_proj", + "model.layers.48.mlp.experts.92.up_proj", + "model.layers.48.mlp.experts.93.down_proj", + "model.layers.48.mlp.experts.93.gate_proj", + "model.layers.48.mlp.experts.93.up_proj", + "model.layers.48.mlp.experts.94.down_proj", + "model.layers.48.mlp.experts.94.gate_proj", + "model.layers.48.mlp.experts.94.up_proj", + "model.layers.48.mlp.experts.95.down_proj", + "model.layers.48.mlp.experts.95.gate_proj", + "model.layers.48.mlp.experts.95.up_proj", + "model.layers.48.mlp.experts.96.down_proj", + "model.layers.48.mlp.experts.96.gate_proj", + "model.layers.48.mlp.experts.96.up_proj", + "model.layers.48.mlp.experts.97.down_proj", + "model.layers.48.mlp.experts.97.gate_proj", + "model.layers.48.mlp.experts.97.up_proj", + "model.layers.48.mlp.experts.98.down_proj", + "model.layers.48.mlp.experts.98.gate_proj", + "model.layers.48.mlp.experts.98.up_proj", + "model.layers.48.mlp.experts.99.down_proj", + "model.layers.48.mlp.experts.99.gate_proj", + "model.layers.48.mlp.experts.99.up_proj", + "model.layers.49.mlp.experts.0.down_proj", + "model.layers.49.mlp.experts.0.gate_proj", + "model.layers.49.mlp.experts.0.up_proj", + "model.layers.49.mlp.experts.1.down_proj", + "model.layers.49.mlp.experts.1.gate_proj", + "model.layers.49.mlp.experts.1.up_proj", + "model.layers.49.mlp.experts.10.down_proj", + "model.layers.49.mlp.experts.10.gate_proj", + "model.layers.49.mlp.experts.10.up_proj", + "model.layers.49.mlp.experts.100.down_proj", + "model.layers.49.mlp.experts.100.gate_proj", + "model.layers.49.mlp.experts.100.up_proj", + "model.layers.49.mlp.experts.101.down_proj", + "model.layers.49.mlp.experts.101.gate_proj", + "model.layers.49.mlp.experts.101.up_proj", + "model.layers.49.mlp.experts.102.down_proj", + "model.layers.49.mlp.experts.102.gate_proj", + "model.layers.49.mlp.experts.102.up_proj", + "model.layers.49.mlp.experts.103.down_proj", + "model.layers.49.mlp.experts.103.gate_proj", + "model.layers.49.mlp.experts.103.up_proj", + "model.layers.49.mlp.experts.104.down_proj", + "model.layers.49.mlp.experts.104.gate_proj", + "model.layers.49.mlp.experts.104.up_proj", + "model.layers.49.mlp.experts.105.down_proj", + "model.layers.49.mlp.experts.105.gate_proj", + "model.layers.49.mlp.experts.105.up_proj", + "model.layers.49.mlp.experts.106.down_proj", + "model.layers.49.mlp.experts.106.gate_proj", + "model.layers.49.mlp.experts.106.up_proj", + "model.layers.49.mlp.experts.107.down_proj", + "model.layers.49.mlp.experts.107.gate_proj", + "model.layers.49.mlp.experts.107.up_proj", + "model.layers.49.mlp.experts.108.down_proj", + "model.layers.49.mlp.experts.108.gate_proj", + "model.layers.49.mlp.experts.108.up_proj", + "model.layers.49.mlp.experts.109.down_proj", + "model.layers.49.mlp.experts.109.gate_proj", + "model.layers.49.mlp.experts.109.up_proj", + "model.layers.49.mlp.experts.11.down_proj", + "model.layers.49.mlp.experts.11.gate_proj", + "model.layers.49.mlp.experts.11.up_proj", + "model.layers.49.mlp.experts.110.down_proj", + "model.layers.49.mlp.experts.110.gate_proj", + "model.layers.49.mlp.experts.110.up_proj", + "model.layers.49.mlp.experts.111.down_proj", + "model.layers.49.mlp.experts.111.gate_proj", + "model.layers.49.mlp.experts.111.up_proj", + "model.layers.49.mlp.experts.112.down_proj", + "model.layers.49.mlp.experts.112.gate_proj", + "model.layers.49.mlp.experts.112.up_proj", + "model.layers.49.mlp.experts.113.down_proj", + "model.layers.49.mlp.experts.113.gate_proj", + "model.layers.49.mlp.experts.113.up_proj", + "model.layers.49.mlp.experts.114.down_proj", + "model.layers.49.mlp.experts.114.gate_proj", + "model.layers.49.mlp.experts.114.up_proj", + "model.layers.49.mlp.experts.115.down_proj", + "model.layers.49.mlp.experts.115.gate_proj", + "model.layers.49.mlp.experts.115.up_proj", + "model.layers.49.mlp.experts.116.down_proj", + "model.layers.49.mlp.experts.116.gate_proj", + "model.layers.49.mlp.experts.116.up_proj", + "model.layers.49.mlp.experts.117.down_proj", + "model.layers.49.mlp.experts.117.gate_proj", + "model.layers.49.mlp.experts.117.up_proj", + "model.layers.49.mlp.experts.118.down_proj", + "model.layers.49.mlp.experts.118.gate_proj", + "model.layers.49.mlp.experts.118.up_proj", + "model.layers.49.mlp.experts.119.down_proj", + "model.layers.49.mlp.experts.119.gate_proj", + "model.layers.49.mlp.experts.119.up_proj", + "model.layers.49.mlp.experts.12.down_proj", + "model.layers.49.mlp.experts.12.gate_proj", + "model.layers.49.mlp.experts.12.up_proj", + "model.layers.49.mlp.experts.120.down_proj", + "model.layers.49.mlp.experts.120.gate_proj", + "model.layers.49.mlp.experts.120.up_proj", + "model.layers.49.mlp.experts.121.down_proj", + "model.layers.49.mlp.experts.121.gate_proj", + "model.layers.49.mlp.experts.121.up_proj", + "model.layers.49.mlp.experts.122.down_proj", + "model.layers.49.mlp.experts.122.gate_proj", + "model.layers.49.mlp.experts.122.up_proj", + "model.layers.49.mlp.experts.123.down_proj", + "model.layers.49.mlp.experts.123.gate_proj", + "model.layers.49.mlp.experts.123.up_proj", + "model.layers.49.mlp.experts.124.down_proj", + "model.layers.49.mlp.experts.124.gate_proj", + "model.layers.49.mlp.experts.124.up_proj", + "model.layers.49.mlp.experts.125.down_proj", + "model.layers.49.mlp.experts.125.gate_proj", + "model.layers.49.mlp.experts.125.up_proj", + "model.layers.49.mlp.experts.126.down_proj", + "model.layers.49.mlp.experts.126.gate_proj", + "model.layers.49.mlp.experts.126.up_proj", + "model.layers.49.mlp.experts.127.down_proj", + "model.layers.49.mlp.experts.127.gate_proj", + "model.layers.49.mlp.experts.127.up_proj", + "model.layers.49.mlp.experts.128.down_proj", + "model.layers.49.mlp.experts.128.gate_proj", + "model.layers.49.mlp.experts.128.up_proj", + "model.layers.49.mlp.experts.129.down_proj", + "model.layers.49.mlp.experts.129.gate_proj", + "model.layers.49.mlp.experts.129.up_proj", + "model.layers.49.mlp.experts.13.down_proj", + "model.layers.49.mlp.experts.13.gate_proj", + "model.layers.49.mlp.experts.13.up_proj", + "model.layers.49.mlp.experts.130.down_proj", + "model.layers.49.mlp.experts.130.gate_proj", + "model.layers.49.mlp.experts.130.up_proj", + "model.layers.49.mlp.experts.131.down_proj", + "model.layers.49.mlp.experts.131.gate_proj", + "model.layers.49.mlp.experts.131.up_proj", + "model.layers.49.mlp.experts.132.down_proj", + "model.layers.49.mlp.experts.132.gate_proj", + "model.layers.49.mlp.experts.132.up_proj", + "model.layers.49.mlp.experts.133.down_proj", + "model.layers.49.mlp.experts.133.gate_proj", + "model.layers.49.mlp.experts.133.up_proj", + "model.layers.49.mlp.experts.134.down_proj", + "model.layers.49.mlp.experts.134.gate_proj", + "model.layers.49.mlp.experts.134.up_proj", + "model.layers.49.mlp.experts.135.down_proj", + "model.layers.49.mlp.experts.135.gate_proj", + "model.layers.49.mlp.experts.135.up_proj", + "model.layers.49.mlp.experts.136.down_proj", + "model.layers.49.mlp.experts.136.gate_proj", + "model.layers.49.mlp.experts.136.up_proj", + "model.layers.49.mlp.experts.137.down_proj", + "model.layers.49.mlp.experts.137.gate_proj", + "model.layers.49.mlp.experts.137.up_proj", + "model.layers.49.mlp.experts.138.down_proj", + "model.layers.49.mlp.experts.138.gate_proj", + "model.layers.49.mlp.experts.138.up_proj", + "model.layers.49.mlp.experts.139.down_proj", + "model.layers.49.mlp.experts.139.gate_proj", + "model.layers.49.mlp.experts.139.up_proj", + "model.layers.49.mlp.experts.14.down_proj", + "model.layers.49.mlp.experts.14.gate_proj", + "model.layers.49.mlp.experts.14.up_proj", + "model.layers.49.mlp.experts.140.down_proj", + "model.layers.49.mlp.experts.140.gate_proj", + "model.layers.49.mlp.experts.140.up_proj", + "model.layers.49.mlp.experts.141.down_proj", + "model.layers.49.mlp.experts.141.gate_proj", + "model.layers.49.mlp.experts.141.up_proj", + "model.layers.49.mlp.experts.142.down_proj", + "model.layers.49.mlp.experts.142.gate_proj", + "model.layers.49.mlp.experts.142.up_proj", + "model.layers.49.mlp.experts.143.down_proj", + "model.layers.49.mlp.experts.143.gate_proj", + "model.layers.49.mlp.experts.143.up_proj", + "model.layers.49.mlp.experts.144.down_proj", + "model.layers.49.mlp.experts.144.gate_proj", + "model.layers.49.mlp.experts.144.up_proj", + "model.layers.49.mlp.experts.145.down_proj", + "model.layers.49.mlp.experts.145.gate_proj", + "model.layers.49.mlp.experts.145.up_proj", + "model.layers.49.mlp.experts.146.down_proj", + "model.layers.49.mlp.experts.146.gate_proj", + "model.layers.49.mlp.experts.146.up_proj", + "model.layers.49.mlp.experts.147.down_proj", + "model.layers.49.mlp.experts.147.gate_proj", + "model.layers.49.mlp.experts.147.up_proj", + "model.layers.49.mlp.experts.148.down_proj", + "model.layers.49.mlp.experts.148.gate_proj", + "model.layers.49.mlp.experts.148.up_proj", + "model.layers.49.mlp.experts.149.down_proj", + "model.layers.49.mlp.experts.149.gate_proj", + "model.layers.49.mlp.experts.149.up_proj", + "model.layers.49.mlp.experts.15.down_proj", + "model.layers.49.mlp.experts.15.gate_proj", + "model.layers.49.mlp.experts.15.up_proj", + "model.layers.49.mlp.experts.150.down_proj", + "model.layers.49.mlp.experts.150.gate_proj", + "model.layers.49.mlp.experts.150.up_proj", + "model.layers.49.mlp.experts.151.down_proj", + "model.layers.49.mlp.experts.151.gate_proj", + "model.layers.49.mlp.experts.151.up_proj", + "model.layers.49.mlp.experts.152.down_proj", + "model.layers.49.mlp.experts.152.gate_proj", + "model.layers.49.mlp.experts.152.up_proj", + "model.layers.49.mlp.experts.153.down_proj", + "model.layers.49.mlp.experts.153.gate_proj", + "model.layers.49.mlp.experts.153.up_proj", + "model.layers.49.mlp.experts.154.down_proj", + "model.layers.49.mlp.experts.154.gate_proj", + "model.layers.49.mlp.experts.154.up_proj", + "model.layers.49.mlp.experts.155.down_proj", + "model.layers.49.mlp.experts.155.gate_proj", + "model.layers.49.mlp.experts.155.up_proj", + "model.layers.49.mlp.experts.156.down_proj", + "model.layers.49.mlp.experts.156.gate_proj", + "model.layers.49.mlp.experts.156.up_proj", + "model.layers.49.mlp.experts.157.down_proj", + "model.layers.49.mlp.experts.157.gate_proj", + "model.layers.49.mlp.experts.157.up_proj", + "model.layers.49.mlp.experts.158.down_proj", + "model.layers.49.mlp.experts.158.gate_proj", + "model.layers.49.mlp.experts.158.up_proj", + "model.layers.49.mlp.experts.159.down_proj", + "model.layers.49.mlp.experts.159.gate_proj", + "model.layers.49.mlp.experts.159.up_proj", + "model.layers.49.mlp.experts.16.down_proj", + "model.layers.49.mlp.experts.16.gate_proj", + "model.layers.49.mlp.experts.16.up_proj", + "model.layers.49.mlp.experts.160.down_proj", + "model.layers.49.mlp.experts.160.gate_proj", + "model.layers.49.mlp.experts.160.up_proj", + "model.layers.49.mlp.experts.161.down_proj", + "model.layers.49.mlp.experts.161.gate_proj", + "model.layers.49.mlp.experts.161.up_proj", + "model.layers.49.mlp.experts.162.down_proj", + "model.layers.49.mlp.experts.162.gate_proj", + "model.layers.49.mlp.experts.162.up_proj", + "model.layers.49.mlp.experts.163.down_proj", + "model.layers.49.mlp.experts.163.gate_proj", + "model.layers.49.mlp.experts.163.up_proj", + "model.layers.49.mlp.experts.164.down_proj", + "model.layers.49.mlp.experts.164.gate_proj", + "model.layers.49.mlp.experts.164.up_proj", + "model.layers.49.mlp.experts.165.down_proj", + "model.layers.49.mlp.experts.165.gate_proj", + "model.layers.49.mlp.experts.165.up_proj", + "model.layers.49.mlp.experts.166.down_proj", + "model.layers.49.mlp.experts.166.gate_proj", + "model.layers.49.mlp.experts.166.up_proj", + "model.layers.49.mlp.experts.167.down_proj", + "model.layers.49.mlp.experts.167.gate_proj", + "model.layers.49.mlp.experts.167.up_proj", + "model.layers.49.mlp.experts.168.down_proj", + "model.layers.49.mlp.experts.168.gate_proj", + "model.layers.49.mlp.experts.168.up_proj", + "model.layers.49.mlp.experts.169.down_proj", + "model.layers.49.mlp.experts.169.gate_proj", + "model.layers.49.mlp.experts.169.up_proj", + "model.layers.49.mlp.experts.17.down_proj", + "model.layers.49.mlp.experts.17.gate_proj", + "model.layers.49.mlp.experts.17.up_proj", + "model.layers.49.mlp.experts.170.down_proj", + "model.layers.49.mlp.experts.170.gate_proj", + "model.layers.49.mlp.experts.170.up_proj", + "model.layers.49.mlp.experts.171.down_proj", + "model.layers.49.mlp.experts.171.gate_proj", + "model.layers.49.mlp.experts.171.up_proj", + "model.layers.49.mlp.experts.172.down_proj", + "model.layers.49.mlp.experts.172.gate_proj", + "model.layers.49.mlp.experts.172.up_proj", + "model.layers.49.mlp.experts.173.down_proj", + "model.layers.49.mlp.experts.173.gate_proj", + "model.layers.49.mlp.experts.173.up_proj", + "model.layers.49.mlp.experts.174.down_proj", + "model.layers.49.mlp.experts.174.gate_proj", + "model.layers.49.mlp.experts.174.up_proj", + "model.layers.49.mlp.experts.175.down_proj", + "model.layers.49.mlp.experts.175.gate_proj", + "model.layers.49.mlp.experts.175.up_proj", + "model.layers.49.mlp.experts.176.down_proj", + "model.layers.49.mlp.experts.176.gate_proj", + "model.layers.49.mlp.experts.176.up_proj", + "model.layers.49.mlp.experts.177.down_proj", + "model.layers.49.mlp.experts.177.gate_proj", + "model.layers.49.mlp.experts.177.up_proj", + "model.layers.49.mlp.experts.178.down_proj", + "model.layers.49.mlp.experts.178.gate_proj", + "model.layers.49.mlp.experts.178.up_proj", + "model.layers.49.mlp.experts.179.down_proj", + "model.layers.49.mlp.experts.179.gate_proj", + "model.layers.49.mlp.experts.179.up_proj", + "model.layers.49.mlp.experts.18.down_proj", + "model.layers.49.mlp.experts.18.gate_proj", + "model.layers.49.mlp.experts.18.up_proj", + "model.layers.49.mlp.experts.180.down_proj", + "model.layers.49.mlp.experts.180.gate_proj", + "model.layers.49.mlp.experts.180.up_proj", + "model.layers.49.mlp.experts.181.down_proj", + "model.layers.49.mlp.experts.181.gate_proj", + "model.layers.49.mlp.experts.181.up_proj", + "model.layers.49.mlp.experts.182.down_proj", + "model.layers.49.mlp.experts.182.gate_proj", + "model.layers.49.mlp.experts.182.up_proj", + "model.layers.49.mlp.experts.183.down_proj", + "model.layers.49.mlp.experts.183.gate_proj", + "model.layers.49.mlp.experts.183.up_proj", + "model.layers.49.mlp.experts.184.down_proj", + "model.layers.49.mlp.experts.184.gate_proj", + "model.layers.49.mlp.experts.184.up_proj", + "model.layers.49.mlp.experts.185.down_proj", + "model.layers.49.mlp.experts.185.gate_proj", + "model.layers.49.mlp.experts.185.up_proj", + "model.layers.49.mlp.experts.186.down_proj", + "model.layers.49.mlp.experts.186.gate_proj", + "model.layers.49.mlp.experts.186.up_proj", + "model.layers.49.mlp.experts.187.down_proj", + "model.layers.49.mlp.experts.187.gate_proj", + "model.layers.49.mlp.experts.187.up_proj", + "model.layers.49.mlp.experts.188.down_proj", + "model.layers.49.mlp.experts.188.gate_proj", + "model.layers.49.mlp.experts.188.up_proj", + "model.layers.49.mlp.experts.189.down_proj", + "model.layers.49.mlp.experts.189.gate_proj", + "model.layers.49.mlp.experts.189.up_proj", + "model.layers.49.mlp.experts.19.down_proj", + "model.layers.49.mlp.experts.19.gate_proj", + "model.layers.49.mlp.experts.19.up_proj", + "model.layers.49.mlp.experts.190.down_proj", + "model.layers.49.mlp.experts.190.gate_proj", + "model.layers.49.mlp.experts.190.up_proj", + "model.layers.49.mlp.experts.191.down_proj", + "model.layers.49.mlp.experts.191.gate_proj", + "model.layers.49.mlp.experts.191.up_proj", + "model.layers.49.mlp.experts.192.down_proj", + "model.layers.49.mlp.experts.192.gate_proj", + "model.layers.49.mlp.experts.192.up_proj", + "model.layers.49.mlp.experts.193.down_proj", + "model.layers.49.mlp.experts.193.gate_proj", + "model.layers.49.mlp.experts.193.up_proj", + "model.layers.49.mlp.experts.194.down_proj", + "model.layers.49.mlp.experts.194.gate_proj", + "model.layers.49.mlp.experts.194.up_proj", + "model.layers.49.mlp.experts.195.down_proj", + "model.layers.49.mlp.experts.195.gate_proj", + "model.layers.49.mlp.experts.195.up_proj", + "model.layers.49.mlp.experts.196.down_proj", + "model.layers.49.mlp.experts.196.gate_proj", + "model.layers.49.mlp.experts.196.up_proj", + "model.layers.49.mlp.experts.197.down_proj", + "model.layers.49.mlp.experts.197.gate_proj", + "model.layers.49.mlp.experts.197.up_proj", + "model.layers.49.mlp.experts.198.down_proj", + "model.layers.49.mlp.experts.198.gate_proj", + "model.layers.49.mlp.experts.198.up_proj", + "model.layers.49.mlp.experts.199.down_proj", + "model.layers.49.mlp.experts.199.gate_proj", + "model.layers.49.mlp.experts.199.up_proj", + "model.layers.49.mlp.experts.2.down_proj", + "model.layers.49.mlp.experts.2.gate_proj", + "model.layers.49.mlp.experts.2.up_proj", + "model.layers.49.mlp.experts.20.down_proj", + "model.layers.49.mlp.experts.20.gate_proj", + "model.layers.49.mlp.experts.20.up_proj", + "model.layers.49.mlp.experts.200.down_proj", + "model.layers.49.mlp.experts.200.gate_proj", + "model.layers.49.mlp.experts.200.up_proj", + "model.layers.49.mlp.experts.201.down_proj", + "model.layers.49.mlp.experts.201.gate_proj", + "model.layers.49.mlp.experts.201.up_proj", + "model.layers.49.mlp.experts.202.down_proj", + "model.layers.49.mlp.experts.202.gate_proj", + "model.layers.49.mlp.experts.202.up_proj", + "model.layers.49.mlp.experts.203.down_proj", + "model.layers.49.mlp.experts.203.gate_proj", + "model.layers.49.mlp.experts.203.up_proj", + "model.layers.49.mlp.experts.204.down_proj", + "model.layers.49.mlp.experts.204.gate_proj", + "model.layers.49.mlp.experts.204.up_proj", + "model.layers.49.mlp.experts.205.down_proj", + "model.layers.49.mlp.experts.205.gate_proj", + "model.layers.49.mlp.experts.205.up_proj", + "model.layers.49.mlp.experts.206.down_proj", + "model.layers.49.mlp.experts.206.gate_proj", + "model.layers.49.mlp.experts.206.up_proj", + "model.layers.49.mlp.experts.207.down_proj", + "model.layers.49.mlp.experts.207.gate_proj", + "model.layers.49.mlp.experts.207.up_proj", + "model.layers.49.mlp.experts.208.down_proj", + "model.layers.49.mlp.experts.208.gate_proj", + "model.layers.49.mlp.experts.208.up_proj", + "model.layers.49.mlp.experts.209.down_proj", + "model.layers.49.mlp.experts.209.gate_proj", + "model.layers.49.mlp.experts.209.up_proj", + "model.layers.49.mlp.experts.21.down_proj", + "model.layers.49.mlp.experts.21.gate_proj", + "model.layers.49.mlp.experts.21.up_proj", + "model.layers.49.mlp.experts.210.down_proj", + "model.layers.49.mlp.experts.210.gate_proj", + "model.layers.49.mlp.experts.210.up_proj", + "model.layers.49.mlp.experts.211.down_proj", + "model.layers.49.mlp.experts.211.gate_proj", + "model.layers.49.mlp.experts.211.up_proj", + "model.layers.49.mlp.experts.212.down_proj", + "model.layers.49.mlp.experts.212.gate_proj", + "model.layers.49.mlp.experts.212.up_proj", + "model.layers.49.mlp.experts.213.down_proj", + "model.layers.49.mlp.experts.213.gate_proj", + "model.layers.49.mlp.experts.213.up_proj", + "model.layers.49.mlp.experts.214.down_proj", + "model.layers.49.mlp.experts.214.gate_proj", + "model.layers.49.mlp.experts.214.up_proj", + "model.layers.49.mlp.experts.215.down_proj", + "model.layers.49.mlp.experts.215.gate_proj", + "model.layers.49.mlp.experts.215.up_proj", + "model.layers.49.mlp.experts.216.down_proj", + "model.layers.49.mlp.experts.216.gate_proj", + "model.layers.49.mlp.experts.216.up_proj", + "model.layers.49.mlp.experts.217.down_proj", + "model.layers.49.mlp.experts.217.gate_proj", + "model.layers.49.mlp.experts.217.up_proj", + "model.layers.49.mlp.experts.218.down_proj", + "model.layers.49.mlp.experts.218.gate_proj", + "model.layers.49.mlp.experts.218.up_proj", + "model.layers.49.mlp.experts.219.down_proj", + "model.layers.49.mlp.experts.219.gate_proj", + "model.layers.49.mlp.experts.219.up_proj", + "model.layers.49.mlp.experts.22.down_proj", + "model.layers.49.mlp.experts.22.gate_proj", + "model.layers.49.mlp.experts.22.up_proj", + "model.layers.49.mlp.experts.220.down_proj", + "model.layers.49.mlp.experts.220.gate_proj", + "model.layers.49.mlp.experts.220.up_proj", + "model.layers.49.mlp.experts.221.down_proj", + "model.layers.49.mlp.experts.221.gate_proj", + "model.layers.49.mlp.experts.221.up_proj", + "model.layers.49.mlp.experts.222.down_proj", + "model.layers.49.mlp.experts.222.gate_proj", + "model.layers.49.mlp.experts.222.up_proj", + "model.layers.49.mlp.experts.223.down_proj", + "model.layers.49.mlp.experts.223.gate_proj", + "model.layers.49.mlp.experts.223.up_proj", + "model.layers.49.mlp.experts.224.down_proj", + "model.layers.49.mlp.experts.224.gate_proj", + "model.layers.49.mlp.experts.224.up_proj", + "model.layers.49.mlp.experts.225.down_proj", + "model.layers.49.mlp.experts.225.gate_proj", + "model.layers.49.mlp.experts.225.up_proj", + "model.layers.49.mlp.experts.226.down_proj", + "model.layers.49.mlp.experts.226.gate_proj", + "model.layers.49.mlp.experts.226.up_proj", + "model.layers.49.mlp.experts.227.down_proj", + "model.layers.49.mlp.experts.227.gate_proj", + "model.layers.49.mlp.experts.227.up_proj", + "model.layers.49.mlp.experts.228.down_proj", + "model.layers.49.mlp.experts.228.gate_proj", + "model.layers.49.mlp.experts.228.up_proj", + "model.layers.49.mlp.experts.229.down_proj", + "model.layers.49.mlp.experts.229.gate_proj", + "model.layers.49.mlp.experts.229.up_proj", + "model.layers.49.mlp.experts.23.down_proj", + "model.layers.49.mlp.experts.23.gate_proj", + "model.layers.49.mlp.experts.23.up_proj", + "model.layers.49.mlp.experts.230.down_proj", + "model.layers.49.mlp.experts.230.gate_proj", + "model.layers.49.mlp.experts.230.up_proj", + "model.layers.49.mlp.experts.231.down_proj", + "model.layers.49.mlp.experts.231.gate_proj", + "model.layers.49.mlp.experts.231.up_proj", + "model.layers.49.mlp.experts.232.down_proj", + "model.layers.49.mlp.experts.232.gate_proj", + "model.layers.49.mlp.experts.232.up_proj", + "model.layers.49.mlp.experts.233.down_proj", + "model.layers.49.mlp.experts.233.gate_proj", + "model.layers.49.mlp.experts.233.up_proj", + "model.layers.49.mlp.experts.234.down_proj", + "model.layers.49.mlp.experts.234.gate_proj", + "model.layers.49.mlp.experts.234.up_proj", + "model.layers.49.mlp.experts.235.down_proj", + "model.layers.49.mlp.experts.235.gate_proj", + "model.layers.49.mlp.experts.235.up_proj", + "model.layers.49.mlp.experts.236.down_proj", + "model.layers.49.mlp.experts.236.gate_proj", + "model.layers.49.mlp.experts.236.up_proj", + "model.layers.49.mlp.experts.237.down_proj", + "model.layers.49.mlp.experts.237.gate_proj", + "model.layers.49.mlp.experts.237.up_proj", + "model.layers.49.mlp.experts.238.down_proj", + "model.layers.49.mlp.experts.238.gate_proj", + "model.layers.49.mlp.experts.238.up_proj", + "model.layers.49.mlp.experts.239.down_proj", + "model.layers.49.mlp.experts.239.gate_proj", + "model.layers.49.mlp.experts.239.up_proj", + "model.layers.49.mlp.experts.24.down_proj", + "model.layers.49.mlp.experts.24.gate_proj", + "model.layers.49.mlp.experts.24.up_proj", + "model.layers.49.mlp.experts.240.down_proj", + "model.layers.49.mlp.experts.240.gate_proj", + "model.layers.49.mlp.experts.240.up_proj", + "model.layers.49.mlp.experts.241.down_proj", + "model.layers.49.mlp.experts.241.gate_proj", + "model.layers.49.mlp.experts.241.up_proj", + "model.layers.49.mlp.experts.242.down_proj", + "model.layers.49.mlp.experts.242.gate_proj", + "model.layers.49.mlp.experts.242.up_proj", + "model.layers.49.mlp.experts.243.down_proj", + "model.layers.49.mlp.experts.243.gate_proj", + "model.layers.49.mlp.experts.243.up_proj", + "model.layers.49.mlp.experts.244.down_proj", + "model.layers.49.mlp.experts.244.gate_proj", + "model.layers.49.mlp.experts.244.up_proj", + "model.layers.49.mlp.experts.245.down_proj", + "model.layers.49.mlp.experts.245.gate_proj", + "model.layers.49.mlp.experts.245.up_proj", + "model.layers.49.mlp.experts.246.down_proj", + "model.layers.49.mlp.experts.246.gate_proj", + "model.layers.49.mlp.experts.246.up_proj", + "model.layers.49.mlp.experts.247.down_proj", + "model.layers.49.mlp.experts.247.gate_proj", + "model.layers.49.mlp.experts.247.up_proj", + "model.layers.49.mlp.experts.248.down_proj", + "model.layers.49.mlp.experts.248.gate_proj", + "model.layers.49.mlp.experts.248.up_proj", + "model.layers.49.mlp.experts.249.down_proj", + "model.layers.49.mlp.experts.249.gate_proj", + "model.layers.49.mlp.experts.249.up_proj", + "model.layers.49.mlp.experts.25.down_proj", + "model.layers.49.mlp.experts.25.gate_proj", + "model.layers.49.mlp.experts.25.up_proj", + "model.layers.49.mlp.experts.250.down_proj", + "model.layers.49.mlp.experts.250.gate_proj", + "model.layers.49.mlp.experts.250.up_proj", + "model.layers.49.mlp.experts.251.down_proj", + "model.layers.49.mlp.experts.251.gate_proj", + "model.layers.49.mlp.experts.251.up_proj", + "model.layers.49.mlp.experts.252.down_proj", + "model.layers.49.mlp.experts.252.gate_proj", + "model.layers.49.mlp.experts.252.up_proj", + "model.layers.49.mlp.experts.253.down_proj", + "model.layers.49.mlp.experts.253.gate_proj", + "model.layers.49.mlp.experts.253.up_proj", + "model.layers.49.mlp.experts.254.down_proj", + "model.layers.49.mlp.experts.254.gate_proj", + "model.layers.49.mlp.experts.254.up_proj", + "model.layers.49.mlp.experts.255.down_proj", + "model.layers.49.mlp.experts.255.gate_proj", + "model.layers.49.mlp.experts.255.up_proj", + "model.layers.49.mlp.experts.26.down_proj", + "model.layers.49.mlp.experts.26.gate_proj", + "model.layers.49.mlp.experts.26.up_proj", + "model.layers.49.mlp.experts.27.down_proj", + "model.layers.49.mlp.experts.27.gate_proj", + "model.layers.49.mlp.experts.27.up_proj", + "model.layers.49.mlp.experts.28.down_proj", + "model.layers.49.mlp.experts.28.gate_proj", + "model.layers.49.mlp.experts.28.up_proj", + "model.layers.49.mlp.experts.29.down_proj", + "model.layers.49.mlp.experts.29.gate_proj", + "model.layers.49.mlp.experts.29.up_proj", + "model.layers.49.mlp.experts.3.down_proj", + "model.layers.49.mlp.experts.3.gate_proj", + "model.layers.49.mlp.experts.3.up_proj", + "model.layers.49.mlp.experts.30.down_proj", + "model.layers.49.mlp.experts.30.gate_proj", + "model.layers.49.mlp.experts.30.up_proj", + "model.layers.49.mlp.experts.31.down_proj", + "model.layers.49.mlp.experts.31.gate_proj", + "model.layers.49.mlp.experts.31.up_proj", + "model.layers.49.mlp.experts.32.down_proj", + "model.layers.49.mlp.experts.32.gate_proj", + "model.layers.49.mlp.experts.32.up_proj", + "model.layers.49.mlp.experts.33.down_proj", + "model.layers.49.mlp.experts.33.gate_proj", + "model.layers.49.mlp.experts.33.up_proj", + "model.layers.49.mlp.experts.34.down_proj", + "model.layers.49.mlp.experts.34.gate_proj", + "model.layers.49.mlp.experts.34.up_proj", + "model.layers.49.mlp.experts.35.down_proj", + "model.layers.49.mlp.experts.35.gate_proj", + "model.layers.49.mlp.experts.35.up_proj", + "model.layers.49.mlp.experts.36.down_proj", + "model.layers.49.mlp.experts.36.gate_proj", + "model.layers.49.mlp.experts.36.up_proj", + "model.layers.49.mlp.experts.37.down_proj", + "model.layers.49.mlp.experts.37.gate_proj", + "model.layers.49.mlp.experts.37.up_proj", + "model.layers.49.mlp.experts.38.down_proj", + "model.layers.49.mlp.experts.38.gate_proj", + "model.layers.49.mlp.experts.38.up_proj", + "model.layers.49.mlp.experts.39.down_proj", + "model.layers.49.mlp.experts.39.gate_proj", + "model.layers.49.mlp.experts.39.up_proj", + "model.layers.49.mlp.experts.4.down_proj", + "model.layers.49.mlp.experts.4.gate_proj", + "model.layers.49.mlp.experts.4.up_proj", + "model.layers.49.mlp.experts.40.down_proj", + "model.layers.49.mlp.experts.40.gate_proj", + "model.layers.49.mlp.experts.40.up_proj", + "model.layers.49.mlp.experts.41.down_proj", + "model.layers.49.mlp.experts.41.gate_proj", + "model.layers.49.mlp.experts.41.up_proj", + "model.layers.49.mlp.experts.42.down_proj", + "model.layers.49.mlp.experts.42.gate_proj", + "model.layers.49.mlp.experts.42.up_proj", + "model.layers.49.mlp.experts.43.down_proj", + "model.layers.49.mlp.experts.43.gate_proj", + "model.layers.49.mlp.experts.43.up_proj", + "model.layers.49.mlp.experts.44.down_proj", + "model.layers.49.mlp.experts.44.gate_proj", + "model.layers.49.mlp.experts.44.up_proj", + "model.layers.49.mlp.experts.45.down_proj", + "model.layers.49.mlp.experts.45.gate_proj", + "model.layers.49.mlp.experts.45.up_proj", + "model.layers.49.mlp.experts.46.down_proj", + "model.layers.49.mlp.experts.46.gate_proj", + "model.layers.49.mlp.experts.46.up_proj", + "model.layers.49.mlp.experts.47.down_proj", + "model.layers.49.mlp.experts.47.gate_proj", + "model.layers.49.mlp.experts.47.up_proj", + "model.layers.49.mlp.experts.48.down_proj", + "model.layers.49.mlp.experts.48.gate_proj", + "model.layers.49.mlp.experts.48.up_proj", + "model.layers.49.mlp.experts.49.down_proj", + "model.layers.49.mlp.experts.49.gate_proj", + "model.layers.49.mlp.experts.49.up_proj", + "model.layers.49.mlp.experts.5.down_proj", + "model.layers.49.mlp.experts.5.gate_proj", + "model.layers.49.mlp.experts.5.up_proj", + "model.layers.49.mlp.experts.50.down_proj", + "model.layers.49.mlp.experts.50.gate_proj", + "model.layers.49.mlp.experts.50.up_proj", + "model.layers.49.mlp.experts.51.down_proj", + "model.layers.49.mlp.experts.51.gate_proj", + "model.layers.49.mlp.experts.51.up_proj", + "model.layers.49.mlp.experts.52.down_proj", + "model.layers.49.mlp.experts.52.gate_proj", + "model.layers.49.mlp.experts.52.up_proj", + "model.layers.49.mlp.experts.53.down_proj", + "model.layers.49.mlp.experts.53.gate_proj", + "model.layers.49.mlp.experts.53.up_proj", + "model.layers.49.mlp.experts.54.down_proj", + "model.layers.49.mlp.experts.54.gate_proj", + "model.layers.49.mlp.experts.54.up_proj", + "model.layers.49.mlp.experts.55.down_proj", + "model.layers.49.mlp.experts.55.gate_proj", + "model.layers.49.mlp.experts.55.up_proj", + "model.layers.49.mlp.experts.56.down_proj", + "model.layers.49.mlp.experts.56.gate_proj", + "model.layers.49.mlp.experts.56.up_proj", + "model.layers.49.mlp.experts.57.down_proj", + "model.layers.49.mlp.experts.57.gate_proj", + "model.layers.49.mlp.experts.57.up_proj", + "model.layers.49.mlp.experts.58.down_proj", + "model.layers.49.mlp.experts.58.gate_proj", + "model.layers.49.mlp.experts.58.up_proj", + "model.layers.49.mlp.experts.59.down_proj", + "model.layers.49.mlp.experts.59.gate_proj", + "model.layers.49.mlp.experts.59.up_proj", + "model.layers.49.mlp.experts.6.down_proj", + "model.layers.49.mlp.experts.6.gate_proj", + "model.layers.49.mlp.experts.6.up_proj", + "model.layers.49.mlp.experts.60.down_proj", + "model.layers.49.mlp.experts.60.gate_proj", + "model.layers.49.mlp.experts.60.up_proj", + "model.layers.49.mlp.experts.61.down_proj", + "model.layers.49.mlp.experts.61.gate_proj", + "model.layers.49.mlp.experts.61.up_proj", + "model.layers.49.mlp.experts.62.down_proj", + "model.layers.49.mlp.experts.62.gate_proj", + "model.layers.49.mlp.experts.62.up_proj", + "model.layers.49.mlp.experts.63.down_proj", + "model.layers.49.mlp.experts.63.gate_proj", + "model.layers.49.mlp.experts.63.up_proj", + "model.layers.49.mlp.experts.64.down_proj", + "model.layers.49.mlp.experts.64.gate_proj", + "model.layers.49.mlp.experts.64.up_proj", + "model.layers.49.mlp.experts.65.down_proj", + "model.layers.49.mlp.experts.65.gate_proj", + "model.layers.49.mlp.experts.65.up_proj", + "model.layers.49.mlp.experts.66.down_proj", + "model.layers.49.mlp.experts.66.gate_proj", + "model.layers.49.mlp.experts.66.up_proj", + "model.layers.49.mlp.experts.67.down_proj", + "model.layers.49.mlp.experts.67.gate_proj", + "model.layers.49.mlp.experts.67.up_proj", + "model.layers.49.mlp.experts.68.down_proj", + "model.layers.49.mlp.experts.68.gate_proj", + "model.layers.49.mlp.experts.68.up_proj", + "model.layers.49.mlp.experts.69.down_proj", + "model.layers.49.mlp.experts.69.gate_proj", + "model.layers.49.mlp.experts.69.up_proj", + "model.layers.49.mlp.experts.7.down_proj", + "model.layers.49.mlp.experts.7.gate_proj", + "model.layers.49.mlp.experts.7.up_proj", + "model.layers.49.mlp.experts.70.down_proj", + "model.layers.49.mlp.experts.70.gate_proj", + "model.layers.49.mlp.experts.70.up_proj", + "model.layers.49.mlp.experts.71.down_proj", + "model.layers.49.mlp.experts.71.gate_proj", + "model.layers.49.mlp.experts.71.up_proj", + "model.layers.49.mlp.experts.72.down_proj", + "model.layers.49.mlp.experts.72.gate_proj", + "model.layers.49.mlp.experts.72.up_proj", + "model.layers.49.mlp.experts.73.down_proj", + "model.layers.49.mlp.experts.73.gate_proj", + "model.layers.49.mlp.experts.73.up_proj", + "model.layers.49.mlp.experts.74.down_proj", + "model.layers.49.mlp.experts.74.gate_proj", + "model.layers.49.mlp.experts.74.up_proj", + "model.layers.49.mlp.experts.75.down_proj", + "model.layers.49.mlp.experts.75.gate_proj", + "model.layers.49.mlp.experts.75.up_proj", + "model.layers.49.mlp.experts.76.down_proj", + "model.layers.49.mlp.experts.76.gate_proj", + "model.layers.49.mlp.experts.76.up_proj", + "model.layers.49.mlp.experts.77.down_proj", + "model.layers.49.mlp.experts.77.gate_proj", + "model.layers.49.mlp.experts.77.up_proj", + "model.layers.49.mlp.experts.78.down_proj", + "model.layers.49.mlp.experts.78.gate_proj", + "model.layers.49.mlp.experts.78.up_proj", + "model.layers.49.mlp.experts.79.down_proj", + "model.layers.49.mlp.experts.79.gate_proj", + "model.layers.49.mlp.experts.79.up_proj", + "model.layers.49.mlp.experts.8.down_proj", + "model.layers.49.mlp.experts.8.gate_proj", + "model.layers.49.mlp.experts.8.up_proj", + "model.layers.49.mlp.experts.80.down_proj", + "model.layers.49.mlp.experts.80.gate_proj", + "model.layers.49.mlp.experts.80.up_proj", + "model.layers.49.mlp.experts.81.down_proj", + "model.layers.49.mlp.experts.81.gate_proj", + "model.layers.49.mlp.experts.81.up_proj", + "model.layers.49.mlp.experts.82.down_proj", + "model.layers.49.mlp.experts.82.gate_proj", + "model.layers.49.mlp.experts.82.up_proj", + "model.layers.49.mlp.experts.83.down_proj", + "model.layers.49.mlp.experts.83.gate_proj", + "model.layers.49.mlp.experts.83.up_proj", + "model.layers.49.mlp.experts.84.down_proj", + "model.layers.49.mlp.experts.84.gate_proj", + "model.layers.49.mlp.experts.84.up_proj", + "model.layers.49.mlp.experts.85.down_proj", + "model.layers.49.mlp.experts.85.gate_proj", + "model.layers.49.mlp.experts.85.up_proj", + "model.layers.49.mlp.experts.86.down_proj", + "model.layers.49.mlp.experts.86.gate_proj", + "model.layers.49.mlp.experts.86.up_proj", + "model.layers.49.mlp.experts.87.down_proj", + "model.layers.49.mlp.experts.87.gate_proj", + "model.layers.49.mlp.experts.87.up_proj", + "model.layers.49.mlp.experts.88.down_proj", + "model.layers.49.mlp.experts.88.gate_proj", + "model.layers.49.mlp.experts.88.up_proj", + "model.layers.49.mlp.experts.89.down_proj", + "model.layers.49.mlp.experts.89.gate_proj", + "model.layers.49.mlp.experts.89.up_proj", + "model.layers.49.mlp.experts.9.down_proj", + "model.layers.49.mlp.experts.9.gate_proj", + "model.layers.49.mlp.experts.9.up_proj", + "model.layers.49.mlp.experts.90.down_proj", + "model.layers.49.mlp.experts.90.gate_proj", + "model.layers.49.mlp.experts.90.up_proj", + "model.layers.49.mlp.experts.91.down_proj", + "model.layers.49.mlp.experts.91.gate_proj", + "model.layers.49.mlp.experts.91.up_proj", + "model.layers.49.mlp.experts.92.down_proj", + "model.layers.49.mlp.experts.92.gate_proj", + "model.layers.49.mlp.experts.92.up_proj", + "model.layers.49.mlp.experts.93.down_proj", + "model.layers.49.mlp.experts.93.gate_proj", + "model.layers.49.mlp.experts.93.up_proj", + "model.layers.49.mlp.experts.94.down_proj", + "model.layers.49.mlp.experts.94.gate_proj", + "model.layers.49.mlp.experts.94.up_proj", + "model.layers.49.mlp.experts.95.down_proj", + "model.layers.49.mlp.experts.95.gate_proj", + "model.layers.49.mlp.experts.95.up_proj", + "model.layers.49.mlp.experts.96.down_proj", + "model.layers.49.mlp.experts.96.gate_proj", + "model.layers.49.mlp.experts.96.up_proj", + "model.layers.49.mlp.experts.97.down_proj", + "model.layers.49.mlp.experts.97.gate_proj", + "model.layers.49.mlp.experts.97.up_proj", + "model.layers.49.mlp.experts.98.down_proj", + "model.layers.49.mlp.experts.98.gate_proj", + "model.layers.49.mlp.experts.98.up_proj", + "model.layers.49.mlp.experts.99.down_proj", + "model.layers.49.mlp.experts.99.gate_proj", + "model.layers.49.mlp.experts.99.up_proj", + "model.layers.5.mlp.experts.0.down_proj", + "model.layers.5.mlp.experts.0.gate_proj", + "model.layers.5.mlp.experts.0.up_proj", + "model.layers.5.mlp.experts.1.down_proj", + "model.layers.5.mlp.experts.1.gate_proj", + "model.layers.5.mlp.experts.1.up_proj", + "model.layers.5.mlp.experts.10.down_proj", + "model.layers.5.mlp.experts.10.gate_proj", + "model.layers.5.mlp.experts.10.up_proj", + "model.layers.5.mlp.experts.100.down_proj", + "model.layers.5.mlp.experts.100.gate_proj", + "model.layers.5.mlp.experts.100.up_proj", + "model.layers.5.mlp.experts.101.down_proj", + "model.layers.5.mlp.experts.101.gate_proj", + "model.layers.5.mlp.experts.101.up_proj", + "model.layers.5.mlp.experts.102.down_proj", + "model.layers.5.mlp.experts.102.gate_proj", + "model.layers.5.mlp.experts.102.up_proj", + "model.layers.5.mlp.experts.103.down_proj", + "model.layers.5.mlp.experts.103.gate_proj", + "model.layers.5.mlp.experts.103.up_proj", + "model.layers.5.mlp.experts.104.down_proj", + "model.layers.5.mlp.experts.104.gate_proj", + "model.layers.5.mlp.experts.104.up_proj", + "model.layers.5.mlp.experts.105.down_proj", + "model.layers.5.mlp.experts.105.gate_proj", + "model.layers.5.mlp.experts.105.up_proj", + "model.layers.5.mlp.experts.106.down_proj", + "model.layers.5.mlp.experts.106.gate_proj", + "model.layers.5.mlp.experts.106.up_proj", + "model.layers.5.mlp.experts.107.down_proj", + "model.layers.5.mlp.experts.107.gate_proj", + "model.layers.5.mlp.experts.107.up_proj", + "model.layers.5.mlp.experts.108.down_proj", + "model.layers.5.mlp.experts.108.gate_proj", + "model.layers.5.mlp.experts.108.up_proj", + "model.layers.5.mlp.experts.109.down_proj", + "model.layers.5.mlp.experts.109.gate_proj", + "model.layers.5.mlp.experts.109.up_proj", + "model.layers.5.mlp.experts.11.down_proj", + "model.layers.5.mlp.experts.11.gate_proj", + "model.layers.5.mlp.experts.11.up_proj", + "model.layers.5.mlp.experts.110.down_proj", + "model.layers.5.mlp.experts.110.gate_proj", + "model.layers.5.mlp.experts.110.up_proj", + "model.layers.5.mlp.experts.111.down_proj", + "model.layers.5.mlp.experts.111.gate_proj", + "model.layers.5.mlp.experts.111.up_proj", + "model.layers.5.mlp.experts.112.down_proj", + "model.layers.5.mlp.experts.112.gate_proj", + "model.layers.5.mlp.experts.112.up_proj", + "model.layers.5.mlp.experts.113.down_proj", + "model.layers.5.mlp.experts.113.gate_proj", + "model.layers.5.mlp.experts.113.up_proj", + "model.layers.5.mlp.experts.114.down_proj", + "model.layers.5.mlp.experts.114.gate_proj", + "model.layers.5.mlp.experts.114.up_proj", + "model.layers.5.mlp.experts.115.down_proj", + "model.layers.5.mlp.experts.115.gate_proj", + "model.layers.5.mlp.experts.115.up_proj", + "model.layers.5.mlp.experts.116.down_proj", + "model.layers.5.mlp.experts.116.gate_proj", + "model.layers.5.mlp.experts.116.up_proj", + "model.layers.5.mlp.experts.117.down_proj", + "model.layers.5.mlp.experts.117.gate_proj", + "model.layers.5.mlp.experts.117.up_proj", + "model.layers.5.mlp.experts.118.down_proj", + "model.layers.5.mlp.experts.118.gate_proj", + "model.layers.5.mlp.experts.118.up_proj", + "model.layers.5.mlp.experts.119.down_proj", + "model.layers.5.mlp.experts.119.gate_proj", + "model.layers.5.mlp.experts.119.up_proj", + "model.layers.5.mlp.experts.12.down_proj", + "model.layers.5.mlp.experts.12.gate_proj", + "model.layers.5.mlp.experts.12.up_proj", + "model.layers.5.mlp.experts.120.down_proj", + "model.layers.5.mlp.experts.120.gate_proj", + "model.layers.5.mlp.experts.120.up_proj", + "model.layers.5.mlp.experts.121.down_proj", + "model.layers.5.mlp.experts.121.gate_proj", + "model.layers.5.mlp.experts.121.up_proj", + "model.layers.5.mlp.experts.122.down_proj", + "model.layers.5.mlp.experts.122.gate_proj", + "model.layers.5.mlp.experts.122.up_proj", + "model.layers.5.mlp.experts.123.down_proj", + "model.layers.5.mlp.experts.123.gate_proj", + "model.layers.5.mlp.experts.123.up_proj", + "model.layers.5.mlp.experts.124.down_proj", + "model.layers.5.mlp.experts.124.gate_proj", + "model.layers.5.mlp.experts.124.up_proj", + "model.layers.5.mlp.experts.125.down_proj", + "model.layers.5.mlp.experts.125.gate_proj", + "model.layers.5.mlp.experts.125.up_proj", + "model.layers.5.mlp.experts.126.down_proj", + "model.layers.5.mlp.experts.126.gate_proj", + "model.layers.5.mlp.experts.126.up_proj", + "model.layers.5.mlp.experts.127.down_proj", + "model.layers.5.mlp.experts.127.gate_proj", + "model.layers.5.mlp.experts.127.up_proj", + "model.layers.5.mlp.experts.128.down_proj", + "model.layers.5.mlp.experts.128.gate_proj", + "model.layers.5.mlp.experts.128.up_proj", + "model.layers.5.mlp.experts.129.down_proj", + "model.layers.5.mlp.experts.129.gate_proj", + "model.layers.5.mlp.experts.129.up_proj", + "model.layers.5.mlp.experts.13.down_proj", + "model.layers.5.mlp.experts.13.gate_proj", + "model.layers.5.mlp.experts.13.up_proj", + "model.layers.5.mlp.experts.130.down_proj", + "model.layers.5.mlp.experts.130.gate_proj", + "model.layers.5.mlp.experts.130.up_proj", + "model.layers.5.mlp.experts.131.down_proj", + "model.layers.5.mlp.experts.131.gate_proj", + "model.layers.5.mlp.experts.131.up_proj", + "model.layers.5.mlp.experts.132.down_proj", + "model.layers.5.mlp.experts.132.gate_proj", + "model.layers.5.mlp.experts.132.up_proj", + "model.layers.5.mlp.experts.133.down_proj", + "model.layers.5.mlp.experts.133.gate_proj", + "model.layers.5.mlp.experts.133.up_proj", + "model.layers.5.mlp.experts.134.down_proj", + "model.layers.5.mlp.experts.134.gate_proj", + "model.layers.5.mlp.experts.134.up_proj", + "model.layers.5.mlp.experts.135.down_proj", + "model.layers.5.mlp.experts.135.gate_proj", + "model.layers.5.mlp.experts.135.up_proj", + "model.layers.5.mlp.experts.136.down_proj", + "model.layers.5.mlp.experts.136.gate_proj", + "model.layers.5.mlp.experts.136.up_proj", + "model.layers.5.mlp.experts.137.down_proj", + "model.layers.5.mlp.experts.137.gate_proj", + "model.layers.5.mlp.experts.137.up_proj", + "model.layers.5.mlp.experts.138.down_proj", + "model.layers.5.mlp.experts.138.gate_proj", + "model.layers.5.mlp.experts.138.up_proj", + "model.layers.5.mlp.experts.139.down_proj", + "model.layers.5.mlp.experts.139.gate_proj", + "model.layers.5.mlp.experts.139.up_proj", + "model.layers.5.mlp.experts.14.down_proj", + "model.layers.5.mlp.experts.14.gate_proj", + "model.layers.5.mlp.experts.14.up_proj", + "model.layers.5.mlp.experts.140.down_proj", + "model.layers.5.mlp.experts.140.gate_proj", + "model.layers.5.mlp.experts.140.up_proj", + "model.layers.5.mlp.experts.141.down_proj", + "model.layers.5.mlp.experts.141.gate_proj", + "model.layers.5.mlp.experts.141.up_proj", + "model.layers.5.mlp.experts.142.down_proj", + "model.layers.5.mlp.experts.142.gate_proj", + "model.layers.5.mlp.experts.142.up_proj", + "model.layers.5.mlp.experts.143.down_proj", + "model.layers.5.mlp.experts.143.gate_proj", + "model.layers.5.mlp.experts.143.up_proj", + "model.layers.5.mlp.experts.144.down_proj", + "model.layers.5.mlp.experts.144.gate_proj", + "model.layers.5.mlp.experts.144.up_proj", + "model.layers.5.mlp.experts.145.down_proj", + "model.layers.5.mlp.experts.145.gate_proj", + "model.layers.5.mlp.experts.145.up_proj", + "model.layers.5.mlp.experts.146.down_proj", + "model.layers.5.mlp.experts.146.gate_proj", + "model.layers.5.mlp.experts.146.up_proj", + "model.layers.5.mlp.experts.147.down_proj", + "model.layers.5.mlp.experts.147.gate_proj", + "model.layers.5.mlp.experts.147.up_proj", + "model.layers.5.mlp.experts.148.down_proj", + "model.layers.5.mlp.experts.148.gate_proj", + "model.layers.5.mlp.experts.148.up_proj", + "model.layers.5.mlp.experts.149.down_proj", + "model.layers.5.mlp.experts.149.gate_proj", + "model.layers.5.mlp.experts.149.up_proj", + "model.layers.5.mlp.experts.15.down_proj", + "model.layers.5.mlp.experts.15.gate_proj", + "model.layers.5.mlp.experts.15.up_proj", + "model.layers.5.mlp.experts.150.down_proj", + "model.layers.5.mlp.experts.150.gate_proj", + "model.layers.5.mlp.experts.150.up_proj", + "model.layers.5.mlp.experts.151.down_proj", + "model.layers.5.mlp.experts.151.gate_proj", + "model.layers.5.mlp.experts.151.up_proj", + "model.layers.5.mlp.experts.152.down_proj", + "model.layers.5.mlp.experts.152.gate_proj", + "model.layers.5.mlp.experts.152.up_proj", + "model.layers.5.mlp.experts.153.down_proj", + "model.layers.5.mlp.experts.153.gate_proj", + "model.layers.5.mlp.experts.153.up_proj", + "model.layers.5.mlp.experts.154.down_proj", + "model.layers.5.mlp.experts.154.gate_proj", + "model.layers.5.mlp.experts.154.up_proj", + "model.layers.5.mlp.experts.155.down_proj", + "model.layers.5.mlp.experts.155.gate_proj", + "model.layers.5.mlp.experts.155.up_proj", + "model.layers.5.mlp.experts.156.down_proj", + "model.layers.5.mlp.experts.156.gate_proj", + "model.layers.5.mlp.experts.156.up_proj", + "model.layers.5.mlp.experts.157.down_proj", + "model.layers.5.mlp.experts.157.gate_proj", + "model.layers.5.mlp.experts.157.up_proj", + "model.layers.5.mlp.experts.158.down_proj", + "model.layers.5.mlp.experts.158.gate_proj", + "model.layers.5.mlp.experts.158.up_proj", + "model.layers.5.mlp.experts.159.down_proj", + "model.layers.5.mlp.experts.159.gate_proj", + "model.layers.5.mlp.experts.159.up_proj", + "model.layers.5.mlp.experts.16.down_proj", + "model.layers.5.mlp.experts.16.gate_proj", + "model.layers.5.mlp.experts.16.up_proj", + "model.layers.5.mlp.experts.160.down_proj", + "model.layers.5.mlp.experts.160.gate_proj", + "model.layers.5.mlp.experts.160.up_proj", + "model.layers.5.mlp.experts.161.down_proj", + "model.layers.5.mlp.experts.161.gate_proj", + "model.layers.5.mlp.experts.161.up_proj", + "model.layers.5.mlp.experts.162.down_proj", + "model.layers.5.mlp.experts.162.gate_proj", + "model.layers.5.mlp.experts.162.up_proj", + "model.layers.5.mlp.experts.163.down_proj", + "model.layers.5.mlp.experts.163.gate_proj", + "model.layers.5.mlp.experts.163.up_proj", + "model.layers.5.mlp.experts.164.down_proj", + "model.layers.5.mlp.experts.164.gate_proj", + "model.layers.5.mlp.experts.164.up_proj", + "model.layers.5.mlp.experts.165.down_proj", + "model.layers.5.mlp.experts.165.gate_proj", + "model.layers.5.mlp.experts.165.up_proj", + "model.layers.5.mlp.experts.166.down_proj", + "model.layers.5.mlp.experts.166.gate_proj", + "model.layers.5.mlp.experts.166.up_proj", + "model.layers.5.mlp.experts.167.down_proj", + "model.layers.5.mlp.experts.167.gate_proj", + "model.layers.5.mlp.experts.167.up_proj", + "model.layers.5.mlp.experts.168.down_proj", + "model.layers.5.mlp.experts.168.gate_proj", + "model.layers.5.mlp.experts.168.up_proj", + "model.layers.5.mlp.experts.169.down_proj", + "model.layers.5.mlp.experts.169.gate_proj", + "model.layers.5.mlp.experts.169.up_proj", + "model.layers.5.mlp.experts.17.down_proj", + "model.layers.5.mlp.experts.17.gate_proj", + "model.layers.5.mlp.experts.17.up_proj", + "model.layers.5.mlp.experts.170.down_proj", + "model.layers.5.mlp.experts.170.gate_proj", + "model.layers.5.mlp.experts.170.up_proj", + "model.layers.5.mlp.experts.171.down_proj", + "model.layers.5.mlp.experts.171.gate_proj", + "model.layers.5.mlp.experts.171.up_proj", + "model.layers.5.mlp.experts.172.down_proj", + "model.layers.5.mlp.experts.172.gate_proj", + "model.layers.5.mlp.experts.172.up_proj", + "model.layers.5.mlp.experts.173.down_proj", + "model.layers.5.mlp.experts.173.gate_proj", + "model.layers.5.mlp.experts.173.up_proj", + "model.layers.5.mlp.experts.174.down_proj", + "model.layers.5.mlp.experts.174.gate_proj", + "model.layers.5.mlp.experts.174.up_proj", + "model.layers.5.mlp.experts.175.down_proj", + "model.layers.5.mlp.experts.175.gate_proj", + "model.layers.5.mlp.experts.175.up_proj", + "model.layers.5.mlp.experts.176.down_proj", + "model.layers.5.mlp.experts.176.gate_proj", + "model.layers.5.mlp.experts.176.up_proj", + "model.layers.5.mlp.experts.177.down_proj", + "model.layers.5.mlp.experts.177.gate_proj", + "model.layers.5.mlp.experts.177.up_proj", + "model.layers.5.mlp.experts.178.down_proj", + "model.layers.5.mlp.experts.178.gate_proj", + "model.layers.5.mlp.experts.178.up_proj", + "model.layers.5.mlp.experts.179.down_proj", + "model.layers.5.mlp.experts.179.gate_proj", + "model.layers.5.mlp.experts.179.up_proj", + "model.layers.5.mlp.experts.18.down_proj", + "model.layers.5.mlp.experts.18.gate_proj", + "model.layers.5.mlp.experts.18.up_proj", + "model.layers.5.mlp.experts.180.down_proj", + "model.layers.5.mlp.experts.180.gate_proj", + "model.layers.5.mlp.experts.180.up_proj", + "model.layers.5.mlp.experts.181.down_proj", + "model.layers.5.mlp.experts.181.gate_proj", + "model.layers.5.mlp.experts.181.up_proj", + "model.layers.5.mlp.experts.182.down_proj", + "model.layers.5.mlp.experts.182.gate_proj", + "model.layers.5.mlp.experts.182.up_proj", + "model.layers.5.mlp.experts.183.down_proj", + "model.layers.5.mlp.experts.183.gate_proj", + "model.layers.5.mlp.experts.183.up_proj", + "model.layers.5.mlp.experts.184.down_proj", + "model.layers.5.mlp.experts.184.gate_proj", + "model.layers.5.mlp.experts.184.up_proj", + "model.layers.5.mlp.experts.185.down_proj", + "model.layers.5.mlp.experts.185.gate_proj", + "model.layers.5.mlp.experts.185.up_proj", + "model.layers.5.mlp.experts.186.down_proj", + "model.layers.5.mlp.experts.186.gate_proj", + "model.layers.5.mlp.experts.186.up_proj", + "model.layers.5.mlp.experts.187.down_proj", + "model.layers.5.mlp.experts.187.gate_proj", + "model.layers.5.mlp.experts.187.up_proj", + "model.layers.5.mlp.experts.188.down_proj", + "model.layers.5.mlp.experts.188.gate_proj", + "model.layers.5.mlp.experts.188.up_proj", + "model.layers.5.mlp.experts.189.down_proj", + "model.layers.5.mlp.experts.189.gate_proj", + "model.layers.5.mlp.experts.189.up_proj", + "model.layers.5.mlp.experts.19.down_proj", + "model.layers.5.mlp.experts.19.gate_proj", + "model.layers.5.mlp.experts.19.up_proj", + "model.layers.5.mlp.experts.190.down_proj", + "model.layers.5.mlp.experts.190.gate_proj", + "model.layers.5.mlp.experts.190.up_proj", + "model.layers.5.mlp.experts.191.down_proj", + "model.layers.5.mlp.experts.191.gate_proj", + "model.layers.5.mlp.experts.191.up_proj", + "model.layers.5.mlp.experts.192.down_proj", + "model.layers.5.mlp.experts.192.gate_proj", + "model.layers.5.mlp.experts.192.up_proj", + "model.layers.5.mlp.experts.193.down_proj", + "model.layers.5.mlp.experts.193.gate_proj", + "model.layers.5.mlp.experts.193.up_proj", + "model.layers.5.mlp.experts.194.down_proj", + "model.layers.5.mlp.experts.194.gate_proj", + "model.layers.5.mlp.experts.194.up_proj", + "model.layers.5.mlp.experts.195.down_proj", + "model.layers.5.mlp.experts.195.gate_proj", + "model.layers.5.mlp.experts.195.up_proj", + "model.layers.5.mlp.experts.196.down_proj", + "model.layers.5.mlp.experts.196.gate_proj", + "model.layers.5.mlp.experts.196.up_proj", + "model.layers.5.mlp.experts.197.down_proj", + "model.layers.5.mlp.experts.197.gate_proj", + "model.layers.5.mlp.experts.197.up_proj", + "model.layers.5.mlp.experts.198.down_proj", + "model.layers.5.mlp.experts.198.gate_proj", + "model.layers.5.mlp.experts.198.up_proj", + "model.layers.5.mlp.experts.199.down_proj", + "model.layers.5.mlp.experts.199.gate_proj", + "model.layers.5.mlp.experts.199.up_proj", + "model.layers.5.mlp.experts.2.down_proj", + "model.layers.5.mlp.experts.2.gate_proj", + "model.layers.5.mlp.experts.2.up_proj", + "model.layers.5.mlp.experts.20.down_proj", + "model.layers.5.mlp.experts.20.gate_proj", + "model.layers.5.mlp.experts.20.up_proj", + "model.layers.5.mlp.experts.200.down_proj", + "model.layers.5.mlp.experts.200.gate_proj", + "model.layers.5.mlp.experts.200.up_proj", + "model.layers.5.mlp.experts.201.down_proj", + "model.layers.5.mlp.experts.201.gate_proj", + "model.layers.5.mlp.experts.201.up_proj", + "model.layers.5.mlp.experts.202.down_proj", + "model.layers.5.mlp.experts.202.gate_proj", + "model.layers.5.mlp.experts.202.up_proj", + "model.layers.5.mlp.experts.203.down_proj", + "model.layers.5.mlp.experts.203.gate_proj", + "model.layers.5.mlp.experts.203.up_proj", + "model.layers.5.mlp.experts.204.down_proj", + "model.layers.5.mlp.experts.204.gate_proj", + "model.layers.5.mlp.experts.204.up_proj", + "model.layers.5.mlp.experts.205.down_proj", + "model.layers.5.mlp.experts.205.gate_proj", + "model.layers.5.mlp.experts.205.up_proj", + "model.layers.5.mlp.experts.206.down_proj", + "model.layers.5.mlp.experts.206.gate_proj", + "model.layers.5.mlp.experts.206.up_proj", + "model.layers.5.mlp.experts.207.down_proj", + "model.layers.5.mlp.experts.207.gate_proj", + "model.layers.5.mlp.experts.207.up_proj", + "model.layers.5.mlp.experts.208.down_proj", + "model.layers.5.mlp.experts.208.gate_proj", + "model.layers.5.mlp.experts.208.up_proj", + "model.layers.5.mlp.experts.209.down_proj", + "model.layers.5.mlp.experts.209.gate_proj", + "model.layers.5.mlp.experts.209.up_proj", + "model.layers.5.mlp.experts.21.down_proj", + "model.layers.5.mlp.experts.21.gate_proj", + "model.layers.5.mlp.experts.21.up_proj", + "model.layers.5.mlp.experts.210.down_proj", + "model.layers.5.mlp.experts.210.gate_proj", + "model.layers.5.mlp.experts.210.up_proj", + "model.layers.5.mlp.experts.211.down_proj", + "model.layers.5.mlp.experts.211.gate_proj", + "model.layers.5.mlp.experts.211.up_proj", + "model.layers.5.mlp.experts.212.down_proj", + "model.layers.5.mlp.experts.212.gate_proj", + "model.layers.5.mlp.experts.212.up_proj", + "model.layers.5.mlp.experts.213.down_proj", + "model.layers.5.mlp.experts.213.gate_proj", + "model.layers.5.mlp.experts.213.up_proj", + "model.layers.5.mlp.experts.214.down_proj", + "model.layers.5.mlp.experts.214.gate_proj", + "model.layers.5.mlp.experts.214.up_proj", + "model.layers.5.mlp.experts.215.down_proj", + "model.layers.5.mlp.experts.215.gate_proj", + "model.layers.5.mlp.experts.215.up_proj", + "model.layers.5.mlp.experts.216.down_proj", + "model.layers.5.mlp.experts.216.gate_proj", + "model.layers.5.mlp.experts.216.up_proj", + "model.layers.5.mlp.experts.217.down_proj", + "model.layers.5.mlp.experts.217.gate_proj", + "model.layers.5.mlp.experts.217.up_proj", + "model.layers.5.mlp.experts.218.down_proj", + "model.layers.5.mlp.experts.218.gate_proj", + "model.layers.5.mlp.experts.218.up_proj", + "model.layers.5.mlp.experts.219.down_proj", + "model.layers.5.mlp.experts.219.gate_proj", + "model.layers.5.mlp.experts.219.up_proj", + "model.layers.5.mlp.experts.22.down_proj", + "model.layers.5.mlp.experts.22.gate_proj", + "model.layers.5.mlp.experts.22.up_proj", + "model.layers.5.mlp.experts.220.down_proj", + "model.layers.5.mlp.experts.220.gate_proj", + "model.layers.5.mlp.experts.220.up_proj", + "model.layers.5.mlp.experts.221.down_proj", + "model.layers.5.mlp.experts.221.gate_proj", + "model.layers.5.mlp.experts.221.up_proj", + "model.layers.5.mlp.experts.222.down_proj", + "model.layers.5.mlp.experts.222.gate_proj", + "model.layers.5.mlp.experts.222.up_proj", + "model.layers.5.mlp.experts.223.down_proj", + "model.layers.5.mlp.experts.223.gate_proj", + "model.layers.5.mlp.experts.223.up_proj", + "model.layers.5.mlp.experts.224.down_proj", + "model.layers.5.mlp.experts.224.gate_proj", + "model.layers.5.mlp.experts.224.up_proj", + "model.layers.5.mlp.experts.225.down_proj", + "model.layers.5.mlp.experts.225.gate_proj", + "model.layers.5.mlp.experts.225.up_proj", + "model.layers.5.mlp.experts.226.down_proj", + "model.layers.5.mlp.experts.226.gate_proj", + "model.layers.5.mlp.experts.226.up_proj", + "model.layers.5.mlp.experts.227.down_proj", + "model.layers.5.mlp.experts.227.gate_proj", + "model.layers.5.mlp.experts.227.up_proj", + "model.layers.5.mlp.experts.228.down_proj", + "model.layers.5.mlp.experts.228.gate_proj", + "model.layers.5.mlp.experts.228.up_proj", + "model.layers.5.mlp.experts.229.down_proj", + "model.layers.5.mlp.experts.229.gate_proj", + "model.layers.5.mlp.experts.229.up_proj", + "model.layers.5.mlp.experts.23.down_proj", + "model.layers.5.mlp.experts.23.gate_proj", + "model.layers.5.mlp.experts.23.up_proj", + "model.layers.5.mlp.experts.230.down_proj", + "model.layers.5.mlp.experts.230.gate_proj", + "model.layers.5.mlp.experts.230.up_proj", + "model.layers.5.mlp.experts.231.down_proj", + "model.layers.5.mlp.experts.231.gate_proj", + "model.layers.5.mlp.experts.231.up_proj", + "model.layers.5.mlp.experts.232.down_proj", + "model.layers.5.mlp.experts.232.gate_proj", + "model.layers.5.mlp.experts.232.up_proj", + "model.layers.5.mlp.experts.233.down_proj", + "model.layers.5.mlp.experts.233.gate_proj", + "model.layers.5.mlp.experts.233.up_proj", + "model.layers.5.mlp.experts.234.down_proj", + "model.layers.5.mlp.experts.234.gate_proj", + "model.layers.5.mlp.experts.234.up_proj", + "model.layers.5.mlp.experts.235.down_proj", + "model.layers.5.mlp.experts.235.gate_proj", + "model.layers.5.mlp.experts.235.up_proj", + "model.layers.5.mlp.experts.236.down_proj", + "model.layers.5.mlp.experts.236.gate_proj", + "model.layers.5.mlp.experts.236.up_proj", + "model.layers.5.mlp.experts.237.down_proj", + "model.layers.5.mlp.experts.237.gate_proj", + "model.layers.5.mlp.experts.237.up_proj", + "model.layers.5.mlp.experts.238.down_proj", + "model.layers.5.mlp.experts.238.gate_proj", + "model.layers.5.mlp.experts.238.up_proj", + "model.layers.5.mlp.experts.239.down_proj", + "model.layers.5.mlp.experts.239.gate_proj", + "model.layers.5.mlp.experts.239.up_proj", + "model.layers.5.mlp.experts.24.down_proj", + "model.layers.5.mlp.experts.24.gate_proj", + "model.layers.5.mlp.experts.24.up_proj", + "model.layers.5.mlp.experts.240.down_proj", + "model.layers.5.mlp.experts.240.gate_proj", + "model.layers.5.mlp.experts.240.up_proj", + "model.layers.5.mlp.experts.241.down_proj", + "model.layers.5.mlp.experts.241.gate_proj", + "model.layers.5.mlp.experts.241.up_proj", + "model.layers.5.mlp.experts.242.down_proj", + "model.layers.5.mlp.experts.242.gate_proj", + "model.layers.5.mlp.experts.242.up_proj", + "model.layers.5.mlp.experts.243.down_proj", + "model.layers.5.mlp.experts.243.gate_proj", + "model.layers.5.mlp.experts.243.up_proj", + "model.layers.5.mlp.experts.244.down_proj", + "model.layers.5.mlp.experts.244.gate_proj", + "model.layers.5.mlp.experts.244.up_proj", + "model.layers.5.mlp.experts.245.down_proj", + "model.layers.5.mlp.experts.245.gate_proj", + "model.layers.5.mlp.experts.245.up_proj", + "model.layers.5.mlp.experts.246.down_proj", + "model.layers.5.mlp.experts.246.gate_proj", + "model.layers.5.mlp.experts.246.up_proj", + "model.layers.5.mlp.experts.247.down_proj", + "model.layers.5.mlp.experts.247.gate_proj", + "model.layers.5.mlp.experts.247.up_proj", + "model.layers.5.mlp.experts.248.down_proj", + "model.layers.5.mlp.experts.248.gate_proj", + "model.layers.5.mlp.experts.248.up_proj", + "model.layers.5.mlp.experts.249.down_proj", + "model.layers.5.mlp.experts.249.gate_proj", + "model.layers.5.mlp.experts.249.up_proj", + "model.layers.5.mlp.experts.25.down_proj", + "model.layers.5.mlp.experts.25.gate_proj", + "model.layers.5.mlp.experts.25.up_proj", + "model.layers.5.mlp.experts.250.down_proj", + "model.layers.5.mlp.experts.250.gate_proj", + "model.layers.5.mlp.experts.250.up_proj", + "model.layers.5.mlp.experts.251.down_proj", + "model.layers.5.mlp.experts.251.gate_proj", + "model.layers.5.mlp.experts.251.up_proj", + "model.layers.5.mlp.experts.252.down_proj", + "model.layers.5.mlp.experts.252.gate_proj", + "model.layers.5.mlp.experts.252.up_proj", + "model.layers.5.mlp.experts.253.down_proj", + "model.layers.5.mlp.experts.253.gate_proj", + "model.layers.5.mlp.experts.253.up_proj", + "model.layers.5.mlp.experts.254.down_proj", + "model.layers.5.mlp.experts.254.gate_proj", + "model.layers.5.mlp.experts.254.up_proj", + "model.layers.5.mlp.experts.255.down_proj", + "model.layers.5.mlp.experts.255.gate_proj", + "model.layers.5.mlp.experts.255.up_proj", + "model.layers.5.mlp.experts.26.down_proj", + "model.layers.5.mlp.experts.26.gate_proj", + "model.layers.5.mlp.experts.26.up_proj", + "model.layers.5.mlp.experts.27.down_proj", + "model.layers.5.mlp.experts.27.gate_proj", + "model.layers.5.mlp.experts.27.up_proj", + "model.layers.5.mlp.experts.28.down_proj", + "model.layers.5.mlp.experts.28.gate_proj", + "model.layers.5.mlp.experts.28.up_proj", + "model.layers.5.mlp.experts.29.down_proj", + "model.layers.5.mlp.experts.29.gate_proj", + "model.layers.5.mlp.experts.29.up_proj", + "model.layers.5.mlp.experts.3.down_proj", + "model.layers.5.mlp.experts.3.gate_proj", + "model.layers.5.mlp.experts.3.up_proj", + "model.layers.5.mlp.experts.30.down_proj", + "model.layers.5.mlp.experts.30.gate_proj", + "model.layers.5.mlp.experts.30.up_proj", + "model.layers.5.mlp.experts.31.down_proj", + "model.layers.5.mlp.experts.31.gate_proj", + "model.layers.5.mlp.experts.31.up_proj", + "model.layers.5.mlp.experts.32.down_proj", + "model.layers.5.mlp.experts.32.gate_proj", + "model.layers.5.mlp.experts.32.up_proj", + "model.layers.5.mlp.experts.33.down_proj", + "model.layers.5.mlp.experts.33.gate_proj", + "model.layers.5.mlp.experts.33.up_proj", + "model.layers.5.mlp.experts.34.down_proj", + "model.layers.5.mlp.experts.34.gate_proj", + "model.layers.5.mlp.experts.34.up_proj", + "model.layers.5.mlp.experts.35.down_proj", + "model.layers.5.mlp.experts.35.gate_proj", + "model.layers.5.mlp.experts.35.up_proj", + "model.layers.5.mlp.experts.36.down_proj", + "model.layers.5.mlp.experts.36.gate_proj", + "model.layers.5.mlp.experts.36.up_proj", + "model.layers.5.mlp.experts.37.down_proj", + "model.layers.5.mlp.experts.37.gate_proj", + "model.layers.5.mlp.experts.37.up_proj", + "model.layers.5.mlp.experts.38.down_proj", + "model.layers.5.mlp.experts.38.gate_proj", + "model.layers.5.mlp.experts.38.up_proj", + "model.layers.5.mlp.experts.39.down_proj", + "model.layers.5.mlp.experts.39.gate_proj", + "model.layers.5.mlp.experts.39.up_proj", + "model.layers.5.mlp.experts.4.down_proj", + "model.layers.5.mlp.experts.4.gate_proj", + "model.layers.5.mlp.experts.4.up_proj", + "model.layers.5.mlp.experts.40.down_proj", + "model.layers.5.mlp.experts.40.gate_proj", + "model.layers.5.mlp.experts.40.up_proj", + "model.layers.5.mlp.experts.41.down_proj", + "model.layers.5.mlp.experts.41.gate_proj", + "model.layers.5.mlp.experts.41.up_proj", + "model.layers.5.mlp.experts.42.down_proj", + "model.layers.5.mlp.experts.42.gate_proj", + "model.layers.5.mlp.experts.42.up_proj", + "model.layers.5.mlp.experts.43.down_proj", + "model.layers.5.mlp.experts.43.gate_proj", + "model.layers.5.mlp.experts.43.up_proj", + "model.layers.5.mlp.experts.44.down_proj", + "model.layers.5.mlp.experts.44.gate_proj", + "model.layers.5.mlp.experts.44.up_proj", + "model.layers.5.mlp.experts.45.down_proj", + "model.layers.5.mlp.experts.45.gate_proj", + "model.layers.5.mlp.experts.45.up_proj", + "model.layers.5.mlp.experts.46.down_proj", + "model.layers.5.mlp.experts.46.gate_proj", + "model.layers.5.mlp.experts.46.up_proj", + "model.layers.5.mlp.experts.47.down_proj", + "model.layers.5.mlp.experts.47.gate_proj", + "model.layers.5.mlp.experts.47.up_proj", + "model.layers.5.mlp.experts.48.down_proj", + "model.layers.5.mlp.experts.48.gate_proj", + "model.layers.5.mlp.experts.48.up_proj", + "model.layers.5.mlp.experts.49.down_proj", + "model.layers.5.mlp.experts.49.gate_proj", + "model.layers.5.mlp.experts.49.up_proj", + "model.layers.5.mlp.experts.5.down_proj", + "model.layers.5.mlp.experts.5.gate_proj", + "model.layers.5.mlp.experts.5.up_proj", + "model.layers.5.mlp.experts.50.down_proj", + "model.layers.5.mlp.experts.50.gate_proj", + "model.layers.5.mlp.experts.50.up_proj", + "model.layers.5.mlp.experts.51.down_proj", + "model.layers.5.mlp.experts.51.gate_proj", + "model.layers.5.mlp.experts.51.up_proj", + "model.layers.5.mlp.experts.52.down_proj", + "model.layers.5.mlp.experts.52.gate_proj", + "model.layers.5.mlp.experts.52.up_proj", + "model.layers.5.mlp.experts.53.down_proj", + "model.layers.5.mlp.experts.53.gate_proj", + "model.layers.5.mlp.experts.53.up_proj", + "model.layers.5.mlp.experts.54.down_proj", + "model.layers.5.mlp.experts.54.gate_proj", + "model.layers.5.mlp.experts.54.up_proj", + "model.layers.5.mlp.experts.55.down_proj", + "model.layers.5.mlp.experts.55.gate_proj", + "model.layers.5.mlp.experts.55.up_proj", + "model.layers.5.mlp.experts.56.down_proj", + "model.layers.5.mlp.experts.56.gate_proj", + "model.layers.5.mlp.experts.56.up_proj", + "model.layers.5.mlp.experts.57.down_proj", + "model.layers.5.mlp.experts.57.gate_proj", + "model.layers.5.mlp.experts.57.up_proj", + "model.layers.5.mlp.experts.58.down_proj", + "model.layers.5.mlp.experts.58.gate_proj", + "model.layers.5.mlp.experts.58.up_proj", + "model.layers.5.mlp.experts.59.down_proj", + "model.layers.5.mlp.experts.59.gate_proj", + "model.layers.5.mlp.experts.59.up_proj", + "model.layers.5.mlp.experts.6.down_proj", + "model.layers.5.mlp.experts.6.gate_proj", + "model.layers.5.mlp.experts.6.up_proj", + "model.layers.5.mlp.experts.60.down_proj", + "model.layers.5.mlp.experts.60.gate_proj", + "model.layers.5.mlp.experts.60.up_proj", + "model.layers.5.mlp.experts.61.down_proj", + "model.layers.5.mlp.experts.61.gate_proj", + "model.layers.5.mlp.experts.61.up_proj", + "model.layers.5.mlp.experts.62.down_proj", + "model.layers.5.mlp.experts.62.gate_proj", + "model.layers.5.mlp.experts.62.up_proj", + "model.layers.5.mlp.experts.63.down_proj", + "model.layers.5.mlp.experts.63.gate_proj", + "model.layers.5.mlp.experts.63.up_proj", + "model.layers.5.mlp.experts.64.down_proj", + "model.layers.5.mlp.experts.64.gate_proj", + "model.layers.5.mlp.experts.64.up_proj", + "model.layers.5.mlp.experts.65.down_proj", + "model.layers.5.mlp.experts.65.gate_proj", + "model.layers.5.mlp.experts.65.up_proj", + "model.layers.5.mlp.experts.66.down_proj", + "model.layers.5.mlp.experts.66.gate_proj", + "model.layers.5.mlp.experts.66.up_proj", + "model.layers.5.mlp.experts.67.down_proj", + "model.layers.5.mlp.experts.67.gate_proj", + "model.layers.5.mlp.experts.67.up_proj", + "model.layers.5.mlp.experts.68.down_proj", + "model.layers.5.mlp.experts.68.gate_proj", + "model.layers.5.mlp.experts.68.up_proj", + "model.layers.5.mlp.experts.69.down_proj", + "model.layers.5.mlp.experts.69.gate_proj", + "model.layers.5.mlp.experts.69.up_proj", + "model.layers.5.mlp.experts.7.down_proj", + "model.layers.5.mlp.experts.7.gate_proj", + "model.layers.5.mlp.experts.7.up_proj", + "model.layers.5.mlp.experts.70.down_proj", + "model.layers.5.mlp.experts.70.gate_proj", + "model.layers.5.mlp.experts.70.up_proj", + "model.layers.5.mlp.experts.71.down_proj", + "model.layers.5.mlp.experts.71.gate_proj", + "model.layers.5.mlp.experts.71.up_proj", + "model.layers.5.mlp.experts.72.down_proj", + "model.layers.5.mlp.experts.72.gate_proj", + "model.layers.5.mlp.experts.72.up_proj", + "model.layers.5.mlp.experts.73.down_proj", + "model.layers.5.mlp.experts.73.gate_proj", + "model.layers.5.mlp.experts.73.up_proj", + "model.layers.5.mlp.experts.74.down_proj", + "model.layers.5.mlp.experts.74.gate_proj", + "model.layers.5.mlp.experts.74.up_proj", + "model.layers.5.mlp.experts.75.down_proj", + "model.layers.5.mlp.experts.75.gate_proj", + "model.layers.5.mlp.experts.75.up_proj", + "model.layers.5.mlp.experts.76.down_proj", + "model.layers.5.mlp.experts.76.gate_proj", + "model.layers.5.mlp.experts.76.up_proj", + "model.layers.5.mlp.experts.77.down_proj", + "model.layers.5.mlp.experts.77.gate_proj", + "model.layers.5.mlp.experts.77.up_proj", + "model.layers.5.mlp.experts.78.down_proj", + "model.layers.5.mlp.experts.78.gate_proj", + "model.layers.5.mlp.experts.78.up_proj", + "model.layers.5.mlp.experts.79.down_proj", + "model.layers.5.mlp.experts.79.gate_proj", + "model.layers.5.mlp.experts.79.up_proj", + "model.layers.5.mlp.experts.8.down_proj", + "model.layers.5.mlp.experts.8.gate_proj", + "model.layers.5.mlp.experts.8.up_proj", + "model.layers.5.mlp.experts.80.down_proj", + "model.layers.5.mlp.experts.80.gate_proj", + "model.layers.5.mlp.experts.80.up_proj", + "model.layers.5.mlp.experts.81.down_proj", + "model.layers.5.mlp.experts.81.gate_proj", + "model.layers.5.mlp.experts.81.up_proj", + "model.layers.5.mlp.experts.82.down_proj", + "model.layers.5.mlp.experts.82.gate_proj", + "model.layers.5.mlp.experts.82.up_proj", + "model.layers.5.mlp.experts.83.down_proj", + "model.layers.5.mlp.experts.83.gate_proj", + "model.layers.5.mlp.experts.83.up_proj", + "model.layers.5.mlp.experts.84.down_proj", + "model.layers.5.mlp.experts.84.gate_proj", + "model.layers.5.mlp.experts.84.up_proj", + "model.layers.5.mlp.experts.85.down_proj", + "model.layers.5.mlp.experts.85.gate_proj", + "model.layers.5.mlp.experts.85.up_proj", + "model.layers.5.mlp.experts.86.down_proj", + "model.layers.5.mlp.experts.86.gate_proj", + "model.layers.5.mlp.experts.86.up_proj", + "model.layers.5.mlp.experts.87.down_proj", + "model.layers.5.mlp.experts.87.gate_proj", + "model.layers.5.mlp.experts.87.up_proj", + "model.layers.5.mlp.experts.88.down_proj", + "model.layers.5.mlp.experts.88.gate_proj", + "model.layers.5.mlp.experts.88.up_proj", + "model.layers.5.mlp.experts.89.down_proj", + "model.layers.5.mlp.experts.89.gate_proj", + "model.layers.5.mlp.experts.89.up_proj", + "model.layers.5.mlp.experts.9.down_proj", + "model.layers.5.mlp.experts.9.gate_proj", + "model.layers.5.mlp.experts.9.up_proj", + "model.layers.5.mlp.experts.90.down_proj", + "model.layers.5.mlp.experts.90.gate_proj", + "model.layers.5.mlp.experts.90.up_proj", + "model.layers.5.mlp.experts.91.down_proj", + "model.layers.5.mlp.experts.91.gate_proj", + "model.layers.5.mlp.experts.91.up_proj", + "model.layers.5.mlp.experts.92.down_proj", + "model.layers.5.mlp.experts.92.gate_proj", + "model.layers.5.mlp.experts.92.up_proj", + "model.layers.5.mlp.experts.93.down_proj", + "model.layers.5.mlp.experts.93.gate_proj", + "model.layers.5.mlp.experts.93.up_proj", + "model.layers.5.mlp.experts.94.down_proj", + "model.layers.5.mlp.experts.94.gate_proj", + "model.layers.5.mlp.experts.94.up_proj", + "model.layers.5.mlp.experts.95.down_proj", + "model.layers.5.mlp.experts.95.gate_proj", + "model.layers.5.mlp.experts.95.up_proj", + "model.layers.5.mlp.experts.96.down_proj", + "model.layers.5.mlp.experts.96.gate_proj", + "model.layers.5.mlp.experts.96.up_proj", + "model.layers.5.mlp.experts.97.down_proj", + "model.layers.5.mlp.experts.97.gate_proj", + "model.layers.5.mlp.experts.97.up_proj", + "model.layers.5.mlp.experts.98.down_proj", + "model.layers.5.mlp.experts.98.gate_proj", + "model.layers.5.mlp.experts.98.up_proj", + "model.layers.5.mlp.experts.99.down_proj", + "model.layers.5.mlp.experts.99.gate_proj", + "model.layers.5.mlp.experts.99.up_proj", + "model.layers.50.mlp.experts.0.down_proj", + "model.layers.50.mlp.experts.0.gate_proj", + "model.layers.50.mlp.experts.0.up_proj", + "model.layers.50.mlp.experts.1.down_proj", + "model.layers.50.mlp.experts.1.gate_proj", + "model.layers.50.mlp.experts.1.up_proj", + "model.layers.50.mlp.experts.10.down_proj", + "model.layers.50.mlp.experts.10.gate_proj", + "model.layers.50.mlp.experts.10.up_proj", + "model.layers.50.mlp.experts.100.down_proj", + "model.layers.50.mlp.experts.100.gate_proj", + "model.layers.50.mlp.experts.100.up_proj", + "model.layers.50.mlp.experts.101.down_proj", + "model.layers.50.mlp.experts.101.gate_proj", + "model.layers.50.mlp.experts.101.up_proj", + "model.layers.50.mlp.experts.102.down_proj", + "model.layers.50.mlp.experts.102.gate_proj", + "model.layers.50.mlp.experts.102.up_proj", + "model.layers.50.mlp.experts.103.down_proj", + "model.layers.50.mlp.experts.103.gate_proj", + "model.layers.50.mlp.experts.103.up_proj", + "model.layers.50.mlp.experts.104.down_proj", + "model.layers.50.mlp.experts.104.gate_proj", + "model.layers.50.mlp.experts.104.up_proj", + "model.layers.50.mlp.experts.105.down_proj", + "model.layers.50.mlp.experts.105.gate_proj", + "model.layers.50.mlp.experts.105.up_proj", + "model.layers.50.mlp.experts.106.down_proj", + "model.layers.50.mlp.experts.106.gate_proj", + "model.layers.50.mlp.experts.106.up_proj", + "model.layers.50.mlp.experts.107.down_proj", + "model.layers.50.mlp.experts.107.gate_proj", + "model.layers.50.mlp.experts.107.up_proj", + "model.layers.50.mlp.experts.108.down_proj", + "model.layers.50.mlp.experts.108.gate_proj", + "model.layers.50.mlp.experts.108.up_proj", + "model.layers.50.mlp.experts.109.down_proj", + "model.layers.50.mlp.experts.109.gate_proj", + "model.layers.50.mlp.experts.109.up_proj", + "model.layers.50.mlp.experts.11.down_proj", + "model.layers.50.mlp.experts.11.gate_proj", + "model.layers.50.mlp.experts.11.up_proj", + "model.layers.50.mlp.experts.110.down_proj", + "model.layers.50.mlp.experts.110.gate_proj", + "model.layers.50.mlp.experts.110.up_proj", + "model.layers.50.mlp.experts.111.down_proj", + "model.layers.50.mlp.experts.111.gate_proj", + "model.layers.50.mlp.experts.111.up_proj", + "model.layers.50.mlp.experts.112.down_proj", + "model.layers.50.mlp.experts.112.gate_proj", + "model.layers.50.mlp.experts.112.up_proj", + "model.layers.50.mlp.experts.113.down_proj", + "model.layers.50.mlp.experts.113.gate_proj", + "model.layers.50.mlp.experts.113.up_proj", + "model.layers.50.mlp.experts.114.down_proj", + "model.layers.50.mlp.experts.114.gate_proj", + "model.layers.50.mlp.experts.114.up_proj", + "model.layers.50.mlp.experts.115.down_proj", + "model.layers.50.mlp.experts.115.gate_proj", + "model.layers.50.mlp.experts.115.up_proj", + "model.layers.50.mlp.experts.116.down_proj", + "model.layers.50.mlp.experts.116.gate_proj", + "model.layers.50.mlp.experts.116.up_proj", + "model.layers.50.mlp.experts.117.down_proj", + "model.layers.50.mlp.experts.117.gate_proj", + "model.layers.50.mlp.experts.117.up_proj", + "model.layers.50.mlp.experts.118.down_proj", + "model.layers.50.mlp.experts.118.gate_proj", + "model.layers.50.mlp.experts.118.up_proj", + "model.layers.50.mlp.experts.119.down_proj", + "model.layers.50.mlp.experts.119.gate_proj", + "model.layers.50.mlp.experts.119.up_proj", + "model.layers.50.mlp.experts.12.down_proj", + "model.layers.50.mlp.experts.12.gate_proj", + "model.layers.50.mlp.experts.12.up_proj", + "model.layers.50.mlp.experts.120.down_proj", + "model.layers.50.mlp.experts.120.gate_proj", + "model.layers.50.mlp.experts.120.up_proj", + "model.layers.50.mlp.experts.121.down_proj", + "model.layers.50.mlp.experts.121.gate_proj", + "model.layers.50.mlp.experts.121.up_proj", + "model.layers.50.mlp.experts.122.down_proj", + "model.layers.50.mlp.experts.122.gate_proj", + "model.layers.50.mlp.experts.122.up_proj", + "model.layers.50.mlp.experts.123.down_proj", + "model.layers.50.mlp.experts.123.gate_proj", + "model.layers.50.mlp.experts.123.up_proj", + "model.layers.50.mlp.experts.124.down_proj", + "model.layers.50.mlp.experts.124.gate_proj", + "model.layers.50.mlp.experts.124.up_proj", + "model.layers.50.mlp.experts.125.down_proj", + "model.layers.50.mlp.experts.125.gate_proj", + "model.layers.50.mlp.experts.125.up_proj", + "model.layers.50.mlp.experts.126.down_proj", + "model.layers.50.mlp.experts.126.gate_proj", + "model.layers.50.mlp.experts.126.up_proj", + "model.layers.50.mlp.experts.127.down_proj", + "model.layers.50.mlp.experts.127.gate_proj", + "model.layers.50.mlp.experts.127.up_proj", + "model.layers.50.mlp.experts.128.down_proj", + "model.layers.50.mlp.experts.128.gate_proj", + "model.layers.50.mlp.experts.128.up_proj", + "model.layers.50.mlp.experts.129.down_proj", + "model.layers.50.mlp.experts.129.gate_proj", + "model.layers.50.mlp.experts.129.up_proj", + "model.layers.50.mlp.experts.13.down_proj", + "model.layers.50.mlp.experts.13.gate_proj", + "model.layers.50.mlp.experts.13.up_proj", + "model.layers.50.mlp.experts.130.down_proj", + "model.layers.50.mlp.experts.130.gate_proj", + "model.layers.50.mlp.experts.130.up_proj", + "model.layers.50.mlp.experts.131.down_proj", + "model.layers.50.mlp.experts.131.gate_proj", + "model.layers.50.mlp.experts.131.up_proj", + "model.layers.50.mlp.experts.132.down_proj", + "model.layers.50.mlp.experts.132.gate_proj", + "model.layers.50.mlp.experts.132.up_proj", + "model.layers.50.mlp.experts.133.down_proj", + "model.layers.50.mlp.experts.133.gate_proj", + "model.layers.50.mlp.experts.133.up_proj", + "model.layers.50.mlp.experts.134.down_proj", + "model.layers.50.mlp.experts.134.gate_proj", + "model.layers.50.mlp.experts.134.up_proj", + "model.layers.50.mlp.experts.135.down_proj", + "model.layers.50.mlp.experts.135.gate_proj", + "model.layers.50.mlp.experts.135.up_proj", + "model.layers.50.mlp.experts.136.down_proj", + "model.layers.50.mlp.experts.136.gate_proj", + "model.layers.50.mlp.experts.136.up_proj", + "model.layers.50.mlp.experts.137.down_proj", + "model.layers.50.mlp.experts.137.gate_proj", + "model.layers.50.mlp.experts.137.up_proj", + "model.layers.50.mlp.experts.138.down_proj", + "model.layers.50.mlp.experts.138.gate_proj", + "model.layers.50.mlp.experts.138.up_proj", + "model.layers.50.mlp.experts.139.down_proj", + "model.layers.50.mlp.experts.139.gate_proj", + "model.layers.50.mlp.experts.139.up_proj", + "model.layers.50.mlp.experts.14.down_proj", + "model.layers.50.mlp.experts.14.gate_proj", + "model.layers.50.mlp.experts.14.up_proj", + "model.layers.50.mlp.experts.140.down_proj", + "model.layers.50.mlp.experts.140.gate_proj", + "model.layers.50.mlp.experts.140.up_proj", + "model.layers.50.mlp.experts.141.down_proj", + "model.layers.50.mlp.experts.141.gate_proj", + "model.layers.50.mlp.experts.141.up_proj", + "model.layers.50.mlp.experts.142.down_proj", + "model.layers.50.mlp.experts.142.gate_proj", + "model.layers.50.mlp.experts.142.up_proj", + "model.layers.50.mlp.experts.143.down_proj", + "model.layers.50.mlp.experts.143.gate_proj", + "model.layers.50.mlp.experts.143.up_proj", + "model.layers.50.mlp.experts.144.down_proj", + "model.layers.50.mlp.experts.144.gate_proj", + "model.layers.50.mlp.experts.144.up_proj", + "model.layers.50.mlp.experts.145.down_proj", + "model.layers.50.mlp.experts.145.gate_proj", + "model.layers.50.mlp.experts.145.up_proj", + "model.layers.50.mlp.experts.146.down_proj", + "model.layers.50.mlp.experts.146.gate_proj", + "model.layers.50.mlp.experts.146.up_proj", + "model.layers.50.mlp.experts.147.down_proj", + "model.layers.50.mlp.experts.147.gate_proj", + "model.layers.50.mlp.experts.147.up_proj", + "model.layers.50.mlp.experts.148.down_proj", + "model.layers.50.mlp.experts.148.gate_proj", + "model.layers.50.mlp.experts.148.up_proj", + "model.layers.50.mlp.experts.149.down_proj", + "model.layers.50.mlp.experts.149.gate_proj", + "model.layers.50.mlp.experts.149.up_proj", + "model.layers.50.mlp.experts.15.down_proj", + "model.layers.50.mlp.experts.15.gate_proj", + "model.layers.50.mlp.experts.15.up_proj", + "model.layers.50.mlp.experts.150.down_proj", + "model.layers.50.mlp.experts.150.gate_proj", + "model.layers.50.mlp.experts.150.up_proj", + "model.layers.50.mlp.experts.151.down_proj", + "model.layers.50.mlp.experts.151.gate_proj", + "model.layers.50.mlp.experts.151.up_proj", + "model.layers.50.mlp.experts.152.down_proj", + "model.layers.50.mlp.experts.152.gate_proj", + "model.layers.50.mlp.experts.152.up_proj", + "model.layers.50.mlp.experts.153.down_proj", + "model.layers.50.mlp.experts.153.gate_proj", + "model.layers.50.mlp.experts.153.up_proj", + "model.layers.50.mlp.experts.154.down_proj", + "model.layers.50.mlp.experts.154.gate_proj", + "model.layers.50.mlp.experts.154.up_proj", + "model.layers.50.mlp.experts.155.down_proj", + "model.layers.50.mlp.experts.155.gate_proj", + "model.layers.50.mlp.experts.155.up_proj", + "model.layers.50.mlp.experts.156.down_proj", + "model.layers.50.mlp.experts.156.gate_proj", + "model.layers.50.mlp.experts.156.up_proj", + "model.layers.50.mlp.experts.157.down_proj", + "model.layers.50.mlp.experts.157.gate_proj", + "model.layers.50.mlp.experts.157.up_proj", + "model.layers.50.mlp.experts.158.down_proj", + "model.layers.50.mlp.experts.158.gate_proj", + "model.layers.50.mlp.experts.158.up_proj", + "model.layers.50.mlp.experts.159.down_proj", + "model.layers.50.mlp.experts.159.gate_proj", + "model.layers.50.mlp.experts.159.up_proj", + "model.layers.50.mlp.experts.16.down_proj", + "model.layers.50.mlp.experts.16.gate_proj", + "model.layers.50.mlp.experts.16.up_proj", + "model.layers.50.mlp.experts.160.down_proj", + "model.layers.50.mlp.experts.160.gate_proj", + "model.layers.50.mlp.experts.160.up_proj", + "model.layers.50.mlp.experts.161.down_proj", + "model.layers.50.mlp.experts.161.gate_proj", + "model.layers.50.mlp.experts.161.up_proj", + "model.layers.50.mlp.experts.162.down_proj", + "model.layers.50.mlp.experts.162.gate_proj", + "model.layers.50.mlp.experts.162.up_proj", + "model.layers.50.mlp.experts.163.down_proj", + "model.layers.50.mlp.experts.163.gate_proj", + "model.layers.50.mlp.experts.163.up_proj", + "model.layers.50.mlp.experts.164.down_proj", + "model.layers.50.mlp.experts.164.gate_proj", + "model.layers.50.mlp.experts.164.up_proj", + "model.layers.50.mlp.experts.165.down_proj", + "model.layers.50.mlp.experts.165.gate_proj", + "model.layers.50.mlp.experts.165.up_proj", + "model.layers.50.mlp.experts.166.down_proj", + "model.layers.50.mlp.experts.166.gate_proj", + "model.layers.50.mlp.experts.166.up_proj", + "model.layers.50.mlp.experts.167.down_proj", + "model.layers.50.mlp.experts.167.gate_proj", + "model.layers.50.mlp.experts.167.up_proj", + "model.layers.50.mlp.experts.168.down_proj", + "model.layers.50.mlp.experts.168.gate_proj", + "model.layers.50.mlp.experts.168.up_proj", + "model.layers.50.mlp.experts.169.down_proj", + "model.layers.50.mlp.experts.169.gate_proj", + "model.layers.50.mlp.experts.169.up_proj", + "model.layers.50.mlp.experts.17.down_proj", + "model.layers.50.mlp.experts.17.gate_proj", + "model.layers.50.mlp.experts.17.up_proj", + "model.layers.50.mlp.experts.170.down_proj", + "model.layers.50.mlp.experts.170.gate_proj", + "model.layers.50.mlp.experts.170.up_proj", + "model.layers.50.mlp.experts.171.down_proj", + "model.layers.50.mlp.experts.171.gate_proj", + "model.layers.50.mlp.experts.171.up_proj", + "model.layers.50.mlp.experts.172.down_proj", + "model.layers.50.mlp.experts.172.gate_proj", + "model.layers.50.mlp.experts.172.up_proj", + "model.layers.50.mlp.experts.173.down_proj", + "model.layers.50.mlp.experts.173.gate_proj", + "model.layers.50.mlp.experts.173.up_proj", + "model.layers.50.mlp.experts.174.down_proj", + "model.layers.50.mlp.experts.174.gate_proj", + "model.layers.50.mlp.experts.174.up_proj", + "model.layers.50.mlp.experts.175.down_proj", + "model.layers.50.mlp.experts.175.gate_proj", + "model.layers.50.mlp.experts.175.up_proj", + "model.layers.50.mlp.experts.176.down_proj", + "model.layers.50.mlp.experts.176.gate_proj", + "model.layers.50.mlp.experts.176.up_proj", + "model.layers.50.mlp.experts.177.down_proj", + "model.layers.50.mlp.experts.177.gate_proj", + "model.layers.50.mlp.experts.177.up_proj", + "model.layers.50.mlp.experts.178.down_proj", + "model.layers.50.mlp.experts.178.gate_proj", + "model.layers.50.mlp.experts.178.up_proj", + "model.layers.50.mlp.experts.179.down_proj", + "model.layers.50.mlp.experts.179.gate_proj", + "model.layers.50.mlp.experts.179.up_proj", + "model.layers.50.mlp.experts.18.down_proj", + "model.layers.50.mlp.experts.18.gate_proj", + "model.layers.50.mlp.experts.18.up_proj", + "model.layers.50.mlp.experts.180.down_proj", + "model.layers.50.mlp.experts.180.gate_proj", + "model.layers.50.mlp.experts.180.up_proj", + "model.layers.50.mlp.experts.181.down_proj", + "model.layers.50.mlp.experts.181.gate_proj", + "model.layers.50.mlp.experts.181.up_proj", + "model.layers.50.mlp.experts.182.down_proj", + "model.layers.50.mlp.experts.182.gate_proj", + "model.layers.50.mlp.experts.182.up_proj", + "model.layers.50.mlp.experts.183.down_proj", + "model.layers.50.mlp.experts.183.gate_proj", + "model.layers.50.mlp.experts.183.up_proj", + "model.layers.50.mlp.experts.184.down_proj", + "model.layers.50.mlp.experts.184.gate_proj", + "model.layers.50.mlp.experts.184.up_proj", + "model.layers.50.mlp.experts.185.down_proj", + "model.layers.50.mlp.experts.185.gate_proj", + "model.layers.50.mlp.experts.185.up_proj", + "model.layers.50.mlp.experts.186.down_proj", + "model.layers.50.mlp.experts.186.gate_proj", + "model.layers.50.mlp.experts.186.up_proj", + "model.layers.50.mlp.experts.187.down_proj", + "model.layers.50.mlp.experts.187.gate_proj", + "model.layers.50.mlp.experts.187.up_proj", + "model.layers.50.mlp.experts.188.down_proj", + "model.layers.50.mlp.experts.188.gate_proj", + "model.layers.50.mlp.experts.188.up_proj", + "model.layers.50.mlp.experts.189.down_proj", + "model.layers.50.mlp.experts.189.gate_proj", + "model.layers.50.mlp.experts.189.up_proj", + "model.layers.50.mlp.experts.19.down_proj", + "model.layers.50.mlp.experts.19.gate_proj", + "model.layers.50.mlp.experts.19.up_proj", + "model.layers.50.mlp.experts.190.down_proj", + "model.layers.50.mlp.experts.190.gate_proj", + "model.layers.50.mlp.experts.190.up_proj", + "model.layers.50.mlp.experts.191.down_proj", + "model.layers.50.mlp.experts.191.gate_proj", + "model.layers.50.mlp.experts.191.up_proj", + "model.layers.50.mlp.experts.192.down_proj", + "model.layers.50.mlp.experts.192.gate_proj", + "model.layers.50.mlp.experts.192.up_proj", + "model.layers.50.mlp.experts.193.down_proj", + "model.layers.50.mlp.experts.193.gate_proj", + "model.layers.50.mlp.experts.193.up_proj", + "model.layers.50.mlp.experts.194.down_proj", + "model.layers.50.mlp.experts.194.gate_proj", + "model.layers.50.mlp.experts.194.up_proj", + "model.layers.50.mlp.experts.195.down_proj", + "model.layers.50.mlp.experts.195.gate_proj", + "model.layers.50.mlp.experts.195.up_proj", + "model.layers.50.mlp.experts.196.down_proj", + "model.layers.50.mlp.experts.196.gate_proj", + "model.layers.50.mlp.experts.196.up_proj", + "model.layers.50.mlp.experts.197.down_proj", + "model.layers.50.mlp.experts.197.gate_proj", + "model.layers.50.mlp.experts.197.up_proj", + "model.layers.50.mlp.experts.198.down_proj", + "model.layers.50.mlp.experts.198.gate_proj", + "model.layers.50.mlp.experts.198.up_proj", + "model.layers.50.mlp.experts.199.down_proj", + "model.layers.50.mlp.experts.199.gate_proj", + "model.layers.50.mlp.experts.199.up_proj", + "model.layers.50.mlp.experts.2.down_proj", + "model.layers.50.mlp.experts.2.gate_proj", + "model.layers.50.mlp.experts.2.up_proj", + "model.layers.50.mlp.experts.20.down_proj", + "model.layers.50.mlp.experts.20.gate_proj", + "model.layers.50.mlp.experts.20.up_proj", + "model.layers.50.mlp.experts.200.down_proj", + "model.layers.50.mlp.experts.200.gate_proj", + "model.layers.50.mlp.experts.200.up_proj", + "model.layers.50.mlp.experts.201.down_proj", + "model.layers.50.mlp.experts.201.gate_proj", + "model.layers.50.mlp.experts.201.up_proj", + "model.layers.50.mlp.experts.202.down_proj", + "model.layers.50.mlp.experts.202.gate_proj", + "model.layers.50.mlp.experts.202.up_proj", + "model.layers.50.mlp.experts.203.down_proj", + "model.layers.50.mlp.experts.203.gate_proj", + "model.layers.50.mlp.experts.203.up_proj", + "model.layers.50.mlp.experts.204.down_proj", + "model.layers.50.mlp.experts.204.gate_proj", + "model.layers.50.mlp.experts.204.up_proj", + "model.layers.50.mlp.experts.205.down_proj", + "model.layers.50.mlp.experts.205.gate_proj", + "model.layers.50.mlp.experts.205.up_proj", + "model.layers.50.mlp.experts.206.down_proj", + "model.layers.50.mlp.experts.206.gate_proj", + "model.layers.50.mlp.experts.206.up_proj", + "model.layers.50.mlp.experts.207.down_proj", + "model.layers.50.mlp.experts.207.gate_proj", + "model.layers.50.mlp.experts.207.up_proj", + "model.layers.50.mlp.experts.208.down_proj", + "model.layers.50.mlp.experts.208.gate_proj", + "model.layers.50.mlp.experts.208.up_proj", + "model.layers.50.mlp.experts.209.down_proj", + "model.layers.50.mlp.experts.209.gate_proj", + "model.layers.50.mlp.experts.209.up_proj", + "model.layers.50.mlp.experts.21.down_proj", + "model.layers.50.mlp.experts.21.gate_proj", + "model.layers.50.mlp.experts.21.up_proj", + "model.layers.50.mlp.experts.210.down_proj", + "model.layers.50.mlp.experts.210.gate_proj", + "model.layers.50.mlp.experts.210.up_proj", + "model.layers.50.mlp.experts.211.down_proj", + "model.layers.50.mlp.experts.211.gate_proj", + "model.layers.50.mlp.experts.211.up_proj", + "model.layers.50.mlp.experts.212.down_proj", + "model.layers.50.mlp.experts.212.gate_proj", + "model.layers.50.mlp.experts.212.up_proj", + "model.layers.50.mlp.experts.213.down_proj", + "model.layers.50.mlp.experts.213.gate_proj", + "model.layers.50.mlp.experts.213.up_proj", + "model.layers.50.mlp.experts.214.down_proj", + "model.layers.50.mlp.experts.214.gate_proj", + "model.layers.50.mlp.experts.214.up_proj", + "model.layers.50.mlp.experts.215.down_proj", + "model.layers.50.mlp.experts.215.gate_proj", + "model.layers.50.mlp.experts.215.up_proj", + "model.layers.50.mlp.experts.216.down_proj", + "model.layers.50.mlp.experts.216.gate_proj", + "model.layers.50.mlp.experts.216.up_proj", + "model.layers.50.mlp.experts.217.down_proj", + "model.layers.50.mlp.experts.217.gate_proj", + "model.layers.50.mlp.experts.217.up_proj", + "model.layers.50.mlp.experts.218.down_proj", + "model.layers.50.mlp.experts.218.gate_proj", + "model.layers.50.mlp.experts.218.up_proj", + "model.layers.50.mlp.experts.219.down_proj", + "model.layers.50.mlp.experts.219.gate_proj", + "model.layers.50.mlp.experts.219.up_proj", + "model.layers.50.mlp.experts.22.down_proj", + "model.layers.50.mlp.experts.22.gate_proj", + "model.layers.50.mlp.experts.22.up_proj", + "model.layers.50.mlp.experts.220.down_proj", + "model.layers.50.mlp.experts.220.gate_proj", + "model.layers.50.mlp.experts.220.up_proj", + "model.layers.50.mlp.experts.221.down_proj", + "model.layers.50.mlp.experts.221.gate_proj", + "model.layers.50.mlp.experts.221.up_proj", + "model.layers.50.mlp.experts.222.down_proj", + "model.layers.50.mlp.experts.222.gate_proj", + "model.layers.50.mlp.experts.222.up_proj", + "model.layers.50.mlp.experts.223.down_proj", + "model.layers.50.mlp.experts.223.gate_proj", + "model.layers.50.mlp.experts.223.up_proj", + "model.layers.50.mlp.experts.224.down_proj", + "model.layers.50.mlp.experts.224.gate_proj", + "model.layers.50.mlp.experts.224.up_proj", + "model.layers.50.mlp.experts.225.down_proj", + "model.layers.50.mlp.experts.225.gate_proj", + "model.layers.50.mlp.experts.225.up_proj", + "model.layers.50.mlp.experts.226.down_proj", + "model.layers.50.mlp.experts.226.gate_proj", + "model.layers.50.mlp.experts.226.up_proj", + "model.layers.50.mlp.experts.227.down_proj", + "model.layers.50.mlp.experts.227.gate_proj", + "model.layers.50.mlp.experts.227.up_proj", + "model.layers.50.mlp.experts.228.down_proj", + "model.layers.50.mlp.experts.228.gate_proj", + "model.layers.50.mlp.experts.228.up_proj", + "model.layers.50.mlp.experts.229.down_proj", + "model.layers.50.mlp.experts.229.gate_proj", + "model.layers.50.mlp.experts.229.up_proj", + "model.layers.50.mlp.experts.23.down_proj", + "model.layers.50.mlp.experts.23.gate_proj", + "model.layers.50.mlp.experts.23.up_proj", + "model.layers.50.mlp.experts.230.down_proj", + "model.layers.50.mlp.experts.230.gate_proj", + "model.layers.50.mlp.experts.230.up_proj", + "model.layers.50.mlp.experts.231.down_proj", + "model.layers.50.mlp.experts.231.gate_proj", + "model.layers.50.mlp.experts.231.up_proj", + "model.layers.50.mlp.experts.232.down_proj", + "model.layers.50.mlp.experts.232.gate_proj", + "model.layers.50.mlp.experts.232.up_proj", + "model.layers.50.mlp.experts.233.down_proj", + "model.layers.50.mlp.experts.233.gate_proj", + "model.layers.50.mlp.experts.233.up_proj", + "model.layers.50.mlp.experts.234.down_proj", + "model.layers.50.mlp.experts.234.gate_proj", + "model.layers.50.mlp.experts.234.up_proj", + "model.layers.50.mlp.experts.235.down_proj", + "model.layers.50.mlp.experts.235.gate_proj", + "model.layers.50.mlp.experts.235.up_proj", + "model.layers.50.mlp.experts.236.down_proj", + "model.layers.50.mlp.experts.236.gate_proj", + "model.layers.50.mlp.experts.236.up_proj", + "model.layers.50.mlp.experts.237.down_proj", + "model.layers.50.mlp.experts.237.gate_proj", + "model.layers.50.mlp.experts.237.up_proj", + "model.layers.50.mlp.experts.238.down_proj", + "model.layers.50.mlp.experts.238.gate_proj", + "model.layers.50.mlp.experts.238.up_proj", + "model.layers.50.mlp.experts.239.down_proj", + "model.layers.50.mlp.experts.239.gate_proj", + "model.layers.50.mlp.experts.239.up_proj", + "model.layers.50.mlp.experts.24.down_proj", + "model.layers.50.mlp.experts.24.gate_proj", + "model.layers.50.mlp.experts.24.up_proj", + "model.layers.50.mlp.experts.240.down_proj", + "model.layers.50.mlp.experts.240.gate_proj", + "model.layers.50.mlp.experts.240.up_proj", + "model.layers.50.mlp.experts.241.down_proj", + "model.layers.50.mlp.experts.241.gate_proj", + "model.layers.50.mlp.experts.241.up_proj", + "model.layers.50.mlp.experts.242.down_proj", + "model.layers.50.mlp.experts.242.gate_proj", + "model.layers.50.mlp.experts.242.up_proj", + "model.layers.50.mlp.experts.243.down_proj", + "model.layers.50.mlp.experts.243.gate_proj", + "model.layers.50.mlp.experts.243.up_proj", + "model.layers.50.mlp.experts.244.down_proj", + "model.layers.50.mlp.experts.244.gate_proj", + "model.layers.50.mlp.experts.244.up_proj", + "model.layers.50.mlp.experts.245.down_proj", + "model.layers.50.mlp.experts.245.gate_proj", + "model.layers.50.mlp.experts.245.up_proj", + "model.layers.50.mlp.experts.246.down_proj", + "model.layers.50.mlp.experts.246.gate_proj", + "model.layers.50.mlp.experts.246.up_proj", + "model.layers.50.mlp.experts.247.down_proj", + "model.layers.50.mlp.experts.247.gate_proj", + "model.layers.50.mlp.experts.247.up_proj", + "model.layers.50.mlp.experts.248.down_proj", + "model.layers.50.mlp.experts.248.gate_proj", + "model.layers.50.mlp.experts.248.up_proj", + "model.layers.50.mlp.experts.249.down_proj", + "model.layers.50.mlp.experts.249.gate_proj", + "model.layers.50.mlp.experts.249.up_proj", + "model.layers.50.mlp.experts.25.down_proj", + "model.layers.50.mlp.experts.25.gate_proj", + "model.layers.50.mlp.experts.25.up_proj", + "model.layers.50.mlp.experts.250.down_proj", + "model.layers.50.mlp.experts.250.gate_proj", + "model.layers.50.mlp.experts.250.up_proj", + "model.layers.50.mlp.experts.251.down_proj", + "model.layers.50.mlp.experts.251.gate_proj", + "model.layers.50.mlp.experts.251.up_proj", + "model.layers.50.mlp.experts.252.down_proj", + "model.layers.50.mlp.experts.252.gate_proj", + "model.layers.50.mlp.experts.252.up_proj", + "model.layers.50.mlp.experts.253.down_proj", + "model.layers.50.mlp.experts.253.gate_proj", + "model.layers.50.mlp.experts.253.up_proj", + "model.layers.50.mlp.experts.254.down_proj", + "model.layers.50.mlp.experts.254.gate_proj", + "model.layers.50.mlp.experts.254.up_proj", + "model.layers.50.mlp.experts.255.down_proj", + "model.layers.50.mlp.experts.255.gate_proj", + "model.layers.50.mlp.experts.255.up_proj", + "model.layers.50.mlp.experts.26.down_proj", + "model.layers.50.mlp.experts.26.gate_proj", + "model.layers.50.mlp.experts.26.up_proj", + "model.layers.50.mlp.experts.27.down_proj", + "model.layers.50.mlp.experts.27.gate_proj", + "model.layers.50.mlp.experts.27.up_proj", + "model.layers.50.mlp.experts.28.down_proj", + "model.layers.50.mlp.experts.28.gate_proj", + "model.layers.50.mlp.experts.28.up_proj", + "model.layers.50.mlp.experts.29.down_proj", + "model.layers.50.mlp.experts.29.gate_proj", + "model.layers.50.mlp.experts.29.up_proj", + "model.layers.50.mlp.experts.3.down_proj", + "model.layers.50.mlp.experts.3.gate_proj", + "model.layers.50.mlp.experts.3.up_proj", + "model.layers.50.mlp.experts.30.down_proj", + "model.layers.50.mlp.experts.30.gate_proj", + "model.layers.50.mlp.experts.30.up_proj", + "model.layers.50.mlp.experts.31.down_proj", + "model.layers.50.mlp.experts.31.gate_proj", + "model.layers.50.mlp.experts.31.up_proj", + "model.layers.50.mlp.experts.32.down_proj", + "model.layers.50.mlp.experts.32.gate_proj", + "model.layers.50.mlp.experts.32.up_proj", + "model.layers.50.mlp.experts.33.down_proj", + "model.layers.50.mlp.experts.33.gate_proj", + "model.layers.50.mlp.experts.33.up_proj", + "model.layers.50.mlp.experts.34.down_proj", + "model.layers.50.mlp.experts.34.gate_proj", + "model.layers.50.mlp.experts.34.up_proj", + "model.layers.50.mlp.experts.35.down_proj", + "model.layers.50.mlp.experts.35.gate_proj", + "model.layers.50.mlp.experts.35.up_proj", + "model.layers.50.mlp.experts.36.down_proj", + "model.layers.50.mlp.experts.36.gate_proj", + "model.layers.50.mlp.experts.36.up_proj", + "model.layers.50.mlp.experts.37.down_proj", + "model.layers.50.mlp.experts.37.gate_proj", + "model.layers.50.mlp.experts.37.up_proj", + "model.layers.50.mlp.experts.38.down_proj", + "model.layers.50.mlp.experts.38.gate_proj", + "model.layers.50.mlp.experts.38.up_proj", + "model.layers.50.mlp.experts.39.down_proj", + "model.layers.50.mlp.experts.39.gate_proj", + "model.layers.50.mlp.experts.39.up_proj", + "model.layers.50.mlp.experts.4.down_proj", + "model.layers.50.mlp.experts.4.gate_proj", + "model.layers.50.mlp.experts.4.up_proj", + "model.layers.50.mlp.experts.40.down_proj", + "model.layers.50.mlp.experts.40.gate_proj", + "model.layers.50.mlp.experts.40.up_proj", + "model.layers.50.mlp.experts.41.down_proj", + "model.layers.50.mlp.experts.41.gate_proj", + "model.layers.50.mlp.experts.41.up_proj", + "model.layers.50.mlp.experts.42.down_proj", + "model.layers.50.mlp.experts.42.gate_proj", + "model.layers.50.mlp.experts.42.up_proj", + "model.layers.50.mlp.experts.43.down_proj", + "model.layers.50.mlp.experts.43.gate_proj", + "model.layers.50.mlp.experts.43.up_proj", + "model.layers.50.mlp.experts.44.down_proj", + "model.layers.50.mlp.experts.44.gate_proj", + "model.layers.50.mlp.experts.44.up_proj", + "model.layers.50.mlp.experts.45.down_proj", + "model.layers.50.mlp.experts.45.gate_proj", + "model.layers.50.mlp.experts.45.up_proj", + "model.layers.50.mlp.experts.46.down_proj", + "model.layers.50.mlp.experts.46.gate_proj", + "model.layers.50.mlp.experts.46.up_proj", + "model.layers.50.mlp.experts.47.down_proj", + "model.layers.50.mlp.experts.47.gate_proj", + "model.layers.50.mlp.experts.47.up_proj", + "model.layers.50.mlp.experts.48.down_proj", + "model.layers.50.mlp.experts.48.gate_proj", + "model.layers.50.mlp.experts.48.up_proj", + "model.layers.50.mlp.experts.49.down_proj", + "model.layers.50.mlp.experts.49.gate_proj", + "model.layers.50.mlp.experts.49.up_proj", + "model.layers.50.mlp.experts.5.down_proj", + "model.layers.50.mlp.experts.5.gate_proj", + "model.layers.50.mlp.experts.5.up_proj", + "model.layers.50.mlp.experts.50.down_proj", + "model.layers.50.mlp.experts.50.gate_proj", + "model.layers.50.mlp.experts.50.up_proj", + "model.layers.50.mlp.experts.51.down_proj", + "model.layers.50.mlp.experts.51.gate_proj", + "model.layers.50.mlp.experts.51.up_proj", + "model.layers.50.mlp.experts.52.down_proj", + "model.layers.50.mlp.experts.52.gate_proj", + "model.layers.50.mlp.experts.52.up_proj", + "model.layers.50.mlp.experts.53.down_proj", + "model.layers.50.mlp.experts.53.gate_proj", + "model.layers.50.mlp.experts.53.up_proj", + "model.layers.50.mlp.experts.54.down_proj", + "model.layers.50.mlp.experts.54.gate_proj", + "model.layers.50.mlp.experts.54.up_proj", + "model.layers.50.mlp.experts.55.down_proj", + "model.layers.50.mlp.experts.55.gate_proj", + "model.layers.50.mlp.experts.55.up_proj", + "model.layers.50.mlp.experts.56.down_proj", + "model.layers.50.mlp.experts.56.gate_proj", + "model.layers.50.mlp.experts.56.up_proj", + "model.layers.50.mlp.experts.57.down_proj", + "model.layers.50.mlp.experts.57.gate_proj", + "model.layers.50.mlp.experts.57.up_proj", + "model.layers.50.mlp.experts.58.down_proj", + "model.layers.50.mlp.experts.58.gate_proj", + "model.layers.50.mlp.experts.58.up_proj", + "model.layers.50.mlp.experts.59.down_proj", + "model.layers.50.mlp.experts.59.gate_proj", + "model.layers.50.mlp.experts.59.up_proj", + "model.layers.50.mlp.experts.6.down_proj", + "model.layers.50.mlp.experts.6.gate_proj", + "model.layers.50.mlp.experts.6.up_proj", + "model.layers.50.mlp.experts.60.down_proj", + "model.layers.50.mlp.experts.60.gate_proj", + "model.layers.50.mlp.experts.60.up_proj", + "model.layers.50.mlp.experts.61.down_proj", + "model.layers.50.mlp.experts.61.gate_proj", + "model.layers.50.mlp.experts.61.up_proj", + "model.layers.50.mlp.experts.62.down_proj", + "model.layers.50.mlp.experts.62.gate_proj", + "model.layers.50.mlp.experts.62.up_proj", + "model.layers.50.mlp.experts.63.down_proj", + "model.layers.50.mlp.experts.63.gate_proj", + "model.layers.50.mlp.experts.63.up_proj", + "model.layers.50.mlp.experts.64.down_proj", + "model.layers.50.mlp.experts.64.gate_proj", + "model.layers.50.mlp.experts.64.up_proj", + "model.layers.50.mlp.experts.65.down_proj", + "model.layers.50.mlp.experts.65.gate_proj", + "model.layers.50.mlp.experts.65.up_proj", + "model.layers.50.mlp.experts.66.down_proj", + "model.layers.50.mlp.experts.66.gate_proj", + "model.layers.50.mlp.experts.66.up_proj", + "model.layers.50.mlp.experts.67.down_proj", + "model.layers.50.mlp.experts.67.gate_proj", + "model.layers.50.mlp.experts.67.up_proj", + "model.layers.50.mlp.experts.68.down_proj", + "model.layers.50.mlp.experts.68.gate_proj", + "model.layers.50.mlp.experts.68.up_proj", + "model.layers.50.mlp.experts.69.down_proj", + "model.layers.50.mlp.experts.69.gate_proj", + "model.layers.50.mlp.experts.69.up_proj", + "model.layers.50.mlp.experts.7.down_proj", + "model.layers.50.mlp.experts.7.gate_proj", + "model.layers.50.mlp.experts.7.up_proj", + "model.layers.50.mlp.experts.70.down_proj", + "model.layers.50.mlp.experts.70.gate_proj", + "model.layers.50.mlp.experts.70.up_proj", + "model.layers.50.mlp.experts.71.down_proj", + "model.layers.50.mlp.experts.71.gate_proj", + "model.layers.50.mlp.experts.71.up_proj", + "model.layers.50.mlp.experts.72.down_proj", + "model.layers.50.mlp.experts.72.gate_proj", + "model.layers.50.mlp.experts.72.up_proj", + "model.layers.50.mlp.experts.73.down_proj", + "model.layers.50.mlp.experts.73.gate_proj", + "model.layers.50.mlp.experts.73.up_proj", + "model.layers.50.mlp.experts.74.down_proj", + "model.layers.50.mlp.experts.74.gate_proj", + "model.layers.50.mlp.experts.74.up_proj", + "model.layers.50.mlp.experts.75.down_proj", + "model.layers.50.mlp.experts.75.gate_proj", + "model.layers.50.mlp.experts.75.up_proj", + "model.layers.50.mlp.experts.76.down_proj", + "model.layers.50.mlp.experts.76.gate_proj", + "model.layers.50.mlp.experts.76.up_proj", + "model.layers.50.mlp.experts.77.down_proj", + "model.layers.50.mlp.experts.77.gate_proj", + "model.layers.50.mlp.experts.77.up_proj", + "model.layers.50.mlp.experts.78.down_proj", + "model.layers.50.mlp.experts.78.gate_proj", + "model.layers.50.mlp.experts.78.up_proj", + "model.layers.50.mlp.experts.79.down_proj", + "model.layers.50.mlp.experts.79.gate_proj", + "model.layers.50.mlp.experts.79.up_proj", + "model.layers.50.mlp.experts.8.down_proj", + "model.layers.50.mlp.experts.8.gate_proj", + "model.layers.50.mlp.experts.8.up_proj", + "model.layers.50.mlp.experts.80.down_proj", + "model.layers.50.mlp.experts.80.gate_proj", + "model.layers.50.mlp.experts.80.up_proj", + "model.layers.50.mlp.experts.81.down_proj", + "model.layers.50.mlp.experts.81.gate_proj", + "model.layers.50.mlp.experts.81.up_proj", + "model.layers.50.mlp.experts.82.down_proj", + "model.layers.50.mlp.experts.82.gate_proj", + "model.layers.50.mlp.experts.82.up_proj", + "model.layers.50.mlp.experts.83.down_proj", + "model.layers.50.mlp.experts.83.gate_proj", + "model.layers.50.mlp.experts.83.up_proj", + "model.layers.50.mlp.experts.84.down_proj", + "model.layers.50.mlp.experts.84.gate_proj", + "model.layers.50.mlp.experts.84.up_proj", + "model.layers.50.mlp.experts.85.down_proj", + "model.layers.50.mlp.experts.85.gate_proj", + "model.layers.50.mlp.experts.85.up_proj", + "model.layers.50.mlp.experts.86.down_proj", + "model.layers.50.mlp.experts.86.gate_proj", + "model.layers.50.mlp.experts.86.up_proj", + "model.layers.50.mlp.experts.87.down_proj", + "model.layers.50.mlp.experts.87.gate_proj", + "model.layers.50.mlp.experts.87.up_proj", + "model.layers.50.mlp.experts.88.down_proj", + "model.layers.50.mlp.experts.88.gate_proj", + "model.layers.50.mlp.experts.88.up_proj", + "model.layers.50.mlp.experts.89.down_proj", + "model.layers.50.mlp.experts.89.gate_proj", + "model.layers.50.mlp.experts.89.up_proj", + "model.layers.50.mlp.experts.9.down_proj", + "model.layers.50.mlp.experts.9.gate_proj", + "model.layers.50.mlp.experts.9.up_proj", + "model.layers.50.mlp.experts.90.down_proj", + "model.layers.50.mlp.experts.90.gate_proj", + "model.layers.50.mlp.experts.90.up_proj", + "model.layers.50.mlp.experts.91.down_proj", + "model.layers.50.mlp.experts.91.gate_proj", + "model.layers.50.mlp.experts.91.up_proj", + "model.layers.50.mlp.experts.92.down_proj", + "model.layers.50.mlp.experts.92.gate_proj", + "model.layers.50.mlp.experts.92.up_proj", + "model.layers.50.mlp.experts.93.down_proj", + "model.layers.50.mlp.experts.93.gate_proj", + "model.layers.50.mlp.experts.93.up_proj", + "model.layers.50.mlp.experts.94.down_proj", + "model.layers.50.mlp.experts.94.gate_proj", + "model.layers.50.mlp.experts.94.up_proj", + "model.layers.50.mlp.experts.95.down_proj", + "model.layers.50.mlp.experts.95.gate_proj", + "model.layers.50.mlp.experts.95.up_proj", + "model.layers.50.mlp.experts.96.down_proj", + "model.layers.50.mlp.experts.96.gate_proj", + "model.layers.50.mlp.experts.96.up_proj", + "model.layers.50.mlp.experts.97.down_proj", + "model.layers.50.mlp.experts.97.gate_proj", + "model.layers.50.mlp.experts.97.up_proj", + "model.layers.50.mlp.experts.98.down_proj", + "model.layers.50.mlp.experts.98.gate_proj", + "model.layers.50.mlp.experts.98.up_proj", + "model.layers.50.mlp.experts.99.down_proj", + "model.layers.50.mlp.experts.99.gate_proj", + "model.layers.50.mlp.experts.99.up_proj", + "model.layers.51.mlp.experts.0.down_proj", + "model.layers.51.mlp.experts.0.gate_proj", + "model.layers.51.mlp.experts.0.up_proj", + "model.layers.51.mlp.experts.1.down_proj", + "model.layers.51.mlp.experts.1.gate_proj", + "model.layers.51.mlp.experts.1.up_proj", + "model.layers.51.mlp.experts.10.down_proj", + "model.layers.51.mlp.experts.10.gate_proj", + "model.layers.51.mlp.experts.10.up_proj", + "model.layers.51.mlp.experts.100.down_proj", + "model.layers.51.mlp.experts.100.gate_proj", + "model.layers.51.mlp.experts.100.up_proj", + "model.layers.51.mlp.experts.101.down_proj", + "model.layers.51.mlp.experts.101.gate_proj", + "model.layers.51.mlp.experts.101.up_proj", + "model.layers.51.mlp.experts.102.down_proj", + "model.layers.51.mlp.experts.102.gate_proj", + "model.layers.51.mlp.experts.102.up_proj", + "model.layers.51.mlp.experts.103.down_proj", + "model.layers.51.mlp.experts.103.gate_proj", + "model.layers.51.mlp.experts.103.up_proj", + "model.layers.51.mlp.experts.104.down_proj", + "model.layers.51.mlp.experts.104.gate_proj", + "model.layers.51.mlp.experts.104.up_proj", + "model.layers.51.mlp.experts.105.down_proj", + "model.layers.51.mlp.experts.105.gate_proj", + "model.layers.51.mlp.experts.105.up_proj", + "model.layers.51.mlp.experts.106.down_proj", + "model.layers.51.mlp.experts.106.gate_proj", + "model.layers.51.mlp.experts.106.up_proj", + "model.layers.51.mlp.experts.107.down_proj", + "model.layers.51.mlp.experts.107.gate_proj", + "model.layers.51.mlp.experts.107.up_proj", + "model.layers.51.mlp.experts.108.down_proj", + "model.layers.51.mlp.experts.108.gate_proj", + "model.layers.51.mlp.experts.108.up_proj", + "model.layers.51.mlp.experts.109.down_proj", + "model.layers.51.mlp.experts.109.gate_proj", + "model.layers.51.mlp.experts.109.up_proj", + "model.layers.51.mlp.experts.11.down_proj", + "model.layers.51.mlp.experts.11.gate_proj", + "model.layers.51.mlp.experts.11.up_proj", + "model.layers.51.mlp.experts.110.down_proj", + "model.layers.51.mlp.experts.110.gate_proj", + "model.layers.51.mlp.experts.110.up_proj", + "model.layers.51.mlp.experts.111.down_proj", + "model.layers.51.mlp.experts.111.gate_proj", + "model.layers.51.mlp.experts.111.up_proj", + "model.layers.51.mlp.experts.112.down_proj", + "model.layers.51.mlp.experts.112.gate_proj", + "model.layers.51.mlp.experts.112.up_proj", + "model.layers.51.mlp.experts.113.down_proj", + "model.layers.51.mlp.experts.113.gate_proj", + "model.layers.51.mlp.experts.113.up_proj", + "model.layers.51.mlp.experts.114.down_proj", + "model.layers.51.mlp.experts.114.gate_proj", + "model.layers.51.mlp.experts.114.up_proj", + "model.layers.51.mlp.experts.115.down_proj", + "model.layers.51.mlp.experts.115.gate_proj", + "model.layers.51.mlp.experts.115.up_proj", + "model.layers.51.mlp.experts.116.down_proj", + "model.layers.51.mlp.experts.116.gate_proj", + "model.layers.51.mlp.experts.116.up_proj", + "model.layers.51.mlp.experts.117.down_proj", + "model.layers.51.mlp.experts.117.gate_proj", + "model.layers.51.mlp.experts.117.up_proj", + "model.layers.51.mlp.experts.118.down_proj", + "model.layers.51.mlp.experts.118.gate_proj", + "model.layers.51.mlp.experts.118.up_proj", + "model.layers.51.mlp.experts.119.down_proj", + "model.layers.51.mlp.experts.119.gate_proj", + "model.layers.51.mlp.experts.119.up_proj", + "model.layers.51.mlp.experts.12.down_proj", + "model.layers.51.mlp.experts.12.gate_proj", + "model.layers.51.mlp.experts.12.up_proj", + "model.layers.51.mlp.experts.120.down_proj", + "model.layers.51.mlp.experts.120.gate_proj", + "model.layers.51.mlp.experts.120.up_proj", + "model.layers.51.mlp.experts.121.down_proj", + "model.layers.51.mlp.experts.121.gate_proj", + "model.layers.51.mlp.experts.121.up_proj", + "model.layers.51.mlp.experts.122.down_proj", + "model.layers.51.mlp.experts.122.gate_proj", + "model.layers.51.mlp.experts.122.up_proj", + "model.layers.51.mlp.experts.123.down_proj", + "model.layers.51.mlp.experts.123.gate_proj", + "model.layers.51.mlp.experts.123.up_proj", + "model.layers.51.mlp.experts.124.down_proj", + "model.layers.51.mlp.experts.124.gate_proj", + "model.layers.51.mlp.experts.124.up_proj", + "model.layers.51.mlp.experts.125.down_proj", + "model.layers.51.mlp.experts.125.gate_proj", + "model.layers.51.mlp.experts.125.up_proj", + "model.layers.51.mlp.experts.126.down_proj", + "model.layers.51.mlp.experts.126.gate_proj", + "model.layers.51.mlp.experts.126.up_proj", + "model.layers.51.mlp.experts.127.down_proj", + "model.layers.51.mlp.experts.127.gate_proj", + "model.layers.51.mlp.experts.127.up_proj", + "model.layers.51.mlp.experts.128.down_proj", + "model.layers.51.mlp.experts.128.gate_proj", + "model.layers.51.mlp.experts.128.up_proj", + "model.layers.51.mlp.experts.129.down_proj", + "model.layers.51.mlp.experts.129.gate_proj", + "model.layers.51.mlp.experts.129.up_proj", + "model.layers.51.mlp.experts.13.down_proj", + "model.layers.51.mlp.experts.13.gate_proj", + "model.layers.51.mlp.experts.13.up_proj", + "model.layers.51.mlp.experts.130.down_proj", + "model.layers.51.mlp.experts.130.gate_proj", + "model.layers.51.mlp.experts.130.up_proj", + "model.layers.51.mlp.experts.131.down_proj", + "model.layers.51.mlp.experts.131.gate_proj", + "model.layers.51.mlp.experts.131.up_proj", + "model.layers.51.mlp.experts.132.down_proj", + "model.layers.51.mlp.experts.132.gate_proj", + "model.layers.51.mlp.experts.132.up_proj", + "model.layers.51.mlp.experts.133.down_proj", + "model.layers.51.mlp.experts.133.gate_proj", + "model.layers.51.mlp.experts.133.up_proj", + "model.layers.51.mlp.experts.134.down_proj", + "model.layers.51.mlp.experts.134.gate_proj", + "model.layers.51.mlp.experts.134.up_proj", + "model.layers.51.mlp.experts.135.down_proj", + "model.layers.51.mlp.experts.135.gate_proj", + "model.layers.51.mlp.experts.135.up_proj", + "model.layers.51.mlp.experts.136.down_proj", + "model.layers.51.mlp.experts.136.gate_proj", + "model.layers.51.mlp.experts.136.up_proj", + "model.layers.51.mlp.experts.137.down_proj", + "model.layers.51.mlp.experts.137.gate_proj", + "model.layers.51.mlp.experts.137.up_proj", + "model.layers.51.mlp.experts.138.down_proj", + "model.layers.51.mlp.experts.138.gate_proj", + "model.layers.51.mlp.experts.138.up_proj", + "model.layers.51.mlp.experts.139.down_proj", + "model.layers.51.mlp.experts.139.gate_proj", + "model.layers.51.mlp.experts.139.up_proj", + "model.layers.51.mlp.experts.14.down_proj", + "model.layers.51.mlp.experts.14.gate_proj", + "model.layers.51.mlp.experts.14.up_proj", + "model.layers.51.mlp.experts.140.down_proj", + "model.layers.51.mlp.experts.140.gate_proj", + "model.layers.51.mlp.experts.140.up_proj", + "model.layers.51.mlp.experts.141.down_proj", + "model.layers.51.mlp.experts.141.gate_proj", + "model.layers.51.mlp.experts.141.up_proj", + "model.layers.51.mlp.experts.142.down_proj", + "model.layers.51.mlp.experts.142.gate_proj", + "model.layers.51.mlp.experts.142.up_proj", + "model.layers.51.mlp.experts.143.down_proj", + "model.layers.51.mlp.experts.143.gate_proj", + "model.layers.51.mlp.experts.143.up_proj", + "model.layers.51.mlp.experts.144.down_proj", + "model.layers.51.mlp.experts.144.gate_proj", + "model.layers.51.mlp.experts.144.up_proj", + "model.layers.51.mlp.experts.145.down_proj", + "model.layers.51.mlp.experts.145.gate_proj", + "model.layers.51.mlp.experts.145.up_proj", + "model.layers.51.mlp.experts.146.down_proj", + "model.layers.51.mlp.experts.146.gate_proj", + "model.layers.51.mlp.experts.146.up_proj", + "model.layers.51.mlp.experts.147.down_proj", + "model.layers.51.mlp.experts.147.gate_proj", + "model.layers.51.mlp.experts.147.up_proj", + "model.layers.51.mlp.experts.148.down_proj", + "model.layers.51.mlp.experts.148.gate_proj", + "model.layers.51.mlp.experts.148.up_proj", + "model.layers.51.mlp.experts.149.down_proj", + "model.layers.51.mlp.experts.149.gate_proj", + "model.layers.51.mlp.experts.149.up_proj", + "model.layers.51.mlp.experts.15.down_proj", + "model.layers.51.mlp.experts.15.gate_proj", + "model.layers.51.mlp.experts.15.up_proj", + "model.layers.51.mlp.experts.150.down_proj", + "model.layers.51.mlp.experts.150.gate_proj", + "model.layers.51.mlp.experts.150.up_proj", + "model.layers.51.mlp.experts.151.down_proj", + "model.layers.51.mlp.experts.151.gate_proj", + "model.layers.51.mlp.experts.151.up_proj", + "model.layers.51.mlp.experts.152.down_proj", + "model.layers.51.mlp.experts.152.gate_proj", + "model.layers.51.mlp.experts.152.up_proj", + "model.layers.51.mlp.experts.153.down_proj", + "model.layers.51.mlp.experts.153.gate_proj", + "model.layers.51.mlp.experts.153.up_proj", + "model.layers.51.mlp.experts.154.down_proj", + "model.layers.51.mlp.experts.154.gate_proj", + "model.layers.51.mlp.experts.154.up_proj", + "model.layers.51.mlp.experts.155.down_proj", + "model.layers.51.mlp.experts.155.gate_proj", + "model.layers.51.mlp.experts.155.up_proj", + "model.layers.51.mlp.experts.156.down_proj", + "model.layers.51.mlp.experts.156.gate_proj", + "model.layers.51.mlp.experts.156.up_proj", + "model.layers.51.mlp.experts.157.down_proj", + "model.layers.51.mlp.experts.157.gate_proj", + "model.layers.51.mlp.experts.157.up_proj", + "model.layers.51.mlp.experts.158.down_proj", + "model.layers.51.mlp.experts.158.gate_proj", + "model.layers.51.mlp.experts.158.up_proj", + "model.layers.51.mlp.experts.159.down_proj", + "model.layers.51.mlp.experts.159.gate_proj", + "model.layers.51.mlp.experts.159.up_proj", + "model.layers.51.mlp.experts.16.down_proj", + "model.layers.51.mlp.experts.16.gate_proj", + "model.layers.51.mlp.experts.16.up_proj", + "model.layers.51.mlp.experts.160.down_proj", + "model.layers.51.mlp.experts.160.gate_proj", + "model.layers.51.mlp.experts.160.up_proj", + "model.layers.51.mlp.experts.161.down_proj", + "model.layers.51.mlp.experts.161.gate_proj", + "model.layers.51.mlp.experts.161.up_proj", + "model.layers.51.mlp.experts.162.down_proj", + "model.layers.51.mlp.experts.162.gate_proj", + "model.layers.51.mlp.experts.162.up_proj", + "model.layers.51.mlp.experts.163.down_proj", + "model.layers.51.mlp.experts.163.gate_proj", + "model.layers.51.mlp.experts.163.up_proj", + "model.layers.51.mlp.experts.164.down_proj", + "model.layers.51.mlp.experts.164.gate_proj", + "model.layers.51.mlp.experts.164.up_proj", + "model.layers.51.mlp.experts.165.down_proj", + "model.layers.51.mlp.experts.165.gate_proj", + "model.layers.51.mlp.experts.165.up_proj", + "model.layers.51.mlp.experts.166.down_proj", + "model.layers.51.mlp.experts.166.gate_proj", + "model.layers.51.mlp.experts.166.up_proj", + "model.layers.51.mlp.experts.167.down_proj", + "model.layers.51.mlp.experts.167.gate_proj", + "model.layers.51.mlp.experts.167.up_proj", + "model.layers.51.mlp.experts.168.down_proj", + "model.layers.51.mlp.experts.168.gate_proj", + "model.layers.51.mlp.experts.168.up_proj", + "model.layers.51.mlp.experts.169.down_proj", + "model.layers.51.mlp.experts.169.gate_proj", + "model.layers.51.mlp.experts.169.up_proj", + "model.layers.51.mlp.experts.17.down_proj", + "model.layers.51.mlp.experts.17.gate_proj", + "model.layers.51.mlp.experts.17.up_proj", + "model.layers.51.mlp.experts.170.down_proj", + "model.layers.51.mlp.experts.170.gate_proj", + "model.layers.51.mlp.experts.170.up_proj", + "model.layers.51.mlp.experts.171.down_proj", + "model.layers.51.mlp.experts.171.gate_proj", + "model.layers.51.mlp.experts.171.up_proj", + "model.layers.51.mlp.experts.172.down_proj", + "model.layers.51.mlp.experts.172.gate_proj", + "model.layers.51.mlp.experts.172.up_proj", + "model.layers.51.mlp.experts.173.down_proj", + "model.layers.51.mlp.experts.173.gate_proj", + "model.layers.51.mlp.experts.173.up_proj", + "model.layers.51.mlp.experts.174.down_proj", + "model.layers.51.mlp.experts.174.gate_proj", + "model.layers.51.mlp.experts.174.up_proj", + "model.layers.51.mlp.experts.175.down_proj", + "model.layers.51.mlp.experts.175.gate_proj", + "model.layers.51.mlp.experts.175.up_proj", + "model.layers.51.mlp.experts.176.down_proj", + "model.layers.51.mlp.experts.176.gate_proj", + "model.layers.51.mlp.experts.176.up_proj", + "model.layers.51.mlp.experts.177.down_proj", + "model.layers.51.mlp.experts.177.gate_proj", + "model.layers.51.mlp.experts.177.up_proj", + "model.layers.51.mlp.experts.178.down_proj", + "model.layers.51.mlp.experts.178.gate_proj", + "model.layers.51.mlp.experts.178.up_proj", + "model.layers.51.mlp.experts.179.down_proj", + "model.layers.51.mlp.experts.179.gate_proj", + "model.layers.51.mlp.experts.179.up_proj", + "model.layers.51.mlp.experts.18.down_proj", + "model.layers.51.mlp.experts.18.gate_proj", + "model.layers.51.mlp.experts.18.up_proj", + "model.layers.51.mlp.experts.180.down_proj", + "model.layers.51.mlp.experts.180.gate_proj", + "model.layers.51.mlp.experts.180.up_proj", + "model.layers.51.mlp.experts.181.down_proj", + "model.layers.51.mlp.experts.181.gate_proj", + "model.layers.51.mlp.experts.181.up_proj", + "model.layers.51.mlp.experts.182.down_proj", + "model.layers.51.mlp.experts.182.gate_proj", + "model.layers.51.mlp.experts.182.up_proj", + "model.layers.51.mlp.experts.183.down_proj", + "model.layers.51.mlp.experts.183.gate_proj", + "model.layers.51.mlp.experts.183.up_proj", + "model.layers.51.mlp.experts.184.down_proj", + "model.layers.51.mlp.experts.184.gate_proj", + "model.layers.51.mlp.experts.184.up_proj", + "model.layers.51.mlp.experts.185.down_proj", + "model.layers.51.mlp.experts.185.gate_proj", + "model.layers.51.mlp.experts.185.up_proj", + "model.layers.51.mlp.experts.186.down_proj", + "model.layers.51.mlp.experts.186.gate_proj", + "model.layers.51.mlp.experts.186.up_proj", + "model.layers.51.mlp.experts.187.down_proj", + "model.layers.51.mlp.experts.187.gate_proj", + "model.layers.51.mlp.experts.187.up_proj", + "model.layers.51.mlp.experts.188.down_proj", + "model.layers.51.mlp.experts.188.gate_proj", + "model.layers.51.mlp.experts.188.up_proj", + "model.layers.51.mlp.experts.189.down_proj", + "model.layers.51.mlp.experts.189.gate_proj", + "model.layers.51.mlp.experts.189.up_proj", + "model.layers.51.mlp.experts.19.down_proj", + "model.layers.51.mlp.experts.19.gate_proj", + "model.layers.51.mlp.experts.19.up_proj", + "model.layers.51.mlp.experts.190.down_proj", + "model.layers.51.mlp.experts.190.gate_proj", + "model.layers.51.mlp.experts.190.up_proj", + "model.layers.51.mlp.experts.191.down_proj", + "model.layers.51.mlp.experts.191.gate_proj", + "model.layers.51.mlp.experts.191.up_proj", + "model.layers.51.mlp.experts.192.down_proj", + "model.layers.51.mlp.experts.192.gate_proj", + "model.layers.51.mlp.experts.192.up_proj", + "model.layers.51.mlp.experts.193.down_proj", + "model.layers.51.mlp.experts.193.gate_proj", + "model.layers.51.mlp.experts.193.up_proj", + "model.layers.51.mlp.experts.194.down_proj", + "model.layers.51.mlp.experts.194.gate_proj", + "model.layers.51.mlp.experts.194.up_proj", + "model.layers.51.mlp.experts.195.down_proj", + "model.layers.51.mlp.experts.195.gate_proj", + "model.layers.51.mlp.experts.195.up_proj", + "model.layers.51.mlp.experts.196.down_proj", + "model.layers.51.mlp.experts.196.gate_proj", + "model.layers.51.mlp.experts.196.up_proj", + "model.layers.51.mlp.experts.197.down_proj", + "model.layers.51.mlp.experts.197.gate_proj", + "model.layers.51.mlp.experts.197.up_proj", + "model.layers.51.mlp.experts.198.down_proj", + "model.layers.51.mlp.experts.198.gate_proj", + "model.layers.51.mlp.experts.198.up_proj", + "model.layers.51.mlp.experts.199.down_proj", + "model.layers.51.mlp.experts.199.gate_proj", + "model.layers.51.mlp.experts.199.up_proj", + "model.layers.51.mlp.experts.2.down_proj", + "model.layers.51.mlp.experts.2.gate_proj", + "model.layers.51.mlp.experts.2.up_proj", + "model.layers.51.mlp.experts.20.down_proj", + "model.layers.51.mlp.experts.20.gate_proj", + "model.layers.51.mlp.experts.20.up_proj", + "model.layers.51.mlp.experts.200.down_proj", + "model.layers.51.mlp.experts.200.gate_proj", + "model.layers.51.mlp.experts.200.up_proj", + "model.layers.51.mlp.experts.201.down_proj", + "model.layers.51.mlp.experts.201.gate_proj", + "model.layers.51.mlp.experts.201.up_proj", + "model.layers.51.mlp.experts.202.down_proj", + "model.layers.51.mlp.experts.202.gate_proj", + "model.layers.51.mlp.experts.202.up_proj", + "model.layers.51.mlp.experts.203.down_proj", + "model.layers.51.mlp.experts.203.gate_proj", + "model.layers.51.mlp.experts.203.up_proj", + "model.layers.51.mlp.experts.204.down_proj", + "model.layers.51.mlp.experts.204.gate_proj", + "model.layers.51.mlp.experts.204.up_proj", + "model.layers.51.mlp.experts.205.down_proj", + "model.layers.51.mlp.experts.205.gate_proj", + "model.layers.51.mlp.experts.205.up_proj", + "model.layers.51.mlp.experts.206.down_proj", + "model.layers.51.mlp.experts.206.gate_proj", + "model.layers.51.mlp.experts.206.up_proj", + "model.layers.51.mlp.experts.207.down_proj", + "model.layers.51.mlp.experts.207.gate_proj", + "model.layers.51.mlp.experts.207.up_proj", + "model.layers.51.mlp.experts.208.down_proj", + "model.layers.51.mlp.experts.208.gate_proj", + "model.layers.51.mlp.experts.208.up_proj", + "model.layers.51.mlp.experts.209.down_proj", + "model.layers.51.mlp.experts.209.gate_proj", + "model.layers.51.mlp.experts.209.up_proj", + "model.layers.51.mlp.experts.21.down_proj", + "model.layers.51.mlp.experts.21.gate_proj", + "model.layers.51.mlp.experts.21.up_proj", + "model.layers.51.mlp.experts.210.down_proj", + "model.layers.51.mlp.experts.210.gate_proj", + "model.layers.51.mlp.experts.210.up_proj", + "model.layers.51.mlp.experts.211.down_proj", + "model.layers.51.mlp.experts.211.gate_proj", + "model.layers.51.mlp.experts.211.up_proj", + "model.layers.51.mlp.experts.212.down_proj", + "model.layers.51.mlp.experts.212.gate_proj", + "model.layers.51.mlp.experts.212.up_proj", + "model.layers.51.mlp.experts.213.down_proj", + "model.layers.51.mlp.experts.213.gate_proj", + "model.layers.51.mlp.experts.213.up_proj", + "model.layers.51.mlp.experts.214.down_proj", + "model.layers.51.mlp.experts.214.gate_proj", + "model.layers.51.mlp.experts.214.up_proj", + "model.layers.51.mlp.experts.215.down_proj", + "model.layers.51.mlp.experts.215.gate_proj", + "model.layers.51.mlp.experts.215.up_proj", + "model.layers.51.mlp.experts.216.down_proj", + "model.layers.51.mlp.experts.216.gate_proj", + "model.layers.51.mlp.experts.216.up_proj", + "model.layers.51.mlp.experts.217.down_proj", + "model.layers.51.mlp.experts.217.gate_proj", + "model.layers.51.mlp.experts.217.up_proj", + "model.layers.51.mlp.experts.218.down_proj", + "model.layers.51.mlp.experts.218.gate_proj", + "model.layers.51.mlp.experts.218.up_proj", + "model.layers.51.mlp.experts.219.down_proj", + "model.layers.51.mlp.experts.219.gate_proj", + "model.layers.51.mlp.experts.219.up_proj", + "model.layers.51.mlp.experts.22.down_proj", + "model.layers.51.mlp.experts.22.gate_proj", + "model.layers.51.mlp.experts.22.up_proj", + "model.layers.51.mlp.experts.220.down_proj", + "model.layers.51.mlp.experts.220.gate_proj", + "model.layers.51.mlp.experts.220.up_proj", + "model.layers.51.mlp.experts.221.down_proj", + "model.layers.51.mlp.experts.221.gate_proj", + "model.layers.51.mlp.experts.221.up_proj", + "model.layers.51.mlp.experts.222.down_proj", + "model.layers.51.mlp.experts.222.gate_proj", + "model.layers.51.mlp.experts.222.up_proj", + "model.layers.51.mlp.experts.223.down_proj", + "model.layers.51.mlp.experts.223.gate_proj", + "model.layers.51.mlp.experts.223.up_proj", + "model.layers.51.mlp.experts.224.down_proj", + "model.layers.51.mlp.experts.224.gate_proj", + "model.layers.51.mlp.experts.224.up_proj", + "model.layers.51.mlp.experts.225.down_proj", + "model.layers.51.mlp.experts.225.gate_proj", + "model.layers.51.mlp.experts.225.up_proj", + "model.layers.51.mlp.experts.226.down_proj", + "model.layers.51.mlp.experts.226.gate_proj", + "model.layers.51.mlp.experts.226.up_proj", + "model.layers.51.mlp.experts.227.down_proj", + "model.layers.51.mlp.experts.227.gate_proj", + "model.layers.51.mlp.experts.227.up_proj", + "model.layers.51.mlp.experts.228.down_proj", + "model.layers.51.mlp.experts.228.gate_proj", + "model.layers.51.mlp.experts.228.up_proj", + "model.layers.51.mlp.experts.229.down_proj", + "model.layers.51.mlp.experts.229.gate_proj", + "model.layers.51.mlp.experts.229.up_proj", + "model.layers.51.mlp.experts.23.down_proj", + "model.layers.51.mlp.experts.23.gate_proj", + "model.layers.51.mlp.experts.23.up_proj", + "model.layers.51.mlp.experts.230.down_proj", + "model.layers.51.mlp.experts.230.gate_proj", + "model.layers.51.mlp.experts.230.up_proj", + "model.layers.51.mlp.experts.231.down_proj", + "model.layers.51.mlp.experts.231.gate_proj", + "model.layers.51.mlp.experts.231.up_proj", + "model.layers.51.mlp.experts.232.down_proj", + "model.layers.51.mlp.experts.232.gate_proj", + "model.layers.51.mlp.experts.232.up_proj", + "model.layers.51.mlp.experts.233.down_proj", + "model.layers.51.mlp.experts.233.gate_proj", + "model.layers.51.mlp.experts.233.up_proj", + "model.layers.51.mlp.experts.234.down_proj", + "model.layers.51.mlp.experts.234.gate_proj", + "model.layers.51.mlp.experts.234.up_proj", + "model.layers.51.mlp.experts.235.down_proj", + "model.layers.51.mlp.experts.235.gate_proj", + "model.layers.51.mlp.experts.235.up_proj", + "model.layers.51.mlp.experts.236.down_proj", + "model.layers.51.mlp.experts.236.gate_proj", + "model.layers.51.mlp.experts.236.up_proj", + "model.layers.51.mlp.experts.237.down_proj", + "model.layers.51.mlp.experts.237.gate_proj", + "model.layers.51.mlp.experts.237.up_proj", + "model.layers.51.mlp.experts.238.down_proj", + "model.layers.51.mlp.experts.238.gate_proj", + "model.layers.51.mlp.experts.238.up_proj", + "model.layers.51.mlp.experts.239.down_proj", + "model.layers.51.mlp.experts.239.gate_proj", + "model.layers.51.mlp.experts.239.up_proj", + "model.layers.51.mlp.experts.24.down_proj", + "model.layers.51.mlp.experts.24.gate_proj", + "model.layers.51.mlp.experts.24.up_proj", + "model.layers.51.mlp.experts.240.down_proj", + "model.layers.51.mlp.experts.240.gate_proj", + "model.layers.51.mlp.experts.240.up_proj", + "model.layers.51.mlp.experts.241.down_proj", + "model.layers.51.mlp.experts.241.gate_proj", + "model.layers.51.mlp.experts.241.up_proj", + "model.layers.51.mlp.experts.242.down_proj", + "model.layers.51.mlp.experts.242.gate_proj", + "model.layers.51.mlp.experts.242.up_proj", + "model.layers.51.mlp.experts.243.down_proj", + "model.layers.51.mlp.experts.243.gate_proj", + "model.layers.51.mlp.experts.243.up_proj", + "model.layers.51.mlp.experts.244.down_proj", + "model.layers.51.mlp.experts.244.gate_proj", + "model.layers.51.mlp.experts.244.up_proj", + "model.layers.51.mlp.experts.245.down_proj", + "model.layers.51.mlp.experts.245.gate_proj", + "model.layers.51.mlp.experts.245.up_proj", + "model.layers.51.mlp.experts.246.down_proj", + "model.layers.51.mlp.experts.246.gate_proj", + "model.layers.51.mlp.experts.246.up_proj", + "model.layers.51.mlp.experts.247.down_proj", + "model.layers.51.mlp.experts.247.gate_proj", + "model.layers.51.mlp.experts.247.up_proj", + "model.layers.51.mlp.experts.248.down_proj", + "model.layers.51.mlp.experts.248.gate_proj", + "model.layers.51.mlp.experts.248.up_proj", + "model.layers.51.mlp.experts.249.down_proj", + "model.layers.51.mlp.experts.249.gate_proj", + "model.layers.51.mlp.experts.249.up_proj", + "model.layers.51.mlp.experts.25.down_proj", + "model.layers.51.mlp.experts.25.gate_proj", + "model.layers.51.mlp.experts.25.up_proj", + "model.layers.51.mlp.experts.250.down_proj", + "model.layers.51.mlp.experts.250.gate_proj", + "model.layers.51.mlp.experts.250.up_proj", + "model.layers.51.mlp.experts.251.down_proj", + "model.layers.51.mlp.experts.251.gate_proj", + "model.layers.51.mlp.experts.251.up_proj", + "model.layers.51.mlp.experts.252.down_proj", + "model.layers.51.mlp.experts.252.gate_proj", + "model.layers.51.mlp.experts.252.up_proj", + "model.layers.51.mlp.experts.253.down_proj", + "model.layers.51.mlp.experts.253.gate_proj", + "model.layers.51.mlp.experts.253.up_proj", + "model.layers.51.mlp.experts.254.down_proj", + "model.layers.51.mlp.experts.254.gate_proj", + "model.layers.51.mlp.experts.254.up_proj", + "model.layers.51.mlp.experts.255.down_proj", + "model.layers.51.mlp.experts.255.gate_proj", + "model.layers.51.mlp.experts.255.up_proj", + "model.layers.51.mlp.experts.26.down_proj", + "model.layers.51.mlp.experts.26.gate_proj", + "model.layers.51.mlp.experts.26.up_proj", + "model.layers.51.mlp.experts.27.down_proj", + "model.layers.51.mlp.experts.27.gate_proj", + "model.layers.51.mlp.experts.27.up_proj", + "model.layers.51.mlp.experts.28.down_proj", + "model.layers.51.mlp.experts.28.gate_proj", + "model.layers.51.mlp.experts.28.up_proj", + "model.layers.51.mlp.experts.29.down_proj", + "model.layers.51.mlp.experts.29.gate_proj", + "model.layers.51.mlp.experts.29.up_proj", + "model.layers.51.mlp.experts.3.down_proj", + "model.layers.51.mlp.experts.3.gate_proj", + "model.layers.51.mlp.experts.3.up_proj", + "model.layers.51.mlp.experts.30.down_proj", + "model.layers.51.mlp.experts.30.gate_proj", + "model.layers.51.mlp.experts.30.up_proj", + "model.layers.51.mlp.experts.31.down_proj", + "model.layers.51.mlp.experts.31.gate_proj", + "model.layers.51.mlp.experts.31.up_proj", + "model.layers.51.mlp.experts.32.down_proj", + "model.layers.51.mlp.experts.32.gate_proj", + "model.layers.51.mlp.experts.32.up_proj", + "model.layers.51.mlp.experts.33.down_proj", + "model.layers.51.mlp.experts.33.gate_proj", + "model.layers.51.mlp.experts.33.up_proj", + "model.layers.51.mlp.experts.34.down_proj", + "model.layers.51.mlp.experts.34.gate_proj", + "model.layers.51.mlp.experts.34.up_proj", + "model.layers.51.mlp.experts.35.down_proj", + "model.layers.51.mlp.experts.35.gate_proj", + "model.layers.51.mlp.experts.35.up_proj", + "model.layers.51.mlp.experts.36.down_proj", + "model.layers.51.mlp.experts.36.gate_proj", + "model.layers.51.mlp.experts.36.up_proj", + "model.layers.51.mlp.experts.37.down_proj", + "model.layers.51.mlp.experts.37.gate_proj", + "model.layers.51.mlp.experts.37.up_proj", + "model.layers.51.mlp.experts.38.down_proj", + "model.layers.51.mlp.experts.38.gate_proj", + "model.layers.51.mlp.experts.38.up_proj", + "model.layers.51.mlp.experts.39.down_proj", + "model.layers.51.mlp.experts.39.gate_proj", + "model.layers.51.mlp.experts.39.up_proj", + "model.layers.51.mlp.experts.4.down_proj", + "model.layers.51.mlp.experts.4.gate_proj", + "model.layers.51.mlp.experts.4.up_proj", + "model.layers.51.mlp.experts.40.down_proj", + "model.layers.51.mlp.experts.40.gate_proj", + "model.layers.51.mlp.experts.40.up_proj", + "model.layers.51.mlp.experts.41.down_proj", + "model.layers.51.mlp.experts.41.gate_proj", + "model.layers.51.mlp.experts.41.up_proj", + "model.layers.51.mlp.experts.42.down_proj", + "model.layers.51.mlp.experts.42.gate_proj", + "model.layers.51.mlp.experts.42.up_proj", + "model.layers.51.mlp.experts.43.down_proj", + "model.layers.51.mlp.experts.43.gate_proj", + "model.layers.51.mlp.experts.43.up_proj", + "model.layers.51.mlp.experts.44.down_proj", + "model.layers.51.mlp.experts.44.gate_proj", + "model.layers.51.mlp.experts.44.up_proj", + "model.layers.51.mlp.experts.45.down_proj", + "model.layers.51.mlp.experts.45.gate_proj", + "model.layers.51.mlp.experts.45.up_proj", + "model.layers.51.mlp.experts.46.down_proj", + "model.layers.51.mlp.experts.46.gate_proj", + "model.layers.51.mlp.experts.46.up_proj", + "model.layers.51.mlp.experts.47.down_proj", + "model.layers.51.mlp.experts.47.gate_proj", + "model.layers.51.mlp.experts.47.up_proj", + "model.layers.51.mlp.experts.48.down_proj", + "model.layers.51.mlp.experts.48.gate_proj", + "model.layers.51.mlp.experts.48.up_proj", + "model.layers.51.mlp.experts.49.down_proj", + "model.layers.51.mlp.experts.49.gate_proj", + "model.layers.51.mlp.experts.49.up_proj", + "model.layers.51.mlp.experts.5.down_proj", + "model.layers.51.mlp.experts.5.gate_proj", + "model.layers.51.mlp.experts.5.up_proj", + "model.layers.51.mlp.experts.50.down_proj", + "model.layers.51.mlp.experts.50.gate_proj", + "model.layers.51.mlp.experts.50.up_proj", + "model.layers.51.mlp.experts.51.down_proj", + "model.layers.51.mlp.experts.51.gate_proj", + "model.layers.51.mlp.experts.51.up_proj", + "model.layers.51.mlp.experts.52.down_proj", + "model.layers.51.mlp.experts.52.gate_proj", + "model.layers.51.mlp.experts.52.up_proj", + "model.layers.51.mlp.experts.53.down_proj", + "model.layers.51.mlp.experts.53.gate_proj", + "model.layers.51.mlp.experts.53.up_proj", + "model.layers.51.mlp.experts.54.down_proj", + "model.layers.51.mlp.experts.54.gate_proj", + "model.layers.51.mlp.experts.54.up_proj", + "model.layers.51.mlp.experts.55.down_proj", + "model.layers.51.mlp.experts.55.gate_proj", + "model.layers.51.mlp.experts.55.up_proj", + "model.layers.51.mlp.experts.56.down_proj", + "model.layers.51.mlp.experts.56.gate_proj", + "model.layers.51.mlp.experts.56.up_proj", + "model.layers.51.mlp.experts.57.down_proj", + "model.layers.51.mlp.experts.57.gate_proj", + "model.layers.51.mlp.experts.57.up_proj", + "model.layers.51.mlp.experts.58.down_proj", + "model.layers.51.mlp.experts.58.gate_proj", + "model.layers.51.mlp.experts.58.up_proj", + "model.layers.51.mlp.experts.59.down_proj", + "model.layers.51.mlp.experts.59.gate_proj", + "model.layers.51.mlp.experts.59.up_proj", + "model.layers.51.mlp.experts.6.down_proj", + "model.layers.51.mlp.experts.6.gate_proj", + "model.layers.51.mlp.experts.6.up_proj", + "model.layers.51.mlp.experts.60.down_proj", + "model.layers.51.mlp.experts.60.gate_proj", + "model.layers.51.mlp.experts.60.up_proj", + "model.layers.51.mlp.experts.61.down_proj", + "model.layers.51.mlp.experts.61.gate_proj", + "model.layers.51.mlp.experts.61.up_proj", + "model.layers.51.mlp.experts.62.down_proj", + "model.layers.51.mlp.experts.62.gate_proj", + "model.layers.51.mlp.experts.62.up_proj", + "model.layers.51.mlp.experts.63.down_proj", + "model.layers.51.mlp.experts.63.gate_proj", + "model.layers.51.mlp.experts.63.up_proj", + "model.layers.51.mlp.experts.64.down_proj", + "model.layers.51.mlp.experts.64.gate_proj", + "model.layers.51.mlp.experts.64.up_proj", + "model.layers.51.mlp.experts.65.down_proj", + "model.layers.51.mlp.experts.65.gate_proj", + "model.layers.51.mlp.experts.65.up_proj", + "model.layers.51.mlp.experts.66.down_proj", + "model.layers.51.mlp.experts.66.gate_proj", + "model.layers.51.mlp.experts.66.up_proj", + "model.layers.51.mlp.experts.67.down_proj", + "model.layers.51.mlp.experts.67.gate_proj", + "model.layers.51.mlp.experts.67.up_proj", + "model.layers.51.mlp.experts.68.down_proj", + "model.layers.51.mlp.experts.68.gate_proj", + "model.layers.51.mlp.experts.68.up_proj", + "model.layers.51.mlp.experts.69.down_proj", + "model.layers.51.mlp.experts.69.gate_proj", + "model.layers.51.mlp.experts.69.up_proj", + "model.layers.51.mlp.experts.7.down_proj", + "model.layers.51.mlp.experts.7.gate_proj", + "model.layers.51.mlp.experts.7.up_proj", + "model.layers.51.mlp.experts.70.down_proj", + "model.layers.51.mlp.experts.70.gate_proj", + "model.layers.51.mlp.experts.70.up_proj", + "model.layers.51.mlp.experts.71.down_proj", + "model.layers.51.mlp.experts.71.gate_proj", + "model.layers.51.mlp.experts.71.up_proj", + "model.layers.51.mlp.experts.72.down_proj", + "model.layers.51.mlp.experts.72.gate_proj", + "model.layers.51.mlp.experts.72.up_proj", + "model.layers.51.mlp.experts.73.down_proj", + "model.layers.51.mlp.experts.73.gate_proj", + "model.layers.51.mlp.experts.73.up_proj", + "model.layers.51.mlp.experts.74.down_proj", + "model.layers.51.mlp.experts.74.gate_proj", + "model.layers.51.mlp.experts.74.up_proj", + "model.layers.51.mlp.experts.75.down_proj", + "model.layers.51.mlp.experts.75.gate_proj", + "model.layers.51.mlp.experts.75.up_proj", + "model.layers.51.mlp.experts.76.down_proj", + "model.layers.51.mlp.experts.76.gate_proj", + "model.layers.51.mlp.experts.76.up_proj", + "model.layers.51.mlp.experts.77.down_proj", + "model.layers.51.mlp.experts.77.gate_proj", + "model.layers.51.mlp.experts.77.up_proj", + "model.layers.51.mlp.experts.78.down_proj", + "model.layers.51.mlp.experts.78.gate_proj", + "model.layers.51.mlp.experts.78.up_proj", + "model.layers.51.mlp.experts.79.down_proj", + "model.layers.51.mlp.experts.79.gate_proj", + "model.layers.51.mlp.experts.79.up_proj", + "model.layers.51.mlp.experts.8.down_proj", + "model.layers.51.mlp.experts.8.gate_proj", + "model.layers.51.mlp.experts.8.up_proj", + "model.layers.51.mlp.experts.80.down_proj", + "model.layers.51.mlp.experts.80.gate_proj", + "model.layers.51.mlp.experts.80.up_proj", + "model.layers.51.mlp.experts.81.down_proj", + "model.layers.51.mlp.experts.81.gate_proj", + "model.layers.51.mlp.experts.81.up_proj", + "model.layers.51.mlp.experts.82.down_proj", + "model.layers.51.mlp.experts.82.gate_proj", + "model.layers.51.mlp.experts.82.up_proj", + "model.layers.51.mlp.experts.83.down_proj", + "model.layers.51.mlp.experts.83.gate_proj", + "model.layers.51.mlp.experts.83.up_proj", + "model.layers.51.mlp.experts.84.down_proj", + "model.layers.51.mlp.experts.84.gate_proj", + "model.layers.51.mlp.experts.84.up_proj", + "model.layers.51.mlp.experts.85.down_proj", + "model.layers.51.mlp.experts.85.gate_proj", + "model.layers.51.mlp.experts.85.up_proj", + "model.layers.51.mlp.experts.86.down_proj", + "model.layers.51.mlp.experts.86.gate_proj", + "model.layers.51.mlp.experts.86.up_proj", + "model.layers.51.mlp.experts.87.down_proj", + "model.layers.51.mlp.experts.87.gate_proj", + "model.layers.51.mlp.experts.87.up_proj", + "model.layers.51.mlp.experts.88.down_proj", + "model.layers.51.mlp.experts.88.gate_proj", + "model.layers.51.mlp.experts.88.up_proj", + "model.layers.51.mlp.experts.89.down_proj", + "model.layers.51.mlp.experts.89.gate_proj", + "model.layers.51.mlp.experts.89.up_proj", + "model.layers.51.mlp.experts.9.down_proj", + "model.layers.51.mlp.experts.9.gate_proj", + "model.layers.51.mlp.experts.9.up_proj", + "model.layers.51.mlp.experts.90.down_proj", + "model.layers.51.mlp.experts.90.gate_proj", + "model.layers.51.mlp.experts.90.up_proj", + "model.layers.51.mlp.experts.91.down_proj", + "model.layers.51.mlp.experts.91.gate_proj", + "model.layers.51.mlp.experts.91.up_proj", + "model.layers.51.mlp.experts.92.down_proj", + "model.layers.51.mlp.experts.92.gate_proj", + "model.layers.51.mlp.experts.92.up_proj", + "model.layers.51.mlp.experts.93.down_proj", + "model.layers.51.mlp.experts.93.gate_proj", + "model.layers.51.mlp.experts.93.up_proj", + "model.layers.51.mlp.experts.94.down_proj", + "model.layers.51.mlp.experts.94.gate_proj", + "model.layers.51.mlp.experts.94.up_proj", + "model.layers.51.mlp.experts.95.down_proj", + "model.layers.51.mlp.experts.95.gate_proj", + "model.layers.51.mlp.experts.95.up_proj", + "model.layers.51.mlp.experts.96.down_proj", + "model.layers.51.mlp.experts.96.gate_proj", + "model.layers.51.mlp.experts.96.up_proj", + "model.layers.51.mlp.experts.97.down_proj", + "model.layers.51.mlp.experts.97.gate_proj", + "model.layers.51.mlp.experts.97.up_proj", + "model.layers.51.mlp.experts.98.down_proj", + "model.layers.51.mlp.experts.98.gate_proj", + "model.layers.51.mlp.experts.98.up_proj", + "model.layers.51.mlp.experts.99.down_proj", + "model.layers.51.mlp.experts.99.gate_proj", + "model.layers.51.mlp.experts.99.up_proj", + "model.layers.52.mlp.experts.0.down_proj", + "model.layers.52.mlp.experts.0.gate_proj", + "model.layers.52.mlp.experts.0.up_proj", + "model.layers.52.mlp.experts.1.down_proj", + "model.layers.52.mlp.experts.1.gate_proj", + "model.layers.52.mlp.experts.1.up_proj", + "model.layers.52.mlp.experts.10.down_proj", + "model.layers.52.mlp.experts.10.gate_proj", + "model.layers.52.mlp.experts.10.up_proj", + "model.layers.52.mlp.experts.100.down_proj", + "model.layers.52.mlp.experts.100.gate_proj", + "model.layers.52.mlp.experts.100.up_proj", + "model.layers.52.mlp.experts.101.down_proj", + "model.layers.52.mlp.experts.101.gate_proj", + "model.layers.52.mlp.experts.101.up_proj", + "model.layers.52.mlp.experts.102.down_proj", + "model.layers.52.mlp.experts.102.gate_proj", + "model.layers.52.mlp.experts.102.up_proj", + "model.layers.52.mlp.experts.103.down_proj", + "model.layers.52.mlp.experts.103.gate_proj", + "model.layers.52.mlp.experts.103.up_proj", + "model.layers.52.mlp.experts.104.down_proj", + "model.layers.52.mlp.experts.104.gate_proj", + "model.layers.52.mlp.experts.104.up_proj", + "model.layers.52.mlp.experts.105.down_proj", + "model.layers.52.mlp.experts.105.gate_proj", + "model.layers.52.mlp.experts.105.up_proj", + "model.layers.52.mlp.experts.106.down_proj", + "model.layers.52.mlp.experts.106.gate_proj", + "model.layers.52.mlp.experts.106.up_proj", + "model.layers.52.mlp.experts.107.down_proj", + "model.layers.52.mlp.experts.107.gate_proj", + "model.layers.52.mlp.experts.107.up_proj", + "model.layers.52.mlp.experts.108.down_proj", + "model.layers.52.mlp.experts.108.gate_proj", + "model.layers.52.mlp.experts.108.up_proj", + "model.layers.52.mlp.experts.109.down_proj", + "model.layers.52.mlp.experts.109.gate_proj", + "model.layers.52.mlp.experts.109.up_proj", + "model.layers.52.mlp.experts.11.down_proj", + "model.layers.52.mlp.experts.11.gate_proj", + "model.layers.52.mlp.experts.11.up_proj", + "model.layers.52.mlp.experts.110.down_proj", + "model.layers.52.mlp.experts.110.gate_proj", + "model.layers.52.mlp.experts.110.up_proj", + "model.layers.52.mlp.experts.111.down_proj", + "model.layers.52.mlp.experts.111.gate_proj", + "model.layers.52.mlp.experts.111.up_proj", + "model.layers.52.mlp.experts.112.down_proj", + "model.layers.52.mlp.experts.112.gate_proj", + "model.layers.52.mlp.experts.112.up_proj", + "model.layers.52.mlp.experts.113.down_proj", + "model.layers.52.mlp.experts.113.gate_proj", + "model.layers.52.mlp.experts.113.up_proj", + "model.layers.52.mlp.experts.114.down_proj", + "model.layers.52.mlp.experts.114.gate_proj", + "model.layers.52.mlp.experts.114.up_proj", + "model.layers.52.mlp.experts.115.down_proj", + "model.layers.52.mlp.experts.115.gate_proj", + "model.layers.52.mlp.experts.115.up_proj", + "model.layers.52.mlp.experts.116.down_proj", + "model.layers.52.mlp.experts.116.gate_proj", + "model.layers.52.mlp.experts.116.up_proj", + "model.layers.52.mlp.experts.117.down_proj", + "model.layers.52.mlp.experts.117.gate_proj", + "model.layers.52.mlp.experts.117.up_proj", + "model.layers.52.mlp.experts.118.down_proj", + "model.layers.52.mlp.experts.118.gate_proj", + "model.layers.52.mlp.experts.118.up_proj", + "model.layers.52.mlp.experts.119.down_proj", + "model.layers.52.mlp.experts.119.gate_proj", + "model.layers.52.mlp.experts.119.up_proj", + "model.layers.52.mlp.experts.12.down_proj", + "model.layers.52.mlp.experts.12.gate_proj", + "model.layers.52.mlp.experts.12.up_proj", + "model.layers.52.mlp.experts.120.down_proj", + "model.layers.52.mlp.experts.120.gate_proj", + "model.layers.52.mlp.experts.120.up_proj", + "model.layers.52.mlp.experts.121.down_proj", + "model.layers.52.mlp.experts.121.gate_proj", + "model.layers.52.mlp.experts.121.up_proj", + "model.layers.52.mlp.experts.122.down_proj", + "model.layers.52.mlp.experts.122.gate_proj", + "model.layers.52.mlp.experts.122.up_proj", + "model.layers.52.mlp.experts.123.down_proj", + "model.layers.52.mlp.experts.123.gate_proj", + "model.layers.52.mlp.experts.123.up_proj", + "model.layers.52.mlp.experts.124.down_proj", + "model.layers.52.mlp.experts.124.gate_proj", + "model.layers.52.mlp.experts.124.up_proj", + "model.layers.52.mlp.experts.125.down_proj", + "model.layers.52.mlp.experts.125.gate_proj", + "model.layers.52.mlp.experts.125.up_proj", + "model.layers.52.mlp.experts.126.down_proj", + "model.layers.52.mlp.experts.126.gate_proj", + "model.layers.52.mlp.experts.126.up_proj", + "model.layers.52.mlp.experts.127.down_proj", + "model.layers.52.mlp.experts.127.gate_proj", + "model.layers.52.mlp.experts.127.up_proj", + "model.layers.52.mlp.experts.128.down_proj", + "model.layers.52.mlp.experts.128.gate_proj", + "model.layers.52.mlp.experts.128.up_proj", + "model.layers.52.mlp.experts.129.down_proj", + "model.layers.52.mlp.experts.129.gate_proj", + "model.layers.52.mlp.experts.129.up_proj", + "model.layers.52.mlp.experts.13.down_proj", + "model.layers.52.mlp.experts.13.gate_proj", + "model.layers.52.mlp.experts.13.up_proj", + "model.layers.52.mlp.experts.130.down_proj", + "model.layers.52.mlp.experts.130.gate_proj", + "model.layers.52.mlp.experts.130.up_proj", + "model.layers.52.mlp.experts.131.down_proj", + "model.layers.52.mlp.experts.131.gate_proj", + "model.layers.52.mlp.experts.131.up_proj", + "model.layers.52.mlp.experts.132.down_proj", + "model.layers.52.mlp.experts.132.gate_proj", + "model.layers.52.mlp.experts.132.up_proj", + "model.layers.52.mlp.experts.133.down_proj", + "model.layers.52.mlp.experts.133.gate_proj", + "model.layers.52.mlp.experts.133.up_proj", + "model.layers.52.mlp.experts.134.down_proj", + "model.layers.52.mlp.experts.134.gate_proj", + "model.layers.52.mlp.experts.134.up_proj", + "model.layers.52.mlp.experts.135.down_proj", + "model.layers.52.mlp.experts.135.gate_proj", + "model.layers.52.mlp.experts.135.up_proj", + "model.layers.52.mlp.experts.136.down_proj", + "model.layers.52.mlp.experts.136.gate_proj", + "model.layers.52.mlp.experts.136.up_proj", + "model.layers.52.mlp.experts.137.down_proj", + "model.layers.52.mlp.experts.137.gate_proj", + "model.layers.52.mlp.experts.137.up_proj", + "model.layers.52.mlp.experts.138.down_proj", + "model.layers.52.mlp.experts.138.gate_proj", + "model.layers.52.mlp.experts.138.up_proj", + "model.layers.52.mlp.experts.139.down_proj", + "model.layers.52.mlp.experts.139.gate_proj", + "model.layers.52.mlp.experts.139.up_proj", + "model.layers.52.mlp.experts.14.down_proj", + "model.layers.52.mlp.experts.14.gate_proj", + "model.layers.52.mlp.experts.14.up_proj", + "model.layers.52.mlp.experts.140.down_proj", + "model.layers.52.mlp.experts.140.gate_proj", + "model.layers.52.mlp.experts.140.up_proj", + "model.layers.52.mlp.experts.141.down_proj", + "model.layers.52.mlp.experts.141.gate_proj", + "model.layers.52.mlp.experts.141.up_proj", + "model.layers.52.mlp.experts.142.down_proj", + "model.layers.52.mlp.experts.142.gate_proj", + "model.layers.52.mlp.experts.142.up_proj", + "model.layers.52.mlp.experts.143.down_proj", + "model.layers.52.mlp.experts.143.gate_proj", + "model.layers.52.mlp.experts.143.up_proj", + "model.layers.52.mlp.experts.144.down_proj", + "model.layers.52.mlp.experts.144.gate_proj", + "model.layers.52.mlp.experts.144.up_proj", + "model.layers.52.mlp.experts.145.down_proj", + "model.layers.52.mlp.experts.145.gate_proj", + "model.layers.52.mlp.experts.145.up_proj", + "model.layers.52.mlp.experts.146.down_proj", + "model.layers.52.mlp.experts.146.gate_proj", + "model.layers.52.mlp.experts.146.up_proj", + "model.layers.52.mlp.experts.147.down_proj", + "model.layers.52.mlp.experts.147.gate_proj", + "model.layers.52.mlp.experts.147.up_proj", + "model.layers.52.mlp.experts.148.down_proj", + "model.layers.52.mlp.experts.148.gate_proj", + "model.layers.52.mlp.experts.148.up_proj", + "model.layers.52.mlp.experts.149.down_proj", + "model.layers.52.mlp.experts.149.gate_proj", + "model.layers.52.mlp.experts.149.up_proj", + "model.layers.52.mlp.experts.15.down_proj", + "model.layers.52.mlp.experts.15.gate_proj", + "model.layers.52.mlp.experts.15.up_proj", + "model.layers.52.mlp.experts.150.down_proj", + "model.layers.52.mlp.experts.150.gate_proj", + "model.layers.52.mlp.experts.150.up_proj", + "model.layers.52.mlp.experts.151.down_proj", + "model.layers.52.mlp.experts.151.gate_proj", + "model.layers.52.mlp.experts.151.up_proj", + "model.layers.52.mlp.experts.152.down_proj", + "model.layers.52.mlp.experts.152.gate_proj", + "model.layers.52.mlp.experts.152.up_proj", + "model.layers.52.mlp.experts.153.down_proj", + "model.layers.52.mlp.experts.153.gate_proj", + "model.layers.52.mlp.experts.153.up_proj", + "model.layers.52.mlp.experts.154.down_proj", + "model.layers.52.mlp.experts.154.gate_proj", + "model.layers.52.mlp.experts.154.up_proj", + "model.layers.52.mlp.experts.155.down_proj", + "model.layers.52.mlp.experts.155.gate_proj", + "model.layers.52.mlp.experts.155.up_proj", + "model.layers.52.mlp.experts.156.down_proj", + "model.layers.52.mlp.experts.156.gate_proj", + "model.layers.52.mlp.experts.156.up_proj", + "model.layers.52.mlp.experts.157.down_proj", + "model.layers.52.mlp.experts.157.gate_proj", + "model.layers.52.mlp.experts.157.up_proj", + "model.layers.52.mlp.experts.158.down_proj", + "model.layers.52.mlp.experts.158.gate_proj", + "model.layers.52.mlp.experts.158.up_proj", + "model.layers.52.mlp.experts.159.down_proj", + "model.layers.52.mlp.experts.159.gate_proj", + "model.layers.52.mlp.experts.159.up_proj", + "model.layers.52.mlp.experts.16.down_proj", + "model.layers.52.mlp.experts.16.gate_proj", + "model.layers.52.mlp.experts.16.up_proj", + "model.layers.52.mlp.experts.160.down_proj", + "model.layers.52.mlp.experts.160.gate_proj", + "model.layers.52.mlp.experts.160.up_proj", + "model.layers.52.mlp.experts.161.down_proj", + "model.layers.52.mlp.experts.161.gate_proj", + "model.layers.52.mlp.experts.161.up_proj", + "model.layers.52.mlp.experts.162.down_proj", + "model.layers.52.mlp.experts.162.gate_proj", + "model.layers.52.mlp.experts.162.up_proj", + "model.layers.52.mlp.experts.163.down_proj", + "model.layers.52.mlp.experts.163.gate_proj", + "model.layers.52.mlp.experts.163.up_proj", + "model.layers.52.mlp.experts.164.down_proj", + "model.layers.52.mlp.experts.164.gate_proj", + "model.layers.52.mlp.experts.164.up_proj", + "model.layers.52.mlp.experts.165.down_proj", + "model.layers.52.mlp.experts.165.gate_proj", + "model.layers.52.mlp.experts.165.up_proj", + "model.layers.52.mlp.experts.166.down_proj", + "model.layers.52.mlp.experts.166.gate_proj", + "model.layers.52.mlp.experts.166.up_proj", + "model.layers.52.mlp.experts.167.down_proj", + "model.layers.52.mlp.experts.167.gate_proj", + "model.layers.52.mlp.experts.167.up_proj", + "model.layers.52.mlp.experts.168.down_proj", + "model.layers.52.mlp.experts.168.gate_proj", + "model.layers.52.mlp.experts.168.up_proj", + "model.layers.52.mlp.experts.169.down_proj", + "model.layers.52.mlp.experts.169.gate_proj", + "model.layers.52.mlp.experts.169.up_proj", + "model.layers.52.mlp.experts.17.down_proj", + "model.layers.52.mlp.experts.17.gate_proj", + "model.layers.52.mlp.experts.17.up_proj", + "model.layers.52.mlp.experts.170.down_proj", + "model.layers.52.mlp.experts.170.gate_proj", + "model.layers.52.mlp.experts.170.up_proj", + "model.layers.52.mlp.experts.171.down_proj", + "model.layers.52.mlp.experts.171.gate_proj", + "model.layers.52.mlp.experts.171.up_proj", + "model.layers.52.mlp.experts.172.down_proj", + "model.layers.52.mlp.experts.172.gate_proj", + "model.layers.52.mlp.experts.172.up_proj", + "model.layers.52.mlp.experts.173.down_proj", + "model.layers.52.mlp.experts.173.gate_proj", + "model.layers.52.mlp.experts.173.up_proj", + "model.layers.52.mlp.experts.174.down_proj", + "model.layers.52.mlp.experts.174.gate_proj", + "model.layers.52.mlp.experts.174.up_proj", + "model.layers.52.mlp.experts.175.down_proj", + "model.layers.52.mlp.experts.175.gate_proj", + "model.layers.52.mlp.experts.175.up_proj", + "model.layers.52.mlp.experts.176.down_proj", + "model.layers.52.mlp.experts.176.gate_proj", + "model.layers.52.mlp.experts.176.up_proj", + "model.layers.52.mlp.experts.177.down_proj", + "model.layers.52.mlp.experts.177.gate_proj", + "model.layers.52.mlp.experts.177.up_proj", + "model.layers.52.mlp.experts.178.down_proj", + "model.layers.52.mlp.experts.178.gate_proj", + "model.layers.52.mlp.experts.178.up_proj", + "model.layers.52.mlp.experts.179.down_proj", + "model.layers.52.mlp.experts.179.gate_proj", + "model.layers.52.mlp.experts.179.up_proj", + "model.layers.52.mlp.experts.18.down_proj", + "model.layers.52.mlp.experts.18.gate_proj", + "model.layers.52.mlp.experts.18.up_proj", + "model.layers.52.mlp.experts.180.down_proj", + "model.layers.52.mlp.experts.180.gate_proj", + "model.layers.52.mlp.experts.180.up_proj", + "model.layers.52.mlp.experts.181.down_proj", + "model.layers.52.mlp.experts.181.gate_proj", + "model.layers.52.mlp.experts.181.up_proj", + "model.layers.52.mlp.experts.182.down_proj", + "model.layers.52.mlp.experts.182.gate_proj", + "model.layers.52.mlp.experts.182.up_proj", + "model.layers.52.mlp.experts.183.down_proj", + "model.layers.52.mlp.experts.183.gate_proj", + "model.layers.52.mlp.experts.183.up_proj", + "model.layers.52.mlp.experts.184.down_proj", + "model.layers.52.mlp.experts.184.gate_proj", + "model.layers.52.mlp.experts.184.up_proj", + "model.layers.52.mlp.experts.185.down_proj", + "model.layers.52.mlp.experts.185.gate_proj", + "model.layers.52.mlp.experts.185.up_proj", + "model.layers.52.mlp.experts.186.down_proj", + "model.layers.52.mlp.experts.186.gate_proj", + "model.layers.52.mlp.experts.186.up_proj", + "model.layers.52.mlp.experts.187.down_proj", + "model.layers.52.mlp.experts.187.gate_proj", + "model.layers.52.mlp.experts.187.up_proj", + "model.layers.52.mlp.experts.188.down_proj", + "model.layers.52.mlp.experts.188.gate_proj", + "model.layers.52.mlp.experts.188.up_proj", + "model.layers.52.mlp.experts.189.down_proj", + "model.layers.52.mlp.experts.189.gate_proj", + "model.layers.52.mlp.experts.189.up_proj", + "model.layers.52.mlp.experts.19.down_proj", + "model.layers.52.mlp.experts.19.gate_proj", + "model.layers.52.mlp.experts.19.up_proj", + "model.layers.52.mlp.experts.190.down_proj", + "model.layers.52.mlp.experts.190.gate_proj", + "model.layers.52.mlp.experts.190.up_proj", + "model.layers.52.mlp.experts.191.down_proj", + "model.layers.52.mlp.experts.191.gate_proj", + "model.layers.52.mlp.experts.191.up_proj", + "model.layers.52.mlp.experts.192.down_proj", + "model.layers.52.mlp.experts.192.gate_proj", + "model.layers.52.mlp.experts.192.up_proj", + "model.layers.52.mlp.experts.193.down_proj", + "model.layers.52.mlp.experts.193.gate_proj", + "model.layers.52.mlp.experts.193.up_proj", + "model.layers.52.mlp.experts.194.down_proj", + "model.layers.52.mlp.experts.194.gate_proj", + "model.layers.52.mlp.experts.194.up_proj", + "model.layers.52.mlp.experts.195.down_proj", + "model.layers.52.mlp.experts.195.gate_proj", + "model.layers.52.mlp.experts.195.up_proj", + "model.layers.52.mlp.experts.196.down_proj", + "model.layers.52.mlp.experts.196.gate_proj", + "model.layers.52.mlp.experts.196.up_proj", + "model.layers.52.mlp.experts.197.down_proj", + "model.layers.52.mlp.experts.197.gate_proj", + "model.layers.52.mlp.experts.197.up_proj", + "model.layers.52.mlp.experts.198.down_proj", + "model.layers.52.mlp.experts.198.gate_proj", + "model.layers.52.mlp.experts.198.up_proj", + "model.layers.52.mlp.experts.199.down_proj", + "model.layers.52.mlp.experts.199.gate_proj", + "model.layers.52.mlp.experts.199.up_proj", + "model.layers.52.mlp.experts.2.down_proj", + "model.layers.52.mlp.experts.2.gate_proj", + "model.layers.52.mlp.experts.2.up_proj", + "model.layers.52.mlp.experts.20.down_proj", + "model.layers.52.mlp.experts.20.gate_proj", + "model.layers.52.mlp.experts.20.up_proj", + "model.layers.52.mlp.experts.200.down_proj", + "model.layers.52.mlp.experts.200.gate_proj", + "model.layers.52.mlp.experts.200.up_proj", + "model.layers.52.mlp.experts.201.down_proj", + "model.layers.52.mlp.experts.201.gate_proj", + "model.layers.52.mlp.experts.201.up_proj", + "model.layers.52.mlp.experts.202.down_proj", + "model.layers.52.mlp.experts.202.gate_proj", + "model.layers.52.mlp.experts.202.up_proj", + "model.layers.52.mlp.experts.203.down_proj", + "model.layers.52.mlp.experts.203.gate_proj", + "model.layers.52.mlp.experts.203.up_proj", + "model.layers.52.mlp.experts.204.down_proj", + "model.layers.52.mlp.experts.204.gate_proj", + "model.layers.52.mlp.experts.204.up_proj", + "model.layers.52.mlp.experts.205.down_proj", + "model.layers.52.mlp.experts.205.gate_proj", + "model.layers.52.mlp.experts.205.up_proj", + "model.layers.52.mlp.experts.206.down_proj", + "model.layers.52.mlp.experts.206.gate_proj", + "model.layers.52.mlp.experts.206.up_proj", + "model.layers.52.mlp.experts.207.down_proj", + "model.layers.52.mlp.experts.207.gate_proj", + "model.layers.52.mlp.experts.207.up_proj", + "model.layers.52.mlp.experts.208.down_proj", + "model.layers.52.mlp.experts.208.gate_proj", + "model.layers.52.mlp.experts.208.up_proj", + "model.layers.52.mlp.experts.209.down_proj", + "model.layers.52.mlp.experts.209.gate_proj", + "model.layers.52.mlp.experts.209.up_proj", + "model.layers.52.mlp.experts.21.down_proj", + "model.layers.52.mlp.experts.21.gate_proj", + "model.layers.52.mlp.experts.21.up_proj", + "model.layers.52.mlp.experts.210.down_proj", + "model.layers.52.mlp.experts.210.gate_proj", + "model.layers.52.mlp.experts.210.up_proj", + "model.layers.52.mlp.experts.211.down_proj", + "model.layers.52.mlp.experts.211.gate_proj", + "model.layers.52.mlp.experts.211.up_proj", + "model.layers.52.mlp.experts.212.down_proj", + "model.layers.52.mlp.experts.212.gate_proj", + "model.layers.52.mlp.experts.212.up_proj", + "model.layers.52.mlp.experts.213.down_proj", + "model.layers.52.mlp.experts.213.gate_proj", + "model.layers.52.mlp.experts.213.up_proj", + "model.layers.52.mlp.experts.214.down_proj", + "model.layers.52.mlp.experts.214.gate_proj", + "model.layers.52.mlp.experts.214.up_proj", + "model.layers.52.mlp.experts.215.down_proj", + "model.layers.52.mlp.experts.215.gate_proj", + "model.layers.52.mlp.experts.215.up_proj", + "model.layers.52.mlp.experts.216.down_proj", + "model.layers.52.mlp.experts.216.gate_proj", + "model.layers.52.mlp.experts.216.up_proj", + "model.layers.52.mlp.experts.217.down_proj", + "model.layers.52.mlp.experts.217.gate_proj", + "model.layers.52.mlp.experts.217.up_proj", + "model.layers.52.mlp.experts.218.down_proj", + "model.layers.52.mlp.experts.218.gate_proj", + "model.layers.52.mlp.experts.218.up_proj", + "model.layers.52.mlp.experts.219.down_proj", + "model.layers.52.mlp.experts.219.gate_proj", + "model.layers.52.mlp.experts.219.up_proj", + "model.layers.52.mlp.experts.22.down_proj", + "model.layers.52.mlp.experts.22.gate_proj", + "model.layers.52.mlp.experts.22.up_proj", + "model.layers.52.mlp.experts.220.down_proj", + "model.layers.52.mlp.experts.220.gate_proj", + "model.layers.52.mlp.experts.220.up_proj", + "model.layers.52.mlp.experts.221.down_proj", + "model.layers.52.mlp.experts.221.gate_proj", + "model.layers.52.mlp.experts.221.up_proj", + "model.layers.52.mlp.experts.222.down_proj", + "model.layers.52.mlp.experts.222.gate_proj", + "model.layers.52.mlp.experts.222.up_proj", + "model.layers.52.mlp.experts.223.down_proj", + "model.layers.52.mlp.experts.223.gate_proj", + "model.layers.52.mlp.experts.223.up_proj", + "model.layers.52.mlp.experts.224.down_proj", + "model.layers.52.mlp.experts.224.gate_proj", + "model.layers.52.mlp.experts.224.up_proj", + "model.layers.52.mlp.experts.225.down_proj", + "model.layers.52.mlp.experts.225.gate_proj", + "model.layers.52.mlp.experts.225.up_proj", + "model.layers.52.mlp.experts.226.down_proj", + "model.layers.52.mlp.experts.226.gate_proj", + "model.layers.52.mlp.experts.226.up_proj", + "model.layers.52.mlp.experts.227.down_proj", + "model.layers.52.mlp.experts.227.gate_proj", + "model.layers.52.mlp.experts.227.up_proj", + "model.layers.52.mlp.experts.228.down_proj", + "model.layers.52.mlp.experts.228.gate_proj", + "model.layers.52.mlp.experts.228.up_proj", + "model.layers.52.mlp.experts.229.down_proj", + "model.layers.52.mlp.experts.229.gate_proj", + "model.layers.52.mlp.experts.229.up_proj", + "model.layers.52.mlp.experts.23.down_proj", + "model.layers.52.mlp.experts.23.gate_proj", + "model.layers.52.mlp.experts.23.up_proj", + "model.layers.52.mlp.experts.230.down_proj", + "model.layers.52.mlp.experts.230.gate_proj", + "model.layers.52.mlp.experts.230.up_proj", + "model.layers.52.mlp.experts.231.down_proj", + "model.layers.52.mlp.experts.231.gate_proj", + "model.layers.52.mlp.experts.231.up_proj", + "model.layers.52.mlp.experts.232.down_proj", + "model.layers.52.mlp.experts.232.gate_proj", + "model.layers.52.mlp.experts.232.up_proj", + "model.layers.52.mlp.experts.233.down_proj", + "model.layers.52.mlp.experts.233.gate_proj", + "model.layers.52.mlp.experts.233.up_proj", + "model.layers.52.mlp.experts.234.down_proj", + "model.layers.52.mlp.experts.234.gate_proj", + "model.layers.52.mlp.experts.234.up_proj", + "model.layers.52.mlp.experts.235.down_proj", + "model.layers.52.mlp.experts.235.gate_proj", + "model.layers.52.mlp.experts.235.up_proj", + "model.layers.52.mlp.experts.236.down_proj", + "model.layers.52.mlp.experts.236.gate_proj", + "model.layers.52.mlp.experts.236.up_proj", + "model.layers.52.mlp.experts.237.down_proj", + "model.layers.52.mlp.experts.237.gate_proj", + "model.layers.52.mlp.experts.237.up_proj", + "model.layers.52.mlp.experts.238.down_proj", + "model.layers.52.mlp.experts.238.gate_proj", + "model.layers.52.mlp.experts.238.up_proj", + "model.layers.52.mlp.experts.239.down_proj", + "model.layers.52.mlp.experts.239.gate_proj", + "model.layers.52.mlp.experts.239.up_proj", + "model.layers.52.mlp.experts.24.down_proj", + "model.layers.52.mlp.experts.24.gate_proj", + "model.layers.52.mlp.experts.24.up_proj", + "model.layers.52.mlp.experts.240.down_proj", + "model.layers.52.mlp.experts.240.gate_proj", + "model.layers.52.mlp.experts.240.up_proj", + "model.layers.52.mlp.experts.241.down_proj", + "model.layers.52.mlp.experts.241.gate_proj", + "model.layers.52.mlp.experts.241.up_proj", + "model.layers.52.mlp.experts.242.down_proj", + "model.layers.52.mlp.experts.242.gate_proj", + "model.layers.52.mlp.experts.242.up_proj", + "model.layers.52.mlp.experts.243.down_proj", + "model.layers.52.mlp.experts.243.gate_proj", + "model.layers.52.mlp.experts.243.up_proj", + "model.layers.52.mlp.experts.244.down_proj", + "model.layers.52.mlp.experts.244.gate_proj", + "model.layers.52.mlp.experts.244.up_proj", + "model.layers.52.mlp.experts.245.down_proj", + "model.layers.52.mlp.experts.245.gate_proj", + "model.layers.52.mlp.experts.245.up_proj", + "model.layers.52.mlp.experts.246.down_proj", + "model.layers.52.mlp.experts.246.gate_proj", + "model.layers.52.mlp.experts.246.up_proj", + "model.layers.52.mlp.experts.247.down_proj", + "model.layers.52.mlp.experts.247.gate_proj", + "model.layers.52.mlp.experts.247.up_proj", + "model.layers.52.mlp.experts.248.down_proj", + "model.layers.52.mlp.experts.248.gate_proj", + "model.layers.52.mlp.experts.248.up_proj", + "model.layers.52.mlp.experts.249.down_proj", + "model.layers.52.mlp.experts.249.gate_proj", + "model.layers.52.mlp.experts.249.up_proj", + "model.layers.52.mlp.experts.25.down_proj", + "model.layers.52.mlp.experts.25.gate_proj", + "model.layers.52.mlp.experts.25.up_proj", + "model.layers.52.mlp.experts.250.down_proj", + "model.layers.52.mlp.experts.250.gate_proj", + "model.layers.52.mlp.experts.250.up_proj", + "model.layers.52.mlp.experts.251.down_proj", + "model.layers.52.mlp.experts.251.gate_proj", + "model.layers.52.mlp.experts.251.up_proj", + "model.layers.52.mlp.experts.252.down_proj", + "model.layers.52.mlp.experts.252.gate_proj", + "model.layers.52.mlp.experts.252.up_proj", + "model.layers.52.mlp.experts.253.down_proj", + "model.layers.52.mlp.experts.253.gate_proj", + "model.layers.52.mlp.experts.253.up_proj", + "model.layers.52.mlp.experts.254.down_proj", + "model.layers.52.mlp.experts.254.gate_proj", + "model.layers.52.mlp.experts.254.up_proj", + "model.layers.52.mlp.experts.255.down_proj", + "model.layers.52.mlp.experts.255.gate_proj", + "model.layers.52.mlp.experts.255.up_proj", + "model.layers.52.mlp.experts.26.down_proj", + "model.layers.52.mlp.experts.26.gate_proj", + "model.layers.52.mlp.experts.26.up_proj", + "model.layers.52.mlp.experts.27.down_proj", + "model.layers.52.mlp.experts.27.gate_proj", + "model.layers.52.mlp.experts.27.up_proj", + "model.layers.52.mlp.experts.28.down_proj", + "model.layers.52.mlp.experts.28.gate_proj", + "model.layers.52.mlp.experts.28.up_proj", + "model.layers.52.mlp.experts.29.down_proj", + "model.layers.52.mlp.experts.29.gate_proj", + "model.layers.52.mlp.experts.29.up_proj", + "model.layers.52.mlp.experts.3.down_proj", + "model.layers.52.mlp.experts.3.gate_proj", + "model.layers.52.mlp.experts.3.up_proj", + "model.layers.52.mlp.experts.30.down_proj", + "model.layers.52.mlp.experts.30.gate_proj", + "model.layers.52.mlp.experts.30.up_proj", + "model.layers.52.mlp.experts.31.down_proj", + "model.layers.52.mlp.experts.31.gate_proj", + "model.layers.52.mlp.experts.31.up_proj", + "model.layers.52.mlp.experts.32.down_proj", + "model.layers.52.mlp.experts.32.gate_proj", + "model.layers.52.mlp.experts.32.up_proj", + "model.layers.52.mlp.experts.33.down_proj", + "model.layers.52.mlp.experts.33.gate_proj", + "model.layers.52.mlp.experts.33.up_proj", + "model.layers.52.mlp.experts.34.down_proj", + "model.layers.52.mlp.experts.34.gate_proj", + "model.layers.52.mlp.experts.34.up_proj", + "model.layers.52.mlp.experts.35.down_proj", + "model.layers.52.mlp.experts.35.gate_proj", + "model.layers.52.mlp.experts.35.up_proj", + "model.layers.52.mlp.experts.36.down_proj", + "model.layers.52.mlp.experts.36.gate_proj", + "model.layers.52.mlp.experts.36.up_proj", + "model.layers.52.mlp.experts.37.down_proj", + "model.layers.52.mlp.experts.37.gate_proj", + "model.layers.52.mlp.experts.37.up_proj", + "model.layers.52.mlp.experts.38.down_proj", + "model.layers.52.mlp.experts.38.gate_proj", + "model.layers.52.mlp.experts.38.up_proj", + "model.layers.52.mlp.experts.39.down_proj", + "model.layers.52.mlp.experts.39.gate_proj", + "model.layers.52.mlp.experts.39.up_proj", + "model.layers.52.mlp.experts.4.down_proj", + "model.layers.52.mlp.experts.4.gate_proj", + "model.layers.52.mlp.experts.4.up_proj", + "model.layers.52.mlp.experts.40.down_proj", + "model.layers.52.mlp.experts.40.gate_proj", + "model.layers.52.mlp.experts.40.up_proj", + "model.layers.52.mlp.experts.41.down_proj", + "model.layers.52.mlp.experts.41.gate_proj", + "model.layers.52.mlp.experts.41.up_proj", + "model.layers.52.mlp.experts.42.down_proj", + "model.layers.52.mlp.experts.42.gate_proj", + "model.layers.52.mlp.experts.42.up_proj", + "model.layers.52.mlp.experts.43.down_proj", + "model.layers.52.mlp.experts.43.gate_proj", + "model.layers.52.mlp.experts.43.up_proj", + "model.layers.52.mlp.experts.44.down_proj", + "model.layers.52.mlp.experts.44.gate_proj", + "model.layers.52.mlp.experts.44.up_proj", + "model.layers.52.mlp.experts.45.down_proj", + "model.layers.52.mlp.experts.45.gate_proj", + "model.layers.52.mlp.experts.45.up_proj", + "model.layers.52.mlp.experts.46.down_proj", + "model.layers.52.mlp.experts.46.gate_proj", + "model.layers.52.mlp.experts.46.up_proj", + "model.layers.52.mlp.experts.47.down_proj", + "model.layers.52.mlp.experts.47.gate_proj", + "model.layers.52.mlp.experts.47.up_proj", + "model.layers.52.mlp.experts.48.down_proj", + "model.layers.52.mlp.experts.48.gate_proj", + "model.layers.52.mlp.experts.48.up_proj", + "model.layers.52.mlp.experts.49.down_proj", + "model.layers.52.mlp.experts.49.gate_proj", + "model.layers.52.mlp.experts.49.up_proj", + "model.layers.52.mlp.experts.5.down_proj", + "model.layers.52.mlp.experts.5.gate_proj", + "model.layers.52.mlp.experts.5.up_proj", + "model.layers.52.mlp.experts.50.down_proj", + "model.layers.52.mlp.experts.50.gate_proj", + "model.layers.52.mlp.experts.50.up_proj", + "model.layers.52.mlp.experts.51.down_proj", + "model.layers.52.mlp.experts.51.gate_proj", + "model.layers.52.mlp.experts.51.up_proj", + "model.layers.52.mlp.experts.52.down_proj", + "model.layers.52.mlp.experts.52.gate_proj", + "model.layers.52.mlp.experts.52.up_proj", + "model.layers.52.mlp.experts.53.down_proj", + "model.layers.52.mlp.experts.53.gate_proj", + "model.layers.52.mlp.experts.53.up_proj", + "model.layers.52.mlp.experts.54.down_proj", + "model.layers.52.mlp.experts.54.gate_proj", + "model.layers.52.mlp.experts.54.up_proj", + "model.layers.52.mlp.experts.55.down_proj", + "model.layers.52.mlp.experts.55.gate_proj", + "model.layers.52.mlp.experts.55.up_proj", + "model.layers.52.mlp.experts.56.down_proj", + "model.layers.52.mlp.experts.56.gate_proj", + "model.layers.52.mlp.experts.56.up_proj", + "model.layers.52.mlp.experts.57.down_proj", + "model.layers.52.mlp.experts.57.gate_proj", + "model.layers.52.mlp.experts.57.up_proj", + "model.layers.52.mlp.experts.58.down_proj", + "model.layers.52.mlp.experts.58.gate_proj", + "model.layers.52.mlp.experts.58.up_proj", + "model.layers.52.mlp.experts.59.down_proj", + "model.layers.52.mlp.experts.59.gate_proj", + "model.layers.52.mlp.experts.59.up_proj", + "model.layers.52.mlp.experts.6.down_proj", + "model.layers.52.mlp.experts.6.gate_proj", + "model.layers.52.mlp.experts.6.up_proj", + "model.layers.52.mlp.experts.60.down_proj", + "model.layers.52.mlp.experts.60.gate_proj", + "model.layers.52.mlp.experts.60.up_proj", + "model.layers.52.mlp.experts.61.down_proj", + "model.layers.52.mlp.experts.61.gate_proj", + "model.layers.52.mlp.experts.61.up_proj", + "model.layers.52.mlp.experts.62.down_proj", + "model.layers.52.mlp.experts.62.gate_proj", + "model.layers.52.mlp.experts.62.up_proj", + "model.layers.52.mlp.experts.63.down_proj", + "model.layers.52.mlp.experts.63.gate_proj", + "model.layers.52.mlp.experts.63.up_proj", + "model.layers.52.mlp.experts.64.down_proj", + "model.layers.52.mlp.experts.64.gate_proj", + "model.layers.52.mlp.experts.64.up_proj", + "model.layers.52.mlp.experts.65.down_proj", + "model.layers.52.mlp.experts.65.gate_proj", + "model.layers.52.mlp.experts.65.up_proj", + "model.layers.52.mlp.experts.66.down_proj", + "model.layers.52.mlp.experts.66.gate_proj", + "model.layers.52.mlp.experts.66.up_proj", + "model.layers.52.mlp.experts.67.down_proj", + "model.layers.52.mlp.experts.67.gate_proj", + "model.layers.52.mlp.experts.67.up_proj", + "model.layers.52.mlp.experts.68.down_proj", + "model.layers.52.mlp.experts.68.gate_proj", + "model.layers.52.mlp.experts.68.up_proj", + "model.layers.52.mlp.experts.69.down_proj", + "model.layers.52.mlp.experts.69.gate_proj", + "model.layers.52.mlp.experts.69.up_proj", + "model.layers.52.mlp.experts.7.down_proj", + "model.layers.52.mlp.experts.7.gate_proj", + "model.layers.52.mlp.experts.7.up_proj", + "model.layers.52.mlp.experts.70.down_proj", + "model.layers.52.mlp.experts.70.gate_proj", + "model.layers.52.mlp.experts.70.up_proj", + "model.layers.52.mlp.experts.71.down_proj", + "model.layers.52.mlp.experts.71.gate_proj", + "model.layers.52.mlp.experts.71.up_proj", + "model.layers.52.mlp.experts.72.down_proj", + "model.layers.52.mlp.experts.72.gate_proj", + "model.layers.52.mlp.experts.72.up_proj", + "model.layers.52.mlp.experts.73.down_proj", + "model.layers.52.mlp.experts.73.gate_proj", + "model.layers.52.mlp.experts.73.up_proj", + "model.layers.52.mlp.experts.74.down_proj", + "model.layers.52.mlp.experts.74.gate_proj", + "model.layers.52.mlp.experts.74.up_proj", + "model.layers.52.mlp.experts.75.down_proj", + "model.layers.52.mlp.experts.75.gate_proj", + "model.layers.52.mlp.experts.75.up_proj", + "model.layers.52.mlp.experts.76.down_proj", + "model.layers.52.mlp.experts.76.gate_proj", + "model.layers.52.mlp.experts.76.up_proj", + "model.layers.52.mlp.experts.77.down_proj", + "model.layers.52.mlp.experts.77.gate_proj", + "model.layers.52.mlp.experts.77.up_proj", + "model.layers.52.mlp.experts.78.down_proj", + "model.layers.52.mlp.experts.78.gate_proj", + "model.layers.52.mlp.experts.78.up_proj", + "model.layers.52.mlp.experts.79.down_proj", + "model.layers.52.mlp.experts.79.gate_proj", + "model.layers.52.mlp.experts.79.up_proj", + "model.layers.52.mlp.experts.8.down_proj", + "model.layers.52.mlp.experts.8.gate_proj", + "model.layers.52.mlp.experts.8.up_proj", + "model.layers.52.mlp.experts.80.down_proj", + "model.layers.52.mlp.experts.80.gate_proj", + "model.layers.52.mlp.experts.80.up_proj", + "model.layers.52.mlp.experts.81.down_proj", + "model.layers.52.mlp.experts.81.gate_proj", + "model.layers.52.mlp.experts.81.up_proj", + "model.layers.52.mlp.experts.82.down_proj", + "model.layers.52.mlp.experts.82.gate_proj", + "model.layers.52.mlp.experts.82.up_proj", + "model.layers.52.mlp.experts.83.down_proj", + "model.layers.52.mlp.experts.83.gate_proj", + "model.layers.52.mlp.experts.83.up_proj", + "model.layers.52.mlp.experts.84.down_proj", + "model.layers.52.mlp.experts.84.gate_proj", + "model.layers.52.mlp.experts.84.up_proj", + "model.layers.52.mlp.experts.85.down_proj", + "model.layers.52.mlp.experts.85.gate_proj", + "model.layers.52.mlp.experts.85.up_proj", + "model.layers.52.mlp.experts.86.down_proj", + "model.layers.52.mlp.experts.86.gate_proj", + "model.layers.52.mlp.experts.86.up_proj", + "model.layers.52.mlp.experts.87.down_proj", + "model.layers.52.mlp.experts.87.gate_proj", + "model.layers.52.mlp.experts.87.up_proj", + "model.layers.52.mlp.experts.88.down_proj", + "model.layers.52.mlp.experts.88.gate_proj", + "model.layers.52.mlp.experts.88.up_proj", + "model.layers.52.mlp.experts.89.down_proj", + "model.layers.52.mlp.experts.89.gate_proj", + "model.layers.52.mlp.experts.89.up_proj", + "model.layers.52.mlp.experts.9.down_proj", + "model.layers.52.mlp.experts.9.gate_proj", + "model.layers.52.mlp.experts.9.up_proj", + "model.layers.52.mlp.experts.90.down_proj", + "model.layers.52.mlp.experts.90.gate_proj", + "model.layers.52.mlp.experts.90.up_proj", + "model.layers.52.mlp.experts.91.down_proj", + "model.layers.52.mlp.experts.91.gate_proj", + "model.layers.52.mlp.experts.91.up_proj", + "model.layers.52.mlp.experts.92.down_proj", + "model.layers.52.mlp.experts.92.gate_proj", + "model.layers.52.mlp.experts.92.up_proj", + "model.layers.52.mlp.experts.93.down_proj", + "model.layers.52.mlp.experts.93.gate_proj", + "model.layers.52.mlp.experts.93.up_proj", + "model.layers.52.mlp.experts.94.down_proj", + "model.layers.52.mlp.experts.94.gate_proj", + "model.layers.52.mlp.experts.94.up_proj", + "model.layers.52.mlp.experts.95.down_proj", + "model.layers.52.mlp.experts.95.gate_proj", + "model.layers.52.mlp.experts.95.up_proj", + "model.layers.52.mlp.experts.96.down_proj", + "model.layers.52.mlp.experts.96.gate_proj", + "model.layers.52.mlp.experts.96.up_proj", + "model.layers.52.mlp.experts.97.down_proj", + "model.layers.52.mlp.experts.97.gate_proj", + "model.layers.52.mlp.experts.97.up_proj", + "model.layers.52.mlp.experts.98.down_proj", + "model.layers.52.mlp.experts.98.gate_proj", + "model.layers.52.mlp.experts.98.up_proj", + "model.layers.52.mlp.experts.99.down_proj", + "model.layers.52.mlp.experts.99.gate_proj", + "model.layers.52.mlp.experts.99.up_proj", + "model.layers.53.mlp.experts.0.down_proj", + "model.layers.53.mlp.experts.0.gate_proj", + "model.layers.53.mlp.experts.0.up_proj", + "model.layers.53.mlp.experts.1.down_proj", + "model.layers.53.mlp.experts.1.gate_proj", + "model.layers.53.mlp.experts.1.up_proj", + "model.layers.53.mlp.experts.10.down_proj", + "model.layers.53.mlp.experts.10.gate_proj", + "model.layers.53.mlp.experts.10.up_proj", + "model.layers.53.mlp.experts.100.down_proj", + "model.layers.53.mlp.experts.100.gate_proj", + "model.layers.53.mlp.experts.100.up_proj", + "model.layers.53.mlp.experts.101.down_proj", + "model.layers.53.mlp.experts.101.gate_proj", + "model.layers.53.mlp.experts.101.up_proj", + "model.layers.53.mlp.experts.102.down_proj", + "model.layers.53.mlp.experts.102.gate_proj", + "model.layers.53.mlp.experts.102.up_proj", + "model.layers.53.mlp.experts.103.down_proj", + "model.layers.53.mlp.experts.103.gate_proj", + "model.layers.53.mlp.experts.103.up_proj", + "model.layers.53.mlp.experts.104.down_proj", + "model.layers.53.mlp.experts.104.gate_proj", + "model.layers.53.mlp.experts.104.up_proj", + "model.layers.53.mlp.experts.105.down_proj", + "model.layers.53.mlp.experts.105.gate_proj", + "model.layers.53.mlp.experts.105.up_proj", + "model.layers.53.mlp.experts.106.down_proj", + "model.layers.53.mlp.experts.106.gate_proj", + "model.layers.53.mlp.experts.106.up_proj", + "model.layers.53.mlp.experts.107.down_proj", + "model.layers.53.mlp.experts.107.gate_proj", + "model.layers.53.mlp.experts.107.up_proj", + "model.layers.53.mlp.experts.108.down_proj", + "model.layers.53.mlp.experts.108.gate_proj", + "model.layers.53.mlp.experts.108.up_proj", + "model.layers.53.mlp.experts.109.down_proj", + "model.layers.53.mlp.experts.109.gate_proj", + "model.layers.53.mlp.experts.109.up_proj", + "model.layers.53.mlp.experts.11.down_proj", + "model.layers.53.mlp.experts.11.gate_proj", + "model.layers.53.mlp.experts.11.up_proj", + "model.layers.53.mlp.experts.110.down_proj", + "model.layers.53.mlp.experts.110.gate_proj", + "model.layers.53.mlp.experts.110.up_proj", + "model.layers.53.mlp.experts.111.down_proj", + "model.layers.53.mlp.experts.111.gate_proj", + "model.layers.53.mlp.experts.111.up_proj", + "model.layers.53.mlp.experts.112.down_proj", + "model.layers.53.mlp.experts.112.gate_proj", + "model.layers.53.mlp.experts.112.up_proj", + "model.layers.53.mlp.experts.113.down_proj", + "model.layers.53.mlp.experts.113.gate_proj", + "model.layers.53.mlp.experts.113.up_proj", + "model.layers.53.mlp.experts.114.down_proj", + "model.layers.53.mlp.experts.114.gate_proj", + "model.layers.53.mlp.experts.114.up_proj", + "model.layers.53.mlp.experts.115.down_proj", + "model.layers.53.mlp.experts.115.gate_proj", + "model.layers.53.mlp.experts.115.up_proj", + "model.layers.53.mlp.experts.116.down_proj", + "model.layers.53.mlp.experts.116.gate_proj", + "model.layers.53.mlp.experts.116.up_proj", + "model.layers.53.mlp.experts.117.down_proj", + "model.layers.53.mlp.experts.117.gate_proj", + "model.layers.53.mlp.experts.117.up_proj", + "model.layers.53.mlp.experts.118.down_proj", + "model.layers.53.mlp.experts.118.gate_proj", + "model.layers.53.mlp.experts.118.up_proj", + "model.layers.53.mlp.experts.119.down_proj", + "model.layers.53.mlp.experts.119.gate_proj", + "model.layers.53.mlp.experts.119.up_proj", + "model.layers.53.mlp.experts.12.down_proj", + "model.layers.53.mlp.experts.12.gate_proj", + "model.layers.53.mlp.experts.12.up_proj", + "model.layers.53.mlp.experts.120.down_proj", + "model.layers.53.mlp.experts.120.gate_proj", + "model.layers.53.mlp.experts.120.up_proj", + "model.layers.53.mlp.experts.121.down_proj", + "model.layers.53.mlp.experts.121.gate_proj", + "model.layers.53.mlp.experts.121.up_proj", + "model.layers.53.mlp.experts.122.down_proj", + "model.layers.53.mlp.experts.122.gate_proj", + "model.layers.53.mlp.experts.122.up_proj", + "model.layers.53.mlp.experts.123.down_proj", + "model.layers.53.mlp.experts.123.gate_proj", + "model.layers.53.mlp.experts.123.up_proj", + "model.layers.53.mlp.experts.124.down_proj", + "model.layers.53.mlp.experts.124.gate_proj", + "model.layers.53.mlp.experts.124.up_proj", + "model.layers.53.mlp.experts.125.down_proj", + "model.layers.53.mlp.experts.125.gate_proj", + "model.layers.53.mlp.experts.125.up_proj", + "model.layers.53.mlp.experts.126.down_proj", + "model.layers.53.mlp.experts.126.gate_proj", + "model.layers.53.mlp.experts.126.up_proj", + "model.layers.53.mlp.experts.127.down_proj", + "model.layers.53.mlp.experts.127.gate_proj", + "model.layers.53.mlp.experts.127.up_proj", + "model.layers.53.mlp.experts.128.down_proj", + "model.layers.53.mlp.experts.128.gate_proj", + "model.layers.53.mlp.experts.128.up_proj", + "model.layers.53.mlp.experts.129.down_proj", + "model.layers.53.mlp.experts.129.gate_proj", + "model.layers.53.mlp.experts.129.up_proj", + "model.layers.53.mlp.experts.13.down_proj", + "model.layers.53.mlp.experts.13.gate_proj", + "model.layers.53.mlp.experts.13.up_proj", + "model.layers.53.mlp.experts.130.down_proj", + "model.layers.53.mlp.experts.130.gate_proj", + "model.layers.53.mlp.experts.130.up_proj", + "model.layers.53.mlp.experts.131.down_proj", + "model.layers.53.mlp.experts.131.gate_proj", + "model.layers.53.mlp.experts.131.up_proj", + "model.layers.53.mlp.experts.132.down_proj", + "model.layers.53.mlp.experts.132.gate_proj", + "model.layers.53.mlp.experts.132.up_proj", + "model.layers.53.mlp.experts.133.down_proj", + "model.layers.53.mlp.experts.133.gate_proj", + "model.layers.53.mlp.experts.133.up_proj", + "model.layers.53.mlp.experts.134.down_proj", + "model.layers.53.mlp.experts.134.gate_proj", + "model.layers.53.mlp.experts.134.up_proj", + "model.layers.53.mlp.experts.135.down_proj", + "model.layers.53.mlp.experts.135.gate_proj", + "model.layers.53.mlp.experts.135.up_proj", + "model.layers.53.mlp.experts.136.down_proj", + "model.layers.53.mlp.experts.136.gate_proj", + "model.layers.53.mlp.experts.136.up_proj", + "model.layers.53.mlp.experts.137.down_proj", + "model.layers.53.mlp.experts.137.gate_proj", + "model.layers.53.mlp.experts.137.up_proj", + "model.layers.53.mlp.experts.138.down_proj", + "model.layers.53.mlp.experts.138.gate_proj", + "model.layers.53.mlp.experts.138.up_proj", + "model.layers.53.mlp.experts.139.down_proj", + "model.layers.53.mlp.experts.139.gate_proj", + "model.layers.53.mlp.experts.139.up_proj", + "model.layers.53.mlp.experts.14.down_proj", + "model.layers.53.mlp.experts.14.gate_proj", + "model.layers.53.mlp.experts.14.up_proj", + "model.layers.53.mlp.experts.140.down_proj", + "model.layers.53.mlp.experts.140.gate_proj", + "model.layers.53.mlp.experts.140.up_proj", + "model.layers.53.mlp.experts.141.down_proj", + "model.layers.53.mlp.experts.141.gate_proj", + "model.layers.53.mlp.experts.141.up_proj", + "model.layers.53.mlp.experts.142.down_proj", + "model.layers.53.mlp.experts.142.gate_proj", + "model.layers.53.mlp.experts.142.up_proj", + "model.layers.53.mlp.experts.143.down_proj", + "model.layers.53.mlp.experts.143.gate_proj", + "model.layers.53.mlp.experts.143.up_proj", + "model.layers.53.mlp.experts.144.down_proj", + "model.layers.53.mlp.experts.144.gate_proj", + "model.layers.53.mlp.experts.144.up_proj", + "model.layers.53.mlp.experts.145.down_proj", + "model.layers.53.mlp.experts.145.gate_proj", + "model.layers.53.mlp.experts.145.up_proj", + "model.layers.53.mlp.experts.146.down_proj", + "model.layers.53.mlp.experts.146.gate_proj", + "model.layers.53.mlp.experts.146.up_proj", + "model.layers.53.mlp.experts.147.down_proj", + "model.layers.53.mlp.experts.147.gate_proj", + "model.layers.53.mlp.experts.147.up_proj", + "model.layers.53.mlp.experts.148.down_proj", + "model.layers.53.mlp.experts.148.gate_proj", + "model.layers.53.mlp.experts.148.up_proj", + "model.layers.53.mlp.experts.149.down_proj", + "model.layers.53.mlp.experts.149.gate_proj", + "model.layers.53.mlp.experts.149.up_proj", + "model.layers.53.mlp.experts.15.down_proj", + "model.layers.53.mlp.experts.15.gate_proj", + "model.layers.53.mlp.experts.15.up_proj", + "model.layers.53.mlp.experts.150.down_proj", + "model.layers.53.mlp.experts.150.gate_proj", + "model.layers.53.mlp.experts.150.up_proj", + "model.layers.53.mlp.experts.151.down_proj", + "model.layers.53.mlp.experts.151.gate_proj", + "model.layers.53.mlp.experts.151.up_proj", + "model.layers.53.mlp.experts.152.down_proj", + "model.layers.53.mlp.experts.152.gate_proj", + "model.layers.53.mlp.experts.152.up_proj", + "model.layers.53.mlp.experts.153.down_proj", + "model.layers.53.mlp.experts.153.gate_proj", + "model.layers.53.mlp.experts.153.up_proj", + "model.layers.53.mlp.experts.154.down_proj", + "model.layers.53.mlp.experts.154.gate_proj", + "model.layers.53.mlp.experts.154.up_proj", + "model.layers.53.mlp.experts.155.down_proj", + "model.layers.53.mlp.experts.155.gate_proj", + "model.layers.53.mlp.experts.155.up_proj", + "model.layers.53.mlp.experts.156.down_proj", + "model.layers.53.mlp.experts.156.gate_proj", + "model.layers.53.mlp.experts.156.up_proj", + "model.layers.53.mlp.experts.157.down_proj", + "model.layers.53.mlp.experts.157.gate_proj", + "model.layers.53.mlp.experts.157.up_proj", + "model.layers.53.mlp.experts.158.down_proj", + "model.layers.53.mlp.experts.158.gate_proj", + "model.layers.53.mlp.experts.158.up_proj", + "model.layers.53.mlp.experts.159.down_proj", + "model.layers.53.mlp.experts.159.gate_proj", + "model.layers.53.mlp.experts.159.up_proj", + "model.layers.53.mlp.experts.16.down_proj", + "model.layers.53.mlp.experts.16.gate_proj", + "model.layers.53.mlp.experts.16.up_proj", + "model.layers.53.mlp.experts.160.down_proj", + "model.layers.53.mlp.experts.160.gate_proj", + "model.layers.53.mlp.experts.160.up_proj", + "model.layers.53.mlp.experts.161.down_proj", + "model.layers.53.mlp.experts.161.gate_proj", + "model.layers.53.mlp.experts.161.up_proj", + "model.layers.53.mlp.experts.162.down_proj", + "model.layers.53.mlp.experts.162.gate_proj", + "model.layers.53.mlp.experts.162.up_proj", + "model.layers.53.mlp.experts.163.down_proj", + "model.layers.53.mlp.experts.163.gate_proj", + "model.layers.53.mlp.experts.163.up_proj", + "model.layers.53.mlp.experts.164.down_proj", + "model.layers.53.mlp.experts.164.gate_proj", + "model.layers.53.mlp.experts.164.up_proj", + "model.layers.53.mlp.experts.165.down_proj", + "model.layers.53.mlp.experts.165.gate_proj", + "model.layers.53.mlp.experts.165.up_proj", + "model.layers.53.mlp.experts.166.down_proj", + "model.layers.53.mlp.experts.166.gate_proj", + "model.layers.53.mlp.experts.166.up_proj", + "model.layers.53.mlp.experts.167.down_proj", + "model.layers.53.mlp.experts.167.gate_proj", + "model.layers.53.mlp.experts.167.up_proj", + "model.layers.53.mlp.experts.168.down_proj", + "model.layers.53.mlp.experts.168.gate_proj", + "model.layers.53.mlp.experts.168.up_proj", + "model.layers.53.mlp.experts.169.down_proj", + "model.layers.53.mlp.experts.169.gate_proj", + "model.layers.53.mlp.experts.169.up_proj", + "model.layers.53.mlp.experts.17.down_proj", + "model.layers.53.mlp.experts.17.gate_proj", + "model.layers.53.mlp.experts.17.up_proj", + "model.layers.53.mlp.experts.170.down_proj", + "model.layers.53.mlp.experts.170.gate_proj", + "model.layers.53.mlp.experts.170.up_proj", + "model.layers.53.mlp.experts.171.down_proj", + "model.layers.53.mlp.experts.171.gate_proj", + "model.layers.53.mlp.experts.171.up_proj", + "model.layers.53.mlp.experts.172.down_proj", + "model.layers.53.mlp.experts.172.gate_proj", + "model.layers.53.mlp.experts.172.up_proj", + "model.layers.53.mlp.experts.173.down_proj", + "model.layers.53.mlp.experts.173.gate_proj", + "model.layers.53.mlp.experts.173.up_proj", + "model.layers.53.mlp.experts.174.down_proj", + "model.layers.53.mlp.experts.174.gate_proj", + "model.layers.53.mlp.experts.174.up_proj", + "model.layers.53.mlp.experts.175.down_proj", + "model.layers.53.mlp.experts.175.gate_proj", + "model.layers.53.mlp.experts.175.up_proj", + "model.layers.53.mlp.experts.176.down_proj", + "model.layers.53.mlp.experts.176.gate_proj", + "model.layers.53.mlp.experts.176.up_proj", + "model.layers.53.mlp.experts.177.down_proj", + "model.layers.53.mlp.experts.177.gate_proj", + "model.layers.53.mlp.experts.177.up_proj", + "model.layers.53.mlp.experts.178.down_proj", + "model.layers.53.mlp.experts.178.gate_proj", + "model.layers.53.mlp.experts.178.up_proj", + "model.layers.53.mlp.experts.179.down_proj", + "model.layers.53.mlp.experts.179.gate_proj", + "model.layers.53.mlp.experts.179.up_proj", + "model.layers.53.mlp.experts.18.down_proj", + "model.layers.53.mlp.experts.18.gate_proj", + "model.layers.53.mlp.experts.18.up_proj", + "model.layers.53.mlp.experts.180.down_proj", + "model.layers.53.mlp.experts.180.gate_proj", + "model.layers.53.mlp.experts.180.up_proj", + "model.layers.53.mlp.experts.181.down_proj", + "model.layers.53.mlp.experts.181.gate_proj", + "model.layers.53.mlp.experts.181.up_proj", + "model.layers.53.mlp.experts.182.down_proj", + "model.layers.53.mlp.experts.182.gate_proj", + "model.layers.53.mlp.experts.182.up_proj", + "model.layers.53.mlp.experts.183.down_proj", + "model.layers.53.mlp.experts.183.gate_proj", + "model.layers.53.mlp.experts.183.up_proj", + "model.layers.53.mlp.experts.184.down_proj", + "model.layers.53.mlp.experts.184.gate_proj", + "model.layers.53.mlp.experts.184.up_proj", + "model.layers.53.mlp.experts.185.down_proj", + "model.layers.53.mlp.experts.185.gate_proj", + "model.layers.53.mlp.experts.185.up_proj", + "model.layers.53.mlp.experts.186.down_proj", + "model.layers.53.mlp.experts.186.gate_proj", + "model.layers.53.mlp.experts.186.up_proj", + "model.layers.53.mlp.experts.187.down_proj", + "model.layers.53.mlp.experts.187.gate_proj", + "model.layers.53.mlp.experts.187.up_proj", + "model.layers.53.mlp.experts.188.down_proj", + "model.layers.53.mlp.experts.188.gate_proj", + "model.layers.53.mlp.experts.188.up_proj", + "model.layers.53.mlp.experts.189.down_proj", + "model.layers.53.mlp.experts.189.gate_proj", + "model.layers.53.mlp.experts.189.up_proj", + "model.layers.53.mlp.experts.19.down_proj", + "model.layers.53.mlp.experts.19.gate_proj", + "model.layers.53.mlp.experts.19.up_proj", + "model.layers.53.mlp.experts.190.down_proj", + "model.layers.53.mlp.experts.190.gate_proj", + "model.layers.53.mlp.experts.190.up_proj", + "model.layers.53.mlp.experts.191.down_proj", + "model.layers.53.mlp.experts.191.gate_proj", + "model.layers.53.mlp.experts.191.up_proj", + "model.layers.53.mlp.experts.192.down_proj", + "model.layers.53.mlp.experts.192.gate_proj", + "model.layers.53.mlp.experts.192.up_proj", + "model.layers.53.mlp.experts.193.down_proj", + "model.layers.53.mlp.experts.193.gate_proj", + "model.layers.53.mlp.experts.193.up_proj", + "model.layers.53.mlp.experts.194.down_proj", + "model.layers.53.mlp.experts.194.gate_proj", + "model.layers.53.mlp.experts.194.up_proj", + "model.layers.53.mlp.experts.195.down_proj", + "model.layers.53.mlp.experts.195.gate_proj", + "model.layers.53.mlp.experts.195.up_proj", + "model.layers.53.mlp.experts.196.down_proj", + "model.layers.53.mlp.experts.196.gate_proj", + "model.layers.53.mlp.experts.196.up_proj", + "model.layers.53.mlp.experts.197.down_proj", + "model.layers.53.mlp.experts.197.gate_proj", + "model.layers.53.mlp.experts.197.up_proj", + "model.layers.53.mlp.experts.198.down_proj", + "model.layers.53.mlp.experts.198.gate_proj", + "model.layers.53.mlp.experts.198.up_proj", + "model.layers.53.mlp.experts.199.down_proj", + "model.layers.53.mlp.experts.199.gate_proj", + "model.layers.53.mlp.experts.199.up_proj", + "model.layers.53.mlp.experts.2.down_proj", + "model.layers.53.mlp.experts.2.gate_proj", + "model.layers.53.mlp.experts.2.up_proj", + "model.layers.53.mlp.experts.20.down_proj", + "model.layers.53.mlp.experts.20.gate_proj", + "model.layers.53.mlp.experts.20.up_proj", + "model.layers.53.mlp.experts.200.down_proj", + "model.layers.53.mlp.experts.200.gate_proj", + "model.layers.53.mlp.experts.200.up_proj", + "model.layers.53.mlp.experts.201.down_proj", + "model.layers.53.mlp.experts.201.gate_proj", + "model.layers.53.mlp.experts.201.up_proj", + "model.layers.53.mlp.experts.202.down_proj", + "model.layers.53.mlp.experts.202.gate_proj", + "model.layers.53.mlp.experts.202.up_proj", + "model.layers.53.mlp.experts.203.down_proj", + "model.layers.53.mlp.experts.203.gate_proj", + "model.layers.53.mlp.experts.203.up_proj", + "model.layers.53.mlp.experts.204.down_proj", + "model.layers.53.mlp.experts.204.gate_proj", + "model.layers.53.mlp.experts.204.up_proj", + "model.layers.53.mlp.experts.205.down_proj", + "model.layers.53.mlp.experts.205.gate_proj", + "model.layers.53.mlp.experts.205.up_proj", + "model.layers.53.mlp.experts.206.down_proj", + "model.layers.53.mlp.experts.206.gate_proj", + "model.layers.53.mlp.experts.206.up_proj", + "model.layers.53.mlp.experts.207.down_proj", + "model.layers.53.mlp.experts.207.gate_proj", + "model.layers.53.mlp.experts.207.up_proj", + "model.layers.53.mlp.experts.208.down_proj", + "model.layers.53.mlp.experts.208.gate_proj", + "model.layers.53.mlp.experts.208.up_proj", + "model.layers.53.mlp.experts.209.down_proj", + "model.layers.53.mlp.experts.209.gate_proj", + "model.layers.53.mlp.experts.209.up_proj", + "model.layers.53.mlp.experts.21.down_proj", + "model.layers.53.mlp.experts.21.gate_proj", + "model.layers.53.mlp.experts.21.up_proj", + "model.layers.53.mlp.experts.210.down_proj", + "model.layers.53.mlp.experts.210.gate_proj", + "model.layers.53.mlp.experts.210.up_proj", + "model.layers.53.mlp.experts.211.down_proj", + "model.layers.53.mlp.experts.211.gate_proj", + "model.layers.53.mlp.experts.211.up_proj", + "model.layers.53.mlp.experts.212.down_proj", + "model.layers.53.mlp.experts.212.gate_proj", + "model.layers.53.mlp.experts.212.up_proj", + "model.layers.53.mlp.experts.213.down_proj", + "model.layers.53.mlp.experts.213.gate_proj", + "model.layers.53.mlp.experts.213.up_proj", + "model.layers.53.mlp.experts.214.down_proj", + "model.layers.53.mlp.experts.214.gate_proj", + "model.layers.53.mlp.experts.214.up_proj", + "model.layers.53.mlp.experts.215.down_proj", + "model.layers.53.mlp.experts.215.gate_proj", + "model.layers.53.mlp.experts.215.up_proj", + "model.layers.53.mlp.experts.216.down_proj", + "model.layers.53.mlp.experts.216.gate_proj", + "model.layers.53.mlp.experts.216.up_proj", + "model.layers.53.mlp.experts.217.down_proj", + "model.layers.53.mlp.experts.217.gate_proj", + "model.layers.53.mlp.experts.217.up_proj", + "model.layers.53.mlp.experts.218.down_proj", + "model.layers.53.mlp.experts.218.gate_proj", + "model.layers.53.mlp.experts.218.up_proj", + "model.layers.53.mlp.experts.219.down_proj", + "model.layers.53.mlp.experts.219.gate_proj", + "model.layers.53.mlp.experts.219.up_proj", + "model.layers.53.mlp.experts.22.down_proj", + "model.layers.53.mlp.experts.22.gate_proj", + "model.layers.53.mlp.experts.22.up_proj", + "model.layers.53.mlp.experts.220.down_proj", + "model.layers.53.mlp.experts.220.gate_proj", + "model.layers.53.mlp.experts.220.up_proj", + "model.layers.53.mlp.experts.221.down_proj", + "model.layers.53.mlp.experts.221.gate_proj", + "model.layers.53.mlp.experts.221.up_proj", + "model.layers.53.mlp.experts.222.down_proj", + "model.layers.53.mlp.experts.222.gate_proj", + "model.layers.53.mlp.experts.222.up_proj", + "model.layers.53.mlp.experts.223.down_proj", + "model.layers.53.mlp.experts.223.gate_proj", + "model.layers.53.mlp.experts.223.up_proj", + "model.layers.53.mlp.experts.224.down_proj", + "model.layers.53.mlp.experts.224.gate_proj", + "model.layers.53.mlp.experts.224.up_proj", + "model.layers.53.mlp.experts.225.down_proj", + "model.layers.53.mlp.experts.225.gate_proj", + "model.layers.53.mlp.experts.225.up_proj", + "model.layers.53.mlp.experts.226.down_proj", + "model.layers.53.mlp.experts.226.gate_proj", + "model.layers.53.mlp.experts.226.up_proj", + "model.layers.53.mlp.experts.227.down_proj", + "model.layers.53.mlp.experts.227.gate_proj", + "model.layers.53.mlp.experts.227.up_proj", + "model.layers.53.mlp.experts.228.down_proj", + "model.layers.53.mlp.experts.228.gate_proj", + "model.layers.53.mlp.experts.228.up_proj", + "model.layers.53.mlp.experts.229.down_proj", + "model.layers.53.mlp.experts.229.gate_proj", + "model.layers.53.mlp.experts.229.up_proj", + "model.layers.53.mlp.experts.23.down_proj", + "model.layers.53.mlp.experts.23.gate_proj", + "model.layers.53.mlp.experts.23.up_proj", + "model.layers.53.mlp.experts.230.down_proj", + "model.layers.53.mlp.experts.230.gate_proj", + "model.layers.53.mlp.experts.230.up_proj", + "model.layers.53.mlp.experts.231.down_proj", + "model.layers.53.mlp.experts.231.gate_proj", + "model.layers.53.mlp.experts.231.up_proj", + "model.layers.53.mlp.experts.232.down_proj", + "model.layers.53.mlp.experts.232.gate_proj", + "model.layers.53.mlp.experts.232.up_proj", + "model.layers.53.mlp.experts.233.down_proj", + "model.layers.53.mlp.experts.233.gate_proj", + "model.layers.53.mlp.experts.233.up_proj", + "model.layers.53.mlp.experts.234.down_proj", + "model.layers.53.mlp.experts.234.gate_proj", + "model.layers.53.mlp.experts.234.up_proj", + "model.layers.53.mlp.experts.235.down_proj", + "model.layers.53.mlp.experts.235.gate_proj", + "model.layers.53.mlp.experts.235.up_proj", + "model.layers.53.mlp.experts.236.down_proj", + "model.layers.53.mlp.experts.236.gate_proj", + "model.layers.53.mlp.experts.236.up_proj", + "model.layers.53.mlp.experts.237.down_proj", + "model.layers.53.mlp.experts.237.gate_proj", + "model.layers.53.mlp.experts.237.up_proj", + "model.layers.53.mlp.experts.238.down_proj", + "model.layers.53.mlp.experts.238.gate_proj", + "model.layers.53.mlp.experts.238.up_proj", + "model.layers.53.mlp.experts.239.down_proj", + "model.layers.53.mlp.experts.239.gate_proj", + "model.layers.53.mlp.experts.239.up_proj", + "model.layers.53.mlp.experts.24.down_proj", + "model.layers.53.mlp.experts.24.gate_proj", + "model.layers.53.mlp.experts.24.up_proj", + "model.layers.53.mlp.experts.240.down_proj", + "model.layers.53.mlp.experts.240.gate_proj", + "model.layers.53.mlp.experts.240.up_proj", + "model.layers.53.mlp.experts.241.down_proj", + "model.layers.53.mlp.experts.241.gate_proj", + "model.layers.53.mlp.experts.241.up_proj", + "model.layers.53.mlp.experts.242.down_proj", + "model.layers.53.mlp.experts.242.gate_proj", + "model.layers.53.mlp.experts.242.up_proj", + "model.layers.53.mlp.experts.243.down_proj", + "model.layers.53.mlp.experts.243.gate_proj", + "model.layers.53.mlp.experts.243.up_proj", + "model.layers.53.mlp.experts.244.down_proj", + "model.layers.53.mlp.experts.244.gate_proj", + "model.layers.53.mlp.experts.244.up_proj", + "model.layers.53.mlp.experts.245.down_proj", + "model.layers.53.mlp.experts.245.gate_proj", + "model.layers.53.mlp.experts.245.up_proj", + "model.layers.53.mlp.experts.246.down_proj", + "model.layers.53.mlp.experts.246.gate_proj", + "model.layers.53.mlp.experts.246.up_proj", + "model.layers.53.mlp.experts.247.down_proj", + "model.layers.53.mlp.experts.247.gate_proj", + "model.layers.53.mlp.experts.247.up_proj", + "model.layers.53.mlp.experts.248.down_proj", + "model.layers.53.mlp.experts.248.gate_proj", + "model.layers.53.mlp.experts.248.up_proj", + "model.layers.53.mlp.experts.249.down_proj", + "model.layers.53.mlp.experts.249.gate_proj", + "model.layers.53.mlp.experts.249.up_proj", + "model.layers.53.mlp.experts.25.down_proj", + "model.layers.53.mlp.experts.25.gate_proj", + "model.layers.53.mlp.experts.25.up_proj", + "model.layers.53.mlp.experts.250.down_proj", + "model.layers.53.mlp.experts.250.gate_proj", + "model.layers.53.mlp.experts.250.up_proj", + "model.layers.53.mlp.experts.251.down_proj", + "model.layers.53.mlp.experts.251.gate_proj", + "model.layers.53.mlp.experts.251.up_proj", + "model.layers.53.mlp.experts.252.down_proj", + "model.layers.53.mlp.experts.252.gate_proj", + "model.layers.53.mlp.experts.252.up_proj", + "model.layers.53.mlp.experts.253.down_proj", + "model.layers.53.mlp.experts.253.gate_proj", + "model.layers.53.mlp.experts.253.up_proj", + "model.layers.53.mlp.experts.254.down_proj", + "model.layers.53.mlp.experts.254.gate_proj", + "model.layers.53.mlp.experts.254.up_proj", + "model.layers.53.mlp.experts.255.down_proj", + "model.layers.53.mlp.experts.255.gate_proj", + "model.layers.53.mlp.experts.255.up_proj", + "model.layers.53.mlp.experts.26.down_proj", + "model.layers.53.mlp.experts.26.gate_proj", + "model.layers.53.mlp.experts.26.up_proj", + "model.layers.53.mlp.experts.27.down_proj", + "model.layers.53.mlp.experts.27.gate_proj", + "model.layers.53.mlp.experts.27.up_proj", + "model.layers.53.mlp.experts.28.down_proj", + "model.layers.53.mlp.experts.28.gate_proj", + "model.layers.53.mlp.experts.28.up_proj", + "model.layers.53.mlp.experts.29.down_proj", + "model.layers.53.mlp.experts.29.gate_proj", + "model.layers.53.mlp.experts.29.up_proj", + "model.layers.53.mlp.experts.3.down_proj", + "model.layers.53.mlp.experts.3.gate_proj", + "model.layers.53.mlp.experts.3.up_proj", + "model.layers.53.mlp.experts.30.down_proj", + "model.layers.53.mlp.experts.30.gate_proj", + "model.layers.53.mlp.experts.30.up_proj", + "model.layers.53.mlp.experts.31.down_proj", + "model.layers.53.mlp.experts.31.gate_proj", + "model.layers.53.mlp.experts.31.up_proj", + "model.layers.53.mlp.experts.32.down_proj", + "model.layers.53.mlp.experts.32.gate_proj", + "model.layers.53.mlp.experts.32.up_proj", + "model.layers.53.mlp.experts.33.down_proj", + "model.layers.53.mlp.experts.33.gate_proj", + "model.layers.53.mlp.experts.33.up_proj", + "model.layers.53.mlp.experts.34.down_proj", + "model.layers.53.mlp.experts.34.gate_proj", + "model.layers.53.mlp.experts.34.up_proj", + "model.layers.53.mlp.experts.35.down_proj", + "model.layers.53.mlp.experts.35.gate_proj", + "model.layers.53.mlp.experts.35.up_proj", + "model.layers.53.mlp.experts.36.down_proj", + "model.layers.53.mlp.experts.36.gate_proj", + "model.layers.53.mlp.experts.36.up_proj", + "model.layers.53.mlp.experts.37.down_proj", + "model.layers.53.mlp.experts.37.gate_proj", + "model.layers.53.mlp.experts.37.up_proj", + "model.layers.53.mlp.experts.38.down_proj", + "model.layers.53.mlp.experts.38.gate_proj", + "model.layers.53.mlp.experts.38.up_proj", + "model.layers.53.mlp.experts.39.down_proj", + "model.layers.53.mlp.experts.39.gate_proj", + "model.layers.53.mlp.experts.39.up_proj", + "model.layers.53.mlp.experts.4.down_proj", + "model.layers.53.mlp.experts.4.gate_proj", + "model.layers.53.mlp.experts.4.up_proj", + "model.layers.53.mlp.experts.40.down_proj", + "model.layers.53.mlp.experts.40.gate_proj", + "model.layers.53.mlp.experts.40.up_proj", + "model.layers.53.mlp.experts.41.down_proj", + "model.layers.53.mlp.experts.41.gate_proj", + "model.layers.53.mlp.experts.41.up_proj", + "model.layers.53.mlp.experts.42.down_proj", + "model.layers.53.mlp.experts.42.gate_proj", + "model.layers.53.mlp.experts.42.up_proj", + "model.layers.53.mlp.experts.43.down_proj", + "model.layers.53.mlp.experts.43.gate_proj", + "model.layers.53.mlp.experts.43.up_proj", + "model.layers.53.mlp.experts.44.down_proj", + "model.layers.53.mlp.experts.44.gate_proj", + "model.layers.53.mlp.experts.44.up_proj", + "model.layers.53.mlp.experts.45.down_proj", + "model.layers.53.mlp.experts.45.gate_proj", + "model.layers.53.mlp.experts.45.up_proj", + "model.layers.53.mlp.experts.46.down_proj", + "model.layers.53.mlp.experts.46.gate_proj", + "model.layers.53.mlp.experts.46.up_proj", + "model.layers.53.mlp.experts.47.down_proj", + "model.layers.53.mlp.experts.47.gate_proj", + "model.layers.53.mlp.experts.47.up_proj", + "model.layers.53.mlp.experts.48.down_proj", + "model.layers.53.mlp.experts.48.gate_proj", + "model.layers.53.mlp.experts.48.up_proj", + "model.layers.53.mlp.experts.49.down_proj", + "model.layers.53.mlp.experts.49.gate_proj", + "model.layers.53.mlp.experts.49.up_proj", + "model.layers.53.mlp.experts.5.down_proj", + "model.layers.53.mlp.experts.5.gate_proj", + "model.layers.53.mlp.experts.5.up_proj", + "model.layers.53.mlp.experts.50.down_proj", + "model.layers.53.mlp.experts.50.gate_proj", + "model.layers.53.mlp.experts.50.up_proj", + "model.layers.53.mlp.experts.51.down_proj", + "model.layers.53.mlp.experts.51.gate_proj", + "model.layers.53.mlp.experts.51.up_proj", + "model.layers.53.mlp.experts.52.down_proj", + "model.layers.53.mlp.experts.52.gate_proj", + "model.layers.53.mlp.experts.52.up_proj", + "model.layers.53.mlp.experts.53.down_proj", + "model.layers.53.mlp.experts.53.gate_proj", + "model.layers.53.mlp.experts.53.up_proj", + "model.layers.53.mlp.experts.54.down_proj", + "model.layers.53.mlp.experts.54.gate_proj", + "model.layers.53.mlp.experts.54.up_proj", + "model.layers.53.mlp.experts.55.down_proj", + "model.layers.53.mlp.experts.55.gate_proj", + "model.layers.53.mlp.experts.55.up_proj", + "model.layers.53.mlp.experts.56.down_proj", + "model.layers.53.mlp.experts.56.gate_proj", + "model.layers.53.mlp.experts.56.up_proj", + "model.layers.53.mlp.experts.57.down_proj", + "model.layers.53.mlp.experts.57.gate_proj", + "model.layers.53.mlp.experts.57.up_proj", + "model.layers.53.mlp.experts.58.down_proj", + "model.layers.53.mlp.experts.58.gate_proj", + "model.layers.53.mlp.experts.58.up_proj", + "model.layers.53.mlp.experts.59.down_proj", + "model.layers.53.mlp.experts.59.gate_proj", + "model.layers.53.mlp.experts.59.up_proj", + "model.layers.53.mlp.experts.6.down_proj", + "model.layers.53.mlp.experts.6.gate_proj", + "model.layers.53.mlp.experts.6.up_proj", + "model.layers.53.mlp.experts.60.down_proj", + "model.layers.53.mlp.experts.60.gate_proj", + "model.layers.53.mlp.experts.60.up_proj", + "model.layers.53.mlp.experts.61.down_proj", + "model.layers.53.mlp.experts.61.gate_proj", + "model.layers.53.mlp.experts.61.up_proj", + "model.layers.53.mlp.experts.62.down_proj", + "model.layers.53.mlp.experts.62.gate_proj", + "model.layers.53.mlp.experts.62.up_proj", + "model.layers.53.mlp.experts.63.down_proj", + "model.layers.53.mlp.experts.63.gate_proj", + "model.layers.53.mlp.experts.63.up_proj", + "model.layers.53.mlp.experts.64.down_proj", + "model.layers.53.mlp.experts.64.gate_proj", + "model.layers.53.mlp.experts.64.up_proj", + "model.layers.53.mlp.experts.65.down_proj", + "model.layers.53.mlp.experts.65.gate_proj", + "model.layers.53.mlp.experts.65.up_proj", + "model.layers.53.mlp.experts.66.down_proj", + "model.layers.53.mlp.experts.66.gate_proj", + "model.layers.53.mlp.experts.66.up_proj", + "model.layers.53.mlp.experts.67.down_proj", + "model.layers.53.mlp.experts.67.gate_proj", + "model.layers.53.mlp.experts.67.up_proj", + "model.layers.53.mlp.experts.68.down_proj", + "model.layers.53.mlp.experts.68.gate_proj", + "model.layers.53.mlp.experts.68.up_proj", + "model.layers.53.mlp.experts.69.down_proj", + "model.layers.53.mlp.experts.69.gate_proj", + "model.layers.53.mlp.experts.69.up_proj", + "model.layers.53.mlp.experts.7.down_proj", + "model.layers.53.mlp.experts.7.gate_proj", + "model.layers.53.mlp.experts.7.up_proj", + "model.layers.53.mlp.experts.70.down_proj", + "model.layers.53.mlp.experts.70.gate_proj", + "model.layers.53.mlp.experts.70.up_proj", + "model.layers.53.mlp.experts.71.down_proj", + "model.layers.53.mlp.experts.71.gate_proj", + "model.layers.53.mlp.experts.71.up_proj", + "model.layers.53.mlp.experts.72.down_proj", + "model.layers.53.mlp.experts.72.gate_proj", + "model.layers.53.mlp.experts.72.up_proj", + "model.layers.53.mlp.experts.73.down_proj", + "model.layers.53.mlp.experts.73.gate_proj", + "model.layers.53.mlp.experts.73.up_proj", + "model.layers.53.mlp.experts.74.down_proj", + "model.layers.53.mlp.experts.74.gate_proj", + "model.layers.53.mlp.experts.74.up_proj", + "model.layers.53.mlp.experts.75.down_proj", + "model.layers.53.mlp.experts.75.gate_proj", + "model.layers.53.mlp.experts.75.up_proj", + "model.layers.53.mlp.experts.76.down_proj", + "model.layers.53.mlp.experts.76.gate_proj", + "model.layers.53.mlp.experts.76.up_proj", + "model.layers.53.mlp.experts.77.down_proj", + "model.layers.53.mlp.experts.77.gate_proj", + "model.layers.53.mlp.experts.77.up_proj", + "model.layers.53.mlp.experts.78.down_proj", + "model.layers.53.mlp.experts.78.gate_proj", + "model.layers.53.mlp.experts.78.up_proj", + "model.layers.53.mlp.experts.79.down_proj", + "model.layers.53.mlp.experts.79.gate_proj", + "model.layers.53.mlp.experts.79.up_proj", + "model.layers.53.mlp.experts.8.down_proj", + "model.layers.53.mlp.experts.8.gate_proj", + "model.layers.53.mlp.experts.8.up_proj", + "model.layers.53.mlp.experts.80.down_proj", + "model.layers.53.mlp.experts.80.gate_proj", + "model.layers.53.mlp.experts.80.up_proj", + "model.layers.53.mlp.experts.81.down_proj", + "model.layers.53.mlp.experts.81.gate_proj", + "model.layers.53.mlp.experts.81.up_proj", + "model.layers.53.mlp.experts.82.down_proj", + "model.layers.53.mlp.experts.82.gate_proj", + "model.layers.53.mlp.experts.82.up_proj", + "model.layers.53.mlp.experts.83.down_proj", + "model.layers.53.mlp.experts.83.gate_proj", + "model.layers.53.mlp.experts.83.up_proj", + "model.layers.53.mlp.experts.84.down_proj", + "model.layers.53.mlp.experts.84.gate_proj", + "model.layers.53.mlp.experts.84.up_proj", + "model.layers.53.mlp.experts.85.down_proj", + "model.layers.53.mlp.experts.85.gate_proj", + "model.layers.53.mlp.experts.85.up_proj", + "model.layers.53.mlp.experts.86.down_proj", + "model.layers.53.mlp.experts.86.gate_proj", + "model.layers.53.mlp.experts.86.up_proj", + "model.layers.53.mlp.experts.87.down_proj", + "model.layers.53.mlp.experts.87.gate_proj", + "model.layers.53.mlp.experts.87.up_proj", + "model.layers.53.mlp.experts.88.down_proj", + "model.layers.53.mlp.experts.88.gate_proj", + "model.layers.53.mlp.experts.88.up_proj", + "model.layers.53.mlp.experts.89.down_proj", + "model.layers.53.mlp.experts.89.gate_proj", + "model.layers.53.mlp.experts.89.up_proj", + "model.layers.53.mlp.experts.9.down_proj", + "model.layers.53.mlp.experts.9.gate_proj", + "model.layers.53.mlp.experts.9.up_proj", + "model.layers.53.mlp.experts.90.down_proj", + "model.layers.53.mlp.experts.90.gate_proj", + "model.layers.53.mlp.experts.90.up_proj", + "model.layers.53.mlp.experts.91.down_proj", + "model.layers.53.mlp.experts.91.gate_proj", + "model.layers.53.mlp.experts.91.up_proj", + "model.layers.53.mlp.experts.92.down_proj", + "model.layers.53.mlp.experts.92.gate_proj", + "model.layers.53.mlp.experts.92.up_proj", + "model.layers.53.mlp.experts.93.down_proj", + "model.layers.53.mlp.experts.93.gate_proj", + "model.layers.53.mlp.experts.93.up_proj", + "model.layers.53.mlp.experts.94.down_proj", + "model.layers.53.mlp.experts.94.gate_proj", + "model.layers.53.mlp.experts.94.up_proj", + "model.layers.53.mlp.experts.95.down_proj", + "model.layers.53.mlp.experts.95.gate_proj", + "model.layers.53.mlp.experts.95.up_proj", + "model.layers.53.mlp.experts.96.down_proj", + "model.layers.53.mlp.experts.96.gate_proj", + "model.layers.53.mlp.experts.96.up_proj", + "model.layers.53.mlp.experts.97.down_proj", + "model.layers.53.mlp.experts.97.gate_proj", + "model.layers.53.mlp.experts.97.up_proj", + "model.layers.53.mlp.experts.98.down_proj", + "model.layers.53.mlp.experts.98.gate_proj", + "model.layers.53.mlp.experts.98.up_proj", + "model.layers.53.mlp.experts.99.down_proj", + "model.layers.53.mlp.experts.99.gate_proj", + "model.layers.53.mlp.experts.99.up_proj", + "model.layers.54.mlp.experts.0.down_proj", + "model.layers.54.mlp.experts.0.gate_proj", + "model.layers.54.mlp.experts.0.up_proj", + "model.layers.54.mlp.experts.1.down_proj", + "model.layers.54.mlp.experts.1.gate_proj", + "model.layers.54.mlp.experts.1.up_proj", + "model.layers.54.mlp.experts.10.down_proj", + "model.layers.54.mlp.experts.10.gate_proj", + "model.layers.54.mlp.experts.10.up_proj", + "model.layers.54.mlp.experts.100.down_proj", + "model.layers.54.mlp.experts.100.gate_proj", + "model.layers.54.mlp.experts.100.up_proj", + "model.layers.54.mlp.experts.101.down_proj", + "model.layers.54.mlp.experts.101.gate_proj", + "model.layers.54.mlp.experts.101.up_proj", + "model.layers.54.mlp.experts.102.down_proj", + "model.layers.54.mlp.experts.102.gate_proj", + "model.layers.54.mlp.experts.102.up_proj", + "model.layers.54.mlp.experts.103.down_proj", + "model.layers.54.mlp.experts.103.gate_proj", + "model.layers.54.mlp.experts.103.up_proj", + "model.layers.54.mlp.experts.104.down_proj", + "model.layers.54.mlp.experts.104.gate_proj", + "model.layers.54.mlp.experts.104.up_proj", + "model.layers.54.mlp.experts.105.down_proj", + "model.layers.54.mlp.experts.105.gate_proj", + "model.layers.54.mlp.experts.105.up_proj", + "model.layers.54.mlp.experts.106.down_proj", + "model.layers.54.mlp.experts.106.gate_proj", + "model.layers.54.mlp.experts.106.up_proj", + "model.layers.54.mlp.experts.107.down_proj", + "model.layers.54.mlp.experts.107.gate_proj", + "model.layers.54.mlp.experts.107.up_proj", + "model.layers.54.mlp.experts.108.down_proj", + "model.layers.54.mlp.experts.108.gate_proj", + "model.layers.54.mlp.experts.108.up_proj", + "model.layers.54.mlp.experts.109.down_proj", + "model.layers.54.mlp.experts.109.gate_proj", + "model.layers.54.mlp.experts.109.up_proj", + "model.layers.54.mlp.experts.11.down_proj", + "model.layers.54.mlp.experts.11.gate_proj", + "model.layers.54.mlp.experts.11.up_proj", + "model.layers.54.mlp.experts.110.down_proj", + "model.layers.54.mlp.experts.110.gate_proj", + "model.layers.54.mlp.experts.110.up_proj", + "model.layers.54.mlp.experts.111.down_proj", + "model.layers.54.mlp.experts.111.gate_proj", + "model.layers.54.mlp.experts.111.up_proj", + "model.layers.54.mlp.experts.112.down_proj", + "model.layers.54.mlp.experts.112.gate_proj", + "model.layers.54.mlp.experts.112.up_proj", + "model.layers.54.mlp.experts.113.down_proj", + "model.layers.54.mlp.experts.113.gate_proj", + "model.layers.54.mlp.experts.113.up_proj", + "model.layers.54.mlp.experts.114.down_proj", + "model.layers.54.mlp.experts.114.gate_proj", + "model.layers.54.mlp.experts.114.up_proj", + "model.layers.54.mlp.experts.115.down_proj", + "model.layers.54.mlp.experts.115.gate_proj", + "model.layers.54.mlp.experts.115.up_proj", + "model.layers.54.mlp.experts.116.down_proj", + "model.layers.54.mlp.experts.116.gate_proj", + "model.layers.54.mlp.experts.116.up_proj", + "model.layers.54.mlp.experts.117.down_proj", + "model.layers.54.mlp.experts.117.gate_proj", + "model.layers.54.mlp.experts.117.up_proj", + "model.layers.54.mlp.experts.118.down_proj", + "model.layers.54.mlp.experts.118.gate_proj", + "model.layers.54.mlp.experts.118.up_proj", + "model.layers.54.mlp.experts.119.down_proj", + "model.layers.54.mlp.experts.119.gate_proj", + "model.layers.54.mlp.experts.119.up_proj", + "model.layers.54.mlp.experts.12.down_proj", + "model.layers.54.mlp.experts.12.gate_proj", + "model.layers.54.mlp.experts.12.up_proj", + "model.layers.54.mlp.experts.120.down_proj", + "model.layers.54.mlp.experts.120.gate_proj", + "model.layers.54.mlp.experts.120.up_proj", + "model.layers.54.mlp.experts.121.down_proj", + "model.layers.54.mlp.experts.121.gate_proj", + "model.layers.54.mlp.experts.121.up_proj", + "model.layers.54.mlp.experts.122.down_proj", + "model.layers.54.mlp.experts.122.gate_proj", + "model.layers.54.mlp.experts.122.up_proj", + "model.layers.54.mlp.experts.123.down_proj", + "model.layers.54.mlp.experts.123.gate_proj", + "model.layers.54.mlp.experts.123.up_proj", + "model.layers.54.mlp.experts.124.down_proj", + "model.layers.54.mlp.experts.124.gate_proj", + "model.layers.54.mlp.experts.124.up_proj", + "model.layers.54.mlp.experts.125.down_proj", + "model.layers.54.mlp.experts.125.gate_proj", + "model.layers.54.mlp.experts.125.up_proj", + "model.layers.54.mlp.experts.126.down_proj", + "model.layers.54.mlp.experts.126.gate_proj", + "model.layers.54.mlp.experts.126.up_proj", + "model.layers.54.mlp.experts.127.down_proj", + "model.layers.54.mlp.experts.127.gate_proj", + "model.layers.54.mlp.experts.127.up_proj", + "model.layers.54.mlp.experts.128.down_proj", + "model.layers.54.mlp.experts.128.gate_proj", + "model.layers.54.mlp.experts.128.up_proj", + "model.layers.54.mlp.experts.129.down_proj", + "model.layers.54.mlp.experts.129.gate_proj", + "model.layers.54.mlp.experts.129.up_proj", + "model.layers.54.mlp.experts.13.down_proj", + "model.layers.54.mlp.experts.13.gate_proj", + "model.layers.54.mlp.experts.13.up_proj", + "model.layers.54.mlp.experts.130.down_proj", + "model.layers.54.mlp.experts.130.gate_proj", + "model.layers.54.mlp.experts.130.up_proj", + "model.layers.54.mlp.experts.131.down_proj", + "model.layers.54.mlp.experts.131.gate_proj", + "model.layers.54.mlp.experts.131.up_proj", + "model.layers.54.mlp.experts.132.down_proj", + "model.layers.54.mlp.experts.132.gate_proj", + "model.layers.54.mlp.experts.132.up_proj", + "model.layers.54.mlp.experts.133.down_proj", + "model.layers.54.mlp.experts.133.gate_proj", + "model.layers.54.mlp.experts.133.up_proj", + "model.layers.54.mlp.experts.134.down_proj", + "model.layers.54.mlp.experts.134.gate_proj", + "model.layers.54.mlp.experts.134.up_proj", + "model.layers.54.mlp.experts.135.down_proj", + "model.layers.54.mlp.experts.135.gate_proj", + "model.layers.54.mlp.experts.135.up_proj", + "model.layers.54.mlp.experts.136.down_proj", + "model.layers.54.mlp.experts.136.gate_proj", + "model.layers.54.mlp.experts.136.up_proj", + "model.layers.54.mlp.experts.137.down_proj", + "model.layers.54.mlp.experts.137.gate_proj", + "model.layers.54.mlp.experts.137.up_proj", + "model.layers.54.mlp.experts.138.down_proj", + "model.layers.54.mlp.experts.138.gate_proj", + "model.layers.54.mlp.experts.138.up_proj", + "model.layers.54.mlp.experts.139.down_proj", + "model.layers.54.mlp.experts.139.gate_proj", + "model.layers.54.mlp.experts.139.up_proj", + "model.layers.54.mlp.experts.14.down_proj", + "model.layers.54.mlp.experts.14.gate_proj", + "model.layers.54.mlp.experts.14.up_proj", + "model.layers.54.mlp.experts.140.down_proj", + "model.layers.54.mlp.experts.140.gate_proj", + "model.layers.54.mlp.experts.140.up_proj", + "model.layers.54.mlp.experts.141.down_proj", + "model.layers.54.mlp.experts.141.gate_proj", + "model.layers.54.mlp.experts.141.up_proj", + "model.layers.54.mlp.experts.142.down_proj", + "model.layers.54.mlp.experts.142.gate_proj", + "model.layers.54.mlp.experts.142.up_proj", + "model.layers.54.mlp.experts.143.down_proj", + "model.layers.54.mlp.experts.143.gate_proj", + "model.layers.54.mlp.experts.143.up_proj", + "model.layers.54.mlp.experts.144.down_proj", + "model.layers.54.mlp.experts.144.gate_proj", + "model.layers.54.mlp.experts.144.up_proj", + "model.layers.54.mlp.experts.145.down_proj", + "model.layers.54.mlp.experts.145.gate_proj", + "model.layers.54.mlp.experts.145.up_proj", + "model.layers.54.mlp.experts.146.down_proj", + "model.layers.54.mlp.experts.146.gate_proj", + "model.layers.54.mlp.experts.146.up_proj", + "model.layers.54.mlp.experts.147.down_proj", + "model.layers.54.mlp.experts.147.gate_proj", + "model.layers.54.mlp.experts.147.up_proj", + "model.layers.54.mlp.experts.148.down_proj", + "model.layers.54.mlp.experts.148.gate_proj", + "model.layers.54.mlp.experts.148.up_proj", + "model.layers.54.mlp.experts.149.down_proj", + "model.layers.54.mlp.experts.149.gate_proj", + "model.layers.54.mlp.experts.149.up_proj", + "model.layers.54.mlp.experts.15.down_proj", + "model.layers.54.mlp.experts.15.gate_proj", + "model.layers.54.mlp.experts.15.up_proj", + "model.layers.54.mlp.experts.150.down_proj", + "model.layers.54.mlp.experts.150.gate_proj", + "model.layers.54.mlp.experts.150.up_proj", + "model.layers.54.mlp.experts.151.down_proj", + "model.layers.54.mlp.experts.151.gate_proj", + "model.layers.54.mlp.experts.151.up_proj", + "model.layers.54.mlp.experts.152.down_proj", + "model.layers.54.mlp.experts.152.gate_proj", + "model.layers.54.mlp.experts.152.up_proj", + "model.layers.54.mlp.experts.153.down_proj", + "model.layers.54.mlp.experts.153.gate_proj", + "model.layers.54.mlp.experts.153.up_proj", + "model.layers.54.mlp.experts.154.down_proj", + "model.layers.54.mlp.experts.154.gate_proj", + "model.layers.54.mlp.experts.154.up_proj", + "model.layers.54.mlp.experts.155.down_proj", + "model.layers.54.mlp.experts.155.gate_proj", + "model.layers.54.mlp.experts.155.up_proj", + "model.layers.54.mlp.experts.156.down_proj", + "model.layers.54.mlp.experts.156.gate_proj", + "model.layers.54.mlp.experts.156.up_proj", + "model.layers.54.mlp.experts.157.down_proj", + "model.layers.54.mlp.experts.157.gate_proj", + "model.layers.54.mlp.experts.157.up_proj", + "model.layers.54.mlp.experts.158.down_proj", + "model.layers.54.mlp.experts.158.gate_proj", + "model.layers.54.mlp.experts.158.up_proj", + "model.layers.54.mlp.experts.159.down_proj", + "model.layers.54.mlp.experts.159.gate_proj", + "model.layers.54.mlp.experts.159.up_proj", + "model.layers.54.mlp.experts.16.down_proj", + "model.layers.54.mlp.experts.16.gate_proj", + "model.layers.54.mlp.experts.16.up_proj", + "model.layers.54.mlp.experts.160.down_proj", + "model.layers.54.mlp.experts.160.gate_proj", + "model.layers.54.mlp.experts.160.up_proj", + "model.layers.54.mlp.experts.161.down_proj", + "model.layers.54.mlp.experts.161.gate_proj", + "model.layers.54.mlp.experts.161.up_proj", + "model.layers.54.mlp.experts.162.down_proj", + "model.layers.54.mlp.experts.162.gate_proj", + "model.layers.54.mlp.experts.162.up_proj", + "model.layers.54.mlp.experts.163.down_proj", + "model.layers.54.mlp.experts.163.gate_proj", + "model.layers.54.mlp.experts.163.up_proj", + "model.layers.54.mlp.experts.164.down_proj", + "model.layers.54.mlp.experts.164.gate_proj", + "model.layers.54.mlp.experts.164.up_proj", + "model.layers.54.mlp.experts.165.down_proj", + "model.layers.54.mlp.experts.165.gate_proj", + "model.layers.54.mlp.experts.165.up_proj", + "model.layers.54.mlp.experts.166.down_proj", + "model.layers.54.mlp.experts.166.gate_proj", + "model.layers.54.mlp.experts.166.up_proj", + "model.layers.54.mlp.experts.167.down_proj", + "model.layers.54.mlp.experts.167.gate_proj", + "model.layers.54.mlp.experts.167.up_proj", + "model.layers.54.mlp.experts.168.down_proj", + "model.layers.54.mlp.experts.168.gate_proj", + "model.layers.54.mlp.experts.168.up_proj", + "model.layers.54.mlp.experts.169.down_proj", + "model.layers.54.mlp.experts.169.gate_proj", + "model.layers.54.mlp.experts.169.up_proj", + "model.layers.54.mlp.experts.17.down_proj", + "model.layers.54.mlp.experts.17.gate_proj", + "model.layers.54.mlp.experts.17.up_proj", + "model.layers.54.mlp.experts.170.down_proj", + "model.layers.54.mlp.experts.170.gate_proj", + "model.layers.54.mlp.experts.170.up_proj", + "model.layers.54.mlp.experts.171.down_proj", + "model.layers.54.mlp.experts.171.gate_proj", + "model.layers.54.mlp.experts.171.up_proj", + "model.layers.54.mlp.experts.172.down_proj", + "model.layers.54.mlp.experts.172.gate_proj", + "model.layers.54.mlp.experts.172.up_proj", + "model.layers.54.mlp.experts.173.down_proj", + "model.layers.54.mlp.experts.173.gate_proj", + "model.layers.54.mlp.experts.173.up_proj", + "model.layers.54.mlp.experts.174.down_proj", + "model.layers.54.mlp.experts.174.gate_proj", + "model.layers.54.mlp.experts.174.up_proj", + "model.layers.54.mlp.experts.175.down_proj", + "model.layers.54.mlp.experts.175.gate_proj", + "model.layers.54.mlp.experts.175.up_proj", + "model.layers.54.mlp.experts.176.down_proj", + "model.layers.54.mlp.experts.176.gate_proj", + "model.layers.54.mlp.experts.176.up_proj", + "model.layers.54.mlp.experts.177.down_proj", + "model.layers.54.mlp.experts.177.gate_proj", + "model.layers.54.mlp.experts.177.up_proj", + "model.layers.54.mlp.experts.178.down_proj", + "model.layers.54.mlp.experts.178.gate_proj", + "model.layers.54.mlp.experts.178.up_proj", + "model.layers.54.mlp.experts.179.down_proj", + "model.layers.54.mlp.experts.179.gate_proj", + "model.layers.54.mlp.experts.179.up_proj", + "model.layers.54.mlp.experts.18.down_proj", + "model.layers.54.mlp.experts.18.gate_proj", + "model.layers.54.mlp.experts.18.up_proj", + "model.layers.54.mlp.experts.180.down_proj", + "model.layers.54.mlp.experts.180.gate_proj", + "model.layers.54.mlp.experts.180.up_proj", + "model.layers.54.mlp.experts.181.down_proj", + "model.layers.54.mlp.experts.181.gate_proj", + "model.layers.54.mlp.experts.181.up_proj", + "model.layers.54.mlp.experts.182.down_proj", + "model.layers.54.mlp.experts.182.gate_proj", + "model.layers.54.mlp.experts.182.up_proj", + "model.layers.54.mlp.experts.183.down_proj", + "model.layers.54.mlp.experts.183.gate_proj", + "model.layers.54.mlp.experts.183.up_proj", + "model.layers.54.mlp.experts.184.down_proj", + "model.layers.54.mlp.experts.184.gate_proj", + "model.layers.54.mlp.experts.184.up_proj", + "model.layers.54.mlp.experts.185.down_proj", + "model.layers.54.mlp.experts.185.gate_proj", + "model.layers.54.mlp.experts.185.up_proj", + "model.layers.54.mlp.experts.186.down_proj", + "model.layers.54.mlp.experts.186.gate_proj", + "model.layers.54.mlp.experts.186.up_proj", + "model.layers.54.mlp.experts.187.down_proj", + "model.layers.54.mlp.experts.187.gate_proj", + "model.layers.54.mlp.experts.187.up_proj", + "model.layers.54.mlp.experts.188.down_proj", + "model.layers.54.mlp.experts.188.gate_proj", + "model.layers.54.mlp.experts.188.up_proj", + "model.layers.54.mlp.experts.189.down_proj", + "model.layers.54.mlp.experts.189.gate_proj", + "model.layers.54.mlp.experts.189.up_proj", + "model.layers.54.mlp.experts.19.down_proj", + "model.layers.54.mlp.experts.19.gate_proj", + "model.layers.54.mlp.experts.19.up_proj", + "model.layers.54.mlp.experts.190.down_proj", + "model.layers.54.mlp.experts.190.gate_proj", + "model.layers.54.mlp.experts.190.up_proj", + "model.layers.54.mlp.experts.191.down_proj", + "model.layers.54.mlp.experts.191.gate_proj", + "model.layers.54.mlp.experts.191.up_proj", + "model.layers.54.mlp.experts.192.down_proj", + "model.layers.54.mlp.experts.192.gate_proj", + "model.layers.54.mlp.experts.192.up_proj", + "model.layers.54.mlp.experts.193.down_proj", + "model.layers.54.mlp.experts.193.gate_proj", + "model.layers.54.mlp.experts.193.up_proj", + "model.layers.54.mlp.experts.194.down_proj", + "model.layers.54.mlp.experts.194.gate_proj", + "model.layers.54.mlp.experts.194.up_proj", + "model.layers.54.mlp.experts.195.down_proj", + "model.layers.54.mlp.experts.195.gate_proj", + "model.layers.54.mlp.experts.195.up_proj", + "model.layers.54.mlp.experts.196.down_proj", + "model.layers.54.mlp.experts.196.gate_proj", + "model.layers.54.mlp.experts.196.up_proj", + "model.layers.54.mlp.experts.197.down_proj", + "model.layers.54.mlp.experts.197.gate_proj", + "model.layers.54.mlp.experts.197.up_proj", + "model.layers.54.mlp.experts.198.down_proj", + "model.layers.54.mlp.experts.198.gate_proj", + "model.layers.54.mlp.experts.198.up_proj", + "model.layers.54.mlp.experts.199.down_proj", + "model.layers.54.mlp.experts.199.gate_proj", + "model.layers.54.mlp.experts.199.up_proj", + "model.layers.54.mlp.experts.2.down_proj", + "model.layers.54.mlp.experts.2.gate_proj", + "model.layers.54.mlp.experts.2.up_proj", + "model.layers.54.mlp.experts.20.down_proj", + "model.layers.54.mlp.experts.20.gate_proj", + "model.layers.54.mlp.experts.20.up_proj", + "model.layers.54.mlp.experts.200.down_proj", + "model.layers.54.mlp.experts.200.gate_proj", + "model.layers.54.mlp.experts.200.up_proj", + "model.layers.54.mlp.experts.201.down_proj", + "model.layers.54.mlp.experts.201.gate_proj", + "model.layers.54.mlp.experts.201.up_proj", + "model.layers.54.mlp.experts.202.down_proj", + "model.layers.54.mlp.experts.202.gate_proj", + "model.layers.54.mlp.experts.202.up_proj", + "model.layers.54.mlp.experts.203.down_proj", + "model.layers.54.mlp.experts.203.gate_proj", + "model.layers.54.mlp.experts.203.up_proj", + "model.layers.54.mlp.experts.204.down_proj", + "model.layers.54.mlp.experts.204.gate_proj", + "model.layers.54.mlp.experts.204.up_proj", + "model.layers.54.mlp.experts.205.down_proj", + "model.layers.54.mlp.experts.205.gate_proj", + "model.layers.54.mlp.experts.205.up_proj", + "model.layers.54.mlp.experts.206.down_proj", + "model.layers.54.mlp.experts.206.gate_proj", + "model.layers.54.mlp.experts.206.up_proj", + "model.layers.54.mlp.experts.207.down_proj", + "model.layers.54.mlp.experts.207.gate_proj", + "model.layers.54.mlp.experts.207.up_proj", + "model.layers.54.mlp.experts.208.down_proj", + "model.layers.54.mlp.experts.208.gate_proj", + "model.layers.54.mlp.experts.208.up_proj", + "model.layers.54.mlp.experts.209.down_proj", + "model.layers.54.mlp.experts.209.gate_proj", + "model.layers.54.mlp.experts.209.up_proj", + "model.layers.54.mlp.experts.21.down_proj", + "model.layers.54.mlp.experts.21.gate_proj", + "model.layers.54.mlp.experts.21.up_proj", + "model.layers.54.mlp.experts.210.down_proj", + "model.layers.54.mlp.experts.210.gate_proj", + "model.layers.54.mlp.experts.210.up_proj", + "model.layers.54.mlp.experts.211.down_proj", + "model.layers.54.mlp.experts.211.gate_proj", + "model.layers.54.mlp.experts.211.up_proj", + "model.layers.54.mlp.experts.212.down_proj", + "model.layers.54.mlp.experts.212.gate_proj", + "model.layers.54.mlp.experts.212.up_proj", + "model.layers.54.mlp.experts.213.down_proj", + "model.layers.54.mlp.experts.213.gate_proj", + "model.layers.54.mlp.experts.213.up_proj", + "model.layers.54.mlp.experts.214.down_proj", + "model.layers.54.mlp.experts.214.gate_proj", + "model.layers.54.mlp.experts.214.up_proj", + "model.layers.54.mlp.experts.215.down_proj", + "model.layers.54.mlp.experts.215.gate_proj", + "model.layers.54.mlp.experts.215.up_proj", + "model.layers.54.mlp.experts.216.down_proj", + "model.layers.54.mlp.experts.216.gate_proj", + "model.layers.54.mlp.experts.216.up_proj", + "model.layers.54.mlp.experts.217.down_proj", + "model.layers.54.mlp.experts.217.gate_proj", + "model.layers.54.mlp.experts.217.up_proj", + "model.layers.54.mlp.experts.218.down_proj", + "model.layers.54.mlp.experts.218.gate_proj", + "model.layers.54.mlp.experts.218.up_proj", + "model.layers.54.mlp.experts.219.down_proj", + "model.layers.54.mlp.experts.219.gate_proj", + "model.layers.54.mlp.experts.219.up_proj", + "model.layers.54.mlp.experts.22.down_proj", + "model.layers.54.mlp.experts.22.gate_proj", + "model.layers.54.mlp.experts.22.up_proj", + "model.layers.54.mlp.experts.220.down_proj", + "model.layers.54.mlp.experts.220.gate_proj", + "model.layers.54.mlp.experts.220.up_proj", + "model.layers.54.mlp.experts.221.down_proj", + "model.layers.54.mlp.experts.221.gate_proj", + "model.layers.54.mlp.experts.221.up_proj", + "model.layers.54.mlp.experts.222.down_proj", + "model.layers.54.mlp.experts.222.gate_proj", + "model.layers.54.mlp.experts.222.up_proj", + "model.layers.54.mlp.experts.223.down_proj", + "model.layers.54.mlp.experts.223.gate_proj", + "model.layers.54.mlp.experts.223.up_proj", + "model.layers.54.mlp.experts.224.down_proj", + "model.layers.54.mlp.experts.224.gate_proj", + "model.layers.54.mlp.experts.224.up_proj", + "model.layers.54.mlp.experts.225.down_proj", + "model.layers.54.mlp.experts.225.gate_proj", + "model.layers.54.mlp.experts.225.up_proj", + "model.layers.54.mlp.experts.226.down_proj", + "model.layers.54.mlp.experts.226.gate_proj", + "model.layers.54.mlp.experts.226.up_proj", + "model.layers.54.mlp.experts.227.down_proj", + "model.layers.54.mlp.experts.227.gate_proj", + "model.layers.54.mlp.experts.227.up_proj", + "model.layers.54.mlp.experts.228.down_proj", + "model.layers.54.mlp.experts.228.gate_proj", + "model.layers.54.mlp.experts.228.up_proj", + "model.layers.54.mlp.experts.229.down_proj", + "model.layers.54.mlp.experts.229.gate_proj", + "model.layers.54.mlp.experts.229.up_proj", + "model.layers.54.mlp.experts.23.down_proj", + "model.layers.54.mlp.experts.23.gate_proj", + "model.layers.54.mlp.experts.23.up_proj", + "model.layers.54.mlp.experts.230.down_proj", + "model.layers.54.mlp.experts.230.gate_proj", + "model.layers.54.mlp.experts.230.up_proj", + "model.layers.54.mlp.experts.231.down_proj", + "model.layers.54.mlp.experts.231.gate_proj", + "model.layers.54.mlp.experts.231.up_proj", + "model.layers.54.mlp.experts.232.down_proj", + "model.layers.54.mlp.experts.232.gate_proj", + "model.layers.54.mlp.experts.232.up_proj", + "model.layers.54.mlp.experts.233.down_proj", + "model.layers.54.mlp.experts.233.gate_proj", + "model.layers.54.mlp.experts.233.up_proj", + "model.layers.54.mlp.experts.234.down_proj", + "model.layers.54.mlp.experts.234.gate_proj", + "model.layers.54.mlp.experts.234.up_proj", + "model.layers.54.mlp.experts.235.down_proj", + "model.layers.54.mlp.experts.235.gate_proj", + "model.layers.54.mlp.experts.235.up_proj", + "model.layers.54.mlp.experts.236.down_proj", + "model.layers.54.mlp.experts.236.gate_proj", + "model.layers.54.mlp.experts.236.up_proj", + "model.layers.54.mlp.experts.237.down_proj", + "model.layers.54.mlp.experts.237.gate_proj", + "model.layers.54.mlp.experts.237.up_proj", + "model.layers.54.mlp.experts.238.down_proj", + "model.layers.54.mlp.experts.238.gate_proj", + "model.layers.54.mlp.experts.238.up_proj", + "model.layers.54.mlp.experts.239.down_proj", + "model.layers.54.mlp.experts.239.gate_proj", + "model.layers.54.mlp.experts.239.up_proj", + "model.layers.54.mlp.experts.24.down_proj", + "model.layers.54.mlp.experts.24.gate_proj", + "model.layers.54.mlp.experts.24.up_proj", + "model.layers.54.mlp.experts.240.down_proj", + "model.layers.54.mlp.experts.240.gate_proj", + "model.layers.54.mlp.experts.240.up_proj", + "model.layers.54.mlp.experts.241.down_proj", + "model.layers.54.mlp.experts.241.gate_proj", + "model.layers.54.mlp.experts.241.up_proj", + "model.layers.54.mlp.experts.242.down_proj", + "model.layers.54.mlp.experts.242.gate_proj", + "model.layers.54.mlp.experts.242.up_proj", + "model.layers.54.mlp.experts.243.down_proj", + "model.layers.54.mlp.experts.243.gate_proj", + "model.layers.54.mlp.experts.243.up_proj", + "model.layers.54.mlp.experts.244.down_proj", + "model.layers.54.mlp.experts.244.gate_proj", + "model.layers.54.mlp.experts.244.up_proj", + "model.layers.54.mlp.experts.245.down_proj", + "model.layers.54.mlp.experts.245.gate_proj", + "model.layers.54.mlp.experts.245.up_proj", + "model.layers.54.mlp.experts.246.down_proj", + "model.layers.54.mlp.experts.246.gate_proj", + "model.layers.54.mlp.experts.246.up_proj", + "model.layers.54.mlp.experts.247.down_proj", + "model.layers.54.mlp.experts.247.gate_proj", + "model.layers.54.mlp.experts.247.up_proj", + "model.layers.54.mlp.experts.248.down_proj", + "model.layers.54.mlp.experts.248.gate_proj", + "model.layers.54.mlp.experts.248.up_proj", + "model.layers.54.mlp.experts.249.down_proj", + "model.layers.54.mlp.experts.249.gate_proj", + "model.layers.54.mlp.experts.249.up_proj", + "model.layers.54.mlp.experts.25.down_proj", + "model.layers.54.mlp.experts.25.gate_proj", + "model.layers.54.mlp.experts.25.up_proj", + "model.layers.54.mlp.experts.250.down_proj", + "model.layers.54.mlp.experts.250.gate_proj", + "model.layers.54.mlp.experts.250.up_proj", + "model.layers.54.mlp.experts.251.down_proj", + "model.layers.54.mlp.experts.251.gate_proj", + "model.layers.54.mlp.experts.251.up_proj", + "model.layers.54.mlp.experts.252.down_proj", + "model.layers.54.mlp.experts.252.gate_proj", + "model.layers.54.mlp.experts.252.up_proj", + "model.layers.54.mlp.experts.253.down_proj", + "model.layers.54.mlp.experts.253.gate_proj", + "model.layers.54.mlp.experts.253.up_proj", + "model.layers.54.mlp.experts.254.down_proj", + "model.layers.54.mlp.experts.254.gate_proj", + "model.layers.54.mlp.experts.254.up_proj", + "model.layers.54.mlp.experts.255.down_proj", + "model.layers.54.mlp.experts.255.gate_proj", + "model.layers.54.mlp.experts.255.up_proj", + "model.layers.54.mlp.experts.26.down_proj", + "model.layers.54.mlp.experts.26.gate_proj", + "model.layers.54.mlp.experts.26.up_proj", + "model.layers.54.mlp.experts.27.down_proj", + "model.layers.54.mlp.experts.27.gate_proj", + "model.layers.54.mlp.experts.27.up_proj", + "model.layers.54.mlp.experts.28.down_proj", + "model.layers.54.mlp.experts.28.gate_proj", + "model.layers.54.mlp.experts.28.up_proj", + "model.layers.54.mlp.experts.29.down_proj", + "model.layers.54.mlp.experts.29.gate_proj", + "model.layers.54.mlp.experts.29.up_proj", + "model.layers.54.mlp.experts.3.down_proj", + "model.layers.54.mlp.experts.3.gate_proj", + "model.layers.54.mlp.experts.3.up_proj", + "model.layers.54.mlp.experts.30.down_proj", + "model.layers.54.mlp.experts.30.gate_proj", + "model.layers.54.mlp.experts.30.up_proj", + "model.layers.54.mlp.experts.31.down_proj", + "model.layers.54.mlp.experts.31.gate_proj", + "model.layers.54.mlp.experts.31.up_proj", + "model.layers.54.mlp.experts.32.down_proj", + "model.layers.54.mlp.experts.32.gate_proj", + "model.layers.54.mlp.experts.32.up_proj", + "model.layers.54.mlp.experts.33.down_proj", + "model.layers.54.mlp.experts.33.gate_proj", + "model.layers.54.mlp.experts.33.up_proj", + "model.layers.54.mlp.experts.34.down_proj", + "model.layers.54.mlp.experts.34.gate_proj", + "model.layers.54.mlp.experts.34.up_proj", + "model.layers.54.mlp.experts.35.down_proj", + "model.layers.54.mlp.experts.35.gate_proj", + "model.layers.54.mlp.experts.35.up_proj", + "model.layers.54.mlp.experts.36.down_proj", + "model.layers.54.mlp.experts.36.gate_proj", + "model.layers.54.mlp.experts.36.up_proj", + "model.layers.54.mlp.experts.37.down_proj", + "model.layers.54.mlp.experts.37.gate_proj", + "model.layers.54.mlp.experts.37.up_proj", + "model.layers.54.mlp.experts.38.down_proj", + "model.layers.54.mlp.experts.38.gate_proj", + "model.layers.54.mlp.experts.38.up_proj", + "model.layers.54.mlp.experts.39.down_proj", + "model.layers.54.mlp.experts.39.gate_proj", + "model.layers.54.mlp.experts.39.up_proj", + "model.layers.54.mlp.experts.4.down_proj", + "model.layers.54.mlp.experts.4.gate_proj", + "model.layers.54.mlp.experts.4.up_proj", + "model.layers.54.mlp.experts.40.down_proj", + "model.layers.54.mlp.experts.40.gate_proj", + "model.layers.54.mlp.experts.40.up_proj", + "model.layers.54.mlp.experts.41.down_proj", + "model.layers.54.mlp.experts.41.gate_proj", + "model.layers.54.mlp.experts.41.up_proj", + "model.layers.54.mlp.experts.42.down_proj", + "model.layers.54.mlp.experts.42.gate_proj", + "model.layers.54.mlp.experts.42.up_proj", + "model.layers.54.mlp.experts.43.down_proj", + "model.layers.54.mlp.experts.43.gate_proj", + "model.layers.54.mlp.experts.43.up_proj", + "model.layers.54.mlp.experts.44.down_proj", + "model.layers.54.mlp.experts.44.gate_proj", + "model.layers.54.mlp.experts.44.up_proj", + "model.layers.54.mlp.experts.45.down_proj", + "model.layers.54.mlp.experts.45.gate_proj", + "model.layers.54.mlp.experts.45.up_proj", + "model.layers.54.mlp.experts.46.down_proj", + "model.layers.54.mlp.experts.46.gate_proj", + "model.layers.54.mlp.experts.46.up_proj", + "model.layers.54.mlp.experts.47.down_proj", + "model.layers.54.mlp.experts.47.gate_proj", + "model.layers.54.mlp.experts.47.up_proj", + "model.layers.54.mlp.experts.48.down_proj", + "model.layers.54.mlp.experts.48.gate_proj", + "model.layers.54.mlp.experts.48.up_proj", + "model.layers.54.mlp.experts.49.down_proj", + "model.layers.54.mlp.experts.49.gate_proj", + "model.layers.54.mlp.experts.49.up_proj", + "model.layers.54.mlp.experts.5.down_proj", + "model.layers.54.mlp.experts.5.gate_proj", + "model.layers.54.mlp.experts.5.up_proj", + "model.layers.54.mlp.experts.50.down_proj", + "model.layers.54.mlp.experts.50.gate_proj", + "model.layers.54.mlp.experts.50.up_proj", + "model.layers.54.mlp.experts.51.down_proj", + "model.layers.54.mlp.experts.51.gate_proj", + "model.layers.54.mlp.experts.51.up_proj", + "model.layers.54.mlp.experts.52.down_proj", + "model.layers.54.mlp.experts.52.gate_proj", + "model.layers.54.mlp.experts.52.up_proj", + "model.layers.54.mlp.experts.53.down_proj", + "model.layers.54.mlp.experts.53.gate_proj", + "model.layers.54.mlp.experts.53.up_proj", + "model.layers.54.mlp.experts.54.down_proj", + "model.layers.54.mlp.experts.54.gate_proj", + "model.layers.54.mlp.experts.54.up_proj", + "model.layers.54.mlp.experts.55.down_proj", + "model.layers.54.mlp.experts.55.gate_proj", + "model.layers.54.mlp.experts.55.up_proj", + "model.layers.54.mlp.experts.56.down_proj", + "model.layers.54.mlp.experts.56.gate_proj", + "model.layers.54.mlp.experts.56.up_proj", + "model.layers.54.mlp.experts.57.down_proj", + "model.layers.54.mlp.experts.57.gate_proj", + "model.layers.54.mlp.experts.57.up_proj", + "model.layers.54.mlp.experts.58.down_proj", + "model.layers.54.mlp.experts.58.gate_proj", + "model.layers.54.mlp.experts.58.up_proj", + "model.layers.54.mlp.experts.59.down_proj", + "model.layers.54.mlp.experts.59.gate_proj", + "model.layers.54.mlp.experts.59.up_proj", + "model.layers.54.mlp.experts.6.down_proj", + "model.layers.54.mlp.experts.6.gate_proj", + "model.layers.54.mlp.experts.6.up_proj", + "model.layers.54.mlp.experts.60.down_proj", + "model.layers.54.mlp.experts.60.gate_proj", + "model.layers.54.mlp.experts.60.up_proj", + "model.layers.54.mlp.experts.61.down_proj", + "model.layers.54.mlp.experts.61.gate_proj", + "model.layers.54.mlp.experts.61.up_proj", + "model.layers.54.mlp.experts.62.down_proj", + "model.layers.54.mlp.experts.62.gate_proj", + "model.layers.54.mlp.experts.62.up_proj", + "model.layers.54.mlp.experts.63.down_proj", + "model.layers.54.mlp.experts.63.gate_proj", + "model.layers.54.mlp.experts.63.up_proj", + "model.layers.54.mlp.experts.64.down_proj", + "model.layers.54.mlp.experts.64.gate_proj", + "model.layers.54.mlp.experts.64.up_proj", + "model.layers.54.mlp.experts.65.down_proj", + "model.layers.54.mlp.experts.65.gate_proj", + "model.layers.54.mlp.experts.65.up_proj", + "model.layers.54.mlp.experts.66.down_proj", + "model.layers.54.mlp.experts.66.gate_proj", + "model.layers.54.mlp.experts.66.up_proj", + "model.layers.54.mlp.experts.67.down_proj", + "model.layers.54.mlp.experts.67.gate_proj", + "model.layers.54.mlp.experts.67.up_proj", + "model.layers.54.mlp.experts.68.down_proj", + "model.layers.54.mlp.experts.68.gate_proj", + "model.layers.54.mlp.experts.68.up_proj", + "model.layers.54.mlp.experts.69.down_proj", + "model.layers.54.mlp.experts.69.gate_proj", + "model.layers.54.mlp.experts.69.up_proj", + "model.layers.54.mlp.experts.7.down_proj", + "model.layers.54.mlp.experts.7.gate_proj", + "model.layers.54.mlp.experts.7.up_proj", + "model.layers.54.mlp.experts.70.down_proj", + "model.layers.54.mlp.experts.70.gate_proj", + "model.layers.54.mlp.experts.70.up_proj", + "model.layers.54.mlp.experts.71.down_proj", + "model.layers.54.mlp.experts.71.gate_proj", + "model.layers.54.mlp.experts.71.up_proj", + "model.layers.54.mlp.experts.72.down_proj", + "model.layers.54.mlp.experts.72.gate_proj", + "model.layers.54.mlp.experts.72.up_proj", + "model.layers.54.mlp.experts.73.down_proj", + "model.layers.54.mlp.experts.73.gate_proj", + "model.layers.54.mlp.experts.73.up_proj", + "model.layers.54.mlp.experts.74.down_proj", + "model.layers.54.mlp.experts.74.gate_proj", + "model.layers.54.mlp.experts.74.up_proj", + "model.layers.54.mlp.experts.75.down_proj", + "model.layers.54.mlp.experts.75.gate_proj", + "model.layers.54.mlp.experts.75.up_proj", + "model.layers.54.mlp.experts.76.down_proj", + "model.layers.54.mlp.experts.76.gate_proj", + "model.layers.54.mlp.experts.76.up_proj", + "model.layers.54.mlp.experts.77.down_proj", + "model.layers.54.mlp.experts.77.gate_proj", + "model.layers.54.mlp.experts.77.up_proj", + "model.layers.54.mlp.experts.78.down_proj", + "model.layers.54.mlp.experts.78.gate_proj", + "model.layers.54.mlp.experts.78.up_proj", + "model.layers.54.mlp.experts.79.down_proj", + "model.layers.54.mlp.experts.79.gate_proj", + "model.layers.54.mlp.experts.79.up_proj", + "model.layers.54.mlp.experts.8.down_proj", + "model.layers.54.mlp.experts.8.gate_proj", + "model.layers.54.mlp.experts.8.up_proj", + "model.layers.54.mlp.experts.80.down_proj", + "model.layers.54.mlp.experts.80.gate_proj", + "model.layers.54.mlp.experts.80.up_proj", + "model.layers.54.mlp.experts.81.down_proj", + "model.layers.54.mlp.experts.81.gate_proj", + "model.layers.54.mlp.experts.81.up_proj", + "model.layers.54.mlp.experts.82.down_proj", + "model.layers.54.mlp.experts.82.gate_proj", + "model.layers.54.mlp.experts.82.up_proj", + "model.layers.54.mlp.experts.83.down_proj", + "model.layers.54.mlp.experts.83.gate_proj", + "model.layers.54.mlp.experts.83.up_proj", + "model.layers.54.mlp.experts.84.down_proj", + "model.layers.54.mlp.experts.84.gate_proj", + "model.layers.54.mlp.experts.84.up_proj", + "model.layers.54.mlp.experts.85.down_proj", + "model.layers.54.mlp.experts.85.gate_proj", + "model.layers.54.mlp.experts.85.up_proj", + "model.layers.54.mlp.experts.86.down_proj", + "model.layers.54.mlp.experts.86.gate_proj", + "model.layers.54.mlp.experts.86.up_proj", + "model.layers.54.mlp.experts.87.down_proj", + "model.layers.54.mlp.experts.87.gate_proj", + "model.layers.54.mlp.experts.87.up_proj", + "model.layers.54.mlp.experts.88.down_proj", + "model.layers.54.mlp.experts.88.gate_proj", + "model.layers.54.mlp.experts.88.up_proj", + "model.layers.54.mlp.experts.89.down_proj", + "model.layers.54.mlp.experts.89.gate_proj", + "model.layers.54.mlp.experts.89.up_proj", + "model.layers.54.mlp.experts.9.down_proj", + "model.layers.54.mlp.experts.9.gate_proj", + "model.layers.54.mlp.experts.9.up_proj", + "model.layers.54.mlp.experts.90.down_proj", + "model.layers.54.mlp.experts.90.gate_proj", + "model.layers.54.mlp.experts.90.up_proj", + "model.layers.54.mlp.experts.91.down_proj", + "model.layers.54.mlp.experts.91.gate_proj", + "model.layers.54.mlp.experts.91.up_proj", + "model.layers.54.mlp.experts.92.down_proj", + "model.layers.54.mlp.experts.92.gate_proj", + "model.layers.54.mlp.experts.92.up_proj", + "model.layers.54.mlp.experts.93.down_proj", + "model.layers.54.mlp.experts.93.gate_proj", + "model.layers.54.mlp.experts.93.up_proj", + "model.layers.54.mlp.experts.94.down_proj", + "model.layers.54.mlp.experts.94.gate_proj", + "model.layers.54.mlp.experts.94.up_proj", + "model.layers.54.mlp.experts.95.down_proj", + "model.layers.54.mlp.experts.95.gate_proj", + "model.layers.54.mlp.experts.95.up_proj", + "model.layers.54.mlp.experts.96.down_proj", + "model.layers.54.mlp.experts.96.gate_proj", + "model.layers.54.mlp.experts.96.up_proj", + "model.layers.54.mlp.experts.97.down_proj", + "model.layers.54.mlp.experts.97.gate_proj", + "model.layers.54.mlp.experts.97.up_proj", + "model.layers.54.mlp.experts.98.down_proj", + "model.layers.54.mlp.experts.98.gate_proj", + "model.layers.54.mlp.experts.98.up_proj", + "model.layers.54.mlp.experts.99.down_proj", + "model.layers.54.mlp.experts.99.gate_proj", + "model.layers.54.mlp.experts.99.up_proj", + "model.layers.55.mlp.experts.0.down_proj", + "model.layers.55.mlp.experts.0.gate_proj", + "model.layers.55.mlp.experts.0.up_proj", + "model.layers.55.mlp.experts.1.down_proj", + "model.layers.55.mlp.experts.1.gate_proj", + "model.layers.55.mlp.experts.1.up_proj", + "model.layers.55.mlp.experts.10.down_proj", + "model.layers.55.mlp.experts.10.gate_proj", + "model.layers.55.mlp.experts.10.up_proj", + "model.layers.55.mlp.experts.100.down_proj", + "model.layers.55.mlp.experts.100.gate_proj", + "model.layers.55.mlp.experts.100.up_proj", + "model.layers.55.mlp.experts.101.down_proj", + "model.layers.55.mlp.experts.101.gate_proj", + "model.layers.55.mlp.experts.101.up_proj", + "model.layers.55.mlp.experts.102.down_proj", + "model.layers.55.mlp.experts.102.gate_proj", + "model.layers.55.mlp.experts.102.up_proj", + "model.layers.55.mlp.experts.103.down_proj", + "model.layers.55.mlp.experts.103.gate_proj", + "model.layers.55.mlp.experts.103.up_proj", + "model.layers.55.mlp.experts.104.down_proj", + "model.layers.55.mlp.experts.104.gate_proj", + "model.layers.55.mlp.experts.104.up_proj", + "model.layers.55.mlp.experts.105.down_proj", + "model.layers.55.mlp.experts.105.gate_proj", + "model.layers.55.mlp.experts.105.up_proj", + "model.layers.55.mlp.experts.106.down_proj", + "model.layers.55.mlp.experts.106.gate_proj", + "model.layers.55.mlp.experts.106.up_proj", + "model.layers.55.mlp.experts.107.down_proj", + "model.layers.55.mlp.experts.107.gate_proj", + "model.layers.55.mlp.experts.107.up_proj", + "model.layers.55.mlp.experts.108.down_proj", + "model.layers.55.mlp.experts.108.gate_proj", + "model.layers.55.mlp.experts.108.up_proj", + "model.layers.55.mlp.experts.109.down_proj", + "model.layers.55.mlp.experts.109.gate_proj", + "model.layers.55.mlp.experts.109.up_proj", + "model.layers.55.mlp.experts.11.down_proj", + "model.layers.55.mlp.experts.11.gate_proj", + "model.layers.55.mlp.experts.11.up_proj", + "model.layers.55.mlp.experts.110.down_proj", + "model.layers.55.mlp.experts.110.gate_proj", + "model.layers.55.mlp.experts.110.up_proj", + "model.layers.55.mlp.experts.111.down_proj", + "model.layers.55.mlp.experts.111.gate_proj", + "model.layers.55.mlp.experts.111.up_proj", + "model.layers.55.mlp.experts.112.down_proj", + "model.layers.55.mlp.experts.112.gate_proj", + "model.layers.55.mlp.experts.112.up_proj", + "model.layers.55.mlp.experts.113.down_proj", + "model.layers.55.mlp.experts.113.gate_proj", + "model.layers.55.mlp.experts.113.up_proj", + "model.layers.55.mlp.experts.114.down_proj", + "model.layers.55.mlp.experts.114.gate_proj", + "model.layers.55.mlp.experts.114.up_proj", + "model.layers.55.mlp.experts.115.down_proj", + "model.layers.55.mlp.experts.115.gate_proj", + "model.layers.55.mlp.experts.115.up_proj", + "model.layers.55.mlp.experts.116.down_proj", + "model.layers.55.mlp.experts.116.gate_proj", + "model.layers.55.mlp.experts.116.up_proj", + "model.layers.55.mlp.experts.117.down_proj", + "model.layers.55.mlp.experts.117.gate_proj", + "model.layers.55.mlp.experts.117.up_proj", + "model.layers.55.mlp.experts.118.down_proj", + "model.layers.55.mlp.experts.118.gate_proj", + "model.layers.55.mlp.experts.118.up_proj", + "model.layers.55.mlp.experts.119.down_proj", + "model.layers.55.mlp.experts.119.gate_proj", + "model.layers.55.mlp.experts.119.up_proj", + "model.layers.55.mlp.experts.12.down_proj", + "model.layers.55.mlp.experts.12.gate_proj", + "model.layers.55.mlp.experts.12.up_proj", + "model.layers.55.mlp.experts.120.down_proj", + "model.layers.55.mlp.experts.120.gate_proj", + "model.layers.55.mlp.experts.120.up_proj", + "model.layers.55.mlp.experts.121.down_proj", + "model.layers.55.mlp.experts.121.gate_proj", + "model.layers.55.mlp.experts.121.up_proj", + "model.layers.55.mlp.experts.122.down_proj", + "model.layers.55.mlp.experts.122.gate_proj", + "model.layers.55.mlp.experts.122.up_proj", + "model.layers.55.mlp.experts.123.down_proj", + "model.layers.55.mlp.experts.123.gate_proj", + "model.layers.55.mlp.experts.123.up_proj", + "model.layers.55.mlp.experts.124.down_proj", + "model.layers.55.mlp.experts.124.gate_proj", + "model.layers.55.mlp.experts.124.up_proj", + "model.layers.55.mlp.experts.125.down_proj", + "model.layers.55.mlp.experts.125.gate_proj", + "model.layers.55.mlp.experts.125.up_proj", + "model.layers.55.mlp.experts.126.down_proj", + "model.layers.55.mlp.experts.126.gate_proj", + "model.layers.55.mlp.experts.126.up_proj", + "model.layers.55.mlp.experts.127.down_proj", + "model.layers.55.mlp.experts.127.gate_proj", + "model.layers.55.mlp.experts.127.up_proj", + "model.layers.55.mlp.experts.128.down_proj", + "model.layers.55.mlp.experts.128.gate_proj", + "model.layers.55.mlp.experts.128.up_proj", + "model.layers.55.mlp.experts.129.down_proj", + "model.layers.55.mlp.experts.129.gate_proj", + "model.layers.55.mlp.experts.129.up_proj", + "model.layers.55.mlp.experts.13.down_proj", + "model.layers.55.mlp.experts.13.gate_proj", + "model.layers.55.mlp.experts.13.up_proj", + "model.layers.55.mlp.experts.130.down_proj", + "model.layers.55.mlp.experts.130.gate_proj", + "model.layers.55.mlp.experts.130.up_proj", + "model.layers.55.mlp.experts.131.down_proj", + "model.layers.55.mlp.experts.131.gate_proj", + "model.layers.55.mlp.experts.131.up_proj", + "model.layers.55.mlp.experts.132.down_proj", + "model.layers.55.mlp.experts.132.gate_proj", + "model.layers.55.mlp.experts.132.up_proj", + "model.layers.55.mlp.experts.133.down_proj", + "model.layers.55.mlp.experts.133.gate_proj", + "model.layers.55.mlp.experts.133.up_proj", + "model.layers.55.mlp.experts.134.down_proj", + "model.layers.55.mlp.experts.134.gate_proj", + "model.layers.55.mlp.experts.134.up_proj", + "model.layers.55.mlp.experts.135.down_proj", + "model.layers.55.mlp.experts.135.gate_proj", + "model.layers.55.mlp.experts.135.up_proj", + "model.layers.55.mlp.experts.136.down_proj", + "model.layers.55.mlp.experts.136.gate_proj", + "model.layers.55.mlp.experts.136.up_proj", + "model.layers.55.mlp.experts.137.down_proj", + "model.layers.55.mlp.experts.137.gate_proj", + "model.layers.55.mlp.experts.137.up_proj", + "model.layers.55.mlp.experts.138.down_proj", + "model.layers.55.mlp.experts.138.gate_proj", + "model.layers.55.mlp.experts.138.up_proj", + "model.layers.55.mlp.experts.139.down_proj", + "model.layers.55.mlp.experts.139.gate_proj", + "model.layers.55.mlp.experts.139.up_proj", + "model.layers.55.mlp.experts.14.down_proj", + "model.layers.55.mlp.experts.14.gate_proj", + "model.layers.55.mlp.experts.14.up_proj", + "model.layers.55.mlp.experts.140.down_proj", + "model.layers.55.mlp.experts.140.gate_proj", + "model.layers.55.mlp.experts.140.up_proj", + "model.layers.55.mlp.experts.141.down_proj", + "model.layers.55.mlp.experts.141.gate_proj", + "model.layers.55.mlp.experts.141.up_proj", + "model.layers.55.mlp.experts.142.down_proj", + "model.layers.55.mlp.experts.142.gate_proj", + "model.layers.55.mlp.experts.142.up_proj", + "model.layers.55.mlp.experts.143.down_proj", + "model.layers.55.mlp.experts.143.gate_proj", + "model.layers.55.mlp.experts.143.up_proj", + "model.layers.55.mlp.experts.144.down_proj", + "model.layers.55.mlp.experts.144.gate_proj", + "model.layers.55.mlp.experts.144.up_proj", + "model.layers.55.mlp.experts.145.down_proj", + "model.layers.55.mlp.experts.145.gate_proj", + "model.layers.55.mlp.experts.145.up_proj", + "model.layers.55.mlp.experts.146.down_proj", + "model.layers.55.mlp.experts.146.gate_proj", + "model.layers.55.mlp.experts.146.up_proj", + "model.layers.55.mlp.experts.147.down_proj", + "model.layers.55.mlp.experts.147.gate_proj", + "model.layers.55.mlp.experts.147.up_proj", + "model.layers.55.mlp.experts.148.down_proj", + "model.layers.55.mlp.experts.148.gate_proj", + "model.layers.55.mlp.experts.148.up_proj", + "model.layers.55.mlp.experts.149.down_proj", + "model.layers.55.mlp.experts.149.gate_proj", + "model.layers.55.mlp.experts.149.up_proj", + "model.layers.55.mlp.experts.15.down_proj", + "model.layers.55.mlp.experts.15.gate_proj", + "model.layers.55.mlp.experts.15.up_proj", + "model.layers.55.mlp.experts.150.down_proj", + "model.layers.55.mlp.experts.150.gate_proj", + "model.layers.55.mlp.experts.150.up_proj", + "model.layers.55.mlp.experts.151.down_proj", + "model.layers.55.mlp.experts.151.gate_proj", + "model.layers.55.mlp.experts.151.up_proj", + "model.layers.55.mlp.experts.152.down_proj", + "model.layers.55.mlp.experts.152.gate_proj", + "model.layers.55.mlp.experts.152.up_proj", + "model.layers.55.mlp.experts.153.down_proj", + "model.layers.55.mlp.experts.153.gate_proj", + "model.layers.55.mlp.experts.153.up_proj", + "model.layers.55.mlp.experts.154.down_proj", + "model.layers.55.mlp.experts.154.gate_proj", + "model.layers.55.mlp.experts.154.up_proj", + "model.layers.55.mlp.experts.155.down_proj", + "model.layers.55.mlp.experts.155.gate_proj", + "model.layers.55.mlp.experts.155.up_proj", + "model.layers.55.mlp.experts.156.down_proj", + "model.layers.55.mlp.experts.156.gate_proj", + "model.layers.55.mlp.experts.156.up_proj", + "model.layers.55.mlp.experts.157.down_proj", + "model.layers.55.mlp.experts.157.gate_proj", + "model.layers.55.mlp.experts.157.up_proj", + "model.layers.55.mlp.experts.158.down_proj", + "model.layers.55.mlp.experts.158.gate_proj", + "model.layers.55.mlp.experts.158.up_proj", + "model.layers.55.mlp.experts.159.down_proj", + "model.layers.55.mlp.experts.159.gate_proj", + "model.layers.55.mlp.experts.159.up_proj", + "model.layers.55.mlp.experts.16.down_proj", + "model.layers.55.mlp.experts.16.gate_proj", + "model.layers.55.mlp.experts.16.up_proj", + "model.layers.55.mlp.experts.160.down_proj", + "model.layers.55.mlp.experts.160.gate_proj", + "model.layers.55.mlp.experts.160.up_proj", + "model.layers.55.mlp.experts.161.down_proj", + "model.layers.55.mlp.experts.161.gate_proj", + "model.layers.55.mlp.experts.161.up_proj", + "model.layers.55.mlp.experts.162.down_proj", + "model.layers.55.mlp.experts.162.gate_proj", + "model.layers.55.mlp.experts.162.up_proj", + "model.layers.55.mlp.experts.163.down_proj", + "model.layers.55.mlp.experts.163.gate_proj", + "model.layers.55.mlp.experts.163.up_proj", + "model.layers.55.mlp.experts.164.down_proj", + "model.layers.55.mlp.experts.164.gate_proj", + "model.layers.55.mlp.experts.164.up_proj", + "model.layers.55.mlp.experts.165.down_proj", + "model.layers.55.mlp.experts.165.gate_proj", + "model.layers.55.mlp.experts.165.up_proj", + "model.layers.55.mlp.experts.166.down_proj", + "model.layers.55.mlp.experts.166.gate_proj", + "model.layers.55.mlp.experts.166.up_proj", + "model.layers.55.mlp.experts.167.down_proj", + "model.layers.55.mlp.experts.167.gate_proj", + "model.layers.55.mlp.experts.167.up_proj", + "model.layers.55.mlp.experts.168.down_proj", + "model.layers.55.mlp.experts.168.gate_proj", + "model.layers.55.mlp.experts.168.up_proj", + "model.layers.55.mlp.experts.169.down_proj", + "model.layers.55.mlp.experts.169.gate_proj", + "model.layers.55.mlp.experts.169.up_proj", + "model.layers.55.mlp.experts.17.down_proj", + "model.layers.55.mlp.experts.17.gate_proj", + "model.layers.55.mlp.experts.17.up_proj", + "model.layers.55.mlp.experts.170.down_proj", + "model.layers.55.mlp.experts.170.gate_proj", + "model.layers.55.mlp.experts.170.up_proj", + "model.layers.55.mlp.experts.171.down_proj", + "model.layers.55.mlp.experts.171.gate_proj", + "model.layers.55.mlp.experts.171.up_proj", + "model.layers.55.mlp.experts.172.down_proj", + "model.layers.55.mlp.experts.172.gate_proj", + "model.layers.55.mlp.experts.172.up_proj", + "model.layers.55.mlp.experts.173.down_proj", + "model.layers.55.mlp.experts.173.gate_proj", + "model.layers.55.mlp.experts.173.up_proj", + "model.layers.55.mlp.experts.174.down_proj", + "model.layers.55.mlp.experts.174.gate_proj", + "model.layers.55.mlp.experts.174.up_proj", + "model.layers.55.mlp.experts.175.down_proj", + "model.layers.55.mlp.experts.175.gate_proj", + "model.layers.55.mlp.experts.175.up_proj", + "model.layers.55.mlp.experts.176.down_proj", + "model.layers.55.mlp.experts.176.gate_proj", + "model.layers.55.mlp.experts.176.up_proj", + "model.layers.55.mlp.experts.177.down_proj", + "model.layers.55.mlp.experts.177.gate_proj", + "model.layers.55.mlp.experts.177.up_proj", + "model.layers.55.mlp.experts.178.down_proj", + "model.layers.55.mlp.experts.178.gate_proj", + "model.layers.55.mlp.experts.178.up_proj", + "model.layers.55.mlp.experts.179.down_proj", + "model.layers.55.mlp.experts.179.gate_proj", + "model.layers.55.mlp.experts.179.up_proj", + "model.layers.55.mlp.experts.18.down_proj", + "model.layers.55.mlp.experts.18.gate_proj", + "model.layers.55.mlp.experts.18.up_proj", + "model.layers.55.mlp.experts.180.down_proj", + "model.layers.55.mlp.experts.180.gate_proj", + "model.layers.55.mlp.experts.180.up_proj", + "model.layers.55.mlp.experts.181.down_proj", + "model.layers.55.mlp.experts.181.gate_proj", + "model.layers.55.mlp.experts.181.up_proj", + "model.layers.55.mlp.experts.182.down_proj", + "model.layers.55.mlp.experts.182.gate_proj", + "model.layers.55.mlp.experts.182.up_proj", + "model.layers.55.mlp.experts.183.down_proj", + "model.layers.55.mlp.experts.183.gate_proj", + "model.layers.55.mlp.experts.183.up_proj", + "model.layers.55.mlp.experts.184.down_proj", + "model.layers.55.mlp.experts.184.gate_proj", + "model.layers.55.mlp.experts.184.up_proj", + "model.layers.55.mlp.experts.185.down_proj", + "model.layers.55.mlp.experts.185.gate_proj", + "model.layers.55.mlp.experts.185.up_proj", + "model.layers.55.mlp.experts.186.down_proj", + "model.layers.55.mlp.experts.186.gate_proj", + "model.layers.55.mlp.experts.186.up_proj", + "model.layers.55.mlp.experts.187.down_proj", + "model.layers.55.mlp.experts.187.gate_proj", + "model.layers.55.mlp.experts.187.up_proj", + "model.layers.55.mlp.experts.188.down_proj", + "model.layers.55.mlp.experts.188.gate_proj", + "model.layers.55.mlp.experts.188.up_proj", + "model.layers.55.mlp.experts.189.down_proj", + "model.layers.55.mlp.experts.189.gate_proj", + "model.layers.55.mlp.experts.189.up_proj", + "model.layers.55.mlp.experts.19.down_proj", + "model.layers.55.mlp.experts.19.gate_proj", + "model.layers.55.mlp.experts.19.up_proj", + "model.layers.55.mlp.experts.190.down_proj", + "model.layers.55.mlp.experts.190.gate_proj", + "model.layers.55.mlp.experts.190.up_proj", + "model.layers.55.mlp.experts.191.down_proj", + "model.layers.55.mlp.experts.191.gate_proj", + "model.layers.55.mlp.experts.191.up_proj", + "model.layers.55.mlp.experts.192.down_proj", + "model.layers.55.mlp.experts.192.gate_proj", + "model.layers.55.mlp.experts.192.up_proj", + "model.layers.55.mlp.experts.193.down_proj", + "model.layers.55.mlp.experts.193.gate_proj", + "model.layers.55.mlp.experts.193.up_proj", + "model.layers.55.mlp.experts.194.down_proj", + "model.layers.55.mlp.experts.194.gate_proj", + "model.layers.55.mlp.experts.194.up_proj", + "model.layers.55.mlp.experts.195.down_proj", + "model.layers.55.mlp.experts.195.gate_proj", + "model.layers.55.mlp.experts.195.up_proj", + "model.layers.55.mlp.experts.196.down_proj", + "model.layers.55.mlp.experts.196.gate_proj", + "model.layers.55.mlp.experts.196.up_proj", + "model.layers.55.mlp.experts.197.down_proj", + "model.layers.55.mlp.experts.197.gate_proj", + "model.layers.55.mlp.experts.197.up_proj", + "model.layers.55.mlp.experts.198.down_proj", + "model.layers.55.mlp.experts.198.gate_proj", + "model.layers.55.mlp.experts.198.up_proj", + "model.layers.55.mlp.experts.199.down_proj", + "model.layers.55.mlp.experts.199.gate_proj", + "model.layers.55.mlp.experts.199.up_proj", + "model.layers.55.mlp.experts.2.down_proj", + "model.layers.55.mlp.experts.2.gate_proj", + "model.layers.55.mlp.experts.2.up_proj", + "model.layers.55.mlp.experts.20.down_proj", + "model.layers.55.mlp.experts.20.gate_proj", + "model.layers.55.mlp.experts.20.up_proj", + "model.layers.55.mlp.experts.200.down_proj", + "model.layers.55.mlp.experts.200.gate_proj", + "model.layers.55.mlp.experts.200.up_proj", + "model.layers.55.mlp.experts.201.down_proj", + "model.layers.55.mlp.experts.201.gate_proj", + "model.layers.55.mlp.experts.201.up_proj", + "model.layers.55.mlp.experts.202.down_proj", + "model.layers.55.mlp.experts.202.gate_proj", + "model.layers.55.mlp.experts.202.up_proj", + "model.layers.55.mlp.experts.203.down_proj", + "model.layers.55.mlp.experts.203.gate_proj", + "model.layers.55.mlp.experts.203.up_proj", + "model.layers.55.mlp.experts.204.down_proj", + "model.layers.55.mlp.experts.204.gate_proj", + "model.layers.55.mlp.experts.204.up_proj", + "model.layers.55.mlp.experts.205.down_proj", + "model.layers.55.mlp.experts.205.gate_proj", + "model.layers.55.mlp.experts.205.up_proj", + "model.layers.55.mlp.experts.206.down_proj", + "model.layers.55.mlp.experts.206.gate_proj", + "model.layers.55.mlp.experts.206.up_proj", + "model.layers.55.mlp.experts.207.down_proj", + "model.layers.55.mlp.experts.207.gate_proj", + "model.layers.55.mlp.experts.207.up_proj", + "model.layers.55.mlp.experts.208.down_proj", + "model.layers.55.mlp.experts.208.gate_proj", + "model.layers.55.mlp.experts.208.up_proj", + "model.layers.55.mlp.experts.209.down_proj", + "model.layers.55.mlp.experts.209.gate_proj", + "model.layers.55.mlp.experts.209.up_proj", + "model.layers.55.mlp.experts.21.down_proj", + "model.layers.55.mlp.experts.21.gate_proj", + "model.layers.55.mlp.experts.21.up_proj", + "model.layers.55.mlp.experts.210.down_proj", + "model.layers.55.mlp.experts.210.gate_proj", + "model.layers.55.mlp.experts.210.up_proj", + "model.layers.55.mlp.experts.211.down_proj", + "model.layers.55.mlp.experts.211.gate_proj", + "model.layers.55.mlp.experts.211.up_proj", + "model.layers.55.mlp.experts.212.down_proj", + "model.layers.55.mlp.experts.212.gate_proj", + "model.layers.55.mlp.experts.212.up_proj", + "model.layers.55.mlp.experts.213.down_proj", + "model.layers.55.mlp.experts.213.gate_proj", + "model.layers.55.mlp.experts.213.up_proj", + "model.layers.55.mlp.experts.214.down_proj", + "model.layers.55.mlp.experts.214.gate_proj", + "model.layers.55.mlp.experts.214.up_proj", + "model.layers.55.mlp.experts.215.down_proj", + "model.layers.55.mlp.experts.215.gate_proj", + "model.layers.55.mlp.experts.215.up_proj", + "model.layers.55.mlp.experts.216.down_proj", + "model.layers.55.mlp.experts.216.gate_proj", + "model.layers.55.mlp.experts.216.up_proj", + "model.layers.55.mlp.experts.217.down_proj", + "model.layers.55.mlp.experts.217.gate_proj", + "model.layers.55.mlp.experts.217.up_proj", + "model.layers.55.mlp.experts.218.down_proj", + "model.layers.55.mlp.experts.218.gate_proj", + "model.layers.55.mlp.experts.218.up_proj", + "model.layers.55.mlp.experts.219.down_proj", + "model.layers.55.mlp.experts.219.gate_proj", + "model.layers.55.mlp.experts.219.up_proj", + "model.layers.55.mlp.experts.22.down_proj", + "model.layers.55.mlp.experts.22.gate_proj", + "model.layers.55.mlp.experts.22.up_proj", + "model.layers.55.mlp.experts.220.down_proj", + "model.layers.55.mlp.experts.220.gate_proj", + "model.layers.55.mlp.experts.220.up_proj", + "model.layers.55.mlp.experts.221.down_proj", + "model.layers.55.mlp.experts.221.gate_proj", + "model.layers.55.mlp.experts.221.up_proj", + "model.layers.55.mlp.experts.222.down_proj", + "model.layers.55.mlp.experts.222.gate_proj", + "model.layers.55.mlp.experts.222.up_proj", + "model.layers.55.mlp.experts.223.down_proj", + "model.layers.55.mlp.experts.223.gate_proj", + "model.layers.55.mlp.experts.223.up_proj", + "model.layers.55.mlp.experts.224.down_proj", + "model.layers.55.mlp.experts.224.gate_proj", + "model.layers.55.mlp.experts.224.up_proj", + "model.layers.55.mlp.experts.225.down_proj", + "model.layers.55.mlp.experts.225.gate_proj", + "model.layers.55.mlp.experts.225.up_proj", + "model.layers.55.mlp.experts.226.down_proj", + "model.layers.55.mlp.experts.226.gate_proj", + "model.layers.55.mlp.experts.226.up_proj", + "model.layers.55.mlp.experts.227.down_proj", + "model.layers.55.mlp.experts.227.gate_proj", + "model.layers.55.mlp.experts.227.up_proj", + "model.layers.55.mlp.experts.228.down_proj", + "model.layers.55.mlp.experts.228.gate_proj", + "model.layers.55.mlp.experts.228.up_proj", + "model.layers.55.mlp.experts.229.down_proj", + "model.layers.55.mlp.experts.229.gate_proj", + "model.layers.55.mlp.experts.229.up_proj", + "model.layers.55.mlp.experts.23.down_proj", + "model.layers.55.mlp.experts.23.gate_proj", + "model.layers.55.mlp.experts.23.up_proj", + "model.layers.55.mlp.experts.230.down_proj", + "model.layers.55.mlp.experts.230.gate_proj", + "model.layers.55.mlp.experts.230.up_proj", + "model.layers.55.mlp.experts.231.down_proj", + "model.layers.55.mlp.experts.231.gate_proj", + "model.layers.55.mlp.experts.231.up_proj", + "model.layers.55.mlp.experts.232.down_proj", + "model.layers.55.mlp.experts.232.gate_proj", + "model.layers.55.mlp.experts.232.up_proj", + "model.layers.55.mlp.experts.233.down_proj", + "model.layers.55.mlp.experts.233.gate_proj", + "model.layers.55.mlp.experts.233.up_proj", + "model.layers.55.mlp.experts.234.down_proj", + "model.layers.55.mlp.experts.234.gate_proj", + "model.layers.55.mlp.experts.234.up_proj", + "model.layers.55.mlp.experts.235.down_proj", + "model.layers.55.mlp.experts.235.gate_proj", + "model.layers.55.mlp.experts.235.up_proj", + "model.layers.55.mlp.experts.236.down_proj", + "model.layers.55.mlp.experts.236.gate_proj", + "model.layers.55.mlp.experts.236.up_proj", + "model.layers.55.mlp.experts.237.down_proj", + "model.layers.55.mlp.experts.237.gate_proj", + "model.layers.55.mlp.experts.237.up_proj", + "model.layers.55.mlp.experts.238.down_proj", + "model.layers.55.mlp.experts.238.gate_proj", + "model.layers.55.mlp.experts.238.up_proj", + "model.layers.55.mlp.experts.239.down_proj", + "model.layers.55.mlp.experts.239.gate_proj", + "model.layers.55.mlp.experts.239.up_proj", + "model.layers.55.mlp.experts.24.down_proj", + "model.layers.55.mlp.experts.24.gate_proj", + "model.layers.55.mlp.experts.24.up_proj", + "model.layers.55.mlp.experts.240.down_proj", + "model.layers.55.mlp.experts.240.gate_proj", + "model.layers.55.mlp.experts.240.up_proj", + "model.layers.55.mlp.experts.241.down_proj", + "model.layers.55.mlp.experts.241.gate_proj", + "model.layers.55.mlp.experts.241.up_proj", + "model.layers.55.mlp.experts.242.down_proj", + "model.layers.55.mlp.experts.242.gate_proj", + "model.layers.55.mlp.experts.242.up_proj", + "model.layers.55.mlp.experts.243.down_proj", + "model.layers.55.mlp.experts.243.gate_proj", + "model.layers.55.mlp.experts.243.up_proj", + "model.layers.55.mlp.experts.244.down_proj", + "model.layers.55.mlp.experts.244.gate_proj", + "model.layers.55.mlp.experts.244.up_proj", + "model.layers.55.mlp.experts.245.down_proj", + "model.layers.55.mlp.experts.245.gate_proj", + "model.layers.55.mlp.experts.245.up_proj", + "model.layers.55.mlp.experts.246.down_proj", + "model.layers.55.mlp.experts.246.gate_proj", + "model.layers.55.mlp.experts.246.up_proj", + "model.layers.55.mlp.experts.247.down_proj", + "model.layers.55.mlp.experts.247.gate_proj", + "model.layers.55.mlp.experts.247.up_proj", + "model.layers.55.mlp.experts.248.down_proj", + "model.layers.55.mlp.experts.248.gate_proj", + "model.layers.55.mlp.experts.248.up_proj", + "model.layers.55.mlp.experts.249.down_proj", + "model.layers.55.mlp.experts.249.gate_proj", + "model.layers.55.mlp.experts.249.up_proj", + "model.layers.55.mlp.experts.25.down_proj", + "model.layers.55.mlp.experts.25.gate_proj", + "model.layers.55.mlp.experts.25.up_proj", + "model.layers.55.mlp.experts.250.down_proj", + "model.layers.55.mlp.experts.250.gate_proj", + "model.layers.55.mlp.experts.250.up_proj", + "model.layers.55.mlp.experts.251.down_proj", + "model.layers.55.mlp.experts.251.gate_proj", + "model.layers.55.mlp.experts.251.up_proj", + "model.layers.55.mlp.experts.252.down_proj", + "model.layers.55.mlp.experts.252.gate_proj", + "model.layers.55.mlp.experts.252.up_proj", + "model.layers.55.mlp.experts.253.down_proj", + "model.layers.55.mlp.experts.253.gate_proj", + "model.layers.55.mlp.experts.253.up_proj", + "model.layers.55.mlp.experts.254.down_proj", + "model.layers.55.mlp.experts.254.gate_proj", + "model.layers.55.mlp.experts.254.up_proj", + "model.layers.55.mlp.experts.255.down_proj", + "model.layers.55.mlp.experts.255.gate_proj", + "model.layers.55.mlp.experts.255.up_proj", + "model.layers.55.mlp.experts.26.down_proj", + "model.layers.55.mlp.experts.26.gate_proj", + "model.layers.55.mlp.experts.26.up_proj", + "model.layers.55.mlp.experts.27.down_proj", + "model.layers.55.mlp.experts.27.gate_proj", + "model.layers.55.mlp.experts.27.up_proj", + "model.layers.55.mlp.experts.28.down_proj", + "model.layers.55.mlp.experts.28.gate_proj", + "model.layers.55.mlp.experts.28.up_proj", + "model.layers.55.mlp.experts.29.down_proj", + "model.layers.55.mlp.experts.29.gate_proj", + "model.layers.55.mlp.experts.29.up_proj", + "model.layers.55.mlp.experts.3.down_proj", + "model.layers.55.mlp.experts.3.gate_proj", + "model.layers.55.mlp.experts.3.up_proj", + "model.layers.55.mlp.experts.30.down_proj", + "model.layers.55.mlp.experts.30.gate_proj", + "model.layers.55.mlp.experts.30.up_proj", + "model.layers.55.mlp.experts.31.down_proj", + "model.layers.55.mlp.experts.31.gate_proj", + "model.layers.55.mlp.experts.31.up_proj", + "model.layers.55.mlp.experts.32.down_proj", + "model.layers.55.mlp.experts.32.gate_proj", + "model.layers.55.mlp.experts.32.up_proj", + "model.layers.55.mlp.experts.33.down_proj", + "model.layers.55.mlp.experts.33.gate_proj", + "model.layers.55.mlp.experts.33.up_proj", + "model.layers.55.mlp.experts.34.down_proj", + "model.layers.55.mlp.experts.34.gate_proj", + "model.layers.55.mlp.experts.34.up_proj", + "model.layers.55.mlp.experts.35.down_proj", + "model.layers.55.mlp.experts.35.gate_proj", + "model.layers.55.mlp.experts.35.up_proj", + "model.layers.55.mlp.experts.36.down_proj", + "model.layers.55.mlp.experts.36.gate_proj", + "model.layers.55.mlp.experts.36.up_proj", + "model.layers.55.mlp.experts.37.down_proj", + "model.layers.55.mlp.experts.37.gate_proj", + "model.layers.55.mlp.experts.37.up_proj", + "model.layers.55.mlp.experts.38.down_proj", + "model.layers.55.mlp.experts.38.gate_proj", + "model.layers.55.mlp.experts.38.up_proj", + "model.layers.55.mlp.experts.39.down_proj", + "model.layers.55.mlp.experts.39.gate_proj", + "model.layers.55.mlp.experts.39.up_proj", + "model.layers.55.mlp.experts.4.down_proj", + "model.layers.55.mlp.experts.4.gate_proj", + "model.layers.55.mlp.experts.4.up_proj", + "model.layers.55.mlp.experts.40.down_proj", + "model.layers.55.mlp.experts.40.gate_proj", + "model.layers.55.mlp.experts.40.up_proj", + "model.layers.55.mlp.experts.41.down_proj", + "model.layers.55.mlp.experts.41.gate_proj", + "model.layers.55.mlp.experts.41.up_proj", + "model.layers.55.mlp.experts.42.down_proj", + "model.layers.55.mlp.experts.42.gate_proj", + "model.layers.55.mlp.experts.42.up_proj", + "model.layers.55.mlp.experts.43.down_proj", + "model.layers.55.mlp.experts.43.gate_proj", + "model.layers.55.mlp.experts.43.up_proj", + "model.layers.55.mlp.experts.44.down_proj", + "model.layers.55.mlp.experts.44.gate_proj", + "model.layers.55.mlp.experts.44.up_proj", + "model.layers.55.mlp.experts.45.down_proj", + "model.layers.55.mlp.experts.45.gate_proj", + "model.layers.55.mlp.experts.45.up_proj", + "model.layers.55.mlp.experts.46.down_proj", + "model.layers.55.mlp.experts.46.gate_proj", + "model.layers.55.mlp.experts.46.up_proj", + "model.layers.55.mlp.experts.47.down_proj", + "model.layers.55.mlp.experts.47.gate_proj", + "model.layers.55.mlp.experts.47.up_proj", + "model.layers.55.mlp.experts.48.down_proj", + "model.layers.55.mlp.experts.48.gate_proj", + "model.layers.55.mlp.experts.48.up_proj", + "model.layers.55.mlp.experts.49.down_proj", + "model.layers.55.mlp.experts.49.gate_proj", + "model.layers.55.mlp.experts.49.up_proj", + "model.layers.55.mlp.experts.5.down_proj", + "model.layers.55.mlp.experts.5.gate_proj", + "model.layers.55.mlp.experts.5.up_proj", + "model.layers.55.mlp.experts.50.down_proj", + "model.layers.55.mlp.experts.50.gate_proj", + "model.layers.55.mlp.experts.50.up_proj", + "model.layers.55.mlp.experts.51.down_proj", + "model.layers.55.mlp.experts.51.gate_proj", + "model.layers.55.mlp.experts.51.up_proj", + "model.layers.55.mlp.experts.52.down_proj", + "model.layers.55.mlp.experts.52.gate_proj", + "model.layers.55.mlp.experts.52.up_proj", + "model.layers.55.mlp.experts.53.down_proj", + "model.layers.55.mlp.experts.53.gate_proj", + "model.layers.55.mlp.experts.53.up_proj", + "model.layers.55.mlp.experts.54.down_proj", + "model.layers.55.mlp.experts.54.gate_proj", + "model.layers.55.mlp.experts.54.up_proj", + "model.layers.55.mlp.experts.55.down_proj", + "model.layers.55.mlp.experts.55.gate_proj", + "model.layers.55.mlp.experts.55.up_proj", + "model.layers.55.mlp.experts.56.down_proj", + "model.layers.55.mlp.experts.56.gate_proj", + "model.layers.55.mlp.experts.56.up_proj", + "model.layers.55.mlp.experts.57.down_proj", + "model.layers.55.mlp.experts.57.gate_proj", + "model.layers.55.mlp.experts.57.up_proj", + "model.layers.55.mlp.experts.58.down_proj", + "model.layers.55.mlp.experts.58.gate_proj", + "model.layers.55.mlp.experts.58.up_proj", + "model.layers.55.mlp.experts.59.down_proj", + "model.layers.55.mlp.experts.59.gate_proj", + "model.layers.55.mlp.experts.59.up_proj", + "model.layers.55.mlp.experts.6.down_proj", + "model.layers.55.mlp.experts.6.gate_proj", + "model.layers.55.mlp.experts.6.up_proj", + "model.layers.55.mlp.experts.60.down_proj", + "model.layers.55.mlp.experts.60.gate_proj", + "model.layers.55.mlp.experts.60.up_proj", + "model.layers.55.mlp.experts.61.down_proj", + "model.layers.55.mlp.experts.61.gate_proj", + "model.layers.55.mlp.experts.61.up_proj", + "model.layers.55.mlp.experts.62.down_proj", + "model.layers.55.mlp.experts.62.gate_proj", + "model.layers.55.mlp.experts.62.up_proj", + "model.layers.55.mlp.experts.63.down_proj", + "model.layers.55.mlp.experts.63.gate_proj", + "model.layers.55.mlp.experts.63.up_proj", + "model.layers.55.mlp.experts.64.down_proj", + "model.layers.55.mlp.experts.64.gate_proj", + "model.layers.55.mlp.experts.64.up_proj", + "model.layers.55.mlp.experts.65.down_proj", + "model.layers.55.mlp.experts.65.gate_proj", + "model.layers.55.mlp.experts.65.up_proj", + "model.layers.55.mlp.experts.66.down_proj", + "model.layers.55.mlp.experts.66.gate_proj", + "model.layers.55.mlp.experts.66.up_proj", + "model.layers.55.mlp.experts.67.down_proj", + "model.layers.55.mlp.experts.67.gate_proj", + "model.layers.55.mlp.experts.67.up_proj", + "model.layers.55.mlp.experts.68.down_proj", + "model.layers.55.mlp.experts.68.gate_proj", + "model.layers.55.mlp.experts.68.up_proj", + "model.layers.55.mlp.experts.69.down_proj", + "model.layers.55.mlp.experts.69.gate_proj", + "model.layers.55.mlp.experts.69.up_proj", + "model.layers.55.mlp.experts.7.down_proj", + "model.layers.55.mlp.experts.7.gate_proj", + "model.layers.55.mlp.experts.7.up_proj", + "model.layers.55.mlp.experts.70.down_proj", + "model.layers.55.mlp.experts.70.gate_proj", + "model.layers.55.mlp.experts.70.up_proj", + "model.layers.55.mlp.experts.71.down_proj", + "model.layers.55.mlp.experts.71.gate_proj", + "model.layers.55.mlp.experts.71.up_proj", + "model.layers.55.mlp.experts.72.down_proj", + "model.layers.55.mlp.experts.72.gate_proj", + "model.layers.55.mlp.experts.72.up_proj", + "model.layers.55.mlp.experts.73.down_proj", + "model.layers.55.mlp.experts.73.gate_proj", + "model.layers.55.mlp.experts.73.up_proj", + "model.layers.55.mlp.experts.74.down_proj", + "model.layers.55.mlp.experts.74.gate_proj", + "model.layers.55.mlp.experts.74.up_proj", + "model.layers.55.mlp.experts.75.down_proj", + "model.layers.55.mlp.experts.75.gate_proj", + "model.layers.55.mlp.experts.75.up_proj", + "model.layers.55.mlp.experts.76.down_proj", + "model.layers.55.mlp.experts.76.gate_proj", + "model.layers.55.mlp.experts.76.up_proj", + "model.layers.55.mlp.experts.77.down_proj", + "model.layers.55.mlp.experts.77.gate_proj", + "model.layers.55.mlp.experts.77.up_proj", + "model.layers.55.mlp.experts.78.down_proj", + "model.layers.55.mlp.experts.78.gate_proj", + "model.layers.55.mlp.experts.78.up_proj", + "model.layers.55.mlp.experts.79.down_proj", + "model.layers.55.mlp.experts.79.gate_proj", + "model.layers.55.mlp.experts.79.up_proj", + "model.layers.55.mlp.experts.8.down_proj", + "model.layers.55.mlp.experts.8.gate_proj", + "model.layers.55.mlp.experts.8.up_proj", + "model.layers.55.mlp.experts.80.down_proj", + "model.layers.55.mlp.experts.80.gate_proj", + "model.layers.55.mlp.experts.80.up_proj", + "model.layers.55.mlp.experts.81.down_proj", + "model.layers.55.mlp.experts.81.gate_proj", + "model.layers.55.mlp.experts.81.up_proj", + "model.layers.55.mlp.experts.82.down_proj", + "model.layers.55.mlp.experts.82.gate_proj", + "model.layers.55.mlp.experts.82.up_proj", + "model.layers.55.mlp.experts.83.down_proj", + "model.layers.55.mlp.experts.83.gate_proj", + "model.layers.55.mlp.experts.83.up_proj", + "model.layers.55.mlp.experts.84.down_proj", + "model.layers.55.mlp.experts.84.gate_proj", + "model.layers.55.mlp.experts.84.up_proj", + "model.layers.55.mlp.experts.85.down_proj", + "model.layers.55.mlp.experts.85.gate_proj", + "model.layers.55.mlp.experts.85.up_proj", + "model.layers.55.mlp.experts.86.down_proj", + "model.layers.55.mlp.experts.86.gate_proj", + "model.layers.55.mlp.experts.86.up_proj", + "model.layers.55.mlp.experts.87.down_proj", + "model.layers.55.mlp.experts.87.gate_proj", + "model.layers.55.mlp.experts.87.up_proj", + "model.layers.55.mlp.experts.88.down_proj", + "model.layers.55.mlp.experts.88.gate_proj", + "model.layers.55.mlp.experts.88.up_proj", + "model.layers.55.mlp.experts.89.down_proj", + "model.layers.55.mlp.experts.89.gate_proj", + "model.layers.55.mlp.experts.89.up_proj", + "model.layers.55.mlp.experts.9.down_proj", + "model.layers.55.mlp.experts.9.gate_proj", + "model.layers.55.mlp.experts.9.up_proj", + "model.layers.55.mlp.experts.90.down_proj", + "model.layers.55.mlp.experts.90.gate_proj", + "model.layers.55.mlp.experts.90.up_proj", + "model.layers.55.mlp.experts.91.down_proj", + "model.layers.55.mlp.experts.91.gate_proj", + "model.layers.55.mlp.experts.91.up_proj", + "model.layers.55.mlp.experts.92.down_proj", + "model.layers.55.mlp.experts.92.gate_proj", + "model.layers.55.mlp.experts.92.up_proj", + "model.layers.55.mlp.experts.93.down_proj", + "model.layers.55.mlp.experts.93.gate_proj", + "model.layers.55.mlp.experts.93.up_proj", + "model.layers.55.mlp.experts.94.down_proj", + "model.layers.55.mlp.experts.94.gate_proj", + "model.layers.55.mlp.experts.94.up_proj", + "model.layers.55.mlp.experts.95.down_proj", + "model.layers.55.mlp.experts.95.gate_proj", + "model.layers.55.mlp.experts.95.up_proj", + "model.layers.55.mlp.experts.96.down_proj", + "model.layers.55.mlp.experts.96.gate_proj", + "model.layers.55.mlp.experts.96.up_proj", + "model.layers.55.mlp.experts.97.down_proj", + "model.layers.55.mlp.experts.97.gate_proj", + "model.layers.55.mlp.experts.97.up_proj", + "model.layers.55.mlp.experts.98.down_proj", + "model.layers.55.mlp.experts.98.gate_proj", + "model.layers.55.mlp.experts.98.up_proj", + "model.layers.55.mlp.experts.99.down_proj", + "model.layers.55.mlp.experts.99.gate_proj", + "model.layers.55.mlp.experts.99.up_proj", + "model.layers.56.mlp.experts.0.down_proj", + "model.layers.56.mlp.experts.0.gate_proj", + "model.layers.56.mlp.experts.0.up_proj", + "model.layers.56.mlp.experts.1.down_proj", + "model.layers.56.mlp.experts.1.gate_proj", + "model.layers.56.mlp.experts.1.up_proj", + "model.layers.56.mlp.experts.10.down_proj", + "model.layers.56.mlp.experts.10.gate_proj", + "model.layers.56.mlp.experts.10.up_proj", + "model.layers.56.mlp.experts.100.down_proj", + "model.layers.56.mlp.experts.100.gate_proj", + "model.layers.56.mlp.experts.100.up_proj", + "model.layers.56.mlp.experts.101.down_proj", + "model.layers.56.mlp.experts.101.gate_proj", + "model.layers.56.mlp.experts.101.up_proj", + "model.layers.56.mlp.experts.102.down_proj", + "model.layers.56.mlp.experts.102.gate_proj", + "model.layers.56.mlp.experts.102.up_proj", + "model.layers.56.mlp.experts.103.down_proj", + "model.layers.56.mlp.experts.103.gate_proj", + "model.layers.56.mlp.experts.103.up_proj", + "model.layers.56.mlp.experts.104.down_proj", + "model.layers.56.mlp.experts.104.gate_proj", + "model.layers.56.mlp.experts.104.up_proj", + "model.layers.56.mlp.experts.105.down_proj", + "model.layers.56.mlp.experts.105.gate_proj", + "model.layers.56.mlp.experts.105.up_proj", + "model.layers.56.mlp.experts.106.down_proj", + "model.layers.56.mlp.experts.106.gate_proj", + "model.layers.56.mlp.experts.106.up_proj", + "model.layers.56.mlp.experts.107.down_proj", + "model.layers.56.mlp.experts.107.gate_proj", + "model.layers.56.mlp.experts.107.up_proj", + "model.layers.56.mlp.experts.108.down_proj", + "model.layers.56.mlp.experts.108.gate_proj", + "model.layers.56.mlp.experts.108.up_proj", + "model.layers.56.mlp.experts.109.down_proj", + "model.layers.56.mlp.experts.109.gate_proj", + "model.layers.56.mlp.experts.109.up_proj", + "model.layers.56.mlp.experts.11.down_proj", + "model.layers.56.mlp.experts.11.gate_proj", + "model.layers.56.mlp.experts.11.up_proj", + "model.layers.56.mlp.experts.110.down_proj", + "model.layers.56.mlp.experts.110.gate_proj", + "model.layers.56.mlp.experts.110.up_proj", + "model.layers.56.mlp.experts.111.down_proj", + "model.layers.56.mlp.experts.111.gate_proj", + "model.layers.56.mlp.experts.111.up_proj", + "model.layers.56.mlp.experts.112.down_proj", + "model.layers.56.mlp.experts.112.gate_proj", + "model.layers.56.mlp.experts.112.up_proj", + "model.layers.56.mlp.experts.113.down_proj", + "model.layers.56.mlp.experts.113.gate_proj", + "model.layers.56.mlp.experts.113.up_proj", + "model.layers.56.mlp.experts.114.down_proj", + "model.layers.56.mlp.experts.114.gate_proj", + "model.layers.56.mlp.experts.114.up_proj", + "model.layers.56.mlp.experts.115.down_proj", + "model.layers.56.mlp.experts.115.gate_proj", + "model.layers.56.mlp.experts.115.up_proj", + "model.layers.56.mlp.experts.116.down_proj", + "model.layers.56.mlp.experts.116.gate_proj", + "model.layers.56.mlp.experts.116.up_proj", + "model.layers.56.mlp.experts.117.down_proj", + "model.layers.56.mlp.experts.117.gate_proj", + "model.layers.56.mlp.experts.117.up_proj", + "model.layers.56.mlp.experts.118.down_proj", + "model.layers.56.mlp.experts.118.gate_proj", + "model.layers.56.mlp.experts.118.up_proj", + "model.layers.56.mlp.experts.119.down_proj", + "model.layers.56.mlp.experts.119.gate_proj", + "model.layers.56.mlp.experts.119.up_proj", + "model.layers.56.mlp.experts.12.down_proj", + "model.layers.56.mlp.experts.12.gate_proj", + "model.layers.56.mlp.experts.12.up_proj", + "model.layers.56.mlp.experts.120.down_proj", + "model.layers.56.mlp.experts.120.gate_proj", + "model.layers.56.mlp.experts.120.up_proj", + "model.layers.56.mlp.experts.121.down_proj", + "model.layers.56.mlp.experts.121.gate_proj", + "model.layers.56.mlp.experts.121.up_proj", + "model.layers.56.mlp.experts.122.down_proj", + "model.layers.56.mlp.experts.122.gate_proj", + "model.layers.56.mlp.experts.122.up_proj", + "model.layers.56.mlp.experts.123.down_proj", + "model.layers.56.mlp.experts.123.gate_proj", + "model.layers.56.mlp.experts.123.up_proj", + "model.layers.56.mlp.experts.124.down_proj", + "model.layers.56.mlp.experts.124.gate_proj", + "model.layers.56.mlp.experts.124.up_proj", + "model.layers.56.mlp.experts.125.down_proj", + "model.layers.56.mlp.experts.125.gate_proj", + "model.layers.56.mlp.experts.125.up_proj", + "model.layers.56.mlp.experts.126.down_proj", + "model.layers.56.mlp.experts.126.gate_proj", + "model.layers.56.mlp.experts.126.up_proj", + "model.layers.56.mlp.experts.127.down_proj", + "model.layers.56.mlp.experts.127.gate_proj", + "model.layers.56.mlp.experts.127.up_proj", + "model.layers.56.mlp.experts.128.down_proj", + "model.layers.56.mlp.experts.128.gate_proj", + "model.layers.56.mlp.experts.128.up_proj", + "model.layers.56.mlp.experts.129.down_proj", + "model.layers.56.mlp.experts.129.gate_proj", + "model.layers.56.mlp.experts.129.up_proj", + "model.layers.56.mlp.experts.13.down_proj", + "model.layers.56.mlp.experts.13.gate_proj", + "model.layers.56.mlp.experts.13.up_proj", + "model.layers.56.mlp.experts.130.down_proj", + "model.layers.56.mlp.experts.130.gate_proj", + "model.layers.56.mlp.experts.130.up_proj", + "model.layers.56.mlp.experts.131.down_proj", + "model.layers.56.mlp.experts.131.gate_proj", + "model.layers.56.mlp.experts.131.up_proj", + "model.layers.56.mlp.experts.132.down_proj", + "model.layers.56.mlp.experts.132.gate_proj", + "model.layers.56.mlp.experts.132.up_proj", + "model.layers.56.mlp.experts.133.down_proj", + "model.layers.56.mlp.experts.133.gate_proj", + "model.layers.56.mlp.experts.133.up_proj", + "model.layers.56.mlp.experts.134.down_proj", + "model.layers.56.mlp.experts.134.gate_proj", + "model.layers.56.mlp.experts.134.up_proj", + "model.layers.56.mlp.experts.135.down_proj", + "model.layers.56.mlp.experts.135.gate_proj", + "model.layers.56.mlp.experts.135.up_proj", + "model.layers.56.mlp.experts.136.down_proj", + "model.layers.56.mlp.experts.136.gate_proj", + "model.layers.56.mlp.experts.136.up_proj", + "model.layers.56.mlp.experts.137.down_proj", + "model.layers.56.mlp.experts.137.gate_proj", + "model.layers.56.mlp.experts.137.up_proj", + "model.layers.56.mlp.experts.138.down_proj", + "model.layers.56.mlp.experts.138.gate_proj", + "model.layers.56.mlp.experts.138.up_proj", + "model.layers.56.mlp.experts.139.down_proj", + "model.layers.56.mlp.experts.139.gate_proj", + "model.layers.56.mlp.experts.139.up_proj", + "model.layers.56.mlp.experts.14.down_proj", + "model.layers.56.mlp.experts.14.gate_proj", + "model.layers.56.mlp.experts.14.up_proj", + "model.layers.56.mlp.experts.140.down_proj", + "model.layers.56.mlp.experts.140.gate_proj", + "model.layers.56.mlp.experts.140.up_proj", + "model.layers.56.mlp.experts.141.down_proj", + "model.layers.56.mlp.experts.141.gate_proj", + "model.layers.56.mlp.experts.141.up_proj", + "model.layers.56.mlp.experts.142.down_proj", + "model.layers.56.mlp.experts.142.gate_proj", + "model.layers.56.mlp.experts.142.up_proj", + "model.layers.56.mlp.experts.143.down_proj", + "model.layers.56.mlp.experts.143.gate_proj", + "model.layers.56.mlp.experts.143.up_proj", + "model.layers.56.mlp.experts.144.down_proj", + "model.layers.56.mlp.experts.144.gate_proj", + "model.layers.56.mlp.experts.144.up_proj", + "model.layers.56.mlp.experts.145.down_proj", + "model.layers.56.mlp.experts.145.gate_proj", + "model.layers.56.mlp.experts.145.up_proj", + "model.layers.56.mlp.experts.146.down_proj", + "model.layers.56.mlp.experts.146.gate_proj", + "model.layers.56.mlp.experts.146.up_proj", + "model.layers.56.mlp.experts.147.down_proj", + "model.layers.56.mlp.experts.147.gate_proj", + "model.layers.56.mlp.experts.147.up_proj", + "model.layers.56.mlp.experts.148.down_proj", + "model.layers.56.mlp.experts.148.gate_proj", + "model.layers.56.mlp.experts.148.up_proj", + "model.layers.56.mlp.experts.149.down_proj", + "model.layers.56.mlp.experts.149.gate_proj", + "model.layers.56.mlp.experts.149.up_proj", + "model.layers.56.mlp.experts.15.down_proj", + "model.layers.56.mlp.experts.15.gate_proj", + "model.layers.56.mlp.experts.15.up_proj", + "model.layers.56.mlp.experts.150.down_proj", + "model.layers.56.mlp.experts.150.gate_proj", + "model.layers.56.mlp.experts.150.up_proj", + "model.layers.56.mlp.experts.151.down_proj", + "model.layers.56.mlp.experts.151.gate_proj", + "model.layers.56.mlp.experts.151.up_proj", + "model.layers.56.mlp.experts.152.down_proj", + "model.layers.56.mlp.experts.152.gate_proj", + "model.layers.56.mlp.experts.152.up_proj", + "model.layers.56.mlp.experts.153.down_proj", + "model.layers.56.mlp.experts.153.gate_proj", + "model.layers.56.mlp.experts.153.up_proj", + "model.layers.56.mlp.experts.154.down_proj", + "model.layers.56.mlp.experts.154.gate_proj", + "model.layers.56.mlp.experts.154.up_proj", + "model.layers.56.mlp.experts.155.down_proj", + "model.layers.56.mlp.experts.155.gate_proj", + "model.layers.56.mlp.experts.155.up_proj", + "model.layers.56.mlp.experts.156.down_proj", + "model.layers.56.mlp.experts.156.gate_proj", + "model.layers.56.mlp.experts.156.up_proj", + "model.layers.56.mlp.experts.157.down_proj", + "model.layers.56.mlp.experts.157.gate_proj", + "model.layers.56.mlp.experts.157.up_proj", + "model.layers.56.mlp.experts.158.down_proj", + "model.layers.56.mlp.experts.158.gate_proj", + "model.layers.56.mlp.experts.158.up_proj", + "model.layers.56.mlp.experts.159.down_proj", + "model.layers.56.mlp.experts.159.gate_proj", + "model.layers.56.mlp.experts.159.up_proj", + "model.layers.56.mlp.experts.16.down_proj", + "model.layers.56.mlp.experts.16.gate_proj", + "model.layers.56.mlp.experts.16.up_proj", + "model.layers.56.mlp.experts.160.down_proj", + "model.layers.56.mlp.experts.160.gate_proj", + "model.layers.56.mlp.experts.160.up_proj", + "model.layers.56.mlp.experts.161.down_proj", + "model.layers.56.mlp.experts.161.gate_proj", + "model.layers.56.mlp.experts.161.up_proj", + "model.layers.56.mlp.experts.162.down_proj", + "model.layers.56.mlp.experts.162.gate_proj", + "model.layers.56.mlp.experts.162.up_proj", + "model.layers.56.mlp.experts.163.down_proj", + "model.layers.56.mlp.experts.163.gate_proj", + "model.layers.56.mlp.experts.163.up_proj", + "model.layers.56.mlp.experts.164.down_proj", + "model.layers.56.mlp.experts.164.gate_proj", + "model.layers.56.mlp.experts.164.up_proj", + "model.layers.56.mlp.experts.165.down_proj", + "model.layers.56.mlp.experts.165.gate_proj", + "model.layers.56.mlp.experts.165.up_proj", + "model.layers.56.mlp.experts.166.down_proj", + "model.layers.56.mlp.experts.166.gate_proj", + "model.layers.56.mlp.experts.166.up_proj", + "model.layers.56.mlp.experts.167.down_proj", + "model.layers.56.mlp.experts.167.gate_proj", + "model.layers.56.mlp.experts.167.up_proj", + "model.layers.56.mlp.experts.168.down_proj", + "model.layers.56.mlp.experts.168.gate_proj", + "model.layers.56.mlp.experts.168.up_proj", + "model.layers.56.mlp.experts.169.down_proj", + "model.layers.56.mlp.experts.169.gate_proj", + "model.layers.56.mlp.experts.169.up_proj", + "model.layers.56.mlp.experts.17.down_proj", + "model.layers.56.mlp.experts.17.gate_proj", + "model.layers.56.mlp.experts.17.up_proj", + "model.layers.56.mlp.experts.170.down_proj", + "model.layers.56.mlp.experts.170.gate_proj", + "model.layers.56.mlp.experts.170.up_proj", + "model.layers.56.mlp.experts.171.down_proj", + "model.layers.56.mlp.experts.171.gate_proj", + "model.layers.56.mlp.experts.171.up_proj", + "model.layers.56.mlp.experts.172.down_proj", + "model.layers.56.mlp.experts.172.gate_proj", + "model.layers.56.mlp.experts.172.up_proj", + "model.layers.56.mlp.experts.173.down_proj", + "model.layers.56.mlp.experts.173.gate_proj", + "model.layers.56.mlp.experts.173.up_proj", + "model.layers.56.mlp.experts.174.down_proj", + "model.layers.56.mlp.experts.174.gate_proj", + "model.layers.56.mlp.experts.174.up_proj", + "model.layers.56.mlp.experts.175.down_proj", + "model.layers.56.mlp.experts.175.gate_proj", + "model.layers.56.mlp.experts.175.up_proj", + "model.layers.56.mlp.experts.176.down_proj", + "model.layers.56.mlp.experts.176.gate_proj", + "model.layers.56.mlp.experts.176.up_proj", + "model.layers.56.mlp.experts.177.down_proj", + "model.layers.56.mlp.experts.177.gate_proj", + "model.layers.56.mlp.experts.177.up_proj", + "model.layers.56.mlp.experts.178.down_proj", + "model.layers.56.mlp.experts.178.gate_proj", + "model.layers.56.mlp.experts.178.up_proj", + "model.layers.56.mlp.experts.179.down_proj", + "model.layers.56.mlp.experts.179.gate_proj", + "model.layers.56.mlp.experts.179.up_proj", + "model.layers.56.mlp.experts.18.down_proj", + "model.layers.56.mlp.experts.18.gate_proj", + "model.layers.56.mlp.experts.18.up_proj", + "model.layers.56.mlp.experts.180.down_proj", + "model.layers.56.mlp.experts.180.gate_proj", + "model.layers.56.mlp.experts.180.up_proj", + "model.layers.56.mlp.experts.181.down_proj", + "model.layers.56.mlp.experts.181.gate_proj", + "model.layers.56.mlp.experts.181.up_proj", + "model.layers.56.mlp.experts.182.down_proj", + "model.layers.56.mlp.experts.182.gate_proj", + "model.layers.56.mlp.experts.182.up_proj", + "model.layers.56.mlp.experts.183.down_proj", + "model.layers.56.mlp.experts.183.gate_proj", + "model.layers.56.mlp.experts.183.up_proj", + "model.layers.56.mlp.experts.184.down_proj", + "model.layers.56.mlp.experts.184.gate_proj", + "model.layers.56.mlp.experts.184.up_proj", + "model.layers.56.mlp.experts.185.down_proj", + "model.layers.56.mlp.experts.185.gate_proj", + "model.layers.56.mlp.experts.185.up_proj", + "model.layers.56.mlp.experts.186.down_proj", + "model.layers.56.mlp.experts.186.gate_proj", + "model.layers.56.mlp.experts.186.up_proj", + "model.layers.56.mlp.experts.187.down_proj", + "model.layers.56.mlp.experts.187.gate_proj", + "model.layers.56.mlp.experts.187.up_proj", + "model.layers.56.mlp.experts.188.down_proj", + "model.layers.56.mlp.experts.188.gate_proj", + "model.layers.56.mlp.experts.188.up_proj", + "model.layers.56.mlp.experts.189.down_proj", + "model.layers.56.mlp.experts.189.gate_proj", + "model.layers.56.mlp.experts.189.up_proj", + "model.layers.56.mlp.experts.19.down_proj", + "model.layers.56.mlp.experts.19.gate_proj", + "model.layers.56.mlp.experts.19.up_proj", + "model.layers.56.mlp.experts.190.down_proj", + "model.layers.56.mlp.experts.190.gate_proj", + "model.layers.56.mlp.experts.190.up_proj", + "model.layers.56.mlp.experts.191.down_proj", + "model.layers.56.mlp.experts.191.gate_proj", + "model.layers.56.mlp.experts.191.up_proj", + "model.layers.56.mlp.experts.192.down_proj", + "model.layers.56.mlp.experts.192.gate_proj", + "model.layers.56.mlp.experts.192.up_proj", + "model.layers.56.mlp.experts.193.down_proj", + "model.layers.56.mlp.experts.193.gate_proj", + "model.layers.56.mlp.experts.193.up_proj", + "model.layers.56.mlp.experts.194.down_proj", + "model.layers.56.mlp.experts.194.gate_proj", + "model.layers.56.mlp.experts.194.up_proj", + "model.layers.56.mlp.experts.195.down_proj", + "model.layers.56.mlp.experts.195.gate_proj", + "model.layers.56.mlp.experts.195.up_proj", + "model.layers.56.mlp.experts.196.down_proj", + "model.layers.56.mlp.experts.196.gate_proj", + "model.layers.56.mlp.experts.196.up_proj", + "model.layers.56.mlp.experts.197.down_proj", + "model.layers.56.mlp.experts.197.gate_proj", + "model.layers.56.mlp.experts.197.up_proj", + "model.layers.56.mlp.experts.198.down_proj", + "model.layers.56.mlp.experts.198.gate_proj", + "model.layers.56.mlp.experts.198.up_proj", + "model.layers.56.mlp.experts.199.down_proj", + "model.layers.56.mlp.experts.199.gate_proj", + "model.layers.56.mlp.experts.199.up_proj", + "model.layers.56.mlp.experts.2.down_proj", + "model.layers.56.mlp.experts.2.gate_proj", + "model.layers.56.mlp.experts.2.up_proj", + "model.layers.56.mlp.experts.20.down_proj", + "model.layers.56.mlp.experts.20.gate_proj", + "model.layers.56.mlp.experts.20.up_proj", + "model.layers.56.mlp.experts.200.down_proj", + "model.layers.56.mlp.experts.200.gate_proj", + "model.layers.56.mlp.experts.200.up_proj", + "model.layers.56.mlp.experts.201.down_proj", + "model.layers.56.mlp.experts.201.gate_proj", + "model.layers.56.mlp.experts.201.up_proj", + "model.layers.56.mlp.experts.202.down_proj", + "model.layers.56.mlp.experts.202.gate_proj", + "model.layers.56.mlp.experts.202.up_proj", + "model.layers.56.mlp.experts.203.down_proj", + "model.layers.56.mlp.experts.203.gate_proj", + "model.layers.56.mlp.experts.203.up_proj", + "model.layers.56.mlp.experts.204.down_proj", + "model.layers.56.mlp.experts.204.gate_proj", + "model.layers.56.mlp.experts.204.up_proj", + "model.layers.56.mlp.experts.205.down_proj", + "model.layers.56.mlp.experts.205.gate_proj", + "model.layers.56.mlp.experts.205.up_proj", + "model.layers.56.mlp.experts.206.down_proj", + "model.layers.56.mlp.experts.206.gate_proj", + "model.layers.56.mlp.experts.206.up_proj", + "model.layers.56.mlp.experts.207.down_proj", + "model.layers.56.mlp.experts.207.gate_proj", + "model.layers.56.mlp.experts.207.up_proj", + "model.layers.56.mlp.experts.208.down_proj", + "model.layers.56.mlp.experts.208.gate_proj", + "model.layers.56.mlp.experts.208.up_proj", + "model.layers.56.mlp.experts.209.down_proj", + "model.layers.56.mlp.experts.209.gate_proj", + "model.layers.56.mlp.experts.209.up_proj", + "model.layers.56.mlp.experts.21.down_proj", + "model.layers.56.mlp.experts.21.gate_proj", + "model.layers.56.mlp.experts.21.up_proj", + "model.layers.56.mlp.experts.210.down_proj", + "model.layers.56.mlp.experts.210.gate_proj", + "model.layers.56.mlp.experts.210.up_proj", + "model.layers.56.mlp.experts.211.down_proj", + "model.layers.56.mlp.experts.211.gate_proj", + "model.layers.56.mlp.experts.211.up_proj", + "model.layers.56.mlp.experts.212.down_proj", + "model.layers.56.mlp.experts.212.gate_proj", + "model.layers.56.mlp.experts.212.up_proj", + "model.layers.56.mlp.experts.213.down_proj", + "model.layers.56.mlp.experts.213.gate_proj", + "model.layers.56.mlp.experts.213.up_proj", + "model.layers.56.mlp.experts.214.down_proj", + "model.layers.56.mlp.experts.214.gate_proj", + "model.layers.56.mlp.experts.214.up_proj", + "model.layers.56.mlp.experts.215.down_proj", + "model.layers.56.mlp.experts.215.gate_proj", + "model.layers.56.mlp.experts.215.up_proj", + "model.layers.56.mlp.experts.216.down_proj", + "model.layers.56.mlp.experts.216.gate_proj", + "model.layers.56.mlp.experts.216.up_proj", + "model.layers.56.mlp.experts.217.down_proj", + "model.layers.56.mlp.experts.217.gate_proj", + "model.layers.56.mlp.experts.217.up_proj", + "model.layers.56.mlp.experts.218.down_proj", + "model.layers.56.mlp.experts.218.gate_proj", + "model.layers.56.mlp.experts.218.up_proj", + "model.layers.56.mlp.experts.219.down_proj", + "model.layers.56.mlp.experts.219.gate_proj", + "model.layers.56.mlp.experts.219.up_proj", + "model.layers.56.mlp.experts.22.down_proj", + "model.layers.56.mlp.experts.22.gate_proj", + "model.layers.56.mlp.experts.22.up_proj", + "model.layers.56.mlp.experts.220.down_proj", + "model.layers.56.mlp.experts.220.gate_proj", + "model.layers.56.mlp.experts.220.up_proj", + "model.layers.56.mlp.experts.221.down_proj", + "model.layers.56.mlp.experts.221.gate_proj", + "model.layers.56.mlp.experts.221.up_proj", + "model.layers.56.mlp.experts.222.down_proj", + "model.layers.56.mlp.experts.222.gate_proj", + "model.layers.56.mlp.experts.222.up_proj", + "model.layers.56.mlp.experts.223.down_proj", + "model.layers.56.mlp.experts.223.gate_proj", + "model.layers.56.mlp.experts.223.up_proj", + "model.layers.56.mlp.experts.224.down_proj", + "model.layers.56.mlp.experts.224.gate_proj", + "model.layers.56.mlp.experts.224.up_proj", + "model.layers.56.mlp.experts.225.down_proj", + "model.layers.56.mlp.experts.225.gate_proj", + "model.layers.56.mlp.experts.225.up_proj", + "model.layers.56.mlp.experts.226.down_proj", + "model.layers.56.mlp.experts.226.gate_proj", + "model.layers.56.mlp.experts.226.up_proj", + "model.layers.56.mlp.experts.227.down_proj", + "model.layers.56.mlp.experts.227.gate_proj", + "model.layers.56.mlp.experts.227.up_proj", + "model.layers.56.mlp.experts.228.down_proj", + "model.layers.56.mlp.experts.228.gate_proj", + "model.layers.56.mlp.experts.228.up_proj", + "model.layers.56.mlp.experts.229.down_proj", + "model.layers.56.mlp.experts.229.gate_proj", + "model.layers.56.mlp.experts.229.up_proj", + "model.layers.56.mlp.experts.23.down_proj", + "model.layers.56.mlp.experts.23.gate_proj", + "model.layers.56.mlp.experts.23.up_proj", + "model.layers.56.mlp.experts.230.down_proj", + "model.layers.56.mlp.experts.230.gate_proj", + "model.layers.56.mlp.experts.230.up_proj", + "model.layers.56.mlp.experts.231.down_proj", + "model.layers.56.mlp.experts.231.gate_proj", + "model.layers.56.mlp.experts.231.up_proj", + "model.layers.56.mlp.experts.232.down_proj", + "model.layers.56.mlp.experts.232.gate_proj", + "model.layers.56.mlp.experts.232.up_proj", + "model.layers.56.mlp.experts.233.down_proj", + "model.layers.56.mlp.experts.233.gate_proj", + "model.layers.56.mlp.experts.233.up_proj", + "model.layers.56.mlp.experts.234.down_proj", + "model.layers.56.mlp.experts.234.gate_proj", + "model.layers.56.mlp.experts.234.up_proj", + "model.layers.56.mlp.experts.235.down_proj", + "model.layers.56.mlp.experts.235.gate_proj", + "model.layers.56.mlp.experts.235.up_proj", + "model.layers.56.mlp.experts.236.down_proj", + "model.layers.56.mlp.experts.236.gate_proj", + "model.layers.56.mlp.experts.236.up_proj", + "model.layers.56.mlp.experts.237.down_proj", + "model.layers.56.mlp.experts.237.gate_proj", + "model.layers.56.mlp.experts.237.up_proj", + "model.layers.56.mlp.experts.238.down_proj", + "model.layers.56.mlp.experts.238.gate_proj", + "model.layers.56.mlp.experts.238.up_proj", + "model.layers.56.mlp.experts.239.down_proj", + "model.layers.56.mlp.experts.239.gate_proj", + "model.layers.56.mlp.experts.239.up_proj", + "model.layers.56.mlp.experts.24.down_proj", + "model.layers.56.mlp.experts.24.gate_proj", + "model.layers.56.mlp.experts.24.up_proj", + "model.layers.56.mlp.experts.240.down_proj", + "model.layers.56.mlp.experts.240.gate_proj", + "model.layers.56.mlp.experts.240.up_proj", + "model.layers.56.mlp.experts.241.down_proj", + "model.layers.56.mlp.experts.241.gate_proj", + "model.layers.56.mlp.experts.241.up_proj", + "model.layers.56.mlp.experts.242.down_proj", + "model.layers.56.mlp.experts.242.gate_proj", + "model.layers.56.mlp.experts.242.up_proj", + "model.layers.56.mlp.experts.243.down_proj", + "model.layers.56.mlp.experts.243.gate_proj", + "model.layers.56.mlp.experts.243.up_proj", + "model.layers.56.mlp.experts.244.down_proj", + "model.layers.56.mlp.experts.244.gate_proj", + "model.layers.56.mlp.experts.244.up_proj", + "model.layers.56.mlp.experts.245.down_proj", + "model.layers.56.mlp.experts.245.gate_proj", + "model.layers.56.mlp.experts.245.up_proj", + "model.layers.56.mlp.experts.246.down_proj", + "model.layers.56.mlp.experts.246.gate_proj", + "model.layers.56.mlp.experts.246.up_proj", + "model.layers.56.mlp.experts.247.down_proj", + "model.layers.56.mlp.experts.247.gate_proj", + "model.layers.56.mlp.experts.247.up_proj", + "model.layers.56.mlp.experts.248.down_proj", + "model.layers.56.mlp.experts.248.gate_proj", + "model.layers.56.mlp.experts.248.up_proj", + "model.layers.56.mlp.experts.249.down_proj", + "model.layers.56.mlp.experts.249.gate_proj", + "model.layers.56.mlp.experts.249.up_proj", + "model.layers.56.mlp.experts.25.down_proj", + "model.layers.56.mlp.experts.25.gate_proj", + "model.layers.56.mlp.experts.25.up_proj", + "model.layers.56.mlp.experts.250.down_proj", + "model.layers.56.mlp.experts.250.gate_proj", + "model.layers.56.mlp.experts.250.up_proj", + "model.layers.56.mlp.experts.251.down_proj", + "model.layers.56.mlp.experts.251.gate_proj", + "model.layers.56.mlp.experts.251.up_proj", + "model.layers.56.mlp.experts.252.down_proj", + "model.layers.56.mlp.experts.252.gate_proj", + "model.layers.56.mlp.experts.252.up_proj", + "model.layers.56.mlp.experts.253.down_proj", + "model.layers.56.mlp.experts.253.gate_proj", + "model.layers.56.mlp.experts.253.up_proj", + "model.layers.56.mlp.experts.254.down_proj", + "model.layers.56.mlp.experts.254.gate_proj", + "model.layers.56.mlp.experts.254.up_proj", + "model.layers.56.mlp.experts.255.down_proj", + "model.layers.56.mlp.experts.255.gate_proj", + "model.layers.56.mlp.experts.255.up_proj", + "model.layers.56.mlp.experts.26.down_proj", + "model.layers.56.mlp.experts.26.gate_proj", + "model.layers.56.mlp.experts.26.up_proj", + "model.layers.56.mlp.experts.27.down_proj", + "model.layers.56.mlp.experts.27.gate_proj", + "model.layers.56.mlp.experts.27.up_proj", + "model.layers.56.mlp.experts.28.down_proj", + "model.layers.56.mlp.experts.28.gate_proj", + "model.layers.56.mlp.experts.28.up_proj", + "model.layers.56.mlp.experts.29.down_proj", + "model.layers.56.mlp.experts.29.gate_proj", + "model.layers.56.mlp.experts.29.up_proj", + "model.layers.56.mlp.experts.3.down_proj", + "model.layers.56.mlp.experts.3.gate_proj", + "model.layers.56.mlp.experts.3.up_proj", + "model.layers.56.mlp.experts.30.down_proj", + "model.layers.56.mlp.experts.30.gate_proj", + "model.layers.56.mlp.experts.30.up_proj", + "model.layers.56.mlp.experts.31.down_proj", + "model.layers.56.mlp.experts.31.gate_proj", + "model.layers.56.mlp.experts.31.up_proj", + "model.layers.56.mlp.experts.32.down_proj", + "model.layers.56.mlp.experts.32.gate_proj", + "model.layers.56.mlp.experts.32.up_proj", + "model.layers.56.mlp.experts.33.down_proj", + "model.layers.56.mlp.experts.33.gate_proj", + "model.layers.56.mlp.experts.33.up_proj", + "model.layers.56.mlp.experts.34.down_proj", + "model.layers.56.mlp.experts.34.gate_proj", + "model.layers.56.mlp.experts.34.up_proj", + "model.layers.56.mlp.experts.35.down_proj", + "model.layers.56.mlp.experts.35.gate_proj", + "model.layers.56.mlp.experts.35.up_proj", + "model.layers.56.mlp.experts.36.down_proj", + "model.layers.56.mlp.experts.36.gate_proj", + "model.layers.56.mlp.experts.36.up_proj", + "model.layers.56.mlp.experts.37.down_proj", + "model.layers.56.mlp.experts.37.gate_proj", + "model.layers.56.mlp.experts.37.up_proj", + "model.layers.56.mlp.experts.38.down_proj", + "model.layers.56.mlp.experts.38.gate_proj", + "model.layers.56.mlp.experts.38.up_proj", + "model.layers.56.mlp.experts.39.down_proj", + "model.layers.56.mlp.experts.39.gate_proj", + "model.layers.56.mlp.experts.39.up_proj", + "model.layers.56.mlp.experts.4.down_proj", + "model.layers.56.mlp.experts.4.gate_proj", + "model.layers.56.mlp.experts.4.up_proj", + "model.layers.56.mlp.experts.40.down_proj", + "model.layers.56.mlp.experts.40.gate_proj", + "model.layers.56.mlp.experts.40.up_proj", + "model.layers.56.mlp.experts.41.down_proj", + "model.layers.56.mlp.experts.41.gate_proj", + "model.layers.56.mlp.experts.41.up_proj", + "model.layers.56.mlp.experts.42.down_proj", + "model.layers.56.mlp.experts.42.gate_proj", + "model.layers.56.mlp.experts.42.up_proj", + "model.layers.56.mlp.experts.43.down_proj", + "model.layers.56.mlp.experts.43.gate_proj", + "model.layers.56.mlp.experts.43.up_proj", + "model.layers.56.mlp.experts.44.down_proj", + "model.layers.56.mlp.experts.44.gate_proj", + "model.layers.56.mlp.experts.44.up_proj", + "model.layers.56.mlp.experts.45.down_proj", + "model.layers.56.mlp.experts.45.gate_proj", + "model.layers.56.mlp.experts.45.up_proj", + "model.layers.56.mlp.experts.46.down_proj", + "model.layers.56.mlp.experts.46.gate_proj", + "model.layers.56.mlp.experts.46.up_proj", + "model.layers.56.mlp.experts.47.down_proj", + "model.layers.56.mlp.experts.47.gate_proj", + "model.layers.56.mlp.experts.47.up_proj", + "model.layers.56.mlp.experts.48.down_proj", + "model.layers.56.mlp.experts.48.gate_proj", + "model.layers.56.mlp.experts.48.up_proj", + "model.layers.56.mlp.experts.49.down_proj", + "model.layers.56.mlp.experts.49.gate_proj", + "model.layers.56.mlp.experts.49.up_proj", + "model.layers.56.mlp.experts.5.down_proj", + "model.layers.56.mlp.experts.5.gate_proj", + "model.layers.56.mlp.experts.5.up_proj", + "model.layers.56.mlp.experts.50.down_proj", + "model.layers.56.mlp.experts.50.gate_proj", + "model.layers.56.mlp.experts.50.up_proj", + "model.layers.56.mlp.experts.51.down_proj", + "model.layers.56.mlp.experts.51.gate_proj", + "model.layers.56.mlp.experts.51.up_proj", + "model.layers.56.mlp.experts.52.down_proj", + "model.layers.56.mlp.experts.52.gate_proj", + "model.layers.56.mlp.experts.52.up_proj", + "model.layers.56.mlp.experts.53.down_proj", + "model.layers.56.mlp.experts.53.gate_proj", + "model.layers.56.mlp.experts.53.up_proj", + "model.layers.56.mlp.experts.54.down_proj", + "model.layers.56.mlp.experts.54.gate_proj", + "model.layers.56.mlp.experts.54.up_proj", + "model.layers.56.mlp.experts.55.down_proj", + "model.layers.56.mlp.experts.55.gate_proj", + "model.layers.56.mlp.experts.55.up_proj", + "model.layers.56.mlp.experts.56.down_proj", + "model.layers.56.mlp.experts.56.gate_proj", + "model.layers.56.mlp.experts.56.up_proj", + "model.layers.56.mlp.experts.57.down_proj", + "model.layers.56.mlp.experts.57.gate_proj", + "model.layers.56.mlp.experts.57.up_proj", + "model.layers.56.mlp.experts.58.down_proj", + "model.layers.56.mlp.experts.58.gate_proj", + "model.layers.56.mlp.experts.58.up_proj", + "model.layers.56.mlp.experts.59.down_proj", + "model.layers.56.mlp.experts.59.gate_proj", + "model.layers.56.mlp.experts.59.up_proj", + "model.layers.56.mlp.experts.6.down_proj", + "model.layers.56.mlp.experts.6.gate_proj", + "model.layers.56.mlp.experts.6.up_proj", + "model.layers.56.mlp.experts.60.down_proj", + "model.layers.56.mlp.experts.60.gate_proj", + "model.layers.56.mlp.experts.60.up_proj", + "model.layers.56.mlp.experts.61.down_proj", + "model.layers.56.mlp.experts.61.gate_proj", + "model.layers.56.mlp.experts.61.up_proj", + "model.layers.56.mlp.experts.62.down_proj", + "model.layers.56.mlp.experts.62.gate_proj", + "model.layers.56.mlp.experts.62.up_proj", + "model.layers.56.mlp.experts.63.down_proj", + "model.layers.56.mlp.experts.63.gate_proj", + "model.layers.56.mlp.experts.63.up_proj", + "model.layers.56.mlp.experts.64.down_proj", + "model.layers.56.mlp.experts.64.gate_proj", + "model.layers.56.mlp.experts.64.up_proj", + "model.layers.56.mlp.experts.65.down_proj", + "model.layers.56.mlp.experts.65.gate_proj", + "model.layers.56.mlp.experts.65.up_proj", + "model.layers.56.mlp.experts.66.down_proj", + "model.layers.56.mlp.experts.66.gate_proj", + "model.layers.56.mlp.experts.66.up_proj", + "model.layers.56.mlp.experts.67.down_proj", + "model.layers.56.mlp.experts.67.gate_proj", + "model.layers.56.mlp.experts.67.up_proj", + "model.layers.56.mlp.experts.68.down_proj", + "model.layers.56.mlp.experts.68.gate_proj", + "model.layers.56.mlp.experts.68.up_proj", + "model.layers.56.mlp.experts.69.down_proj", + "model.layers.56.mlp.experts.69.gate_proj", + "model.layers.56.mlp.experts.69.up_proj", + "model.layers.56.mlp.experts.7.down_proj", + "model.layers.56.mlp.experts.7.gate_proj", + "model.layers.56.mlp.experts.7.up_proj", + "model.layers.56.mlp.experts.70.down_proj", + "model.layers.56.mlp.experts.70.gate_proj", + "model.layers.56.mlp.experts.70.up_proj", + "model.layers.56.mlp.experts.71.down_proj", + "model.layers.56.mlp.experts.71.gate_proj", + "model.layers.56.mlp.experts.71.up_proj", + "model.layers.56.mlp.experts.72.down_proj", + "model.layers.56.mlp.experts.72.gate_proj", + "model.layers.56.mlp.experts.72.up_proj", + "model.layers.56.mlp.experts.73.down_proj", + "model.layers.56.mlp.experts.73.gate_proj", + "model.layers.56.mlp.experts.73.up_proj", + "model.layers.56.mlp.experts.74.down_proj", + "model.layers.56.mlp.experts.74.gate_proj", + "model.layers.56.mlp.experts.74.up_proj", + "model.layers.56.mlp.experts.75.down_proj", + "model.layers.56.mlp.experts.75.gate_proj", + "model.layers.56.mlp.experts.75.up_proj", + "model.layers.56.mlp.experts.76.down_proj", + "model.layers.56.mlp.experts.76.gate_proj", + "model.layers.56.mlp.experts.76.up_proj", + "model.layers.56.mlp.experts.77.down_proj", + "model.layers.56.mlp.experts.77.gate_proj", + "model.layers.56.mlp.experts.77.up_proj", + "model.layers.56.mlp.experts.78.down_proj", + "model.layers.56.mlp.experts.78.gate_proj", + "model.layers.56.mlp.experts.78.up_proj", + "model.layers.56.mlp.experts.79.down_proj", + "model.layers.56.mlp.experts.79.gate_proj", + "model.layers.56.mlp.experts.79.up_proj", + "model.layers.56.mlp.experts.8.down_proj", + "model.layers.56.mlp.experts.8.gate_proj", + "model.layers.56.mlp.experts.8.up_proj", + "model.layers.56.mlp.experts.80.down_proj", + "model.layers.56.mlp.experts.80.gate_proj", + "model.layers.56.mlp.experts.80.up_proj", + "model.layers.56.mlp.experts.81.down_proj", + "model.layers.56.mlp.experts.81.gate_proj", + "model.layers.56.mlp.experts.81.up_proj", + "model.layers.56.mlp.experts.82.down_proj", + "model.layers.56.mlp.experts.82.gate_proj", + "model.layers.56.mlp.experts.82.up_proj", + "model.layers.56.mlp.experts.83.down_proj", + "model.layers.56.mlp.experts.83.gate_proj", + "model.layers.56.mlp.experts.83.up_proj", + "model.layers.56.mlp.experts.84.down_proj", + "model.layers.56.mlp.experts.84.gate_proj", + "model.layers.56.mlp.experts.84.up_proj", + "model.layers.56.mlp.experts.85.down_proj", + "model.layers.56.mlp.experts.85.gate_proj", + "model.layers.56.mlp.experts.85.up_proj", + "model.layers.56.mlp.experts.86.down_proj", + "model.layers.56.mlp.experts.86.gate_proj", + "model.layers.56.mlp.experts.86.up_proj", + "model.layers.56.mlp.experts.87.down_proj", + "model.layers.56.mlp.experts.87.gate_proj", + "model.layers.56.mlp.experts.87.up_proj", + "model.layers.56.mlp.experts.88.down_proj", + "model.layers.56.mlp.experts.88.gate_proj", + "model.layers.56.mlp.experts.88.up_proj", + "model.layers.56.mlp.experts.89.down_proj", + "model.layers.56.mlp.experts.89.gate_proj", + "model.layers.56.mlp.experts.89.up_proj", + "model.layers.56.mlp.experts.9.down_proj", + "model.layers.56.mlp.experts.9.gate_proj", + "model.layers.56.mlp.experts.9.up_proj", + "model.layers.56.mlp.experts.90.down_proj", + "model.layers.56.mlp.experts.90.gate_proj", + "model.layers.56.mlp.experts.90.up_proj", + "model.layers.56.mlp.experts.91.down_proj", + "model.layers.56.mlp.experts.91.gate_proj", + "model.layers.56.mlp.experts.91.up_proj", + "model.layers.56.mlp.experts.92.down_proj", + "model.layers.56.mlp.experts.92.gate_proj", + "model.layers.56.mlp.experts.92.up_proj", + "model.layers.56.mlp.experts.93.down_proj", + "model.layers.56.mlp.experts.93.gate_proj", + "model.layers.56.mlp.experts.93.up_proj", + "model.layers.56.mlp.experts.94.down_proj", + "model.layers.56.mlp.experts.94.gate_proj", + "model.layers.56.mlp.experts.94.up_proj", + "model.layers.56.mlp.experts.95.down_proj", + "model.layers.56.mlp.experts.95.gate_proj", + "model.layers.56.mlp.experts.95.up_proj", + "model.layers.56.mlp.experts.96.down_proj", + "model.layers.56.mlp.experts.96.gate_proj", + "model.layers.56.mlp.experts.96.up_proj", + "model.layers.56.mlp.experts.97.down_proj", + "model.layers.56.mlp.experts.97.gate_proj", + "model.layers.56.mlp.experts.97.up_proj", + "model.layers.56.mlp.experts.98.down_proj", + "model.layers.56.mlp.experts.98.gate_proj", + "model.layers.56.mlp.experts.98.up_proj", + "model.layers.56.mlp.experts.99.down_proj", + "model.layers.56.mlp.experts.99.gate_proj", + "model.layers.56.mlp.experts.99.up_proj", + "model.layers.57.mlp.experts.0.down_proj", + "model.layers.57.mlp.experts.0.gate_proj", + "model.layers.57.mlp.experts.0.up_proj", + "model.layers.57.mlp.experts.1.down_proj", + "model.layers.57.mlp.experts.1.gate_proj", + "model.layers.57.mlp.experts.1.up_proj", + "model.layers.57.mlp.experts.10.down_proj", + "model.layers.57.mlp.experts.10.gate_proj", + "model.layers.57.mlp.experts.10.up_proj", + "model.layers.57.mlp.experts.100.down_proj", + "model.layers.57.mlp.experts.100.gate_proj", + "model.layers.57.mlp.experts.100.up_proj", + "model.layers.57.mlp.experts.101.down_proj", + "model.layers.57.mlp.experts.101.gate_proj", + "model.layers.57.mlp.experts.101.up_proj", + "model.layers.57.mlp.experts.102.down_proj", + "model.layers.57.mlp.experts.102.gate_proj", + "model.layers.57.mlp.experts.102.up_proj", + "model.layers.57.mlp.experts.103.down_proj", + "model.layers.57.mlp.experts.103.gate_proj", + "model.layers.57.mlp.experts.103.up_proj", + "model.layers.57.mlp.experts.104.down_proj", + "model.layers.57.mlp.experts.104.gate_proj", + "model.layers.57.mlp.experts.104.up_proj", + "model.layers.57.mlp.experts.105.down_proj", + "model.layers.57.mlp.experts.105.gate_proj", + "model.layers.57.mlp.experts.105.up_proj", + "model.layers.57.mlp.experts.106.down_proj", + "model.layers.57.mlp.experts.106.gate_proj", + "model.layers.57.mlp.experts.106.up_proj", + "model.layers.57.mlp.experts.107.down_proj", + "model.layers.57.mlp.experts.107.gate_proj", + "model.layers.57.mlp.experts.107.up_proj", + "model.layers.57.mlp.experts.108.down_proj", + "model.layers.57.mlp.experts.108.gate_proj", + "model.layers.57.mlp.experts.108.up_proj", + "model.layers.57.mlp.experts.109.down_proj", + "model.layers.57.mlp.experts.109.gate_proj", + "model.layers.57.mlp.experts.109.up_proj", + "model.layers.57.mlp.experts.11.down_proj", + "model.layers.57.mlp.experts.11.gate_proj", + "model.layers.57.mlp.experts.11.up_proj", + "model.layers.57.mlp.experts.110.down_proj", + "model.layers.57.mlp.experts.110.gate_proj", + "model.layers.57.mlp.experts.110.up_proj", + "model.layers.57.mlp.experts.111.down_proj", + "model.layers.57.mlp.experts.111.gate_proj", + "model.layers.57.mlp.experts.111.up_proj", + "model.layers.57.mlp.experts.112.down_proj", + "model.layers.57.mlp.experts.112.gate_proj", + "model.layers.57.mlp.experts.112.up_proj", + "model.layers.57.mlp.experts.113.down_proj", + "model.layers.57.mlp.experts.113.gate_proj", + "model.layers.57.mlp.experts.113.up_proj", + "model.layers.57.mlp.experts.114.down_proj", + "model.layers.57.mlp.experts.114.gate_proj", + "model.layers.57.mlp.experts.114.up_proj", + "model.layers.57.mlp.experts.115.down_proj", + "model.layers.57.mlp.experts.115.gate_proj", + "model.layers.57.mlp.experts.115.up_proj", + "model.layers.57.mlp.experts.116.down_proj", + "model.layers.57.mlp.experts.116.gate_proj", + "model.layers.57.mlp.experts.116.up_proj", + "model.layers.57.mlp.experts.117.down_proj", + "model.layers.57.mlp.experts.117.gate_proj", + "model.layers.57.mlp.experts.117.up_proj", + "model.layers.57.mlp.experts.118.down_proj", + "model.layers.57.mlp.experts.118.gate_proj", + "model.layers.57.mlp.experts.118.up_proj", + "model.layers.57.mlp.experts.119.down_proj", + "model.layers.57.mlp.experts.119.gate_proj", + "model.layers.57.mlp.experts.119.up_proj", + "model.layers.57.mlp.experts.12.down_proj", + "model.layers.57.mlp.experts.12.gate_proj", + "model.layers.57.mlp.experts.12.up_proj", + "model.layers.57.mlp.experts.120.down_proj", + "model.layers.57.mlp.experts.120.gate_proj", + "model.layers.57.mlp.experts.120.up_proj", + "model.layers.57.mlp.experts.121.down_proj", + "model.layers.57.mlp.experts.121.gate_proj", + "model.layers.57.mlp.experts.121.up_proj", + "model.layers.57.mlp.experts.122.down_proj", + "model.layers.57.mlp.experts.122.gate_proj", + "model.layers.57.mlp.experts.122.up_proj", + "model.layers.57.mlp.experts.123.down_proj", + "model.layers.57.mlp.experts.123.gate_proj", + "model.layers.57.mlp.experts.123.up_proj", + "model.layers.57.mlp.experts.124.down_proj", + "model.layers.57.mlp.experts.124.gate_proj", + "model.layers.57.mlp.experts.124.up_proj", + "model.layers.57.mlp.experts.125.down_proj", + "model.layers.57.mlp.experts.125.gate_proj", + "model.layers.57.mlp.experts.125.up_proj", + "model.layers.57.mlp.experts.126.down_proj", + "model.layers.57.mlp.experts.126.gate_proj", + "model.layers.57.mlp.experts.126.up_proj", + "model.layers.57.mlp.experts.127.down_proj", + "model.layers.57.mlp.experts.127.gate_proj", + "model.layers.57.mlp.experts.127.up_proj", + "model.layers.57.mlp.experts.128.down_proj", + "model.layers.57.mlp.experts.128.gate_proj", + "model.layers.57.mlp.experts.128.up_proj", + "model.layers.57.mlp.experts.129.down_proj", + "model.layers.57.mlp.experts.129.gate_proj", + "model.layers.57.mlp.experts.129.up_proj", + "model.layers.57.mlp.experts.13.down_proj", + "model.layers.57.mlp.experts.13.gate_proj", + "model.layers.57.mlp.experts.13.up_proj", + "model.layers.57.mlp.experts.130.down_proj", + "model.layers.57.mlp.experts.130.gate_proj", + "model.layers.57.mlp.experts.130.up_proj", + "model.layers.57.mlp.experts.131.down_proj", + "model.layers.57.mlp.experts.131.gate_proj", + "model.layers.57.mlp.experts.131.up_proj", + "model.layers.57.mlp.experts.132.down_proj", + "model.layers.57.mlp.experts.132.gate_proj", + "model.layers.57.mlp.experts.132.up_proj", + "model.layers.57.mlp.experts.133.down_proj", + "model.layers.57.mlp.experts.133.gate_proj", + "model.layers.57.mlp.experts.133.up_proj", + "model.layers.57.mlp.experts.134.down_proj", + "model.layers.57.mlp.experts.134.gate_proj", + "model.layers.57.mlp.experts.134.up_proj", + "model.layers.57.mlp.experts.135.down_proj", + "model.layers.57.mlp.experts.135.gate_proj", + "model.layers.57.mlp.experts.135.up_proj", + "model.layers.57.mlp.experts.136.down_proj", + "model.layers.57.mlp.experts.136.gate_proj", + "model.layers.57.mlp.experts.136.up_proj", + "model.layers.57.mlp.experts.137.down_proj", + "model.layers.57.mlp.experts.137.gate_proj", + "model.layers.57.mlp.experts.137.up_proj", + "model.layers.57.mlp.experts.138.down_proj", + "model.layers.57.mlp.experts.138.gate_proj", + "model.layers.57.mlp.experts.138.up_proj", + "model.layers.57.mlp.experts.139.down_proj", + "model.layers.57.mlp.experts.139.gate_proj", + "model.layers.57.mlp.experts.139.up_proj", + "model.layers.57.mlp.experts.14.down_proj", + "model.layers.57.mlp.experts.14.gate_proj", + "model.layers.57.mlp.experts.14.up_proj", + "model.layers.57.mlp.experts.140.down_proj", + "model.layers.57.mlp.experts.140.gate_proj", + "model.layers.57.mlp.experts.140.up_proj", + "model.layers.57.mlp.experts.141.down_proj", + "model.layers.57.mlp.experts.141.gate_proj", + "model.layers.57.mlp.experts.141.up_proj", + "model.layers.57.mlp.experts.142.down_proj", + "model.layers.57.mlp.experts.142.gate_proj", + "model.layers.57.mlp.experts.142.up_proj", + "model.layers.57.mlp.experts.143.down_proj", + "model.layers.57.mlp.experts.143.gate_proj", + "model.layers.57.mlp.experts.143.up_proj", + "model.layers.57.mlp.experts.144.down_proj", + "model.layers.57.mlp.experts.144.gate_proj", + "model.layers.57.mlp.experts.144.up_proj", + "model.layers.57.mlp.experts.145.down_proj", + "model.layers.57.mlp.experts.145.gate_proj", + "model.layers.57.mlp.experts.145.up_proj", + "model.layers.57.mlp.experts.146.down_proj", + "model.layers.57.mlp.experts.146.gate_proj", + "model.layers.57.mlp.experts.146.up_proj", + "model.layers.57.mlp.experts.147.down_proj", + "model.layers.57.mlp.experts.147.gate_proj", + "model.layers.57.mlp.experts.147.up_proj", + "model.layers.57.mlp.experts.148.down_proj", + "model.layers.57.mlp.experts.148.gate_proj", + "model.layers.57.mlp.experts.148.up_proj", + "model.layers.57.mlp.experts.149.down_proj", + "model.layers.57.mlp.experts.149.gate_proj", + "model.layers.57.mlp.experts.149.up_proj", + "model.layers.57.mlp.experts.15.down_proj", + "model.layers.57.mlp.experts.15.gate_proj", + "model.layers.57.mlp.experts.15.up_proj", + "model.layers.57.mlp.experts.150.down_proj", + "model.layers.57.mlp.experts.150.gate_proj", + "model.layers.57.mlp.experts.150.up_proj", + "model.layers.57.mlp.experts.151.down_proj", + "model.layers.57.mlp.experts.151.gate_proj", + "model.layers.57.mlp.experts.151.up_proj", + "model.layers.57.mlp.experts.152.down_proj", + "model.layers.57.mlp.experts.152.gate_proj", + "model.layers.57.mlp.experts.152.up_proj", + "model.layers.57.mlp.experts.153.down_proj", + "model.layers.57.mlp.experts.153.gate_proj", + "model.layers.57.mlp.experts.153.up_proj", + "model.layers.57.mlp.experts.154.down_proj", + "model.layers.57.mlp.experts.154.gate_proj", + "model.layers.57.mlp.experts.154.up_proj", + "model.layers.57.mlp.experts.155.down_proj", + "model.layers.57.mlp.experts.155.gate_proj", + "model.layers.57.mlp.experts.155.up_proj", + "model.layers.57.mlp.experts.156.down_proj", + "model.layers.57.mlp.experts.156.gate_proj", + "model.layers.57.mlp.experts.156.up_proj", + "model.layers.57.mlp.experts.157.down_proj", + "model.layers.57.mlp.experts.157.gate_proj", + "model.layers.57.mlp.experts.157.up_proj", + "model.layers.57.mlp.experts.158.down_proj", + "model.layers.57.mlp.experts.158.gate_proj", + "model.layers.57.mlp.experts.158.up_proj", + "model.layers.57.mlp.experts.159.down_proj", + "model.layers.57.mlp.experts.159.gate_proj", + "model.layers.57.mlp.experts.159.up_proj", + "model.layers.57.mlp.experts.16.down_proj", + "model.layers.57.mlp.experts.16.gate_proj", + "model.layers.57.mlp.experts.16.up_proj", + "model.layers.57.mlp.experts.160.down_proj", + "model.layers.57.mlp.experts.160.gate_proj", + "model.layers.57.mlp.experts.160.up_proj", + "model.layers.57.mlp.experts.161.down_proj", + "model.layers.57.mlp.experts.161.gate_proj", + "model.layers.57.mlp.experts.161.up_proj", + "model.layers.57.mlp.experts.162.down_proj", + "model.layers.57.mlp.experts.162.gate_proj", + "model.layers.57.mlp.experts.162.up_proj", + "model.layers.57.mlp.experts.163.down_proj", + "model.layers.57.mlp.experts.163.gate_proj", + "model.layers.57.mlp.experts.163.up_proj", + "model.layers.57.mlp.experts.164.down_proj", + "model.layers.57.mlp.experts.164.gate_proj", + "model.layers.57.mlp.experts.164.up_proj", + "model.layers.57.mlp.experts.165.down_proj", + "model.layers.57.mlp.experts.165.gate_proj", + "model.layers.57.mlp.experts.165.up_proj", + "model.layers.57.mlp.experts.166.down_proj", + "model.layers.57.mlp.experts.166.gate_proj", + "model.layers.57.mlp.experts.166.up_proj", + "model.layers.57.mlp.experts.167.down_proj", + "model.layers.57.mlp.experts.167.gate_proj", + "model.layers.57.mlp.experts.167.up_proj", + "model.layers.57.mlp.experts.168.down_proj", + "model.layers.57.mlp.experts.168.gate_proj", + "model.layers.57.mlp.experts.168.up_proj", + "model.layers.57.mlp.experts.169.down_proj", + "model.layers.57.mlp.experts.169.gate_proj", + "model.layers.57.mlp.experts.169.up_proj", + "model.layers.57.mlp.experts.17.down_proj", + "model.layers.57.mlp.experts.17.gate_proj", + "model.layers.57.mlp.experts.17.up_proj", + "model.layers.57.mlp.experts.170.down_proj", + "model.layers.57.mlp.experts.170.gate_proj", + "model.layers.57.mlp.experts.170.up_proj", + "model.layers.57.mlp.experts.171.down_proj", + "model.layers.57.mlp.experts.171.gate_proj", + "model.layers.57.mlp.experts.171.up_proj", + "model.layers.57.mlp.experts.172.down_proj", + "model.layers.57.mlp.experts.172.gate_proj", + "model.layers.57.mlp.experts.172.up_proj", + "model.layers.57.mlp.experts.173.down_proj", + "model.layers.57.mlp.experts.173.gate_proj", + "model.layers.57.mlp.experts.173.up_proj", + "model.layers.57.mlp.experts.174.down_proj", + "model.layers.57.mlp.experts.174.gate_proj", + "model.layers.57.mlp.experts.174.up_proj", + "model.layers.57.mlp.experts.175.down_proj", + "model.layers.57.mlp.experts.175.gate_proj", + "model.layers.57.mlp.experts.175.up_proj", + "model.layers.57.mlp.experts.176.down_proj", + "model.layers.57.mlp.experts.176.gate_proj", + "model.layers.57.mlp.experts.176.up_proj", + "model.layers.57.mlp.experts.177.down_proj", + "model.layers.57.mlp.experts.177.gate_proj", + "model.layers.57.mlp.experts.177.up_proj", + "model.layers.57.mlp.experts.178.down_proj", + "model.layers.57.mlp.experts.178.gate_proj", + "model.layers.57.mlp.experts.178.up_proj", + "model.layers.57.mlp.experts.179.down_proj", + "model.layers.57.mlp.experts.179.gate_proj", + "model.layers.57.mlp.experts.179.up_proj", + "model.layers.57.mlp.experts.18.down_proj", + "model.layers.57.mlp.experts.18.gate_proj", + "model.layers.57.mlp.experts.18.up_proj", + "model.layers.57.mlp.experts.180.down_proj", + "model.layers.57.mlp.experts.180.gate_proj", + "model.layers.57.mlp.experts.180.up_proj", + "model.layers.57.mlp.experts.181.down_proj", + "model.layers.57.mlp.experts.181.gate_proj", + "model.layers.57.mlp.experts.181.up_proj", + "model.layers.57.mlp.experts.182.down_proj", + "model.layers.57.mlp.experts.182.gate_proj", + "model.layers.57.mlp.experts.182.up_proj", + "model.layers.57.mlp.experts.183.down_proj", + "model.layers.57.mlp.experts.183.gate_proj", + "model.layers.57.mlp.experts.183.up_proj", + "model.layers.57.mlp.experts.184.down_proj", + "model.layers.57.mlp.experts.184.gate_proj", + "model.layers.57.mlp.experts.184.up_proj", + "model.layers.57.mlp.experts.185.down_proj", + "model.layers.57.mlp.experts.185.gate_proj", + "model.layers.57.mlp.experts.185.up_proj", + "model.layers.57.mlp.experts.186.down_proj", + "model.layers.57.mlp.experts.186.gate_proj", + "model.layers.57.mlp.experts.186.up_proj", + "model.layers.57.mlp.experts.187.down_proj", + "model.layers.57.mlp.experts.187.gate_proj", + "model.layers.57.mlp.experts.187.up_proj", + "model.layers.57.mlp.experts.188.down_proj", + "model.layers.57.mlp.experts.188.gate_proj", + "model.layers.57.mlp.experts.188.up_proj", + "model.layers.57.mlp.experts.189.down_proj", + "model.layers.57.mlp.experts.189.gate_proj", + "model.layers.57.mlp.experts.189.up_proj", + "model.layers.57.mlp.experts.19.down_proj", + "model.layers.57.mlp.experts.19.gate_proj", + "model.layers.57.mlp.experts.19.up_proj", + "model.layers.57.mlp.experts.190.down_proj", + "model.layers.57.mlp.experts.190.gate_proj", + "model.layers.57.mlp.experts.190.up_proj", + "model.layers.57.mlp.experts.191.down_proj", + "model.layers.57.mlp.experts.191.gate_proj", + "model.layers.57.mlp.experts.191.up_proj", + "model.layers.57.mlp.experts.192.down_proj", + "model.layers.57.mlp.experts.192.gate_proj", + "model.layers.57.mlp.experts.192.up_proj", + "model.layers.57.mlp.experts.193.down_proj", + "model.layers.57.mlp.experts.193.gate_proj", + "model.layers.57.mlp.experts.193.up_proj", + "model.layers.57.mlp.experts.194.down_proj", + "model.layers.57.mlp.experts.194.gate_proj", + "model.layers.57.mlp.experts.194.up_proj", + "model.layers.57.mlp.experts.195.down_proj", + "model.layers.57.mlp.experts.195.gate_proj", + "model.layers.57.mlp.experts.195.up_proj", + "model.layers.57.mlp.experts.196.down_proj", + "model.layers.57.mlp.experts.196.gate_proj", + "model.layers.57.mlp.experts.196.up_proj", + "model.layers.57.mlp.experts.197.down_proj", + "model.layers.57.mlp.experts.197.gate_proj", + "model.layers.57.mlp.experts.197.up_proj", + "model.layers.57.mlp.experts.198.down_proj", + "model.layers.57.mlp.experts.198.gate_proj", + "model.layers.57.mlp.experts.198.up_proj", + "model.layers.57.mlp.experts.199.down_proj", + "model.layers.57.mlp.experts.199.gate_proj", + "model.layers.57.mlp.experts.199.up_proj", + "model.layers.57.mlp.experts.2.down_proj", + "model.layers.57.mlp.experts.2.gate_proj", + "model.layers.57.mlp.experts.2.up_proj", + "model.layers.57.mlp.experts.20.down_proj", + "model.layers.57.mlp.experts.20.gate_proj", + "model.layers.57.mlp.experts.20.up_proj", + "model.layers.57.mlp.experts.200.down_proj", + "model.layers.57.mlp.experts.200.gate_proj", + "model.layers.57.mlp.experts.200.up_proj", + "model.layers.57.mlp.experts.201.down_proj", + "model.layers.57.mlp.experts.201.gate_proj", + "model.layers.57.mlp.experts.201.up_proj", + "model.layers.57.mlp.experts.202.down_proj", + "model.layers.57.mlp.experts.202.gate_proj", + "model.layers.57.mlp.experts.202.up_proj", + "model.layers.57.mlp.experts.203.down_proj", + "model.layers.57.mlp.experts.203.gate_proj", + "model.layers.57.mlp.experts.203.up_proj", + "model.layers.57.mlp.experts.204.down_proj", + "model.layers.57.mlp.experts.204.gate_proj", + "model.layers.57.mlp.experts.204.up_proj", + "model.layers.57.mlp.experts.205.down_proj", + "model.layers.57.mlp.experts.205.gate_proj", + "model.layers.57.mlp.experts.205.up_proj", + "model.layers.57.mlp.experts.206.down_proj", + "model.layers.57.mlp.experts.206.gate_proj", + "model.layers.57.mlp.experts.206.up_proj", + "model.layers.57.mlp.experts.207.down_proj", + "model.layers.57.mlp.experts.207.gate_proj", + "model.layers.57.mlp.experts.207.up_proj", + "model.layers.57.mlp.experts.208.down_proj", + "model.layers.57.mlp.experts.208.gate_proj", + "model.layers.57.mlp.experts.208.up_proj", + "model.layers.57.mlp.experts.209.down_proj", + "model.layers.57.mlp.experts.209.gate_proj", + "model.layers.57.mlp.experts.209.up_proj", + "model.layers.57.mlp.experts.21.down_proj", + "model.layers.57.mlp.experts.21.gate_proj", + "model.layers.57.mlp.experts.21.up_proj", + "model.layers.57.mlp.experts.210.down_proj", + "model.layers.57.mlp.experts.210.gate_proj", + "model.layers.57.mlp.experts.210.up_proj", + "model.layers.57.mlp.experts.211.down_proj", + "model.layers.57.mlp.experts.211.gate_proj", + "model.layers.57.mlp.experts.211.up_proj", + "model.layers.57.mlp.experts.212.down_proj", + "model.layers.57.mlp.experts.212.gate_proj", + "model.layers.57.mlp.experts.212.up_proj", + "model.layers.57.mlp.experts.213.down_proj", + "model.layers.57.mlp.experts.213.gate_proj", + "model.layers.57.mlp.experts.213.up_proj", + "model.layers.57.mlp.experts.214.down_proj", + "model.layers.57.mlp.experts.214.gate_proj", + "model.layers.57.mlp.experts.214.up_proj", + "model.layers.57.mlp.experts.215.down_proj", + "model.layers.57.mlp.experts.215.gate_proj", + "model.layers.57.mlp.experts.215.up_proj", + "model.layers.57.mlp.experts.216.down_proj", + "model.layers.57.mlp.experts.216.gate_proj", + "model.layers.57.mlp.experts.216.up_proj", + "model.layers.57.mlp.experts.217.down_proj", + "model.layers.57.mlp.experts.217.gate_proj", + "model.layers.57.mlp.experts.217.up_proj", + "model.layers.57.mlp.experts.218.down_proj", + "model.layers.57.mlp.experts.218.gate_proj", + "model.layers.57.mlp.experts.218.up_proj", + "model.layers.57.mlp.experts.219.down_proj", + "model.layers.57.mlp.experts.219.gate_proj", + "model.layers.57.mlp.experts.219.up_proj", + "model.layers.57.mlp.experts.22.down_proj", + "model.layers.57.mlp.experts.22.gate_proj", + "model.layers.57.mlp.experts.22.up_proj", + "model.layers.57.mlp.experts.220.down_proj", + "model.layers.57.mlp.experts.220.gate_proj", + "model.layers.57.mlp.experts.220.up_proj", + "model.layers.57.mlp.experts.221.down_proj", + "model.layers.57.mlp.experts.221.gate_proj", + "model.layers.57.mlp.experts.221.up_proj", + "model.layers.57.mlp.experts.222.down_proj", + "model.layers.57.mlp.experts.222.gate_proj", + "model.layers.57.mlp.experts.222.up_proj", + "model.layers.57.mlp.experts.223.down_proj", + "model.layers.57.mlp.experts.223.gate_proj", + "model.layers.57.mlp.experts.223.up_proj", + "model.layers.57.mlp.experts.224.down_proj", + "model.layers.57.mlp.experts.224.gate_proj", + "model.layers.57.mlp.experts.224.up_proj", + "model.layers.57.mlp.experts.225.down_proj", + "model.layers.57.mlp.experts.225.gate_proj", + "model.layers.57.mlp.experts.225.up_proj", + "model.layers.57.mlp.experts.226.down_proj", + "model.layers.57.mlp.experts.226.gate_proj", + "model.layers.57.mlp.experts.226.up_proj", + "model.layers.57.mlp.experts.227.down_proj", + "model.layers.57.mlp.experts.227.gate_proj", + "model.layers.57.mlp.experts.227.up_proj", + "model.layers.57.mlp.experts.228.down_proj", + "model.layers.57.mlp.experts.228.gate_proj", + "model.layers.57.mlp.experts.228.up_proj", + "model.layers.57.mlp.experts.229.down_proj", + "model.layers.57.mlp.experts.229.gate_proj", + "model.layers.57.mlp.experts.229.up_proj", + "model.layers.57.mlp.experts.23.down_proj", + "model.layers.57.mlp.experts.23.gate_proj", + "model.layers.57.mlp.experts.23.up_proj", + "model.layers.57.mlp.experts.230.down_proj", + "model.layers.57.mlp.experts.230.gate_proj", + "model.layers.57.mlp.experts.230.up_proj", + "model.layers.57.mlp.experts.231.down_proj", + "model.layers.57.mlp.experts.231.gate_proj", + "model.layers.57.mlp.experts.231.up_proj", + "model.layers.57.mlp.experts.232.down_proj", + "model.layers.57.mlp.experts.232.gate_proj", + "model.layers.57.mlp.experts.232.up_proj", + "model.layers.57.mlp.experts.233.down_proj", + "model.layers.57.mlp.experts.233.gate_proj", + "model.layers.57.mlp.experts.233.up_proj", + "model.layers.57.mlp.experts.234.down_proj", + "model.layers.57.mlp.experts.234.gate_proj", + "model.layers.57.mlp.experts.234.up_proj", + "model.layers.57.mlp.experts.235.down_proj", + "model.layers.57.mlp.experts.235.gate_proj", + "model.layers.57.mlp.experts.235.up_proj", + "model.layers.57.mlp.experts.236.down_proj", + "model.layers.57.mlp.experts.236.gate_proj", + "model.layers.57.mlp.experts.236.up_proj", + "model.layers.57.mlp.experts.237.down_proj", + "model.layers.57.mlp.experts.237.gate_proj", + "model.layers.57.mlp.experts.237.up_proj", + "model.layers.57.mlp.experts.238.down_proj", + "model.layers.57.mlp.experts.238.gate_proj", + "model.layers.57.mlp.experts.238.up_proj", + "model.layers.57.mlp.experts.239.down_proj", + "model.layers.57.mlp.experts.239.gate_proj", + "model.layers.57.mlp.experts.239.up_proj", + "model.layers.57.mlp.experts.24.down_proj", + "model.layers.57.mlp.experts.24.gate_proj", + "model.layers.57.mlp.experts.24.up_proj", + "model.layers.57.mlp.experts.240.down_proj", + "model.layers.57.mlp.experts.240.gate_proj", + "model.layers.57.mlp.experts.240.up_proj", + "model.layers.57.mlp.experts.241.down_proj", + "model.layers.57.mlp.experts.241.gate_proj", + "model.layers.57.mlp.experts.241.up_proj", + "model.layers.57.mlp.experts.242.down_proj", + "model.layers.57.mlp.experts.242.gate_proj", + "model.layers.57.mlp.experts.242.up_proj", + "model.layers.57.mlp.experts.243.down_proj", + "model.layers.57.mlp.experts.243.gate_proj", + "model.layers.57.mlp.experts.243.up_proj", + "model.layers.57.mlp.experts.244.down_proj", + "model.layers.57.mlp.experts.244.gate_proj", + "model.layers.57.mlp.experts.244.up_proj", + "model.layers.57.mlp.experts.245.down_proj", + "model.layers.57.mlp.experts.245.gate_proj", + "model.layers.57.mlp.experts.245.up_proj", + "model.layers.57.mlp.experts.246.down_proj", + "model.layers.57.mlp.experts.246.gate_proj", + "model.layers.57.mlp.experts.246.up_proj", + "model.layers.57.mlp.experts.247.down_proj", + "model.layers.57.mlp.experts.247.gate_proj", + "model.layers.57.mlp.experts.247.up_proj", + "model.layers.57.mlp.experts.248.down_proj", + "model.layers.57.mlp.experts.248.gate_proj", + "model.layers.57.mlp.experts.248.up_proj", + "model.layers.57.mlp.experts.249.down_proj", + "model.layers.57.mlp.experts.249.gate_proj", + "model.layers.57.mlp.experts.249.up_proj", + "model.layers.57.mlp.experts.25.down_proj", + "model.layers.57.mlp.experts.25.gate_proj", + "model.layers.57.mlp.experts.25.up_proj", + "model.layers.57.mlp.experts.250.down_proj", + "model.layers.57.mlp.experts.250.gate_proj", + "model.layers.57.mlp.experts.250.up_proj", + "model.layers.57.mlp.experts.251.down_proj", + "model.layers.57.mlp.experts.251.gate_proj", + "model.layers.57.mlp.experts.251.up_proj", + "model.layers.57.mlp.experts.252.down_proj", + "model.layers.57.mlp.experts.252.gate_proj", + "model.layers.57.mlp.experts.252.up_proj", + "model.layers.57.mlp.experts.253.down_proj", + "model.layers.57.mlp.experts.253.gate_proj", + "model.layers.57.mlp.experts.253.up_proj", + "model.layers.57.mlp.experts.254.down_proj", + "model.layers.57.mlp.experts.254.gate_proj", + "model.layers.57.mlp.experts.254.up_proj", + "model.layers.57.mlp.experts.255.down_proj", + "model.layers.57.mlp.experts.255.gate_proj", + "model.layers.57.mlp.experts.255.up_proj", + "model.layers.57.mlp.experts.26.down_proj", + "model.layers.57.mlp.experts.26.gate_proj", + "model.layers.57.mlp.experts.26.up_proj", + "model.layers.57.mlp.experts.27.down_proj", + "model.layers.57.mlp.experts.27.gate_proj", + "model.layers.57.mlp.experts.27.up_proj", + "model.layers.57.mlp.experts.28.down_proj", + "model.layers.57.mlp.experts.28.gate_proj", + "model.layers.57.mlp.experts.28.up_proj", + "model.layers.57.mlp.experts.29.down_proj", + "model.layers.57.mlp.experts.29.gate_proj", + "model.layers.57.mlp.experts.29.up_proj", + "model.layers.57.mlp.experts.3.down_proj", + "model.layers.57.mlp.experts.3.gate_proj", + "model.layers.57.mlp.experts.3.up_proj", + "model.layers.57.mlp.experts.30.down_proj", + "model.layers.57.mlp.experts.30.gate_proj", + "model.layers.57.mlp.experts.30.up_proj", + "model.layers.57.mlp.experts.31.down_proj", + "model.layers.57.mlp.experts.31.gate_proj", + "model.layers.57.mlp.experts.31.up_proj", + "model.layers.57.mlp.experts.32.down_proj", + "model.layers.57.mlp.experts.32.gate_proj", + "model.layers.57.mlp.experts.32.up_proj", + "model.layers.57.mlp.experts.33.down_proj", + "model.layers.57.mlp.experts.33.gate_proj", + "model.layers.57.mlp.experts.33.up_proj", + "model.layers.57.mlp.experts.34.down_proj", + "model.layers.57.mlp.experts.34.gate_proj", + "model.layers.57.mlp.experts.34.up_proj", + "model.layers.57.mlp.experts.35.down_proj", + "model.layers.57.mlp.experts.35.gate_proj", + "model.layers.57.mlp.experts.35.up_proj", + "model.layers.57.mlp.experts.36.down_proj", + "model.layers.57.mlp.experts.36.gate_proj", + "model.layers.57.mlp.experts.36.up_proj", + "model.layers.57.mlp.experts.37.down_proj", + "model.layers.57.mlp.experts.37.gate_proj", + "model.layers.57.mlp.experts.37.up_proj", + "model.layers.57.mlp.experts.38.down_proj", + "model.layers.57.mlp.experts.38.gate_proj", + "model.layers.57.mlp.experts.38.up_proj", + "model.layers.57.mlp.experts.39.down_proj", + "model.layers.57.mlp.experts.39.gate_proj", + "model.layers.57.mlp.experts.39.up_proj", + "model.layers.57.mlp.experts.4.down_proj", + "model.layers.57.mlp.experts.4.gate_proj", + "model.layers.57.mlp.experts.4.up_proj", + "model.layers.57.mlp.experts.40.down_proj", + "model.layers.57.mlp.experts.40.gate_proj", + "model.layers.57.mlp.experts.40.up_proj", + "model.layers.57.mlp.experts.41.down_proj", + "model.layers.57.mlp.experts.41.gate_proj", + "model.layers.57.mlp.experts.41.up_proj", + "model.layers.57.mlp.experts.42.down_proj", + "model.layers.57.mlp.experts.42.gate_proj", + "model.layers.57.mlp.experts.42.up_proj", + "model.layers.57.mlp.experts.43.down_proj", + "model.layers.57.mlp.experts.43.gate_proj", + "model.layers.57.mlp.experts.43.up_proj", + "model.layers.57.mlp.experts.44.down_proj", + "model.layers.57.mlp.experts.44.gate_proj", + "model.layers.57.mlp.experts.44.up_proj", + "model.layers.57.mlp.experts.45.down_proj", + "model.layers.57.mlp.experts.45.gate_proj", + "model.layers.57.mlp.experts.45.up_proj", + "model.layers.57.mlp.experts.46.down_proj", + "model.layers.57.mlp.experts.46.gate_proj", + "model.layers.57.mlp.experts.46.up_proj", + "model.layers.57.mlp.experts.47.down_proj", + "model.layers.57.mlp.experts.47.gate_proj", + "model.layers.57.mlp.experts.47.up_proj", + "model.layers.57.mlp.experts.48.down_proj", + "model.layers.57.mlp.experts.48.gate_proj", + "model.layers.57.mlp.experts.48.up_proj", + "model.layers.57.mlp.experts.49.down_proj", + "model.layers.57.mlp.experts.49.gate_proj", + "model.layers.57.mlp.experts.49.up_proj", + "model.layers.57.mlp.experts.5.down_proj", + "model.layers.57.mlp.experts.5.gate_proj", + "model.layers.57.mlp.experts.5.up_proj", + "model.layers.57.mlp.experts.50.down_proj", + "model.layers.57.mlp.experts.50.gate_proj", + "model.layers.57.mlp.experts.50.up_proj", + "model.layers.57.mlp.experts.51.down_proj", + "model.layers.57.mlp.experts.51.gate_proj", + "model.layers.57.mlp.experts.51.up_proj", + "model.layers.57.mlp.experts.52.down_proj", + "model.layers.57.mlp.experts.52.gate_proj", + "model.layers.57.mlp.experts.52.up_proj", + "model.layers.57.mlp.experts.53.down_proj", + "model.layers.57.mlp.experts.53.gate_proj", + "model.layers.57.mlp.experts.53.up_proj", + "model.layers.57.mlp.experts.54.down_proj", + "model.layers.57.mlp.experts.54.gate_proj", + "model.layers.57.mlp.experts.54.up_proj", + "model.layers.57.mlp.experts.55.down_proj", + "model.layers.57.mlp.experts.55.gate_proj", + "model.layers.57.mlp.experts.55.up_proj", + "model.layers.57.mlp.experts.56.down_proj", + "model.layers.57.mlp.experts.56.gate_proj", + "model.layers.57.mlp.experts.56.up_proj", + "model.layers.57.mlp.experts.57.down_proj", + "model.layers.57.mlp.experts.57.gate_proj", + "model.layers.57.mlp.experts.57.up_proj", + "model.layers.57.mlp.experts.58.down_proj", + "model.layers.57.mlp.experts.58.gate_proj", + "model.layers.57.mlp.experts.58.up_proj", + "model.layers.57.mlp.experts.59.down_proj", + "model.layers.57.mlp.experts.59.gate_proj", + "model.layers.57.mlp.experts.59.up_proj", + "model.layers.57.mlp.experts.6.down_proj", + "model.layers.57.mlp.experts.6.gate_proj", + "model.layers.57.mlp.experts.6.up_proj", + "model.layers.57.mlp.experts.60.down_proj", + "model.layers.57.mlp.experts.60.gate_proj", + "model.layers.57.mlp.experts.60.up_proj", + "model.layers.57.mlp.experts.61.down_proj", + "model.layers.57.mlp.experts.61.gate_proj", + "model.layers.57.mlp.experts.61.up_proj", + "model.layers.57.mlp.experts.62.down_proj", + "model.layers.57.mlp.experts.62.gate_proj", + "model.layers.57.mlp.experts.62.up_proj", + "model.layers.57.mlp.experts.63.down_proj", + "model.layers.57.mlp.experts.63.gate_proj", + "model.layers.57.mlp.experts.63.up_proj", + "model.layers.57.mlp.experts.64.down_proj", + "model.layers.57.mlp.experts.64.gate_proj", + "model.layers.57.mlp.experts.64.up_proj", + "model.layers.57.mlp.experts.65.down_proj", + "model.layers.57.mlp.experts.65.gate_proj", + "model.layers.57.mlp.experts.65.up_proj", + "model.layers.57.mlp.experts.66.down_proj", + "model.layers.57.mlp.experts.66.gate_proj", + "model.layers.57.mlp.experts.66.up_proj", + "model.layers.57.mlp.experts.67.down_proj", + "model.layers.57.mlp.experts.67.gate_proj", + "model.layers.57.mlp.experts.67.up_proj", + "model.layers.57.mlp.experts.68.down_proj", + "model.layers.57.mlp.experts.68.gate_proj", + "model.layers.57.mlp.experts.68.up_proj", + "model.layers.57.mlp.experts.69.down_proj", + "model.layers.57.mlp.experts.69.gate_proj", + "model.layers.57.mlp.experts.69.up_proj", + "model.layers.57.mlp.experts.7.down_proj", + "model.layers.57.mlp.experts.7.gate_proj", + "model.layers.57.mlp.experts.7.up_proj", + "model.layers.57.mlp.experts.70.down_proj", + "model.layers.57.mlp.experts.70.gate_proj", + "model.layers.57.mlp.experts.70.up_proj", + "model.layers.57.mlp.experts.71.down_proj", + "model.layers.57.mlp.experts.71.gate_proj", + "model.layers.57.mlp.experts.71.up_proj", + "model.layers.57.mlp.experts.72.down_proj", + "model.layers.57.mlp.experts.72.gate_proj", + "model.layers.57.mlp.experts.72.up_proj", + "model.layers.57.mlp.experts.73.down_proj", + "model.layers.57.mlp.experts.73.gate_proj", + "model.layers.57.mlp.experts.73.up_proj", + "model.layers.57.mlp.experts.74.down_proj", + "model.layers.57.mlp.experts.74.gate_proj", + "model.layers.57.mlp.experts.74.up_proj", + "model.layers.57.mlp.experts.75.down_proj", + "model.layers.57.mlp.experts.75.gate_proj", + "model.layers.57.mlp.experts.75.up_proj", + "model.layers.57.mlp.experts.76.down_proj", + "model.layers.57.mlp.experts.76.gate_proj", + "model.layers.57.mlp.experts.76.up_proj", + "model.layers.57.mlp.experts.77.down_proj", + "model.layers.57.mlp.experts.77.gate_proj", + "model.layers.57.mlp.experts.77.up_proj", + "model.layers.57.mlp.experts.78.down_proj", + "model.layers.57.mlp.experts.78.gate_proj", + "model.layers.57.mlp.experts.78.up_proj", + "model.layers.57.mlp.experts.79.down_proj", + "model.layers.57.mlp.experts.79.gate_proj", + "model.layers.57.mlp.experts.79.up_proj", + "model.layers.57.mlp.experts.8.down_proj", + "model.layers.57.mlp.experts.8.gate_proj", + "model.layers.57.mlp.experts.8.up_proj", + "model.layers.57.mlp.experts.80.down_proj", + "model.layers.57.mlp.experts.80.gate_proj", + "model.layers.57.mlp.experts.80.up_proj", + "model.layers.57.mlp.experts.81.down_proj", + "model.layers.57.mlp.experts.81.gate_proj", + "model.layers.57.mlp.experts.81.up_proj", + "model.layers.57.mlp.experts.82.down_proj", + "model.layers.57.mlp.experts.82.gate_proj", + "model.layers.57.mlp.experts.82.up_proj", + "model.layers.57.mlp.experts.83.down_proj", + "model.layers.57.mlp.experts.83.gate_proj", + "model.layers.57.mlp.experts.83.up_proj", + "model.layers.57.mlp.experts.84.down_proj", + "model.layers.57.mlp.experts.84.gate_proj", + "model.layers.57.mlp.experts.84.up_proj", + "model.layers.57.mlp.experts.85.down_proj", + "model.layers.57.mlp.experts.85.gate_proj", + "model.layers.57.mlp.experts.85.up_proj", + "model.layers.57.mlp.experts.86.down_proj", + "model.layers.57.mlp.experts.86.gate_proj", + "model.layers.57.mlp.experts.86.up_proj", + "model.layers.57.mlp.experts.87.down_proj", + "model.layers.57.mlp.experts.87.gate_proj", + "model.layers.57.mlp.experts.87.up_proj", + "model.layers.57.mlp.experts.88.down_proj", + "model.layers.57.mlp.experts.88.gate_proj", + "model.layers.57.mlp.experts.88.up_proj", + "model.layers.57.mlp.experts.89.down_proj", + "model.layers.57.mlp.experts.89.gate_proj", + "model.layers.57.mlp.experts.89.up_proj", + "model.layers.57.mlp.experts.9.down_proj", + "model.layers.57.mlp.experts.9.gate_proj", + "model.layers.57.mlp.experts.9.up_proj", + "model.layers.57.mlp.experts.90.down_proj", + "model.layers.57.mlp.experts.90.gate_proj", + "model.layers.57.mlp.experts.90.up_proj", + "model.layers.57.mlp.experts.91.down_proj", + "model.layers.57.mlp.experts.91.gate_proj", + "model.layers.57.mlp.experts.91.up_proj", + "model.layers.57.mlp.experts.92.down_proj", + "model.layers.57.mlp.experts.92.gate_proj", + "model.layers.57.mlp.experts.92.up_proj", + "model.layers.57.mlp.experts.93.down_proj", + "model.layers.57.mlp.experts.93.gate_proj", + "model.layers.57.mlp.experts.93.up_proj", + "model.layers.57.mlp.experts.94.down_proj", + "model.layers.57.mlp.experts.94.gate_proj", + "model.layers.57.mlp.experts.94.up_proj", + "model.layers.57.mlp.experts.95.down_proj", + "model.layers.57.mlp.experts.95.gate_proj", + "model.layers.57.mlp.experts.95.up_proj", + "model.layers.57.mlp.experts.96.down_proj", + "model.layers.57.mlp.experts.96.gate_proj", + "model.layers.57.mlp.experts.96.up_proj", + "model.layers.57.mlp.experts.97.down_proj", + "model.layers.57.mlp.experts.97.gate_proj", + "model.layers.57.mlp.experts.97.up_proj", + "model.layers.57.mlp.experts.98.down_proj", + "model.layers.57.mlp.experts.98.gate_proj", + "model.layers.57.mlp.experts.98.up_proj", + "model.layers.57.mlp.experts.99.down_proj", + "model.layers.57.mlp.experts.99.gate_proj", + "model.layers.57.mlp.experts.99.up_proj", + "model.layers.58.mlp.experts.0.down_proj", + "model.layers.58.mlp.experts.0.gate_proj", + "model.layers.58.mlp.experts.0.up_proj", + "model.layers.58.mlp.experts.1.down_proj", + "model.layers.58.mlp.experts.1.gate_proj", + "model.layers.58.mlp.experts.1.up_proj", + "model.layers.58.mlp.experts.10.down_proj", + "model.layers.58.mlp.experts.10.gate_proj", + "model.layers.58.mlp.experts.10.up_proj", + "model.layers.58.mlp.experts.100.down_proj", + "model.layers.58.mlp.experts.100.gate_proj", + "model.layers.58.mlp.experts.100.up_proj", + "model.layers.58.mlp.experts.101.down_proj", + "model.layers.58.mlp.experts.101.gate_proj", + "model.layers.58.mlp.experts.101.up_proj", + "model.layers.58.mlp.experts.102.down_proj", + "model.layers.58.mlp.experts.102.gate_proj", + "model.layers.58.mlp.experts.102.up_proj", + "model.layers.58.mlp.experts.103.down_proj", + "model.layers.58.mlp.experts.103.gate_proj", + "model.layers.58.mlp.experts.103.up_proj", + "model.layers.58.mlp.experts.104.down_proj", + "model.layers.58.mlp.experts.104.gate_proj", + "model.layers.58.mlp.experts.104.up_proj", + "model.layers.58.mlp.experts.105.down_proj", + "model.layers.58.mlp.experts.105.gate_proj", + "model.layers.58.mlp.experts.105.up_proj", + "model.layers.58.mlp.experts.106.down_proj", + "model.layers.58.mlp.experts.106.gate_proj", + "model.layers.58.mlp.experts.106.up_proj", + "model.layers.58.mlp.experts.107.down_proj", + "model.layers.58.mlp.experts.107.gate_proj", + "model.layers.58.mlp.experts.107.up_proj", + "model.layers.58.mlp.experts.108.down_proj", + "model.layers.58.mlp.experts.108.gate_proj", + "model.layers.58.mlp.experts.108.up_proj", + "model.layers.58.mlp.experts.109.down_proj", + "model.layers.58.mlp.experts.109.gate_proj", + "model.layers.58.mlp.experts.109.up_proj", + "model.layers.58.mlp.experts.11.down_proj", + "model.layers.58.mlp.experts.11.gate_proj", + "model.layers.58.mlp.experts.11.up_proj", + "model.layers.58.mlp.experts.110.down_proj", + "model.layers.58.mlp.experts.110.gate_proj", + "model.layers.58.mlp.experts.110.up_proj", + "model.layers.58.mlp.experts.111.down_proj", + "model.layers.58.mlp.experts.111.gate_proj", + "model.layers.58.mlp.experts.111.up_proj", + "model.layers.58.mlp.experts.112.down_proj", + "model.layers.58.mlp.experts.112.gate_proj", + "model.layers.58.mlp.experts.112.up_proj", + "model.layers.58.mlp.experts.113.down_proj", + "model.layers.58.mlp.experts.113.gate_proj", + "model.layers.58.mlp.experts.113.up_proj", + "model.layers.58.mlp.experts.114.down_proj", + "model.layers.58.mlp.experts.114.gate_proj", + "model.layers.58.mlp.experts.114.up_proj", + "model.layers.58.mlp.experts.115.down_proj", + "model.layers.58.mlp.experts.115.gate_proj", + "model.layers.58.mlp.experts.115.up_proj", + "model.layers.58.mlp.experts.116.down_proj", + "model.layers.58.mlp.experts.116.gate_proj", + "model.layers.58.mlp.experts.116.up_proj", + "model.layers.58.mlp.experts.117.down_proj", + "model.layers.58.mlp.experts.117.gate_proj", + "model.layers.58.mlp.experts.117.up_proj", + "model.layers.58.mlp.experts.118.down_proj", + "model.layers.58.mlp.experts.118.gate_proj", + "model.layers.58.mlp.experts.118.up_proj", + "model.layers.58.mlp.experts.119.down_proj", + "model.layers.58.mlp.experts.119.gate_proj", + "model.layers.58.mlp.experts.119.up_proj", + "model.layers.58.mlp.experts.12.down_proj", + "model.layers.58.mlp.experts.12.gate_proj", + "model.layers.58.mlp.experts.12.up_proj", + "model.layers.58.mlp.experts.120.down_proj", + "model.layers.58.mlp.experts.120.gate_proj", + "model.layers.58.mlp.experts.120.up_proj", + "model.layers.58.mlp.experts.121.down_proj", + "model.layers.58.mlp.experts.121.gate_proj", + "model.layers.58.mlp.experts.121.up_proj", + "model.layers.58.mlp.experts.122.down_proj", + "model.layers.58.mlp.experts.122.gate_proj", + "model.layers.58.mlp.experts.122.up_proj", + "model.layers.58.mlp.experts.123.down_proj", + "model.layers.58.mlp.experts.123.gate_proj", + "model.layers.58.mlp.experts.123.up_proj", + "model.layers.58.mlp.experts.124.down_proj", + "model.layers.58.mlp.experts.124.gate_proj", + "model.layers.58.mlp.experts.124.up_proj", + "model.layers.58.mlp.experts.125.down_proj", + "model.layers.58.mlp.experts.125.gate_proj", + "model.layers.58.mlp.experts.125.up_proj", + "model.layers.58.mlp.experts.126.down_proj", + "model.layers.58.mlp.experts.126.gate_proj", + "model.layers.58.mlp.experts.126.up_proj", + "model.layers.58.mlp.experts.127.down_proj", + "model.layers.58.mlp.experts.127.gate_proj", + "model.layers.58.mlp.experts.127.up_proj", + "model.layers.58.mlp.experts.128.down_proj", + "model.layers.58.mlp.experts.128.gate_proj", + "model.layers.58.mlp.experts.128.up_proj", + "model.layers.58.mlp.experts.129.down_proj", + "model.layers.58.mlp.experts.129.gate_proj", + "model.layers.58.mlp.experts.129.up_proj", + "model.layers.58.mlp.experts.13.down_proj", + "model.layers.58.mlp.experts.13.gate_proj", + "model.layers.58.mlp.experts.13.up_proj", + "model.layers.58.mlp.experts.130.down_proj", + "model.layers.58.mlp.experts.130.gate_proj", + "model.layers.58.mlp.experts.130.up_proj", + "model.layers.58.mlp.experts.131.down_proj", + "model.layers.58.mlp.experts.131.gate_proj", + "model.layers.58.mlp.experts.131.up_proj", + "model.layers.58.mlp.experts.132.down_proj", + "model.layers.58.mlp.experts.132.gate_proj", + "model.layers.58.mlp.experts.132.up_proj", + "model.layers.58.mlp.experts.133.down_proj", + "model.layers.58.mlp.experts.133.gate_proj", + "model.layers.58.mlp.experts.133.up_proj", + "model.layers.58.mlp.experts.134.down_proj", + "model.layers.58.mlp.experts.134.gate_proj", + "model.layers.58.mlp.experts.134.up_proj", + "model.layers.58.mlp.experts.135.down_proj", + "model.layers.58.mlp.experts.135.gate_proj", + "model.layers.58.mlp.experts.135.up_proj", + "model.layers.58.mlp.experts.136.down_proj", + "model.layers.58.mlp.experts.136.gate_proj", + "model.layers.58.mlp.experts.136.up_proj", + "model.layers.58.mlp.experts.137.down_proj", + "model.layers.58.mlp.experts.137.gate_proj", + "model.layers.58.mlp.experts.137.up_proj", + "model.layers.58.mlp.experts.138.down_proj", + "model.layers.58.mlp.experts.138.gate_proj", + "model.layers.58.mlp.experts.138.up_proj", + "model.layers.58.mlp.experts.139.down_proj", + "model.layers.58.mlp.experts.139.gate_proj", + "model.layers.58.mlp.experts.139.up_proj", + "model.layers.58.mlp.experts.14.down_proj", + "model.layers.58.mlp.experts.14.gate_proj", + "model.layers.58.mlp.experts.14.up_proj", + "model.layers.58.mlp.experts.140.down_proj", + "model.layers.58.mlp.experts.140.gate_proj", + "model.layers.58.mlp.experts.140.up_proj", + "model.layers.58.mlp.experts.141.down_proj", + "model.layers.58.mlp.experts.141.gate_proj", + "model.layers.58.mlp.experts.141.up_proj", + "model.layers.58.mlp.experts.142.down_proj", + "model.layers.58.mlp.experts.142.gate_proj", + "model.layers.58.mlp.experts.142.up_proj", + "model.layers.58.mlp.experts.143.down_proj", + "model.layers.58.mlp.experts.143.gate_proj", + "model.layers.58.mlp.experts.143.up_proj", + "model.layers.58.mlp.experts.144.down_proj", + "model.layers.58.mlp.experts.144.gate_proj", + "model.layers.58.mlp.experts.144.up_proj", + "model.layers.58.mlp.experts.145.down_proj", + "model.layers.58.mlp.experts.145.gate_proj", + "model.layers.58.mlp.experts.145.up_proj", + "model.layers.58.mlp.experts.146.down_proj", + "model.layers.58.mlp.experts.146.gate_proj", + "model.layers.58.mlp.experts.146.up_proj", + "model.layers.58.mlp.experts.147.down_proj", + "model.layers.58.mlp.experts.147.gate_proj", + "model.layers.58.mlp.experts.147.up_proj", + "model.layers.58.mlp.experts.148.down_proj", + "model.layers.58.mlp.experts.148.gate_proj", + "model.layers.58.mlp.experts.148.up_proj", + "model.layers.58.mlp.experts.149.down_proj", + "model.layers.58.mlp.experts.149.gate_proj", + "model.layers.58.mlp.experts.149.up_proj", + "model.layers.58.mlp.experts.15.down_proj", + "model.layers.58.mlp.experts.15.gate_proj", + "model.layers.58.mlp.experts.15.up_proj", + "model.layers.58.mlp.experts.150.down_proj", + "model.layers.58.mlp.experts.150.gate_proj", + "model.layers.58.mlp.experts.150.up_proj", + "model.layers.58.mlp.experts.151.down_proj", + "model.layers.58.mlp.experts.151.gate_proj", + "model.layers.58.mlp.experts.151.up_proj", + "model.layers.58.mlp.experts.152.down_proj", + "model.layers.58.mlp.experts.152.gate_proj", + "model.layers.58.mlp.experts.152.up_proj", + "model.layers.58.mlp.experts.153.down_proj", + "model.layers.58.mlp.experts.153.gate_proj", + "model.layers.58.mlp.experts.153.up_proj", + "model.layers.58.mlp.experts.154.down_proj", + "model.layers.58.mlp.experts.154.gate_proj", + "model.layers.58.mlp.experts.154.up_proj", + "model.layers.58.mlp.experts.155.down_proj", + "model.layers.58.mlp.experts.155.gate_proj", + "model.layers.58.mlp.experts.155.up_proj", + "model.layers.58.mlp.experts.156.down_proj", + "model.layers.58.mlp.experts.156.gate_proj", + "model.layers.58.mlp.experts.156.up_proj", + "model.layers.58.mlp.experts.157.down_proj", + "model.layers.58.mlp.experts.157.gate_proj", + "model.layers.58.mlp.experts.157.up_proj", + "model.layers.58.mlp.experts.158.down_proj", + "model.layers.58.mlp.experts.158.gate_proj", + "model.layers.58.mlp.experts.158.up_proj", + "model.layers.58.mlp.experts.159.down_proj", + "model.layers.58.mlp.experts.159.gate_proj", + "model.layers.58.mlp.experts.159.up_proj", + "model.layers.58.mlp.experts.16.down_proj", + "model.layers.58.mlp.experts.16.gate_proj", + "model.layers.58.mlp.experts.16.up_proj", + "model.layers.58.mlp.experts.160.down_proj", + "model.layers.58.mlp.experts.160.gate_proj", + "model.layers.58.mlp.experts.160.up_proj", + "model.layers.58.mlp.experts.161.down_proj", + "model.layers.58.mlp.experts.161.gate_proj", + "model.layers.58.mlp.experts.161.up_proj", + "model.layers.58.mlp.experts.162.down_proj", + "model.layers.58.mlp.experts.162.gate_proj", + "model.layers.58.mlp.experts.162.up_proj", + "model.layers.58.mlp.experts.163.down_proj", + "model.layers.58.mlp.experts.163.gate_proj", + "model.layers.58.mlp.experts.163.up_proj", + "model.layers.58.mlp.experts.164.down_proj", + "model.layers.58.mlp.experts.164.gate_proj", + "model.layers.58.mlp.experts.164.up_proj", + "model.layers.58.mlp.experts.165.down_proj", + "model.layers.58.mlp.experts.165.gate_proj", + "model.layers.58.mlp.experts.165.up_proj", + "model.layers.58.mlp.experts.166.down_proj", + "model.layers.58.mlp.experts.166.gate_proj", + "model.layers.58.mlp.experts.166.up_proj", + "model.layers.58.mlp.experts.167.down_proj", + "model.layers.58.mlp.experts.167.gate_proj", + "model.layers.58.mlp.experts.167.up_proj", + "model.layers.58.mlp.experts.168.down_proj", + "model.layers.58.mlp.experts.168.gate_proj", + "model.layers.58.mlp.experts.168.up_proj", + "model.layers.58.mlp.experts.169.down_proj", + "model.layers.58.mlp.experts.169.gate_proj", + "model.layers.58.mlp.experts.169.up_proj", + "model.layers.58.mlp.experts.17.down_proj", + "model.layers.58.mlp.experts.17.gate_proj", + "model.layers.58.mlp.experts.17.up_proj", + "model.layers.58.mlp.experts.170.down_proj", + "model.layers.58.mlp.experts.170.gate_proj", + "model.layers.58.mlp.experts.170.up_proj", + "model.layers.58.mlp.experts.171.down_proj", + "model.layers.58.mlp.experts.171.gate_proj", + "model.layers.58.mlp.experts.171.up_proj", + "model.layers.58.mlp.experts.172.down_proj", + "model.layers.58.mlp.experts.172.gate_proj", + "model.layers.58.mlp.experts.172.up_proj", + "model.layers.58.mlp.experts.173.down_proj", + "model.layers.58.mlp.experts.173.gate_proj", + "model.layers.58.mlp.experts.173.up_proj", + "model.layers.58.mlp.experts.174.down_proj", + "model.layers.58.mlp.experts.174.gate_proj", + "model.layers.58.mlp.experts.174.up_proj", + "model.layers.58.mlp.experts.175.down_proj", + "model.layers.58.mlp.experts.175.gate_proj", + "model.layers.58.mlp.experts.175.up_proj", + "model.layers.58.mlp.experts.176.down_proj", + "model.layers.58.mlp.experts.176.gate_proj", + "model.layers.58.mlp.experts.176.up_proj", + "model.layers.58.mlp.experts.177.down_proj", + "model.layers.58.mlp.experts.177.gate_proj", + "model.layers.58.mlp.experts.177.up_proj", + "model.layers.58.mlp.experts.178.down_proj", + "model.layers.58.mlp.experts.178.gate_proj", + "model.layers.58.mlp.experts.178.up_proj", + "model.layers.58.mlp.experts.179.down_proj", + "model.layers.58.mlp.experts.179.gate_proj", + "model.layers.58.mlp.experts.179.up_proj", + "model.layers.58.mlp.experts.18.down_proj", + "model.layers.58.mlp.experts.18.gate_proj", + "model.layers.58.mlp.experts.18.up_proj", + "model.layers.58.mlp.experts.180.down_proj", + "model.layers.58.mlp.experts.180.gate_proj", + "model.layers.58.mlp.experts.180.up_proj", + "model.layers.58.mlp.experts.181.down_proj", + "model.layers.58.mlp.experts.181.gate_proj", + "model.layers.58.mlp.experts.181.up_proj", + "model.layers.58.mlp.experts.182.down_proj", + "model.layers.58.mlp.experts.182.gate_proj", + "model.layers.58.mlp.experts.182.up_proj", + "model.layers.58.mlp.experts.183.down_proj", + "model.layers.58.mlp.experts.183.gate_proj", + "model.layers.58.mlp.experts.183.up_proj", + "model.layers.58.mlp.experts.184.down_proj", + "model.layers.58.mlp.experts.184.gate_proj", + "model.layers.58.mlp.experts.184.up_proj", + "model.layers.58.mlp.experts.185.down_proj", + "model.layers.58.mlp.experts.185.gate_proj", + "model.layers.58.mlp.experts.185.up_proj", + "model.layers.58.mlp.experts.186.down_proj", + "model.layers.58.mlp.experts.186.gate_proj", + "model.layers.58.mlp.experts.186.up_proj", + "model.layers.58.mlp.experts.187.down_proj", + "model.layers.58.mlp.experts.187.gate_proj", + "model.layers.58.mlp.experts.187.up_proj", + "model.layers.58.mlp.experts.188.down_proj", + "model.layers.58.mlp.experts.188.gate_proj", + "model.layers.58.mlp.experts.188.up_proj", + "model.layers.58.mlp.experts.189.down_proj", + "model.layers.58.mlp.experts.189.gate_proj", + "model.layers.58.mlp.experts.189.up_proj", + "model.layers.58.mlp.experts.19.down_proj", + "model.layers.58.mlp.experts.19.gate_proj", + "model.layers.58.mlp.experts.19.up_proj", + "model.layers.58.mlp.experts.190.down_proj", + "model.layers.58.mlp.experts.190.gate_proj", + "model.layers.58.mlp.experts.190.up_proj", + "model.layers.58.mlp.experts.191.down_proj", + "model.layers.58.mlp.experts.191.gate_proj", + "model.layers.58.mlp.experts.191.up_proj", + "model.layers.58.mlp.experts.192.down_proj", + "model.layers.58.mlp.experts.192.gate_proj", + "model.layers.58.mlp.experts.192.up_proj", + "model.layers.58.mlp.experts.193.down_proj", + "model.layers.58.mlp.experts.193.gate_proj", + "model.layers.58.mlp.experts.193.up_proj", + "model.layers.58.mlp.experts.194.down_proj", + "model.layers.58.mlp.experts.194.gate_proj", + "model.layers.58.mlp.experts.194.up_proj", + "model.layers.58.mlp.experts.195.down_proj", + "model.layers.58.mlp.experts.195.gate_proj", + "model.layers.58.mlp.experts.195.up_proj", + "model.layers.58.mlp.experts.196.down_proj", + "model.layers.58.mlp.experts.196.gate_proj", + "model.layers.58.mlp.experts.196.up_proj", + "model.layers.58.mlp.experts.197.down_proj", + "model.layers.58.mlp.experts.197.gate_proj", + "model.layers.58.mlp.experts.197.up_proj", + "model.layers.58.mlp.experts.198.down_proj", + "model.layers.58.mlp.experts.198.gate_proj", + "model.layers.58.mlp.experts.198.up_proj", + "model.layers.58.mlp.experts.199.down_proj", + "model.layers.58.mlp.experts.199.gate_proj", + "model.layers.58.mlp.experts.199.up_proj", + "model.layers.58.mlp.experts.2.down_proj", + "model.layers.58.mlp.experts.2.gate_proj", + "model.layers.58.mlp.experts.2.up_proj", + "model.layers.58.mlp.experts.20.down_proj", + "model.layers.58.mlp.experts.20.gate_proj", + "model.layers.58.mlp.experts.20.up_proj", + "model.layers.58.mlp.experts.200.down_proj", + "model.layers.58.mlp.experts.200.gate_proj", + "model.layers.58.mlp.experts.200.up_proj", + "model.layers.58.mlp.experts.201.down_proj", + "model.layers.58.mlp.experts.201.gate_proj", + "model.layers.58.mlp.experts.201.up_proj", + "model.layers.58.mlp.experts.202.down_proj", + "model.layers.58.mlp.experts.202.gate_proj", + "model.layers.58.mlp.experts.202.up_proj", + "model.layers.58.mlp.experts.203.down_proj", + "model.layers.58.mlp.experts.203.gate_proj", + "model.layers.58.mlp.experts.203.up_proj", + "model.layers.58.mlp.experts.204.down_proj", + "model.layers.58.mlp.experts.204.gate_proj", + "model.layers.58.mlp.experts.204.up_proj", + "model.layers.58.mlp.experts.205.down_proj", + "model.layers.58.mlp.experts.205.gate_proj", + "model.layers.58.mlp.experts.205.up_proj", + "model.layers.58.mlp.experts.206.down_proj", + "model.layers.58.mlp.experts.206.gate_proj", + "model.layers.58.mlp.experts.206.up_proj", + "model.layers.58.mlp.experts.207.down_proj", + "model.layers.58.mlp.experts.207.gate_proj", + "model.layers.58.mlp.experts.207.up_proj", + "model.layers.58.mlp.experts.208.down_proj", + "model.layers.58.mlp.experts.208.gate_proj", + "model.layers.58.mlp.experts.208.up_proj", + "model.layers.58.mlp.experts.209.down_proj", + "model.layers.58.mlp.experts.209.gate_proj", + "model.layers.58.mlp.experts.209.up_proj", + "model.layers.58.mlp.experts.21.down_proj", + "model.layers.58.mlp.experts.21.gate_proj", + "model.layers.58.mlp.experts.21.up_proj", + "model.layers.58.mlp.experts.210.down_proj", + "model.layers.58.mlp.experts.210.gate_proj", + "model.layers.58.mlp.experts.210.up_proj", + "model.layers.58.mlp.experts.211.down_proj", + "model.layers.58.mlp.experts.211.gate_proj", + "model.layers.58.mlp.experts.211.up_proj", + "model.layers.58.mlp.experts.212.down_proj", + "model.layers.58.mlp.experts.212.gate_proj", + "model.layers.58.mlp.experts.212.up_proj", + "model.layers.58.mlp.experts.213.down_proj", + "model.layers.58.mlp.experts.213.gate_proj", + "model.layers.58.mlp.experts.213.up_proj", + "model.layers.58.mlp.experts.214.down_proj", + "model.layers.58.mlp.experts.214.gate_proj", + "model.layers.58.mlp.experts.214.up_proj", + "model.layers.58.mlp.experts.215.down_proj", + "model.layers.58.mlp.experts.215.gate_proj", + "model.layers.58.mlp.experts.215.up_proj", + "model.layers.58.mlp.experts.216.down_proj", + "model.layers.58.mlp.experts.216.gate_proj", + "model.layers.58.mlp.experts.216.up_proj", + "model.layers.58.mlp.experts.217.down_proj", + "model.layers.58.mlp.experts.217.gate_proj", + "model.layers.58.mlp.experts.217.up_proj", + "model.layers.58.mlp.experts.218.down_proj", + "model.layers.58.mlp.experts.218.gate_proj", + "model.layers.58.mlp.experts.218.up_proj", + "model.layers.58.mlp.experts.219.down_proj", + "model.layers.58.mlp.experts.219.gate_proj", + "model.layers.58.mlp.experts.219.up_proj", + "model.layers.58.mlp.experts.22.down_proj", + "model.layers.58.mlp.experts.22.gate_proj", + "model.layers.58.mlp.experts.22.up_proj", + "model.layers.58.mlp.experts.220.down_proj", + "model.layers.58.mlp.experts.220.gate_proj", + "model.layers.58.mlp.experts.220.up_proj", + "model.layers.58.mlp.experts.221.down_proj", + "model.layers.58.mlp.experts.221.gate_proj", + "model.layers.58.mlp.experts.221.up_proj", + "model.layers.58.mlp.experts.222.down_proj", + "model.layers.58.mlp.experts.222.gate_proj", + "model.layers.58.mlp.experts.222.up_proj", + "model.layers.58.mlp.experts.223.down_proj", + "model.layers.58.mlp.experts.223.gate_proj", + "model.layers.58.mlp.experts.223.up_proj", + "model.layers.58.mlp.experts.224.down_proj", + "model.layers.58.mlp.experts.224.gate_proj", + "model.layers.58.mlp.experts.224.up_proj", + "model.layers.58.mlp.experts.225.down_proj", + "model.layers.58.mlp.experts.225.gate_proj", + "model.layers.58.mlp.experts.225.up_proj", + "model.layers.58.mlp.experts.226.down_proj", + "model.layers.58.mlp.experts.226.gate_proj", + "model.layers.58.mlp.experts.226.up_proj", + "model.layers.58.mlp.experts.227.down_proj", + "model.layers.58.mlp.experts.227.gate_proj", + "model.layers.58.mlp.experts.227.up_proj", + "model.layers.58.mlp.experts.228.down_proj", + "model.layers.58.mlp.experts.228.gate_proj", + "model.layers.58.mlp.experts.228.up_proj", + "model.layers.58.mlp.experts.229.down_proj", + "model.layers.58.mlp.experts.229.gate_proj", + "model.layers.58.mlp.experts.229.up_proj", + "model.layers.58.mlp.experts.23.down_proj", + "model.layers.58.mlp.experts.23.gate_proj", + "model.layers.58.mlp.experts.23.up_proj", + "model.layers.58.mlp.experts.230.down_proj", + "model.layers.58.mlp.experts.230.gate_proj", + "model.layers.58.mlp.experts.230.up_proj", + "model.layers.58.mlp.experts.231.down_proj", + "model.layers.58.mlp.experts.231.gate_proj", + "model.layers.58.mlp.experts.231.up_proj", + "model.layers.58.mlp.experts.232.down_proj", + "model.layers.58.mlp.experts.232.gate_proj", + "model.layers.58.mlp.experts.232.up_proj", + "model.layers.58.mlp.experts.233.down_proj", + "model.layers.58.mlp.experts.233.gate_proj", + "model.layers.58.mlp.experts.233.up_proj", + "model.layers.58.mlp.experts.234.down_proj", + "model.layers.58.mlp.experts.234.gate_proj", + "model.layers.58.mlp.experts.234.up_proj", + "model.layers.58.mlp.experts.235.down_proj", + "model.layers.58.mlp.experts.235.gate_proj", + "model.layers.58.mlp.experts.235.up_proj", + "model.layers.58.mlp.experts.236.down_proj", + "model.layers.58.mlp.experts.236.gate_proj", + "model.layers.58.mlp.experts.236.up_proj", + "model.layers.58.mlp.experts.237.down_proj", + "model.layers.58.mlp.experts.237.gate_proj", + "model.layers.58.mlp.experts.237.up_proj", + "model.layers.58.mlp.experts.238.down_proj", + "model.layers.58.mlp.experts.238.gate_proj", + "model.layers.58.mlp.experts.238.up_proj", + "model.layers.58.mlp.experts.239.down_proj", + "model.layers.58.mlp.experts.239.gate_proj", + "model.layers.58.mlp.experts.239.up_proj", + "model.layers.58.mlp.experts.24.down_proj", + "model.layers.58.mlp.experts.24.gate_proj", + "model.layers.58.mlp.experts.24.up_proj", + "model.layers.58.mlp.experts.240.down_proj", + "model.layers.58.mlp.experts.240.gate_proj", + "model.layers.58.mlp.experts.240.up_proj", + "model.layers.58.mlp.experts.241.down_proj", + "model.layers.58.mlp.experts.241.gate_proj", + "model.layers.58.mlp.experts.241.up_proj", + "model.layers.58.mlp.experts.242.down_proj", + "model.layers.58.mlp.experts.242.gate_proj", + "model.layers.58.mlp.experts.242.up_proj", + "model.layers.58.mlp.experts.243.down_proj", + "model.layers.58.mlp.experts.243.gate_proj", + "model.layers.58.mlp.experts.243.up_proj", + "model.layers.58.mlp.experts.244.down_proj", + "model.layers.58.mlp.experts.244.gate_proj", + "model.layers.58.mlp.experts.244.up_proj", + "model.layers.58.mlp.experts.245.down_proj", + "model.layers.58.mlp.experts.245.gate_proj", + "model.layers.58.mlp.experts.245.up_proj", + "model.layers.58.mlp.experts.246.down_proj", + "model.layers.58.mlp.experts.246.gate_proj", + "model.layers.58.mlp.experts.246.up_proj", + "model.layers.58.mlp.experts.247.down_proj", + "model.layers.58.mlp.experts.247.gate_proj", + "model.layers.58.mlp.experts.247.up_proj", + "model.layers.58.mlp.experts.248.down_proj", + "model.layers.58.mlp.experts.248.gate_proj", + "model.layers.58.mlp.experts.248.up_proj", + "model.layers.58.mlp.experts.249.down_proj", + "model.layers.58.mlp.experts.249.gate_proj", + "model.layers.58.mlp.experts.249.up_proj", + "model.layers.58.mlp.experts.25.down_proj", + "model.layers.58.mlp.experts.25.gate_proj", + "model.layers.58.mlp.experts.25.up_proj", + "model.layers.58.mlp.experts.250.down_proj", + "model.layers.58.mlp.experts.250.gate_proj", + "model.layers.58.mlp.experts.250.up_proj", + "model.layers.58.mlp.experts.251.down_proj", + "model.layers.58.mlp.experts.251.gate_proj", + "model.layers.58.mlp.experts.251.up_proj", + "model.layers.58.mlp.experts.252.down_proj", + "model.layers.58.mlp.experts.252.gate_proj", + "model.layers.58.mlp.experts.252.up_proj", + "model.layers.58.mlp.experts.253.down_proj", + "model.layers.58.mlp.experts.253.gate_proj", + "model.layers.58.mlp.experts.253.up_proj", + "model.layers.58.mlp.experts.254.down_proj", + "model.layers.58.mlp.experts.254.gate_proj", + "model.layers.58.mlp.experts.254.up_proj", + "model.layers.58.mlp.experts.255.down_proj", + "model.layers.58.mlp.experts.255.gate_proj", + "model.layers.58.mlp.experts.255.up_proj", + "model.layers.58.mlp.experts.26.down_proj", + "model.layers.58.mlp.experts.26.gate_proj", + "model.layers.58.mlp.experts.26.up_proj", + "model.layers.58.mlp.experts.27.down_proj", + "model.layers.58.mlp.experts.27.gate_proj", + "model.layers.58.mlp.experts.27.up_proj", + "model.layers.58.mlp.experts.28.down_proj", + "model.layers.58.mlp.experts.28.gate_proj", + "model.layers.58.mlp.experts.28.up_proj", + "model.layers.58.mlp.experts.29.down_proj", + "model.layers.58.mlp.experts.29.gate_proj", + "model.layers.58.mlp.experts.29.up_proj", + "model.layers.58.mlp.experts.3.down_proj", + "model.layers.58.mlp.experts.3.gate_proj", + "model.layers.58.mlp.experts.3.up_proj", + "model.layers.58.mlp.experts.30.down_proj", + "model.layers.58.mlp.experts.30.gate_proj", + "model.layers.58.mlp.experts.30.up_proj", + "model.layers.58.mlp.experts.31.down_proj", + "model.layers.58.mlp.experts.31.gate_proj", + "model.layers.58.mlp.experts.31.up_proj", + "model.layers.58.mlp.experts.32.down_proj", + "model.layers.58.mlp.experts.32.gate_proj", + "model.layers.58.mlp.experts.32.up_proj", + "model.layers.58.mlp.experts.33.down_proj", + "model.layers.58.mlp.experts.33.gate_proj", + "model.layers.58.mlp.experts.33.up_proj", + "model.layers.58.mlp.experts.34.down_proj", + "model.layers.58.mlp.experts.34.gate_proj", + "model.layers.58.mlp.experts.34.up_proj", + "model.layers.58.mlp.experts.35.down_proj", + "model.layers.58.mlp.experts.35.gate_proj", + "model.layers.58.mlp.experts.35.up_proj", + "model.layers.58.mlp.experts.36.down_proj", + "model.layers.58.mlp.experts.36.gate_proj", + "model.layers.58.mlp.experts.36.up_proj", + "model.layers.58.mlp.experts.37.down_proj", + "model.layers.58.mlp.experts.37.gate_proj", + "model.layers.58.mlp.experts.37.up_proj", + "model.layers.58.mlp.experts.38.down_proj", + "model.layers.58.mlp.experts.38.gate_proj", + "model.layers.58.mlp.experts.38.up_proj", + "model.layers.58.mlp.experts.39.down_proj", + "model.layers.58.mlp.experts.39.gate_proj", + "model.layers.58.mlp.experts.39.up_proj", + "model.layers.58.mlp.experts.4.down_proj", + "model.layers.58.mlp.experts.4.gate_proj", + "model.layers.58.mlp.experts.4.up_proj", + "model.layers.58.mlp.experts.40.down_proj", + "model.layers.58.mlp.experts.40.gate_proj", + "model.layers.58.mlp.experts.40.up_proj", + "model.layers.58.mlp.experts.41.down_proj", + "model.layers.58.mlp.experts.41.gate_proj", + "model.layers.58.mlp.experts.41.up_proj", + "model.layers.58.mlp.experts.42.down_proj", + "model.layers.58.mlp.experts.42.gate_proj", + "model.layers.58.mlp.experts.42.up_proj", + "model.layers.58.mlp.experts.43.down_proj", + "model.layers.58.mlp.experts.43.gate_proj", + "model.layers.58.mlp.experts.43.up_proj", + "model.layers.58.mlp.experts.44.down_proj", + "model.layers.58.mlp.experts.44.gate_proj", + "model.layers.58.mlp.experts.44.up_proj", + "model.layers.58.mlp.experts.45.down_proj", + "model.layers.58.mlp.experts.45.gate_proj", + "model.layers.58.mlp.experts.45.up_proj", + "model.layers.58.mlp.experts.46.down_proj", + "model.layers.58.mlp.experts.46.gate_proj", + "model.layers.58.mlp.experts.46.up_proj", + "model.layers.58.mlp.experts.47.down_proj", + "model.layers.58.mlp.experts.47.gate_proj", + "model.layers.58.mlp.experts.47.up_proj", + "model.layers.58.mlp.experts.48.down_proj", + "model.layers.58.mlp.experts.48.gate_proj", + "model.layers.58.mlp.experts.48.up_proj", + "model.layers.58.mlp.experts.49.down_proj", + "model.layers.58.mlp.experts.49.gate_proj", + "model.layers.58.mlp.experts.49.up_proj", + "model.layers.58.mlp.experts.5.down_proj", + "model.layers.58.mlp.experts.5.gate_proj", + "model.layers.58.mlp.experts.5.up_proj", + "model.layers.58.mlp.experts.50.down_proj", + "model.layers.58.mlp.experts.50.gate_proj", + "model.layers.58.mlp.experts.50.up_proj", + "model.layers.58.mlp.experts.51.down_proj", + "model.layers.58.mlp.experts.51.gate_proj", + "model.layers.58.mlp.experts.51.up_proj", + "model.layers.58.mlp.experts.52.down_proj", + "model.layers.58.mlp.experts.52.gate_proj", + "model.layers.58.mlp.experts.52.up_proj", + "model.layers.58.mlp.experts.53.down_proj", + "model.layers.58.mlp.experts.53.gate_proj", + "model.layers.58.mlp.experts.53.up_proj", + "model.layers.58.mlp.experts.54.down_proj", + "model.layers.58.mlp.experts.54.gate_proj", + "model.layers.58.mlp.experts.54.up_proj", + "model.layers.58.mlp.experts.55.down_proj", + "model.layers.58.mlp.experts.55.gate_proj", + "model.layers.58.mlp.experts.55.up_proj", + "model.layers.58.mlp.experts.56.down_proj", + "model.layers.58.mlp.experts.56.gate_proj", + "model.layers.58.mlp.experts.56.up_proj", + "model.layers.58.mlp.experts.57.down_proj", + "model.layers.58.mlp.experts.57.gate_proj", + "model.layers.58.mlp.experts.57.up_proj", + "model.layers.58.mlp.experts.58.down_proj", + "model.layers.58.mlp.experts.58.gate_proj", + "model.layers.58.mlp.experts.58.up_proj", + "model.layers.58.mlp.experts.59.down_proj", + "model.layers.58.mlp.experts.59.gate_proj", + "model.layers.58.mlp.experts.59.up_proj", + "model.layers.58.mlp.experts.6.down_proj", + "model.layers.58.mlp.experts.6.gate_proj", + "model.layers.58.mlp.experts.6.up_proj", + "model.layers.58.mlp.experts.60.down_proj", + "model.layers.58.mlp.experts.60.gate_proj", + "model.layers.58.mlp.experts.60.up_proj", + "model.layers.58.mlp.experts.61.down_proj", + "model.layers.58.mlp.experts.61.gate_proj", + "model.layers.58.mlp.experts.61.up_proj", + "model.layers.58.mlp.experts.62.down_proj", + "model.layers.58.mlp.experts.62.gate_proj", + "model.layers.58.mlp.experts.62.up_proj", + "model.layers.58.mlp.experts.63.down_proj", + "model.layers.58.mlp.experts.63.gate_proj", + "model.layers.58.mlp.experts.63.up_proj", + "model.layers.58.mlp.experts.64.down_proj", + "model.layers.58.mlp.experts.64.gate_proj", + "model.layers.58.mlp.experts.64.up_proj", + "model.layers.58.mlp.experts.65.down_proj", + "model.layers.58.mlp.experts.65.gate_proj", + "model.layers.58.mlp.experts.65.up_proj", + "model.layers.58.mlp.experts.66.down_proj", + "model.layers.58.mlp.experts.66.gate_proj", + "model.layers.58.mlp.experts.66.up_proj", + "model.layers.58.mlp.experts.67.down_proj", + "model.layers.58.mlp.experts.67.gate_proj", + "model.layers.58.mlp.experts.67.up_proj", + "model.layers.58.mlp.experts.68.down_proj", + "model.layers.58.mlp.experts.68.gate_proj", + "model.layers.58.mlp.experts.68.up_proj", + "model.layers.58.mlp.experts.69.down_proj", + "model.layers.58.mlp.experts.69.gate_proj", + "model.layers.58.mlp.experts.69.up_proj", + "model.layers.58.mlp.experts.7.down_proj", + "model.layers.58.mlp.experts.7.gate_proj", + "model.layers.58.mlp.experts.7.up_proj", + "model.layers.58.mlp.experts.70.down_proj", + "model.layers.58.mlp.experts.70.gate_proj", + "model.layers.58.mlp.experts.70.up_proj", + "model.layers.58.mlp.experts.71.down_proj", + "model.layers.58.mlp.experts.71.gate_proj", + "model.layers.58.mlp.experts.71.up_proj", + "model.layers.58.mlp.experts.72.down_proj", + "model.layers.58.mlp.experts.72.gate_proj", + "model.layers.58.mlp.experts.72.up_proj", + "model.layers.58.mlp.experts.73.down_proj", + "model.layers.58.mlp.experts.73.gate_proj", + "model.layers.58.mlp.experts.73.up_proj", + "model.layers.58.mlp.experts.74.down_proj", + "model.layers.58.mlp.experts.74.gate_proj", + "model.layers.58.mlp.experts.74.up_proj", + "model.layers.58.mlp.experts.75.down_proj", + "model.layers.58.mlp.experts.75.gate_proj", + "model.layers.58.mlp.experts.75.up_proj", + "model.layers.58.mlp.experts.76.down_proj", + "model.layers.58.mlp.experts.76.gate_proj", + "model.layers.58.mlp.experts.76.up_proj", + "model.layers.58.mlp.experts.77.down_proj", + "model.layers.58.mlp.experts.77.gate_proj", + "model.layers.58.mlp.experts.77.up_proj", + "model.layers.58.mlp.experts.78.down_proj", + "model.layers.58.mlp.experts.78.gate_proj", + "model.layers.58.mlp.experts.78.up_proj", + "model.layers.58.mlp.experts.79.down_proj", + "model.layers.58.mlp.experts.79.gate_proj", + "model.layers.58.mlp.experts.79.up_proj", + "model.layers.58.mlp.experts.8.down_proj", + "model.layers.58.mlp.experts.8.gate_proj", + "model.layers.58.mlp.experts.8.up_proj", + "model.layers.58.mlp.experts.80.down_proj", + "model.layers.58.mlp.experts.80.gate_proj", + "model.layers.58.mlp.experts.80.up_proj", + "model.layers.58.mlp.experts.81.down_proj", + "model.layers.58.mlp.experts.81.gate_proj", + "model.layers.58.mlp.experts.81.up_proj", + "model.layers.58.mlp.experts.82.down_proj", + "model.layers.58.mlp.experts.82.gate_proj", + "model.layers.58.mlp.experts.82.up_proj", + "model.layers.58.mlp.experts.83.down_proj", + "model.layers.58.mlp.experts.83.gate_proj", + "model.layers.58.mlp.experts.83.up_proj", + "model.layers.58.mlp.experts.84.down_proj", + "model.layers.58.mlp.experts.84.gate_proj", + "model.layers.58.mlp.experts.84.up_proj", + "model.layers.58.mlp.experts.85.down_proj", + "model.layers.58.mlp.experts.85.gate_proj", + "model.layers.58.mlp.experts.85.up_proj", + "model.layers.58.mlp.experts.86.down_proj", + "model.layers.58.mlp.experts.86.gate_proj", + "model.layers.58.mlp.experts.86.up_proj", + "model.layers.58.mlp.experts.87.down_proj", + "model.layers.58.mlp.experts.87.gate_proj", + "model.layers.58.mlp.experts.87.up_proj", + "model.layers.58.mlp.experts.88.down_proj", + "model.layers.58.mlp.experts.88.gate_proj", + "model.layers.58.mlp.experts.88.up_proj", + "model.layers.58.mlp.experts.89.down_proj", + "model.layers.58.mlp.experts.89.gate_proj", + "model.layers.58.mlp.experts.89.up_proj", + "model.layers.58.mlp.experts.9.down_proj", + "model.layers.58.mlp.experts.9.gate_proj", + "model.layers.58.mlp.experts.9.up_proj", + "model.layers.58.mlp.experts.90.down_proj", + "model.layers.58.mlp.experts.90.gate_proj", + "model.layers.58.mlp.experts.90.up_proj", + "model.layers.58.mlp.experts.91.down_proj", + "model.layers.58.mlp.experts.91.gate_proj", + "model.layers.58.mlp.experts.91.up_proj", + "model.layers.58.mlp.experts.92.down_proj", + "model.layers.58.mlp.experts.92.gate_proj", + "model.layers.58.mlp.experts.92.up_proj", + "model.layers.58.mlp.experts.93.down_proj", + "model.layers.58.mlp.experts.93.gate_proj", + "model.layers.58.mlp.experts.93.up_proj", + "model.layers.58.mlp.experts.94.down_proj", + "model.layers.58.mlp.experts.94.gate_proj", + "model.layers.58.mlp.experts.94.up_proj", + "model.layers.58.mlp.experts.95.down_proj", + "model.layers.58.mlp.experts.95.gate_proj", + "model.layers.58.mlp.experts.95.up_proj", + "model.layers.58.mlp.experts.96.down_proj", + "model.layers.58.mlp.experts.96.gate_proj", + "model.layers.58.mlp.experts.96.up_proj", + "model.layers.58.mlp.experts.97.down_proj", + "model.layers.58.mlp.experts.97.gate_proj", + "model.layers.58.mlp.experts.97.up_proj", + "model.layers.58.mlp.experts.98.down_proj", + "model.layers.58.mlp.experts.98.gate_proj", + "model.layers.58.mlp.experts.98.up_proj", + "model.layers.58.mlp.experts.99.down_proj", + "model.layers.58.mlp.experts.99.gate_proj", + "model.layers.58.mlp.experts.99.up_proj", + "model.layers.59.mlp.experts.0.down_proj", + "model.layers.59.mlp.experts.0.gate_proj", + "model.layers.59.mlp.experts.0.up_proj", + "model.layers.59.mlp.experts.1.down_proj", + "model.layers.59.mlp.experts.1.gate_proj", + "model.layers.59.mlp.experts.1.up_proj", + "model.layers.59.mlp.experts.10.down_proj", + "model.layers.59.mlp.experts.10.gate_proj", + "model.layers.59.mlp.experts.10.up_proj", + "model.layers.59.mlp.experts.100.down_proj", + "model.layers.59.mlp.experts.100.gate_proj", + "model.layers.59.mlp.experts.100.up_proj", + "model.layers.59.mlp.experts.101.down_proj", + "model.layers.59.mlp.experts.101.gate_proj", + "model.layers.59.mlp.experts.101.up_proj", + "model.layers.59.mlp.experts.102.down_proj", + "model.layers.59.mlp.experts.102.gate_proj", + "model.layers.59.mlp.experts.102.up_proj", + "model.layers.59.mlp.experts.103.down_proj", + "model.layers.59.mlp.experts.103.gate_proj", + "model.layers.59.mlp.experts.103.up_proj", + "model.layers.59.mlp.experts.104.down_proj", + "model.layers.59.mlp.experts.104.gate_proj", + "model.layers.59.mlp.experts.104.up_proj", + "model.layers.59.mlp.experts.105.down_proj", + "model.layers.59.mlp.experts.105.gate_proj", + "model.layers.59.mlp.experts.105.up_proj", + "model.layers.59.mlp.experts.106.down_proj", + "model.layers.59.mlp.experts.106.gate_proj", + "model.layers.59.mlp.experts.106.up_proj", + "model.layers.59.mlp.experts.107.down_proj", + "model.layers.59.mlp.experts.107.gate_proj", + "model.layers.59.mlp.experts.107.up_proj", + "model.layers.59.mlp.experts.108.down_proj", + "model.layers.59.mlp.experts.108.gate_proj", + "model.layers.59.mlp.experts.108.up_proj", + "model.layers.59.mlp.experts.109.down_proj", + "model.layers.59.mlp.experts.109.gate_proj", + "model.layers.59.mlp.experts.109.up_proj", + "model.layers.59.mlp.experts.11.down_proj", + "model.layers.59.mlp.experts.11.gate_proj", + "model.layers.59.mlp.experts.11.up_proj", + "model.layers.59.mlp.experts.110.down_proj", + "model.layers.59.mlp.experts.110.gate_proj", + "model.layers.59.mlp.experts.110.up_proj", + "model.layers.59.mlp.experts.111.down_proj", + "model.layers.59.mlp.experts.111.gate_proj", + "model.layers.59.mlp.experts.111.up_proj", + "model.layers.59.mlp.experts.112.down_proj", + "model.layers.59.mlp.experts.112.gate_proj", + "model.layers.59.mlp.experts.112.up_proj", + "model.layers.59.mlp.experts.113.down_proj", + "model.layers.59.mlp.experts.113.gate_proj", + "model.layers.59.mlp.experts.113.up_proj", + "model.layers.59.mlp.experts.114.down_proj", + "model.layers.59.mlp.experts.114.gate_proj", + "model.layers.59.mlp.experts.114.up_proj", + "model.layers.59.mlp.experts.115.down_proj", + "model.layers.59.mlp.experts.115.gate_proj", + "model.layers.59.mlp.experts.115.up_proj", + "model.layers.59.mlp.experts.116.down_proj", + "model.layers.59.mlp.experts.116.gate_proj", + "model.layers.59.mlp.experts.116.up_proj", + "model.layers.59.mlp.experts.117.down_proj", + "model.layers.59.mlp.experts.117.gate_proj", + "model.layers.59.mlp.experts.117.up_proj", + "model.layers.59.mlp.experts.118.down_proj", + "model.layers.59.mlp.experts.118.gate_proj", + "model.layers.59.mlp.experts.118.up_proj", + "model.layers.59.mlp.experts.119.down_proj", + "model.layers.59.mlp.experts.119.gate_proj", + "model.layers.59.mlp.experts.119.up_proj", + "model.layers.59.mlp.experts.12.down_proj", + "model.layers.59.mlp.experts.12.gate_proj", + "model.layers.59.mlp.experts.12.up_proj", + "model.layers.59.mlp.experts.120.down_proj", + "model.layers.59.mlp.experts.120.gate_proj", + "model.layers.59.mlp.experts.120.up_proj", + "model.layers.59.mlp.experts.121.down_proj", + "model.layers.59.mlp.experts.121.gate_proj", + "model.layers.59.mlp.experts.121.up_proj", + "model.layers.59.mlp.experts.122.down_proj", + "model.layers.59.mlp.experts.122.gate_proj", + "model.layers.59.mlp.experts.122.up_proj", + "model.layers.59.mlp.experts.123.down_proj", + "model.layers.59.mlp.experts.123.gate_proj", + "model.layers.59.mlp.experts.123.up_proj", + "model.layers.59.mlp.experts.124.down_proj", + "model.layers.59.mlp.experts.124.gate_proj", + "model.layers.59.mlp.experts.124.up_proj", + "model.layers.59.mlp.experts.125.down_proj", + "model.layers.59.mlp.experts.125.gate_proj", + "model.layers.59.mlp.experts.125.up_proj", + "model.layers.59.mlp.experts.126.down_proj", + "model.layers.59.mlp.experts.126.gate_proj", + "model.layers.59.mlp.experts.126.up_proj", + "model.layers.59.mlp.experts.127.down_proj", + "model.layers.59.mlp.experts.127.gate_proj", + "model.layers.59.mlp.experts.127.up_proj", + "model.layers.59.mlp.experts.128.down_proj", + "model.layers.59.mlp.experts.128.gate_proj", + "model.layers.59.mlp.experts.128.up_proj", + "model.layers.59.mlp.experts.129.down_proj", + "model.layers.59.mlp.experts.129.gate_proj", + "model.layers.59.mlp.experts.129.up_proj", + "model.layers.59.mlp.experts.13.down_proj", + "model.layers.59.mlp.experts.13.gate_proj", + "model.layers.59.mlp.experts.13.up_proj", + "model.layers.59.mlp.experts.130.down_proj", + "model.layers.59.mlp.experts.130.gate_proj", + "model.layers.59.mlp.experts.130.up_proj", + "model.layers.59.mlp.experts.131.down_proj", + "model.layers.59.mlp.experts.131.gate_proj", + "model.layers.59.mlp.experts.131.up_proj", + "model.layers.59.mlp.experts.132.down_proj", + "model.layers.59.mlp.experts.132.gate_proj", + "model.layers.59.mlp.experts.132.up_proj", + "model.layers.59.mlp.experts.133.down_proj", + "model.layers.59.mlp.experts.133.gate_proj", + "model.layers.59.mlp.experts.133.up_proj", + "model.layers.59.mlp.experts.134.down_proj", + "model.layers.59.mlp.experts.134.gate_proj", + "model.layers.59.mlp.experts.134.up_proj", + "model.layers.59.mlp.experts.135.down_proj", + "model.layers.59.mlp.experts.135.gate_proj", + "model.layers.59.mlp.experts.135.up_proj", + "model.layers.59.mlp.experts.136.down_proj", + "model.layers.59.mlp.experts.136.gate_proj", + "model.layers.59.mlp.experts.136.up_proj", + "model.layers.59.mlp.experts.137.down_proj", + "model.layers.59.mlp.experts.137.gate_proj", + "model.layers.59.mlp.experts.137.up_proj", + "model.layers.59.mlp.experts.138.down_proj", + "model.layers.59.mlp.experts.138.gate_proj", + "model.layers.59.mlp.experts.138.up_proj", + "model.layers.59.mlp.experts.139.down_proj", + "model.layers.59.mlp.experts.139.gate_proj", + "model.layers.59.mlp.experts.139.up_proj", + "model.layers.59.mlp.experts.14.down_proj", + "model.layers.59.mlp.experts.14.gate_proj", + "model.layers.59.mlp.experts.14.up_proj", + "model.layers.59.mlp.experts.140.down_proj", + "model.layers.59.mlp.experts.140.gate_proj", + "model.layers.59.mlp.experts.140.up_proj", + "model.layers.59.mlp.experts.141.down_proj", + "model.layers.59.mlp.experts.141.gate_proj", + "model.layers.59.mlp.experts.141.up_proj", + "model.layers.59.mlp.experts.142.down_proj", + "model.layers.59.mlp.experts.142.gate_proj", + "model.layers.59.mlp.experts.142.up_proj", + "model.layers.59.mlp.experts.143.down_proj", + "model.layers.59.mlp.experts.143.gate_proj", + "model.layers.59.mlp.experts.143.up_proj", + "model.layers.59.mlp.experts.144.down_proj", + "model.layers.59.mlp.experts.144.gate_proj", + "model.layers.59.mlp.experts.144.up_proj", + "model.layers.59.mlp.experts.145.down_proj", + "model.layers.59.mlp.experts.145.gate_proj", + "model.layers.59.mlp.experts.145.up_proj", + "model.layers.59.mlp.experts.146.down_proj", + "model.layers.59.mlp.experts.146.gate_proj", + "model.layers.59.mlp.experts.146.up_proj", + "model.layers.59.mlp.experts.147.down_proj", + "model.layers.59.mlp.experts.147.gate_proj", + "model.layers.59.mlp.experts.147.up_proj", + "model.layers.59.mlp.experts.148.down_proj", + "model.layers.59.mlp.experts.148.gate_proj", + "model.layers.59.mlp.experts.148.up_proj", + "model.layers.59.mlp.experts.149.down_proj", + "model.layers.59.mlp.experts.149.gate_proj", + "model.layers.59.mlp.experts.149.up_proj", + "model.layers.59.mlp.experts.15.down_proj", + "model.layers.59.mlp.experts.15.gate_proj", + "model.layers.59.mlp.experts.15.up_proj", + "model.layers.59.mlp.experts.150.down_proj", + "model.layers.59.mlp.experts.150.gate_proj", + "model.layers.59.mlp.experts.150.up_proj", + "model.layers.59.mlp.experts.151.down_proj", + "model.layers.59.mlp.experts.151.gate_proj", + "model.layers.59.mlp.experts.151.up_proj", + "model.layers.59.mlp.experts.152.down_proj", + "model.layers.59.mlp.experts.152.gate_proj", + "model.layers.59.mlp.experts.152.up_proj", + "model.layers.59.mlp.experts.153.down_proj", + "model.layers.59.mlp.experts.153.gate_proj", + "model.layers.59.mlp.experts.153.up_proj", + "model.layers.59.mlp.experts.154.down_proj", + "model.layers.59.mlp.experts.154.gate_proj", + "model.layers.59.mlp.experts.154.up_proj", + "model.layers.59.mlp.experts.155.down_proj", + "model.layers.59.mlp.experts.155.gate_proj", + "model.layers.59.mlp.experts.155.up_proj", + "model.layers.59.mlp.experts.156.down_proj", + "model.layers.59.mlp.experts.156.gate_proj", + "model.layers.59.mlp.experts.156.up_proj", + "model.layers.59.mlp.experts.157.down_proj", + "model.layers.59.mlp.experts.157.gate_proj", + "model.layers.59.mlp.experts.157.up_proj", + "model.layers.59.mlp.experts.158.down_proj", + "model.layers.59.mlp.experts.158.gate_proj", + "model.layers.59.mlp.experts.158.up_proj", + "model.layers.59.mlp.experts.159.down_proj", + "model.layers.59.mlp.experts.159.gate_proj", + "model.layers.59.mlp.experts.159.up_proj", + "model.layers.59.mlp.experts.16.down_proj", + "model.layers.59.mlp.experts.16.gate_proj", + "model.layers.59.mlp.experts.16.up_proj", + "model.layers.59.mlp.experts.160.down_proj", + "model.layers.59.mlp.experts.160.gate_proj", + "model.layers.59.mlp.experts.160.up_proj", + "model.layers.59.mlp.experts.161.down_proj", + "model.layers.59.mlp.experts.161.gate_proj", + "model.layers.59.mlp.experts.161.up_proj", + "model.layers.59.mlp.experts.162.down_proj", + "model.layers.59.mlp.experts.162.gate_proj", + "model.layers.59.mlp.experts.162.up_proj", + "model.layers.59.mlp.experts.163.down_proj", + "model.layers.59.mlp.experts.163.gate_proj", + "model.layers.59.mlp.experts.163.up_proj", + "model.layers.59.mlp.experts.164.down_proj", + "model.layers.59.mlp.experts.164.gate_proj", + "model.layers.59.mlp.experts.164.up_proj", + "model.layers.59.mlp.experts.165.down_proj", + "model.layers.59.mlp.experts.165.gate_proj", + "model.layers.59.mlp.experts.165.up_proj", + "model.layers.59.mlp.experts.166.down_proj", + "model.layers.59.mlp.experts.166.gate_proj", + "model.layers.59.mlp.experts.166.up_proj", + "model.layers.59.mlp.experts.167.down_proj", + "model.layers.59.mlp.experts.167.gate_proj", + "model.layers.59.mlp.experts.167.up_proj", + "model.layers.59.mlp.experts.168.down_proj", + "model.layers.59.mlp.experts.168.gate_proj", + "model.layers.59.mlp.experts.168.up_proj", + "model.layers.59.mlp.experts.169.down_proj", + "model.layers.59.mlp.experts.169.gate_proj", + "model.layers.59.mlp.experts.169.up_proj", + "model.layers.59.mlp.experts.17.down_proj", + "model.layers.59.mlp.experts.17.gate_proj", + "model.layers.59.mlp.experts.17.up_proj", + "model.layers.59.mlp.experts.170.down_proj", + "model.layers.59.mlp.experts.170.gate_proj", + "model.layers.59.mlp.experts.170.up_proj", + "model.layers.59.mlp.experts.171.down_proj", + "model.layers.59.mlp.experts.171.gate_proj", + "model.layers.59.mlp.experts.171.up_proj", + "model.layers.59.mlp.experts.172.down_proj", + "model.layers.59.mlp.experts.172.gate_proj", + "model.layers.59.mlp.experts.172.up_proj", + "model.layers.59.mlp.experts.173.down_proj", + "model.layers.59.mlp.experts.173.gate_proj", + "model.layers.59.mlp.experts.173.up_proj", + "model.layers.59.mlp.experts.174.down_proj", + "model.layers.59.mlp.experts.174.gate_proj", + "model.layers.59.mlp.experts.174.up_proj", + "model.layers.59.mlp.experts.175.down_proj", + "model.layers.59.mlp.experts.175.gate_proj", + "model.layers.59.mlp.experts.175.up_proj", + "model.layers.59.mlp.experts.176.down_proj", + "model.layers.59.mlp.experts.176.gate_proj", + "model.layers.59.mlp.experts.176.up_proj", + "model.layers.59.mlp.experts.177.down_proj", + "model.layers.59.mlp.experts.177.gate_proj", + "model.layers.59.mlp.experts.177.up_proj", + "model.layers.59.mlp.experts.178.down_proj", + "model.layers.59.mlp.experts.178.gate_proj", + "model.layers.59.mlp.experts.178.up_proj", + "model.layers.59.mlp.experts.179.down_proj", + "model.layers.59.mlp.experts.179.gate_proj", + "model.layers.59.mlp.experts.179.up_proj", + "model.layers.59.mlp.experts.18.down_proj", + "model.layers.59.mlp.experts.18.gate_proj", + "model.layers.59.mlp.experts.18.up_proj", + "model.layers.59.mlp.experts.180.down_proj", + "model.layers.59.mlp.experts.180.gate_proj", + "model.layers.59.mlp.experts.180.up_proj", + "model.layers.59.mlp.experts.181.down_proj", + "model.layers.59.mlp.experts.181.gate_proj", + "model.layers.59.mlp.experts.181.up_proj", + "model.layers.59.mlp.experts.182.down_proj", + "model.layers.59.mlp.experts.182.gate_proj", + "model.layers.59.mlp.experts.182.up_proj", + "model.layers.59.mlp.experts.183.down_proj", + "model.layers.59.mlp.experts.183.gate_proj", + "model.layers.59.mlp.experts.183.up_proj", + "model.layers.59.mlp.experts.184.down_proj", + "model.layers.59.mlp.experts.184.gate_proj", + "model.layers.59.mlp.experts.184.up_proj", + "model.layers.59.mlp.experts.185.down_proj", + "model.layers.59.mlp.experts.185.gate_proj", + "model.layers.59.mlp.experts.185.up_proj", + "model.layers.59.mlp.experts.186.down_proj", + "model.layers.59.mlp.experts.186.gate_proj", + "model.layers.59.mlp.experts.186.up_proj", + "model.layers.59.mlp.experts.187.down_proj", + "model.layers.59.mlp.experts.187.gate_proj", + "model.layers.59.mlp.experts.187.up_proj", + "model.layers.59.mlp.experts.188.down_proj", + "model.layers.59.mlp.experts.188.gate_proj", + "model.layers.59.mlp.experts.188.up_proj", + "model.layers.59.mlp.experts.189.down_proj", + "model.layers.59.mlp.experts.189.gate_proj", + "model.layers.59.mlp.experts.189.up_proj", + "model.layers.59.mlp.experts.19.down_proj", + "model.layers.59.mlp.experts.19.gate_proj", + "model.layers.59.mlp.experts.19.up_proj", + "model.layers.59.mlp.experts.190.down_proj", + "model.layers.59.mlp.experts.190.gate_proj", + "model.layers.59.mlp.experts.190.up_proj", + "model.layers.59.mlp.experts.191.down_proj", + "model.layers.59.mlp.experts.191.gate_proj", + "model.layers.59.mlp.experts.191.up_proj", + "model.layers.59.mlp.experts.192.down_proj", + "model.layers.59.mlp.experts.192.gate_proj", + "model.layers.59.mlp.experts.192.up_proj", + "model.layers.59.mlp.experts.193.down_proj", + "model.layers.59.mlp.experts.193.gate_proj", + "model.layers.59.mlp.experts.193.up_proj", + "model.layers.59.mlp.experts.194.down_proj", + "model.layers.59.mlp.experts.194.gate_proj", + "model.layers.59.mlp.experts.194.up_proj", + "model.layers.59.mlp.experts.195.down_proj", + "model.layers.59.mlp.experts.195.gate_proj", + "model.layers.59.mlp.experts.195.up_proj", + "model.layers.59.mlp.experts.196.down_proj", + "model.layers.59.mlp.experts.196.gate_proj", + "model.layers.59.mlp.experts.196.up_proj", + "model.layers.59.mlp.experts.197.down_proj", + "model.layers.59.mlp.experts.197.gate_proj", + "model.layers.59.mlp.experts.197.up_proj", + "model.layers.59.mlp.experts.198.down_proj", + "model.layers.59.mlp.experts.198.gate_proj", + "model.layers.59.mlp.experts.198.up_proj", + "model.layers.59.mlp.experts.199.down_proj", + "model.layers.59.mlp.experts.199.gate_proj", + "model.layers.59.mlp.experts.199.up_proj", + "model.layers.59.mlp.experts.2.down_proj", + "model.layers.59.mlp.experts.2.gate_proj", + "model.layers.59.mlp.experts.2.up_proj", + "model.layers.59.mlp.experts.20.down_proj", + "model.layers.59.mlp.experts.20.gate_proj", + "model.layers.59.mlp.experts.20.up_proj", + "model.layers.59.mlp.experts.200.down_proj", + "model.layers.59.mlp.experts.200.gate_proj", + "model.layers.59.mlp.experts.200.up_proj", + "model.layers.59.mlp.experts.201.down_proj", + "model.layers.59.mlp.experts.201.gate_proj", + "model.layers.59.mlp.experts.201.up_proj", + "model.layers.59.mlp.experts.202.down_proj", + "model.layers.59.mlp.experts.202.gate_proj", + "model.layers.59.mlp.experts.202.up_proj", + "model.layers.59.mlp.experts.203.down_proj", + "model.layers.59.mlp.experts.203.gate_proj", + "model.layers.59.mlp.experts.203.up_proj", + "model.layers.59.mlp.experts.204.down_proj", + "model.layers.59.mlp.experts.204.gate_proj", + "model.layers.59.mlp.experts.204.up_proj", + "model.layers.59.mlp.experts.205.down_proj", + "model.layers.59.mlp.experts.205.gate_proj", + "model.layers.59.mlp.experts.205.up_proj", + "model.layers.59.mlp.experts.206.down_proj", + "model.layers.59.mlp.experts.206.gate_proj", + "model.layers.59.mlp.experts.206.up_proj", + "model.layers.59.mlp.experts.207.down_proj", + "model.layers.59.mlp.experts.207.gate_proj", + "model.layers.59.mlp.experts.207.up_proj", + "model.layers.59.mlp.experts.208.down_proj", + "model.layers.59.mlp.experts.208.gate_proj", + "model.layers.59.mlp.experts.208.up_proj", + "model.layers.59.mlp.experts.209.down_proj", + "model.layers.59.mlp.experts.209.gate_proj", + "model.layers.59.mlp.experts.209.up_proj", + "model.layers.59.mlp.experts.21.down_proj", + "model.layers.59.mlp.experts.21.gate_proj", + "model.layers.59.mlp.experts.21.up_proj", + "model.layers.59.mlp.experts.210.down_proj", + "model.layers.59.mlp.experts.210.gate_proj", + "model.layers.59.mlp.experts.210.up_proj", + "model.layers.59.mlp.experts.211.down_proj", + "model.layers.59.mlp.experts.211.gate_proj", + "model.layers.59.mlp.experts.211.up_proj", + "model.layers.59.mlp.experts.212.down_proj", + "model.layers.59.mlp.experts.212.gate_proj", + "model.layers.59.mlp.experts.212.up_proj", + "model.layers.59.mlp.experts.213.down_proj", + "model.layers.59.mlp.experts.213.gate_proj", + "model.layers.59.mlp.experts.213.up_proj", + "model.layers.59.mlp.experts.214.down_proj", + "model.layers.59.mlp.experts.214.gate_proj", + "model.layers.59.mlp.experts.214.up_proj", + "model.layers.59.mlp.experts.215.down_proj", + "model.layers.59.mlp.experts.215.gate_proj", + "model.layers.59.mlp.experts.215.up_proj", + "model.layers.59.mlp.experts.216.down_proj", + "model.layers.59.mlp.experts.216.gate_proj", + "model.layers.59.mlp.experts.216.up_proj", + "model.layers.59.mlp.experts.217.down_proj", + "model.layers.59.mlp.experts.217.gate_proj", + "model.layers.59.mlp.experts.217.up_proj", + "model.layers.59.mlp.experts.218.down_proj", + "model.layers.59.mlp.experts.218.gate_proj", + "model.layers.59.mlp.experts.218.up_proj", + "model.layers.59.mlp.experts.219.down_proj", + "model.layers.59.mlp.experts.219.gate_proj", + "model.layers.59.mlp.experts.219.up_proj", + "model.layers.59.mlp.experts.22.down_proj", + "model.layers.59.mlp.experts.22.gate_proj", + "model.layers.59.mlp.experts.22.up_proj", + "model.layers.59.mlp.experts.220.down_proj", + "model.layers.59.mlp.experts.220.gate_proj", + "model.layers.59.mlp.experts.220.up_proj", + "model.layers.59.mlp.experts.221.down_proj", + "model.layers.59.mlp.experts.221.gate_proj", + "model.layers.59.mlp.experts.221.up_proj", + "model.layers.59.mlp.experts.222.down_proj", + "model.layers.59.mlp.experts.222.gate_proj", + "model.layers.59.mlp.experts.222.up_proj", + "model.layers.59.mlp.experts.223.down_proj", + "model.layers.59.mlp.experts.223.gate_proj", + "model.layers.59.mlp.experts.223.up_proj", + "model.layers.59.mlp.experts.224.down_proj", + "model.layers.59.mlp.experts.224.gate_proj", + "model.layers.59.mlp.experts.224.up_proj", + "model.layers.59.mlp.experts.225.down_proj", + "model.layers.59.mlp.experts.225.gate_proj", + "model.layers.59.mlp.experts.225.up_proj", + "model.layers.59.mlp.experts.226.down_proj", + "model.layers.59.mlp.experts.226.gate_proj", + "model.layers.59.mlp.experts.226.up_proj", + "model.layers.59.mlp.experts.227.down_proj", + "model.layers.59.mlp.experts.227.gate_proj", + "model.layers.59.mlp.experts.227.up_proj", + "model.layers.59.mlp.experts.228.down_proj", + "model.layers.59.mlp.experts.228.gate_proj", + "model.layers.59.mlp.experts.228.up_proj", + "model.layers.59.mlp.experts.229.down_proj", + "model.layers.59.mlp.experts.229.gate_proj", + "model.layers.59.mlp.experts.229.up_proj", + "model.layers.59.mlp.experts.23.down_proj", + "model.layers.59.mlp.experts.23.gate_proj", + "model.layers.59.mlp.experts.23.up_proj", + "model.layers.59.mlp.experts.230.down_proj", + "model.layers.59.mlp.experts.230.gate_proj", + "model.layers.59.mlp.experts.230.up_proj", + "model.layers.59.mlp.experts.231.down_proj", + "model.layers.59.mlp.experts.231.gate_proj", + "model.layers.59.mlp.experts.231.up_proj", + "model.layers.59.mlp.experts.232.down_proj", + "model.layers.59.mlp.experts.232.gate_proj", + "model.layers.59.mlp.experts.232.up_proj", + "model.layers.59.mlp.experts.233.down_proj", + "model.layers.59.mlp.experts.233.gate_proj", + "model.layers.59.mlp.experts.233.up_proj", + "model.layers.59.mlp.experts.234.down_proj", + "model.layers.59.mlp.experts.234.gate_proj", + "model.layers.59.mlp.experts.234.up_proj", + "model.layers.59.mlp.experts.235.down_proj", + "model.layers.59.mlp.experts.235.gate_proj", + "model.layers.59.mlp.experts.235.up_proj", + "model.layers.59.mlp.experts.236.down_proj", + "model.layers.59.mlp.experts.236.gate_proj", + "model.layers.59.mlp.experts.236.up_proj", + "model.layers.59.mlp.experts.237.down_proj", + "model.layers.59.mlp.experts.237.gate_proj", + "model.layers.59.mlp.experts.237.up_proj", + "model.layers.59.mlp.experts.238.down_proj", + "model.layers.59.mlp.experts.238.gate_proj", + "model.layers.59.mlp.experts.238.up_proj", + "model.layers.59.mlp.experts.239.down_proj", + "model.layers.59.mlp.experts.239.gate_proj", + "model.layers.59.mlp.experts.239.up_proj", + "model.layers.59.mlp.experts.24.down_proj", + "model.layers.59.mlp.experts.24.gate_proj", + "model.layers.59.mlp.experts.24.up_proj", + "model.layers.59.mlp.experts.240.down_proj", + "model.layers.59.mlp.experts.240.gate_proj", + "model.layers.59.mlp.experts.240.up_proj", + "model.layers.59.mlp.experts.241.down_proj", + "model.layers.59.mlp.experts.241.gate_proj", + "model.layers.59.mlp.experts.241.up_proj", + "model.layers.59.mlp.experts.242.down_proj", + "model.layers.59.mlp.experts.242.gate_proj", + "model.layers.59.mlp.experts.242.up_proj", + "model.layers.59.mlp.experts.243.down_proj", + "model.layers.59.mlp.experts.243.gate_proj", + "model.layers.59.mlp.experts.243.up_proj", + "model.layers.59.mlp.experts.244.down_proj", + "model.layers.59.mlp.experts.244.gate_proj", + "model.layers.59.mlp.experts.244.up_proj", + "model.layers.59.mlp.experts.245.down_proj", + "model.layers.59.mlp.experts.245.gate_proj", + "model.layers.59.mlp.experts.245.up_proj", + "model.layers.59.mlp.experts.246.down_proj", + "model.layers.59.mlp.experts.246.gate_proj", + "model.layers.59.mlp.experts.246.up_proj", + "model.layers.59.mlp.experts.247.down_proj", + "model.layers.59.mlp.experts.247.gate_proj", + "model.layers.59.mlp.experts.247.up_proj", + "model.layers.59.mlp.experts.248.down_proj", + "model.layers.59.mlp.experts.248.gate_proj", + "model.layers.59.mlp.experts.248.up_proj", + "model.layers.59.mlp.experts.249.down_proj", + "model.layers.59.mlp.experts.249.gate_proj", + "model.layers.59.mlp.experts.249.up_proj", + "model.layers.59.mlp.experts.25.down_proj", + "model.layers.59.mlp.experts.25.gate_proj", + "model.layers.59.mlp.experts.25.up_proj", + "model.layers.59.mlp.experts.250.down_proj", + "model.layers.59.mlp.experts.250.gate_proj", + "model.layers.59.mlp.experts.250.up_proj", + "model.layers.59.mlp.experts.251.down_proj", + "model.layers.59.mlp.experts.251.gate_proj", + "model.layers.59.mlp.experts.251.up_proj", + "model.layers.59.mlp.experts.252.down_proj", + "model.layers.59.mlp.experts.252.gate_proj", + "model.layers.59.mlp.experts.252.up_proj", + "model.layers.59.mlp.experts.253.down_proj", + "model.layers.59.mlp.experts.253.gate_proj", + "model.layers.59.mlp.experts.253.up_proj", + "model.layers.59.mlp.experts.254.down_proj", + "model.layers.59.mlp.experts.254.gate_proj", + "model.layers.59.mlp.experts.254.up_proj", + "model.layers.59.mlp.experts.255.down_proj", + "model.layers.59.mlp.experts.255.gate_proj", + "model.layers.59.mlp.experts.255.up_proj", + "model.layers.59.mlp.experts.26.down_proj", + "model.layers.59.mlp.experts.26.gate_proj", + "model.layers.59.mlp.experts.26.up_proj", + "model.layers.59.mlp.experts.27.down_proj", + "model.layers.59.mlp.experts.27.gate_proj", + "model.layers.59.mlp.experts.27.up_proj", + "model.layers.59.mlp.experts.28.down_proj", + "model.layers.59.mlp.experts.28.gate_proj", + "model.layers.59.mlp.experts.28.up_proj", + "model.layers.59.mlp.experts.29.down_proj", + "model.layers.59.mlp.experts.29.gate_proj", + "model.layers.59.mlp.experts.29.up_proj", + "model.layers.59.mlp.experts.3.down_proj", + "model.layers.59.mlp.experts.3.gate_proj", + "model.layers.59.mlp.experts.3.up_proj", + "model.layers.59.mlp.experts.30.down_proj", + "model.layers.59.mlp.experts.30.gate_proj", + "model.layers.59.mlp.experts.30.up_proj", + "model.layers.59.mlp.experts.31.down_proj", + "model.layers.59.mlp.experts.31.gate_proj", + "model.layers.59.mlp.experts.31.up_proj", + "model.layers.59.mlp.experts.32.down_proj", + "model.layers.59.mlp.experts.32.gate_proj", + "model.layers.59.mlp.experts.32.up_proj", + "model.layers.59.mlp.experts.33.down_proj", + "model.layers.59.mlp.experts.33.gate_proj", + "model.layers.59.mlp.experts.33.up_proj", + "model.layers.59.mlp.experts.34.down_proj", + "model.layers.59.mlp.experts.34.gate_proj", + "model.layers.59.mlp.experts.34.up_proj", + "model.layers.59.mlp.experts.35.down_proj", + "model.layers.59.mlp.experts.35.gate_proj", + "model.layers.59.mlp.experts.35.up_proj", + "model.layers.59.mlp.experts.36.down_proj", + "model.layers.59.mlp.experts.36.gate_proj", + "model.layers.59.mlp.experts.36.up_proj", + "model.layers.59.mlp.experts.37.down_proj", + "model.layers.59.mlp.experts.37.gate_proj", + "model.layers.59.mlp.experts.37.up_proj", + "model.layers.59.mlp.experts.38.down_proj", + "model.layers.59.mlp.experts.38.gate_proj", + "model.layers.59.mlp.experts.38.up_proj", + "model.layers.59.mlp.experts.39.down_proj", + "model.layers.59.mlp.experts.39.gate_proj", + "model.layers.59.mlp.experts.39.up_proj", + "model.layers.59.mlp.experts.4.down_proj", + "model.layers.59.mlp.experts.4.gate_proj", + "model.layers.59.mlp.experts.4.up_proj", + "model.layers.59.mlp.experts.40.down_proj", + "model.layers.59.mlp.experts.40.gate_proj", + "model.layers.59.mlp.experts.40.up_proj", + "model.layers.59.mlp.experts.41.down_proj", + "model.layers.59.mlp.experts.41.gate_proj", + "model.layers.59.mlp.experts.41.up_proj", + "model.layers.59.mlp.experts.42.down_proj", + "model.layers.59.mlp.experts.42.gate_proj", + "model.layers.59.mlp.experts.42.up_proj", + "model.layers.59.mlp.experts.43.down_proj", + "model.layers.59.mlp.experts.43.gate_proj", + "model.layers.59.mlp.experts.43.up_proj", + "model.layers.59.mlp.experts.44.down_proj", + "model.layers.59.mlp.experts.44.gate_proj", + "model.layers.59.mlp.experts.44.up_proj", + "model.layers.59.mlp.experts.45.down_proj", + "model.layers.59.mlp.experts.45.gate_proj", + "model.layers.59.mlp.experts.45.up_proj", + "model.layers.59.mlp.experts.46.down_proj", + "model.layers.59.mlp.experts.46.gate_proj", + "model.layers.59.mlp.experts.46.up_proj", + "model.layers.59.mlp.experts.47.down_proj", + "model.layers.59.mlp.experts.47.gate_proj", + "model.layers.59.mlp.experts.47.up_proj", + "model.layers.59.mlp.experts.48.down_proj", + "model.layers.59.mlp.experts.48.gate_proj", + "model.layers.59.mlp.experts.48.up_proj", + "model.layers.59.mlp.experts.49.down_proj", + "model.layers.59.mlp.experts.49.gate_proj", + "model.layers.59.mlp.experts.49.up_proj", + "model.layers.59.mlp.experts.5.down_proj", + "model.layers.59.mlp.experts.5.gate_proj", + "model.layers.59.mlp.experts.5.up_proj", + "model.layers.59.mlp.experts.50.down_proj", + "model.layers.59.mlp.experts.50.gate_proj", + "model.layers.59.mlp.experts.50.up_proj", + "model.layers.59.mlp.experts.51.down_proj", + "model.layers.59.mlp.experts.51.gate_proj", + "model.layers.59.mlp.experts.51.up_proj", + "model.layers.59.mlp.experts.52.down_proj", + "model.layers.59.mlp.experts.52.gate_proj", + "model.layers.59.mlp.experts.52.up_proj", + "model.layers.59.mlp.experts.53.down_proj", + "model.layers.59.mlp.experts.53.gate_proj", + "model.layers.59.mlp.experts.53.up_proj", + "model.layers.59.mlp.experts.54.down_proj", + "model.layers.59.mlp.experts.54.gate_proj", + "model.layers.59.mlp.experts.54.up_proj", + "model.layers.59.mlp.experts.55.down_proj", + "model.layers.59.mlp.experts.55.gate_proj", + "model.layers.59.mlp.experts.55.up_proj", + "model.layers.59.mlp.experts.56.down_proj", + "model.layers.59.mlp.experts.56.gate_proj", + "model.layers.59.mlp.experts.56.up_proj", + "model.layers.59.mlp.experts.57.down_proj", + "model.layers.59.mlp.experts.57.gate_proj", + "model.layers.59.mlp.experts.57.up_proj", + "model.layers.59.mlp.experts.58.down_proj", + "model.layers.59.mlp.experts.58.gate_proj", + "model.layers.59.mlp.experts.58.up_proj", + "model.layers.59.mlp.experts.59.down_proj", + "model.layers.59.mlp.experts.59.gate_proj", + "model.layers.59.mlp.experts.59.up_proj", + "model.layers.59.mlp.experts.6.down_proj", + "model.layers.59.mlp.experts.6.gate_proj", + "model.layers.59.mlp.experts.6.up_proj", + "model.layers.59.mlp.experts.60.down_proj", + "model.layers.59.mlp.experts.60.gate_proj", + "model.layers.59.mlp.experts.60.up_proj", + "model.layers.59.mlp.experts.61.down_proj", + "model.layers.59.mlp.experts.61.gate_proj", + "model.layers.59.mlp.experts.61.up_proj", + "model.layers.59.mlp.experts.62.down_proj", + "model.layers.59.mlp.experts.62.gate_proj", + "model.layers.59.mlp.experts.62.up_proj", + "model.layers.59.mlp.experts.63.down_proj", + "model.layers.59.mlp.experts.63.gate_proj", + "model.layers.59.mlp.experts.63.up_proj", + "model.layers.59.mlp.experts.64.down_proj", + "model.layers.59.mlp.experts.64.gate_proj", + "model.layers.59.mlp.experts.64.up_proj", + "model.layers.59.mlp.experts.65.down_proj", + "model.layers.59.mlp.experts.65.gate_proj", + "model.layers.59.mlp.experts.65.up_proj", + "model.layers.59.mlp.experts.66.down_proj", + "model.layers.59.mlp.experts.66.gate_proj", + "model.layers.59.mlp.experts.66.up_proj", + "model.layers.59.mlp.experts.67.down_proj", + "model.layers.59.mlp.experts.67.gate_proj", + "model.layers.59.mlp.experts.67.up_proj", + "model.layers.59.mlp.experts.68.down_proj", + "model.layers.59.mlp.experts.68.gate_proj", + "model.layers.59.mlp.experts.68.up_proj", + "model.layers.59.mlp.experts.69.down_proj", + "model.layers.59.mlp.experts.69.gate_proj", + "model.layers.59.mlp.experts.69.up_proj", + "model.layers.59.mlp.experts.7.down_proj", + "model.layers.59.mlp.experts.7.gate_proj", + "model.layers.59.mlp.experts.7.up_proj", + "model.layers.59.mlp.experts.70.down_proj", + "model.layers.59.mlp.experts.70.gate_proj", + "model.layers.59.mlp.experts.70.up_proj", + "model.layers.59.mlp.experts.71.down_proj", + "model.layers.59.mlp.experts.71.gate_proj", + "model.layers.59.mlp.experts.71.up_proj", + "model.layers.59.mlp.experts.72.down_proj", + "model.layers.59.mlp.experts.72.gate_proj", + "model.layers.59.mlp.experts.72.up_proj", + "model.layers.59.mlp.experts.73.down_proj", + "model.layers.59.mlp.experts.73.gate_proj", + "model.layers.59.mlp.experts.73.up_proj", + "model.layers.59.mlp.experts.74.down_proj", + "model.layers.59.mlp.experts.74.gate_proj", + "model.layers.59.mlp.experts.74.up_proj", + "model.layers.59.mlp.experts.75.down_proj", + "model.layers.59.mlp.experts.75.gate_proj", + "model.layers.59.mlp.experts.75.up_proj", + "model.layers.59.mlp.experts.76.down_proj", + "model.layers.59.mlp.experts.76.gate_proj", + "model.layers.59.mlp.experts.76.up_proj", + "model.layers.59.mlp.experts.77.down_proj", + "model.layers.59.mlp.experts.77.gate_proj", + "model.layers.59.mlp.experts.77.up_proj", + "model.layers.59.mlp.experts.78.down_proj", + "model.layers.59.mlp.experts.78.gate_proj", + "model.layers.59.mlp.experts.78.up_proj", + "model.layers.59.mlp.experts.79.down_proj", + "model.layers.59.mlp.experts.79.gate_proj", + "model.layers.59.mlp.experts.79.up_proj", + "model.layers.59.mlp.experts.8.down_proj", + "model.layers.59.mlp.experts.8.gate_proj", + "model.layers.59.mlp.experts.8.up_proj", + "model.layers.59.mlp.experts.80.down_proj", + "model.layers.59.mlp.experts.80.gate_proj", + "model.layers.59.mlp.experts.80.up_proj", + "model.layers.59.mlp.experts.81.down_proj", + "model.layers.59.mlp.experts.81.gate_proj", + "model.layers.59.mlp.experts.81.up_proj", + "model.layers.59.mlp.experts.82.down_proj", + "model.layers.59.mlp.experts.82.gate_proj", + "model.layers.59.mlp.experts.82.up_proj", + "model.layers.59.mlp.experts.83.down_proj", + "model.layers.59.mlp.experts.83.gate_proj", + "model.layers.59.mlp.experts.83.up_proj", + "model.layers.59.mlp.experts.84.down_proj", + "model.layers.59.mlp.experts.84.gate_proj", + "model.layers.59.mlp.experts.84.up_proj", + "model.layers.59.mlp.experts.85.down_proj", + "model.layers.59.mlp.experts.85.gate_proj", + "model.layers.59.mlp.experts.85.up_proj", + "model.layers.59.mlp.experts.86.down_proj", + "model.layers.59.mlp.experts.86.gate_proj", + "model.layers.59.mlp.experts.86.up_proj", + "model.layers.59.mlp.experts.87.down_proj", + "model.layers.59.mlp.experts.87.gate_proj", + "model.layers.59.mlp.experts.87.up_proj", + "model.layers.59.mlp.experts.88.down_proj", + "model.layers.59.mlp.experts.88.gate_proj", + "model.layers.59.mlp.experts.88.up_proj", + "model.layers.59.mlp.experts.89.down_proj", + "model.layers.59.mlp.experts.89.gate_proj", + "model.layers.59.mlp.experts.89.up_proj", + "model.layers.59.mlp.experts.9.down_proj", + "model.layers.59.mlp.experts.9.gate_proj", + "model.layers.59.mlp.experts.9.up_proj", + "model.layers.59.mlp.experts.90.down_proj", + "model.layers.59.mlp.experts.90.gate_proj", + "model.layers.59.mlp.experts.90.up_proj", + "model.layers.59.mlp.experts.91.down_proj", + "model.layers.59.mlp.experts.91.gate_proj", + "model.layers.59.mlp.experts.91.up_proj", + "model.layers.59.mlp.experts.92.down_proj", + "model.layers.59.mlp.experts.92.gate_proj", + "model.layers.59.mlp.experts.92.up_proj", + "model.layers.59.mlp.experts.93.down_proj", + "model.layers.59.mlp.experts.93.gate_proj", + "model.layers.59.mlp.experts.93.up_proj", + "model.layers.59.mlp.experts.94.down_proj", + "model.layers.59.mlp.experts.94.gate_proj", + "model.layers.59.mlp.experts.94.up_proj", + "model.layers.59.mlp.experts.95.down_proj", + "model.layers.59.mlp.experts.95.gate_proj", + "model.layers.59.mlp.experts.95.up_proj", + "model.layers.59.mlp.experts.96.down_proj", + "model.layers.59.mlp.experts.96.gate_proj", + "model.layers.59.mlp.experts.96.up_proj", + "model.layers.59.mlp.experts.97.down_proj", + "model.layers.59.mlp.experts.97.gate_proj", + "model.layers.59.mlp.experts.97.up_proj", + "model.layers.59.mlp.experts.98.down_proj", + "model.layers.59.mlp.experts.98.gate_proj", + "model.layers.59.mlp.experts.98.up_proj", + "model.layers.59.mlp.experts.99.down_proj", + "model.layers.59.mlp.experts.99.gate_proj", + "model.layers.59.mlp.experts.99.up_proj", + "model.layers.6.mlp.experts.0.down_proj", + "model.layers.6.mlp.experts.0.gate_proj", + "model.layers.6.mlp.experts.0.up_proj", + "model.layers.6.mlp.experts.1.down_proj", + "model.layers.6.mlp.experts.1.gate_proj", + "model.layers.6.mlp.experts.1.up_proj", + "model.layers.6.mlp.experts.10.down_proj", + "model.layers.6.mlp.experts.10.gate_proj", + "model.layers.6.mlp.experts.10.up_proj", + "model.layers.6.mlp.experts.100.down_proj", + "model.layers.6.mlp.experts.100.gate_proj", + "model.layers.6.mlp.experts.100.up_proj", + "model.layers.6.mlp.experts.101.down_proj", + "model.layers.6.mlp.experts.101.gate_proj", + "model.layers.6.mlp.experts.101.up_proj", + "model.layers.6.mlp.experts.102.down_proj", + "model.layers.6.mlp.experts.102.gate_proj", + "model.layers.6.mlp.experts.102.up_proj", + "model.layers.6.mlp.experts.103.down_proj", + "model.layers.6.mlp.experts.103.gate_proj", + "model.layers.6.mlp.experts.103.up_proj", + "model.layers.6.mlp.experts.104.down_proj", + "model.layers.6.mlp.experts.104.gate_proj", + "model.layers.6.mlp.experts.104.up_proj", + "model.layers.6.mlp.experts.105.down_proj", + "model.layers.6.mlp.experts.105.gate_proj", + "model.layers.6.mlp.experts.105.up_proj", + "model.layers.6.mlp.experts.106.down_proj", + "model.layers.6.mlp.experts.106.gate_proj", + "model.layers.6.mlp.experts.106.up_proj", + "model.layers.6.mlp.experts.107.down_proj", + "model.layers.6.mlp.experts.107.gate_proj", + "model.layers.6.mlp.experts.107.up_proj", + "model.layers.6.mlp.experts.108.down_proj", + "model.layers.6.mlp.experts.108.gate_proj", + "model.layers.6.mlp.experts.108.up_proj", + "model.layers.6.mlp.experts.109.down_proj", + "model.layers.6.mlp.experts.109.gate_proj", + "model.layers.6.mlp.experts.109.up_proj", + "model.layers.6.mlp.experts.11.down_proj", + "model.layers.6.mlp.experts.11.gate_proj", + "model.layers.6.mlp.experts.11.up_proj", + "model.layers.6.mlp.experts.110.down_proj", + "model.layers.6.mlp.experts.110.gate_proj", + "model.layers.6.mlp.experts.110.up_proj", + "model.layers.6.mlp.experts.111.down_proj", + "model.layers.6.mlp.experts.111.gate_proj", + "model.layers.6.mlp.experts.111.up_proj", + "model.layers.6.mlp.experts.112.down_proj", + "model.layers.6.mlp.experts.112.gate_proj", + "model.layers.6.mlp.experts.112.up_proj", + "model.layers.6.mlp.experts.113.down_proj", + "model.layers.6.mlp.experts.113.gate_proj", + "model.layers.6.mlp.experts.113.up_proj", + "model.layers.6.mlp.experts.114.down_proj", + "model.layers.6.mlp.experts.114.gate_proj", + "model.layers.6.mlp.experts.114.up_proj", + "model.layers.6.mlp.experts.115.down_proj", + "model.layers.6.mlp.experts.115.gate_proj", + "model.layers.6.mlp.experts.115.up_proj", + "model.layers.6.mlp.experts.116.down_proj", + "model.layers.6.mlp.experts.116.gate_proj", + "model.layers.6.mlp.experts.116.up_proj", + "model.layers.6.mlp.experts.117.down_proj", + "model.layers.6.mlp.experts.117.gate_proj", + "model.layers.6.mlp.experts.117.up_proj", + "model.layers.6.mlp.experts.118.down_proj", + "model.layers.6.mlp.experts.118.gate_proj", + "model.layers.6.mlp.experts.118.up_proj", + "model.layers.6.mlp.experts.119.down_proj", + "model.layers.6.mlp.experts.119.gate_proj", + "model.layers.6.mlp.experts.119.up_proj", + "model.layers.6.mlp.experts.12.down_proj", + "model.layers.6.mlp.experts.12.gate_proj", + "model.layers.6.mlp.experts.12.up_proj", + "model.layers.6.mlp.experts.120.down_proj", + "model.layers.6.mlp.experts.120.gate_proj", + "model.layers.6.mlp.experts.120.up_proj", + "model.layers.6.mlp.experts.121.down_proj", + "model.layers.6.mlp.experts.121.gate_proj", + "model.layers.6.mlp.experts.121.up_proj", + "model.layers.6.mlp.experts.122.down_proj", + "model.layers.6.mlp.experts.122.gate_proj", + "model.layers.6.mlp.experts.122.up_proj", + "model.layers.6.mlp.experts.123.down_proj", + "model.layers.6.mlp.experts.123.gate_proj", + "model.layers.6.mlp.experts.123.up_proj", + "model.layers.6.mlp.experts.124.down_proj", + "model.layers.6.mlp.experts.124.gate_proj", + "model.layers.6.mlp.experts.124.up_proj", + "model.layers.6.mlp.experts.125.down_proj", + "model.layers.6.mlp.experts.125.gate_proj", + "model.layers.6.mlp.experts.125.up_proj", + "model.layers.6.mlp.experts.126.down_proj", + "model.layers.6.mlp.experts.126.gate_proj", + "model.layers.6.mlp.experts.126.up_proj", + "model.layers.6.mlp.experts.127.down_proj", + "model.layers.6.mlp.experts.127.gate_proj", + "model.layers.6.mlp.experts.127.up_proj", + "model.layers.6.mlp.experts.128.down_proj", + "model.layers.6.mlp.experts.128.gate_proj", + "model.layers.6.mlp.experts.128.up_proj", + "model.layers.6.mlp.experts.129.down_proj", + "model.layers.6.mlp.experts.129.gate_proj", + "model.layers.6.mlp.experts.129.up_proj", + "model.layers.6.mlp.experts.13.down_proj", + "model.layers.6.mlp.experts.13.gate_proj", + "model.layers.6.mlp.experts.13.up_proj", + "model.layers.6.mlp.experts.130.down_proj", + "model.layers.6.mlp.experts.130.gate_proj", + "model.layers.6.mlp.experts.130.up_proj", + "model.layers.6.mlp.experts.131.down_proj", + "model.layers.6.mlp.experts.131.gate_proj", + "model.layers.6.mlp.experts.131.up_proj", + "model.layers.6.mlp.experts.132.down_proj", + "model.layers.6.mlp.experts.132.gate_proj", + "model.layers.6.mlp.experts.132.up_proj", + "model.layers.6.mlp.experts.133.down_proj", + "model.layers.6.mlp.experts.133.gate_proj", + "model.layers.6.mlp.experts.133.up_proj", + "model.layers.6.mlp.experts.134.down_proj", + "model.layers.6.mlp.experts.134.gate_proj", + "model.layers.6.mlp.experts.134.up_proj", + "model.layers.6.mlp.experts.135.down_proj", + "model.layers.6.mlp.experts.135.gate_proj", + "model.layers.6.mlp.experts.135.up_proj", + "model.layers.6.mlp.experts.136.down_proj", + "model.layers.6.mlp.experts.136.gate_proj", + "model.layers.6.mlp.experts.136.up_proj", + "model.layers.6.mlp.experts.137.down_proj", + "model.layers.6.mlp.experts.137.gate_proj", + "model.layers.6.mlp.experts.137.up_proj", + "model.layers.6.mlp.experts.138.down_proj", + "model.layers.6.mlp.experts.138.gate_proj", + "model.layers.6.mlp.experts.138.up_proj", + "model.layers.6.mlp.experts.139.down_proj", + "model.layers.6.mlp.experts.139.gate_proj", + "model.layers.6.mlp.experts.139.up_proj", + "model.layers.6.mlp.experts.14.down_proj", + "model.layers.6.mlp.experts.14.gate_proj", + "model.layers.6.mlp.experts.14.up_proj", + "model.layers.6.mlp.experts.140.down_proj", + "model.layers.6.mlp.experts.140.gate_proj", + "model.layers.6.mlp.experts.140.up_proj", + "model.layers.6.mlp.experts.141.down_proj", + "model.layers.6.mlp.experts.141.gate_proj", + "model.layers.6.mlp.experts.141.up_proj", + "model.layers.6.mlp.experts.142.down_proj", + "model.layers.6.mlp.experts.142.gate_proj", + "model.layers.6.mlp.experts.142.up_proj", + "model.layers.6.mlp.experts.143.down_proj", + "model.layers.6.mlp.experts.143.gate_proj", + "model.layers.6.mlp.experts.143.up_proj", + "model.layers.6.mlp.experts.144.down_proj", + "model.layers.6.mlp.experts.144.gate_proj", + "model.layers.6.mlp.experts.144.up_proj", + "model.layers.6.mlp.experts.145.down_proj", + "model.layers.6.mlp.experts.145.gate_proj", + "model.layers.6.mlp.experts.145.up_proj", + "model.layers.6.mlp.experts.146.down_proj", + "model.layers.6.mlp.experts.146.gate_proj", + "model.layers.6.mlp.experts.146.up_proj", + "model.layers.6.mlp.experts.147.down_proj", + "model.layers.6.mlp.experts.147.gate_proj", + "model.layers.6.mlp.experts.147.up_proj", + "model.layers.6.mlp.experts.148.down_proj", + "model.layers.6.mlp.experts.148.gate_proj", + "model.layers.6.mlp.experts.148.up_proj", + "model.layers.6.mlp.experts.149.down_proj", + "model.layers.6.mlp.experts.149.gate_proj", + "model.layers.6.mlp.experts.149.up_proj", + "model.layers.6.mlp.experts.15.down_proj", + "model.layers.6.mlp.experts.15.gate_proj", + "model.layers.6.mlp.experts.15.up_proj", + "model.layers.6.mlp.experts.150.down_proj", + "model.layers.6.mlp.experts.150.gate_proj", + "model.layers.6.mlp.experts.150.up_proj", + "model.layers.6.mlp.experts.151.down_proj", + "model.layers.6.mlp.experts.151.gate_proj", + "model.layers.6.mlp.experts.151.up_proj", + "model.layers.6.mlp.experts.152.down_proj", + "model.layers.6.mlp.experts.152.gate_proj", + "model.layers.6.mlp.experts.152.up_proj", + "model.layers.6.mlp.experts.153.down_proj", + "model.layers.6.mlp.experts.153.gate_proj", + "model.layers.6.mlp.experts.153.up_proj", + "model.layers.6.mlp.experts.154.down_proj", + "model.layers.6.mlp.experts.154.gate_proj", + "model.layers.6.mlp.experts.154.up_proj", + "model.layers.6.mlp.experts.155.down_proj", + "model.layers.6.mlp.experts.155.gate_proj", + "model.layers.6.mlp.experts.155.up_proj", + "model.layers.6.mlp.experts.156.down_proj", + "model.layers.6.mlp.experts.156.gate_proj", + "model.layers.6.mlp.experts.156.up_proj", + "model.layers.6.mlp.experts.157.down_proj", + "model.layers.6.mlp.experts.157.gate_proj", + "model.layers.6.mlp.experts.157.up_proj", + "model.layers.6.mlp.experts.158.down_proj", + "model.layers.6.mlp.experts.158.gate_proj", + "model.layers.6.mlp.experts.158.up_proj", + "model.layers.6.mlp.experts.159.down_proj", + "model.layers.6.mlp.experts.159.gate_proj", + "model.layers.6.mlp.experts.159.up_proj", + "model.layers.6.mlp.experts.16.down_proj", + "model.layers.6.mlp.experts.16.gate_proj", + "model.layers.6.mlp.experts.16.up_proj", + "model.layers.6.mlp.experts.160.down_proj", + "model.layers.6.mlp.experts.160.gate_proj", + "model.layers.6.mlp.experts.160.up_proj", + "model.layers.6.mlp.experts.161.down_proj", + "model.layers.6.mlp.experts.161.gate_proj", + "model.layers.6.mlp.experts.161.up_proj", + "model.layers.6.mlp.experts.162.down_proj", + "model.layers.6.mlp.experts.162.gate_proj", + "model.layers.6.mlp.experts.162.up_proj", + "model.layers.6.mlp.experts.163.down_proj", + "model.layers.6.mlp.experts.163.gate_proj", + "model.layers.6.mlp.experts.163.up_proj", + "model.layers.6.mlp.experts.164.down_proj", + "model.layers.6.mlp.experts.164.gate_proj", + "model.layers.6.mlp.experts.164.up_proj", + "model.layers.6.mlp.experts.165.down_proj", + "model.layers.6.mlp.experts.165.gate_proj", + "model.layers.6.mlp.experts.165.up_proj", + "model.layers.6.mlp.experts.166.down_proj", + "model.layers.6.mlp.experts.166.gate_proj", + "model.layers.6.mlp.experts.166.up_proj", + "model.layers.6.mlp.experts.167.down_proj", + "model.layers.6.mlp.experts.167.gate_proj", + "model.layers.6.mlp.experts.167.up_proj", + "model.layers.6.mlp.experts.168.down_proj", + "model.layers.6.mlp.experts.168.gate_proj", + "model.layers.6.mlp.experts.168.up_proj", + "model.layers.6.mlp.experts.169.down_proj", + "model.layers.6.mlp.experts.169.gate_proj", + "model.layers.6.mlp.experts.169.up_proj", + "model.layers.6.mlp.experts.17.down_proj", + "model.layers.6.mlp.experts.17.gate_proj", + "model.layers.6.mlp.experts.17.up_proj", + "model.layers.6.mlp.experts.170.down_proj", + "model.layers.6.mlp.experts.170.gate_proj", + "model.layers.6.mlp.experts.170.up_proj", + "model.layers.6.mlp.experts.171.down_proj", + "model.layers.6.mlp.experts.171.gate_proj", + "model.layers.6.mlp.experts.171.up_proj", + "model.layers.6.mlp.experts.172.down_proj", + "model.layers.6.mlp.experts.172.gate_proj", + "model.layers.6.mlp.experts.172.up_proj", + "model.layers.6.mlp.experts.173.down_proj", + "model.layers.6.mlp.experts.173.gate_proj", + "model.layers.6.mlp.experts.173.up_proj", + "model.layers.6.mlp.experts.174.down_proj", + "model.layers.6.mlp.experts.174.gate_proj", + "model.layers.6.mlp.experts.174.up_proj", + "model.layers.6.mlp.experts.175.down_proj", + "model.layers.6.mlp.experts.175.gate_proj", + "model.layers.6.mlp.experts.175.up_proj", + "model.layers.6.mlp.experts.176.down_proj", + "model.layers.6.mlp.experts.176.gate_proj", + "model.layers.6.mlp.experts.176.up_proj", + "model.layers.6.mlp.experts.177.down_proj", + "model.layers.6.mlp.experts.177.gate_proj", + "model.layers.6.mlp.experts.177.up_proj", + "model.layers.6.mlp.experts.178.down_proj", + "model.layers.6.mlp.experts.178.gate_proj", + "model.layers.6.mlp.experts.178.up_proj", + "model.layers.6.mlp.experts.179.down_proj", + "model.layers.6.mlp.experts.179.gate_proj", + "model.layers.6.mlp.experts.179.up_proj", + "model.layers.6.mlp.experts.18.down_proj", + "model.layers.6.mlp.experts.18.gate_proj", + "model.layers.6.mlp.experts.18.up_proj", + "model.layers.6.mlp.experts.180.down_proj", + "model.layers.6.mlp.experts.180.gate_proj", + "model.layers.6.mlp.experts.180.up_proj", + "model.layers.6.mlp.experts.181.down_proj", + "model.layers.6.mlp.experts.181.gate_proj", + "model.layers.6.mlp.experts.181.up_proj", + "model.layers.6.mlp.experts.182.down_proj", + "model.layers.6.mlp.experts.182.gate_proj", + "model.layers.6.mlp.experts.182.up_proj", + "model.layers.6.mlp.experts.183.down_proj", + "model.layers.6.mlp.experts.183.gate_proj", + "model.layers.6.mlp.experts.183.up_proj", + "model.layers.6.mlp.experts.184.down_proj", + "model.layers.6.mlp.experts.184.gate_proj", + "model.layers.6.mlp.experts.184.up_proj", + "model.layers.6.mlp.experts.185.down_proj", + "model.layers.6.mlp.experts.185.gate_proj", + "model.layers.6.mlp.experts.185.up_proj", + "model.layers.6.mlp.experts.186.down_proj", + "model.layers.6.mlp.experts.186.gate_proj", + "model.layers.6.mlp.experts.186.up_proj", + "model.layers.6.mlp.experts.187.down_proj", + "model.layers.6.mlp.experts.187.gate_proj", + "model.layers.6.mlp.experts.187.up_proj", + "model.layers.6.mlp.experts.188.down_proj", + "model.layers.6.mlp.experts.188.gate_proj", + "model.layers.6.mlp.experts.188.up_proj", + "model.layers.6.mlp.experts.189.down_proj", + "model.layers.6.mlp.experts.189.gate_proj", + "model.layers.6.mlp.experts.189.up_proj", + "model.layers.6.mlp.experts.19.down_proj", + "model.layers.6.mlp.experts.19.gate_proj", + "model.layers.6.mlp.experts.19.up_proj", + "model.layers.6.mlp.experts.190.down_proj", + "model.layers.6.mlp.experts.190.gate_proj", + "model.layers.6.mlp.experts.190.up_proj", + "model.layers.6.mlp.experts.191.down_proj", + "model.layers.6.mlp.experts.191.gate_proj", + "model.layers.6.mlp.experts.191.up_proj", + "model.layers.6.mlp.experts.192.down_proj", + "model.layers.6.mlp.experts.192.gate_proj", + "model.layers.6.mlp.experts.192.up_proj", + "model.layers.6.mlp.experts.193.down_proj", + "model.layers.6.mlp.experts.193.gate_proj", + "model.layers.6.mlp.experts.193.up_proj", + "model.layers.6.mlp.experts.194.down_proj", + "model.layers.6.mlp.experts.194.gate_proj", + "model.layers.6.mlp.experts.194.up_proj", + "model.layers.6.mlp.experts.195.down_proj", + "model.layers.6.mlp.experts.195.gate_proj", + "model.layers.6.mlp.experts.195.up_proj", + "model.layers.6.mlp.experts.196.down_proj", + "model.layers.6.mlp.experts.196.gate_proj", + "model.layers.6.mlp.experts.196.up_proj", + "model.layers.6.mlp.experts.197.down_proj", + "model.layers.6.mlp.experts.197.gate_proj", + "model.layers.6.mlp.experts.197.up_proj", + "model.layers.6.mlp.experts.198.down_proj", + "model.layers.6.mlp.experts.198.gate_proj", + "model.layers.6.mlp.experts.198.up_proj", + "model.layers.6.mlp.experts.199.down_proj", + "model.layers.6.mlp.experts.199.gate_proj", + "model.layers.6.mlp.experts.199.up_proj", + "model.layers.6.mlp.experts.2.down_proj", + "model.layers.6.mlp.experts.2.gate_proj", + "model.layers.6.mlp.experts.2.up_proj", + "model.layers.6.mlp.experts.20.down_proj", + "model.layers.6.mlp.experts.20.gate_proj", + "model.layers.6.mlp.experts.20.up_proj", + "model.layers.6.mlp.experts.200.down_proj", + "model.layers.6.mlp.experts.200.gate_proj", + "model.layers.6.mlp.experts.200.up_proj", + "model.layers.6.mlp.experts.201.down_proj", + "model.layers.6.mlp.experts.201.gate_proj", + "model.layers.6.mlp.experts.201.up_proj", + "model.layers.6.mlp.experts.202.down_proj", + "model.layers.6.mlp.experts.202.gate_proj", + "model.layers.6.mlp.experts.202.up_proj", + "model.layers.6.mlp.experts.203.down_proj", + "model.layers.6.mlp.experts.203.gate_proj", + "model.layers.6.mlp.experts.203.up_proj", + "model.layers.6.mlp.experts.204.down_proj", + "model.layers.6.mlp.experts.204.gate_proj", + "model.layers.6.mlp.experts.204.up_proj", + "model.layers.6.mlp.experts.205.down_proj", + "model.layers.6.mlp.experts.205.gate_proj", + "model.layers.6.mlp.experts.205.up_proj", + "model.layers.6.mlp.experts.206.down_proj", + "model.layers.6.mlp.experts.206.gate_proj", + "model.layers.6.mlp.experts.206.up_proj", + "model.layers.6.mlp.experts.207.down_proj", + "model.layers.6.mlp.experts.207.gate_proj", + "model.layers.6.mlp.experts.207.up_proj", + "model.layers.6.mlp.experts.208.down_proj", + "model.layers.6.mlp.experts.208.gate_proj", + "model.layers.6.mlp.experts.208.up_proj", + "model.layers.6.mlp.experts.209.down_proj", + "model.layers.6.mlp.experts.209.gate_proj", + "model.layers.6.mlp.experts.209.up_proj", + "model.layers.6.mlp.experts.21.down_proj", + "model.layers.6.mlp.experts.21.gate_proj", + "model.layers.6.mlp.experts.21.up_proj", + "model.layers.6.mlp.experts.210.down_proj", + "model.layers.6.mlp.experts.210.gate_proj", + "model.layers.6.mlp.experts.210.up_proj", + "model.layers.6.mlp.experts.211.down_proj", + "model.layers.6.mlp.experts.211.gate_proj", + "model.layers.6.mlp.experts.211.up_proj", + "model.layers.6.mlp.experts.212.down_proj", + "model.layers.6.mlp.experts.212.gate_proj", + "model.layers.6.mlp.experts.212.up_proj", + "model.layers.6.mlp.experts.213.down_proj", + "model.layers.6.mlp.experts.213.gate_proj", + "model.layers.6.mlp.experts.213.up_proj", + "model.layers.6.mlp.experts.214.down_proj", + "model.layers.6.mlp.experts.214.gate_proj", + "model.layers.6.mlp.experts.214.up_proj", + "model.layers.6.mlp.experts.215.down_proj", + "model.layers.6.mlp.experts.215.gate_proj", + "model.layers.6.mlp.experts.215.up_proj", + "model.layers.6.mlp.experts.216.down_proj", + "model.layers.6.mlp.experts.216.gate_proj", + "model.layers.6.mlp.experts.216.up_proj", + "model.layers.6.mlp.experts.217.down_proj", + "model.layers.6.mlp.experts.217.gate_proj", + "model.layers.6.mlp.experts.217.up_proj", + "model.layers.6.mlp.experts.218.down_proj", + "model.layers.6.mlp.experts.218.gate_proj", + "model.layers.6.mlp.experts.218.up_proj", + "model.layers.6.mlp.experts.219.down_proj", + "model.layers.6.mlp.experts.219.gate_proj", + "model.layers.6.mlp.experts.219.up_proj", + "model.layers.6.mlp.experts.22.down_proj", + "model.layers.6.mlp.experts.22.gate_proj", + "model.layers.6.mlp.experts.22.up_proj", + "model.layers.6.mlp.experts.220.down_proj", + "model.layers.6.mlp.experts.220.gate_proj", + "model.layers.6.mlp.experts.220.up_proj", + "model.layers.6.mlp.experts.221.down_proj", + "model.layers.6.mlp.experts.221.gate_proj", + "model.layers.6.mlp.experts.221.up_proj", + "model.layers.6.mlp.experts.222.down_proj", + "model.layers.6.mlp.experts.222.gate_proj", + "model.layers.6.mlp.experts.222.up_proj", + "model.layers.6.mlp.experts.223.down_proj", + "model.layers.6.mlp.experts.223.gate_proj", + "model.layers.6.mlp.experts.223.up_proj", + "model.layers.6.mlp.experts.224.down_proj", + "model.layers.6.mlp.experts.224.gate_proj", + "model.layers.6.mlp.experts.224.up_proj", + "model.layers.6.mlp.experts.225.down_proj", + "model.layers.6.mlp.experts.225.gate_proj", + "model.layers.6.mlp.experts.225.up_proj", + "model.layers.6.mlp.experts.226.down_proj", + "model.layers.6.mlp.experts.226.gate_proj", + "model.layers.6.mlp.experts.226.up_proj", + "model.layers.6.mlp.experts.227.down_proj", + "model.layers.6.mlp.experts.227.gate_proj", + "model.layers.6.mlp.experts.227.up_proj", + "model.layers.6.mlp.experts.228.down_proj", + "model.layers.6.mlp.experts.228.gate_proj", + "model.layers.6.mlp.experts.228.up_proj", + "model.layers.6.mlp.experts.229.down_proj", + "model.layers.6.mlp.experts.229.gate_proj", + "model.layers.6.mlp.experts.229.up_proj", + "model.layers.6.mlp.experts.23.down_proj", + "model.layers.6.mlp.experts.23.gate_proj", + "model.layers.6.mlp.experts.23.up_proj", + "model.layers.6.mlp.experts.230.down_proj", + "model.layers.6.mlp.experts.230.gate_proj", + "model.layers.6.mlp.experts.230.up_proj", + "model.layers.6.mlp.experts.231.down_proj", + "model.layers.6.mlp.experts.231.gate_proj", + "model.layers.6.mlp.experts.231.up_proj", + "model.layers.6.mlp.experts.232.down_proj", + "model.layers.6.mlp.experts.232.gate_proj", + "model.layers.6.mlp.experts.232.up_proj", + "model.layers.6.mlp.experts.233.down_proj", + "model.layers.6.mlp.experts.233.gate_proj", + "model.layers.6.mlp.experts.233.up_proj", + "model.layers.6.mlp.experts.234.down_proj", + "model.layers.6.mlp.experts.234.gate_proj", + "model.layers.6.mlp.experts.234.up_proj", + "model.layers.6.mlp.experts.235.down_proj", + "model.layers.6.mlp.experts.235.gate_proj", + "model.layers.6.mlp.experts.235.up_proj", + "model.layers.6.mlp.experts.236.down_proj", + "model.layers.6.mlp.experts.236.gate_proj", + "model.layers.6.mlp.experts.236.up_proj", + "model.layers.6.mlp.experts.237.down_proj", + "model.layers.6.mlp.experts.237.gate_proj", + "model.layers.6.mlp.experts.237.up_proj", + "model.layers.6.mlp.experts.238.down_proj", + "model.layers.6.mlp.experts.238.gate_proj", + "model.layers.6.mlp.experts.238.up_proj", + "model.layers.6.mlp.experts.239.down_proj", + "model.layers.6.mlp.experts.239.gate_proj", + "model.layers.6.mlp.experts.239.up_proj", + "model.layers.6.mlp.experts.24.down_proj", + "model.layers.6.mlp.experts.24.gate_proj", + "model.layers.6.mlp.experts.24.up_proj", + "model.layers.6.mlp.experts.240.down_proj", + "model.layers.6.mlp.experts.240.gate_proj", + "model.layers.6.mlp.experts.240.up_proj", + "model.layers.6.mlp.experts.241.down_proj", + "model.layers.6.mlp.experts.241.gate_proj", + "model.layers.6.mlp.experts.241.up_proj", + "model.layers.6.mlp.experts.242.down_proj", + "model.layers.6.mlp.experts.242.gate_proj", + "model.layers.6.mlp.experts.242.up_proj", + "model.layers.6.mlp.experts.243.down_proj", + "model.layers.6.mlp.experts.243.gate_proj", + "model.layers.6.mlp.experts.243.up_proj", + "model.layers.6.mlp.experts.244.down_proj", + "model.layers.6.mlp.experts.244.gate_proj", + "model.layers.6.mlp.experts.244.up_proj", + "model.layers.6.mlp.experts.245.down_proj", + "model.layers.6.mlp.experts.245.gate_proj", + "model.layers.6.mlp.experts.245.up_proj", + "model.layers.6.mlp.experts.246.down_proj", + "model.layers.6.mlp.experts.246.gate_proj", + "model.layers.6.mlp.experts.246.up_proj", + "model.layers.6.mlp.experts.247.down_proj", + "model.layers.6.mlp.experts.247.gate_proj", + "model.layers.6.mlp.experts.247.up_proj", + "model.layers.6.mlp.experts.248.down_proj", + "model.layers.6.mlp.experts.248.gate_proj", + "model.layers.6.mlp.experts.248.up_proj", + "model.layers.6.mlp.experts.249.down_proj", + "model.layers.6.mlp.experts.249.gate_proj", + "model.layers.6.mlp.experts.249.up_proj", + "model.layers.6.mlp.experts.25.down_proj", + "model.layers.6.mlp.experts.25.gate_proj", + "model.layers.6.mlp.experts.25.up_proj", + "model.layers.6.mlp.experts.250.down_proj", + "model.layers.6.mlp.experts.250.gate_proj", + "model.layers.6.mlp.experts.250.up_proj", + "model.layers.6.mlp.experts.251.down_proj", + "model.layers.6.mlp.experts.251.gate_proj", + "model.layers.6.mlp.experts.251.up_proj", + "model.layers.6.mlp.experts.252.down_proj", + "model.layers.6.mlp.experts.252.gate_proj", + "model.layers.6.mlp.experts.252.up_proj", + "model.layers.6.mlp.experts.253.down_proj", + "model.layers.6.mlp.experts.253.gate_proj", + "model.layers.6.mlp.experts.253.up_proj", + "model.layers.6.mlp.experts.254.down_proj", + "model.layers.6.mlp.experts.254.gate_proj", + "model.layers.6.mlp.experts.254.up_proj", + "model.layers.6.mlp.experts.255.down_proj", + "model.layers.6.mlp.experts.255.gate_proj", + "model.layers.6.mlp.experts.255.up_proj", + "model.layers.6.mlp.experts.26.down_proj", + "model.layers.6.mlp.experts.26.gate_proj", + "model.layers.6.mlp.experts.26.up_proj", + "model.layers.6.mlp.experts.27.down_proj", + "model.layers.6.mlp.experts.27.gate_proj", + "model.layers.6.mlp.experts.27.up_proj", + "model.layers.6.mlp.experts.28.down_proj", + "model.layers.6.mlp.experts.28.gate_proj", + "model.layers.6.mlp.experts.28.up_proj", + "model.layers.6.mlp.experts.29.down_proj", + "model.layers.6.mlp.experts.29.gate_proj", + "model.layers.6.mlp.experts.29.up_proj", + "model.layers.6.mlp.experts.3.down_proj", + "model.layers.6.mlp.experts.3.gate_proj", + "model.layers.6.mlp.experts.3.up_proj", + "model.layers.6.mlp.experts.30.down_proj", + "model.layers.6.mlp.experts.30.gate_proj", + "model.layers.6.mlp.experts.30.up_proj", + "model.layers.6.mlp.experts.31.down_proj", + "model.layers.6.mlp.experts.31.gate_proj", + "model.layers.6.mlp.experts.31.up_proj", + "model.layers.6.mlp.experts.32.down_proj", + "model.layers.6.mlp.experts.32.gate_proj", + "model.layers.6.mlp.experts.32.up_proj", + "model.layers.6.mlp.experts.33.down_proj", + "model.layers.6.mlp.experts.33.gate_proj", + "model.layers.6.mlp.experts.33.up_proj", + "model.layers.6.mlp.experts.34.down_proj", + "model.layers.6.mlp.experts.34.gate_proj", + "model.layers.6.mlp.experts.34.up_proj", + "model.layers.6.mlp.experts.35.down_proj", + "model.layers.6.mlp.experts.35.gate_proj", + "model.layers.6.mlp.experts.35.up_proj", + "model.layers.6.mlp.experts.36.down_proj", + "model.layers.6.mlp.experts.36.gate_proj", + "model.layers.6.mlp.experts.36.up_proj", + "model.layers.6.mlp.experts.37.down_proj", + "model.layers.6.mlp.experts.37.gate_proj", + "model.layers.6.mlp.experts.37.up_proj", + "model.layers.6.mlp.experts.38.down_proj", + "model.layers.6.mlp.experts.38.gate_proj", + "model.layers.6.mlp.experts.38.up_proj", + "model.layers.6.mlp.experts.39.down_proj", + "model.layers.6.mlp.experts.39.gate_proj", + "model.layers.6.mlp.experts.39.up_proj", + "model.layers.6.mlp.experts.4.down_proj", + "model.layers.6.mlp.experts.4.gate_proj", + "model.layers.6.mlp.experts.4.up_proj", + "model.layers.6.mlp.experts.40.down_proj", + "model.layers.6.mlp.experts.40.gate_proj", + "model.layers.6.mlp.experts.40.up_proj", + "model.layers.6.mlp.experts.41.down_proj", + "model.layers.6.mlp.experts.41.gate_proj", + "model.layers.6.mlp.experts.41.up_proj", + "model.layers.6.mlp.experts.42.down_proj", + "model.layers.6.mlp.experts.42.gate_proj", + "model.layers.6.mlp.experts.42.up_proj", + "model.layers.6.mlp.experts.43.down_proj", + "model.layers.6.mlp.experts.43.gate_proj", + "model.layers.6.mlp.experts.43.up_proj", + "model.layers.6.mlp.experts.44.down_proj", + "model.layers.6.mlp.experts.44.gate_proj", + "model.layers.6.mlp.experts.44.up_proj", + "model.layers.6.mlp.experts.45.down_proj", + "model.layers.6.mlp.experts.45.gate_proj", + "model.layers.6.mlp.experts.45.up_proj", + "model.layers.6.mlp.experts.46.down_proj", + "model.layers.6.mlp.experts.46.gate_proj", + "model.layers.6.mlp.experts.46.up_proj", + "model.layers.6.mlp.experts.47.down_proj", + "model.layers.6.mlp.experts.47.gate_proj", + "model.layers.6.mlp.experts.47.up_proj", + "model.layers.6.mlp.experts.48.down_proj", + "model.layers.6.mlp.experts.48.gate_proj", + "model.layers.6.mlp.experts.48.up_proj", + "model.layers.6.mlp.experts.49.down_proj", + "model.layers.6.mlp.experts.49.gate_proj", + "model.layers.6.mlp.experts.49.up_proj", + "model.layers.6.mlp.experts.5.down_proj", + "model.layers.6.mlp.experts.5.gate_proj", + "model.layers.6.mlp.experts.5.up_proj", + "model.layers.6.mlp.experts.50.down_proj", + "model.layers.6.mlp.experts.50.gate_proj", + "model.layers.6.mlp.experts.50.up_proj", + "model.layers.6.mlp.experts.51.down_proj", + "model.layers.6.mlp.experts.51.gate_proj", + "model.layers.6.mlp.experts.51.up_proj", + "model.layers.6.mlp.experts.52.down_proj", + "model.layers.6.mlp.experts.52.gate_proj", + "model.layers.6.mlp.experts.52.up_proj", + "model.layers.6.mlp.experts.53.down_proj", + "model.layers.6.mlp.experts.53.gate_proj", + "model.layers.6.mlp.experts.53.up_proj", + "model.layers.6.mlp.experts.54.down_proj", + "model.layers.6.mlp.experts.54.gate_proj", + "model.layers.6.mlp.experts.54.up_proj", + "model.layers.6.mlp.experts.55.down_proj", + "model.layers.6.mlp.experts.55.gate_proj", + "model.layers.6.mlp.experts.55.up_proj", + "model.layers.6.mlp.experts.56.down_proj", + "model.layers.6.mlp.experts.56.gate_proj", + "model.layers.6.mlp.experts.56.up_proj", + "model.layers.6.mlp.experts.57.down_proj", + "model.layers.6.mlp.experts.57.gate_proj", + "model.layers.6.mlp.experts.57.up_proj", + "model.layers.6.mlp.experts.58.down_proj", + "model.layers.6.mlp.experts.58.gate_proj", + "model.layers.6.mlp.experts.58.up_proj", + "model.layers.6.mlp.experts.59.down_proj", + "model.layers.6.mlp.experts.59.gate_proj", + "model.layers.6.mlp.experts.59.up_proj", + "model.layers.6.mlp.experts.6.down_proj", + "model.layers.6.mlp.experts.6.gate_proj", + "model.layers.6.mlp.experts.6.up_proj", + "model.layers.6.mlp.experts.60.down_proj", + "model.layers.6.mlp.experts.60.gate_proj", + "model.layers.6.mlp.experts.60.up_proj", + "model.layers.6.mlp.experts.61.down_proj", + "model.layers.6.mlp.experts.61.gate_proj", + "model.layers.6.mlp.experts.61.up_proj", + "model.layers.6.mlp.experts.62.down_proj", + "model.layers.6.mlp.experts.62.gate_proj", + "model.layers.6.mlp.experts.62.up_proj", + "model.layers.6.mlp.experts.63.down_proj", + "model.layers.6.mlp.experts.63.gate_proj", + "model.layers.6.mlp.experts.63.up_proj", + "model.layers.6.mlp.experts.64.down_proj", + "model.layers.6.mlp.experts.64.gate_proj", + "model.layers.6.mlp.experts.64.up_proj", + "model.layers.6.mlp.experts.65.down_proj", + "model.layers.6.mlp.experts.65.gate_proj", + "model.layers.6.mlp.experts.65.up_proj", + "model.layers.6.mlp.experts.66.down_proj", + "model.layers.6.mlp.experts.66.gate_proj", + "model.layers.6.mlp.experts.66.up_proj", + "model.layers.6.mlp.experts.67.down_proj", + "model.layers.6.mlp.experts.67.gate_proj", + "model.layers.6.mlp.experts.67.up_proj", + "model.layers.6.mlp.experts.68.down_proj", + "model.layers.6.mlp.experts.68.gate_proj", + "model.layers.6.mlp.experts.68.up_proj", + "model.layers.6.mlp.experts.69.down_proj", + "model.layers.6.mlp.experts.69.gate_proj", + "model.layers.6.mlp.experts.69.up_proj", + "model.layers.6.mlp.experts.7.down_proj", + "model.layers.6.mlp.experts.7.gate_proj", + "model.layers.6.mlp.experts.7.up_proj", + "model.layers.6.mlp.experts.70.down_proj", + "model.layers.6.mlp.experts.70.gate_proj", + "model.layers.6.mlp.experts.70.up_proj", + "model.layers.6.mlp.experts.71.down_proj", + "model.layers.6.mlp.experts.71.gate_proj", + "model.layers.6.mlp.experts.71.up_proj", + "model.layers.6.mlp.experts.72.down_proj", + "model.layers.6.mlp.experts.72.gate_proj", + "model.layers.6.mlp.experts.72.up_proj", + "model.layers.6.mlp.experts.73.down_proj", + "model.layers.6.mlp.experts.73.gate_proj", + "model.layers.6.mlp.experts.73.up_proj", + "model.layers.6.mlp.experts.74.down_proj", + "model.layers.6.mlp.experts.74.gate_proj", + "model.layers.6.mlp.experts.74.up_proj", + "model.layers.6.mlp.experts.75.down_proj", + "model.layers.6.mlp.experts.75.gate_proj", + "model.layers.6.mlp.experts.75.up_proj", + "model.layers.6.mlp.experts.76.down_proj", + "model.layers.6.mlp.experts.76.gate_proj", + "model.layers.6.mlp.experts.76.up_proj", + "model.layers.6.mlp.experts.77.down_proj", + "model.layers.6.mlp.experts.77.gate_proj", + "model.layers.6.mlp.experts.77.up_proj", + "model.layers.6.mlp.experts.78.down_proj", + "model.layers.6.mlp.experts.78.gate_proj", + "model.layers.6.mlp.experts.78.up_proj", + "model.layers.6.mlp.experts.79.down_proj", + "model.layers.6.mlp.experts.79.gate_proj", + "model.layers.6.mlp.experts.79.up_proj", + "model.layers.6.mlp.experts.8.down_proj", + "model.layers.6.mlp.experts.8.gate_proj", + "model.layers.6.mlp.experts.8.up_proj", + "model.layers.6.mlp.experts.80.down_proj", + "model.layers.6.mlp.experts.80.gate_proj", + "model.layers.6.mlp.experts.80.up_proj", + "model.layers.6.mlp.experts.81.down_proj", + "model.layers.6.mlp.experts.81.gate_proj", + "model.layers.6.mlp.experts.81.up_proj", + "model.layers.6.mlp.experts.82.down_proj", + "model.layers.6.mlp.experts.82.gate_proj", + "model.layers.6.mlp.experts.82.up_proj", + "model.layers.6.mlp.experts.83.down_proj", + "model.layers.6.mlp.experts.83.gate_proj", + "model.layers.6.mlp.experts.83.up_proj", + "model.layers.6.mlp.experts.84.down_proj", + "model.layers.6.mlp.experts.84.gate_proj", + "model.layers.6.mlp.experts.84.up_proj", + "model.layers.6.mlp.experts.85.down_proj", + "model.layers.6.mlp.experts.85.gate_proj", + "model.layers.6.mlp.experts.85.up_proj", + "model.layers.6.mlp.experts.86.down_proj", + "model.layers.6.mlp.experts.86.gate_proj", + "model.layers.6.mlp.experts.86.up_proj", + "model.layers.6.mlp.experts.87.down_proj", + "model.layers.6.mlp.experts.87.gate_proj", + "model.layers.6.mlp.experts.87.up_proj", + "model.layers.6.mlp.experts.88.down_proj", + "model.layers.6.mlp.experts.88.gate_proj", + "model.layers.6.mlp.experts.88.up_proj", + "model.layers.6.mlp.experts.89.down_proj", + "model.layers.6.mlp.experts.89.gate_proj", + "model.layers.6.mlp.experts.89.up_proj", + "model.layers.6.mlp.experts.9.down_proj", + "model.layers.6.mlp.experts.9.gate_proj", + "model.layers.6.mlp.experts.9.up_proj", + "model.layers.6.mlp.experts.90.down_proj", + "model.layers.6.mlp.experts.90.gate_proj", + "model.layers.6.mlp.experts.90.up_proj", + "model.layers.6.mlp.experts.91.down_proj", + "model.layers.6.mlp.experts.91.gate_proj", + "model.layers.6.mlp.experts.91.up_proj", + "model.layers.6.mlp.experts.92.down_proj", + "model.layers.6.mlp.experts.92.gate_proj", + "model.layers.6.mlp.experts.92.up_proj", + "model.layers.6.mlp.experts.93.down_proj", + "model.layers.6.mlp.experts.93.gate_proj", + "model.layers.6.mlp.experts.93.up_proj", + "model.layers.6.mlp.experts.94.down_proj", + "model.layers.6.mlp.experts.94.gate_proj", + "model.layers.6.mlp.experts.94.up_proj", + "model.layers.6.mlp.experts.95.down_proj", + "model.layers.6.mlp.experts.95.gate_proj", + "model.layers.6.mlp.experts.95.up_proj", + "model.layers.6.mlp.experts.96.down_proj", + "model.layers.6.mlp.experts.96.gate_proj", + "model.layers.6.mlp.experts.96.up_proj", + "model.layers.6.mlp.experts.97.down_proj", + "model.layers.6.mlp.experts.97.gate_proj", + "model.layers.6.mlp.experts.97.up_proj", + "model.layers.6.mlp.experts.98.down_proj", + "model.layers.6.mlp.experts.98.gate_proj", + "model.layers.6.mlp.experts.98.up_proj", + "model.layers.6.mlp.experts.99.down_proj", + "model.layers.6.mlp.experts.99.gate_proj", + "model.layers.6.mlp.experts.99.up_proj", + "model.layers.60.mlp.experts.0.down_proj", + "model.layers.60.mlp.experts.0.gate_proj", + "model.layers.60.mlp.experts.0.up_proj", + "model.layers.60.mlp.experts.1.down_proj", + "model.layers.60.mlp.experts.1.gate_proj", + "model.layers.60.mlp.experts.1.up_proj", + "model.layers.60.mlp.experts.10.down_proj", + "model.layers.60.mlp.experts.10.gate_proj", + "model.layers.60.mlp.experts.10.up_proj", + "model.layers.60.mlp.experts.100.down_proj", + "model.layers.60.mlp.experts.100.gate_proj", + "model.layers.60.mlp.experts.100.up_proj", + "model.layers.60.mlp.experts.101.down_proj", + "model.layers.60.mlp.experts.101.gate_proj", + "model.layers.60.mlp.experts.101.up_proj", + "model.layers.60.mlp.experts.102.down_proj", + "model.layers.60.mlp.experts.102.gate_proj", + "model.layers.60.mlp.experts.102.up_proj", + "model.layers.60.mlp.experts.103.down_proj", + "model.layers.60.mlp.experts.103.gate_proj", + "model.layers.60.mlp.experts.103.up_proj", + "model.layers.60.mlp.experts.104.down_proj", + "model.layers.60.mlp.experts.104.gate_proj", + "model.layers.60.mlp.experts.104.up_proj", + "model.layers.60.mlp.experts.105.down_proj", + "model.layers.60.mlp.experts.105.gate_proj", + "model.layers.60.mlp.experts.105.up_proj", + "model.layers.60.mlp.experts.106.down_proj", + "model.layers.60.mlp.experts.106.gate_proj", + "model.layers.60.mlp.experts.106.up_proj", + "model.layers.60.mlp.experts.107.down_proj", + "model.layers.60.mlp.experts.107.gate_proj", + "model.layers.60.mlp.experts.107.up_proj", + "model.layers.60.mlp.experts.108.down_proj", + "model.layers.60.mlp.experts.108.gate_proj", + "model.layers.60.mlp.experts.108.up_proj", + "model.layers.60.mlp.experts.109.down_proj", + "model.layers.60.mlp.experts.109.gate_proj", + "model.layers.60.mlp.experts.109.up_proj", + "model.layers.60.mlp.experts.11.down_proj", + "model.layers.60.mlp.experts.11.gate_proj", + "model.layers.60.mlp.experts.11.up_proj", + "model.layers.60.mlp.experts.110.down_proj", + "model.layers.60.mlp.experts.110.gate_proj", + "model.layers.60.mlp.experts.110.up_proj", + "model.layers.60.mlp.experts.111.down_proj", + "model.layers.60.mlp.experts.111.gate_proj", + "model.layers.60.mlp.experts.111.up_proj", + "model.layers.60.mlp.experts.112.down_proj", + "model.layers.60.mlp.experts.112.gate_proj", + "model.layers.60.mlp.experts.112.up_proj", + "model.layers.60.mlp.experts.113.down_proj", + "model.layers.60.mlp.experts.113.gate_proj", + "model.layers.60.mlp.experts.113.up_proj", + "model.layers.60.mlp.experts.114.down_proj", + "model.layers.60.mlp.experts.114.gate_proj", + "model.layers.60.mlp.experts.114.up_proj", + "model.layers.60.mlp.experts.115.down_proj", + "model.layers.60.mlp.experts.115.gate_proj", + "model.layers.60.mlp.experts.115.up_proj", + "model.layers.60.mlp.experts.116.down_proj", + "model.layers.60.mlp.experts.116.gate_proj", + "model.layers.60.mlp.experts.116.up_proj", + "model.layers.60.mlp.experts.117.down_proj", + "model.layers.60.mlp.experts.117.gate_proj", + "model.layers.60.mlp.experts.117.up_proj", + "model.layers.60.mlp.experts.118.down_proj", + "model.layers.60.mlp.experts.118.gate_proj", + "model.layers.60.mlp.experts.118.up_proj", + "model.layers.60.mlp.experts.119.down_proj", + "model.layers.60.mlp.experts.119.gate_proj", + "model.layers.60.mlp.experts.119.up_proj", + "model.layers.60.mlp.experts.12.down_proj", + "model.layers.60.mlp.experts.12.gate_proj", + "model.layers.60.mlp.experts.12.up_proj", + "model.layers.60.mlp.experts.120.down_proj", + "model.layers.60.mlp.experts.120.gate_proj", + "model.layers.60.mlp.experts.120.up_proj", + "model.layers.60.mlp.experts.121.down_proj", + "model.layers.60.mlp.experts.121.gate_proj", + "model.layers.60.mlp.experts.121.up_proj", + "model.layers.60.mlp.experts.122.down_proj", + "model.layers.60.mlp.experts.122.gate_proj", + "model.layers.60.mlp.experts.122.up_proj", + "model.layers.60.mlp.experts.123.down_proj", + "model.layers.60.mlp.experts.123.gate_proj", + "model.layers.60.mlp.experts.123.up_proj", + "model.layers.60.mlp.experts.124.down_proj", + "model.layers.60.mlp.experts.124.gate_proj", + "model.layers.60.mlp.experts.124.up_proj", + "model.layers.60.mlp.experts.125.down_proj", + "model.layers.60.mlp.experts.125.gate_proj", + "model.layers.60.mlp.experts.125.up_proj", + "model.layers.60.mlp.experts.126.down_proj", + "model.layers.60.mlp.experts.126.gate_proj", + "model.layers.60.mlp.experts.126.up_proj", + "model.layers.60.mlp.experts.127.down_proj", + "model.layers.60.mlp.experts.127.gate_proj", + "model.layers.60.mlp.experts.127.up_proj", + "model.layers.60.mlp.experts.128.down_proj", + "model.layers.60.mlp.experts.128.gate_proj", + "model.layers.60.mlp.experts.128.up_proj", + "model.layers.60.mlp.experts.129.down_proj", + "model.layers.60.mlp.experts.129.gate_proj", + "model.layers.60.mlp.experts.129.up_proj", + "model.layers.60.mlp.experts.13.down_proj", + "model.layers.60.mlp.experts.13.gate_proj", + "model.layers.60.mlp.experts.13.up_proj", + "model.layers.60.mlp.experts.130.down_proj", + "model.layers.60.mlp.experts.130.gate_proj", + "model.layers.60.mlp.experts.130.up_proj", + "model.layers.60.mlp.experts.131.down_proj", + "model.layers.60.mlp.experts.131.gate_proj", + "model.layers.60.mlp.experts.131.up_proj", + "model.layers.60.mlp.experts.132.down_proj", + "model.layers.60.mlp.experts.132.gate_proj", + "model.layers.60.mlp.experts.132.up_proj", + "model.layers.60.mlp.experts.133.down_proj", + "model.layers.60.mlp.experts.133.gate_proj", + "model.layers.60.mlp.experts.133.up_proj", + "model.layers.60.mlp.experts.134.down_proj", + "model.layers.60.mlp.experts.134.gate_proj", + "model.layers.60.mlp.experts.134.up_proj", + "model.layers.60.mlp.experts.135.down_proj", + "model.layers.60.mlp.experts.135.gate_proj", + "model.layers.60.mlp.experts.135.up_proj", + "model.layers.60.mlp.experts.136.down_proj", + "model.layers.60.mlp.experts.136.gate_proj", + "model.layers.60.mlp.experts.136.up_proj", + "model.layers.60.mlp.experts.137.down_proj", + "model.layers.60.mlp.experts.137.gate_proj", + "model.layers.60.mlp.experts.137.up_proj", + "model.layers.60.mlp.experts.138.down_proj", + "model.layers.60.mlp.experts.138.gate_proj", + "model.layers.60.mlp.experts.138.up_proj", + "model.layers.60.mlp.experts.139.down_proj", + "model.layers.60.mlp.experts.139.gate_proj", + "model.layers.60.mlp.experts.139.up_proj", + "model.layers.60.mlp.experts.14.down_proj", + "model.layers.60.mlp.experts.14.gate_proj", + "model.layers.60.mlp.experts.14.up_proj", + "model.layers.60.mlp.experts.140.down_proj", + "model.layers.60.mlp.experts.140.gate_proj", + "model.layers.60.mlp.experts.140.up_proj", + "model.layers.60.mlp.experts.141.down_proj", + "model.layers.60.mlp.experts.141.gate_proj", + "model.layers.60.mlp.experts.141.up_proj", + "model.layers.60.mlp.experts.142.down_proj", + "model.layers.60.mlp.experts.142.gate_proj", + "model.layers.60.mlp.experts.142.up_proj", + "model.layers.60.mlp.experts.143.down_proj", + "model.layers.60.mlp.experts.143.gate_proj", + "model.layers.60.mlp.experts.143.up_proj", + "model.layers.60.mlp.experts.144.down_proj", + "model.layers.60.mlp.experts.144.gate_proj", + "model.layers.60.mlp.experts.144.up_proj", + "model.layers.60.mlp.experts.145.down_proj", + "model.layers.60.mlp.experts.145.gate_proj", + "model.layers.60.mlp.experts.145.up_proj", + "model.layers.60.mlp.experts.146.down_proj", + "model.layers.60.mlp.experts.146.gate_proj", + "model.layers.60.mlp.experts.146.up_proj", + "model.layers.60.mlp.experts.147.down_proj", + "model.layers.60.mlp.experts.147.gate_proj", + "model.layers.60.mlp.experts.147.up_proj", + "model.layers.60.mlp.experts.148.down_proj", + "model.layers.60.mlp.experts.148.gate_proj", + "model.layers.60.mlp.experts.148.up_proj", + "model.layers.60.mlp.experts.149.down_proj", + "model.layers.60.mlp.experts.149.gate_proj", + "model.layers.60.mlp.experts.149.up_proj", + "model.layers.60.mlp.experts.15.down_proj", + "model.layers.60.mlp.experts.15.gate_proj", + "model.layers.60.mlp.experts.15.up_proj", + "model.layers.60.mlp.experts.150.down_proj", + "model.layers.60.mlp.experts.150.gate_proj", + "model.layers.60.mlp.experts.150.up_proj", + "model.layers.60.mlp.experts.151.down_proj", + "model.layers.60.mlp.experts.151.gate_proj", + "model.layers.60.mlp.experts.151.up_proj", + "model.layers.60.mlp.experts.152.down_proj", + "model.layers.60.mlp.experts.152.gate_proj", + "model.layers.60.mlp.experts.152.up_proj", + "model.layers.60.mlp.experts.153.down_proj", + "model.layers.60.mlp.experts.153.gate_proj", + "model.layers.60.mlp.experts.153.up_proj", + "model.layers.60.mlp.experts.154.down_proj", + "model.layers.60.mlp.experts.154.gate_proj", + "model.layers.60.mlp.experts.154.up_proj", + "model.layers.60.mlp.experts.155.down_proj", + "model.layers.60.mlp.experts.155.gate_proj", + "model.layers.60.mlp.experts.155.up_proj", + "model.layers.60.mlp.experts.156.down_proj", + "model.layers.60.mlp.experts.156.gate_proj", + "model.layers.60.mlp.experts.156.up_proj", + "model.layers.60.mlp.experts.157.down_proj", + "model.layers.60.mlp.experts.157.gate_proj", + "model.layers.60.mlp.experts.157.up_proj", + "model.layers.60.mlp.experts.158.down_proj", + "model.layers.60.mlp.experts.158.gate_proj", + "model.layers.60.mlp.experts.158.up_proj", + "model.layers.60.mlp.experts.159.down_proj", + "model.layers.60.mlp.experts.159.gate_proj", + "model.layers.60.mlp.experts.159.up_proj", + "model.layers.60.mlp.experts.16.down_proj", + "model.layers.60.mlp.experts.16.gate_proj", + "model.layers.60.mlp.experts.16.up_proj", + "model.layers.60.mlp.experts.160.down_proj", + "model.layers.60.mlp.experts.160.gate_proj", + "model.layers.60.mlp.experts.160.up_proj", + "model.layers.60.mlp.experts.161.down_proj", + "model.layers.60.mlp.experts.161.gate_proj", + "model.layers.60.mlp.experts.161.up_proj", + "model.layers.60.mlp.experts.162.down_proj", + "model.layers.60.mlp.experts.162.gate_proj", + "model.layers.60.mlp.experts.162.up_proj", + "model.layers.60.mlp.experts.163.down_proj", + "model.layers.60.mlp.experts.163.gate_proj", + "model.layers.60.mlp.experts.163.up_proj", + "model.layers.60.mlp.experts.164.down_proj", + "model.layers.60.mlp.experts.164.gate_proj", + "model.layers.60.mlp.experts.164.up_proj", + "model.layers.60.mlp.experts.165.down_proj", + "model.layers.60.mlp.experts.165.gate_proj", + "model.layers.60.mlp.experts.165.up_proj", + "model.layers.60.mlp.experts.166.down_proj", + "model.layers.60.mlp.experts.166.gate_proj", + "model.layers.60.mlp.experts.166.up_proj", + "model.layers.60.mlp.experts.167.down_proj", + "model.layers.60.mlp.experts.167.gate_proj", + "model.layers.60.mlp.experts.167.up_proj", + "model.layers.60.mlp.experts.168.down_proj", + "model.layers.60.mlp.experts.168.gate_proj", + "model.layers.60.mlp.experts.168.up_proj", + "model.layers.60.mlp.experts.169.down_proj", + "model.layers.60.mlp.experts.169.gate_proj", + "model.layers.60.mlp.experts.169.up_proj", + "model.layers.60.mlp.experts.17.down_proj", + "model.layers.60.mlp.experts.17.gate_proj", + "model.layers.60.mlp.experts.17.up_proj", + "model.layers.60.mlp.experts.170.down_proj", + "model.layers.60.mlp.experts.170.gate_proj", + "model.layers.60.mlp.experts.170.up_proj", + "model.layers.60.mlp.experts.171.down_proj", + "model.layers.60.mlp.experts.171.gate_proj", + "model.layers.60.mlp.experts.171.up_proj", + "model.layers.60.mlp.experts.172.down_proj", + "model.layers.60.mlp.experts.172.gate_proj", + "model.layers.60.mlp.experts.172.up_proj", + "model.layers.60.mlp.experts.173.down_proj", + "model.layers.60.mlp.experts.173.gate_proj", + "model.layers.60.mlp.experts.173.up_proj", + "model.layers.60.mlp.experts.174.down_proj", + "model.layers.60.mlp.experts.174.gate_proj", + "model.layers.60.mlp.experts.174.up_proj", + "model.layers.60.mlp.experts.175.down_proj", + "model.layers.60.mlp.experts.175.gate_proj", + "model.layers.60.mlp.experts.175.up_proj", + "model.layers.60.mlp.experts.176.down_proj", + "model.layers.60.mlp.experts.176.gate_proj", + "model.layers.60.mlp.experts.176.up_proj", + "model.layers.60.mlp.experts.177.down_proj", + "model.layers.60.mlp.experts.177.gate_proj", + "model.layers.60.mlp.experts.177.up_proj", + "model.layers.60.mlp.experts.178.down_proj", + "model.layers.60.mlp.experts.178.gate_proj", + "model.layers.60.mlp.experts.178.up_proj", + "model.layers.60.mlp.experts.179.down_proj", + "model.layers.60.mlp.experts.179.gate_proj", + "model.layers.60.mlp.experts.179.up_proj", + "model.layers.60.mlp.experts.18.down_proj", + "model.layers.60.mlp.experts.18.gate_proj", + "model.layers.60.mlp.experts.18.up_proj", + "model.layers.60.mlp.experts.180.down_proj", + "model.layers.60.mlp.experts.180.gate_proj", + "model.layers.60.mlp.experts.180.up_proj", + "model.layers.60.mlp.experts.181.down_proj", + "model.layers.60.mlp.experts.181.gate_proj", + "model.layers.60.mlp.experts.181.up_proj", + "model.layers.60.mlp.experts.182.down_proj", + "model.layers.60.mlp.experts.182.gate_proj", + "model.layers.60.mlp.experts.182.up_proj", + "model.layers.60.mlp.experts.183.down_proj", + "model.layers.60.mlp.experts.183.gate_proj", + "model.layers.60.mlp.experts.183.up_proj", + "model.layers.60.mlp.experts.184.down_proj", + "model.layers.60.mlp.experts.184.gate_proj", + "model.layers.60.mlp.experts.184.up_proj", + "model.layers.60.mlp.experts.185.down_proj", + "model.layers.60.mlp.experts.185.gate_proj", + "model.layers.60.mlp.experts.185.up_proj", + "model.layers.60.mlp.experts.186.down_proj", + "model.layers.60.mlp.experts.186.gate_proj", + "model.layers.60.mlp.experts.186.up_proj", + "model.layers.60.mlp.experts.187.down_proj", + "model.layers.60.mlp.experts.187.gate_proj", + "model.layers.60.mlp.experts.187.up_proj", + "model.layers.60.mlp.experts.188.down_proj", + "model.layers.60.mlp.experts.188.gate_proj", + "model.layers.60.mlp.experts.188.up_proj", + "model.layers.60.mlp.experts.189.down_proj", + "model.layers.60.mlp.experts.189.gate_proj", + "model.layers.60.mlp.experts.189.up_proj", + "model.layers.60.mlp.experts.19.down_proj", + "model.layers.60.mlp.experts.19.gate_proj", + "model.layers.60.mlp.experts.19.up_proj", + "model.layers.60.mlp.experts.190.down_proj", + "model.layers.60.mlp.experts.190.gate_proj", + "model.layers.60.mlp.experts.190.up_proj", + "model.layers.60.mlp.experts.191.down_proj", + "model.layers.60.mlp.experts.191.gate_proj", + "model.layers.60.mlp.experts.191.up_proj", + "model.layers.60.mlp.experts.192.down_proj", + "model.layers.60.mlp.experts.192.gate_proj", + "model.layers.60.mlp.experts.192.up_proj", + "model.layers.60.mlp.experts.193.down_proj", + "model.layers.60.mlp.experts.193.gate_proj", + "model.layers.60.mlp.experts.193.up_proj", + "model.layers.60.mlp.experts.194.down_proj", + "model.layers.60.mlp.experts.194.gate_proj", + "model.layers.60.mlp.experts.194.up_proj", + "model.layers.60.mlp.experts.195.down_proj", + "model.layers.60.mlp.experts.195.gate_proj", + "model.layers.60.mlp.experts.195.up_proj", + "model.layers.60.mlp.experts.196.down_proj", + "model.layers.60.mlp.experts.196.gate_proj", + "model.layers.60.mlp.experts.196.up_proj", + "model.layers.60.mlp.experts.197.down_proj", + "model.layers.60.mlp.experts.197.gate_proj", + "model.layers.60.mlp.experts.197.up_proj", + "model.layers.60.mlp.experts.198.down_proj", + "model.layers.60.mlp.experts.198.gate_proj", + "model.layers.60.mlp.experts.198.up_proj", + "model.layers.60.mlp.experts.199.down_proj", + "model.layers.60.mlp.experts.199.gate_proj", + "model.layers.60.mlp.experts.199.up_proj", + "model.layers.60.mlp.experts.2.down_proj", + "model.layers.60.mlp.experts.2.gate_proj", + "model.layers.60.mlp.experts.2.up_proj", + "model.layers.60.mlp.experts.20.down_proj", + "model.layers.60.mlp.experts.20.gate_proj", + "model.layers.60.mlp.experts.20.up_proj", + "model.layers.60.mlp.experts.200.down_proj", + "model.layers.60.mlp.experts.200.gate_proj", + "model.layers.60.mlp.experts.200.up_proj", + "model.layers.60.mlp.experts.201.down_proj", + "model.layers.60.mlp.experts.201.gate_proj", + "model.layers.60.mlp.experts.201.up_proj", + "model.layers.60.mlp.experts.202.down_proj", + "model.layers.60.mlp.experts.202.gate_proj", + "model.layers.60.mlp.experts.202.up_proj", + "model.layers.60.mlp.experts.203.down_proj", + "model.layers.60.mlp.experts.203.gate_proj", + "model.layers.60.mlp.experts.203.up_proj", + "model.layers.60.mlp.experts.204.down_proj", + "model.layers.60.mlp.experts.204.gate_proj", + "model.layers.60.mlp.experts.204.up_proj", + "model.layers.60.mlp.experts.205.down_proj", + "model.layers.60.mlp.experts.205.gate_proj", + "model.layers.60.mlp.experts.205.up_proj", + "model.layers.60.mlp.experts.206.down_proj", + "model.layers.60.mlp.experts.206.gate_proj", + "model.layers.60.mlp.experts.206.up_proj", + "model.layers.60.mlp.experts.207.down_proj", + "model.layers.60.mlp.experts.207.gate_proj", + "model.layers.60.mlp.experts.207.up_proj", + "model.layers.60.mlp.experts.208.down_proj", + "model.layers.60.mlp.experts.208.gate_proj", + "model.layers.60.mlp.experts.208.up_proj", + "model.layers.60.mlp.experts.209.down_proj", + "model.layers.60.mlp.experts.209.gate_proj", + "model.layers.60.mlp.experts.209.up_proj", + "model.layers.60.mlp.experts.21.down_proj", + "model.layers.60.mlp.experts.21.gate_proj", + "model.layers.60.mlp.experts.21.up_proj", + "model.layers.60.mlp.experts.210.down_proj", + "model.layers.60.mlp.experts.210.gate_proj", + "model.layers.60.mlp.experts.210.up_proj", + "model.layers.60.mlp.experts.211.down_proj", + "model.layers.60.mlp.experts.211.gate_proj", + "model.layers.60.mlp.experts.211.up_proj", + "model.layers.60.mlp.experts.212.down_proj", + "model.layers.60.mlp.experts.212.gate_proj", + "model.layers.60.mlp.experts.212.up_proj", + "model.layers.60.mlp.experts.213.down_proj", + "model.layers.60.mlp.experts.213.gate_proj", + "model.layers.60.mlp.experts.213.up_proj", + "model.layers.60.mlp.experts.214.down_proj", + "model.layers.60.mlp.experts.214.gate_proj", + "model.layers.60.mlp.experts.214.up_proj", + "model.layers.60.mlp.experts.215.down_proj", + "model.layers.60.mlp.experts.215.gate_proj", + "model.layers.60.mlp.experts.215.up_proj", + "model.layers.60.mlp.experts.216.down_proj", + "model.layers.60.mlp.experts.216.gate_proj", + "model.layers.60.mlp.experts.216.up_proj", + "model.layers.60.mlp.experts.217.down_proj", + "model.layers.60.mlp.experts.217.gate_proj", + "model.layers.60.mlp.experts.217.up_proj", + "model.layers.60.mlp.experts.218.down_proj", + "model.layers.60.mlp.experts.218.gate_proj", + "model.layers.60.mlp.experts.218.up_proj", + "model.layers.60.mlp.experts.219.down_proj", + "model.layers.60.mlp.experts.219.gate_proj", + "model.layers.60.mlp.experts.219.up_proj", + "model.layers.60.mlp.experts.22.down_proj", + "model.layers.60.mlp.experts.22.gate_proj", + "model.layers.60.mlp.experts.22.up_proj", + "model.layers.60.mlp.experts.220.down_proj", + "model.layers.60.mlp.experts.220.gate_proj", + "model.layers.60.mlp.experts.220.up_proj", + "model.layers.60.mlp.experts.221.down_proj", + "model.layers.60.mlp.experts.221.gate_proj", + "model.layers.60.mlp.experts.221.up_proj", + "model.layers.60.mlp.experts.222.down_proj", + "model.layers.60.mlp.experts.222.gate_proj", + "model.layers.60.mlp.experts.222.up_proj", + "model.layers.60.mlp.experts.223.down_proj", + "model.layers.60.mlp.experts.223.gate_proj", + "model.layers.60.mlp.experts.223.up_proj", + "model.layers.60.mlp.experts.224.down_proj", + "model.layers.60.mlp.experts.224.gate_proj", + "model.layers.60.mlp.experts.224.up_proj", + "model.layers.60.mlp.experts.225.down_proj", + "model.layers.60.mlp.experts.225.gate_proj", + "model.layers.60.mlp.experts.225.up_proj", + "model.layers.60.mlp.experts.226.down_proj", + "model.layers.60.mlp.experts.226.gate_proj", + "model.layers.60.mlp.experts.226.up_proj", + "model.layers.60.mlp.experts.227.down_proj", + "model.layers.60.mlp.experts.227.gate_proj", + "model.layers.60.mlp.experts.227.up_proj", + "model.layers.60.mlp.experts.228.down_proj", + "model.layers.60.mlp.experts.228.gate_proj", + "model.layers.60.mlp.experts.228.up_proj", + "model.layers.60.mlp.experts.229.down_proj", + "model.layers.60.mlp.experts.229.gate_proj", + "model.layers.60.mlp.experts.229.up_proj", + "model.layers.60.mlp.experts.23.down_proj", + "model.layers.60.mlp.experts.23.gate_proj", + "model.layers.60.mlp.experts.23.up_proj", + "model.layers.60.mlp.experts.230.down_proj", + "model.layers.60.mlp.experts.230.gate_proj", + "model.layers.60.mlp.experts.230.up_proj", + "model.layers.60.mlp.experts.231.down_proj", + "model.layers.60.mlp.experts.231.gate_proj", + "model.layers.60.mlp.experts.231.up_proj", + "model.layers.60.mlp.experts.232.down_proj", + "model.layers.60.mlp.experts.232.gate_proj", + "model.layers.60.mlp.experts.232.up_proj", + "model.layers.60.mlp.experts.233.down_proj", + "model.layers.60.mlp.experts.233.gate_proj", + "model.layers.60.mlp.experts.233.up_proj", + "model.layers.60.mlp.experts.234.down_proj", + "model.layers.60.mlp.experts.234.gate_proj", + "model.layers.60.mlp.experts.234.up_proj", + "model.layers.60.mlp.experts.235.down_proj", + "model.layers.60.mlp.experts.235.gate_proj", + "model.layers.60.mlp.experts.235.up_proj", + "model.layers.60.mlp.experts.236.down_proj", + "model.layers.60.mlp.experts.236.gate_proj", + "model.layers.60.mlp.experts.236.up_proj", + "model.layers.60.mlp.experts.237.down_proj", + "model.layers.60.mlp.experts.237.gate_proj", + "model.layers.60.mlp.experts.237.up_proj", + "model.layers.60.mlp.experts.238.down_proj", + "model.layers.60.mlp.experts.238.gate_proj", + "model.layers.60.mlp.experts.238.up_proj", + "model.layers.60.mlp.experts.239.down_proj", + "model.layers.60.mlp.experts.239.gate_proj", + "model.layers.60.mlp.experts.239.up_proj", + "model.layers.60.mlp.experts.24.down_proj", + "model.layers.60.mlp.experts.24.gate_proj", + "model.layers.60.mlp.experts.24.up_proj", + "model.layers.60.mlp.experts.240.down_proj", + "model.layers.60.mlp.experts.240.gate_proj", + "model.layers.60.mlp.experts.240.up_proj", + "model.layers.60.mlp.experts.241.down_proj", + "model.layers.60.mlp.experts.241.gate_proj", + "model.layers.60.mlp.experts.241.up_proj", + "model.layers.60.mlp.experts.242.down_proj", + "model.layers.60.mlp.experts.242.gate_proj", + "model.layers.60.mlp.experts.242.up_proj", + "model.layers.60.mlp.experts.243.down_proj", + "model.layers.60.mlp.experts.243.gate_proj", + "model.layers.60.mlp.experts.243.up_proj", + "model.layers.60.mlp.experts.244.down_proj", + "model.layers.60.mlp.experts.244.gate_proj", + "model.layers.60.mlp.experts.244.up_proj", + "model.layers.60.mlp.experts.245.down_proj", + "model.layers.60.mlp.experts.245.gate_proj", + "model.layers.60.mlp.experts.245.up_proj", + "model.layers.60.mlp.experts.246.down_proj", + "model.layers.60.mlp.experts.246.gate_proj", + "model.layers.60.mlp.experts.246.up_proj", + "model.layers.60.mlp.experts.247.down_proj", + "model.layers.60.mlp.experts.247.gate_proj", + "model.layers.60.mlp.experts.247.up_proj", + "model.layers.60.mlp.experts.248.down_proj", + "model.layers.60.mlp.experts.248.gate_proj", + "model.layers.60.mlp.experts.248.up_proj", + "model.layers.60.mlp.experts.249.down_proj", + "model.layers.60.mlp.experts.249.gate_proj", + "model.layers.60.mlp.experts.249.up_proj", + "model.layers.60.mlp.experts.25.down_proj", + "model.layers.60.mlp.experts.25.gate_proj", + "model.layers.60.mlp.experts.25.up_proj", + "model.layers.60.mlp.experts.250.down_proj", + "model.layers.60.mlp.experts.250.gate_proj", + "model.layers.60.mlp.experts.250.up_proj", + "model.layers.60.mlp.experts.251.down_proj", + "model.layers.60.mlp.experts.251.gate_proj", + "model.layers.60.mlp.experts.251.up_proj", + "model.layers.60.mlp.experts.252.down_proj", + "model.layers.60.mlp.experts.252.gate_proj", + "model.layers.60.mlp.experts.252.up_proj", + "model.layers.60.mlp.experts.253.down_proj", + "model.layers.60.mlp.experts.253.gate_proj", + "model.layers.60.mlp.experts.253.up_proj", + "model.layers.60.mlp.experts.254.down_proj", + "model.layers.60.mlp.experts.254.gate_proj", + "model.layers.60.mlp.experts.254.up_proj", + "model.layers.60.mlp.experts.255.down_proj", + "model.layers.60.mlp.experts.255.gate_proj", + "model.layers.60.mlp.experts.255.up_proj", + "model.layers.60.mlp.experts.26.down_proj", + "model.layers.60.mlp.experts.26.gate_proj", + "model.layers.60.mlp.experts.26.up_proj", + "model.layers.60.mlp.experts.27.down_proj", + "model.layers.60.mlp.experts.27.gate_proj", + "model.layers.60.mlp.experts.27.up_proj", + "model.layers.60.mlp.experts.28.down_proj", + "model.layers.60.mlp.experts.28.gate_proj", + "model.layers.60.mlp.experts.28.up_proj", + "model.layers.60.mlp.experts.29.down_proj", + "model.layers.60.mlp.experts.29.gate_proj", + "model.layers.60.mlp.experts.29.up_proj", + "model.layers.60.mlp.experts.3.down_proj", + "model.layers.60.mlp.experts.3.gate_proj", + "model.layers.60.mlp.experts.3.up_proj", + "model.layers.60.mlp.experts.30.down_proj", + "model.layers.60.mlp.experts.30.gate_proj", + "model.layers.60.mlp.experts.30.up_proj", + "model.layers.60.mlp.experts.31.down_proj", + "model.layers.60.mlp.experts.31.gate_proj", + "model.layers.60.mlp.experts.31.up_proj", + "model.layers.60.mlp.experts.32.down_proj", + "model.layers.60.mlp.experts.32.gate_proj", + "model.layers.60.mlp.experts.32.up_proj", + "model.layers.60.mlp.experts.33.down_proj", + "model.layers.60.mlp.experts.33.gate_proj", + "model.layers.60.mlp.experts.33.up_proj", + "model.layers.60.mlp.experts.34.down_proj", + "model.layers.60.mlp.experts.34.gate_proj", + "model.layers.60.mlp.experts.34.up_proj", + "model.layers.60.mlp.experts.35.down_proj", + "model.layers.60.mlp.experts.35.gate_proj", + "model.layers.60.mlp.experts.35.up_proj", + "model.layers.60.mlp.experts.36.down_proj", + "model.layers.60.mlp.experts.36.gate_proj", + "model.layers.60.mlp.experts.36.up_proj", + "model.layers.60.mlp.experts.37.down_proj", + "model.layers.60.mlp.experts.37.gate_proj", + "model.layers.60.mlp.experts.37.up_proj", + "model.layers.60.mlp.experts.38.down_proj", + "model.layers.60.mlp.experts.38.gate_proj", + "model.layers.60.mlp.experts.38.up_proj", + "model.layers.60.mlp.experts.39.down_proj", + "model.layers.60.mlp.experts.39.gate_proj", + "model.layers.60.mlp.experts.39.up_proj", + "model.layers.60.mlp.experts.4.down_proj", + "model.layers.60.mlp.experts.4.gate_proj", + "model.layers.60.mlp.experts.4.up_proj", + "model.layers.60.mlp.experts.40.down_proj", + "model.layers.60.mlp.experts.40.gate_proj", + "model.layers.60.mlp.experts.40.up_proj", + "model.layers.60.mlp.experts.41.down_proj", + "model.layers.60.mlp.experts.41.gate_proj", + "model.layers.60.mlp.experts.41.up_proj", + "model.layers.60.mlp.experts.42.down_proj", + "model.layers.60.mlp.experts.42.gate_proj", + "model.layers.60.mlp.experts.42.up_proj", + "model.layers.60.mlp.experts.43.down_proj", + "model.layers.60.mlp.experts.43.gate_proj", + "model.layers.60.mlp.experts.43.up_proj", + "model.layers.60.mlp.experts.44.down_proj", + "model.layers.60.mlp.experts.44.gate_proj", + "model.layers.60.mlp.experts.44.up_proj", + "model.layers.60.mlp.experts.45.down_proj", + "model.layers.60.mlp.experts.45.gate_proj", + "model.layers.60.mlp.experts.45.up_proj", + "model.layers.60.mlp.experts.46.down_proj", + "model.layers.60.mlp.experts.46.gate_proj", + "model.layers.60.mlp.experts.46.up_proj", + "model.layers.60.mlp.experts.47.down_proj", + "model.layers.60.mlp.experts.47.gate_proj", + "model.layers.60.mlp.experts.47.up_proj", + "model.layers.60.mlp.experts.48.down_proj", + "model.layers.60.mlp.experts.48.gate_proj", + "model.layers.60.mlp.experts.48.up_proj", + "model.layers.60.mlp.experts.49.down_proj", + "model.layers.60.mlp.experts.49.gate_proj", + "model.layers.60.mlp.experts.49.up_proj", + "model.layers.60.mlp.experts.5.down_proj", + "model.layers.60.mlp.experts.5.gate_proj", + "model.layers.60.mlp.experts.5.up_proj", + "model.layers.60.mlp.experts.50.down_proj", + "model.layers.60.mlp.experts.50.gate_proj", + "model.layers.60.mlp.experts.50.up_proj", + "model.layers.60.mlp.experts.51.down_proj", + "model.layers.60.mlp.experts.51.gate_proj", + "model.layers.60.mlp.experts.51.up_proj", + "model.layers.60.mlp.experts.52.down_proj", + "model.layers.60.mlp.experts.52.gate_proj", + "model.layers.60.mlp.experts.52.up_proj", + "model.layers.60.mlp.experts.53.down_proj", + "model.layers.60.mlp.experts.53.gate_proj", + "model.layers.60.mlp.experts.53.up_proj", + "model.layers.60.mlp.experts.54.down_proj", + "model.layers.60.mlp.experts.54.gate_proj", + "model.layers.60.mlp.experts.54.up_proj", + "model.layers.60.mlp.experts.55.down_proj", + "model.layers.60.mlp.experts.55.gate_proj", + "model.layers.60.mlp.experts.55.up_proj", + "model.layers.60.mlp.experts.56.down_proj", + "model.layers.60.mlp.experts.56.gate_proj", + "model.layers.60.mlp.experts.56.up_proj", + "model.layers.60.mlp.experts.57.down_proj", + "model.layers.60.mlp.experts.57.gate_proj", + "model.layers.60.mlp.experts.57.up_proj", + "model.layers.60.mlp.experts.58.down_proj", + "model.layers.60.mlp.experts.58.gate_proj", + "model.layers.60.mlp.experts.58.up_proj", + "model.layers.60.mlp.experts.59.down_proj", + "model.layers.60.mlp.experts.59.gate_proj", + "model.layers.60.mlp.experts.59.up_proj", + "model.layers.60.mlp.experts.6.down_proj", + "model.layers.60.mlp.experts.6.gate_proj", + "model.layers.60.mlp.experts.6.up_proj", + "model.layers.60.mlp.experts.60.down_proj", + "model.layers.60.mlp.experts.60.gate_proj", + "model.layers.60.mlp.experts.60.up_proj", + "model.layers.60.mlp.experts.61.down_proj", + "model.layers.60.mlp.experts.61.gate_proj", + "model.layers.60.mlp.experts.61.up_proj", + "model.layers.60.mlp.experts.62.down_proj", + "model.layers.60.mlp.experts.62.gate_proj", + "model.layers.60.mlp.experts.62.up_proj", + "model.layers.60.mlp.experts.63.down_proj", + "model.layers.60.mlp.experts.63.gate_proj", + "model.layers.60.mlp.experts.63.up_proj", + "model.layers.60.mlp.experts.64.down_proj", + "model.layers.60.mlp.experts.64.gate_proj", + "model.layers.60.mlp.experts.64.up_proj", + "model.layers.60.mlp.experts.65.down_proj", + "model.layers.60.mlp.experts.65.gate_proj", + "model.layers.60.mlp.experts.65.up_proj", + "model.layers.60.mlp.experts.66.down_proj", + "model.layers.60.mlp.experts.66.gate_proj", + "model.layers.60.mlp.experts.66.up_proj", + "model.layers.60.mlp.experts.67.down_proj", + "model.layers.60.mlp.experts.67.gate_proj", + "model.layers.60.mlp.experts.67.up_proj", + "model.layers.60.mlp.experts.68.down_proj", + "model.layers.60.mlp.experts.68.gate_proj", + "model.layers.60.mlp.experts.68.up_proj", + "model.layers.60.mlp.experts.69.down_proj", + "model.layers.60.mlp.experts.69.gate_proj", + "model.layers.60.mlp.experts.69.up_proj", + "model.layers.60.mlp.experts.7.down_proj", + "model.layers.60.mlp.experts.7.gate_proj", + "model.layers.60.mlp.experts.7.up_proj", + "model.layers.60.mlp.experts.70.down_proj", + "model.layers.60.mlp.experts.70.gate_proj", + "model.layers.60.mlp.experts.70.up_proj", + "model.layers.60.mlp.experts.71.down_proj", + "model.layers.60.mlp.experts.71.gate_proj", + "model.layers.60.mlp.experts.71.up_proj", + "model.layers.60.mlp.experts.72.down_proj", + "model.layers.60.mlp.experts.72.gate_proj", + "model.layers.60.mlp.experts.72.up_proj", + "model.layers.60.mlp.experts.73.down_proj", + "model.layers.60.mlp.experts.73.gate_proj", + "model.layers.60.mlp.experts.73.up_proj", + "model.layers.60.mlp.experts.74.down_proj", + "model.layers.60.mlp.experts.74.gate_proj", + "model.layers.60.mlp.experts.74.up_proj", + "model.layers.60.mlp.experts.75.down_proj", + "model.layers.60.mlp.experts.75.gate_proj", + "model.layers.60.mlp.experts.75.up_proj", + "model.layers.60.mlp.experts.76.down_proj", + "model.layers.60.mlp.experts.76.gate_proj", + "model.layers.60.mlp.experts.76.up_proj", + "model.layers.60.mlp.experts.77.down_proj", + "model.layers.60.mlp.experts.77.gate_proj", + "model.layers.60.mlp.experts.77.up_proj", + "model.layers.60.mlp.experts.78.down_proj", + "model.layers.60.mlp.experts.78.gate_proj", + "model.layers.60.mlp.experts.78.up_proj", + "model.layers.60.mlp.experts.79.down_proj", + "model.layers.60.mlp.experts.79.gate_proj", + "model.layers.60.mlp.experts.79.up_proj", + "model.layers.60.mlp.experts.8.down_proj", + "model.layers.60.mlp.experts.8.gate_proj", + "model.layers.60.mlp.experts.8.up_proj", + "model.layers.60.mlp.experts.80.down_proj", + "model.layers.60.mlp.experts.80.gate_proj", + "model.layers.60.mlp.experts.80.up_proj", + "model.layers.60.mlp.experts.81.down_proj", + "model.layers.60.mlp.experts.81.gate_proj", + "model.layers.60.mlp.experts.81.up_proj", + "model.layers.60.mlp.experts.82.down_proj", + "model.layers.60.mlp.experts.82.gate_proj", + "model.layers.60.mlp.experts.82.up_proj", + "model.layers.60.mlp.experts.83.down_proj", + "model.layers.60.mlp.experts.83.gate_proj", + "model.layers.60.mlp.experts.83.up_proj", + "model.layers.60.mlp.experts.84.down_proj", + "model.layers.60.mlp.experts.84.gate_proj", + "model.layers.60.mlp.experts.84.up_proj", + "model.layers.60.mlp.experts.85.down_proj", + "model.layers.60.mlp.experts.85.gate_proj", + "model.layers.60.mlp.experts.85.up_proj", + "model.layers.60.mlp.experts.86.down_proj", + "model.layers.60.mlp.experts.86.gate_proj", + "model.layers.60.mlp.experts.86.up_proj", + "model.layers.60.mlp.experts.87.down_proj", + "model.layers.60.mlp.experts.87.gate_proj", + "model.layers.60.mlp.experts.87.up_proj", + "model.layers.60.mlp.experts.88.down_proj", + "model.layers.60.mlp.experts.88.gate_proj", + "model.layers.60.mlp.experts.88.up_proj", + "model.layers.60.mlp.experts.89.down_proj", + "model.layers.60.mlp.experts.89.gate_proj", + "model.layers.60.mlp.experts.89.up_proj", + "model.layers.60.mlp.experts.9.down_proj", + "model.layers.60.mlp.experts.9.gate_proj", + "model.layers.60.mlp.experts.9.up_proj", + "model.layers.60.mlp.experts.90.down_proj", + "model.layers.60.mlp.experts.90.gate_proj", + "model.layers.60.mlp.experts.90.up_proj", + "model.layers.60.mlp.experts.91.down_proj", + "model.layers.60.mlp.experts.91.gate_proj", + "model.layers.60.mlp.experts.91.up_proj", + "model.layers.60.mlp.experts.92.down_proj", + "model.layers.60.mlp.experts.92.gate_proj", + "model.layers.60.mlp.experts.92.up_proj", + "model.layers.60.mlp.experts.93.down_proj", + "model.layers.60.mlp.experts.93.gate_proj", + "model.layers.60.mlp.experts.93.up_proj", + "model.layers.60.mlp.experts.94.down_proj", + "model.layers.60.mlp.experts.94.gate_proj", + "model.layers.60.mlp.experts.94.up_proj", + "model.layers.60.mlp.experts.95.down_proj", + "model.layers.60.mlp.experts.95.gate_proj", + "model.layers.60.mlp.experts.95.up_proj", + "model.layers.60.mlp.experts.96.down_proj", + "model.layers.60.mlp.experts.96.gate_proj", + "model.layers.60.mlp.experts.96.up_proj", + "model.layers.60.mlp.experts.97.down_proj", + "model.layers.60.mlp.experts.97.gate_proj", + "model.layers.60.mlp.experts.97.up_proj", + "model.layers.60.mlp.experts.98.down_proj", + "model.layers.60.mlp.experts.98.gate_proj", + "model.layers.60.mlp.experts.98.up_proj", + "model.layers.60.mlp.experts.99.down_proj", + "model.layers.60.mlp.experts.99.gate_proj", + "model.layers.60.mlp.experts.99.up_proj", + "model.layers.61.mlp.experts.0.down_proj", + "model.layers.61.mlp.experts.0.gate_proj", + "model.layers.61.mlp.experts.0.up_proj", + "model.layers.61.mlp.experts.1.down_proj", + "model.layers.61.mlp.experts.1.gate_proj", + "model.layers.61.mlp.experts.1.up_proj", + "model.layers.61.mlp.experts.10.down_proj", + "model.layers.61.mlp.experts.10.gate_proj", + "model.layers.61.mlp.experts.10.up_proj", + "model.layers.61.mlp.experts.100.down_proj", + "model.layers.61.mlp.experts.100.gate_proj", + "model.layers.61.mlp.experts.100.up_proj", + "model.layers.61.mlp.experts.101.down_proj", + "model.layers.61.mlp.experts.101.gate_proj", + "model.layers.61.mlp.experts.101.up_proj", + "model.layers.61.mlp.experts.102.down_proj", + "model.layers.61.mlp.experts.102.gate_proj", + "model.layers.61.mlp.experts.102.up_proj", + "model.layers.61.mlp.experts.103.down_proj", + "model.layers.61.mlp.experts.103.gate_proj", + "model.layers.61.mlp.experts.103.up_proj", + "model.layers.61.mlp.experts.104.down_proj", + "model.layers.61.mlp.experts.104.gate_proj", + "model.layers.61.mlp.experts.104.up_proj", + "model.layers.61.mlp.experts.105.down_proj", + "model.layers.61.mlp.experts.105.gate_proj", + "model.layers.61.mlp.experts.105.up_proj", + "model.layers.61.mlp.experts.106.down_proj", + "model.layers.61.mlp.experts.106.gate_proj", + "model.layers.61.mlp.experts.106.up_proj", + "model.layers.61.mlp.experts.107.down_proj", + "model.layers.61.mlp.experts.107.gate_proj", + "model.layers.61.mlp.experts.107.up_proj", + "model.layers.61.mlp.experts.108.down_proj", + "model.layers.61.mlp.experts.108.gate_proj", + "model.layers.61.mlp.experts.108.up_proj", + "model.layers.61.mlp.experts.109.down_proj", + "model.layers.61.mlp.experts.109.gate_proj", + "model.layers.61.mlp.experts.109.up_proj", + "model.layers.61.mlp.experts.11.down_proj", + "model.layers.61.mlp.experts.11.gate_proj", + "model.layers.61.mlp.experts.11.up_proj", + "model.layers.61.mlp.experts.110.down_proj", + "model.layers.61.mlp.experts.110.gate_proj", + "model.layers.61.mlp.experts.110.up_proj", + "model.layers.61.mlp.experts.111.down_proj", + "model.layers.61.mlp.experts.111.gate_proj", + "model.layers.61.mlp.experts.111.up_proj", + "model.layers.61.mlp.experts.112.down_proj", + "model.layers.61.mlp.experts.112.gate_proj", + "model.layers.61.mlp.experts.112.up_proj", + "model.layers.61.mlp.experts.113.down_proj", + "model.layers.61.mlp.experts.113.gate_proj", + "model.layers.61.mlp.experts.113.up_proj", + "model.layers.61.mlp.experts.114.down_proj", + "model.layers.61.mlp.experts.114.gate_proj", + "model.layers.61.mlp.experts.114.up_proj", + "model.layers.61.mlp.experts.115.down_proj", + "model.layers.61.mlp.experts.115.gate_proj", + "model.layers.61.mlp.experts.115.up_proj", + "model.layers.61.mlp.experts.116.down_proj", + "model.layers.61.mlp.experts.116.gate_proj", + "model.layers.61.mlp.experts.116.up_proj", + "model.layers.61.mlp.experts.117.down_proj", + "model.layers.61.mlp.experts.117.gate_proj", + "model.layers.61.mlp.experts.117.up_proj", + "model.layers.61.mlp.experts.118.down_proj", + "model.layers.61.mlp.experts.118.gate_proj", + "model.layers.61.mlp.experts.118.up_proj", + "model.layers.61.mlp.experts.119.down_proj", + "model.layers.61.mlp.experts.119.gate_proj", + "model.layers.61.mlp.experts.119.up_proj", + "model.layers.61.mlp.experts.12.down_proj", + "model.layers.61.mlp.experts.12.gate_proj", + "model.layers.61.mlp.experts.12.up_proj", + "model.layers.61.mlp.experts.120.down_proj", + "model.layers.61.mlp.experts.120.gate_proj", + "model.layers.61.mlp.experts.120.up_proj", + "model.layers.61.mlp.experts.121.down_proj", + "model.layers.61.mlp.experts.121.gate_proj", + "model.layers.61.mlp.experts.121.up_proj", + "model.layers.61.mlp.experts.122.down_proj", + "model.layers.61.mlp.experts.122.gate_proj", + "model.layers.61.mlp.experts.122.up_proj", + "model.layers.61.mlp.experts.123.down_proj", + "model.layers.61.mlp.experts.123.gate_proj", + "model.layers.61.mlp.experts.123.up_proj", + "model.layers.61.mlp.experts.124.down_proj", + "model.layers.61.mlp.experts.124.gate_proj", + "model.layers.61.mlp.experts.124.up_proj", + "model.layers.61.mlp.experts.125.down_proj", + "model.layers.61.mlp.experts.125.gate_proj", + "model.layers.61.mlp.experts.125.up_proj", + "model.layers.61.mlp.experts.126.down_proj", + "model.layers.61.mlp.experts.126.gate_proj", + "model.layers.61.mlp.experts.126.up_proj", + "model.layers.61.mlp.experts.127.down_proj", + "model.layers.61.mlp.experts.127.gate_proj", + "model.layers.61.mlp.experts.127.up_proj", + "model.layers.61.mlp.experts.128.down_proj", + "model.layers.61.mlp.experts.128.gate_proj", + "model.layers.61.mlp.experts.128.up_proj", + "model.layers.61.mlp.experts.129.down_proj", + "model.layers.61.mlp.experts.129.gate_proj", + "model.layers.61.mlp.experts.129.up_proj", + "model.layers.61.mlp.experts.13.down_proj", + "model.layers.61.mlp.experts.13.gate_proj", + "model.layers.61.mlp.experts.13.up_proj", + "model.layers.61.mlp.experts.130.down_proj", + "model.layers.61.mlp.experts.130.gate_proj", + "model.layers.61.mlp.experts.130.up_proj", + "model.layers.61.mlp.experts.131.down_proj", + "model.layers.61.mlp.experts.131.gate_proj", + "model.layers.61.mlp.experts.131.up_proj", + "model.layers.61.mlp.experts.132.down_proj", + "model.layers.61.mlp.experts.132.gate_proj", + "model.layers.61.mlp.experts.132.up_proj", + "model.layers.61.mlp.experts.133.down_proj", + "model.layers.61.mlp.experts.133.gate_proj", + "model.layers.61.mlp.experts.133.up_proj", + "model.layers.61.mlp.experts.134.down_proj", + "model.layers.61.mlp.experts.134.gate_proj", + "model.layers.61.mlp.experts.134.up_proj", + "model.layers.61.mlp.experts.135.down_proj", + "model.layers.61.mlp.experts.135.gate_proj", + "model.layers.61.mlp.experts.135.up_proj", + "model.layers.61.mlp.experts.136.down_proj", + "model.layers.61.mlp.experts.136.gate_proj", + "model.layers.61.mlp.experts.136.up_proj", + "model.layers.61.mlp.experts.137.down_proj", + "model.layers.61.mlp.experts.137.gate_proj", + "model.layers.61.mlp.experts.137.up_proj", + "model.layers.61.mlp.experts.138.down_proj", + "model.layers.61.mlp.experts.138.gate_proj", + "model.layers.61.mlp.experts.138.up_proj", + "model.layers.61.mlp.experts.139.down_proj", + "model.layers.61.mlp.experts.139.gate_proj", + "model.layers.61.mlp.experts.139.up_proj", + "model.layers.61.mlp.experts.14.down_proj", + "model.layers.61.mlp.experts.14.gate_proj", + "model.layers.61.mlp.experts.14.up_proj", + "model.layers.61.mlp.experts.140.down_proj", + "model.layers.61.mlp.experts.140.gate_proj", + "model.layers.61.mlp.experts.140.up_proj", + "model.layers.61.mlp.experts.141.down_proj", + "model.layers.61.mlp.experts.141.gate_proj", + "model.layers.61.mlp.experts.141.up_proj", + "model.layers.61.mlp.experts.142.down_proj", + "model.layers.61.mlp.experts.142.gate_proj", + "model.layers.61.mlp.experts.142.up_proj", + "model.layers.61.mlp.experts.143.down_proj", + "model.layers.61.mlp.experts.143.gate_proj", + "model.layers.61.mlp.experts.143.up_proj", + "model.layers.61.mlp.experts.144.down_proj", + "model.layers.61.mlp.experts.144.gate_proj", + "model.layers.61.mlp.experts.144.up_proj", + "model.layers.61.mlp.experts.145.down_proj", + "model.layers.61.mlp.experts.145.gate_proj", + "model.layers.61.mlp.experts.145.up_proj", + "model.layers.61.mlp.experts.146.down_proj", + "model.layers.61.mlp.experts.146.gate_proj", + "model.layers.61.mlp.experts.146.up_proj", + "model.layers.61.mlp.experts.147.down_proj", + "model.layers.61.mlp.experts.147.gate_proj", + "model.layers.61.mlp.experts.147.up_proj", + "model.layers.61.mlp.experts.148.down_proj", + "model.layers.61.mlp.experts.148.gate_proj", + "model.layers.61.mlp.experts.148.up_proj", + "model.layers.61.mlp.experts.149.down_proj", + "model.layers.61.mlp.experts.149.gate_proj", + "model.layers.61.mlp.experts.149.up_proj", + "model.layers.61.mlp.experts.15.down_proj", + "model.layers.61.mlp.experts.15.gate_proj", + "model.layers.61.mlp.experts.15.up_proj", + "model.layers.61.mlp.experts.150.down_proj", + "model.layers.61.mlp.experts.150.gate_proj", + "model.layers.61.mlp.experts.150.up_proj", + "model.layers.61.mlp.experts.151.down_proj", + "model.layers.61.mlp.experts.151.gate_proj", + "model.layers.61.mlp.experts.151.up_proj", + "model.layers.61.mlp.experts.152.down_proj", + "model.layers.61.mlp.experts.152.gate_proj", + "model.layers.61.mlp.experts.152.up_proj", + "model.layers.61.mlp.experts.153.down_proj", + "model.layers.61.mlp.experts.153.gate_proj", + "model.layers.61.mlp.experts.153.up_proj", + "model.layers.61.mlp.experts.154.down_proj", + "model.layers.61.mlp.experts.154.gate_proj", + "model.layers.61.mlp.experts.154.up_proj", + "model.layers.61.mlp.experts.155.down_proj", + "model.layers.61.mlp.experts.155.gate_proj", + "model.layers.61.mlp.experts.155.up_proj", + "model.layers.61.mlp.experts.156.down_proj", + "model.layers.61.mlp.experts.156.gate_proj", + "model.layers.61.mlp.experts.156.up_proj", + "model.layers.61.mlp.experts.157.down_proj", + "model.layers.61.mlp.experts.157.gate_proj", + "model.layers.61.mlp.experts.157.up_proj", + "model.layers.61.mlp.experts.158.down_proj", + "model.layers.61.mlp.experts.158.gate_proj", + "model.layers.61.mlp.experts.158.up_proj", + "model.layers.61.mlp.experts.159.down_proj", + "model.layers.61.mlp.experts.159.gate_proj", + "model.layers.61.mlp.experts.159.up_proj", + "model.layers.61.mlp.experts.16.down_proj", + "model.layers.61.mlp.experts.16.gate_proj", + "model.layers.61.mlp.experts.16.up_proj", + "model.layers.61.mlp.experts.160.down_proj", + "model.layers.61.mlp.experts.160.gate_proj", + "model.layers.61.mlp.experts.160.up_proj", + "model.layers.61.mlp.experts.161.down_proj", + "model.layers.61.mlp.experts.161.gate_proj", + "model.layers.61.mlp.experts.161.up_proj", + "model.layers.61.mlp.experts.162.down_proj", + "model.layers.61.mlp.experts.162.gate_proj", + "model.layers.61.mlp.experts.162.up_proj", + "model.layers.61.mlp.experts.163.down_proj", + "model.layers.61.mlp.experts.163.gate_proj", + "model.layers.61.mlp.experts.163.up_proj", + "model.layers.61.mlp.experts.164.down_proj", + "model.layers.61.mlp.experts.164.gate_proj", + "model.layers.61.mlp.experts.164.up_proj", + "model.layers.61.mlp.experts.165.down_proj", + "model.layers.61.mlp.experts.165.gate_proj", + "model.layers.61.mlp.experts.165.up_proj", + "model.layers.61.mlp.experts.166.down_proj", + "model.layers.61.mlp.experts.166.gate_proj", + "model.layers.61.mlp.experts.166.up_proj", + "model.layers.61.mlp.experts.167.down_proj", + "model.layers.61.mlp.experts.167.gate_proj", + "model.layers.61.mlp.experts.167.up_proj", + "model.layers.61.mlp.experts.168.down_proj", + "model.layers.61.mlp.experts.168.gate_proj", + "model.layers.61.mlp.experts.168.up_proj", + "model.layers.61.mlp.experts.169.down_proj", + "model.layers.61.mlp.experts.169.gate_proj", + "model.layers.61.mlp.experts.169.up_proj", + "model.layers.61.mlp.experts.17.down_proj", + "model.layers.61.mlp.experts.17.gate_proj", + "model.layers.61.mlp.experts.17.up_proj", + "model.layers.61.mlp.experts.170.down_proj", + "model.layers.61.mlp.experts.170.gate_proj", + "model.layers.61.mlp.experts.170.up_proj", + "model.layers.61.mlp.experts.171.down_proj", + "model.layers.61.mlp.experts.171.gate_proj", + "model.layers.61.mlp.experts.171.up_proj", + "model.layers.61.mlp.experts.172.down_proj", + "model.layers.61.mlp.experts.172.gate_proj", + "model.layers.61.mlp.experts.172.up_proj", + "model.layers.61.mlp.experts.173.down_proj", + "model.layers.61.mlp.experts.173.gate_proj", + "model.layers.61.mlp.experts.173.up_proj", + "model.layers.61.mlp.experts.174.down_proj", + "model.layers.61.mlp.experts.174.gate_proj", + "model.layers.61.mlp.experts.174.up_proj", + "model.layers.61.mlp.experts.175.down_proj", + "model.layers.61.mlp.experts.175.gate_proj", + "model.layers.61.mlp.experts.175.up_proj", + "model.layers.61.mlp.experts.176.down_proj", + "model.layers.61.mlp.experts.176.gate_proj", + "model.layers.61.mlp.experts.176.up_proj", + "model.layers.61.mlp.experts.177.down_proj", + "model.layers.61.mlp.experts.177.gate_proj", + "model.layers.61.mlp.experts.177.up_proj", + "model.layers.61.mlp.experts.178.down_proj", + "model.layers.61.mlp.experts.178.gate_proj", + "model.layers.61.mlp.experts.178.up_proj", + "model.layers.61.mlp.experts.179.down_proj", + "model.layers.61.mlp.experts.179.gate_proj", + "model.layers.61.mlp.experts.179.up_proj", + "model.layers.61.mlp.experts.18.down_proj", + "model.layers.61.mlp.experts.18.gate_proj", + "model.layers.61.mlp.experts.18.up_proj", + "model.layers.61.mlp.experts.180.down_proj", + "model.layers.61.mlp.experts.180.gate_proj", + "model.layers.61.mlp.experts.180.up_proj", + "model.layers.61.mlp.experts.181.down_proj", + "model.layers.61.mlp.experts.181.gate_proj", + "model.layers.61.mlp.experts.181.up_proj", + "model.layers.61.mlp.experts.182.down_proj", + "model.layers.61.mlp.experts.182.gate_proj", + "model.layers.61.mlp.experts.182.up_proj", + "model.layers.61.mlp.experts.183.down_proj", + "model.layers.61.mlp.experts.183.gate_proj", + "model.layers.61.mlp.experts.183.up_proj", + "model.layers.61.mlp.experts.184.down_proj", + "model.layers.61.mlp.experts.184.gate_proj", + "model.layers.61.mlp.experts.184.up_proj", + "model.layers.61.mlp.experts.185.down_proj", + "model.layers.61.mlp.experts.185.gate_proj", + "model.layers.61.mlp.experts.185.up_proj", + "model.layers.61.mlp.experts.186.down_proj", + "model.layers.61.mlp.experts.186.gate_proj", + "model.layers.61.mlp.experts.186.up_proj", + "model.layers.61.mlp.experts.187.down_proj", + "model.layers.61.mlp.experts.187.gate_proj", + "model.layers.61.mlp.experts.187.up_proj", + "model.layers.61.mlp.experts.188.down_proj", + "model.layers.61.mlp.experts.188.gate_proj", + "model.layers.61.mlp.experts.188.up_proj", + "model.layers.61.mlp.experts.189.down_proj", + "model.layers.61.mlp.experts.189.gate_proj", + "model.layers.61.mlp.experts.189.up_proj", + "model.layers.61.mlp.experts.19.down_proj", + "model.layers.61.mlp.experts.19.gate_proj", + "model.layers.61.mlp.experts.19.up_proj", + "model.layers.61.mlp.experts.190.down_proj", + "model.layers.61.mlp.experts.190.gate_proj", + "model.layers.61.mlp.experts.190.up_proj", + "model.layers.61.mlp.experts.191.down_proj", + "model.layers.61.mlp.experts.191.gate_proj", + "model.layers.61.mlp.experts.191.up_proj", + "model.layers.61.mlp.experts.192.down_proj", + "model.layers.61.mlp.experts.192.gate_proj", + "model.layers.61.mlp.experts.192.up_proj", + "model.layers.61.mlp.experts.193.down_proj", + "model.layers.61.mlp.experts.193.gate_proj", + "model.layers.61.mlp.experts.193.up_proj", + "model.layers.61.mlp.experts.194.down_proj", + "model.layers.61.mlp.experts.194.gate_proj", + "model.layers.61.mlp.experts.194.up_proj", + "model.layers.61.mlp.experts.195.down_proj", + "model.layers.61.mlp.experts.195.gate_proj", + "model.layers.61.mlp.experts.195.up_proj", + "model.layers.61.mlp.experts.196.down_proj", + "model.layers.61.mlp.experts.196.gate_proj", + "model.layers.61.mlp.experts.196.up_proj", + "model.layers.61.mlp.experts.197.down_proj", + "model.layers.61.mlp.experts.197.gate_proj", + "model.layers.61.mlp.experts.197.up_proj", + "model.layers.61.mlp.experts.198.down_proj", + "model.layers.61.mlp.experts.198.gate_proj", + "model.layers.61.mlp.experts.198.up_proj", + "model.layers.61.mlp.experts.199.down_proj", + "model.layers.61.mlp.experts.199.gate_proj", + "model.layers.61.mlp.experts.199.up_proj", + "model.layers.61.mlp.experts.2.down_proj", + "model.layers.61.mlp.experts.2.gate_proj", + "model.layers.61.mlp.experts.2.up_proj", + "model.layers.61.mlp.experts.20.down_proj", + "model.layers.61.mlp.experts.20.gate_proj", + "model.layers.61.mlp.experts.20.up_proj", + "model.layers.61.mlp.experts.200.down_proj", + "model.layers.61.mlp.experts.200.gate_proj", + "model.layers.61.mlp.experts.200.up_proj", + "model.layers.61.mlp.experts.201.down_proj", + "model.layers.61.mlp.experts.201.gate_proj", + "model.layers.61.mlp.experts.201.up_proj", + "model.layers.61.mlp.experts.202.down_proj", + "model.layers.61.mlp.experts.202.gate_proj", + "model.layers.61.mlp.experts.202.up_proj", + "model.layers.61.mlp.experts.203.down_proj", + "model.layers.61.mlp.experts.203.gate_proj", + "model.layers.61.mlp.experts.203.up_proj", + "model.layers.61.mlp.experts.204.down_proj", + "model.layers.61.mlp.experts.204.gate_proj", + "model.layers.61.mlp.experts.204.up_proj", + "model.layers.61.mlp.experts.205.down_proj", + "model.layers.61.mlp.experts.205.gate_proj", + "model.layers.61.mlp.experts.205.up_proj", + "model.layers.61.mlp.experts.206.down_proj", + "model.layers.61.mlp.experts.206.gate_proj", + "model.layers.61.mlp.experts.206.up_proj", + "model.layers.61.mlp.experts.207.down_proj", + "model.layers.61.mlp.experts.207.gate_proj", + "model.layers.61.mlp.experts.207.up_proj", + "model.layers.61.mlp.experts.208.down_proj", + "model.layers.61.mlp.experts.208.gate_proj", + "model.layers.61.mlp.experts.208.up_proj", + "model.layers.61.mlp.experts.209.down_proj", + "model.layers.61.mlp.experts.209.gate_proj", + "model.layers.61.mlp.experts.209.up_proj", + "model.layers.61.mlp.experts.21.down_proj", + "model.layers.61.mlp.experts.21.gate_proj", + "model.layers.61.mlp.experts.21.up_proj", + "model.layers.61.mlp.experts.210.down_proj", + "model.layers.61.mlp.experts.210.gate_proj", + "model.layers.61.mlp.experts.210.up_proj", + "model.layers.61.mlp.experts.211.down_proj", + "model.layers.61.mlp.experts.211.gate_proj", + "model.layers.61.mlp.experts.211.up_proj", + "model.layers.61.mlp.experts.212.down_proj", + "model.layers.61.mlp.experts.212.gate_proj", + "model.layers.61.mlp.experts.212.up_proj", + "model.layers.61.mlp.experts.213.down_proj", + "model.layers.61.mlp.experts.213.gate_proj", + "model.layers.61.mlp.experts.213.up_proj", + "model.layers.61.mlp.experts.214.down_proj", + "model.layers.61.mlp.experts.214.gate_proj", + "model.layers.61.mlp.experts.214.up_proj", + "model.layers.61.mlp.experts.215.down_proj", + "model.layers.61.mlp.experts.215.gate_proj", + "model.layers.61.mlp.experts.215.up_proj", + "model.layers.61.mlp.experts.216.down_proj", + "model.layers.61.mlp.experts.216.gate_proj", + "model.layers.61.mlp.experts.216.up_proj", + "model.layers.61.mlp.experts.217.down_proj", + "model.layers.61.mlp.experts.217.gate_proj", + "model.layers.61.mlp.experts.217.up_proj", + "model.layers.61.mlp.experts.218.down_proj", + "model.layers.61.mlp.experts.218.gate_proj", + "model.layers.61.mlp.experts.218.up_proj", + "model.layers.61.mlp.experts.219.down_proj", + "model.layers.61.mlp.experts.219.gate_proj", + "model.layers.61.mlp.experts.219.up_proj", + "model.layers.61.mlp.experts.22.down_proj", + "model.layers.61.mlp.experts.22.gate_proj", + "model.layers.61.mlp.experts.22.up_proj", + "model.layers.61.mlp.experts.220.down_proj", + "model.layers.61.mlp.experts.220.gate_proj", + "model.layers.61.mlp.experts.220.up_proj", + "model.layers.61.mlp.experts.221.down_proj", + "model.layers.61.mlp.experts.221.gate_proj", + "model.layers.61.mlp.experts.221.up_proj", + "model.layers.61.mlp.experts.222.down_proj", + "model.layers.61.mlp.experts.222.gate_proj", + "model.layers.61.mlp.experts.222.up_proj", + "model.layers.61.mlp.experts.223.down_proj", + "model.layers.61.mlp.experts.223.gate_proj", + "model.layers.61.mlp.experts.223.up_proj", + "model.layers.61.mlp.experts.224.down_proj", + "model.layers.61.mlp.experts.224.gate_proj", + "model.layers.61.mlp.experts.224.up_proj", + "model.layers.61.mlp.experts.225.down_proj", + "model.layers.61.mlp.experts.225.gate_proj", + "model.layers.61.mlp.experts.225.up_proj", + "model.layers.61.mlp.experts.226.down_proj", + "model.layers.61.mlp.experts.226.gate_proj", + "model.layers.61.mlp.experts.226.up_proj", + "model.layers.61.mlp.experts.227.down_proj", + "model.layers.61.mlp.experts.227.gate_proj", + "model.layers.61.mlp.experts.227.up_proj", + "model.layers.61.mlp.experts.228.down_proj", + "model.layers.61.mlp.experts.228.gate_proj", + "model.layers.61.mlp.experts.228.up_proj", + "model.layers.61.mlp.experts.229.down_proj", + "model.layers.61.mlp.experts.229.gate_proj", + "model.layers.61.mlp.experts.229.up_proj", + "model.layers.61.mlp.experts.23.down_proj", + "model.layers.61.mlp.experts.23.gate_proj", + "model.layers.61.mlp.experts.23.up_proj", + "model.layers.61.mlp.experts.230.down_proj", + "model.layers.61.mlp.experts.230.gate_proj", + "model.layers.61.mlp.experts.230.up_proj", + "model.layers.61.mlp.experts.231.down_proj", + "model.layers.61.mlp.experts.231.gate_proj", + "model.layers.61.mlp.experts.231.up_proj", + "model.layers.61.mlp.experts.232.down_proj", + "model.layers.61.mlp.experts.232.gate_proj", + "model.layers.61.mlp.experts.232.up_proj", + "model.layers.61.mlp.experts.233.down_proj", + "model.layers.61.mlp.experts.233.gate_proj", + "model.layers.61.mlp.experts.233.up_proj", + "model.layers.61.mlp.experts.234.down_proj", + "model.layers.61.mlp.experts.234.gate_proj", + "model.layers.61.mlp.experts.234.up_proj", + "model.layers.61.mlp.experts.235.down_proj", + "model.layers.61.mlp.experts.235.gate_proj", + "model.layers.61.mlp.experts.235.up_proj", + "model.layers.61.mlp.experts.236.down_proj", + "model.layers.61.mlp.experts.236.gate_proj", + "model.layers.61.mlp.experts.236.up_proj", + "model.layers.61.mlp.experts.237.down_proj", + "model.layers.61.mlp.experts.237.gate_proj", + "model.layers.61.mlp.experts.237.up_proj", + "model.layers.61.mlp.experts.238.down_proj", + "model.layers.61.mlp.experts.238.gate_proj", + "model.layers.61.mlp.experts.238.up_proj", + "model.layers.61.mlp.experts.239.down_proj", + "model.layers.61.mlp.experts.239.gate_proj", + "model.layers.61.mlp.experts.239.up_proj", + "model.layers.61.mlp.experts.24.down_proj", + "model.layers.61.mlp.experts.24.gate_proj", + "model.layers.61.mlp.experts.24.up_proj", + "model.layers.61.mlp.experts.240.down_proj", + "model.layers.61.mlp.experts.240.gate_proj", + "model.layers.61.mlp.experts.240.up_proj", + "model.layers.61.mlp.experts.241.down_proj", + "model.layers.61.mlp.experts.241.gate_proj", + "model.layers.61.mlp.experts.241.up_proj", + "model.layers.61.mlp.experts.242.down_proj", + "model.layers.61.mlp.experts.242.gate_proj", + "model.layers.61.mlp.experts.242.up_proj", + "model.layers.61.mlp.experts.243.down_proj", + "model.layers.61.mlp.experts.243.gate_proj", + "model.layers.61.mlp.experts.243.up_proj", + "model.layers.61.mlp.experts.244.down_proj", + "model.layers.61.mlp.experts.244.gate_proj", + "model.layers.61.mlp.experts.244.up_proj", + "model.layers.61.mlp.experts.245.down_proj", + "model.layers.61.mlp.experts.245.gate_proj", + "model.layers.61.mlp.experts.245.up_proj", + "model.layers.61.mlp.experts.246.down_proj", + "model.layers.61.mlp.experts.246.gate_proj", + "model.layers.61.mlp.experts.246.up_proj", + "model.layers.61.mlp.experts.247.down_proj", + "model.layers.61.mlp.experts.247.gate_proj", + "model.layers.61.mlp.experts.247.up_proj", + "model.layers.61.mlp.experts.248.down_proj", + "model.layers.61.mlp.experts.248.gate_proj", + "model.layers.61.mlp.experts.248.up_proj", + "model.layers.61.mlp.experts.249.down_proj", + "model.layers.61.mlp.experts.249.gate_proj", + "model.layers.61.mlp.experts.249.up_proj", + "model.layers.61.mlp.experts.25.down_proj", + "model.layers.61.mlp.experts.25.gate_proj", + "model.layers.61.mlp.experts.25.up_proj", + "model.layers.61.mlp.experts.250.down_proj", + "model.layers.61.mlp.experts.250.gate_proj", + "model.layers.61.mlp.experts.250.up_proj", + "model.layers.61.mlp.experts.251.down_proj", + "model.layers.61.mlp.experts.251.gate_proj", + "model.layers.61.mlp.experts.251.up_proj", + "model.layers.61.mlp.experts.252.down_proj", + "model.layers.61.mlp.experts.252.gate_proj", + "model.layers.61.mlp.experts.252.up_proj", + "model.layers.61.mlp.experts.253.down_proj", + "model.layers.61.mlp.experts.253.gate_proj", + "model.layers.61.mlp.experts.253.up_proj", + "model.layers.61.mlp.experts.254.down_proj", + "model.layers.61.mlp.experts.254.gate_proj", + "model.layers.61.mlp.experts.254.up_proj", + "model.layers.61.mlp.experts.255.down_proj", + "model.layers.61.mlp.experts.255.gate_proj", + "model.layers.61.mlp.experts.255.up_proj", + "model.layers.61.mlp.experts.26.down_proj", + "model.layers.61.mlp.experts.26.gate_proj", + "model.layers.61.mlp.experts.26.up_proj", + "model.layers.61.mlp.experts.27.down_proj", + "model.layers.61.mlp.experts.27.gate_proj", + "model.layers.61.mlp.experts.27.up_proj", + "model.layers.61.mlp.experts.28.down_proj", + "model.layers.61.mlp.experts.28.gate_proj", + "model.layers.61.mlp.experts.28.up_proj", + "model.layers.61.mlp.experts.29.down_proj", + "model.layers.61.mlp.experts.29.gate_proj", + "model.layers.61.mlp.experts.29.up_proj", + "model.layers.61.mlp.experts.3.down_proj", + "model.layers.61.mlp.experts.3.gate_proj", + "model.layers.61.mlp.experts.3.up_proj", + "model.layers.61.mlp.experts.30.down_proj", + "model.layers.61.mlp.experts.30.gate_proj", + "model.layers.61.mlp.experts.30.up_proj", + "model.layers.61.mlp.experts.31.down_proj", + "model.layers.61.mlp.experts.31.gate_proj", + "model.layers.61.mlp.experts.31.up_proj", + "model.layers.61.mlp.experts.32.down_proj", + "model.layers.61.mlp.experts.32.gate_proj", + "model.layers.61.mlp.experts.32.up_proj", + "model.layers.61.mlp.experts.33.down_proj", + "model.layers.61.mlp.experts.33.gate_proj", + "model.layers.61.mlp.experts.33.up_proj", + "model.layers.61.mlp.experts.34.down_proj", + "model.layers.61.mlp.experts.34.gate_proj", + "model.layers.61.mlp.experts.34.up_proj", + "model.layers.61.mlp.experts.35.down_proj", + "model.layers.61.mlp.experts.35.gate_proj", + "model.layers.61.mlp.experts.35.up_proj", + "model.layers.61.mlp.experts.36.down_proj", + "model.layers.61.mlp.experts.36.gate_proj", + "model.layers.61.mlp.experts.36.up_proj", + "model.layers.61.mlp.experts.37.down_proj", + "model.layers.61.mlp.experts.37.gate_proj", + "model.layers.61.mlp.experts.37.up_proj", + "model.layers.61.mlp.experts.38.down_proj", + "model.layers.61.mlp.experts.38.gate_proj", + "model.layers.61.mlp.experts.38.up_proj", + "model.layers.61.mlp.experts.39.down_proj", + "model.layers.61.mlp.experts.39.gate_proj", + "model.layers.61.mlp.experts.39.up_proj", + "model.layers.61.mlp.experts.4.down_proj", + "model.layers.61.mlp.experts.4.gate_proj", + "model.layers.61.mlp.experts.4.up_proj", + "model.layers.61.mlp.experts.40.down_proj", + "model.layers.61.mlp.experts.40.gate_proj", + "model.layers.61.mlp.experts.40.up_proj", + "model.layers.61.mlp.experts.41.down_proj", + "model.layers.61.mlp.experts.41.gate_proj", + "model.layers.61.mlp.experts.41.up_proj", + "model.layers.61.mlp.experts.42.down_proj", + "model.layers.61.mlp.experts.42.gate_proj", + "model.layers.61.mlp.experts.42.up_proj", + "model.layers.61.mlp.experts.43.down_proj", + "model.layers.61.mlp.experts.43.gate_proj", + "model.layers.61.mlp.experts.43.up_proj", + "model.layers.61.mlp.experts.44.down_proj", + "model.layers.61.mlp.experts.44.gate_proj", + "model.layers.61.mlp.experts.44.up_proj", + "model.layers.61.mlp.experts.45.down_proj", + "model.layers.61.mlp.experts.45.gate_proj", + "model.layers.61.mlp.experts.45.up_proj", + "model.layers.61.mlp.experts.46.down_proj", + "model.layers.61.mlp.experts.46.gate_proj", + "model.layers.61.mlp.experts.46.up_proj", + "model.layers.61.mlp.experts.47.down_proj", + "model.layers.61.mlp.experts.47.gate_proj", + "model.layers.61.mlp.experts.47.up_proj", + "model.layers.61.mlp.experts.48.down_proj", + "model.layers.61.mlp.experts.48.gate_proj", + "model.layers.61.mlp.experts.48.up_proj", + "model.layers.61.mlp.experts.49.down_proj", + "model.layers.61.mlp.experts.49.gate_proj", + "model.layers.61.mlp.experts.49.up_proj", + "model.layers.61.mlp.experts.5.down_proj", + "model.layers.61.mlp.experts.5.gate_proj", + "model.layers.61.mlp.experts.5.up_proj", + "model.layers.61.mlp.experts.50.down_proj", + "model.layers.61.mlp.experts.50.gate_proj", + "model.layers.61.mlp.experts.50.up_proj", + "model.layers.61.mlp.experts.51.down_proj", + "model.layers.61.mlp.experts.51.gate_proj", + "model.layers.61.mlp.experts.51.up_proj", + "model.layers.61.mlp.experts.52.down_proj", + "model.layers.61.mlp.experts.52.gate_proj", + "model.layers.61.mlp.experts.52.up_proj", + "model.layers.61.mlp.experts.53.down_proj", + "model.layers.61.mlp.experts.53.gate_proj", + "model.layers.61.mlp.experts.53.up_proj", + "model.layers.61.mlp.experts.54.down_proj", + "model.layers.61.mlp.experts.54.gate_proj", + "model.layers.61.mlp.experts.54.up_proj", + "model.layers.61.mlp.experts.55.down_proj", + "model.layers.61.mlp.experts.55.gate_proj", + "model.layers.61.mlp.experts.55.up_proj", + "model.layers.61.mlp.experts.56.down_proj", + "model.layers.61.mlp.experts.56.gate_proj", + "model.layers.61.mlp.experts.56.up_proj", + "model.layers.61.mlp.experts.57.down_proj", + "model.layers.61.mlp.experts.57.gate_proj", + "model.layers.61.mlp.experts.57.up_proj", + "model.layers.61.mlp.experts.58.down_proj", + "model.layers.61.mlp.experts.58.gate_proj", + "model.layers.61.mlp.experts.58.up_proj", + "model.layers.61.mlp.experts.59.down_proj", + "model.layers.61.mlp.experts.59.gate_proj", + "model.layers.61.mlp.experts.59.up_proj", + "model.layers.61.mlp.experts.6.down_proj", + "model.layers.61.mlp.experts.6.gate_proj", + "model.layers.61.mlp.experts.6.up_proj", + "model.layers.61.mlp.experts.60.down_proj", + "model.layers.61.mlp.experts.60.gate_proj", + "model.layers.61.mlp.experts.60.up_proj", + "model.layers.61.mlp.experts.61.down_proj", + "model.layers.61.mlp.experts.61.gate_proj", + "model.layers.61.mlp.experts.61.up_proj", + "model.layers.61.mlp.experts.62.down_proj", + "model.layers.61.mlp.experts.62.gate_proj", + "model.layers.61.mlp.experts.62.up_proj", + "model.layers.61.mlp.experts.63.down_proj", + "model.layers.61.mlp.experts.63.gate_proj", + "model.layers.61.mlp.experts.63.up_proj", + "model.layers.61.mlp.experts.64.down_proj", + "model.layers.61.mlp.experts.64.gate_proj", + "model.layers.61.mlp.experts.64.up_proj", + "model.layers.61.mlp.experts.65.down_proj", + "model.layers.61.mlp.experts.65.gate_proj", + "model.layers.61.mlp.experts.65.up_proj", + "model.layers.61.mlp.experts.66.down_proj", + "model.layers.61.mlp.experts.66.gate_proj", + "model.layers.61.mlp.experts.66.up_proj", + "model.layers.61.mlp.experts.67.down_proj", + "model.layers.61.mlp.experts.67.gate_proj", + "model.layers.61.mlp.experts.67.up_proj", + "model.layers.61.mlp.experts.68.down_proj", + "model.layers.61.mlp.experts.68.gate_proj", + "model.layers.61.mlp.experts.68.up_proj", + "model.layers.61.mlp.experts.69.down_proj", + "model.layers.61.mlp.experts.69.gate_proj", + "model.layers.61.mlp.experts.69.up_proj", + "model.layers.61.mlp.experts.7.down_proj", + "model.layers.61.mlp.experts.7.gate_proj", + "model.layers.61.mlp.experts.7.up_proj", + "model.layers.61.mlp.experts.70.down_proj", + "model.layers.61.mlp.experts.70.gate_proj", + "model.layers.61.mlp.experts.70.up_proj", + "model.layers.61.mlp.experts.71.down_proj", + "model.layers.61.mlp.experts.71.gate_proj", + "model.layers.61.mlp.experts.71.up_proj", + "model.layers.61.mlp.experts.72.down_proj", + "model.layers.61.mlp.experts.72.gate_proj", + "model.layers.61.mlp.experts.72.up_proj", + "model.layers.61.mlp.experts.73.down_proj", + "model.layers.61.mlp.experts.73.gate_proj", + "model.layers.61.mlp.experts.73.up_proj", + "model.layers.61.mlp.experts.74.down_proj", + "model.layers.61.mlp.experts.74.gate_proj", + "model.layers.61.mlp.experts.74.up_proj", + "model.layers.61.mlp.experts.75.down_proj", + "model.layers.61.mlp.experts.75.gate_proj", + "model.layers.61.mlp.experts.75.up_proj", + "model.layers.61.mlp.experts.76.down_proj", + "model.layers.61.mlp.experts.76.gate_proj", + "model.layers.61.mlp.experts.76.up_proj", + "model.layers.61.mlp.experts.77.down_proj", + "model.layers.61.mlp.experts.77.gate_proj", + "model.layers.61.mlp.experts.77.up_proj", + "model.layers.61.mlp.experts.78.down_proj", + "model.layers.61.mlp.experts.78.gate_proj", + "model.layers.61.mlp.experts.78.up_proj", + "model.layers.61.mlp.experts.79.down_proj", + "model.layers.61.mlp.experts.79.gate_proj", + "model.layers.61.mlp.experts.79.up_proj", + "model.layers.61.mlp.experts.8.down_proj", + "model.layers.61.mlp.experts.8.gate_proj", + "model.layers.61.mlp.experts.8.up_proj", + "model.layers.61.mlp.experts.80.down_proj", + "model.layers.61.mlp.experts.80.gate_proj", + "model.layers.61.mlp.experts.80.up_proj", + "model.layers.61.mlp.experts.81.down_proj", + "model.layers.61.mlp.experts.81.gate_proj", + "model.layers.61.mlp.experts.81.up_proj", + "model.layers.61.mlp.experts.82.down_proj", + "model.layers.61.mlp.experts.82.gate_proj", + "model.layers.61.mlp.experts.82.up_proj", + "model.layers.61.mlp.experts.83.down_proj", + "model.layers.61.mlp.experts.83.gate_proj", + "model.layers.61.mlp.experts.83.up_proj", + "model.layers.61.mlp.experts.84.down_proj", + "model.layers.61.mlp.experts.84.gate_proj", + "model.layers.61.mlp.experts.84.up_proj", + "model.layers.61.mlp.experts.85.down_proj", + "model.layers.61.mlp.experts.85.gate_proj", + "model.layers.61.mlp.experts.85.up_proj", + "model.layers.61.mlp.experts.86.down_proj", + "model.layers.61.mlp.experts.86.gate_proj", + "model.layers.61.mlp.experts.86.up_proj", + "model.layers.61.mlp.experts.87.down_proj", + "model.layers.61.mlp.experts.87.gate_proj", + "model.layers.61.mlp.experts.87.up_proj", + "model.layers.61.mlp.experts.88.down_proj", + "model.layers.61.mlp.experts.88.gate_proj", + "model.layers.61.mlp.experts.88.up_proj", + "model.layers.61.mlp.experts.89.down_proj", + "model.layers.61.mlp.experts.89.gate_proj", + "model.layers.61.mlp.experts.89.up_proj", + "model.layers.61.mlp.experts.9.down_proj", + "model.layers.61.mlp.experts.9.gate_proj", + "model.layers.61.mlp.experts.9.up_proj", + "model.layers.61.mlp.experts.90.down_proj", + "model.layers.61.mlp.experts.90.gate_proj", + "model.layers.61.mlp.experts.90.up_proj", + "model.layers.61.mlp.experts.91.down_proj", + "model.layers.61.mlp.experts.91.gate_proj", + "model.layers.61.mlp.experts.91.up_proj", + "model.layers.61.mlp.experts.92.down_proj", + "model.layers.61.mlp.experts.92.gate_proj", + "model.layers.61.mlp.experts.92.up_proj", + "model.layers.61.mlp.experts.93.down_proj", + "model.layers.61.mlp.experts.93.gate_proj", + "model.layers.61.mlp.experts.93.up_proj", + "model.layers.61.mlp.experts.94.down_proj", + "model.layers.61.mlp.experts.94.gate_proj", + "model.layers.61.mlp.experts.94.up_proj", + "model.layers.61.mlp.experts.95.down_proj", + "model.layers.61.mlp.experts.95.gate_proj", + "model.layers.61.mlp.experts.95.up_proj", + "model.layers.61.mlp.experts.96.down_proj", + "model.layers.61.mlp.experts.96.gate_proj", + "model.layers.61.mlp.experts.96.up_proj", + "model.layers.61.mlp.experts.97.down_proj", + "model.layers.61.mlp.experts.97.gate_proj", + "model.layers.61.mlp.experts.97.up_proj", + "model.layers.61.mlp.experts.98.down_proj", + "model.layers.61.mlp.experts.98.gate_proj", + "model.layers.61.mlp.experts.98.up_proj", + "model.layers.61.mlp.experts.99.down_proj", + "model.layers.61.mlp.experts.99.gate_proj", + "model.layers.61.mlp.experts.99.up_proj", + "model.layers.62.mlp.experts.0.down_proj", + "model.layers.62.mlp.experts.0.gate_proj", + "model.layers.62.mlp.experts.0.up_proj", + "model.layers.62.mlp.experts.1.down_proj", + "model.layers.62.mlp.experts.1.gate_proj", + "model.layers.62.mlp.experts.1.up_proj", + "model.layers.62.mlp.experts.10.down_proj", + "model.layers.62.mlp.experts.10.gate_proj", + "model.layers.62.mlp.experts.10.up_proj", + "model.layers.62.mlp.experts.100.down_proj", + "model.layers.62.mlp.experts.100.gate_proj", + "model.layers.62.mlp.experts.100.up_proj", + "model.layers.62.mlp.experts.101.down_proj", + "model.layers.62.mlp.experts.101.gate_proj", + "model.layers.62.mlp.experts.101.up_proj", + "model.layers.62.mlp.experts.102.down_proj", + "model.layers.62.mlp.experts.102.gate_proj", + "model.layers.62.mlp.experts.102.up_proj", + "model.layers.62.mlp.experts.103.down_proj", + "model.layers.62.mlp.experts.103.gate_proj", + "model.layers.62.mlp.experts.103.up_proj", + "model.layers.62.mlp.experts.104.down_proj", + "model.layers.62.mlp.experts.104.gate_proj", + "model.layers.62.mlp.experts.104.up_proj", + "model.layers.62.mlp.experts.105.down_proj", + "model.layers.62.mlp.experts.105.gate_proj", + "model.layers.62.mlp.experts.105.up_proj", + "model.layers.62.mlp.experts.106.down_proj", + "model.layers.62.mlp.experts.106.gate_proj", + "model.layers.62.mlp.experts.106.up_proj", + "model.layers.62.mlp.experts.107.down_proj", + "model.layers.62.mlp.experts.107.gate_proj", + "model.layers.62.mlp.experts.107.up_proj", + "model.layers.62.mlp.experts.108.down_proj", + "model.layers.62.mlp.experts.108.gate_proj", + "model.layers.62.mlp.experts.108.up_proj", + "model.layers.62.mlp.experts.109.down_proj", + "model.layers.62.mlp.experts.109.gate_proj", + "model.layers.62.mlp.experts.109.up_proj", + "model.layers.62.mlp.experts.11.down_proj", + "model.layers.62.mlp.experts.11.gate_proj", + "model.layers.62.mlp.experts.11.up_proj", + "model.layers.62.mlp.experts.110.down_proj", + "model.layers.62.mlp.experts.110.gate_proj", + "model.layers.62.mlp.experts.110.up_proj", + "model.layers.62.mlp.experts.111.down_proj", + "model.layers.62.mlp.experts.111.gate_proj", + "model.layers.62.mlp.experts.111.up_proj", + "model.layers.62.mlp.experts.112.down_proj", + "model.layers.62.mlp.experts.112.gate_proj", + "model.layers.62.mlp.experts.112.up_proj", + "model.layers.62.mlp.experts.113.down_proj", + "model.layers.62.mlp.experts.113.gate_proj", + "model.layers.62.mlp.experts.113.up_proj", + "model.layers.62.mlp.experts.114.down_proj", + "model.layers.62.mlp.experts.114.gate_proj", + "model.layers.62.mlp.experts.114.up_proj", + "model.layers.62.mlp.experts.115.down_proj", + "model.layers.62.mlp.experts.115.gate_proj", + "model.layers.62.mlp.experts.115.up_proj", + "model.layers.62.mlp.experts.116.down_proj", + "model.layers.62.mlp.experts.116.gate_proj", + "model.layers.62.mlp.experts.116.up_proj", + "model.layers.62.mlp.experts.117.down_proj", + "model.layers.62.mlp.experts.117.gate_proj", + "model.layers.62.mlp.experts.117.up_proj", + "model.layers.62.mlp.experts.118.down_proj", + "model.layers.62.mlp.experts.118.gate_proj", + "model.layers.62.mlp.experts.118.up_proj", + "model.layers.62.mlp.experts.119.down_proj", + "model.layers.62.mlp.experts.119.gate_proj", + "model.layers.62.mlp.experts.119.up_proj", + "model.layers.62.mlp.experts.12.down_proj", + "model.layers.62.mlp.experts.12.gate_proj", + "model.layers.62.mlp.experts.12.up_proj", + "model.layers.62.mlp.experts.120.down_proj", + "model.layers.62.mlp.experts.120.gate_proj", + "model.layers.62.mlp.experts.120.up_proj", + "model.layers.62.mlp.experts.121.down_proj", + "model.layers.62.mlp.experts.121.gate_proj", + "model.layers.62.mlp.experts.121.up_proj", + "model.layers.62.mlp.experts.122.down_proj", + "model.layers.62.mlp.experts.122.gate_proj", + "model.layers.62.mlp.experts.122.up_proj", + "model.layers.62.mlp.experts.123.down_proj", + "model.layers.62.mlp.experts.123.gate_proj", + "model.layers.62.mlp.experts.123.up_proj", + "model.layers.62.mlp.experts.124.down_proj", + "model.layers.62.mlp.experts.124.gate_proj", + "model.layers.62.mlp.experts.124.up_proj", + "model.layers.62.mlp.experts.125.down_proj", + "model.layers.62.mlp.experts.125.gate_proj", + "model.layers.62.mlp.experts.125.up_proj", + "model.layers.62.mlp.experts.126.down_proj", + "model.layers.62.mlp.experts.126.gate_proj", + "model.layers.62.mlp.experts.126.up_proj", + "model.layers.62.mlp.experts.127.down_proj", + "model.layers.62.mlp.experts.127.gate_proj", + "model.layers.62.mlp.experts.127.up_proj", + "model.layers.62.mlp.experts.128.down_proj", + "model.layers.62.mlp.experts.128.gate_proj", + "model.layers.62.mlp.experts.128.up_proj", + "model.layers.62.mlp.experts.129.down_proj", + "model.layers.62.mlp.experts.129.gate_proj", + "model.layers.62.mlp.experts.129.up_proj", + "model.layers.62.mlp.experts.13.down_proj", + "model.layers.62.mlp.experts.13.gate_proj", + "model.layers.62.mlp.experts.13.up_proj", + "model.layers.62.mlp.experts.130.down_proj", + "model.layers.62.mlp.experts.130.gate_proj", + "model.layers.62.mlp.experts.130.up_proj", + "model.layers.62.mlp.experts.131.down_proj", + "model.layers.62.mlp.experts.131.gate_proj", + "model.layers.62.mlp.experts.131.up_proj", + "model.layers.62.mlp.experts.132.down_proj", + "model.layers.62.mlp.experts.132.gate_proj", + "model.layers.62.mlp.experts.132.up_proj", + "model.layers.62.mlp.experts.133.down_proj", + "model.layers.62.mlp.experts.133.gate_proj", + "model.layers.62.mlp.experts.133.up_proj", + "model.layers.62.mlp.experts.134.down_proj", + "model.layers.62.mlp.experts.134.gate_proj", + "model.layers.62.mlp.experts.134.up_proj", + "model.layers.62.mlp.experts.135.down_proj", + "model.layers.62.mlp.experts.135.gate_proj", + "model.layers.62.mlp.experts.135.up_proj", + "model.layers.62.mlp.experts.136.down_proj", + "model.layers.62.mlp.experts.136.gate_proj", + "model.layers.62.mlp.experts.136.up_proj", + "model.layers.62.mlp.experts.137.down_proj", + "model.layers.62.mlp.experts.137.gate_proj", + "model.layers.62.mlp.experts.137.up_proj", + "model.layers.62.mlp.experts.138.down_proj", + "model.layers.62.mlp.experts.138.gate_proj", + "model.layers.62.mlp.experts.138.up_proj", + "model.layers.62.mlp.experts.139.down_proj", + "model.layers.62.mlp.experts.139.gate_proj", + "model.layers.62.mlp.experts.139.up_proj", + "model.layers.62.mlp.experts.14.down_proj", + "model.layers.62.mlp.experts.14.gate_proj", + "model.layers.62.mlp.experts.14.up_proj", + "model.layers.62.mlp.experts.140.down_proj", + "model.layers.62.mlp.experts.140.gate_proj", + "model.layers.62.mlp.experts.140.up_proj", + "model.layers.62.mlp.experts.141.down_proj", + "model.layers.62.mlp.experts.141.gate_proj", + "model.layers.62.mlp.experts.141.up_proj", + "model.layers.62.mlp.experts.142.down_proj", + "model.layers.62.mlp.experts.142.gate_proj", + "model.layers.62.mlp.experts.142.up_proj", + "model.layers.62.mlp.experts.143.down_proj", + "model.layers.62.mlp.experts.143.gate_proj", + "model.layers.62.mlp.experts.143.up_proj", + "model.layers.62.mlp.experts.144.down_proj", + "model.layers.62.mlp.experts.144.gate_proj", + "model.layers.62.mlp.experts.144.up_proj", + "model.layers.62.mlp.experts.145.down_proj", + "model.layers.62.mlp.experts.145.gate_proj", + "model.layers.62.mlp.experts.145.up_proj", + "model.layers.62.mlp.experts.146.down_proj", + "model.layers.62.mlp.experts.146.gate_proj", + "model.layers.62.mlp.experts.146.up_proj", + "model.layers.62.mlp.experts.147.down_proj", + "model.layers.62.mlp.experts.147.gate_proj", + "model.layers.62.mlp.experts.147.up_proj", + "model.layers.62.mlp.experts.148.down_proj", + "model.layers.62.mlp.experts.148.gate_proj", + "model.layers.62.mlp.experts.148.up_proj", + "model.layers.62.mlp.experts.149.down_proj", + "model.layers.62.mlp.experts.149.gate_proj", + "model.layers.62.mlp.experts.149.up_proj", + "model.layers.62.mlp.experts.15.down_proj", + "model.layers.62.mlp.experts.15.gate_proj", + "model.layers.62.mlp.experts.15.up_proj", + "model.layers.62.mlp.experts.150.down_proj", + "model.layers.62.mlp.experts.150.gate_proj", + "model.layers.62.mlp.experts.150.up_proj", + "model.layers.62.mlp.experts.151.down_proj", + "model.layers.62.mlp.experts.151.gate_proj", + "model.layers.62.mlp.experts.151.up_proj", + "model.layers.62.mlp.experts.152.down_proj", + "model.layers.62.mlp.experts.152.gate_proj", + "model.layers.62.mlp.experts.152.up_proj", + "model.layers.62.mlp.experts.153.down_proj", + "model.layers.62.mlp.experts.153.gate_proj", + "model.layers.62.mlp.experts.153.up_proj", + "model.layers.62.mlp.experts.154.down_proj", + "model.layers.62.mlp.experts.154.gate_proj", + "model.layers.62.mlp.experts.154.up_proj", + "model.layers.62.mlp.experts.155.down_proj", + "model.layers.62.mlp.experts.155.gate_proj", + "model.layers.62.mlp.experts.155.up_proj", + "model.layers.62.mlp.experts.156.down_proj", + "model.layers.62.mlp.experts.156.gate_proj", + "model.layers.62.mlp.experts.156.up_proj", + "model.layers.62.mlp.experts.157.down_proj", + "model.layers.62.mlp.experts.157.gate_proj", + "model.layers.62.mlp.experts.157.up_proj", + "model.layers.62.mlp.experts.158.down_proj", + "model.layers.62.mlp.experts.158.gate_proj", + "model.layers.62.mlp.experts.158.up_proj", + "model.layers.62.mlp.experts.159.down_proj", + "model.layers.62.mlp.experts.159.gate_proj", + "model.layers.62.mlp.experts.159.up_proj", + "model.layers.62.mlp.experts.16.down_proj", + "model.layers.62.mlp.experts.16.gate_proj", + "model.layers.62.mlp.experts.16.up_proj", + "model.layers.62.mlp.experts.160.down_proj", + "model.layers.62.mlp.experts.160.gate_proj", + "model.layers.62.mlp.experts.160.up_proj", + "model.layers.62.mlp.experts.161.down_proj", + "model.layers.62.mlp.experts.161.gate_proj", + "model.layers.62.mlp.experts.161.up_proj", + "model.layers.62.mlp.experts.162.down_proj", + "model.layers.62.mlp.experts.162.gate_proj", + "model.layers.62.mlp.experts.162.up_proj", + "model.layers.62.mlp.experts.163.down_proj", + "model.layers.62.mlp.experts.163.gate_proj", + "model.layers.62.mlp.experts.163.up_proj", + "model.layers.62.mlp.experts.164.down_proj", + "model.layers.62.mlp.experts.164.gate_proj", + "model.layers.62.mlp.experts.164.up_proj", + "model.layers.62.mlp.experts.165.down_proj", + "model.layers.62.mlp.experts.165.gate_proj", + "model.layers.62.mlp.experts.165.up_proj", + "model.layers.62.mlp.experts.166.down_proj", + "model.layers.62.mlp.experts.166.gate_proj", + "model.layers.62.mlp.experts.166.up_proj", + "model.layers.62.mlp.experts.167.down_proj", + "model.layers.62.mlp.experts.167.gate_proj", + "model.layers.62.mlp.experts.167.up_proj", + "model.layers.62.mlp.experts.168.down_proj", + "model.layers.62.mlp.experts.168.gate_proj", + "model.layers.62.mlp.experts.168.up_proj", + "model.layers.62.mlp.experts.169.down_proj", + "model.layers.62.mlp.experts.169.gate_proj", + "model.layers.62.mlp.experts.169.up_proj", + "model.layers.62.mlp.experts.17.down_proj", + "model.layers.62.mlp.experts.17.gate_proj", + "model.layers.62.mlp.experts.17.up_proj", + "model.layers.62.mlp.experts.170.down_proj", + "model.layers.62.mlp.experts.170.gate_proj", + "model.layers.62.mlp.experts.170.up_proj", + "model.layers.62.mlp.experts.171.down_proj", + "model.layers.62.mlp.experts.171.gate_proj", + "model.layers.62.mlp.experts.171.up_proj", + "model.layers.62.mlp.experts.172.down_proj", + "model.layers.62.mlp.experts.172.gate_proj", + "model.layers.62.mlp.experts.172.up_proj", + "model.layers.62.mlp.experts.173.down_proj", + "model.layers.62.mlp.experts.173.gate_proj", + "model.layers.62.mlp.experts.173.up_proj", + "model.layers.62.mlp.experts.174.down_proj", + "model.layers.62.mlp.experts.174.gate_proj", + "model.layers.62.mlp.experts.174.up_proj", + "model.layers.62.mlp.experts.175.down_proj", + "model.layers.62.mlp.experts.175.gate_proj", + "model.layers.62.mlp.experts.175.up_proj", + "model.layers.62.mlp.experts.176.down_proj", + "model.layers.62.mlp.experts.176.gate_proj", + "model.layers.62.mlp.experts.176.up_proj", + "model.layers.62.mlp.experts.177.down_proj", + "model.layers.62.mlp.experts.177.gate_proj", + "model.layers.62.mlp.experts.177.up_proj", + "model.layers.62.mlp.experts.178.down_proj", + "model.layers.62.mlp.experts.178.gate_proj", + "model.layers.62.mlp.experts.178.up_proj", + "model.layers.62.mlp.experts.179.down_proj", + "model.layers.62.mlp.experts.179.gate_proj", + "model.layers.62.mlp.experts.179.up_proj", + "model.layers.62.mlp.experts.18.down_proj", + "model.layers.62.mlp.experts.18.gate_proj", + "model.layers.62.mlp.experts.18.up_proj", + "model.layers.62.mlp.experts.180.down_proj", + "model.layers.62.mlp.experts.180.gate_proj", + "model.layers.62.mlp.experts.180.up_proj", + "model.layers.62.mlp.experts.181.down_proj", + "model.layers.62.mlp.experts.181.gate_proj", + "model.layers.62.mlp.experts.181.up_proj", + "model.layers.62.mlp.experts.182.down_proj", + "model.layers.62.mlp.experts.182.gate_proj", + "model.layers.62.mlp.experts.182.up_proj", + "model.layers.62.mlp.experts.183.down_proj", + "model.layers.62.mlp.experts.183.gate_proj", + "model.layers.62.mlp.experts.183.up_proj", + "model.layers.62.mlp.experts.184.down_proj", + "model.layers.62.mlp.experts.184.gate_proj", + "model.layers.62.mlp.experts.184.up_proj", + "model.layers.62.mlp.experts.185.down_proj", + "model.layers.62.mlp.experts.185.gate_proj", + "model.layers.62.mlp.experts.185.up_proj", + "model.layers.62.mlp.experts.186.down_proj", + "model.layers.62.mlp.experts.186.gate_proj", + "model.layers.62.mlp.experts.186.up_proj", + "model.layers.62.mlp.experts.187.down_proj", + "model.layers.62.mlp.experts.187.gate_proj", + "model.layers.62.mlp.experts.187.up_proj", + "model.layers.62.mlp.experts.188.down_proj", + "model.layers.62.mlp.experts.188.gate_proj", + "model.layers.62.mlp.experts.188.up_proj", + "model.layers.62.mlp.experts.189.down_proj", + "model.layers.62.mlp.experts.189.gate_proj", + "model.layers.62.mlp.experts.189.up_proj", + "model.layers.62.mlp.experts.19.down_proj", + "model.layers.62.mlp.experts.19.gate_proj", + "model.layers.62.mlp.experts.19.up_proj", + "model.layers.62.mlp.experts.190.down_proj", + "model.layers.62.mlp.experts.190.gate_proj", + "model.layers.62.mlp.experts.190.up_proj", + "model.layers.62.mlp.experts.191.down_proj", + "model.layers.62.mlp.experts.191.gate_proj", + "model.layers.62.mlp.experts.191.up_proj", + "model.layers.62.mlp.experts.192.down_proj", + "model.layers.62.mlp.experts.192.gate_proj", + "model.layers.62.mlp.experts.192.up_proj", + "model.layers.62.mlp.experts.193.down_proj", + "model.layers.62.mlp.experts.193.gate_proj", + "model.layers.62.mlp.experts.193.up_proj", + "model.layers.62.mlp.experts.194.down_proj", + "model.layers.62.mlp.experts.194.gate_proj", + "model.layers.62.mlp.experts.194.up_proj", + "model.layers.62.mlp.experts.195.down_proj", + "model.layers.62.mlp.experts.195.gate_proj", + "model.layers.62.mlp.experts.195.up_proj", + "model.layers.62.mlp.experts.196.down_proj", + "model.layers.62.mlp.experts.196.gate_proj", + "model.layers.62.mlp.experts.196.up_proj", + "model.layers.62.mlp.experts.197.down_proj", + "model.layers.62.mlp.experts.197.gate_proj", + "model.layers.62.mlp.experts.197.up_proj", + "model.layers.62.mlp.experts.198.down_proj", + "model.layers.62.mlp.experts.198.gate_proj", + "model.layers.62.mlp.experts.198.up_proj", + "model.layers.62.mlp.experts.199.down_proj", + "model.layers.62.mlp.experts.199.gate_proj", + "model.layers.62.mlp.experts.199.up_proj", + "model.layers.62.mlp.experts.2.down_proj", + "model.layers.62.mlp.experts.2.gate_proj", + "model.layers.62.mlp.experts.2.up_proj", + "model.layers.62.mlp.experts.20.down_proj", + "model.layers.62.mlp.experts.20.gate_proj", + "model.layers.62.mlp.experts.20.up_proj", + "model.layers.62.mlp.experts.200.down_proj", + "model.layers.62.mlp.experts.200.gate_proj", + "model.layers.62.mlp.experts.200.up_proj", + "model.layers.62.mlp.experts.201.down_proj", + "model.layers.62.mlp.experts.201.gate_proj", + "model.layers.62.mlp.experts.201.up_proj", + "model.layers.62.mlp.experts.202.down_proj", + "model.layers.62.mlp.experts.202.gate_proj", + "model.layers.62.mlp.experts.202.up_proj", + "model.layers.62.mlp.experts.203.down_proj", + "model.layers.62.mlp.experts.203.gate_proj", + "model.layers.62.mlp.experts.203.up_proj", + "model.layers.62.mlp.experts.204.down_proj", + "model.layers.62.mlp.experts.204.gate_proj", + "model.layers.62.mlp.experts.204.up_proj", + "model.layers.62.mlp.experts.205.down_proj", + "model.layers.62.mlp.experts.205.gate_proj", + "model.layers.62.mlp.experts.205.up_proj", + "model.layers.62.mlp.experts.206.down_proj", + "model.layers.62.mlp.experts.206.gate_proj", + "model.layers.62.mlp.experts.206.up_proj", + "model.layers.62.mlp.experts.207.down_proj", + "model.layers.62.mlp.experts.207.gate_proj", + "model.layers.62.mlp.experts.207.up_proj", + "model.layers.62.mlp.experts.208.down_proj", + "model.layers.62.mlp.experts.208.gate_proj", + "model.layers.62.mlp.experts.208.up_proj", + "model.layers.62.mlp.experts.209.down_proj", + "model.layers.62.mlp.experts.209.gate_proj", + "model.layers.62.mlp.experts.209.up_proj", + "model.layers.62.mlp.experts.21.down_proj", + "model.layers.62.mlp.experts.21.gate_proj", + "model.layers.62.mlp.experts.21.up_proj", + "model.layers.62.mlp.experts.210.down_proj", + "model.layers.62.mlp.experts.210.gate_proj", + "model.layers.62.mlp.experts.210.up_proj", + "model.layers.62.mlp.experts.211.down_proj", + "model.layers.62.mlp.experts.211.gate_proj", + "model.layers.62.mlp.experts.211.up_proj", + "model.layers.62.mlp.experts.212.down_proj", + "model.layers.62.mlp.experts.212.gate_proj", + "model.layers.62.mlp.experts.212.up_proj", + "model.layers.62.mlp.experts.213.down_proj", + "model.layers.62.mlp.experts.213.gate_proj", + "model.layers.62.mlp.experts.213.up_proj", + "model.layers.62.mlp.experts.214.down_proj", + "model.layers.62.mlp.experts.214.gate_proj", + "model.layers.62.mlp.experts.214.up_proj", + "model.layers.62.mlp.experts.215.down_proj", + "model.layers.62.mlp.experts.215.gate_proj", + "model.layers.62.mlp.experts.215.up_proj", + "model.layers.62.mlp.experts.216.down_proj", + "model.layers.62.mlp.experts.216.gate_proj", + "model.layers.62.mlp.experts.216.up_proj", + "model.layers.62.mlp.experts.217.down_proj", + "model.layers.62.mlp.experts.217.gate_proj", + "model.layers.62.mlp.experts.217.up_proj", + "model.layers.62.mlp.experts.218.down_proj", + "model.layers.62.mlp.experts.218.gate_proj", + "model.layers.62.mlp.experts.218.up_proj", + "model.layers.62.mlp.experts.219.down_proj", + "model.layers.62.mlp.experts.219.gate_proj", + "model.layers.62.mlp.experts.219.up_proj", + "model.layers.62.mlp.experts.22.down_proj", + "model.layers.62.mlp.experts.22.gate_proj", + "model.layers.62.mlp.experts.22.up_proj", + "model.layers.62.mlp.experts.220.down_proj", + "model.layers.62.mlp.experts.220.gate_proj", + "model.layers.62.mlp.experts.220.up_proj", + "model.layers.62.mlp.experts.221.down_proj", + "model.layers.62.mlp.experts.221.gate_proj", + "model.layers.62.mlp.experts.221.up_proj", + "model.layers.62.mlp.experts.222.down_proj", + "model.layers.62.mlp.experts.222.gate_proj", + "model.layers.62.mlp.experts.222.up_proj", + "model.layers.62.mlp.experts.223.down_proj", + "model.layers.62.mlp.experts.223.gate_proj", + "model.layers.62.mlp.experts.223.up_proj", + "model.layers.62.mlp.experts.224.down_proj", + "model.layers.62.mlp.experts.224.gate_proj", + "model.layers.62.mlp.experts.224.up_proj", + "model.layers.62.mlp.experts.225.down_proj", + "model.layers.62.mlp.experts.225.gate_proj", + "model.layers.62.mlp.experts.225.up_proj", + "model.layers.62.mlp.experts.226.down_proj", + "model.layers.62.mlp.experts.226.gate_proj", + "model.layers.62.mlp.experts.226.up_proj", + "model.layers.62.mlp.experts.227.down_proj", + "model.layers.62.mlp.experts.227.gate_proj", + "model.layers.62.mlp.experts.227.up_proj", + "model.layers.62.mlp.experts.228.down_proj", + "model.layers.62.mlp.experts.228.gate_proj", + "model.layers.62.mlp.experts.228.up_proj", + "model.layers.62.mlp.experts.229.down_proj", + "model.layers.62.mlp.experts.229.gate_proj", + "model.layers.62.mlp.experts.229.up_proj", + "model.layers.62.mlp.experts.23.down_proj", + "model.layers.62.mlp.experts.23.gate_proj", + "model.layers.62.mlp.experts.23.up_proj", + "model.layers.62.mlp.experts.230.down_proj", + "model.layers.62.mlp.experts.230.gate_proj", + "model.layers.62.mlp.experts.230.up_proj", + "model.layers.62.mlp.experts.231.down_proj", + "model.layers.62.mlp.experts.231.gate_proj", + "model.layers.62.mlp.experts.231.up_proj", + "model.layers.62.mlp.experts.232.down_proj", + "model.layers.62.mlp.experts.232.gate_proj", + "model.layers.62.mlp.experts.232.up_proj", + "model.layers.62.mlp.experts.233.down_proj", + "model.layers.62.mlp.experts.233.gate_proj", + "model.layers.62.mlp.experts.233.up_proj", + "model.layers.62.mlp.experts.234.down_proj", + "model.layers.62.mlp.experts.234.gate_proj", + "model.layers.62.mlp.experts.234.up_proj", + "model.layers.62.mlp.experts.235.down_proj", + "model.layers.62.mlp.experts.235.gate_proj", + "model.layers.62.mlp.experts.235.up_proj", + "model.layers.62.mlp.experts.236.down_proj", + "model.layers.62.mlp.experts.236.gate_proj", + "model.layers.62.mlp.experts.236.up_proj", + "model.layers.62.mlp.experts.237.down_proj", + "model.layers.62.mlp.experts.237.gate_proj", + "model.layers.62.mlp.experts.237.up_proj", + "model.layers.62.mlp.experts.238.down_proj", + "model.layers.62.mlp.experts.238.gate_proj", + "model.layers.62.mlp.experts.238.up_proj", + "model.layers.62.mlp.experts.239.down_proj", + "model.layers.62.mlp.experts.239.gate_proj", + "model.layers.62.mlp.experts.239.up_proj", + "model.layers.62.mlp.experts.24.down_proj", + "model.layers.62.mlp.experts.24.gate_proj", + "model.layers.62.mlp.experts.24.up_proj", + "model.layers.62.mlp.experts.240.down_proj", + "model.layers.62.mlp.experts.240.gate_proj", + "model.layers.62.mlp.experts.240.up_proj", + "model.layers.62.mlp.experts.241.down_proj", + "model.layers.62.mlp.experts.241.gate_proj", + "model.layers.62.mlp.experts.241.up_proj", + "model.layers.62.mlp.experts.242.down_proj", + "model.layers.62.mlp.experts.242.gate_proj", + "model.layers.62.mlp.experts.242.up_proj", + "model.layers.62.mlp.experts.243.down_proj", + "model.layers.62.mlp.experts.243.gate_proj", + "model.layers.62.mlp.experts.243.up_proj", + "model.layers.62.mlp.experts.244.down_proj", + "model.layers.62.mlp.experts.244.gate_proj", + "model.layers.62.mlp.experts.244.up_proj", + "model.layers.62.mlp.experts.245.down_proj", + "model.layers.62.mlp.experts.245.gate_proj", + "model.layers.62.mlp.experts.245.up_proj", + "model.layers.62.mlp.experts.246.down_proj", + "model.layers.62.mlp.experts.246.gate_proj", + "model.layers.62.mlp.experts.246.up_proj", + "model.layers.62.mlp.experts.247.down_proj", + "model.layers.62.mlp.experts.247.gate_proj", + "model.layers.62.mlp.experts.247.up_proj", + "model.layers.62.mlp.experts.248.down_proj", + "model.layers.62.mlp.experts.248.gate_proj", + "model.layers.62.mlp.experts.248.up_proj", + "model.layers.62.mlp.experts.249.down_proj", + "model.layers.62.mlp.experts.249.gate_proj", + "model.layers.62.mlp.experts.249.up_proj", + "model.layers.62.mlp.experts.25.down_proj", + "model.layers.62.mlp.experts.25.gate_proj", + "model.layers.62.mlp.experts.25.up_proj", + "model.layers.62.mlp.experts.250.down_proj", + "model.layers.62.mlp.experts.250.gate_proj", + "model.layers.62.mlp.experts.250.up_proj", + "model.layers.62.mlp.experts.251.down_proj", + "model.layers.62.mlp.experts.251.gate_proj", + "model.layers.62.mlp.experts.251.up_proj", + "model.layers.62.mlp.experts.252.down_proj", + "model.layers.62.mlp.experts.252.gate_proj", + "model.layers.62.mlp.experts.252.up_proj", + "model.layers.62.mlp.experts.253.down_proj", + "model.layers.62.mlp.experts.253.gate_proj", + "model.layers.62.mlp.experts.253.up_proj", + "model.layers.62.mlp.experts.254.down_proj", + "model.layers.62.mlp.experts.254.gate_proj", + "model.layers.62.mlp.experts.254.up_proj", + "model.layers.62.mlp.experts.255.down_proj", + "model.layers.62.mlp.experts.255.gate_proj", + "model.layers.62.mlp.experts.255.up_proj", + "model.layers.62.mlp.experts.26.down_proj", + "model.layers.62.mlp.experts.26.gate_proj", + "model.layers.62.mlp.experts.26.up_proj", + "model.layers.62.mlp.experts.27.down_proj", + "model.layers.62.mlp.experts.27.gate_proj", + "model.layers.62.mlp.experts.27.up_proj", + "model.layers.62.mlp.experts.28.down_proj", + "model.layers.62.mlp.experts.28.gate_proj", + "model.layers.62.mlp.experts.28.up_proj", + "model.layers.62.mlp.experts.29.down_proj", + "model.layers.62.mlp.experts.29.gate_proj", + "model.layers.62.mlp.experts.29.up_proj", + "model.layers.62.mlp.experts.3.down_proj", + "model.layers.62.mlp.experts.3.gate_proj", + "model.layers.62.mlp.experts.3.up_proj", + "model.layers.62.mlp.experts.30.down_proj", + "model.layers.62.mlp.experts.30.gate_proj", + "model.layers.62.mlp.experts.30.up_proj", + "model.layers.62.mlp.experts.31.down_proj", + "model.layers.62.mlp.experts.31.gate_proj", + "model.layers.62.mlp.experts.31.up_proj", + "model.layers.62.mlp.experts.32.down_proj", + "model.layers.62.mlp.experts.32.gate_proj", + "model.layers.62.mlp.experts.32.up_proj", + "model.layers.62.mlp.experts.33.down_proj", + "model.layers.62.mlp.experts.33.gate_proj", + "model.layers.62.mlp.experts.33.up_proj", + "model.layers.62.mlp.experts.34.down_proj", + "model.layers.62.mlp.experts.34.gate_proj", + "model.layers.62.mlp.experts.34.up_proj", + "model.layers.62.mlp.experts.35.down_proj", + "model.layers.62.mlp.experts.35.gate_proj", + "model.layers.62.mlp.experts.35.up_proj", + "model.layers.62.mlp.experts.36.down_proj", + "model.layers.62.mlp.experts.36.gate_proj", + "model.layers.62.mlp.experts.36.up_proj", + "model.layers.62.mlp.experts.37.down_proj", + "model.layers.62.mlp.experts.37.gate_proj", + "model.layers.62.mlp.experts.37.up_proj", + "model.layers.62.mlp.experts.38.down_proj", + "model.layers.62.mlp.experts.38.gate_proj", + "model.layers.62.mlp.experts.38.up_proj", + "model.layers.62.mlp.experts.39.down_proj", + "model.layers.62.mlp.experts.39.gate_proj", + "model.layers.62.mlp.experts.39.up_proj", + "model.layers.62.mlp.experts.4.down_proj", + "model.layers.62.mlp.experts.4.gate_proj", + "model.layers.62.mlp.experts.4.up_proj", + "model.layers.62.mlp.experts.40.down_proj", + "model.layers.62.mlp.experts.40.gate_proj", + "model.layers.62.mlp.experts.40.up_proj", + "model.layers.62.mlp.experts.41.down_proj", + "model.layers.62.mlp.experts.41.gate_proj", + "model.layers.62.mlp.experts.41.up_proj", + "model.layers.62.mlp.experts.42.down_proj", + "model.layers.62.mlp.experts.42.gate_proj", + "model.layers.62.mlp.experts.42.up_proj", + "model.layers.62.mlp.experts.43.down_proj", + "model.layers.62.mlp.experts.43.gate_proj", + "model.layers.62.mlp.experts.43.up_proj", + "model.layers.62.mlp.experts.44.down_proj", + "model.layers.62.mlp.experts.44.gate_proj", + "model.layers.62.mlp.experts.44.up_proj", + "model.layers.62.mlp.experts.45.down_proj", + "model.layers.62.mlp.experts.45.gate_proj", + "model.layers.62.mlp.experts.45.up_proj", + "model.layers.62.mlp.experts.46.down_proj", + "model.layers.62.mlp.experts.46.gate_proj", + "model.layers.62.mlp.experts.46.up_proj", + "model.layers.62.mlp.experts.47.down_proj", + "model.layers.62.mlp.experts.47.gate_proj", + "model.layers.62.mlp.experts.47.up_proj", + "model.layers.62.mlp.experts.48.down_proj", + "model.layers.62.mlp.experts.48.gate_proj", + "model.layers.62.mlp.experts.48.up_proj", + "model.layers.62.mlp.experts.49.down_proj", + "model.layers.62.mlp.experts.49.gate_proj", + "model.layers.62.mlp.experts.49.up_proj", + "model.layers.62.mlp.experts.5.down_proj", + "model.layers.62.mlp.experts.5.gate_proj", + "model.layers.62.mlp.experts.5.up_proj", + "model.layers.62.mlp.experts.50.down_proj", + "model.layers.62.mlp.experts.50.gate_proj", + "model.layers.62.mlp.experts.50.up_proj", + "model.layers.62.mlp.experts.51.down_proj", + "model.layers.62.mlp.experts.51.gate_proj", + "model.layers.62.mlp.experts.51.up_proj", + "model.layers.62.mlp.experts.52.down_proj", + "model.layers.62.mlp.experts.52.gate_proj", + "model.layers.62.mlp.experts.52.up_proj", + "model.layers.62.mlp.experts.53.down_proj", + "model.layers.62.mlp.experts.53.gate_proj", + "model.layers.62.mlp.experts.53.up_proj", + "model.layers.62.mlp.experts.54.down_proj", + "model.layers.62.mlp.experts.54.gate_proj", + "model.layers.62.mlp.experts.54.up_proj", + "model.layers.62.mlp.experts.55.down_proj", + "model.layers.62.mlp.experts.55.gate_proj", + "model.layers.62.mlp.experts.55.up_proj", + "model.layers.62.mlp.experts.56.down_proj", + "model.layers.62.mlp.experts.56.gate_proj", + "model.layers.62.mlp.experts.56.up_proj", + "model.layers.62.mlp.experts.57.down_proj", + "model.layers.62.mlp.experts.57.gate_proj", + "model.layers.62.mlp.experts.57.up_proj", + "model.layers.62.mlp.experts.58.down_proj", + "model.layers.62.mlp.experts.58.gate_proj", + "model.layers.62.mlp.experts.58.up_proj", + "model.layers.62.mlp.experts.59.down_proj", + "model.layers.62.mlp.experts.59.gate_proj", + "model.layers.62.mlp.experts.59.up_proj", + "model.layers.62.mlp.experts.6.down_proj", + "model.layers.62.mlp.experts.6.gate_proj", + "model.layers.62.mlp.experts.6.up_proj", + "model.layers.62.mlp.experts.60.down_proj", + "model.layers.62.mlp.experts.60.gate_proj", + "model.layers.62.mlp.experts.60.up_proj", + "model.layers.62.mlp.experts.61.down_proj", + "model.layers.62.mlp.experts.61.gate_proj", + "model.layers.62.mlp.experts.61.up_proj", + "model.layers.62.mlp.experts.62.down_proj", + "model.layers.62.mlp.experts.62.gate_proj", + "model.layers.62.mlp.experts.62.up_proj", + "model.layers.62.mlp.experts.63.down_proj", + "model.layers.62.mlp.experts.63.gate_proj", + "model.layers.62.mlp.experts.63.up_proj", + "model.layers.62.mlp.experts.64.down_proj", + "model.layers.62.mlp.experts.64.gate_proj", + "model.layers.62.mlp.experts.64.up_proj", + "model.layers.62.mlp.experts.65.down_proj", + "model.layers.62.mlp.experts.65.gate_proj", + "model.layers.62.mlp.experts.65.up_proj", + "model.layers.62.mlp.experts.66.down_proj", + "model.layers.62.mlp.experts.66.gate_proj", + "model.layers.62.mlp.experts.66.up_proj", + "model.layers.62.mlp.experts.67.down_proj", + "model.layers.62.mlp.experts.67.gate_proj", + "model.layers.62.mlp.experts.67.up_proj", + "model.layers.62.mlp.experts.68.down_proj", + "model.layers.62.mlp.experts.68.gate_proj", + "model.layers.62.mlp.experts.68.up_proj", + "model.layers.62.mlp.experts.69.down_proj", + "model.layers.62.mlp.experts.69.gate_proj", + "model.layers.62.mlp.experts.69.up_proj", + "model.layers.62.mlp.experts.7.down_proj", + "model.layers.62.mlp.experts.7.gate_proj", + "model.layers.62.mlp.experts.7.up_proj", + "model.layers.62.mlp.experts.70.down_proj", + "model.layers.62.mlp.experts.70.gate_proj", + "model.layers.62.mlp.experts.70.up_proj", + "model.layers.62.mlp.experts.71.down_proj", + "model.layers.62.mlp.experts.71.gate_proj", + "model.layers.62.mlp.experts.71.up_proj", + "model.layers.62.mlp.experts.72.down_proj", + "model.layers.62.mlp.experts.72.gate_proj", + "model.layers.62.mlp.experts.72.up_proj", + "model.layers.62.mlp.experts.73.down_proj", + "model.layers.62.mlp.experts.73.gate_proj", + "model.layers.62.mlp.experts.73.up_proj", + "model.layers.62.mlp.experts.74.down_proj", + "model.layers.62.mlp.experts.74.gate_proj", + "model.layers.62.mlp.experts.74.up_proj", + "model.layers.62.mlp.experts.75.down_proj", + "model.layers.62.mlp.experts.75.gate_proj", + "model.layers.62.mlp.experts.75.up_proj", + "model.layers.62.mlp.experts.76.down_proj", + "model.layers.62.mlp.experts.76.gate_proj", + "model.layers.62.mlp.experts.76.up_proj", + "model.layers.62.mlp.experts.77.down_proj", + "model.layers.62.mlp.experts.77.gate_proj", + "model.layers.62.mlp.experts.77.up_proj", + "model.layers.62.mlp.experts.78.down_proj", + "model.layers.62.mlp.experts.78.gate_proj", + "model.layers.62.mlp.experts.78.up_proj", + "model.layers.62.mlp.experts.79.down_proj", + "model.layers.62.mlp.experts.79.gate_proj", + "model.layers.62.mlp.experts.79.up_proj", + "model.layers.62.mlp.experts.8.down_proj", + "model.layers.62.mlp.experts.8.gate_proj", + "model.layers.62.mlp.experts.8.up_proj", + "model.layers.62.mlp.experts.80.down_proj", + "model.layers.62.mlp.experts.80.gate_proj", + "model.layers.62.mlp.experts.80.up_proj", + "model.layers.62.mlp.experts.81.down_proj", + "model.layers.62.mlp.experts.81.gate_proj", + "model.layers.62.mlp.experts.81.up_proj", + "model.layers.62.mlp.experts.82.down_proj", + "model.layers.62.mlp.experts.82.gate_proj", + "model.layers.62.mlp.experts.82.up_proj", + "model.layers.62.mlp.experts.83.down_proj", + "model.layers.62.mlp.experts.83.gate_proj", + "model.layers.62.mlp.experts.83.up_proj", + "model.layers.62.mlp.experts.84.down_proj", + "model.layers.62.mlp.experts.84.gate_proj", + "model.layers.62.mlp.experts.84.up_proj", + "model.layers.62.mlp.experts.85.down_proj", + "model.layers.62.mlp.experts.85.gate_proj", + "model.layers.62.mlp.experts.85.up_proj", + "model.layers.62.mlp.experts.86.down_proj", + "model.layers.62.mlp.experts.86.gate_proj", + "model.layers.62.mlp.experts.86.up_proj", + "model.layers.62.mlp.experts.87.down_proj", + "model.layers.62.mlp.experts.87.gate_proj", + "model.layers.62.mlp.experts.87.up_proj", + "model.layers.62.mlp.experts.88.down_proj", + "model.layers.62.mlp.experts.88.gate_proj", + "model.layers.62.mlp.experts.88.up_proj", + "model.layers.62.mlp.experts.89.down_proj", + "model.layers.62.mlp.experts.89.gate_proj", + "model.layers.62.mlp.experts.89.up_proj", + "model.layers.62.mlp.experts.9.down_proj", + "model.layers.62.mlp.experts.9.gate_proj", + "model.layers.62.mlp.experts.9.up_proj", + "model.layers.62.mlp.experts.90.down_proj", + "model.layers.62.mlp.experts.90.gate_proj", + "model.layers.62.mlp.experts.90.up_proj", + "model.layers.62.mlp.experts.91.down_proj", + "model.layers.62.mlp.experts.91.gate_proj", + "model.layers.62.mlp.experts.91.up_proj", + "model.layers.62.mlp.experts.92.down_proj", + "model.layers.62.mlp.experts.92.gate_proj", + "model.layers.62.mlp.experts.92.up_proj", + "model.layers.62.mlp.experts.93.down_proj", + "model.layers.62.mlp.experts.93.gate_proj", + "model.layers.62.mlp.experts.93.up_proj", + "model.layers.62.mlp.experts.94.down_proj", + "model.layers.62.mlp.experts.94.gate_proj", + "model.layers.62.mlp.experts.94.up_proj", + "model.layers.62.mlp.experts.95.down_proj", + "model.layers.62.mlp.experts.95.gate_proj", + "model.layers.62.mlp.experts.95.up_proj", + "model.layers.62.mlp.experts.96.down_proj", + "model.layers.62.mlp.experts.96.gate_proj", + "model.layers.62.mlp.experts.96.up_proj", + "model.layers.62.mlp.experts.97.down_proj", + "model.layers.62.mlp.experts.97.gate_proj", + "model.layers.62.mlp.experts.97.up_proj", + "model.layers.62.mlp.experts.98.down_proj", + "model.layers.62.mlp.experts.98.gate_proj", + "model.layers.62.mlp.experts.98.up_proj", + "model.layers.62.mlp.experts.99.down_proj", + "model.layers.62.mlp.experts.99.gate_proj", + "model.layers.62.mlp.experts.99.up_proj", + "model.layers.63.mlp.experts.0.down_proj", + "model.layers.63.mlp.experts.0.gate_proj", + "model.layers.63.mlp.experts.0.up_proj", + "model.layers.63.mlp.experts.1.down_proj", + "model.layers.63.mlp.experts.1.gate_proj", + "model.layers.63.mlp.experts.1.up_proj", + "model.layers.63.mlp.experts.10.down_proj", + "model.layers.63.mlp.experts.10.gate_proj", + "model.layers.63.mlp.experts.10.up_proj", + "model.layers.63.mlp.experts.100.down_proj", + "model.layers.63.mlp.experts.100.gate_proj", + "model.layers.63.mlp.experts.100.up_proj", + "model.layers.63.mlp.experts.101.down_proj", + "model.layers.63.mlp.experts.101.gate_proj", + "model.layers.63.mlp.experts.101.up_proj", + "model.layers.63.mlp.experts.102.down_proj", + "model.layers.63.mlp.experts.102.gate_proj", + "model.layers.63.mlp.experts.102.up_proj", + "model.layers.63.mlp.experts.103.down_proj", + "model.layers.63.mlp.experts.103.gate_proj", + "model.layers.63.mlp.experts.103.up_proj", + "model.layers.63.mlp.experts.104.down_proj", + "model.layers.63.mlp.experts.104.gate_proj", + "model.layers.63.mlp.experts.104.up_proj", + "model.layers.63.mlp.experts.105.down_proj", + "model.layers.63.mlp.experts.105.gate_proj", + "model.layers.63.mlp.experts.105.up_proj", + "model.layers.63.mlp.experts.106.down_proj", + "model.layers.63.mlp.experts.106.gate_proj", + "model.layers.63.mlp.experts.106.up_proj", + "model.layers.63.mlp.experts.107.down_proj", + "model.layers.63.mlp.experts.107.gate_proj", + "model.layers.63.mlp.experts.107.up_proj", + "model.layers.63.mlp.experts.108.down_proj", + "model.layers.63.mlp.experts.108.gate_proj", + "model.layers.63.mlp.experts.108.up_proj", + "model.layers.63.mlp.experts.109.down_proj", + "model.layers.63.mlp.experts.109.gate_proj", + "model.layers.63.mlp.experts.109.up_proj", + "model.layers.63.mlp.experts.11.down_proj", + "model.layers.63.mlp.experts.11.gate_proj", + "model.layers.63.mlp.experts.11.up_proj", + "model.layers.63.mlp.experts.110.down_proj", + "model.layers.63.mlp.experts.110.gate_proj", + "model.layers.63.mlp.experts.110.up_proj", + "model.layers.63.mlp.experts.111.down_proj", + "model.layers.63.mlp.experts.111.gate_proj", + "model.layers.63.mlp.experts.111.up_proj", + "model.layers.63.mlp.experts.112.down_proj", + "model.layers.63.mlp.experts.112.gate_proj", + "model.layers.63.mlp.experts.112.up_proj", + "model.layers.63.mlp.experts.113.down_proj", + "model.layers.63.mlp.experts.113.gate_proj", + "model.layers.63.mlp.experts.113.up_proj", + "model.layers.63.mlp.experts.114.down_proj", + "model.layers.63.mlp.experts.114.gate_proj", + "model.layers.63.mlp.experts.114.up_proj", + "model.layers.63.mlp.experts.115.down_proj", + "model.layers.63.mlp.experts.115.gate_proj", + "model.layers.63.mlp.experts.115.up_proj", + "model.layers.63.mlp.experts.116.down_proj", + "model.layers.63.mlp.experts.116.gate_proj", + "model.layers.63.mlp.experts.116.up_proj", + "model.layers.63.mlp.experts.117.down_proj", + "model.layers.63.mlp.experts.117.gate_proj", + "model.layers.63.mlp.experts.117.up_proj", + "model.layers.63.mlp.experts.118.down_proj", + "model.layers.63.mlp.experts.118.gate_proj", + "model.layers.63.mlp.experts.118.up_proj", + "model.layers.63.mlp.experts.119.down_proj", + "model.layers.63.mlp.experts.119.gate_proj", + "model.layers.63.mlp.experts.119.up_proj", + "model.layers.63.mlp.experts.12.down_proj", + "model.layers.63.mlp.experts.12.gate_proj", + "model.layers.63.mlp.experts.12.up_proj", + "model.layers.63.mlp.experts.120.down_proj", + "model.layers.63.mlp.experts.120.gate_proj", + "model.layers.63.mlp.experts.120.up_proj", + "model.layers.63.mlp.experts.121.down_proj", + "model.layers.63.mlp.experts.121.gate_proj", + "model.layers.63.mlp.experts.121.up_proj", + "model.layers.63.mlp.experts.122.down_proj", + "model.layers.63.mlp.experts.122.gate_proj", + "model.layers.63.mlp.experts.122.up_proj", + "model.layers.63.mlp.experts.123.down_proj", + "model.layers.63.mlp.experts.123.gate_proj", + "model.layers.63.mlp.experts.123.up_proj", + "model.layers.63.mlp.experts.124.down_proj", + "model.layers.63.mlp.experts.124.gate_proj", + "model.layers.63.mlp.experts.124.up_proj", + "model.layers.63.mlp.experts.125.down_proj", + "model.layers.63.mlp.experts.125.gate_proj", + "model.layers.63.mlp.experts.125.up_proj", + "model.layers.63.mlp.experts.126.down_proj", + "model.layers.63.mlp.experts.126.gate_proj", + "model.layers.63.mlp.experts.126.up_proj", + "model.layers.63.mlp.experts.127.down_proj", + "model.layers.63.mlp.experts.127.gate_proj", + "model.layers.63.mlp.experts.127.up_proj", + "model.layers.63.mlp.experts.128.down_proj", + "model.layers.63.mlp.experts.128.gate_proj", + "model.layers.63.mlp.experts.128.up_proj", + "model.layers.63.mlp.experts.129.down_proj", + "model.layers.63.mlp.experts.129.gate_proj", + "model.layers.63.mlp.experts.129.up_proj", + "model.layers.63.mlp.experts.13.down_proj", + "model.layers.63.mlp.experts.13.gate_proj", + "model.layers.63.mlp.experts.13.up_proj", + "model.layers.63.mlp.experts.130.down_proj", + "model.layers.63.mlp.experts.130.gate_proj", + "model.layers.63.mlp.experts.130.up_proj", + "model.layers.63.mlp.experts.131.down_proj", + "model.layers.63.mlp.experts.131.gate_proj", + "model.layers.63.mlp.experts.131.up_proj", + "model.layers.63.mlp.experts.132.down_proj", + "model.layers.63.mlp.experts.132.gate_proj", + "model.layers.63.mlp.experts.132.up_proj", + "model.layers.63.mlp.experts.133.down_proj", + "model.layers.63.mlp.experts.133.gate_proj", + "model.layers.63.mlp.experts.133.up_proj", + "model.layers.63.mlp.experts.134.down_proj", + "model.layers.63.mlp.experts.134.gate_proj", + "model.layers.63.mlp.experts.134.up_proj", + "model.layers.63.mlp.experts.135.down_proj", + "model.layers.63.mlp.experts.135.gate_proj", + "model.layers.63.mlp.experts.135.up_proj", + "model.layers.63.mlp.experts.136.down_proj", + "model.layers.63.mlp.experts.136.gate_proj", + "model.layers.63.mlp.experts.136.up_proj", + "model.layers.63.mlp.experts.137.down_proj", + "model.layers.63.mlp.experts.137.gate_proj", + "model.layers.63.mlp.experts.137.up_proj", + "model.layers.63.mlp.experts.138.down_proj", + "model.layers.63.mlp.experts.138.gate_proj", + "model.layers.63.mlp.experts.138.up_proj", + "model.layers.63.mlp.experts.139.down_proj", + "model.layers.63.mlp.experts.139.gate_proj", + "model.layers.63.mlp.experts.139.up_proj", + "model.layers.63.mlp.experts.14.down_proj", + "model.layers.63.mlp.experts.14.gate_proj", + "model.layers.63.mlp.experts.14.up_proj", + "model.layers.63.mlp.experts.140.down_proj", + "model.layers.63.mlp.experts.140.gate_proj", + "model.layers.63.mlp.experts.140.up_proj", + "model.layers.63.mlp.experts.141.down_proj", + "model.layers.63.mlp.experts.141.gate_proj", + "model.layers.63.mlp.experts.141.up_proj", + "model.layers.63.mlp.experts.142.down_proj", + "model.layers.63.mlp.experts.142.gate_proj", + "model.layers.63.mlp.experts.142.up_proj", + "model.layers.63.mlp.experts.143.down_proj", + "model.layers.63.mlp.experts.143.gate_proj", + "model.layers.63.mlp.experts.143.up_proj", + "model.layers.63.mlp.experts.144.down_proj", + "model.layers.63.mlp.experts.144.gate_proj", + "model.layers.63.mlp.experts.144.up_proj", + "model.layers.63.mlp.experts.145.down_proj", + "model.layers.63.mlp.experts.145.gate_proj", + "model.layers.63.mlp.experts.145.up_proj", + "model.layers.63.mlp.experts.146.down_proj", + "model.layers.63.mlp.experts.146.gate_proj", + "model.layers.63.mlp.experts.146.up_proj", + "model.layers.63.mlp.experts.147.down_proj", + "model.layers.63.mlp.experts.147.gate_proj", + "model.layers.63.mlp.experts.147.up_proj", + "model.layers.63.mlp.experts.148.down_proj", + "model.layers.63.mlp.experts.148.gate_proj", + "model.layers.63.mlp.experts.148.up_proj", + "model.layers.63.mlp.experts.149.down_proj", + "model.layers.63.mlp.experts.149.gate_proj", + "model.layers.63.mlp.experts.149.up_proj", + "model.layers.63.mlp.experts.15.down_proj", + "model.layers.63.mlp.experts.15.gate_proj", + "model.layers.63.mlp.experts.15.up_proj", + "model.layers.63.mlp.experts.150.down_proj", + "model.layers.63.mlp.experts.150.gate_proj", + "model.layers.63.mlp.experts.150.up_proj", + "model.layers.63.mlp.experts.151.down_proj", + "model.layers.63.mlp.experts.151.gate_proj", + "model.layers.63.mlp.experts.151.up_proj", + "model.layers.63.mlp.experts.152.down_proj", + "model.layers.63.mlp.experts.152.gate_proj", + "model.layers.63.mlp.experts.152.up_proj", + "model.layers.63.mlp.experts.153.down_proj", + "model.layers.63.mlp.experts.153.gate_proj", + "model.layers.63.mlp.experts.153.up_proj", + "model.layers.63.mlp.experts.154.down_proj", + "model.layers.63.mlp.experts.154.gate_proj", + "model.layers.63.mlp.experts.154.up_proj", + "model.layers.63.mlp.experts.155.down_proj", + "model.layers.63.mlp.experts.155.gate_proj", + "model.layers.63.mlp.experts.155.up_proj", + "model.layers.63.mlp.experts.156.down_proj", + "model.layers.63.mlp.experts.156.gate_proj", + "model.layers.63.mlp.experts.156.up_proj", + "model.layers.63.mlp.experts.157.down_proj", + "model.layers.63.mlp.experts.157.gate_proj", + "model.layers.63.mlp.experts.157.up_proj", + "model.layers.63.mlp.experts.158.down_proj", + "model.layers.63.mlp.experts.158.gate_proj", + "model.layers.63.mlp.experts.158.up_proj", + "model.layers.63.mlp.experts.159.down_proj", + "model.layers.63.mlp.experts.159.gate_proj", + "model.layers.63.mlp.experts.159.up_proj", + "model.layers.63.mlp.experts.16.down_proj", + "model.layers.63.mlp.experts.16.gate_proj", + "model.layers.63.mlp.experts.16.up_proj", + "model.layers.63.mlp.experts.160.down_proj", + "model.layers.63.mlp.experts.160.gate_proj", + "model.layers.63.mlp.experts.160.up_proj", + "model.layers.63.mlp.experts.161.down_proj", + "model.layers.63.mlp.experts.161.gate_proj", + "model.layers.63.mlp.experts.161.up_proj", + "model.layers.63.mlp.experts.162.down_proj", + "model.layers.63.mlp.experts.162.gate_proj", + "model.layers.63.mlp.experts.162.up_proj", + "model.layers.63.mlp.experts.163.down_proj", + "model.layers.63.mlp.experts.163.gate_proj", + "model.layers.63.mlp.experts.163.up_proj", + "model.layers.63.mlp.experts.164.down_proj", + "model.layers.63.mlp.experts.164.gate_proj", + "model.layers.63.mlp.experts.164.up_proj", + "model.layers.63.mlp.experts.165.down_proj", + "model.layers.63.mlp.experts.165.gate_proj", + "model.layers.63.mlp.experts.165.up_proj", + "model.layers.63.mlp.experts.166.down_proj", + "model.layers.63.mlp.experts.166.gate_proj", + "model.layers.63.mlp.experts.166.up_proj", + "model.layers.63.mlp.experts.167.down_proj", + "model.layers.63.mlp.experts.167.gate_proj", + "model.layers.63.mlp.experts.167.up_proj", + "model.layers.63.mlp.experts.168.down_proj", + "model.layers.63.mlp.experts.168.gate_proj", + "model.layers.63.mlp.experts.168.up_proj", + "model.layers.63.mlp.experts.169.down_proj", + "model.layers.63.mlp.experts.169.gate_proj", + "model.layers.63.mlp.experts.169.up_proj", + "model.layers.63.mlp.experts.17.down_proj", + "model.layers.63.mlp.experts.17.gate_proj", + "model.layers.63.mlp.experts.17.up_proj", + "model.layers.63.mlp.experts.170.down_proj", + "model.layers.63.mlp.experts.170.gate_proj", + "model.layers.63.mlp.experts.170.up_proj", + "model.layers.63.mlp.experts.171.down_proj", + "model.layers.63.mlp.experts.171.gate_proj", + "model.layers.63.mlp.experts.171.up_proj", + "model.layers.63.mlp.experts.172.down_proj", + "model.layers.63.mlp.experts.172.gate_proj", + "model.layers.63.mlp.experts.172.up_proj", + "model.layers.63.mlp.experts.173.down_proj", + "model.layers.63.mlp.experts.173.gate_proj", + "model.layers.63.mlp.experts.173.up_proj", + "model.layers.63.mlp.experts.174.down_proj", + "model.layers.63.mlp.experts.174.gate_proj", + "model.layers.63.mlp.experts.174.up_proj", + "model.layers.63.mlp.experts.175.down_proj", + "model.layers.63.mlp.experts.175.gate_proj", + "model.layers.63.mlp.experts.175.up_proj", + "model.layers.63.mlp.experts.176.down_proj", + "model.layers.63.mlp.experts.176.gate_proj", + "model.layers.63.mlp.experts.176.up_proj", + "model.layers.63.mlp.experts.177.down_proj", + "model.layers.63.mlp.experts.177.gate_proj", + "model.layers.63.mlp.experts.177.up_proj", + "model.layers.63.mlp.experts.178.down_proj", + "model.layers.63.mlp.experts.178.gate_proj", + "model.layers.63.mlp.experts.178.up_proj", + "model.layers.63.mlp.experts.179.down_proj", + "model.layers.63.mlp.experts.179.gate_proj", + "model.layers.63.mlp.experts.179.up_proj", + "model.layers.63.mlp.experts.18.down_proj", + "model.layers.63.mlp.experts.18.gate_proj", + "model.layers.63.mlp.experts.18.up_proj", + "model.layers.63.mlp.experts.180.down_proj", + "model.layers.63.mlp.experts.180.gate_proj", + "model.layers.63.mlp.experts.180.up_proj", + "model.layers.63.mlp.experts.181.down_proj", + "model.layers.63.mlp.experts.181.gate_proj", + "model.layers.63.mlp.experts.181.up_proj", + "model.layers.63.mlp.experts.182.down_proj", + "model.layers.63.mlp.experts.182.gate_proj", + "model.layers.63.mlp.experts.182.up_proj", + "model.layers.63.mlp.experts.183.down_proj", + "model.layers.63.mlp.experts.183.gate_proj", + "model.layers.63.mlp.experts.183.up_proj", + "model.layers.63.mlp.experts.184.down_proj", + "model.layers.63.mlp.experts.184.gate_proj", + "model.layers.63.mlp.experts.184.up_proj", + "model.layers.63.mlp.experts.185.down_proj", + "model.layers.63.mlp.experts.185.gate_proj", + "model.layers.63.mlp.experts.185.up_proj", + "model.layers.63.mlp.experts.186.down_proj", + "model.layers.63.mlp.experts.186.gate_proj", + "model.layers.63.mlp.experts.186.up_proj", + "model.layers.63.mlp.experts.187.down_proj", + "model.layers.63.mlp.experts.187.gate_proj", + "model.layers.63.mlp.experts.187.up_proj", + "model.layers.63.mlp.experts.188.down_proj", + "model.layers.63.mlp.experts.188.gate_proj", + "model.layers.63.mlp.experts.188.up_proj", + "model.layers.63.mlp.experts.189.down_proj", + "model.layers.63.mlp.experts.189.gate_proj", + "model.layers.63.mlp.experts.189.up_proj", + "model.layers.63.mlp.experts.19.down_proj", + "model.layers.63.mlp.experts.19.gate_proj", + "model.layers.63.mlp.experts.19.up_proj", + "model.layers.63.mlp.experts.190.down_proj", + "model.layers.63.mlp.experts.190.gate_proj", + "model.layers.63.mlp.experts.190.up_proj", + "model.layers.63.mlp.experts.191.down_proj", + "model.layers.63.mlp.experts.191.gate_proj", + "model.layers.63.mlp.experts.191.up_proj", + "model.layers.63.mlp.experts.192.down_proj", + "model.layers.63.mlp.experts.192.gate_proj", + "model.layers.63.mlp.experts.192.up_proj", + "model.layers.63.mlp.experts.193.down_proj", + "model.layers.63.mlp.experts.193.gate_proj", + "model.layers.63.mlp.experts.193.up_proj", + "model.layers.63.mlp.experts.194.down_proj", + "model.layers.63.mlp.experts.194.gate_proj", + "model.layers.63.mlp.experts.194.up_proj", + "model.layers.63.mlp.experts.195.down_proj", + "model.layers.63.mlp.experts.195.gate_proj", + "model.layers.63.mlp.experts.195.up_proj", + "model.layers.63.mlp.experts.196.down_proj", + "model.layers.63.mlp.experts.196.gate_proj", + "model.layers.63.mlp.experts.196.up_proj", + "model.layers.63.mlp.experts.197.down_proj", + "model.layers.63.mlp.experts.197.gate_proj", + "model.layers.63.mlp.experts.197.up_proj", + "model.layers.63.mlp.experts.198.down_proj", + "model.layers.63.mlp.experts.198.gate_proj", + "model.layers.63.mlp.experts.198.up_proj", + "model.layers.63.mlp.experts.199.down_proj", + "model.layers.63.mlp.experts.199.gate_proj", + "model.layers.63.mlp.experts.199.up_proj", + "model.layers.63.mlp.experts.2.down_proj", + "model.layers.63.mlp.experts.2.gate_proj", + "model.layers.63.mlp.experts.2.up_proj", + "model.layers.63.mlp.experts.20.down_proj", + "model.layers.63.mlp.experts.20.gate_proj", + "model.layers.63.mlp.experts.20.up_proj", + "model.layers.63.mlp.experts.200.down_proj", + "model.layers.63.mlp.experts.200.gate_proj", + "model.layers.63.mlp.experts.200.up_proj", + "model.layers.63.mlp.experts.201.down_proj", + "model.layers.63.mlp.experts.201.gate_proj", + "model.layers.63.mlp.experts.201.up_proj", + "model.layers.63.mlp.experts.202.down_proj", + "model.layers.63.mlp.experts.202.gate_proj", + "model.layers.63.mlp.experts.202.up_proj", + "model.layers.63.mlp.experts.203.down_proj", + "model.layers.63.mlp.experts.203.gate_proj", + "model.layers.63.mlp.experts.203.up_proj", + "model.layers.63.mlp.experts.204.down_proj", + "model.layers.63.mlp.experts.204.gate_proj", + "model.layers.63.mlp.experts.204.up_proj", + "model.layers.63.mlp.experts.205.down_proj", + "model.layers.63.mlp.experts.205.gate_proj", + "model.layers.63.mlp.experts.205.up_proj", + "model.layers.63.mlp.experts.206.down_proj", + "model.layers.63.mlp.experts.206.gate_proj", + "model.layers.63.mlp.experts.206.up_proj", + "model.layers.63.mlp.experts.207.down_proj", + "model.layers.63.mlp.experts.207.gate_proj", + "model.layers.63.mlp.experts.207.up_proj", + "model.layers.63.mlp.experts.208.down_proj", + "model.layers.63.mlp.experts.208.gate_proj", + "model.layers.63.mlp.experts.208.up_proj", + "model.layers.63.mlp.experts.209.down_proj", + "model.layers.63.mlp.experts.209.gate_proj", + "model.layers.63.mlp.experts.209.up_proj", + "model.layers.63.mlp.experts.21.down_proj", + "model.layers.63.mlp.experts.21.gate_proj", + "model.layers.63.mlp.experts.21.up_proj", + "model.layers.63.mlp.experts.210.down_proj", + "model.layers.63.mlp.experts.210.gate_proj", + "model.layers.63.mlp.experts.210.up_proj", + "model.layers.63.mlp.experts.211.down_proj", + "model.layers.63.mlp.experts.211.gate_proj", + "model.layers.63.mlp.experts.211.up_proj", + "model.layers.63.mlp.experts.212.down_proj", + "model.layers.63.mlp.experts.212.gate_proj", + "model.layers.63.mlp.experts.212.up_proj", + "model.layers.63.mlp.experts.213.down_proj", + "model.layers.63.mlp.experts.213.gate_proj", + "model.layers.63.mlp.experts.213.up_proj", + "model.layers.63.mlp.experts.214.down_proj", + "model.layers.63.mlp.experts.214.gate_proj", + "model.layers.63.mlp.experts.214.up_proj", + "model.layers.63.mlp.experts.215.down_proj", + "model.layers.63.mlp.experts.215.gate_proj", + "model.layers.63.mlp.experts.215.up_proj", + "model.layers.63.mlp.experts.216.down_proj", + "model.layers.63.mlp.experts.216.gate_proj", + "model.layers.63.mlp.experts.216.up_proj", + "model.layers.63.mlp.experts.217.down_proj", + "model.layers.63.mlp.experts.217.gate_proj", + "model.layers.63.mlp.experts.217.up_proj", + "model.layers.63.mlp.experts.218.down_proj", + "model.layers.63.mlp.experts.218.gate_proj", + "model.layers.63.mlp.experts.218.up_proj", + "model.layers.63.mlp.experts.219.down_proj", + "model.layers.63.mlp.experts.219.gate_proj", + "model.layers.63.mlp.experts.219.up_proj", + "model.layers.63.mlp.experts.22.down_proj", + "model.layers.63.mlp.experts.22.gate_proj", + "model.layers.63.mlp.experts.22.up_proj", + "model.layers.63.mlp.experts.220.down_proj", + "model.layers.63.mlp.experts.220.gate_proj", + "model.layers.63.mlp.experts.220.up_proj", + "model.layers.63.mlp.experts.221.down_proj", + "model.layers.63.mlp.experts.221.gate_proj", + "model.layers.63.mlp.experts.221.up_proj", + "model.layers.63.mlp.experts.222.down_proj", + "model.layers.63.mlp.experts.222.gate_proj", + "model.layers.63.mlp.experts.222.up_proj", + "model.layers.63.mlp.experts.223.down_proj", + "model.layers.63.mlp.experts.223.gate_proj", + "model.layers.63.mlp.experts.223.up_proj", + "model.layers.63.mlp.experts.224.down_proj", + "model.layers.63.mlp.experts.224.gate_proj", + "model.layers.63.mlp.experts.224.up_proj", + "model.layers.63.mlp.experts.225.down_proj", + "model.layers.63.mlp.experts.225.gate_proj", + "model.layers.63.mlp.experts.225.up_proj", + "model.layers.63.mlp.experts.226.down_proj", + "model.layers.63.mlp.experts.226.gate_proj", + "model.layers.63.mlp.experts.226.up_proj", + "model.layers.63.mlp.experts.227.down_proj", + "model.layers.63.mlp.experts.227.gate_proj", + "model.layers.63.mlp.experts.227.up_proj", + "model.layers.63.mlp.experts.228.down_proj", + "model.layers.63.mlp.experts.228.gate_proj", + "model.layers.63.mlp.experts.228.up_proj", + "model.layers.63.mlp.experts.229.down_proj", + "model.layers.63.mlp.experts.229.gate_proj", + "model.layers.63.mlp.experts.229.up_proj", + "model.layers.63.mlp.experts.23.down_proj", + "model.layers.63.mlp.experts.23.gate_proj", + "model.layers.63.mlp.experts.23.up_proj", + "model.layers.63.mlp.experts.230.down_proj", + "model.layers.63.mlp.experts.230.gate_proj", + "model.layers.63.mlp.experts.230.up_proj", + "model.layers.63.mlp.experts.231.down_proj", + "model.layers.63.mlp.experts.231.gate_proj", + "model.layers.63.mlp.experts.231.up_proj", + "model.layers.63.mlp.experts.232.down_proj", + "model.layers.63.mlp.experts.232.gate_proj", + "model.layers.63.mlp.experts.232.up_proj", + "model.layers.63.mlp.experts.233.down_proj", + "model.layers.63.mlp.experts.233.gate_proj", + "model.layers.63.mlp.experts.233.up_proj", + "model.layers.63.mlp.experts.234.down_proj", + "model.layers.63.mlp.experts.234.gate_proj", + "model.layers.63.mlp.experts.234.up_proj", + "model.layers.63.mlp.experts.235.down_proj", + "model.layers.63.mlp.experts.235.gate_proj", + "model.layers.63.mlp.experts.235.up_proj", + "model.layers.63.mlp.experts.236.down_proj", + "model.layers.63.mlp.experts.236.gate_proj", + "model.layers.63.mlp.experts.236.up_proj", + "model.layers.63.mlp.experts.237.down_proj", + "model.layers.63.mlp.experts.237.gate_proj", + "model.layers.63.mlp.experts.237.up_proj", + "model.layers.63.mlp.experts.238.down_proj", + "model.layers.63.mlp.experts.238.gate_proj", + "model.layers.63.mlp.experts.238.up_proj", + "model.layers.63.mlp.experts.239.down_proj", + "model.layers.63.mlp.experts.239.gate_proj", + "model.layers.63.mlp.experts.239.up_proj", + "model.layers.63.mlp.experts.24.down_proj", + "model.layers.63.mlp.experts.24.gate_proj", + "model.layers.63.mlp.experts.24.up_proj", + "model.layers.63.mlp.experts.240.down_proj", + "model.layers.63.mlp.experts.240.gate_proj", + "model.layers.63.mlp.experts.240.up_proj", + "model.layers.63.mlp.experts.241.down_proj", + "model.layers.63.mlp.experts.241.gate_proj", + "model.layers.63.mlp.experts.241.up_proj", + "model.layers.63.mlp.experts.242.down_proj", + "model.layers.63.mlp.experts.242.gate_proj", + "model.layers.63.mlp.experts.242.up_proj", + "model.layers.63.mlp.experts.243.down_proj", + "model.layers.63.mlp.experts.243.gate_proj", + "model.layers.63.mlp.experts.243.up_proj", + "model.layers.63.mlp.experts.244.down_proj", + "model.layers.63.mlp.experts.244.gate_proj", + "model.layers.63.mlp.experts.244.up_proj", + "model.layers.63.mlp.experts.245.down_proj", + "model.layers.63.mlp.experts.245.gate_proj", + "model.layers.63.mlp.experts.245.up_proj", + "model.layers.63.mlp.experts.246.down_proj", + "model.layers.63.mlp.experts.246.gate_proj", + "model.layers.63.mlp.experts.246.up_proj", + "model.layers.63.mlp.experts.247.down_proj", + "model.layers.63.mlp.experts.247.gate_proj", + "model.layers.63.mlp.experts.247.up_proj", + "model.layers.63.mlp.experts.248.down_proj", + "model.layers.63.mlp.experts.248.gate_proj", + "model.layers.63.mlp.experts.248.up_proj", + "model.layers.63.mlp.experts.249.down_proj", + "model.layers.63.mlp.experts.249.gate_proj", + "model.layers.63.mlp.experts.249.up_proj", + "model.layers.63.mlp.experts.25.down_proj", + "model.layers.63.mlp.experts.25.gate_proj", + "model.layers.63.mlp.experts.25.up_proj", + "model.layers.63.mlp.experts.250.down_proj", + "model.layers.63.mlp.experts.250.gate_proj", + "model.layers.63.mlp.experts.250.up_proj", + "model.layers.63.mlp.experts.251.down_proj", + "model.layers.63.mlp.experts.251.gate_proj", + "model.layers.63.mlp.experts.251.up_proj", + "model.layers.63.mlp.experts.252.down_proj", + "model.layers.63.mlp.experts.252.gate_proj", + "model.layers.63.mlp.experts.252.up_proj", + "model.layers.63.mlp.experts.253.down_proj", + "model.layers.63.mlp.experts.253.gate_proj", + "model.layers.63.mlp.experts.253.up_proj", + "model.layers.63.mlp.experts.254.down_proj", + "model.layers.63.mlp.experts.254.gate_proj", + "model.layers.63.mlp.experts.254.up_proj", + "model.layers.63.mlp.experts.255.down_proj", + "model.layers.63.mlp.experts.255.gate_proj", + "model.layers.63.mlp.experts.255.up_proj", + "model.layers.63.mlp.experts.26.down_proj", + "model.layers.63.mlp.experts.26.gate_proj", + "model.layers.63.mlp.experts.26.up_proj", + "model.layers.63.mlp.experts.27.down_proj", + "model.layers.63.mlp.experts.27.gate_proj", + "model.layers.63.mlp.experts.27.up_proj", + "model.layers.63.mlp.experts.28.down_proj", + "model.layers.63.mlp.experts.28.gate_proj", + "model.layers.63.mlp.experts.28.up_proj", + "model.layers.63.mlp.experts.29.down_proj", + "model.layers.63.mlp.experts.29.gate_proj", + "model.layers.63.mlp.experts.29.up_proj", + "model.layers.63.mlp.experts.3.down_proj", + "model.layers.63.mlp.experts.3.gate_proj", + "model.layers.63.mlp.experts.3.up_proj", + "model.layers.63.mlp.experts.30.down_proj", + "model.layers.63.mlp.experts.30.gate_proj", + "model.layers.63.mlp.experts.30.up_proj", + "model.layers.63.mlp.experts.31.down_proj", + "model.layers.63.mlp.experts.31.gate_proj", + "model.layers.63.mlp.experts.31.up_proj", + "model.layers.63.mlp.experts.32.down_proj", + "model.layers.63.mlp.experts.32.gate_proj", + "model.layers.63.mlp.experts.32.up_proj", + "model.layers.63.mlp.experts.33.down_proj", + "model.layers.63.mlp.experts.33.gate_proj", + "model.layers.63.mlp.experts.33.up_proj", + "model.layers.63.mlp.experts.34.down_proj", + "model.layers.63.mlp.experts.34.gate_proj", + "model.layers.63.mlp.experts.34.up_proj", + "model.layers.63.mlp.experts.35.down_proj", + "model.layers.63.mlp.experts.35.gate_proj", + "model.layers.63.mlp.experts.35.up_proj", + "model.layers.63.mlp.experts.36.down_proj", + "model.layers.63.mlp.experts.36.gate_proj", + "model.layers.63.mlp.experts.36.up_proj", + "model.layers.63.mlp.experts.37.down_proj", + "model.layers.63.mlp.experts.37.gate_proj", + "model.layers.63.mlp.experts.37.up_proj", + "model.layers.63.mlp.experts.38.down_proj", + "model.layers.63.mlp.experts.38.gate_proj", + "model.layers.63.mlp.experts.38.up_proj", + "model.layers.63.mlp.experts.39.down_proj", + "model.layers.63.mlp.experts.39.gate_proj", + "model.layers.63.mlp.experts.39.up_proj", + "model.layers.63.mlp.experts.4.down_proj", + "model.layers.63.mlp.experts.4.gate_proj", + "model.layers.63.mlp.experts.4.up_proj", + "model.layers.63.mlp.experts.40.down_proj", + "model.layers.63.mlp.experts.40.gate_proj", + "model.layers.63.mlp.experts.40.up_proj", + "model.layers.63.mlp.experts.41.down_proj", + "model.layers.63.mlp.experts.41.gate_proj", + "model.layers.63.mlp.experts.41.up_proj", + "model.layers.63.mlp.experts.42.down_proj", + "model.layers.63.mlp.experts.42.gate_proj", + "model.layers.63.mlp.experts.42.up_proj", + "model.layers.63.mlp.experts.43.down_proj", + "model.layers.63.mlp.experts.43.gate_proj", + "model.layers.63.mlp.experts.43.up_proj", + "model.layers.63.mlp.experts.44.down_proj", + "model.layers.63.mlp.experts.44.gate_proj", + "model.layers.63.mlp.experts.44.up_proj", + "model.layers.63.mlp.experts.45.down_proj", + "model.layers.63.mlp.experts.45.gate_proj", + "model.layers.63.mlp.experts.45.up_proj", + "model.layers.63.mlp.experts.46.down_proj", + "model.layers.63.mlp.experts.46.gate_proj", + "model.layers.63.mlp.experts.46.up_proj", + "model.layers.63.mlp.experts.47.down_proj", + "model.layers.63.mlp.experts.47.gate_proj", + "model.layers.63.mlp.experts.47.up_proj", + "model.layers.63.mlp.experts.48.down_proj", + "model.layers.63.mlp.experts.48.gate_proj", + "model.layers.63.mlp.experts.48.up_proj", + "model.layers.63.mlp.experts.49.down_proj", + "model.layers.63.mlp.experts.49.gate_proj", + "model.layers.63.mlp.experts.49.up_proj", + "model.layers.63.mlp.experts.5.down_proj", + "model.layers.63.mlp.experts.5.gate_proj", + "model.layers.63.mlp.experts.5.up_proj", + "model.layers.63.mlp.experts.50.down_proj", + "model.layers.63.mlp.experts.50.gate_proj", + "model.layers.63.mlp.experts.50.up_proj", + "model.layers.63.mlp.experts.51.down_proj", + "model.layers.63.mlp.experts.51.gate_proj", + "model.layers.63.mlp.experts.51.up_proj", + "model.layers.63.mlp.experts.52.down_proj", + "model.layers.63.mlp.experts.52.gate_proj", + "model.layers.63.mlp.experts.52.up_proj", + "model.layers.63.mlp.experts.53.down_proj", + "model.layers.63.mlp.experts.53.gate_proj", + "model.layers.63.mlp.experts.53.up_proj", + "model.layers.63.mlp.experts.54.down_proj", + "model.layers.63.mlp.experts.54.gate_proj", + "model.layers.63.mlp.experts.54.up_proj", + "model.layers.63.mlp.experts.55.down_proj", + "model.layers.63.mlp.experts.55.gate_proj", + "model.layers.63.mlp.experts.55.up_proj", + "model.layers.63.mlp.experts.56.down_proj", + "model.layers.63.mlp.experts.56.gate_proj", + "model.layers.63.mlp.experts.56.up_proj", + "model.layers.63.mlp.experts.57.down_proj", + "model.layers.63.mlp.experts.57.gate_proj", + "model.layers.63.mlp.experts.57.up_proj", + "model.layers.63.mlp.experts.58.down_proj", + "model.layers.63.mlp.experts.58.gate_proj", + "model.layers.63.mlp.experts.58.up_proj", + "model.layers.63.mlp.experts.59.down_proj", + "model.layers.63.mlp.experts.59.gate_proj", + "model.layers.63.mlp.experts.59.up_proj", + "model.layers.63.mlp.experts.6.down_proj", + "model.layers.63.mlp.experts.6.gate_proj", + "model.layers.63.mlp.experts.6.up_proj", + "model.layers.63.mlp.experts.60.down_proj", + "model.layers.63.mlp.experts.60.gate_proj", + "model.layers.63.mlp.experts.60.up_proj", + "model.layers.63.mlp.experts.61.down_proj", + "model.layers.63.mlp.experts.61.gate_proj", + "model.layers.63.mlp.experts.61.up_proj", + "model.layers.63.mlp.experts.62.down_proj", + "model.layers.63.mlp.experts.62.gate_proj", + "model.layers.63.mlp.experts.62.up_proj", + "model.layers.63.mlp.experts.63.down_proj", + "model.layers.63.mlp.experts.63.gate_proj", + "model.layers.63.mlp.experts.63.up_proj", + "model.layers.63.mlp.experts.64.down_proj", + "model.layers.63.mlp.experts.64.gate_proj", + "model.layers.63.mlp.experts.64.up_proj", + "model.layers.63.mlp.experts.65.down_proj", + "model.layers.63.mlp.experts.65.gate_proj", + "model.layers.63.mlp.experts.65.up_proj", + "model.layers.63.mlp.experts.66.down_proj", + "model.layers.63.mlp.experts.66.gate_proj", + "model.layers.63.mlp.experts.66.up_proj", + "model.layers.63.mlp.experts.67.down_proj", + "model.layers.63.mlp.experts.67.gate_proj", + "model.layers.63.mlp.experts.67.up_proj", + "model.layers.63.mlp.experts.68.down_proj", + "model.layers.63.mlp.experts.68.gate_proj", + "model.layers.63.mlp.experts.68.up_proj", + "model.layers.63.mlp.experts.69.down_proj", + "model.layers.63.mlp.experts.69.gate_proj", + "model.layers.63.mlp.experts.69.up_proj", + "model.layers.63.mlp.experts.7.down_proj", + "model.layers.63.mlp.experts.7.gate_proj", + "model.layers.63.mlp.experts.7.up_proj", + "model.layers.63.mlp.experts.70.down_proj", + "model.layers.63.mlp.experts.70.gate_proj", + "model.layers.63.mlp.experts.70.up_proj", + "model.layers.63.mlp.experts.71.down_proj", + "model.layers.63.mlp.experts.71.gate_proj", + "model.layers.63.mlp.experts.71.up_proj", + "model.layers.63.mlp.experts.72.down_proj", + "model.layers.63.mlp.experts.72.gate_proj", + "model.layers.63.mlp.experts.72.up_proj", + "model.layers.63.mlp.experts.73.down_proj", + "model.layers.63.mlp.experts.73.gate_proj", + "model.layers.63.mlp.experts.73.up_proj", + "model.layers.63.mlp.experts.74.down_proj", + "model.layers.63.mlp.experts.74.gate_proj", + "model.layers.63.mlp.experts.74.up_proj", + "model.layers.63.mlp.experts.75.down_proj", + "model.layers.63.mlp.experts.75.gate_proj", + "model.layers.63.mlp.experts.75.up_proj", + "model.layers.63.mlp.experts.76.down_proj", + "model.layers.63.mlp.experts.76.gate_proj", + "model.layers.63.mlp.experts.76.up_proj", + "model.layers.63.mlp.experts.77.down_proj", + "model.layers.63.mlp.experts.77.gate_proj", + "model.layers.63.mlp.experts.77.up_proj", + "model.layers.63.mlp.experts.78.down_proj", + "model.layers.63.mlp.experts.78.gate_proj", + "model.layers.63.mlp.experts.78.up_proj", + "model.layers.63.mlp.experts.79.down_proj", + "model.layers.63.mlp.experts.79.gate_proj", + "model.layers.63.mlp.experts.79.up_proj", + "model.layers.63.mlp.experts.8.down_proj", + "model.layers.63.mlp.experts.8.gate_proj", + "model.layers.63.mlp.experts.8.up_proj", + "model.layers.63.mlp.experts.80.down_proj", + "model.layers.63.mlp.experts.80.gate_proj", + "model.layers.63.mlp.experts.80.up_proj", + "model.layers.63.mlp.experts.81.down_proj", + "model.layers.63.mlp.experts.81.gate_proj", + "model.layers.63.mlp.experts.81.up_proj", + "model.layers.63.mlp.experts.82.down_proj", + "model.layers.63.mlp.experts.82.gate_proj", + "model.layers.63.mlp.experts.82.up_proj", + "model.layers.63.mlp.experts.83.down_proj", + "model.layers.63.mlp.experts.83.gate_proj", + "model.layers.63.mlp.experts.83.up_proj", + "model.layers.63.mlp.experts.84.down_proj", + "model.layers.63.mlp.experts.84.gate_proj", + "model.layers.63.mlp.experts.84.up_proj", + "model.layers.63.mlp.experts.85.down_proj", + "model.layers.63.mlp.experts.85.gate_proj", + "model.layers.63.mlp.experts.85.up_proj", + "model.layers.63.mlp.experts.86.down_proj", + "model.layers.63.mlp.experts.86.gate_proj", + "model.layers.63.mlp.experts.86.up_proj", + "model.layers.63.mlp.experts.87.down_proj", + "model.layers.63.mlp.experts.87.gate_proj", + "model.layers.63.mlp.experts.87.up_proj", + "model.layers.63.mlp.experts.88.down_proj", + "model.layers.63.mlp.experts.88.gate_proj", + "model.layers.63.mlp.experts.88.up_proj", + "model.layers.63.mlp.experts.89.down_proj", + "model.layers.63.mlp.experts.89.gate_proj", + "model.layers.63.mlp.experts.89.up_proj", + "model.layers.63.mlp.experts.9.down_proj", + "model.layers.63.mlp.experts.9.gate_proj", + "model.layers.63.mlp.experts.9.up_proj", + "model.layers.63.mlp.experts.90.down_proj", + "model.layers.63.mlp.experts.90.gate_proj", + "model.layers.63.mlp.experts.90.up_proj", + "model.layers.63.mlp.experts.91.down_proj", + "model.layers.63.mlp.experts.91.gate_proj", + "model.layers.63.mlp.experts.91.up_proj", + "model.layers.63.mlp.experts.92.down_proj", + "model.layers.63.mlp.experts.92.gate_proj", + "model.layers.63.mlp.experts.92.up_proj", + "model.layers.63.mlp.experts.93.down_proj", + "model.layers.63.mlp.experts.93.gate_proj", + "model.layers.63.mlp.experts.93.up_proj", + "model.layers.63.mlp.experts.94.down_proj", + "model.layers.63.mlp.experts.94.gate_proj", + "model.layers.63.mlp.experts.94.up_proj", + "model.layers.63.mlp.experts.95.down_proj", + "model.layers.63.mlp.experts.95.gate_proj", + "model.layers.63.mlp.experts.95.up_proj", + "model.layers.63.mlp.experts.96.down_proj", + "model.layers.63.mlp.experts.96.gate_proj", + "model.layers.63.mlp.experts.96.up_proj", + "model.layers.63.mlp.experts.97.down_proj", + "model.layers.63.mlp.experts.97.gate_proj", + "model.layers.63.mlp.experts.97.up_proj", + "model.layers.63.mlp.experts.98.down_proj", + "model.layers.63.mlp.experts.98.gate_proj", + "model.layers.63.mlp.experts.98.up_proj", + "model.layers.63.mlp.experts.99.down_proj", + "model.layers.63.mlp.experts.99.gate_proj", + "model.layers.63.mlp.experts.99.up_proj", + "model.layers.64.mlp.experts.0.down_proj", + "model.layers.64.mlp.experts.0.gate_proj", + "model.layers.64.mlp.experts.0.up_proj", + "model.layers.64.mlp.experts.1.down_proj", + "model.layers.64.mlp.experts.1.gate_proj", + "model.layers.64.mlp.experts.1.up_proj", + "model.layers.64.mlp.experts.10.down_proj", + "model.layers.64.mlp.experts.10.gate_proj", + "model.layers.64.mlp.experts.10.up_proj", + "model.layers.64.mlp.experts.100.down_proj", + "model.layers.64.mlp.experts.100.gate_proj", + "model.layers.64.mlp.experts.100.up_proj", + "model.layers.64.mlp.experts.101.down_proj", + "model.layers.64.mlp.experts.101.gate_proj", + "model.layers.64.mlp.experts.101.up_proj", + "model.layers.64.mlp.experts.102.down_proj", + "model.layers.64.mlp.experts.102.gate_proj", + "model.layers.64.mlp.experts.102.up_proj", + "model.layers.64.mlp.experts.103.down_proj", + "model.layers.64.mlp.experts.103.gate_proj", + "model.layers.64.mlp.experts.103.up_proj", + "model.layers.64.mlp.experts.104.down_proj", + "model.layers.64.mlp.experts.104.gate_proj", + "model.layers.64.mlp.experts.104.up_proj", + "model.layers.64.mlp.experts.105.down_proj", + "model.layers.64.mlp.experts.105.gate_proj", + "model.layers.64.mlp.experts.105.up_proj", + "model.layers.64.mlp.experts.106.down_proj", + "model.layers.64.mlp.experts.106.gate_proj", + "model.layers.64.mlp.experts.106.up_proj", + "model.layers.64.mlp.experts.107.down_proj", + "model.layers.64.mlp.experts.107.gate_proj", + "model.layers.64.mlp.experts.107.up_proj", + "model.layers.64.mlp.experts.108.down_proj", + "model.layers.64.mlp.experts.108.gate_proj", + "model.layers.64.mlp.experts.108.up_proj", + "model.layers.64.mlp.experts.109.down_proj", + "model.layers.64.mlp.experts.109.gate_proj", + "model.layers.64.mlp.experts.109.up_proj", + "model.layers.64.mlp.experts.11.down_proj", + "model.layers.64.mlp.experts.11.gate_proj", + "model.layers.64.mlp.experts.11.up_proj", + "model.layers.64.mlp.experts.110.down_proj", + "model.layers.64.mlp.experts.110.gate_proj", + "model.layers.64.mlp.experts.110.up_proj", + "model.layers.64.mlp.experts.111.down_proj", + "model.layers.64.mlp.experts.111.gate_proj", + "model.layers.64.mlp.experts.111.up_proj", + "model.layers.64.mlp.experts.112.down_proj", + "model.layers.64.mlp.experts.112.gate_proj", + "model.layers.64.mlp.experts.112.up_proj", + "model.layers.64.mlp.experts.113.down_proj", + "model.layers.64.mlp.experts.113.gate_proj", + "model.layers.64.mlp.experts.113.up_proj", + "model.layers.64.mlp.experts.114.down_proj", + "model.layers.64.mlp.experts.114.gate_proj", + "model.layers.64.mlp.experts.114.up_proj", + "model.layers.64.mlp.experts.115.down_proj", + "model.layers.64.mlp.experts.115.gate_proj", + "model.layers.64.mlp.experts.115.up_proj", + "model.layers.64.mlp.experts.116.down_proj", + "model.layers.64.mlp.experts.116.gate_proj", + "model.layers.64.mlp.experts.116.up_proj", + "model.layers.64.mlp.experts.117.down_proj", + "model.layers.64.mlp.experts.117.gate_proj", + "model.layers.64.mlp.experts.117.up_proj", + "model.layers.64.mlp.experts.118.down_proj", + "model.layers.64.mlp.experts.118.gate_proj", + "model.layers.64.mlp.experts.118.up_proj", + "model.layers.64.mlp.experts.119.down_proj", + "model.layers.64.mlp.experts.119.gate_proj", + "model.layers.64.mlp.experts.119.up_proj", + "model.layers.64.mlp.experts.12.down_proj", + "model.layers.64.mlp.experts.12.gate_proj", + "model.layers.64.mlp.experts.12.up_proj", + "model.layers.64.mlp.experts.120.down_proj", + "model.layers.64.mlp.experts.120.gate_proj", + "model.layers.64.mlp.experts.120.up_proj", + "model.layers.64.mlp.experts.121.down_proj", + "model.layers.64.mlp.experts.121.gate_proj", + "model.layers.64.mlp.experts.121.up_proj", + "model.layers.64.mlp.experts.122.down_proj", + "model.layers.64.mlp.experts.122.gate_proj", + "model.layers.64.mlp.experts.122.up_proj", + "model.layers.64.mlp.experts.123.down_proj", + "model.layers.64.mlp.experts.123.gate_proj", + "model.layers.64.mlp.experts.123.up_proj", + "model.layers.64.mlp.experts.124.down_proj", + "model.layers.64.mlp.experts.124.gate_proj", + "model.layers.64.mlp.experts.124.up_proj", + "model.layers.64.mlp.experts.125.down_proj", + "model.layers.64.mlp.experts.125.gate_proj", + "model.layers.64.mlp.experts.125.up_proj", + "model.layers.64.mlp.experts.126.down_proj", + "model.layers.64.mlp.experts.126.gate_proj", + "model.layers.64.mlp.experts.126.up_proj", + "model.layers.64.mlp.experts.127.down_proj", + "model.layers.64.mlp.experts.127.gate_proj", + "model.layers.64.mlp.experts.127.up_proj", + "model.layers.64.mlp.experts.128.down_proj", + "model.layers.64.mlp.experts.128.gate_proj", + "model.layers.64.mlp.experts.128.up_proj", + "model.layers.64.mlp.experts.129.down_proj", + "model.layers.64.mlp.experts.129.gate_proj", + "model.layers.64.mlp.experts.129.up_proj", + "model.layers.64.mlp.experts.13.down_proj", + "model.layers.64.mlp.experts.13.gate_proj", + "model.layers.64.mlp.experts.13.up_proj", + "model.layers.64.mlp.experts.130.down_proj", + "model.layers.64.mlp.experts.130.gate_proj", + "model.layers.64.mlp.experts.130.up_proj", + "model.layers.64.mlp.experts.131.down_proj", + "model.layers.64.mlp.experts.131.gate_proj", + "model.layers.64.mlp.experts.131.up_proj", + "model.layers.64.mlp.experts.132.down_proj", + "model.layers.64.mlp.experts.132.gate_proj", + "model.layers.64.mlp.experts.132.up_proj", + "model.layers.64.mlp.experts.133.down_proj", + "model.layers.64.mlp.experts.133.gate_proj", + "model.layers.64.mlp.experts.133.up_proj", + "model.layers.64.mlp.experts.134.down_proj", + "model.layers.64.mlp.experts.134.gate_proj", + "model.layers.64.mlp.experts.134.up_proj", + "model.layers.64.mlp.experts.135.down_proj", + "model.layers.64.mlp.experts.135.gate_proj", + "model.layers.64.mlp.experts.135.up_proj", + "model.layers.64.mlp.experts.136.down_proj", + "model.layers.64.mlp.experts.136.gate_proj", + "model.layers.64.mlp.experts.136.up_proj", + "model.layers.64.mlp.experts.137.down_proj", + "model.layers.64.mlp.experts.137.gate_proj", + "model.layers.64.mlp.experts.137.up_proj", + "model.layers.64.mlp.experts.138.down_proj", + "model.layers.64.mlp.experts.138.gate_proj", + "model.layers.64.mlp.experts.138.up_proj", + "model.layers.64.mlp.experts.139.down_proj", + "model.layers.64.mlp.experts.139.gate_proj", + "model.layers.64.mlp.experts.139.up_proj", + "model.layers.64.mlp.experts.14.down_proj", + "model.layers.64.mlp.experts.14.gate_proj", + "model.layers.64.mlp.experts.14.up_proj", + "model.layers.64.mlp.experts.140.down_proj", + "model.layers.64.mlp.experts.140.gate_proj", + "model.layers.64.mlp.experts.140.up_proj", + "model.layers.64.mlp.experts.141.down_proj", + "model.layers.64.mlp.experts.141.gate_proj", + "model.layers.64.mlp.experts.141.up_proj", + "model.layers.64.mlp.experts.142.down_proj", + "model.layers.64.mlp.experts.142.gate_proj", + "model.layers.64.mlp.experts.142.up_proj", + "model.layers.64.mlp.experts.143.down_proj", + "model.layers.64.mlp.experts.143.gate_proj", + "model.layers.64.mlp.experts.143.up_proj", + "model.layers.64.mlp.experts.144.down_proj", + "model.layers.64.mlp.experts.144.gate_proj", + "model.layers.64.mlp.experts.144.up_proj", + "model.layers.64.mlp.experts.145.down_proj", + "model.layers.64.mlp.experts.145.gate_proj", + "model.layers.64.mlp.experts.145.up_proj", + "model.layers.64.mlp.experts.146.down_proj", + "model.layers.64.mlp.experts.146.gate_proj", + "model.layers.64.mlp.experts.146.up_proj", + "model.layers.64.mlp.experts.147.down_proj", + "model.layers.64.mlp.experts.147.gate_proj", + "model.layers.64.mlp.experts.147.up_proj", + "model.layers.64.mlp.experts.148.down_proj", + "model.layers.64.mlp.experts.148.gate_proj", + "model.layers.64.mlp.experts.148.up_proj", + "model.layers.64.mlp.experts.149.down_proj", + "model.layers.64.mlp.experts.149.gate_proj", + "model.layers.64.mlp.experts.149.up_proj", + "model.layers.64.mlp.experts.15.down_proj", + "model.layers.64.mlp.experts.15.gate_proj", + "model.layers.64.mlp.experts.15.up_proj", + "model.layers.64.mlp.experts.150.down_proj", + "model.layers.64.mlp.experts.150.gate_proj", + "model.layers.64.mlp.experts.150.up_proj", + "model.layers.64.mlp.experts.151.down_proj", + "model.layers.64.mlp.experts.151.gate_proj", + "model.layers.64.mlp.experts.151.up_proj", + "model.layers.64.mlp.experts.152.down_proj", + "model.layers.64.mlp.experts.152.gate_proj", + "model.layers.64.mlp.experts.152.up_proj", + "model.layers.64.mlp.experts.153.down_proj", + "model.layers.64.mlp.experts.153.gate_proj", + "model.layers.64.mlp.experts.153.up_proj", + "model.layers.64.mlp.experts.154.down_proj", + "model.layers.64.mlp.experts.154.gate_proj", + "model.layers.64.mlp.experts.154.up_proj", + "model.layers.64.mlp.experts.155.down_proj", + "model.layers.64.mlp.experts.155.gate_proj", + "model.layers.64.mlp.experts.155.up_proj", + "model.layers.64.mlp.experts.156.down_proj", + "model.layers.64.mlp.experts.156.gate_proj", + "model.layers.64.mlp.experts.156.up_proj", + "model.layers.64.mlp.experts.157.down_proj", + "model.layers.64.mlp.experts.157.gate_proj", + "model.layers.64.mlp.experts.157.up_proj", + "model.layers.64.mlp.experts.158.down_proj", + "model.layers.64.mlp.experts.158.gate_proj", + "model.layers.64.mlp.experts.158.up_proj", + "model.layers.64.mlp.experts.159.down_proj", + "model.layers.64.mlp.experts.159.gate_proj", + "model.layers.64.mlp.experts.159.up_proj", + "model.layers.64.mlp.experts.16.down_proj", + "model.layers.64.mlp.experts.16.gate_proj", + "model.layers.64.mlp.experts.16.up_proj", + "model.layers.64.mlp.experts.160.down_proj", + "model.layers.64.mlp.experts.160.gate_proj", + "model.layers.64.mlp.experts.160.up_proj", + "model.layers.64.mlp.experts.161.down_proj", + "model.layers.64.mlp.experts.161.gate_proj", + "model.layers.64.mlp.experts.161.up_proj", + "model.layers.64.mlp.experts.162.down_proj", + "model.layers.64.mlp.experts.162.gate_proj", + "model.layers.64.mlp.experts.162.up_proj", + "model.layers.64.mlp.experts.163.down_proj", + "model.layers.64.mlp.experts.163.gate_proj", + "model.layers.64.mlp.experts.163.up_proj", + "model.layers.64.mlp.experts.164.down_proj", + "model.layers.64.mlp.experts.164.gate_proj", + "model.layers.64.mlp.experts.164.up_proj", + "model.layers.64.mlp.experts.165.down_proj", + "model.layers.64.mlp.experts.165.gate_proj", + "model.layers.64.mlp.experts.165.up_proj", + "model.layers.64.mlp.experts.166.down_proj", + "model.layers.64.mlp.experts.166.gate_proj", + "model.layers.64.mlp.experts.166.up_proj", + "model.layers.64.mlp.experts.167.down_proj", + "model.layers.64.mlp.experts.167.gate_proj", + "model.layers.64.mlp.experts.167.up_proj", + "model.layers.64.mlp.experts.168.down_proj", + "model.layers.64.mlp.experts.168.gate_proj", + "model.layers.64.mlp.experts.168.up_proj", + "model.layers.64.mlp.experts.169.down_proj", + "model.layers.64.mlp.experts.169.gate_proj", + "model.layers.64.mlp.experts.169.up_proj", + "model.layers.64.mlp.experts.17.down_proj", + "model.layers.64.mlp.experts.17.gate_proj", + "model.layers.64.mlp.experts.17.up_proj", + "model.layers.64.mlp.experts.170.down_proj", + "model.layers.64.mlp.experts.170.gate_proj", + "model.layers.64.mlp.experts.170.up_proj", + "model.layers.64.mlp.experts.171.down_proj", + "model.layers.64.mlp.experts.171.gate_proj", + "model.layers.64.mlp.experts.171.up_proj", + "model.layers.64.mlp.experts.172.down_proj", + "model.layers.64.mlp.experts.172.gate_proj", + "model.layers.64.mlp.experts.172.up_proj", + "model.layers.64.mlp.experts.173.down_proj", + "model.layers.64.mlp.experts.173.gate_proj", + "model.layers.64.mlp.experts.173.up_proj", + "model.layers.64.mlp.experts.174.down_proj", + "model.layers.64.mlp.experts.174.gate_proj", + "model.layers.64.mlp.experts.174.up_proj", + "model.layers.64.mlp.experts.175.down_proj", + "model.layers.64.mlp.experts.175.gate_proj", + "model.layers.64.mlp.experts.175.up_proj", + "model.layers.64.mlp.experts.176.down_proj", + "model.layers.64.mlp.experts.176.gate_proj", + "model.layers.64.mlp.experts.176.up_proj", + "model.layers.64.mlp.experts.177.down_proj", + "model.layers.64.mlp.experts.177.gate_proj", + "model.layers.64.mlp.experts.177.up_proj", + "model.layers.64.mlp.experts.178.down_proj", + "model.layers.64.mlp.experts.178.gate_proj", + "model.layers.64.mlp.experts.178.up_proj", + "model.layers.64.mlp.experts.179.down_proj", + "model.layers.64.mlp.experts.179.gate_proj", + "model.layers.64.mlp.experts.179.up_proj", + "model.layers.64.mlp.experts.18.down_proj", + "model.layers.64.mlp.experts.18.gate_proj", + "model.layers.64.mlp.experts.18.up_proj", + "model.layers.64.mlp.experts.180.down_proj", + "model.layers.64.mlp.experts.180.gate_proj", + "model.layers.64.mlp.experts.180.up_proj", + "model.layers.64.mlp.experts.181.down_proj", + "model.layers.64.mlp.experts.181.gate_proj", + "model.layers.64.mlp.experts.181.up_proj", + "model.layers.64.mlp.experts.182.down_proj", + "model.layers.64.mlp.experts.182.gate_proj", + "model.layers.64.mlp.experts.182.up_proj", + "model.layers.64.mlp.experts.183.down_proj", + "model.layers.64.mlp.experts.183.gate_proj", + "model.layers.64.mlp.experts.183.up_proj", + "model.layers.64.mlp.experts.184.down_proj", + "model.layers.64.mlp.experts.184.gate_proj", + "model.layers.64.mlp.experts.184.up_proj", + "model.layers.64.mlp.experts.185.down_proj", + "model.layers.64.mlp.experts.185.gate_proj", + "model.layers.64.mlp.experts.185.up_proj", + "model.layers.64.mlp.experts.186.down_proj", + "model.layers.64.mlp.experts.186.gate_proj", + "model.layers.64.mlp.experts.186.up_proj", + "model.layers.64.mlp.experts.187.down_proj", + "model.layers.64.mlp.experts.187.gate_proj", + "model.layers.64.mlp.experts.187.up_proj", + "model.layers.64.mlp.experts.188.down_proj", + "model.layers.64.mlp.experts.188.gate_proj", + "model.layers.64.mlp.experts.188.up_proj", + "model.layers.64.mlp.experts.189.down_proj", + "model.layers.64.mlp.experts.189.gate_proj", + "model.layers.64.mlp.experts.189.up_proj", + "model.layers.64.mlp.experts.19.down_proj", + "model.layers.64.mlp.experts.19.gate_proj", + "model.layers.64.mlp.experts.19.up_proj", + "model.layers.64.mlp.experts.190.down_proj", + "model.layers.64.mlp.experts.190.gate_proj", + "model.layers.64.mlp.experts.190.up_proj", + "model.layers.64.mlp.experts.191.down_proj", + "model.layers.64.mlp.experts.191.gate_proj", + "model.layers.64.mlp.experts.191.up_proj", + "model.layers.64.mlp.experts.192.down_proj", + "model.layers.64.mlp.experts.192.gate_proj", + "model.layers.64.mlp.experts.192.up_proj", + "model.layers.64.mlp.experts.193.down_proj", + "model.layers.64.mlp.experts.193.gate_proj", + "model.layers.64.mlp.experts.193.up_proj", + "model.layers.64.mlp.experts.194.down_proj", + "model.layers.64.mlp.experts.194.gate_proj", + "model.layers.64.mlp.experts.194.up_proj", + "model.layers.64.mlp.experts.195.down_proj", + "model.layers.64.mlp.experts.195.gate_proj", + "model.layers.64.mlp.experts.195.up_proj", + "model.layers.64.mlp.experts.196.down_proj", + "model.layers.64.mlp.experts.196.gate_proj", + "model.layers.64.mlp.experts.196.up_proj", + "model.layers.64.mlp.experts.197.down_proj", + "model.layers.64.mlp.experts.197.gate_proj", + "model.layers.64.mlp.experts.197.up_proj", + "model.layers.64.mlp.experts.198.down_proj", + "model.layers.64.mlp.experts.198.gate_proj", + "model.layers.64.mlp.experts.198.up_proj", + "model.layers.64.mlp.experts.199.down_proj", + "model.layers.64.mlp.experts.199.gate_proj", + "model.layers.64.mlp.experts.199.up_proj", + "model.layers.64.mlp.experts.2.down_proj", + "model.layers.64.mlp.experts.2.gate_proj", + "model.layers.64.mlp.experts.2.up_proj", + "model.layers.64.mlp.experts.20.down_proj", + "model.layers.64.mlp.experts.20.gate_proj", + "model.layers.64.mlp.experts.20.up_proj", + "model.layers.64.mlp.experts.200.down_proj", + "model.layers.64.mlp.experts.200.gate_proj", + "model.layers.64.mlp.experts.200.up_proj", + "model.layers.64.mlp.experts.201.down_proj", + "model.layers.64.mlp.experts.201.gate_proj", + "model.layers.64.mlp.experts.201.up_proj", + "model.layers.64.mlp.experts.202.down_proj", + "model.layers.64.mlp.experts.202.gate_proj", + "model.layers.64.mlp.experts.202.up_proj", + "model.layers.64.mlp.experts.203.down_proj", + "model.layers.64.mlp.experts.203.gate_proj", + "model.layers.64.mlp.experts.203.up_proj", + "model.layers.64.mlp.experts.204.down_proj", + "model.layers.64.mlp.experts.204.gate_proj", + "model.layers.64.mlp.experts.204.up_proj", + "model.layers.64.mlp.experts.205.down_proj", + "model.layers.64.mlp.experts.205.gate_proj", + "model.layers.64.mlp.experts.205.up_proj", + "model.layers.64.mlp.experts.206.down_proj", + "model.layers.64.mlp.experts.206.gate_proj", + "model.layers.64.mlp.experts.206.up_proj", + "model.layers.64.mlp.experts.207.down_proj", + "model.layers.64.mlp.experts.207.gate_proj", + "model.layers.64.mlp.experts.207.up_proj", + "model.layers.64.mlp.experts.208.down_proj", + "model.layers.64.mlp.experts.208.gate_proj", + "model.layers.64.mlp.experts.208.up_proj", + "model.layers.64.mlp.experts.209.down_proj", + "model.layers.64.mlp.experts.209.gate_proj", + "model.layers.64.mlp.experts.209.up_proj", + "model.layers.64.mlp.experts.21.down_proj", + "model.layers.64.mlp.experts.21.gate_proj", + "model.layers.64.mlp.experts.21.up_proj", + "model.layers.64.mlp.experts.210.down_proj", + "model.layers.64.mlp.experts.210.gate_proj", + "model.layers.64.mlp.experts.210.up_proj", + "model.layers.64.mlp.experts.211.down_proj", + "model.layers.64.mlp.experts.211.gate_proj", + "model.layers.64.mlp.experts.211.up_proj", + "model.layers.64.mlp.experts.212.down_proj", + "model.layers.64.mlp.experts.212.gate_proj", + "model.layers.64.mlp.experts.212.up_proj", + "model.layers.64.mlp.experts.213.down_proj", + "model.layers.64.mlp.experts.213.gate_proj", + "model.layers.64.mlp.experts.213.up_proj", + "model.layers.64.mlp.experts.214.down_proj", + "model.layers.64.mlp.experts.214.gate_proj", + "model.layers.64.mlp.experts.214.up_proj", + "model.layers.64.mlp.experts.215.down_proj", + "model.layers.64.mlp.experts.215.gate_proj", + "model.layers.64.mlp.experts.215.up_proj", + "model.layers.64.mlp.experts.216.down_proj", + "model.layers.64.mlp.experts.216.gate_proj", + "model.layers.64.mlp.experts.216.up_proj", + "model.layers.64.mlp.experts.217.down_proj", + "model.layers.64.mlp.experts.217.gate_proj", + "model.layers.64.mlp.experts.217.up_proj", + "model.layers.64.mlp.experts.218.down_proj", + "model.layers.64.mlp.experts.218.gate_proj", + "model.layers.64.mlp.experts.218.up_proj", + "model.layers.64.mlp.experts.219.down_proj", + "model.layers.64.mlp.experts.219.gate_proj", + "model.layers.64.mlp.experts.219.up_proj", + "model.layers.64.mlp.experts.22.down_proj", + "model.layers.64.mlp.experts.22.gate_proj", + "model.layers.64.mlp.experts.22.up_proj", + "model.layers.64.mlp.experts.220.down_proj", + "model.layers.64.mlp.experts.220.gate_proj", + "model.layers.64.mlp.experts.220.up_proj", + "model.layers.64.mlp.experts.221.down_proj", + "model.layers.64.mlp.experts.221.gate_proj", + "model.layers.64.mlp.experts.221.up_proj", + "model.layers.64.mlp.experts.222.down_proj", + "model.layers.64.mlp.experts.222.gate_proj", + "model.layers.64.mlp.experts.222.up_proj", + "model.layers.64.mlp.experts.223.down_proj", + "model.layers.64.mlp.experts.223.gate_proj", + "model.layers.64.mlp.experts.223.up_proj", + "model.layers.64.mlp.experts.224.down_proj", + "model.layers.64.mlp.experts.224.gate_proj", + "model.layers.64.mlp.experts.224.up_proj", + "model.layers.64.mlp.experts.225.down_proj", + "model.layers.64.mlp.experts.225.gate_proj", + "model.layers.64.mlp.experts.225.up_proj", + "model.layers.64.mlp.experts.226.down_proj", + "model.layers.64.mlp.experts.226.gate_proj", + "model.layers.64.mlp.experts.226.up_proj", + "model.layers.64.mlp.experts.227.down_proj", + "model.layers.64.mlp.experts.227.gate_proj", + "model.layers.64.mlp.experts.227.up_proj", + "model.layers.64.mlp.experts.228.down_proj", + "model.layers.64.mlp.experts.228.gate_proj", + "model.layers.64.mlp.experts.228.up_proj", + "model.layers.64.mlp.experts.229.down_proj", + "model.layers.64.mlp.experts.229.gate_proj", + "model.layers.64.mlp.experts.229.up_proj", + "model.layers.64.mlp.experts.23.down_proj", + "model.layers.64.mlp.experts.23.gate_proj", + "model.layers.64.mlp.experts.23.up_proj", + "model.layers.64.mlp.experts.230.down_proj", + "model.layers.64.mlp.experts.230.gate_proj", + "model.layers.64.mlp.experts.230.up_proj", + "model.layers.64.mlp.experts.231.down_proj", + "model.layers.64.mlp.experts.231.gate_proj", + "model.layers.64.mlp.experts.231.up_proj", + "model.layers.64.mlp.experts.232.down_proj", + "model.layers.64.mlp.experts.232.gate_proj", + "model.layers.64.mlp.experts.232.up_proj", + "model.layers.64.mlp.experts.233.down_proj", + "model.layers.64.mlp.experts.233.gate_proj", + "model.layers.64.mlp.experts.233.up_proj", + "model.layers.64.mlp.experts.234.down_proj", + "model.layers.64.mlp.experts.234.gate_proj", + "model.layers.64.mlp.experts.234.up_proj", + "model.layers.64.mlp.experts.235.down_proj", + "model.layers.64.mlp.experts.235.gate_proj", + "model.layers.64.mlp.experts.235.up_proj", + "model.layers.64.mlp.experts.236.down_proj", + "model.layers.64.mlp.experts.236.gate_proj", + "model.layers.64.mlp.experts.236.up_proj", + "model.layers.64.mlp.experts.237.down_proj", + "model.layers.64.mlp.experts.237.gate_proj", + "model.layers.64.mlp.experts.237.up_proj", + "model.layers.64.mlp.experts.238.down_proj", + "model.layers.64.mlp.experts.238.gate_proj", + "model.layers.64.mlp.experts.238.up_proj", + "model.layers.64.mlp.experts.239.down_proj", + "model.layers.64.mlp.experts.239.gate_proj", + "model.layers.64.mlp.experts.239.up_proj", + "model.layers.64.mlp.experts.24.down_proj", + "model.layers.64.mlp.experts.24.gate_proj", + "model.layers.64.mlp.experts.24.up_proj", + "model.layers.64.mlp.experts.240.down_proj", + "model.layers.64.mlp.experts.240.gate_proj", + "model.layers.64.mlp.experts.240.up_proj", + "model.layers.64.mlp.experts.241.down_proj", + "model.layers.64.mlp.experts.241.gate_proj", + "model.layers.64.mlp.experts.241.up_proj", + "model.layers.64.mlp.experts.242.down_proj", + "model.layers.64.mlp.experts.242.gate_proj", + "model.layers.64.mlp.experts.242.up_proj", + "model.layers.64.mlp.experts.243.down_proj", + "model.layers.64.mlp.experts.243.gate_proj", + "model.layers.64.mlp.experts.243.up_proj", + "model.layers.64.mlp.experts.244.down_proj", + "model.layers.64.mlp.experts.244.gate_proj", + "model.layers.64.mlp.experts.244.up_proj", + "model.layers.64.mlp.experts.245.down_proj", + "model.layers.64.mlp.experts.245.gate_proj", + "model.layers.64.mlp.experts.245.up_proj", + "model.layers.64.mlp.experts.246.down_proj", + "model.layers.64.mlp.experts.246.gate_proj", + "model.layers.64.mlp.experts.246.up_proj", + "model.layers.64.mlp.experts.247.down_proj", + "model.layers.64.mlp.experts.247.gate_proj", + "model.layers.64.mlp.experts.247.up_proj", + "model.layers.64.mlp.experts.248.down_proj", + "model.layers.64.mlp.experts.248.gate_proj", + "model.layers.64.mlp.experts.248.up_proj", + "model.layers.64.mlp.experts.249.down_proj", + "model.layers.64.mlp.experts.249.gate_proj", + "model.layers.64.mlp.experts.249.up_proj", + "model.layers.64.mlp.experts.25.down_proj", + "model.layers.64.mlp.experts.25.gate_proj", + "model.layers.64.mlp.experts.25.up_proj", + "model.layers.64.mlp.experts.250.down_proj", + "model.layers.64.mlp.experts.250.gate_proj", + "model.layers.64.mlp.experts.250.up_proj", + "model.layers.64.mlp.experts.251.down_proj", + "model.layers.64.mlp.experts.251.gate_proj", + "model.layers.64.mlp.experts.251.up_proj", + "model.layers.64.mlp.experts.252.down_proj", + "model.layers.64.mlp.experts.252.gate_proj", + "model.layers.64.mlp.experts.252.up_proj", + "model.layers.64.mlp.experts.253.down_proj", + "model.layers.64.mlp.experts.253.gate_proj", + "model.layers.64.mlp.experts.253.up_proj", + "model.layers.64.mlp.experts.254.down_proj", + "model.layers.64.mlp.experts.254.gate_proj", + "model.layers.64.mlp.experts.254.up_proj", + "model.layers.64.mlp.experts.255.down_proj", + "model.layers.64.mlp.experts.255.gate_proj", + "model.layers.64.mlp.experts.255.up_proj", + "model.layers.64.mlp.experts.26.down_proj", + "model.layers.64.mlp.experts.26.gate_proj", + "model.layers.64.mlp.experts.26.up_proj", + "model.layers.64.mlp.experts.27.down_proj", + "model.layers.64.mlp.experts.27.gate_proj", + "model.layers.64.mlp.experts.27.up_proj", + "model.layers.64.mlp.experts.28.down_proj", + "model.layers.64.mlp.experts.28.gate_proj", + "model.layers.64.mlp.experts.28.up_proj", + "model.layers.64.mlp.experts.29.down_proj", + "model.layers.64.mlp.experts.29.gate_proj", + "model.layers.64.mlp.experts.29.up_proj", + "model.layers.64.mlp.experts.3.down_proj", + "model.layers.64.mlp.experts.3.gate_proj", + "model.layers.64.mlp.experts.3.up_proj", + "model.layers.64.mlp.experts.30.down_proj", + "model.layers.64.mlp.experts.30.gate_proj", + "model.layers.64.mlp.experts.30.up_proj", + "model.layers.64.mlp.experts.31.down_proj", + "model.layers.64.mlp.experts.31.gate_proj", + "model.layers.64.mlp.experts.31.up_proj", + "model.layers.64.mlp.experts.32.down_proj", + "model.layers.64.mlp.experts.32.gate_proj", + "model.layers.64.mlp.experts.32.up_proj", + "model.layers.64.mlp.experts.33.down_proj", + "model.layers.64.mlp.experts.33.gate_proj", + "model.layers.64.mlp.experts.33.up_proj", + "model.layers.64.mlp.experts.34.down_proj", + "model.layers.64.mlp.experts.34.gate_proj", + "model.layers.64.mlp.experts.34.up_proj", + "model.layers.64.mlp.experts.35.down_proj", + "model.layers.64.mlp.experts.35.gate_proj", + "model.layers.64.mlp.experts.35.up_proj", + "model.layers.64.mlp.experts.36.down_proj", + "model.layers.64.mlp.experts.36.gate_proj", + "model.layers.64.mlp.experts.36.up_proj", + "model.layers.64.mlp.experts.37.down_proj", + "model.layers.64.mlp.experts.37.gate_proj", + "model.layers.64.mlp.experts.37.up_proj", + "model.layers.64.mlp.experts.38.down_proj", + "model.layers.64.mlp.experts.38.gate_proj", + "model.layers.64.mlp.experts.38.up_proj", + "model.layers.64.mlp.experts.39.down_proj", + "model.layers.64.mlp.experts.39.gate_proj", + "model.layers.64.mlp.experts.39.up_proj", + "model.layers.64.mlp.experts.4.down_proj", + "model.layers.64.mlp.experts.4.gate_proj", + "model.layers.64.mlp.experts.4.up_proj", + "model.layers.64.mlp.experts.40.down_proj", + "model.layers.64.mlp.experts.40.gate_proj", + "model.layers.64.mlp.experts.40.up_proj", + "model.layers.64.mlp.experts.41.down_proj", + "model.layers.64.mlp.experts.41.gate_proj", + "model.layers.64.mlp.experts.41.up_proj", + "model.layers.64.mlp.experts.42.down_proj", + "model.layers.64.mlp.experts.42.gate_proj", + "model.layers.64.mlp.experts.42.up_proj", + "model.layers.64.mlp.experts.43.down_proj", + "model.layers.64.mlp.experts.43.gate_proj", + "model.layers.64.mlp.experts.43.up_proj", + "model.layers.64.mlp.experts.44.down_proj", + "model.layers.64.mlp.experts.44.gate_proj", + "model.layers.64.mlp.experts.44.up_proj", + "model.layers.64.mlp.experts.45.down_proj", + "model.layers.64.mlp.experts.45.gate_proj", + "model.layers.64.mlp.experts.45.up_proj", + "model.layers.64.mlp.experts.46.down_proj", + "model.layers.64.mlp.experts.46.gate_proj", + "model.layers.64.mlp.experts.46.up_proj", + "model.layers.64.mlp.experts.47.down_proj", + "model.layers.64.mlp.experts.47.gate_proj", + "model.layers.64.mlp.experts.47.up_proj", + "model.layers.64.mlp.experts.48.down_proj", + "model.layers.64.mlp.experts.48.gate_proj", + "model.layers.64.mlp.experts.48.up_proj", + "model.layers.64.mlp.experts.49.down_proj", + "model.layers.64.mlp.experts.49.gate_proj", + "model.layers.64.mlp.experts.49.up_proj", + "model.layers.64.mlp.experts.5.down_proj", + "model.layers.64.mlp.experts.5.gate_proj", + "model.layers.64.mlp.experts.5.up_proj", + "model.layers.64.mlp.experts.50.down_proj", + "model.layers.64.mlp.experts.50.gate_proj", + "model.layers.64.mlp.experts.50.up_proj", + "model.layers.64.mlp.experts.51.down_proj", + "model.layers.64.mlp.experts.51.gate_proj", + "model.layers.64.mlp.experts.51.up_proj", + "model.layers.64.mlp.experts.52.down_proj", + "model.layers.64.mlp.experts.52.gate_proj", + "model.layers.64.mlp.experts.52.up_proj", + "model.layers.64.mlp.experts.53.down_proj", + "model.layers.64.mlp.experts.53.gate_proj", + "model.layers.64.mlp.experts.53.up_proj", + "model.layers.64.mlp.experts.54.down_proj", + "model.layers.64.mlp.experts.54.gate_proj", + "model.layers.64.mlp.experts.54.up_proj", + "model.layers.64.mlp.experts.55.down_proj", + "model.layers.64.mlp.experts.55.gate_proj", + "model.layers.64.mlp.experts.55.up_proj", + "model.layers.64.mlp.experts.56.down_proj", + "model.layers.64.mlp.experts.56.gate_proj", + "model.layers.64.mlp.experts.56.up_proj", + "model.layers.64.mlp.experts.57.down_proj", + "model.layers.64.mlp.experts.57.gate_proj", + "model.layers.64.mlp.experts.57.up_proj", + "model.layers.64.mlp.experts.58.down_proj", + "model.layers.64.mlp.experts.58.gate_proj", + "model.layers.64.mlp.experts.58.up_proj", + "model.layers.64.mlp.experts.59.down_proj", + "model.layers.64.mlp.experts.59.gate_proj", + "model.layers.64.mlp.experts.59.up_proj", + "model.layers.64.mlp.experts.6.down_proj", + "model.layers.64.mlp.experts.6.gate_proj", + "model.layers.64.mlp.experts.6.up_proj", + "model.layers.64.mlp.experts.60.down_proj", + "model.layers.64.mlp.experts.60.gate_proj", + "model.layers.64.mlp.experts.60.up_proj", + "model.layers.64.mlp.experts.61.down_proj", + "model.layers.64.mlp.experts.61.gate_proj", + "model.layers.64.mlp.experts.61.up_proj", + "model.layers.64.mlp.experts.62.down_proj", + "model.layers.64.mlp.experts.62.gate_proj", + "model.layers.64.mlp.experts.62.up_proj", + "model.layers.64.mlp.experts.63.down_proj", + "model.layers.64.mlp.experts.63.gate_proj", + "model.layers.64.mlp.experts.63.up_proj", + "model.layers.64.mlp.experts.64.down_proj", + "model.layers.64.mlp.experts.64.gate_proj", + "model.layers.64.mlp.experts.64.up_proj", + "model.layers.64.mlp.experts.65.down_proj", + "model.layers.64.mlp.experts.65.gate_proj", + "model.layers.64.mlp.experts.65.up_proj", + "model.layers.64.mlp.experts.66.down_proj", + "model.layers.64.mlp.experts.66.gate_proj", + "model.layers.64.mlp.experts.66.up_proj", + "model.layers.64.mlp.experts.67.down_proj", + "model.layers.64.mlp.experts.67.gate_proj", + "model.layers.64.mlp.experts.67.up_proj", + "model.layers.64.mlp.experts.68.down_proj", + "model.layers.64.mlp.experts.68.gate_proj", + "model.layers.64.mlp.experts.68.up_proj", + "model.layers.64.mlp.experts.69.down_proj", + "model.layers.64.mlp.experts.69.gate_proj", + "model.layers.64.mlp.experts.69.up_proj", + "model.layers.64.mlp.experts.7.down_proj", + "model.layers.64.mlp.experts.7.gate_proj", + "model.layers.64.mlp.experts.7.up_proj", + "model.layers.64.mlp.experts.70.down_proj", + "model.layers.64.mlp.experts.70.gate_proj", + "model.layers.64.mlp.experts.70.up_proj", + "model.layers.64.mlp.experts.71.down_proj", + "model.layers.64.mlp.experts.71.gate_proj", + "model.layers.64.mlp.experts.71.up_proj", + "model.layers.64.mlp.experts.72.down_proj", + "model.layers.64.mlp.experts.72.gate_proj", + "model.layers.64.mlp.experts.72.up_proj", + "model.layers.64.mlp.experts.73.down_proj", + "model.layers.64.mlp.experts.73.gate_proj", + "model.layers.64.mlp.experts.73.up_proj", + "model.layers.64.mlp.experts.74.down_proj", + "model.layers.64.mlp.experts.74.gate_proj", + "model.layers.64.mlp.experts.74.up_proj", + "model.layers.64.mlp.experts.75.down_proj", + "model.layers.64.mlp.experts.75.gate_proj", + "model.layers.64.mlp.experts.75.up_proj", + "model.layers.64.mlp.experts.76.down_proj", + "model.layers.64.mlp.experts.76.gate_proj", + "model.layers.64.mlp.experts.76.up_proj", + "model.layers.64.mlp.experts.77.down_proj", + "model.layers.64.mlp.experts.77.gate_proj", + "model.layers.64.mlp.experts.77.up_proj", + "model.layers.64.mlp.experts.78.down_proj", + "model.layers.64.mlp.experts.78.gate_proj", + "model.layers.64.mlp.experts.78.up_proj", + "model.layers.64.mlp.experts.79.down_proj", + "model.layers.64.mlp.experts.79.gate_proj", + "model.layers.64.mlp.experts.79.up_proj", + "model.layers.64.mlp.experts.8.down_proj", + "model.layers.64.mlp.experts.8.gate_proj", + "model.layers.64.mlp.experts.8.up_proj", + "model.layers.64.mlp.experts.80.down_proj", + "model.layers.64.mlp.experts.80.gate_proj", + "model.layers.64.mlp.experts.80.up_proj", + "model.layers.64.mlp.experts.81.down_proj", + "model.layers.64.mlp.experts.81.gate_proj", + "model.layers.64.mlp.experts.81.up_proj", + "model.layers.64.mlp.experts.82.down_proj", + "model.layers.64.mlp.experts.82.gate_proj", + "model.layers.64.mlp.experts.82.up_proj", + "model.layers.64.mlp.experts.83.down_proj", + "model.layers.64.mlp.experts.83.gate_proj", + "model.layers.64.mlp.experts.83.up_proj", + "model.layers.64.mlp.experts.84.down_proj", + "model.layers.64.mlp.experts.84.gate_proj", + "model.layers.64.mlp.experts.84.up_proj", + "model.layers.64.mlp.experts.85.down_proj", + "model.layers.64.mlp.experts.85.gate_proj", + "model.layers.64.mlp.experts.85.up_proj", + "model.layers.64.mlp.experts.86.down_proj", + "model.layers.64.mlp.experts.86.gate_proj", + "model.layers.64.mlp.experts.86.up_proj", + "model.layers.64.mlp.experts.87.down_proj", + "model.layers.64.mlp.experts.87.gate_proj", + "model.layers.64.mlp.experts.87.up_proj", + "model.layers.64.mlp.experts.88.down_proj", + "model.layers.64.mlp.experts.88.gate_proj", + "model.layers.64.mlp.experts.88.up_proj", + "model.layers.64.mlp.experts.89.down_proj", + "model.layers.64.mlp.experts.89.gate_proj", + "model.layers.64.mlp.experts.89.up_proj", + "model.layers.64.mlp.experts.9.down_proj", + "model.layers.64.mlp.experts.9.gate_proj", + "model.layers.64.mlp.experts.9.up_proj", + "model.layers.64.mlp.experts.90.down_proj", + "model.layers.64.mlp.experts.90.gate_proj", + "model.layers.64.mlp.experts.90.up_proj", + "model.layers.64.mlp.experts.91.down_proj", + "model.layers.64.mlp.experts.91.gate_proj", + "model.layers.64.mlp.experts.91.up_proj", + "model.layers.64.mlp.experts.92.down_proj", + "model.layers.64.mlp.experts.92.gate_proj", + "model.layers.64.mlp.experts.92.up_proj", + "model.layers.64.mlp.experts.93.down_proj", + "model.layers.64.mlp.experts.93.gate_proj", + "model.layers.64.mlp.experts.93.up_proj", + "model.layers.64.mlp.experts.94.down_proj", + "model.layers.64.mlp.experts.94.gate_proj", + "model.layers.64.mlp.experts.94.up_proj", + "model.layers.64.mlp.experts.95.down_proj", + "model.layers.64.mlp.experts.95.gate_proj", + "model.layers.64.mlp.experts.95.up_proj", + "model.layers.64.mlp.experts.96.down_proj", + "model.layers.64.mlp.experts.96.gate_proj", + "model.layers.64.mlp.experts.96.up_proj", + "model.layers.64.mlp.experts.97.down_proj", + "model.layers.64.mlp.experts.97.gate_proj", + "model.layers.64.mlp.experts.97.up_proj", + "model.layers.64.mlp.experts.98.down_proj", + "model.layers.64.mlp.experts.98.gate_proj", + "model.layers.64.mlp.experts.98.up_proj", + "model.layers.64.mlp.experts.99.down_proj", + "model.layers.64.mlp.experts.99.gate_proj", + "model.layers.64.mlp.experts.99.up_proj", + "model.layers.65.mlp.experts.0.down_proj", + "model.layers.65.mlp.experts.0.gate_proj", + "model.layers.65.mlp.experts.0.up_proj", + "model.layers.65.mlp.experts.1.down_proj", + "model.layers.65.mlp.experts.1.gate_proj", + "model.layers.65.mlp.experts.1.up_proj", + "model.layers.65.mlp.experts.10.down_proj", + "model.layers.65.mlp.experts.10.gate_proj", + "model.layers.65.mlp.experts.10.up_proj", + "model.layers.65.mlp.experts.100.down_proj", + "model.layers.65.mlp.experts.100.gate_proj", + "model.layers.65.mlp.experts.100.up_proj", + "model.layers.65.mlp.experts.101.down_proj", + "model.layers.65.mlp.experts.101.gate_proj", + "model.layers.65.mlp.experts.101.up_proj", + "model.layers.65.mlp.experts.102.down_proj", + "model.layers.65.mlp.experts.102.gate_proj", + "model.layers.65.mlp.experts.102.up_proj", + "model.layers.65.mlp.experts.103.down_proj", + "model.layers.65.mlp.experts.103.gate_proj", + "model.layers.65.mlp.experts.103.up_proj", + "model.layers.65.mlp.experts.104.down_proj", + "model.layers.65.mlp.experts.104.gate_proj", + "model.layers.65.mlp.experts.104.up_proj", + "model.layers.65.mlp.experts.105.down_proj", + "model.layers.65.mlp.experts.105.gate_proj", + "model.layers.65.mlp.experts.105.up_proj", + "model.layers.65.mlp.experts.106.down_proj", + "model.layers.65.mlp.experts.106.gate_proj", + "model.layers.65.mlp.experts.106.up_proj", + "model.layers.65.mlp.experts.107.down_proj", + "model.layers.65.mlp.experts.107.gate_proj", + "model.layers.65.mlp.experts.107.up_proj", + "model.layers.65.mlp.experts.108.down_proj", + "model.layers.65.mlp.experts.108.gate_proj", + "model.layers.65.mlp.experts.108.up_proj", + "model.layers.65.mlp.experts.109.down_proj", + "model.layers.65.mlp.experts.109.gate_proj", + "model.layers.65.mlp.experts.109.up_proj", + "model.layers.65.mlp.experts.11.down_proj", + "model.layers.65.mlp.experts.11.gate_proj", + "model.layers.65.mlp.experts.11.up_proj", + "model.layers.65.mlp.experts.110.down_proj", + "model.layers.65.mlp.experts.110.gate_proj", + "model.layers.65.mlp.experts.110.up_proj", + "model.layers.65.mlp.experts.111.down_proj", + "model.layers.65.mlp.experts.111.gate_proj", + "model.layers.65.mlp.experts.111.up_proj", + "model.layers.65.mlp.experts.112.down_proj", + "model.layers.65.mlp.experts.112.gate_proj", + "model.layers.65.mlp.experts.112.up_proj", + "model.layers.65.mlp.experts.113.down_proj", + "model.layers.65.mlp.experts.113.gate_proj", + "model.layers.65.mlp.experts.113.up_proj", + "model.layers.65.mlp.experts.114.down_proj", + "model.layers.65.mlp.experts.114.gate_proj", + "model.layers.65.mlp.experts.114.up_proj", + "model.layers.65.mlp.experts.115.down_proj", + "model.layers.65.mlp.experts.115.gate_proj", + "model.layers.65.mlp.experts.115.up_proj", + "model.layers.65.mlp.experts.116.down_proj", + "model.layers.65.mlp.experts.116.gate_proj", + "model.layers.65.mlp.experts.116.up_proj", + "model.layers.65.mlp.experts.117.down_proj", + "model.layers.65.mlp.experts.117.gate_proj", + "model.layers.65.mlp.experts.117.up_proj", + "model.layers.65.mlp.experts.118.down_proj", + "model.layers.65.mlp.experts.118.gate_proj", + "model.layers.65.mlp.experts.118.up_proj", + "model.layers.65.mlp.experts.119.down_proj", + "model.layers.65.mlp.experts.119.gate_proj", + "model.layers.65.mlp.experts.119.up_proj", + "model.layers.65.mlp.experts.12.down_proj", + "model.layers.65.mlp.experts.12.gate_proj", + "model.layers.65.mlp.experts.12.up_proj", + "model.layers.65.mlp.experts.120.down_proj", + "model.layers.65.mlp.experts.120.gate_proj", + "model.layers.65.mlp.experts.120.up_proj", + "model.layers.65.mlp.experts.121.down_proj", + "model.layers.65.mlp.experts.121.gate_proj", + "model.layers.65.mlp.experts.121.up_proj", + "model.layers.65.mlp.experts.122.down_proj", + "model.layers.65.mlp.experts.122.gate_proj", + "model.layers.65.mlp.experts.122.up_proj", + "model.layers.65.mlp.experts.123.down_proj", + "model.layers.65.mlp.experts.123.gate_proj", + "model.layers.65.mlp.experts.123.up_proj", + "model.layers.65.mlp.experts.124.down_proj", + "model.layers.65.mlp.experts.124.gate_proj", + "model.layers.65.mlp.experts.124.up_proj", + "model.layers.65.mlp.experts.125.down_proj", + "model.layers.65.mlp.experts.125.gate_proj", + "model.layers.65.mlp.experts.125.up_proj", + "model.layers.65.mlp.experts.126.down_proj", + "model.layers.65.mlp.experts.126.gate_proj", + "model.layers.65.mlp.experts.126.up_proj", + "model.layers.65.mlp.experts.127.down_proj", + "model.layers.65.mlp.experts.127.gate_proj", + "model.layers.65.mlp.experts.127.up_proj", + "model.layers.65.mlp.experts.128.down_proj", + "model.layers.65.mlp.experts.128.gate_proj", + "model.layers.65.mlp.experts.128.up_proj", + "model.layers.65.mlp.experts.129.down_proj", + "model.layers.65.mlp.experts.129.gate_proj", + "model.layers.65.mlp.experts.129.up_proj", + "model.layers.65.mlp.experts.13.down_proj", + "model.layers.65.mlp.experts.13.gate_proj", + "model.layers.65.mlp.experts.13.up_proj", + "model.layers.65.mlp.experts.130.down_proj", + "model.layers.65.mlp.experts.130.gate_proj", + "model.layers.65.mlp.experts.130.up_proj", + "model.layers.65.mlp.experts.131.down_proj", + "model.layers.65.mlp.experts.131.gate_proj", + "model.layers.65.mlp.experts.131.up_proj", + "model.layers.65.mlp.experts.132.down_proj", + "model.layers.65.mlp.experts.132.gate_proj", + "model.layers.65.mlp.experts.132.up_proj", + "model.layers.65.mlp.experts.133.down_proj", + "model.layers.65.mlp.experts.133.gate_proj", + "model.layers.65.mlp.experts.133.up_proj", + "model.layers.65.mlp.experts.134.down_proj", + "model.layers.65.mlp.experts.134.gate_proj", + "model.layers.65.mlp.experts.134.up_proj", + "model.layers.65.mlp.experts.135.down_proj", + "model.layers.65.mlp.experts.135.gate_proj", + "model.layers.65.mlp.experts.135.up_proj", + "model.layers.65.mlp.experts.136.down_proj", + "model.layers.65.mlp.experts.136.gate_proj", + "model.layers.65.mlp.experts.136.up_proj", + "model.layers.65.mlp.experts.137.down_proj", + "model.layers.65.mlp.experts.137.gate_proj", + "model.layers.65.mlp.experts.137.up_proj", + "model.layers.65.mlp.experts.138.down_proj", + "model.layers.65.mlp.experts.138.gate_proj", + "model.layers.65.mlp.experts.138.up_proj", + "model.layers.65.mlp.experts.139.down_proj", + "model.layers.65.mlp.experts.139.gate_proj", + "model.layers.65.mlp.experts.139.up_proj", + "model.layers.65.mlp.experts.14.down_proj", + "model.layers.65.mlp.experts.14.gate_proj", + "model.layers.65.mlp.experts.14.up_proj", + "model.layers.65.mlp.experts.140.down_proj", + "model.layers.65.mlp.experts.140.gate_proj", + "model.layers.65.mlp.experts.140.up_proj", + "model.layers.65.mlp.experts.141.down_proj", + "model.layers.65.mlp.experts.141.gate_proj", + "model.layers.65.mlp.experts.141.up_proj", + "model.layers.65.mlp.experts.142.down_proj", + "model.layers.65.mlp.experts.142.gate_proj", + "model.layers.65.mlp.experts.142.up_proj", + "model.layers.65.mlp.experts.143.down_proj", + "model.layers.65.mlp.experts.143.gate_proj", + "model.layers.65.mlp.experts.143.up_proj", + "model.layers.65.mlp.experts.144.down_proj", + "model.layers.65.mlp.experts.144.gate_proj", + "model.layers.65.mlp.experts.144.up_proj", + "model.layers.65.mlp.experts.145.down_proj", + "model.layers.65.mlp.experts.145.gate_proj", + "model.layers.65.mlp.experts.145.up_proj", + "model.layers.65.mlp.experts.146.down_proj", + "model.layers.65.mlp.experts.146.gate_proj", + "model.layers.65.mlp.experts.146.up_proj", + "model.layers.65.mlp.experts.147.down_proj", + "model.layers.65.mlp.experts.147.gate_proj", + "model.layers.65.mlp.experts.147.up_proj", + "model.layers.65.mlp.experts.148.down_proj", + "model.layers.65.mlp.experts.148.gate_proj", + "model.layers.65.mlp.experts.148.up_proj", + "model.layers.65.mlp.experts.149.down_proj", + "model.layers.65.mlp.experts.149.gate_proj", + "model.layers.65.mlp.experts.149.up_proj", + "model.layers.65.mlp.experts.15.down_proj", + "model.layers.65.mlp.experts.15.gate_proj", + "model.layers.65.mlp.experts.15.up_proj", + "model.layers.65.mlp.experts.150.down_proj", + "model.layers.65.mlp.experts.150.gate_proj", + "model.layers.65.mlp.experts.150.up_proj", + "model.layers.65.mlp.experts.151.down_proj", + "model.layers.65.mlp.experts.151.gate_proj", + "model.layers.65.mlp.experts.151.up_proj", + "model.layers.65.mlp.experts.152.down_proj", + "model.layers.65.mlp.experts.152.gate_proj", + "model.layers.65.mlp.experts.152.up_proj", + "model.layers.65.mlp.experts.153.down_proj", + "model.layers.65.mlp.experts.153.gate_proj", + "model.layers.65.mlp.experts.153.up_proj", + "model.layers.65.mlp.experts.154.down_proj", + "model.layers.65.mlp.experts.154.gate_proj", + "model.layers.65.mlp.experts.154.up_proj", + "model.layers.65.mlp.experts.155.down_proj", + "model.layers.65.mlp.experts.155.gate_proj", + "model.layers.65.mlp.experts.155.up_proj", + "model.layers.65.mlp.experts.156.down_proj", + "model.layers.65.mlp.experts.156.gate_proj", + "model.layers.65.mlp.experts.156.up_proj", + "model.layers.65.mlp.experts.157.down_proj", + "model.layers.65.mlp.experts.157.gate_proj", + "model.layers.65.mlp.experts.157.up_proj", + "model.layers.65.mlp.experts.158.down_proj", + "model.layers.65.mlp.experts.158.gate_proj", + "model.layers.65.mlp.experts.158.up_proj", + "model.layers.65.mlp.experts.159.down_proj", + "model.layers.65.mlp.experts.159.gate_proj", + "model.layers.65.mlp.experts.159.up_proj", + "model.layers.65.mlp.experts.16.down_proj", + "model.layers.65.mlp.experts.16.gate_proj", + "model.layers.65.mlp.experts.16.up_proj", + "model.layers.65.mlp.experts.160.down_proj", + "model.layers.65.mlp.experts.160.gate_proj", + "model.layers.65.mlp.experts.160.up_proj", + "model.layers.65.mlp.experts.161.down_proj", + "model.layers.65.mlp.experts.161.gate_proj", + "model.layers.65.mlp.experts.161.up_proj", + "model.layers.65.mlp.experts.162.down_proj", + "model.layers.65.mlp.experts.162.gate_proj", + "model.layers.65.mlp.experts.162.up_proj", + "model.layers.65.mlp.experts.163.down_proj", + "model.layers.65.mlp.experts.163.gate_proj", + "model.layers.65.mlp.experts.163.up_proj", + "model.layers.65.mlp.experts.164.down_proj", + "model.layers.65.mlp.experts.164.gate_proj", + "model.layers.65.mlp.experts.164.up_proj", + "model.layers.65.mlp.experts.165.down_proj", + "model.layers.65.mlp.experts.165.gate_proj", + "model.layers.65.mlp.experts.165.up_proj", + "model.layers.65.mlp.experts.166.down_proj", + "model.layers.65.mlp.experts.166.gate_proj", + "model.layers.65.mlp.experts.166.up_proj", + "model.layers.65.mlp.experts.167.down_proj", + "model.layers.65.mlp.experts.167.gate_proj", + "model.layers.65.mlp.experts.167.up_proj", + "model.layers.65.mlp.experts.168.down_proj", + "model.layers.65.mlp.experts.168.gate_proj", + "model.layers.65.mlp.experts.168.up_proj", + "model.layers.65.mlp.experts.169.down_proj", + "model.layers.65.mlp.experts.169.gate_proj", + "model.layers.65.mlp.experts.169.up_proj", + "model.layers.65.mlp.experts.17.down_proj", + "model.layers.65.mlp.experts.17.gate_proj", + "model.layers.65.mlp.experts.17.up_proj", + "model.layers.65.mlp.experts.170.down_proj", + "model.layers.65.mlp.experts.170.gate_proj", + "model.layers.65.mlp.experts.170.up_proj", + "model.layers.65.mlp.experts.171.down_proj", + "model.layers.65.mlp.experts.171.gate_proj", + "model.layers.65.mlp.experts.171.up_proj", + "model.layers.65.mlp.experts.172.down_proj", + "model.layers.65.mlp.experts.172.gate_proj", + "model.layers.65.mlp.experts.172.up_proj", + "model.layers.65.mlp.experts.173.down_proj", + "model.layers.65.mlp.experts.173.gate_proj", + "model.layers.65.mlp.experts.173.up_proj", + "model.layers.65.mlp.experts.174.down_proj", + "model.layers.65.mlp.experts.174.gate_proj", + "model.layers.65.mlp.experts.174.up_proj", + "model.layers.65.mlp.experts.175.down_proj", + "model.layers.65.mlp.experts.175.gate_proj", + "model.layers.65.mlp.experts.175.up_proj", + "model.layers.65.mlp.experts.176.down_proj", + "model.layers.65.mlp.experts.176.gate_proj", + "model.layers.65.mlp.experts.176.up_proj", + "model.layers.65.mlp.experts.177.down_proj", + "model.layers.65.mlp.experts.177.gate_proj", + "model.layers.65.mlp.experts.177.up_proj", + "model.layers.65.mlp.experts.178.down_proj", + "model.layers.65.mlp.experts.178.gate_proj", + "model.layers.65.mlp.experts.178.up_proj", + "model.layers.65.mlp.experts.179.down_proj", + "model.layers.65.mlp.experts.179.gate_proj", + "model.layers.65.mlp.experts.179.up_proj", + "model.layers.65.mlp.experts.18.down_proj", + "model.layers.65.mlp.experts.18.gate_proj", + "model.layers.65.mlp.experts.18.up_proj", + "model.layers.65.mlp.experts.180.down_proj", + "model.layers.65.mlp.experts.180.gate_proj", + "model.layers.65.mlp.experts.180.up_proj", + "model.layers.65.mlp.experts.181.down_proj", + "model.layers.65.mlp.experts.181.gate_proj", + "model.layers.65.mlp.experts.181.up_proj", + "model.layers.65.mlp.experts.182.down_proj", + "model.layers.65.mlp.experts.182.gate_proj", + "model.layers.65.mlp.experts.182.up_proj", + "model.layers.65.mlp.experts.183.down_proj", + "model.layers.65.mlp.experts.183.gate_proj", + "model.layers.65.mlp.experts.183.up_proj", + "model.layers.65.mlp.experts.184.down_proj", + "model.layers.65.mlp.experts.184.gate_proj", + "model.layers.65.mlp.experts.184.up_proj", + "model.layers.65.mlp.experts.185.down_proj", + "model.layers.65.mlp.experts.185.gate_proj", + "model.layers.65.mlp.experts.185.up_proj", + "model.layers.65.mlp.experts.186.down_proj", + "model.layers.65.mlp.experts.186.gate_proj", + "model.layers.65.mlp.experts.186.up_proj", + "model.layers.65.mlp.experts.187.down_proj", + "model.layers.65.mlp.experts.187.gate_proj", + "model.layers.65.mlp.experts.187.up_proj", + "model.layers.65.mlp.experts.188.down_proj", + "model.layers.65.mlp.experts.188.gate_proj", + "model.layers.65.mlp.experts.188.up_proj", + "model.layers.65.mlp.experts.189.down_proj", + "model.layers.65.mlp.experts.189.gate_proj", + "model.layers.65.mlp.experts.189.up_proj", + "model.layers.65.mlp.experts.19.down_proj", + "model.layers.65.mlp.experts.19.gate_proj", + "model.layers.65.mlp.experts.19.up_proj", + "model.layers.65.mlp.experts.190.down_proj", + "model.layers.65.mlp.experts.190.gate_proj", + "model.layers.65.mlp.experts.190.up_proj", + "model.layers.65.mlp.experts.191.down_proj", + "model.layers.65.mlp.experts.191.gate_proj", + "model.layers.65.mlp.experts.191.up_proj", + "model.layers.65.mlp.experts.192.down_proj", + "model.layers.65.mlp.experts.192.gate_proj", + "model.layers.65.mlp.experts.192.up_proj", + "model.layers.65.mlp.experts.193.down_proj", + "model.layers.65.mlp.experts.193.gate_proj", + "model.layers.65.mlp.experts.193.up_proj", + "model.layers.65.mlp.experts.194.down_proj", + "model.layers.65.mlp.experts.194.gate_proj", + "model.layers.65.mlp.experts.194.up_proj", + "model.layers.65.mlp.experts.195.down_proj", + "model.layers.65.mlp.experts.195.gate_proj", + "model.layers.65.mlp.experts.195.up_proj", + "model.layers.65.mlp.experts.196.down_proj", + "model.layers.65.mlp.experts.196.gate_proj", + "model.layers.65.mlp.experts.196.up_proj", + "model.layers.65.mlp.experts.197.down_proj", + "model.layers.65.mlp.experts.197.gate_proj", + "model.layers.65.mlp.experts.197.up_proj", + "model.layers.65.mlp.experts.198.down_proj", + "model.layers.65.mlp.experts.198.gate_proj", + "model.layers.65.mlp.experts.198.up_proj", + "model.layers.65.mlp.experts.199.down_proj", + "model.layers.65.mlp.experts.199.gate_proj", + "model.layers.65.mlp.experts.199.up_proj", + "model.layers.65.mlp.experts.2.down_proj", + "model.layers.65.mlp.experts.2.gate_proj", + "model.layers.65.mlp.experts.2.up_proj", + "model.layers.65.mlp.experts.20.down_proj", + "model.layers.65.mlp.experts.20.gate_proj", + "model.layers.65.mlp.experts.20.up_proj", + "model.layers.65.mlp.experts.200.down_proj", + "model.layers.65.mlp.experts.200.gate_proj", + "model.layers.65.mlp.experts.200.up_proj", + "model.layers.65.mlp.experts.201.down_proj", + "model.layers.65.mlp.experts.201.gate_proj", + "model.layers.65.mlp.experts.201.up_proj", + "model.layers.65.mlp.experts.202.down_proj", + "model.layers.65.mlp.experts.202.gate_proj", + "model.layers.65.mlp.experts.202.up_proj", + "model.layers.65.mlp.experts.203.down_proj", + "model.layers.65.mlp.experts.203.gate_proj", + "model.layers.65.mlp.experts.203.up_proj", + "model.layers.65.mlp.experts.204.down_proj", + "model.layers.65.mlp.experts.204.gate_proj", + "model.layers.65.mlp.experts.204.up_proj", + "model.layers.65.mlp.experts.205.down_proj", + "model.layers.65.mlp.experts.205.gate_proj", + "model.layers.65.mlp.experts.205.up_proj", + "model.layers.65.mlp.experts.206.down_proj", + "model.layers.65.mlp.experts.206.gate_proj", + "model.layers.65.mlp.experts.206.up_proj", + "model.layers.65.mlp.experts.207.down_proj", + "model.layers.65.mlp.experts.207.gate_proj", + "model.layers.65.mlp.experts.207.up_proj", + "model.layers.65.mlp.experts.208.down_proj", + "model.layers.65.mlp.experts.208.gate_proj", + "model.layers.65.mlp.experts.208.up_proj", + "model.layers.65.mlp.experts.209.down_proj", + "model.layers.65.mlp.experts.209.gate_proj", + "model.layers.65.mlp.experts.209.up_proj", + "model.layers.65.mlp.experts.21.down_proj", + "model.layers.65.mlp.experts.21.gate_proj", + "model.layers.65.mlp.experts.21.up_proj", + "model.layers.65.mlp.experts.210.down_proj", + "model.layers.65.mlp.experts.210.gate_proj", + "model.layers.65.mlp.experts.210.up_proj", + "model.layers.65.mlp.experts.211.down_proj", + "model.layers.65.mlp.experts.211.gate_proj", + "model.layers.65.mlp.experts.211.up_proj", + "model.layers.65.mlp.experts.212.down_proj", + "model.layers.65.mlp.experts.212.gate_proj", + "model.layers.65.mlp.experts.212.up_proj", + "model.layers.65.mlp.experts.213.down_proj", + "model.layers.65.mlp.experts.213.gate_proj", + "model.layers.65.mlp.experts.213.up_proj", + "model.layers.65.mlp.experts.214.down_proj", + "model.layers.65.mlp.experts.214.gate_proj", + "model.layers.65.mlp.experts.214.up_proj", + "model.layers.65.mlp.experts.215.down_proj", + "model.layers.65.mlp.experts.215.gate_proj", + "model.layers.65.mlp.experts.215.up_proj", + "model.layers.65.mlp.experts.216.down_proj", + "model.layers.65.mlp.experts.216.gate_proj", + "model.layers.65.mlp.experts.216.up_proj", + "model.layers.65.mlp.experts.217.down_proj", + "model.layers.65.mlp.experts.217.gate_proj", + "model.layers.65.mlp.experts.217.up_proj", + "model.layers.65.mlp.experts.218.down_proj", + "model.layers.65.mlp.experts.218.gate_proj", + "model.layers.65.mlp.experts.218.up_proj", + "model.layers.65.mlp.experts.219.down_proj", + "model.layers.65.mlp.experts.219.gate_proj", + "model.layers.65.mlp.experts.219.up_proj", + "model.layers.65.mlp.experts.22.down_proj", + "model.layers.65.mlp.experts.22.gate_proj", + "model.layers.65.mlp.experts.22.up_proj", + "model.layers.65.mlp.experts.220.down_proj", + "model.layers.65.mlp.experts.220.gate_proj", + "model.layers.65.mlp.experts.220.up_proj", + "model.layers.65.mlp.experts.221.down_proj", + "model.layers.65.mlp.experts.221.gate_proj", + "model.layers.65.mlp.experts.221.up_proj", + "model.layers.65.mlp.experts.222.down_proj", + "model.layers.65.mlp.experts.222.gate_proj", + "model.layers.65.mlp.experts.222.up_proj", + "model.layers.65.mlp.experts.223.down_proj", + "model.layers.65.mlp.experts.223.gate_proj", + "model.layers.65.mlp.experts.223.up_proj", + "model.layers.65.mlp.experts.224.down_proj", + "model.layers.65.mlp.experts.224.gate_proj", + "model.layers.65.mlp.experts.224.up_proj", + "model.layers.65.mlp.experts.225.down_proj", + "model.layers.65.mlp.experts.225.gate_proj", + "model.layers.65.mlp.experts.225.up_proj", + "model.layers.65.mlp.experts.226.down_proj", + "model.layers.65.mlp.experts.226.gate_proj", + "model.layers.65.mlp.experts.226.up_proj", + "model.layers.65.mlp.experts.227.down_proj", + "model.layers.65.mlp.experts.227.gate_proj", + "model.layers.65.mlp.experts.227.up_proj", + "model.layers.65.mlp.experts.228.down_proj", + "model.layers.65.mlp.experts.228.gate_proj", + "model.layers.65.mlp.experts.228.up_proj", + "model.layers.65.mlp.experts.229.down_proj", + "model.layers.65.mlp.experts.229.gate_proj", + "model.layers.65.mlp.experts.229.up_proj", + "model.layers.65.mlp.experts.23.down_proj", + "model.layers.65.mlp.experts.23.gate_proj", + "model.layers.65.mlp.experts.23.up_proj", + "model.layers.65.mlp.experts.230.down_proj", + "model.layers.65.mlp.experts.230.gate_proj", + "model.layers.65.mlp.experts.230.up_proj", + "model.layers.65.mlp.experts.231.down_proj", + "model.layers.65.mlp.experts.231.gate_proj", + "model.layers.65.mlp.experts.231.up_proj", + "model.layers.65.mlp.experts.232.down_proj", + "model.layers.65.mlp.experts.232.gate_proj", + "model.layers.65.mlp.experts.232.up_proj", + "model.layers.65.mlp.experts.233.down_proj", + "model.layers.65.mlp.experts.233.gate_proj", + "model.layers.65.mlp.experts.233.up_proj", + "model.layers.65.mlp.experts.234.down_proj", + "model.layers.65.mlp.experts.234.gate_proj", + "model.layers.65.mlp.experts.234.up_proj", + "model.layers.65.mlp.experts.235.down_proj", + "model.layers.65.mlp.experts.235.gate_proj", + "model.layers.65.mlp.experts.235.up_proj", + "model.layers.65.mlp.experts.236.down_proj", + "model.layers.65.mlp.experts.236.gate_proj", + "model.layers.65.mlp.experts.236.up_proj", + "model.layers.65.mlp.experts.237.down_proj", + "model.layers.65.mlp.experts.237.gate_proj", + "model.layers.65.mlp.experts.237.up_proj", + "model.layers.65.mlp.experts.238.down_proj", + "model.layers.65.mlp.experts.238.gate_proj", + "model.layers.65.mlp.experts.238.up_proj", + "model.layers.65.mlp.experts.239.down_proj", + "model.layers.65.mlp.experts.239.gate_proj", + "model.layers.65.mlp.experts.239.up_proj", + "model.layers.65.mlp.experts.24.down_proj", + "model.layers.65.mlp.experts.24.gate_proj", + "model.layers.65.mlp.experts.24.up_proj", + "model.layers.65.mlp.experts.240.down_proj", + "model.layers.65.mlp.experts.240.gate_proj", + "model.layers.65.mlp.experts.240.up_proj", + "model.layers.65.mlp.experts.241.down_proj", + "model.layers.65.mlp.experts.241.gate_proj", + "model.layers.65.mlp.experts.241.up_proj", + "model.layers.65.mlp.experts.242.down_proj", + "model.layers.65.mlp.experts.242.gate_proj", + "model.layers.65.mlp.experts.242.up_proj", + "model.layers.65.mlp.experts.243.down_proj", + "model.layers.65.mlp.experts.243.gate_proj", + "model.layers.65.mlp.experts.243.up_proj", + "model.layers.65.mlp.experts.244.down_proj", + "model.layers.65.mlp.experts.244.gate_proj", + "model.layers.65.mlp.experts.244.up_proj", + "model.layers.65.mlp.experts.245.down_proj", + "model.layers.65.mlp.experts.245.gate_proj", + "model.layers.65.mlp.experts.245.up_proj", + "model.layers.65.mlp.experts.246.down_proj", + "model.layers.65.mlp.experts.246.gate_proj", + "model.layers.65.mlp.experts.246.up_proj", + "model.layers.65.mlp.experts.247.down_proj", + "model.layers.65.mlp.experts.247.gate_proj", + "model.layers.65.mlp.experts.247.up_proj", + "model.layers.65.mlp.experts.248.down_proj", + "model.layers.65.mlp.experts.248.gate_proj", + "model.layers.65.mlp.experts.248.up_proj", + "model.layers.65.mlp.experts.249.down_proj", + "model.layers.65.mlp.experts.249.gate_proj", + "model.layers.65.mlp.experts.249.up_proj", + "model.layers.65.mlp.experts.25.down_proj", + "model.layers.65.mlp.experts.25.gate_proj", + "model.layers.65.mlp.experts.25.up_proj", + "model.layers.65.mlp.experts.250.down_proj", + "model.layers.65.mlp.experts.250.gate_proj", + "model.layers.65.mlp.experts.250.up_proj", + "model.layers.65.mlp.experts.251.down_proj", + "model.layers.65.mlp.experts.251.gate_proj", + "model.layers.65.mlp.experts.251.up_proj", + "model.layers.65.mlp.experts.252.down_proj", + "model.layers.65.mlp.experts.252.gate_proj", + "model.layers.65.mlp.experts.252.up_proj", + "model.layers.65.mlp.experts.253.down_proj", + "model.layers.65.mlp.experts.253.gate_proj", + "model.layers.65.mlp.experts.253.up_proj", + "model.layers.65.mlp.experts.254.down_proj", + "model.layers.65.mlp.experts.254.gate_proj", + "model.layers.65.mlp.experts.254.up_proj", + "model.layers.65.mlp.experts.255.down_proj", + "model.layers.65.mlp.experts.255.gate_proj", + "model.layers.65.mlp.experts.255.up_proj", + "model.layers.65.mlp.experts.26.down_proj", + "model.layers.65.mlp.experts.26.gate_proj", + "model.layers.65.mlp.experts.26.up_proj", + "model.layers.65.mlp.experts.27.down_proj", + "model.layers.65.mlp.experts.27.gate_proj", + "model.layers.65.mlp.experts.27.up_proj", + "model.layers.65.mlp.experts.28.down_proj", + "model.layers.65.mlp.experts.28.gate_proj", + "model.layers.65.mlp.experts.28.up_proj", + "model.layers.65.mlp.experts.29.down_proj", + "model.layers.65.mlp.experts.29.gate_proj", + "model.layers.65.mlp.experts.29.up_proj", + "model.layers.65.mlp.experts.3.down_proj", + "model.layers.65.mlp.experts.3.gate_proj", + "model.layers.65.mlp.experts.3.up_proj", + "model.layers.65.mlp.experts.30.down_proj", + "model.layers.65.mlp.experts.30.gate_proj", + "model.layers.65.mlp.experts.30.up_proj", + "model.layers.65.mlp.experts.31.down_proj", + "model.layers.65.mlp.experts.31.gate_proj", + "model.layers.65.mlp.experts.31.up_proj", + "model.layers.65.mlp.experts.32.down_proj", + "model.layers.65.mlp.experts.32.gate_proj", + "model.layers.65.mlp.experts.32.up_proj", + "model.layers.65.mlp.experts.33.down_proj", + "model.layers.65.mlp.experts.33.gate_proj", + "model.layers.65.mlp.experts.33.up_proj", + "model.layers.65.mlp.experts.34.down_proj", + "model.layers.65.mlp.experts.34.gate_proj", + "model.layers.65.mlp.experts.34.up_proj", + "model.layers.65.mlp.experts.35.down_proj", + "model.layers.65.mlp.experts.35.gate_proj", + "model.layers.65.mlp.experts.35.up_proj", + "model.layers.65.mlp.experts.36.down_proj", + "model.layers.65.mlp.experts.36.gate_proj", + "model.layers.65.mlp.experts.36.up_proj", + "model.layers.65.mlp.experts.37.down_proj", + "model.layers.65.mlp.experts.37.gate_proj", + "model.layers.65.mlp.experts.37.up_proj", + "model.layers.65.mlp.experts.38.down_proj", + "model.layers.65.mlp.experts.38.gate_proj", + "model.layers.65.mlp.experts.38.up_proj", + "model.layers.65.mlp.experts.39.down_proj", + "model.layers.65.mlp.experts.39.gate_proj", + "model.layers.65.mlp.experts.39.up_proj", + "model.layers.65.mlp.experts.4.down_proj", + "model.layers.65.mlp.experts.4.gate_proj", + "model.layers.65.mlp.experts.4.up_proj", + "model.layers.65.mlp.experts.40.down_proj", + "model.layers.65.mlp.experts.40.gate_proj", + "model.layers.65.mlp.experts.40.up_proj", + "model.layers.65.mlp.experts.41.down_proj", + "model.layers.65.mlp.experts.41.gate_proj", + "model.layers.65.mlp.experts.41.up_proj", + "model.layers.65.mlp.experts.42.down_proj", + "model.layers.65.mlp.experts.42.gate_proj", + "model.layers.65.mlp.experts.42.up_proj", + "model.layers.65.mlp.experts.43.down_proj", + "model.layers.65.mlp.experts.43.gate_proj", + "model.layers.65.mlp.experts.43.up_proj", + "model.layers.65.mlp.experts.44.down_proj", + "model.layers.65.mlp.experts.44.gate_proj", + "model.layers.65.mlp.experts.44.up_proj", + "model.layers.65.mlp.experts.45.down_proj", + "model.layers.65.mlp.experts.45.gate_proj", + "model.layers.65.mlp.experts.45.up_proj", + "model.layers.65.mlp.experts.46.down_proj", + "model.layers.65.mlp.experts.46.gate_proj", + "model.layers.65.mlp.experts.46.up_proj", + "model.layers.65.mlp.experts.47.down_proj", + "model.layers.65.mlp.experts.47.gate_proj", + "model.layers.65.mlp.experts.47.up_proj", + "model.layers.65.mlp.experts.48.down_proj", + "model.layers.65.mlp.experts.48.gate_proj", + "model.layers.65.mlp.experts.48.up_proj", + "model.layers.65.mlp.experts.49.down_proj", + "model.layers.65.mlp.experts.49.gate_proj", + "model.layers.65.mlp.experts.49.up_proj", + "model.layers.65.mlp.experts.5.down_proj", + "model.layers.65.mlp.experts.5.gate_proj", + "model.layers.65.mlp.experts.5.up_proj", + "model.layers.65.mlp.experts.50.down_proj", + "model.layers.65.mlp.experts.50.gate_proj", + "model.layers.65.mlp.experts.50.up_proj", + "model.layers.65.mlp.experts.51.down_proj", + "model.layers.65.mlp.experts.51.gate_proj", + "model.layers.65.mlp.experts.51.up_proj", + "model.layers.65.mlp.experts.52.down_proj", + "model.layers.65.mlp.experts.52.gate_proj", + "model.layers.65.mlp.experts.52.up_proj", + "model.layers.65.mlp.experts.53.down_proj", + "model.layers.65.mlp.experts.53.gate_proj", + "model.layers.65.mlp.experts.53.up_proj", + "model.layers.65.mlp.experts.54.down_proj", + "model.layers.65.mlp.experts.54.gate_proj", + "model.layers.65.mlp.experts.54.up_proj", + "model.layers.65.mlp.experts.55.down_proj", + "model.layers.65.mlp.experts.55.gate_proj", + "model.layers.65.mlp.experts.55.up_proj", + "model.layers.65.mlp.experts.56.down_proj", + "model.layers.65.mlp.experts.56.gate_proj", + "model.layers.65.mlp.experts.56.up_proj", + "model.layers.65.mlp.experts.57.down_proj", + "model.layers.65.mlp.experts.57.gate_proj", + "model.layers.65.mlp.experts.57.up_proj", + "model.layers.65.mlp.experts.58.down_proj", + "model.layers.65.mlp.experts.58.gate_proj", + "model.layers.65.mlp.experts.58.up_proj", + "model.layers.65.mlp.experts.59.down_proj", + "model.layers.65.mlp.experts.59.gate_proj", + "model.layers.65.mlp.experts.59.up_proj", + "model.layers.65.mlp.experts.6.down_proj", + "model.layers.65.mlp.experts.6.gate_proj", + "model.layers.65.mlp.experts.6.up_proj", + "model.layers.65.mlp.experts.60.down_proj", + "model.layers.65.mlp.experts.60.gate_proj", + "model.layers.65.mlp.experts.60.up_proj", + "model.layers.65.mlp.experts.61.down_proj", + "model.layers.65.mlp.experts.61.gate_proj", + "model.layers.65.mlp.experts.61.up_proj", + "model.layers.65.mlp.experts.62.down_proj", + "model.layers.65.mlp.experts.62.gate_proj", + "model.layers.65.mlp.experts.62.up_proj", + "model.layers.65.mlp.experts.63.down_proj", + "model.layers.65.mlp.experts.63.gate_proj", + "model.layers.65.mlp.experts.63.up_proj", + "model.layers.65.mlp.experts.64.down_proj", + "model.layers.65.mlp.experts.64.gate_proj", + "model.layers.65.mlp.experts.64.up_proj", + "model.layers.65.mlp.experts.65.down_proj", + "model.layers.65.mlp.experts.65.gate_proj", + "model.layers.65.mlp.experts.65.up_proj", + "model.layers.65.mlp.experts.66.down_proj", + "model.layers.65.mlp.experts.66.gate_proj", + "model.layers.65.mlp.experts.66.up_proj", + "model.layers.65.mlp.experts.67.down_proj", + "model.layers.65.mlp.experts.67.gate_proj", + "model.layers.65.mlp.experts.67.up_proj", + "model.layers.65.mlp.experts.68.down_proj", + "model.layers.65.mlp.experts.68.gate_proj", + "model.layers.65.mlp.experts.68.up_proj", + "model.layers.65.mlp.experts.69.down_proj", + "model.layers.65.mlp.experts.69.gate_proj", + "model.layers.65.mlp.experts.69.up_proj", + "model.layers.65.mlp.experts.7.down_proj", + "model.layers.65.mlp.experts.7.gate_proj", + "model.layers.65.mlp.experts.7.up_proj", + "model.layers.65.mlp.experts.70.down_proj", + "model.layers.65.mlp.experts.70.gate_proj", + "model.layers.65.mlp.experts.70.up_proj", + "model.layers.65.mlp.experts.71.down_proj", + "model.layers.65.mlp.experts.71.gate_proj", + "model.layers.65.mlp.experts.71.up_proj", + "model.layers.65.mlp.experts.72.down_proj", + "model.layers.65.mlp.experts.72.gate_proj", + "model.layers.65.mlp.experts.72.up_proj", + "model.layers.65.mlp.experts.73.down_proj", + "model.layers.65.mlp.experts.73.gate_proj", + "model.layers.65.mlp.experts.73.up_proj", + "model.layers.65.mlp.experts.74.down_proj", + "model.layers.65.mlp.experts.74.gate_proj", + "model.layers.65.mlp.experts.74.up_proj", + "model.layers.65.mlp.experts.75.down_proj", + "model.layers.65.mlp.experts.75.gate_proj", + "model.layers.65.mlp.experts.75.up_proj", + "model.layers.65.mlp.experts.76.down_proj", + "model.layers.65.mlp.experts.76.gate_proj", + "model.layers.65.mlp.experts.76.up_proj", + "model.layers.65.mlp.experts.77.down_proj", + "model.layers.65.mlp.experts.77.gate_proj", + "model.layers.65.mlp.experts.77.up_proj", + "model.layers.65.mlp.experts.78.down_proj", + "model.layers.65.mlp.experts.78.gate_proj", + "model.layers.65.mlp.experts.78.up_proj", + "model.layers.65.mlp.experts.79.down_proj", + "model.layers.65.mlp.experts.79.gate_proj", + "model.layers.65.mlp.experts.79.up_proj", + "model.layers.65.mlp.experts.8.down_proj", + "model.layers.65.mlp.experts.8.gate_proj", + "model.layers.65.mlp.experts.8.up_proj", + "model.layers.65.mlp.experts.80.down_proj", + "model.layers.65.mlp.experts.80.gate_proj", + "model.layers.65.mlp.experts.80.up_proj", + "model.layers.65.mlp.experts.81.down_proj", + "model.layers.65.mlp.experts.81.gate_proj", + "model.layers.65.mlp.experts.81.up_proj", + "model.layers.65.mlp.experts.82.down_proj", + "model.layers.65.mlp.experts.82.gate_proj", + "model.layers.65.mlp.experts.82.up_proj", + "model.layers.65.mlp.experts.83.down_proj", + "model.layers.65.mlp.experts.83.gate_proj", + "model.layers.65.mlp.experts.83.up_proj", + "model.layers.65.mlp.experts.84.down_proj", + "model.layers.65.mlp.experts.84.gate_proj", + "model.layers.65.mlp.experts.84.up_proj", + "model.layers.65.mlp.experts.85.down_proj", + "model.layers.65.mlp.experts.85.gate_proj", + "model.layers.65.mlp.experts.85.up_proj", + "model.layers.65.mlp.experts.86.down_proj", + "model.layers.65.mlp.experts.86.gate_proj", + "model.layers.65.mlp.experts.86.up_proj", + "model.layers.65.mlp.experts.87.down_proj", + "model.layers.65.mlp.experts.87.gate_proj", + "model.layers.65.mlp.experts.87.up_proj", + "model.layers.65.mlp.experts.88.down_proj", + "model.layers.65.mlp.experts.88.gate_proj", + "model.layers.65.mlp.experts.88.up_proj", + "model.layers.65.mlp.experts.89.down_proj", + "model.layers.65.mlp.experts.89.gate_proj", + "model.layers.65.mlp.experts.89.up_proj", + "model.layers.65.mlp.experts.9.down_proj", + "model.layers.65.mlp.experts.9.gate_proj", + "model.layers.65.mlp.experts.9.up_proj", + "model.layers.65.mlp.experts.90.down_proj", + "model.layers.65.mlp.experts.90.gate_proj", + "model.layers.65.mlp.experts.90.up_proj", + "model.layers.65.mlp.experts.91.down_proj", + "model.layers.65.mlp.experts.91.gate_proj", + "model.layers.65.mlp.experts.91.up_proj", + "model.layers.65.mlp.experts.92.down_proj", + "model.layers.65.mlp.experts.92.gate_proj", + "model.layers.65.mlp.experts.92.up_proj", + "model.layers.65.mlp.experts.93.down_proj", + "model.layers.65.mlp.experts.93.gate_proj", + "model.layers.65.mlp.experts.93.up_proj", + "model.layers.65.mlp.experts.94.down_proj", + "model.layers.65.mlp.experts.94.gate_proj", + "model.layers.65.mlp.experts.94.up_proj", + "model.layers.65.mlp.experts.95.down_proj", + "model.layers.65.mlp.experts.95.gate_proj", + "model.layers.65.mlp.experts.95.up_proj", + "model.layers.65.mlp.experts.96.down_proj", + "model.layers.65.mlp.experts.96.gate_proj", + "model.layers.65.mlp.experts.96.up_proj", + "model.layers.65.mlp.experts.97.down_proj", + "model.layers.65.mlp.experts.97.gate_proj", + "model.layers.65.mlp.experts.97.up_proj", + "model.layers.65.mlp.experts.98.down_proj", + "model.layers.65.mlp.experts.98.gate_proj", + "model.layers.65.mlp.experts.98.up_proj", + "model.layers.65.mlp.experts.99.down_proj", + "model.layers.65.mlp.experts.99.gate_proj", + "model.layers.65.mlp.experts.99.up_proj", + "model.layers.66.mlp.experts.0.down_proj", + "model.layers.66.mlp.experts.0.gate_proj", + "model.layers.66.mlp.experts.0.up_proj", + "model.layers.66.mlp.experts.1.down_proj", + "model.layers.66.mlp.experts.1.gate_proj", + "model.layers.66.mlp.experts.1.up_proj", + "model.layers.66.mlp.experts.10.down_proj", + "model.layers.66.mlp.experts.10.gate_proj", + "model.layers.66.mlp.experts.10.up_proj", + "model.layers.66.mlp.experts.100.down_proj", + "model.layers.66.mlp.experts.100.gate_proj", + "model.layers.66.mlp.experts.100.up_proj", + "model.layers.66.mlp.experts.101.down_proj", + "model.layers.66.mlp.experts.101.gate_proj", + "model.layers.66.mlp.experts.101.up_proj", + "model.layers.66.mlp.experts.102.down_proj", + "model.layers.66.mlp.experts.102.gate_proj", + "model.layers.66.mlp.experts.102.up_proj", + "model.layers.66.mlp.experts.103.down_proj", + "model.layers.66.mlp.experts.103.gate_proj", + "model.layers.66.mlp.experts.103.up_proj", + "model.layers.66.mlp.experts.104.down_proj", + "model.layers.66.mlp.experts.104.gate_proj", + "model.layers.66.mlp.experts.104.up_proj", + "model.layers.66.mlp.experts.105.down_proj", + "model.layers.66.mlp.experts.105.gate_proj", + "model.layers.66.mlp.experts.105.up_proj", + "model.layers.66.mlp.experts.106.down_proj", + "model.layers.66.mlp.experts.106.gate_proj", + "model.layers.66.mlp.experts.106.up_proj", + "model.layers.66.mlp.experts.107.down_proj", + "model.layers.66.mlp.experts.107.gate_proj", + "model.layers.66.mlp.experts.107.up_proj", + "model.layers.66.mlp.experts.108.down_proj", + "model.layers.66.mlp.experts.108.gate_proj", + "model.layers.66.mlp.experts.108.up_proj", + "model.layers.66.mlp.experts.109.down_proj", + "model.layers.66.mlp.experts.109.gate_proj", + "model.layers.66.mlp.experts.109.up_proj", + "model.layers.66.mlp.experts.11.down_proj", + "model.layers.66.mlp.experts.11.gate_proj", + "model.layers.66.mlp.experts.11.up_proj", + "model.layers.66.mlp.experts.110.down_proj", + "model.layers.66.mlp.experts.110.gate_proj", + "model.layers.66.mlp.experts.110.up_proj", + "model.layers.66.mlp.experts.111.down_proj", + "model.layers.66.mlp.experts.111.gate_proj", + "model.layers.66.mlp.experts.111.up_proj", + "model.layers.66.mlp.experts.112.down_proj", + "model.layers.66.mlp.experts.112.gate_proj", + "model.layers.66.mlp.experts.112.up_proj", + "model.layers.66.mlp.experts.113.down_proj", + "model.layers.66.mlp.experts.113.gate_proj", + "model.layers.66.mlp.experts.113.up_proj", + "model.layers.66.mlp.experts.114.down_proj", + "model.layers.66.mlp.experts.114.gate_proj", + "model.layers.66.mlp.experts.114.up_proj", + "model.layers.66.mlp.experts.115.down_proj", + "model.layers.66.mlp.experts.115.gate_proj", + "model.layers.66.mlp.experts.115.up_proj", + "model.layers.66.mlp.experts.116.down_proj", + "model.layers.66.mlp.experts.116.gate_proj", + "model.layers.66.mlp.experts.116.up_proj", + "model.layers.66.mlp.experts.117.down_proj", + "model.layers.66.mlp.experts.117.gate_proj", + "model.layers.66.mlp.experts.117.up_proj", + "model.layers.66.mlp.experts.118.down_proj", + "model.layers.66.mlp.experts.118.gate_proj", + "model.layers.66.mlp.experts.118.up_proj", + "model.layers.66.mlp.experts.119.down_proj", + "model.layers.66.mlp.experts.119.gate_proj", + "model.layers.66.mlp.experts.119.up_proj", + "model.layers.66.mlp.experts.12.down_proj", + "model.layers.66.mlp.experts.12.gate_proj", + "model.layers.66.mlp.experts.12.up_proj", + "model.layers.66.mlp.experts.120.down_proj", + "model.layers.66.mlp.experts.120.gate_proj", + "model.layers.66.mlp.experts.120.up_proj", + "model.layers.66.mlp.experts.121.down_proj", + "model.layers.66.mlp.experts.121.gate_proj", + "model.layers.66.mlp.experts.121.up_proj", + "model.layers.66.mlp.experts.122.down_proj", + "model.layers.66.mlp.experts.122.gate_proj", + "model.layers.66.mlp.experts.122.up_proj", + "model.layers.66.mlp.experts.123.down_proj", + "model.layers.66.mlp.experts.123.gate_proj", + "model.layers.66.mlp.experts.123.up_proj", + "model.layers.66.mlp.experts.124.down_proj", + "model.layers.66.mlp.experts.124.gate_proj", + "model.layers.66.mlp.experts.124.up_proj", + "model.layers.66.mlp.experts.125.down_proj", + "model.layers.66.mlp.experts.125.gate_proj", + "model.layers.66.mlp.experts.125.up_proj", + "model.layers.66.mlp.experts.126.down_proj", + "model.layers.66.mlp.experts.126.gate_proj", + "model.layers.66.mlp.experts.126.up_proj", + "model.layers.66.mlp.experts.127.down_proj", + "model.layers.66.mlp.experts.127.gate_proj", + "model.layers.66.mlp.experts.127.up_proj", + "model.layers.66.mlp.experts.128.down_proj", + "model.layers.66.mlp.experts.128.gate_proj", + "model.layers.66.mlp.experts.128.up_proj", + "model.layers.66.mlp.experts.129.down_proj", + "model.layers.66.mlp.experts.129.gate_proj", + "model.layers.66.mlp.experts.129.up_proj", + "model.layers.66.mlp.experts.13.down_proj", + "model.layers.66.mlp.experts.13.gate_proj", + "model.layers.66.mlp.experts.13.up_proj", + "model.layers.66.mlp.experts.130.down_proj", + "model.layers.66.mlp.experts.130.gate_proj", + "model.layers.66.mlp.experts.130.up_proj", + "model.layers.66.mlp.experts.131.down_proj", + "model.layers.66.mlp.experts.131.gate_proj", + "model.layers.66.mlp.experts.131.up_proj", + "model.layers.66.mlp.experts.132.down_proj", + "model.layers.66.mlp.experts.132.gate_proj", + "model.layers.66.mlp.experts.132.up_proj", + "model.layers.66.mlp.experts.133.down_proj", + "model.layers.66.mlp.experts.133.gate_proj", + "model.layers.66.mlp.experts.133.up_proj", + "model.layers.66.mlp.experts.134.down_proj", + "model.layers.66.mlp.experts.134.gate_proj", + "model.layers.66.mlp.experts.134.up_proj", + "model.layers.66.mlp.experts.135.down_proj", + "model.layers.66.mlp.experts.135.gate_proj", + "model.layers.66.mlp.experts.135.up_proj", + "model.layers.66.mlp.experts.136.down_proj", + "model.layers.66.mlp.experts.136.gate_proj", + "model.layers.66.mlp.experts.136.up_proj", + "model.layers.66.mlp.experts.137.down_proj", + "model.layers.66.mlp.experts.137.gate_proj", + "model.layers.66.mlp.experts.137.up_proj", + "model.layers.66.mlp.experts.138.down_proj", + "model.layers.66.mlp.experts.138.gate_proj", + "model.layers.66.mlp.experts.138.up_proj", + "model.layers.66.mlp.experts.139.down_proj", + "model.layers.66.mlp.experts.139.gate_proj", + "model.layers.66.mlp.experts.139.up_proj", + "model.layers.66.mlp.experts.14.down_proj", + "model.layers.66.mlp.experts.14.gate_proj", + "model.layers.66.mlp.experts.14.up_proj", + "model.layers.66.mlp.experts.140.down_proj", + "model.layers.66.mlp.experts.140.gate_proj", + "model.layers.66.mlp.experts.140.up_proj", + "model.layers.66.mlp.experts.141.down_proj", + "model.layers.66.mlp.experts.141.gate_proj", + "model.layers.66.mlp.experts.141.up_proj", + "model.layers.66.mlp.experts.142.down_proj", + "model.layers.66.mlp.experts.142.gate_proj", + "model.layers.66.mlp.experts.142.up_proj", + "model.layers.66.mlp.experts.143.down_proj", + "model.layers.66.mlp.experts.143.gate_proj", + "model.layers.66.mlp.experts.143.up_proj", + "model.layers.66.mlp.experts.144.down_proj", + "model.layers.66.mlp.experts.144.gate_proj", + "model.layers.66.mlp.experts.144.up_proj", + "model.layers.66.mlp.experts.145.down_proj", + "model.layers.66.mlp.experts.145.gate_proj", + "model.layers.66.mlp.experts.145.up_proj", + "model.layers.66.mlp.experts.146.down_proj", + "model.layers.66.mlp.experts.146.gate_proj", + "model.layers.66.mlp.experts.146.up_proj", + "model.layers.66.mlp.experts.147.down_proj", + "model.layers.66.mlp.experts.147.gate_proj", + "model.layers.66.mlp.experts.147.up_proj", + "model.layers.66.mlp.experts.148.down_proj", + "model.layers.66.mlp.experts.148.gate_proj", + "model.layers.66.mlp.experts.148.up_proj", + "model.layers.66.mlp.experts.149.down_proj", + "model.layers.66.mlp.experts.149.gate_proj", + "model.layers.66.mlp.experts.149.up_proj", + "model.layers.66.mlp.experts.15.down_proj", + "model.layers.66.mlp.experts.15.gate_proj", + "model.layers.66.mlp.experts.15.up_proj", + "model.layers.66.mlp.experts.150.down_proj", + "model.layers.66.mlp.experts.150.gate_proj", + "model.layers.66.mlp.experts.150.up_proj", + "model.layers.66.mlp.experts.151.down_proj", + "model.layers.66.mlp.experts.151.gate_proj", + "model.layers.66.mlp.experts.151.up_proj", + "model.layers.66.mlp.experts.152.down_proj", + "model.layers.66.mlp.experts.152.gate_proj", + "model.layers.66.mlp.experts.152.up_proj", + "model.layers.66.mlp.experts.153.down_proj", + "model.layers.66.mlp.experts.153.gate_proj", + "model.layers.66.mlp.experts.153.up_proj", + "model.layers.66.mlp.experts.154.down_proj", + "model.layers.66.mlp.experts.154.gate_proj", + "model.layers.66.mlp.experts.154.up_proj", + "model.layers.66.mlp.experts.155.down_proj", + "model.layers.66.mlp.experts.155.gate_proj", + "model.layers.66.mlp.experts.155.up_proj", + "model.layers.66.mlp.experts.156.down_proj", + "model.layers.66.mlp.experts.156.gate_proj", + "model.layers.66.mlp.experts.156.up_proj", + "model.layers.66.mlp.experts.157.down_proj", + "model.layers.66.mlp.experts.157.gate_proj", + "model.layers.66.mlp.experts.157.up_proj", + "model.layers.66.mlp.experts.158.down_proj", + "model.layers.66.mlp.experts.158.gate_proj", + "model.layers.66.mlp.experts.158.up_proj", + "model.layers.66.mlp.experts.159.down_proj", + "model.layers.66.mlp.experts.159.gate_proj", + "model.layers.66.mlp.experts.159.up_proj", + "model.layers.66.mlp.experts.16.down_proj", + "model.layers.66.mlp.experts.16.gate_proj", + "model.layers.66.mlp.experts.16.up_proj", + "model.layers.66.mlp.experts.160.down_proj", + "model.layers.66.mlp.experts.160.gate_proj", + "model.layers.66.mlp.experts.160.up_proj", + "model.layers.66.mlp.experts.161.down_proj", + "model.layers.66.mlp.experts.161.gate_proj", + "model.layers.66.mlp.experts.161.up_proj", + "model.layers.66.mlp.experts.162.down_proj", + "model.layers.66.mlp.experts.162.gate_proj", + "model.layers.66.mlp.experts.162.up_proj", + "model.layers.66.mlp.experts.163.down_proj", + "model.layers.66.mlp.experts.163.gate_proj", + "model.layers.66.mlp.experts.163.up_proj", + "model.layers.66.mlp.experts.164.down_proj", + "model.layers.66.mlp.experts.164.gate_proj", + "model.layers.66.mlp.experts.164.up_proj", + "model.layers.66.mlp.experts.165.down_proj", + "model.layers.66.mlp.experts.165.gate_proj", + "model.layers.66.mlp.experts.165.up_proj", + "model.layers.66.mlp.experts.166.down_proj", + "model.layers.66.mlp.experts.166.gate_proj", + "model.layers.66.mlp.experts.166.up_proj", + "model.layers.66.mlp.experts.167.down_proj", + "model.layers.66.mlp.experts.167.gate_proj", + "model.layers.66.mlp.experts.167.up_proj", + "model.layers.66.mlp.experts.168.down_proj", + "model.layers.66.mlp.experts.168.gate_proj", + "model.layers.66.mlp.experts.168.up_proj", + "model.layers.66.mlp.experts.169.down_proj", + "model.layers.66.mlp.experts.169.gate_proj", + "model.layers.66.mlp.experts.169.up_proj", + "model.layers.66.mlp.experts.17.down_proj", + "model.layers.66.mlp.experts.17.gate_proj", + "model.layers.66.mlp.experts.17.up_proj", + "model.layers.66.mlp.experts.170.down_proj", + "model.layers.66.mlp.experts.170.gate_proj", + "model.layers.66.mlp.experts.170.up_proj", + "model.layers.66.mlp.experts.171.down_proj", + "model.layers.66.mlp.experts.171.gate_proj", + "model.layers.66.mlp.experts.171.up_proj", + "model.layers.66.mlp.experts.172.down_proj", + "model.layers.66.mlp.experts.172.gate_proj", + "model.layers.66.mlp.experts.172.up_proj", + "model.layers.66.mlp.experts.173.down_proj", + "model.layers.66.mlp.experts.173.gate_proj", + "model.layers.66.mlp.experts.173.up_proj", + "model.layers.66.mlp.experts.174.down_proj", + "model.layers.66.mlp.experts.174.gate_proj", + "model.layers.66.mlp.experts.174.up_proj", + "model.layers.66.mlp.experts.175.down_proj", + "model.layers.66.mlp.experts.175.gate_proj", + "model.layers.66.mlp.experts.175.up_proj", + "model.layers.66.mlp.experts.176.down_proj", + "model.layers.66.mlp.experts.176.gate_proj", + "model.layers.66.mlp.experts.176.up_proj", + "model.layers.66.mlp.experts.177.down_proj", + "model.layers.66.mlp.experts.177.gate_proj", + "model.layers.66.mlp.experts.177.up_proj", + "model.layers.66.mlp.experts.178.down_proj", + "model.layers.66.mlp.experts.178.gate_proj", + "model.layers.66.mlp.experts.178.up_proj", + "model.layers.66.mlp.experts.179.down_proj", + "model.layers.66.mlp.experts.179.gate_proj", + "model.layers.66.mlp.experts.179.up_proj", + "model.layers.66.mlp.experts.18.down_proj", + "model.layers.66.mlp.experts.18.gate_proj", + "model.layers.66.mlp.experts.18.up_proj", + "model.layers.66.mlp.experts.180.down_proj", + "model.layers.66.mlp.experts.180.gate_proj", + "model.layers.66.mlp.experts.180.up_proj", + "model.layers.66.mlp.experts.181.down_proj", + "model.layers.66.mlp.experts.181.gate_proj", + "model.layers.66.mlp.experts.181.up_proj", + "model.layers.66.mlp.experts.182.down_proj", + "model.layers.66.mlp.experts.182.gate_proj", + "model.layers.66.mlp.experts.182.up_proj", + "model.layers.66.mlp.experts.183.down_proj", + "model.layers.66.mlp.experts.183.gate_proj", + "model.layers.66.mlp.experts.183.up_proj", + "model.layers.66.mlp.experts.184.down_proj", + "model.layers.66.mlp.experts.184.gate_proj", + "model.layers.66.mlp.experts.184.up_proj", + "model.layers.66.mlp.experts.185.down_proj", + "model.layers.66.mlp.experts.185.gate_proj", + "model.layers.66.mlp.experts.185.up_proj", + "model.layers.66.mlp.experts.186.down_proj", + "model.layers.66.mlp.experts.186.gate_proj", + "model.layers.66.mlp.experts.186.up_proj", + "model.layers.66.mlp.experts.187.down_proj", + "model.layers.66.mlp.experts.187.gate_proj", + "model.layers.66.mlp.experts.187.up_proj", + "model.layers.66.mlp.experts.188.down_proj", + "model.layers.66.mlp.experts.188.gate_proj", + "model.layers.66.mlp.experts.188.up_proj", + "model.layers.66.mlp.experts.189.down_proj", + "model.layers.66.mlp.experts.189.gate_proj", + "model.layers.66.mlp.experts.189.up_proj", + "model.layers.66.mlp.experts.19.down_proj", + "model.layers.66.mlp.experts.19.gate_proj", + "model.layers.66.mlp.experts.19.up_proj", + "model.layers.66.mlp.experts.190.down_proj", + "model.layers.66.mlp.experts.190.gate_proj", + "model.layers.66.mlp.experts.190.up_proj", + "model.layers.66.mlp.experts.191.down_proj", + "model.layers.66.mlp.experts.191.gate_proj", + "model.layers.66.mlp.experts.191.up_proj", + "model.layers.66.mlp.experts.192.down_proj", + "model.layers.66.mlp.experts.192.gate_proj", + "model.layers.66.mlp.experts.192.up_proj", + "model.layers.66.mlp.experts.193.down_proj", + "model.layers.66.mlp.experts.193.gate_proj", + "model.layers.66.mlp.experts.193.up_proj", + "model.layers.66.mlp.experts.194.down_proj", + "model.layers.66.mlp.experts.194.gate_proj", + "model.layers.66.mlp.experts.194.up_proj", + "model.layers.66.mlp.experts.195.down_proj", + "model.layers.66.mlp.experts.195.gate_proj", + "model.layers.66.mlp.experts.195.up_proj", + "model.layers.66.mlp.experts.196.down_proj", + "model.layers.66.mlp.experts.196.gate_proj", + "model.layers.66.mlp.experts.196.up_proj", + "model.layers.66.mlp.experts.197.down_proj", + "model.layers.66.mlp.experts.197.gate_proj", + "model.layers.66.mlp.experts.197.up_proj", + "model.layers.66.mlp.experts.198.down_proj", + "model.layers.66.mlp.experts.198.gate_proj", + "model.layers.66.mlp.experts.198.up_proj", + "model.layers.66.mlp.experts.199.down_proj", + "model.layers.66.mlp.experts.199.gate_proj", + "model.layers.66.mlp.experts.199.up_proj", + "model.layers.66.mlp.experts.2.down_proj", + "model.layers.66.mlp.experts.2.gate_proj", + "model.layers.66.mlp.experts.2.up_proj", + "model.layers.66.mlp.experts.20.down_proj", + "model.layers.66.mlp.experts.20.gate_proj", + "model.layers.66.mlp.experts.20.up_proj", + "model.layers.66.mlp.experts.200.down_proj", + "model.layers.66.mlp.experts.200.gate_proj", + "model.layers.66.mlp.experts.200.up_proj", + "model.layers.66.mlp.experts.201.down_proj", + "model.layers.66.mlp.experts.201.gate_proj", + "model.layers.66.mlp.experts.201.up_proj", + "model.layers.66.mlp.experts.202.down_proj", + "model.layers.66.mlp.experts.202.gate_proj", + "model.layers.66.mlp.experts.202.up_proj", + "model.layers.66.mlp.experts.203.down_proj", + "model.layers.66.mlp.experts.203.gate_proj", + "model.layers.66.mlp.experts.203.up_proj", + "model.layers.66.mlp.experts.204.down_proj", + "model.layers.66.mlp.experts.204.gate_proj", + "model.layers.66.mlp.experts.204.up_proj", + "model.layers.66.mlp.experts.205.down_proj", + "model.layers.66.mlp.experts.205.gate_proj", + "model.layers.66.mlp.experts.205.up_proj", + "model.layers.66.mlp.experts.206.down_proj", + "model.layers.66.mlp.experts.206.gate_proj", + "model.layers.66.mlp.experts.206.up_proj", + "model.layers.66.mlp.experts.207.down_proj", + "model.layers.66.mlp.experts.207.gate_proj", + "model.layers.66.mlp.experts.207.up_proj", + "model.layers.66.mlp.experts.208.down_proj", + "model.layers.66.mlp.experts.208.gate_proj", + "model.layers.66.mlp.experts.208.up_proj", + "model.layers.66.mlp.experts.209.down_proj", + "model.layers.66.mlp.experts.209.gate_proj", + "model.layers.66.mlp.experts.209.up_proj", + "model.layers.66.mlp.experts.21.down_proj", + "model.layers.66.mlp.experts.21.gate_proj", + "model.layers.66.mlp.experts.21.up_proj", + "model.layers.66.mlp.experts.210.down_proj", + "model.layers.66.mlp.experts.210.gate_proj", + "model.layers.66.mlp.experts.210.up_proj", + "model.layers.66.mlp.experts.211.down_proj", + "model.layers.66.mlp.experts.211.gate_proj", + "model.layers.66.mlp.experts.211.up_proj", + "model.layers.66.mlp.experts.212.down_proj", + "model.layers.66.mlp.experts.212.gate_proj", + "model.layers.66.mlp.experts.212.up_proj", + "model.layers.66.mlp.experts.213.down_proj", + "model.layers.66.mlp.experts.213.gate_proj", + "model.layers.66.mlp.experts.213.up_proj", + "model.layers.66.mlp.experts.214.down_proj", + "model.layers.66.mlp.experts.214.gate_proj", + "model.layers.66.mlp.experts.214.up_proj", + "model.layers.66.mlp.experts.215.down_proj", + "model.layers.66.mlp.experts.215.gate_proj", + "model.layers.66.mlp.experts.215.up_proj", + "model.layers.66.mlp.experts.216.down_proj", + "model.layers.66.mlp.experts.216.gate_proj", + "model.layers.66.mlp.experts.216.up_proj", + "model.layers.66.mlp.experts.217.down_proj", + "model.layers.66.mlp.experts.217.gate_proj", + "model.layers.66.mlp.experts.217.up_proj", + "model.layers.66.mlp.experts.218.down_proj", + "model.layers.66.mlp.experts.218.gate_proj", + "model.layers.66.mlp.experts.218.up_proj", + "model.layers.66.mlp.experts.219.down_proj", + "model.layers.66.mlp.experts.219.gate_proj", + "model.layers.66.mlp.experts.219.up_proj", + "model.layers.66.mlp.experts.22.down_proj", + "model.layers.66.mlp.experts.22.gate_proj", + "model.layers.66.mlp.experts.22.up_proj", + "model.layers.66.mlp.experts.220.down_proj", + "model.layers.66.mlp.experts.220.gate_proj", + "model.layers.66.mlp.experts.220.up_proj", + "model.layers.66.mlp.experts.221.down_proj", + "model.layers.66.mlp.experts.221.gate_proj", + "model.layers.66.mlp.experts.221.up_proj", + "model.layers.66.mlp.experts.222.down_proj", + "model.layers.66.mlp.experts.222.gate_proj", + "model.layers.66.mlp.experts.222.up_proj", + "model.layers.66.mlp.experts.223.down_proj", + "model.layers.66.mlp.experts.223.gate_proj", + "model.layers.66.mlp.experts.223.up_proj", + "model.layers.66.mlp.experts.224.down_proj", + "model.layers.66.mlp.experts.224.gate_proj", + "model.layers.66.mlp.experts.224.up_proj", + "model.layers.66.mlp.experts.225.down_proj", + "model.layers.66.mlp.experts.225.gate_proj", + "model.layers.66.mlp.experts.225.up_proj", + "model.layers.66.mlp.experts.226.down_proj", + "model.layers.66.mlp.experts.226.gate_proj", + "model.layers.66.mlp.experts.226.up_proj", + "model.layers.66.mlp.experts.227.down_proj", + "model.layers.66.mlp.experts.227.gate_proj", + "model.layers.66.mlp.experts.227.up_proj", + "model.layers.66.mlp.experts.228.down_proj", + "model.layers.66.mlp.experts.228.gate_proj", + "model.layers.66.mlp.experts.228.up_proj", + "model.layers.66.mlp.experts.229.down_proj", + "model.layers.66.mlp.experts.229.gate_proj", + "model.layers.66.mlp.experts.229.up_proj", + "model.layers.66.mlp.experts.23.down_proj", + "model.layers.66.mlp.experts.23.gate_proj", + "model.layers.66.mlp.experts.23.up_proj", + "model.layers.66.mlp.experts.230.down_proj", + "model.layers.66.mlp.experts.230.gate_proj", + "model.layers.66.mlp.experts.230.up_proj", + "model.layers.66.mlp.experts.231.down_proj", + "model.layers.66.mlp.experts.231.gate_proj", + "model.layers.66.mlp.experts.231.up_proj", + "model.layers.66.mlp.experts.232.down_proj", + "model.layers.66.mlp.experts.232.gate_proj", + "model.layers.66.mlp.experts.232.up_proj", + "model.layers.66.mlp.experts.233.down_proj", + "model.layers.66.mlp.experts.233.gate_proj", + "model.layers.66.mlp.experts.233.up_proj", + "model.layers.66.mlp.experts.234.down_proj", + "model.layers.66.mlp.experts.234.gate_proj", + "model.layers.66.mlp.experts.234.up_proj", + "model.layers.66.mlp.experts.235.down_proj", + "model.layers.66.mlp.experts.235.gate_proj", + "model.layers.66.mlp.experts.235.up_proj", + "model.layers.66.mlp.experts.236.down_proj", + "model.layers.66.mlp.experts.236.gate_proj", + "model.layers.66.mlp.experts.236.up_proj", + "model.layers.66.mlp.experts.237.down_proj", + "model.layers.66.mlp.experts.237.gate_proj", + "model.layers.66.mlp.experts.237.up_proj", + "model.layers.66.mlp.experts.238.down_proj", + "model.layers.66.mlp.experts.238.gate_proj", + "model.layers.66.mlp.experts.238.up_proj", + "model.layers.66.mlp.experts.239.down_proj", + "model.layers.66.mlp.experts.239.gate_proj", + "model.layers.66.mlp.experts.239.up_proj", + "model.layers.66.mlp.experts.24.down_proj", + "model.layers.66.mlp.experts.24.gate_proj", + "model.layers.66.mlp.experts.24.up_proj", + "model.layers.66.mlp.experts.240.down_proj", + "model.layers.66.mlp.experts.240.gate_proj", + "model.layers.66.mlp.experts.240.up_proj", + "model.layers.66.mlp.experts.241.down_proj", + "model.layers.66.mlp.experts.241.gate_proj", + "model.layers.66.mlp.experts.241.up_proj", + "model.layers.66.mlp.experts.242.down_proj", + "model.layers.66.mlp.experts.242.gate_proj", + "model.layers.66.mlp.experts.242.up_proj", + "model.layers.66.mlp.experts.243.down_proj", + "model.layers.66.mlp.experts.243.gate_proj", + "model.layers.66.mlp.experts.243.up_proj", + "model.layers.66.mlp.experts.244.down_proj", + "model.layers.66.mlp.experts.244.gate_proj", + "model.layers.66.mlp.experts.244.up_proj", + "model.layers.66.mlp.experts.245.down_proj", + "model.layers.66.mlp.experts.245.gate_proj", + "model.layers.66.mlp.experts.245.up_proj", + "model.layers.66.mlp.experts.246.down_proj", + "model.layers.66.mlp.experts.246.gate_proj", + "model.layers.66.mlp.experts.246.up_proj", + "model.layers.66.mlp.experts.247.down_proj", + "model.layers.66.mlp.experts.247.gate_proj", + "model.layers.66.mlp.experts.247.up_proj", + "model.layers.66.mlp.experts.248.down_proj", + "model.layers.66.mlp.experts.248.gate_proj", + "model.layers.66.mlp.experts.248.up_proj", + "model.layers.66.mlp.experts.249.down_proj", + "model.layers.66.mlp.experts.249.gate_proj", + "model.layers.66.mlp.experts.249.up_proj", + "model.layers.66.mlp.experts.25.down_proj", + "model.layers.66.mlp.experts.25.gate_proj", + "model.layers.66.mlp.experts.25.up_proj", + "model.layers.66.mlp.experts.250.down_proj", + "model.layers.66.mlp.experts.250.gate_proj", + "model.layers.66.mlp.experts.250.up_proj", + "model.layers.66.mlp.experts.251.down_proj", + "model.layers.66.mlp.experts.251.gate_proj", + "model.layers.66.mlp.experts.251.up_proj", + "model.layers.66.mlp.experts.252.down_proj", + "model.layers.66.mlp.experts.252.gate_proj", + "model.layers.66.mlp.experts.252.up_proj", + "model.layers.66.mlp.experts.253.down_proj", + "model.layers.66.mlp.experts.253.gate_proj", + "model.layers.66.mlp.experts.253.up_proj", + "model.layers.66.mlp.experts.254.down_proj", + "model.layers.66.mlp.experts.254.gate_proj", + "model.layers.66.mlp.experts.254.up_proj", + "model.layers.66.mlp.experts.255.down_proj", + "model.layers.66.mlp.experts.255.gate_proj", + "model.layers.66.mlp.experts.255.up_proj", + "model.layers.66.mlp.experts.26.down_proj", + "model.layers.66.mlp.experts.26.gate_proj", + "model.layers.66.mlp.experts.26.up_proj", + "model.layers.66.mlp.experts.27.down_proj", + "model.layers.66.mlp.experts.27.gate_proj", + "model.layers.66.mlp.experts.27.up_proj", + "model.layers.66.mlp.experts.28.down_proj", + "model.layers.66.mlp.experts.28.gate_proj", + "model.layers.66.mlp.experts.28.up_proj", + "model.layers.66.mlp.experts.29.down_proj", + "model.layers.66.mlp.experts.29.gate_proj", + "model.layers.66.mlp.experts.29.up_proj", + "model.layers.66.mlp.experts.3.down_proj", + "model.layers.66.mlp.experts.3.gate_proj", + "model.layers.66.mlp.experts.3.up_proj", + "model.layers.66.mlp.experts.30.down_proj", + "model.layers.66.mlp.experts.30.gate_proj", + "model.layers.66.mlp.experts.30.up_proj", + "model.layers.66.mlp.experts.31.down_proj", + "model.layers.66.mlp.experts.31.gate_proj", + "model.layers.66.mlp.experts.31.up_proj", + "model.layers.66.mlp.experts.32.down_proj", + "model.layers.66.mlp.experts.32.gate_proj", + "model.layers.66.mlp.experts.32.up_proj", + "model.layers.66.mlp.experts.33.down_proj", + "model.layers.66.mlp.experts.33.gate_proj", + "model.layers.66.mlp.experts.33.up_proj", + "model.layers.66.mlp.experts.34.down_proj", + "model.layers.66.mlp.experts.34.gate_proj", + "model.layers.66.mlp.experts.34.up_proj", + "model.layers.66.mlp.experts.35.down_proj", + "model.layers.66.mlp.experts.35.gate_proj", + "model.layers.66.mlp.experts.35.up_proj", + "model.layers.66.mlp.experts.36.down_proj", + "model.layers.66.mlp.experts.36.gate_proj", + "model.layers.66.mlp.experts.36.up_proj", + "model.layers.66.mlp.experts.37.down_proj", + "model.layers.66.mlp.experts.37.gate_proj", + "model.layers.66.mlp.experts.37.up_proj", + "model.layers.66.mlp.experts.38.down_proj", + "model.layers.66.mlp.experts.38.gate_proj", + "model.layers.66.mlp.experts.38.up_proj", + "model.layers.66.mlp.experts.39.down_proj", + "model.layers.66.mlp.experts.39.gate_proj", + "model.layers.66.mlp.experts.39.up_proj", + "model.layers.66.mlp.experts.4.down_proj", + "model.layers.66.mlp.experts.4.gate_proj", + "model.layers.66.mlp.experts.4.up_proj", + "model.layers.66.mlp.experts.40.down_proj", + "model.layers.66.mlp.experts.40.gate_proj", + "model.layers.66.mlp.experts.40.up_proj", + "model.layers.66.mlp.experts.41.down_proj", + "model.layers.66.mlp.experts.41.gate_proj", + "model.layers.66.mlp.experts.41.up_proj", + "model.layers.66.mlp.experts.42.down_proj", + "model.layers.66.mlp.experts.42.gate_proj", + "model.layers.66.mlp.experts.42.up_proj", + "model.layers.66.mlp.experts.43.down_proj", + "model.layers.66.mlp.experts.43.gate_proj", + "model.layers.66.mlp.experts.43.up_proj", + "model.layers.66.mlp.experts.44.down_proj", + "model.layers.66.mlp.experts.44.gate_proj", + "model.layers.66.mlp.experts.44.up_proj", + "model.layers.66.mlp.experts.45.down_proj", + "model.layers.66.mlp.experts.45.gate_proj", + "model.layers.66.mlp.experts.45.up_proj", + "model.layers.66.mlp.experts.46.down_proj", + "model.layers.66.mlp.experts.46.gate_proj", + "model.layers.66.mlp.experts.46.up_proj", + "model.layers.66.mlp.experts.47.down_proj", + "model.layers.66.mlp.experts.47.gate_proj", + "model.layers.66.mlp.experts.47.up_proj", + "model.layers.66.mlp.experts.48.down_proj", + "model.layers.66.mlp.experts.48.gate_proj", + "model.layers.66.mlp.experts.48.up_proj", + "model.layers.66.mlp.experts.49.down_proj", + "model.layers.66.mlp.experts.49.gate_proj", + "model.layers.66.mlp.experts.49.up_proj", + "model.layers.66.mlp.experts.5.down_proj", + "model.layers.66.mlp.experts.5.gate_proj", + "model.layers.66.mlp.experts.5.up_proj", + "model.layers.66.mlp.experts.50.down_proj", + "model.layers.66.mlp.experts.50.gate_proj", + "model.layers.66.mlp.experts.50.up_proj", + "model.layers.66.mlp.experts.51.down_proj", + "model.layers.66.mlp.experts.51.gate_proj", + "model.layers.66.mlp.experts.51.up_proj", + "model.layers.66.mlp.experts.52.down_proj", + "model.layers.66.mlp.experts.52.gate_proj", + "model.layers.66.mlp.experts.52.up_proj", + "model.layers.66.mlp.experts.53.down_proj", + "model.layers.66.mlp.experts.53.gate_proj", + "model.layers.66.mlp.experts.53.up_proj", + "model.layers.66.mlp.experts.54.down_proj", + "model.layers.66.mlp.experts.54.gate_proj", + "model.layers.66.mlp.experts.54.up_proj", + "model.layers.66.mlp.experts.55.down_proj", + "model.layers.66.mlp.experts.55.gate_proj", + "model.layers.66.mlp.experts.55.up_proj", + "model.layers.66.mlp.experts.56.down_proj", + "model.layers.66.mlp.experts.56.gate_proj", + "model.layers.66.mlp.experts.56.up_proj", + "model.layers.66.mlp.experts.57.down_proj", + "model.layers.66.mlp.experts.57.gate_proj", + "model.layers.66.mlp.experts.57.up_proj", + "model.layers.66.mlp.experts.58.down_proj", + "model.layers.66.mlp.experts.58.gate_proj", + "model.layers.66.mlp.experts.58.up_proj", + "model.layers.66.mlp.experts.59.down_proj", + "model.layers.66.mlp.experts.59.gate_proj", + "model.layers.66.mlp.experts.59.up_proj", + "model.layers.66.mlp.experts.6.down_proj", + "model.layers.66.mlp.experts.6.gate_proj", + "model.layers.66.mlp.experts.6.up_proj", + "model.layers.66.mlp.experts.60.down_proj", + "model.layers.66.mlp.experts.60.gate_proj", + "model.layers.66.mlp.experts.60.up_proj", + "model.layers.66.mlp.experts.61.down_proj", + "model.layers.66.mlp.experts.61.gate_proj", + "model.layers.66.mlp.experts.61.up_proj", + "model.layers.66.mlp.experts.62.down_proj", + "model.layers.66.mlp.experts.62.gate_proj", + "model.layers.66.mlp.experts.62.up_proj", + "model.layers.66.mlp.experts.63.down_proj", + "model.layers.66.mlp.experts.63.gate_proj", + "model.layers.66.mlp.experts.63.up_proj", + "model.layers.66.mlp.experts.64.down_proj", + "model.layers.66.mlp.experts.64.gate_proj", + "model.layers.66.mlp.experts.64.up_proj", + "model.layers.66.mlp.experts.65.down_proj", + "model.layers.66.mlp.experts.65.gate_proj", + "model.layers.66.mlp.experts.65.up_proj", + "model.layers.66.mlp.experts.66.down_proj", + "model.layers.66.mlp.experts.66.gate_proj", + "model.layers.66.mlp.experts.66.up_proj", + "model.layers.66.mlp.experts.67.down_proj", + "model.layers.66.mlp.experts.67.gate_proj", + "model.layers.66.mlp.experts.67.up_proj", + "model.layers.66.mlp.experts.68.down_proj", + "model.layers.66.mlp.experts.68.gate_proj", + "model.layers.66.mlp.experts.68.up_proj", + "model.layers.66.mlp.experts.69.down_proj", + "model.layers.66.mlp.experts.69.gate_proj", + "model.layers.66.mlp.experts.69.up_proj", + "model.layers.66.mlp.experts.7.down_proj", + "model.layers.66.mlp.experts.7.gate_proj", + "model.layers.66.mlp.experts.7.up_proj", + "model.layers.66.mlp.experts.70.down_proj", + "model.layers.66.mlp.experts.70.gate_proj", + "model.layers.66.mlp.experts.70.up_proj", + "model.layers.66.mlp.experts.71.down_proj", + "model.layers.66.mlp.experts.71.gate_proj", + "model.layers.66.mlp.experts.71.up_proj", + "model.layers.66.mlp.experts.72.down_proj", + "model.layers.66.mlp.experts.72.gate_proj", + "model.layers.66.mlp.experts.72.up_proj", + "model.layers.66.mlp.experts.73.down_proj", + "model.layers.66.mlp.experts.73.gate_proj", + "model.layers.66.mlp.experts.73.up_proj", + "model.layers.66.mlp.experts.74.down_proj", + "model.layers.66.mlp.experts.74.gate_proj", + "model.layers.66.mlp.experts.74.up_proj", + "model.layers.66.mlp.experts.75.down_proj", + "model.layers.66.mlp.experts.75.gate_proj", + "model.layers.66.mlp.experts.75.up_proj", + "model.layers.66.mlp.experts.76.down_proj", + "model.layers.66.mlp.experts.76.gate_proj", + "model.layers.66.mlp.experts.76.up_proj", + "model.layers.66.mlp.experts.77.down_proj", + "model.layers.66.mlp.experts.77.gate_proj", + "model.layers.66.mlp.experts.77.up_proj", + "model.layers.66.mlp.experts.78.down_proj", + "model.layers.66.mlp.experts.78.gate_proj", + "model.layers.66.mlp.experts.78.up_proj", + "model.layers.66.mlp.experts.79.down_proj", + "model.layers.66.mlp.experts.79.gate_proj", + "model.layers.66.mlp.experts.79.up_proj", + "model.layers.66.mlp.experts.8.down_proj", + "model.layers.66.mlp.experts.8.gate_proj", + "model.layers.66.mlp.experts.8.up_proj", + "model.layers.66.mlp.experts.80.down_proj", + "model.layers.66.mlp.experts.80.gate_proj", + "model.layers.66.mlp.experts.80.up_proj", + "model.layers.66.mlp.experts.81.down_proj", + "model.layers.66.mlp.experts.81.gate_proj", + "model.layers.66.mlp.experts.81.up_proj", + "model.layers.66.mlp.experts.82.down_proj", + "model.layers.66.mlp.experts.82.gate_proj", + "model.layers.66.mlp.experts.82.up_proj", + "model.layers.66.mlp.experts.83.down_proj", + "model.layers.66.mlp.experts.83.gate_proj", + "model.layers.66.mlp.experts.83.up_proj", + "model.layers.66.mlp.experts.84.down_proj", + "model.layers.66.mlp.experts.84.gate_proj", + "model.layers.66.mlp.experts.84.up_proj", + "model.layers.66.mlp.experts.85.down_proj", + "model.layers.66.mlp.experts.85.gate_proj", + "model.layers.66.mlp.experts.85.up_proj", + "model.layers.66.mlp.experts.86.down_proj", + "model.layers.66.mlp.experts.86.gate_proj", + "model.layers.66.mlp.experts.86.up_proj", + "model.layers.66.mlp.experts.87.down_proj", + "model.layers.66.mlp.experts.87.gate_proj", + "model.layers.66.mlp.experts.87.up_proj", + "model.layers.66.mlp.experts.88.down_proj", + "model.layers.66.mlp.experts.88.gate_proj", + "model.layers.66.mlp.experts.88.up_proj", + "model.layers.66.mlp.experts.89.down_proj", + "model.layers.66.mlp.experts.89.gate_proj", + "model.layers.66.mlp.experts.89.up_proj", + "model.layers.66.mlp.experts.9.down_proj", + "model.layers.66.mlp.experts.9.gate_proj", + "model.layers.66.mlp.experts.9.up_proj", + "model.layers.66.mlp.experts.90.down_proj", + "model.layers.66.mlp.experts.90.gate_proj", + "model.layers.66.mlp.experts.90.up_proj", + "model.layers.66.mlp.experts.91.down_proj", + "model.layers.66.mlp.experts.91.gate_proj", + "model.layers.66.mlp.experts.91.up_proj", + "model.layers.66.mlp.experts.92.down_proj", + "model.layers.66.mlp.experts.92.gate_proj", + "model.layers.66.mlp.experts.92.up_proj", + "model.layers.66.mlp.experts.93.down_proj", + "model.layers.66.mlp.experts.93.gate_proj", + "model.layers.66.mlp.experts.93.up_proj", + "model.layers.66.mlp.experts.94.down_proj", + "model.layers.66.mlp.experts.94.gate_proj", + "model.layers.66.mlp.experts.94.up_proj", + "model.layers.66.mlp.experts.95.down_proj", + "model.layers.66.mlp.experts.95.gate_proj", + "model.layers.66.mlp.experts.95.up_proj", + "model.layers.66.mlp.experts.96.down_proj", + "model.layers.66.mlp.experts.96.gate_proj", + "model.layers.66.mlp.experts.96.up_proj", + "model.layers.66.mlp.experts.97.down_proj", + "model.layers.66.mlp.experts.97.gate_proj", + "model.layers.66.mlp.experts.97.up_proj", + "model.layers.66.mlp.experts.98.down_proj", + "model.layers.66.mlp.experts.98.gate_proj", + "model.layers.66.mlp.experts.98.up_proj", + "model.layers.66.mlp.experts.99.down_proj", + "model.layers.66.mlp.experts.99.gate_proj", + "model.layers.66.mlp.experts.99.up_proj", + "model.layers.67.mlp.experts.0.down_proj", + "model.layers.67.mlp.experts.0.gate_proj", + "model.layers.67.mlp.experts.0.up_proj", + "model.layers.67.mlp.experts.1.down_proj", + "model.layers.67.mlp.experts.1.gate_proj", + "model.layers.67.mlp.experts.1.up_proj", + "model.layers.67.mlp.experts.10.down_proj", + "model.layers.67.mlp.experts.10.gate_proj", + "model.layers.67.mlp.experts.10.up_proj", + "model.layers.67.mlp.experts.100.down_proj", + "model.layers.67.mlp.experts.100.gate_proj", + "model.layers.67.mlp.experts.100.up_proj", + "model.layers.67.mlp.experts.101.down_proj", + "model.layers.67.mlp.experts.101.gate_proj", + "model.layers.67.mlp.experts.101.up_proj", + "model.layers.67.mlp.experts.102.down_proj", + "model.layers.67.mlp.experts.102.gate_proj", + "model.layers.67.mlp.experts.102.up_proj", + "model.layers.67.mlp.experts.103.down_proj", + "model.layers.67.mlp.experts.103.gate_proj", + "model.layers.67.mlp.experts.103.up_proj", + "model.layers.67.mlp.experts.104.down_proj", + "model.layers.67.mlp.experts.104.gate_proj", + "model.layers.67.mlp.experts.104.up_proj", + "model.layers.67.mlp.experts.105.down_proj", + "model.layers.67.mlp.experts.105.gate_proj", + "model.layers.67.mlp.experts.105.up_proj", + "model.layers.67.mlp.experts.106.down_proj", + "model.layers.67.mlp.experts.106.gate_proj", + "model.layers.67.mlp.experts.106.up_proj", + "model.layers.67.mlp.experts.107.down_proj", + "model.layers.67.mlp.experts.107.gate_proj", + "model.layers.67.mlp.experts.107.up_proj", + "model.layers.67.mlp.experts.108.down_proj", + "model.layers.67.mlp.experts.108.gate_proj", + "model.layers.67.mlp.experts.108.up_proj", + "model.layers.67.mlp.experts.109.down_proj", + "model.layers.67.mlp.experts.109.gate_proj", + "model.layers.67.mlp.experts.109.up_proj", + "model.layers.67.mlp.experts.11.down_proj", + "model.layers.67.mlp.experts.11.gate_proj", + "model.layers.67.mlp.experts.11.up_proj", + "model.layers.67.mlp.experts.110.down_proj", + "model.layers.67.mlp.experts.110.gate_proj", + "model.layers.67.mlp.experts.110.up_proj", + "model.layers.67.mlp.experts.111.down_proj", + "model.layers.67.mlp.experts.111.gate_proj", + "model.layers.67.mlp.experts.111.up_proj", + "model.layers.67.mlp.experts.112.down_proj", + "model.layers.67.mlp.experts.112.gate_proj", + "model.layers.67.mlp.experts.112.up_proj", + "model.layers.67.mlp.experts.113.down_proj", + "model.layers.67.mlp.experts.113.gate_proj", + "model.layers.67.mlp.experts.113.up_proj", + "model.layers.67.mlp.experts.114.down_proj", + "model.layers.67.mlp.experts.114.gate_proj", + "model.layers.67.mlp.experts.114.up_proj", + "model.layers.67.mlp.experts.115.down_proj", + "model.layers.67.mlp.experts.115.gate_proj", + "model.layers.67.mlp.experts.115.up_proj", + "model.layers.67.mlp.experts.116.down_proj", + "model.layers.67.mlp.experts.116.gate_proj", + "model.layers.67.mlp.experts.116.up_proj", + "model.layers.67.mlp.experts.117.down_proj", + "model.layers.67.mlp.experts.117.gate_proj", + "model.layers.67.mlp.experts.117.up_proj", + "model.layers.67.mlp.experts.118.down_proj", + "model.layers.67.mlp.experts.118.gate_proj", + "model.layers.67.mlp.experts.118.up_proj", + "model.layers.67.mlp.experts.119.down_proj", + "model.layers.67.mlp.experts.119.gate_proj", + "model.layers.67.mlp.experts.119.up_proj", + "model.layers.67.mlp.experts.12.down_proj", + "model.layers.67.mlp.experts.12.gate_proj", + "model.layers.67.mlp.experts.12.up_proj", + "model.layers.67.mlp.experts.120.down_proj", + "model.layers.67.mlp.experts.120.gate_proj", + "model.layers.67.mlp.experts.120.up_proj", + "model.layers.67.mlp.experts.121.down_proj", + "model.layers.67.mlp.experts.121.gate_proj", + "model.layers.67.mlp.experts.121.up_proj", + "model.layers.67.mlp.experts.122.down_proj", + "model.layers.67.mlp.experts.122.gate_proj", + "model.layers.67.mlp.experts.122.up_proj", + "model.layers.67.mlp.experts.123.down_proj", + "model.layers.67.mlp.experts.123.gate_proj", + "model.layers.67.mlp.experts.123.up_proj", + "model.layers.67.mlp.experts.124.down_proj", + "model.layers.67.mlp.experts.124.gate_proj", + "model.layers.67.mlp.experts.124.up_proj", + "model.layers.67.mlp.experts.125.down_proj", + "model.layers.67.mlp.experts.125.gate_proj", + "model.layers.67.mlp.experts.125.up_proj", + "model.layers.67.mlp.experts.126.down_proj", + "model.layers.67.mlp.experts.126.gate_proj", + "model.layers.67.mlp.experts.126.up_proj", + "model.layers.67.mlp.experts.127.down_proj", + "model.layers.67.mlp.experts.127.gate_proj", + "model.layers.67.mlp.experts.127.up_proj", + "model.layers.67.mlp.experts.128.down_proj", + "model.layers.67.mlp.experts.128.gate_proj", + "model.layers.67.mlp.experts.128.up_proj", + "model.layers.67.mlp.experts.129.down_proj", + "model.layers.67.mlp.experts.129.gate_proj", + "model.layers.67.mlp.experts.129.up_proj", + "model.layers.67.mlp.experts.13.down_proj", + "model.layers.67.mlp.experts.13.gate_proj", + "model.layers.67.mlp.experts.13.up_proj", + "model.layers.67.mlp.experts.130.down_proj", + "model.layers.67.mlp.experts.130.gate_proj", + "model.layers.67.mlp.experts.130.up_proj", + "model.layers.67.mlp.experts.131.down_proj", + "model.layers.67.mlp.experts.131.gate_proj", + "model.layers.67.mlp.experts.131.up_proj", + "model.layers.67.mlp.experts.132.down_proj", + "model.layers.67.mlp.experts.132.gate_proj", + "model.layers.67.mlp.experts.132.up_proj", + "model.layers.67.mlp.experts.133.down_proj", + "model.layers.67.mlp.experts.133.gate_proj", + "model.layers.67.mlp.experts.133.up_proj", + "model.layers.67.mlp.experts.134.down_proj", + "model.layers.67.mlp.experts.134.gate_proj", + "model.layers.67.mlp.experts.134.up_proj", + "model.layers.67.mlp.experts.135.down_proj", + "model.layers.67.mlp.experts.135.gate_proj", + "model.layers.67.mlp.experts.135.up_proj", + "model.layers.67.mlp.experts.136.down_proj", + "model.layers.67.mlp.experts.136.gate_proj", + "model.layers.67.mlp.experts.136.up_proj", + "model.layers.67.mlp.experts.137.down_proj", + "model.layers.67.mlp.experts.137.gate_proj", + "model.layers.67.mlp.experts.137.up_proj", + "model.layers.67.mlp.experts.138.down_proj", + "model.layers.67.mlp.experts.138.gate_proj", + "model.layers.67.mlp.experts.138.up_proj", + "model.layers.67.mlp.experts.139.down_proj", + "model.layers.67.mlp.experts.139.gate_proj", + "model.layers.67.mlp.experts.139.up_proj", + "model.layers.67.mlp.experts.14.down_proj", + "model.layers.67.mlp.experts.14.gate_proj", + "model.layers.67.mlp.experts.14.up_proj", + "model.layers.67.mlp.experts.140.down_proj", + "model.layers.67.mlp.experts.140.gate_proj", + "model.layers.67.mlp.experts.140.up_proj", + "model.layers.67.mlp.experts.141.down_proj", + "model.layers.67.mlp.experts.141.gate_proj", + "model.layers.67.mlp.experts.141.up_proj", + "model.layers.67.mlp.experts.142.down_proj", + "model.layers.67.mlp.experts.142.gate_proj", + "model.layers.67.mlp.experts.142.up_proj", + "model.layers.67.mlp.experts.143.down_proj", + "model.layers.67.mlp.experts.143.gate_proj", + "model.layers.67.mlp.experts.143.up_proj", + "model.layers.67.mlp.experts.144.down_proj", + "model.layers.67.mlp.experts.144.gate_proj", + "model.layers.67.mlp.experts.144.up_proj", + "model.layers.67.mlp.experts.145.down_proj", + "model.layers.67.mlp.experts.145.gate_proj", + "model.layers.67.mlp.experts.145.up_proj", + "model.layers.67.mlp.experts.146.down_proj", + "model.layers.67.mlp.experts.146.gate_proj", + "model.layers.67.mlp.experts.146.up_proj", + "model.layers.67.mlp.experts.147.down_proj", + "model.layers.67.mlp.experts.147.gate_proj", + "model.layers.67.mlp.experts.147.up_proj", + "model.layers.67.mlp.experts.148.down_proj", + "model.layers.67.mlp.experts.148.gate_proj", + "model.layers.67.mlp.experts.148.up_proj", + "model.layers.67.mlp.experts.149.down_proj", + "model.layers.67.mlp.experts.149.gate_proj", + "model.layers.67.mlp.experts.149.up_proj", + "model.layers.67.mlp.experts.15.down_proj", + "model.layers.67.mlp.experts.15.gate_proj", + "model.layers.67.mlp.experts.15.up_proj", + "model.layers.67.mlp.experts.150.down_proj", + "model.layers.67.mlp.experts.150.gate_proj", + "model.layers.67.mlp.experts.150.up_proj", + "model.layers.67.mlp.experts.151.down_proj", + "model.layers.67.mlp.experts.151.gate_proj", + "model.layers.67.mlp.experts.151.up_proj", + "model.layers.67.mlp.experts.152.down_proj", + "model.layers.67.mlp.experts.152.gate_proj", + "model.layers.67.mlp.experts.152.up_proj", + "model.layers.67.mlp.experts.153.down_proj", + "model.layers.67.mlp.experts.153.gate_proj", + "model.layers.67.mlp.experts.153.up_proj", + "model.layers.67.mlp.experts.154.down_proj", + "model.layers.67.mlp.experts.154.gate_proj", + "model.layers.67.mlp.experts.154.up_proj", + "model.layers.67.mlp.experts.155.down_proj", + "model.layers.67.mlp.experts.155.gate_proj", + "model.layers.67.mlp.experts.155.up_proj", + "model.layers.67.mlp.experts.156.down_proj", + "model.layers.67.mlp.experts.156.gate_proj", + "model.layers.67.mlp.experts.156.up_proj", + "model.layers.67.mlp.experts.157.down_proj", + "model.layers.67.mlp.experts.157.gate_proj", + "model.layers.67.mlp.experts.157.up_proj", + "model.layers.67.mlp.experts.158.down_proj", + "model.layers.67.mlp.experts.158.gate_proj", + "model.layers.67.mlp.experts.158.up_proj", + "model.layers.67.mlp.experts.159.down_proj", + "model.layers.67.mlp.experts.159.gate_proj", + "model.layers.67.mlp.experts.159.up_proj", + "model.layers.67.mlp.experts.16.down_proj", + "model.layers.67.mlp.experts.16.gate_proj", + "model.layers.67.mlp.experts.16.up_proj", + "model.layers.67.mlp.experts.160.down_proj", + "model.layers.67.mlp.experts.160.gate_proj", + "model.layers.67.mlp.experts.160.up_proj", + "model.layers.67.mlp.experts.161.down_proj", + "model.layers.67.mlp.experts.161.gate_proj", + "model.layers.67.mlp.experts.161.up_proj", + "model.layers.67.mlp.experts.162.down_proj", + "model.layers.67.mlp.experts.162.gate_proj", + "model.layers.67.mlp.experts.162.up_proj", + "model.layers.67.mlp.experts.163.down_proj", + "model.layers.67.mlp.experts.163.gate_proj", + "model.layers.67.mlp.experts.163.up_proj", + "model.layers.67.mlp.experts.164.down_proj", + "model.layers.67.mlp.experts.164.gate_proj", + "model.layers.67.mlp.experts.164.up_proj", + "model.layers.67.mlp.experts.165.down_proj", + "model.layers.67.mlp.experts.165.gate_proj", + "model.layers.67.mlp.experts.165.up_proj", + "model.layers.67.mlp.experts.166.down_proj", + "model.layers.67.mlp.experts.166.gate_proj", + "model.layers.67.mlp.experts.166.up_proj", + "model.layers.67.mlp.experts.167.down_proj", + "model.layers.67.mlp.experts.167.gate_proj", + "model.layers.67.mlp.experts.167.up_proj", + "model.layers.67.mlp.experts.168.down_proj", + "model.layers.67.mlp.experts.168.gate_proj", + "model.layers.67.mlp.experts.168.up_proj", + "model.layers.67.mlp.experts.169.down_proj", + "model.layers.67.mlp.experts.169.gate_proj", + "model.layers.67.mlp.experts.169.up_proj", + "model.layers.67.mlp.experts.17.down_proj", + "model.layers.67.mlp.experts.17.gate_proj", + "model.layers.67.mlp.experts.17.up_proj", + "model.layers.67.mlp.experts.170.down_proj", + "model.layers.67.mlp.experts.170.gate_proj", + "model.layers.67.mlp.experts.170.up_proj", + "model.layers.67.mlp.experts.171.down_proj", + "model.layers.67.mlp.experts.171.gate_proj", + "model.layers.67.mlp.experts.171.up_proj", + "model.layers.67.mlp.experts.172.down_proj", + "model.layers.67.mlp.experts.172.gate_proj", + "model.layers.67.mlp.experts.172.up_proj", + "model.layers.67.mlp.experts.173.down_proj", + "model.layers.67.mlp.experts.173.gate_proj", + "model.layers.67.mlp.experts.173.up_proj", + "model.layers.67.mlp.experts.174.down_proj", + "model.layers.67.mlp.experts.174.gate_proj", + "model.layers.67.mlp.experts.174.up_proj", + "model.layers.67.mlp.experts.175.down_proj", + "model.layers.67.mlp.experts.175.gate_proj", + "model.layers.67.mlp.experts.175.up_proj", + "model.layers.67.mlp.experts.176.down_proj", + "model.layers.67.mlp.experts.176.gate_proj", + "model.layers.67.mlp.experts.176.up_proj", + "model.layers.67.mlp.experts.177.down_proj", + "model.layers.67.mlp.experts.177.gate_proj", + "model.layers.67.mlp.experts.177.up_proj", + "model.layers.67.mlp.experts.178.down_proj", + "model.layers.67.mlp.experts.178.gate_proj", + "model.layers.67.mlp.experts.178.up_proj", + "model.layers.67.mlp.experts.179.down_proj", + "model.layers.67.mlp.experts.179.gate_proj", + "model.layers.67.mlp.experts.179.up_proj", + "model.layers.67.mlp.experts.18.down_proj", + "model.layers.67.mlp.experts.18.gate_proj", + "model.layers.67.mlp.experts.18.up_proj", + "model.layers.67.mlp.experts.180.down_proj", + "model.layers.67.mlp.experts.180.gate_proj", + "model.layers.67.mlp.experts.180.up_proj", + "model.layers.67.mlp.experts.181.down_proj", + "model.layers.67.mlp.experts.181.gate_proj", + "model.layers.67.mlp.experts.181.up_proj", + "model.layers.67.mlp.experts.182.down_proj", + "model.layers.67.mlp.experts.182.gate_proj", + "model.layers.67.mlp.experts.182.up_proj", + "model.layers.67.mlp.experts.183.down_proj", + "model.layers.67.mlp.experts.183.gate_proj", + "model.layers.67.mlp.experts.183.up_proj", + "model.layers.67.mlp.experts.184.down_proj", + "model.layers.67.mlp.experts.184.gate_proj", + "model.layers.67.mlp.experts.184.up_proj", + "model.layers.67.mlp.experts.185.down_proj", + "model.layers.67.mlp.experts.185.gate_proj", + "model.layers.67.mlp.experts.185.up_proj", + "model.layers.67.mlp.experts.186.down_proj", + "model.layers.67.mlp.experts.186.gate_proj", + "model.layers.67.mlp.experts.186.up_proj", + "model.layers.67.mlp.experts.187.down_proj", + "model.layers.67.mlp.experts.187.gate_proj", + "model.layers.67.mlp.experts.187.up_proj", + "model.layers.67.mlp.experts.188.down_proj", + "model.layers.67.mlp.experts.188.gate_proj", + "model.layers.67.mlp.experts.188.up_proj", + "model.layers.67.mlp.experts.189.down_proj", + "model.layers.67.mlp.experts.189.gate_proj", + "model.layers.67.mlp.experts.189.up_proj", + "model.layers.67.mlp.experts.19.down_proj", + "model.layers.67.mlp.experts.19.gate_proj", + "model.layers.67.mlp.experts.19.up_proj", + "model.layers.67.mlp.experts.190.down_proj", + "model.layers.67.mlp.experts.190.gate_proj", + "model.layers.67.mlp.experts.190.up_proj", + "model.layers.67.mlp.experts.191.down_proj", + "model.layers.67.mlp.experts.191.gate_proj", + "model.layers.67.mlp.experts.191.up_proj", + "model.layers.67.mlp.experts.192.down_proj", + "model.layers.67.mlp.experts.192.gate_proj", + "model.layers.67.mlp.experts.192.up_proj", + "model.layers.67.mlp.experts.193.down_proj", + "model.layers.67.mlp.experts.193.gate_proj", + "model.layers.67.mlp.experts.193.up_proj", + "model.layers.67.mlp.experts.194.down_proj", + "model.layers.67.mlp.experts.194.gate_proj", + "model.layers.67.mlp.experts.194.up_proj", + "model.layers.67.mlp.experts.195.down_proj", + "model.layers.67.mlp.experts.195.gate_proj", + "model.layers.67.mlp.experts.195.up_proj", + "model.layers.67.mlp.experts.196.down_proj", + "model.layers.67.mlp.experts.196.gate_proj", + "model.layers.67.mlp.experts.196.up_proj", + "model.layers.67.mlp.experts.197.down_proj", + "model.layers.67.mlp.experts.197.gate_proj", + "model.layers.67.mlp.experts.197.up_proj", + "model.layers.67.mlp.experts.198.down_proj", + "model.layers.67.mlp.experts.198.gate_proj", + "model.layers.67.mlp.experts.198.up_proj", + "model.layers.67.mlp.experts.199.down_proj", + "model.layers.67.mlp.experts.199.gate_proj", + "model.layers.67.mlp.experts.199.up_proj", + "model.layers.67.mlp.experts.2.down_proj", + "model.layers.67.mlp.experts.2.gate_proj", + "model.layers.67.mlp.experts.2.up_proj", + "model.layers.67.mlp.experts.20.down_proj", + "model.layers.67.mlp.experts.20.gate_proj", + "model.layers.67.mlp.experts.20.up_proj", + "model.layers.67.mlp.experts.200.down_proj", + "model.layers.67.mlp.experts.200.gate_proj", + "model.layers.67.mlp.experts.200.up_proj", + "model.layers.67.mlp.experts.201.down_proj", + "model.layers.67.mlp.experts.201.gate_proj", + "model.layers.67.mlp.experts.201.up_proj", + "model.layers.67.mlp.experts.202.down_proj", + "model.layers.67.mlp.experts.202.gate_proj", + "model.layers.67.mlp.experts.202.up_proj", + "model.layers.67.mlp.experts.203.down_proj", + "model.layers.67.mlp.experts.203.gate_proj", + "model.layers.67.mlp.experts.203.up_proj", + "model.layers.67.mlp.experts.204.down_proj", + "model.layers.67.mlp.experts.204.gate_proj", + "model.layers.67.mlp.experts.204.up_proj", + "model.layers.67.mlp.experts.205.down_proj", + "model.layers.67.mlp.experts.205.gate_proj", + "model.layers.67.mlp.experts.205.up_proj", + "model.layers.67.mlp.experts.206.down_proj", + "model.layers.67.mlp.experts.206.gate_proj", + "model.layers.67.mlp.experts.206.up_proj", + "model.layers.67.mlp.experts.207.down_proj", + "model.layers.67.mlp.experts.207.gate_proj", + "model.layers.67.mlp.experts.207.up_proj", + "model.layers.67.mlp.experts.208.down_proj", + "model.layers.67.mlp.experts.208.gate_proj", + "model.layers.67.mlp.experts.208.up_proj", + "model.layers.67.mlp.experts.209.down_proj", + "model.layers.67.mlp.experts.209.gate_proj", + "model.layers.67.mlp.experts.209.up_proj", + "model.layers.67.mlp.experts.21.down_proj", + "model.layers.67.mlp.experts.21.gate_proj", + "model.layers.67.mlp.experts.21.up_proj", + "model.layers.67.mlp.experts.210.down_proj", + "model.layers.67.mlp.experts.210.gate_proj", + "model.layers.67.mlp.experts.210.up_proj", + "model.layers.67.mlp.experts.211.down_proj", + "model.layers.67.mlp.experts.211.gate_proj", + "model.layers.67.mlp.experts.211.up_proj", + "model.layers.67.mlp.experts.212.down_proj", + "model.layers.67.mlp.experts.212.gate_proj", + "model.layers.67.mlp.experts.212.up_proj", + "model.layers.67.mlp.experts.213.down_proj", + "model.layers.67.mlp.experts.213.gate_proj", + "model.layers.67.mlp.experts.213.up_proj", + "model.layers.67.mlp.experts.214.down_proj", + "model.layers.67.mlp.experts.214.gate_proj", + "model.layers.67.mlp.experts.214.up_proj", + "model.layers.67.mlp.experts.215.down_proj", + "model.layers.67.mlp.experts.215.gate_proj", + "model.layers.67.mlp.experts.215.up_proj", + "model.layers.67.mlp.experts.216.down_proj", + "model.layers.67.mlp.experts.216.gate_proj", + "model.layers.67.mlp.experts.216.up_proj", + "model.layers.67.mlp.experts.217.down_proj", + "model.layers.67.mlp.experts.217.gate_proj", + "model.layers.67.mlp.experts.217.up_proj", + "model.layers.67.mlp.experts.218.down_proj", + "model.layers.67.mlp.experts.218.gate_proj", + "model.layers.67.mlp.experts.218.up_proj", + "model.layers.67.mlp.experts.219.down_proj", + "model.layers.67.mlp.experts.219.gate_proj", + "model.layers.67.mlp.experts.219.up_proj", + "model.layers.67.mlp.experts.22.down_proj", + "model.layers.67.mlp.experts.22.gate_proj", + "model.layers.67.mlp.experts.22.up_proj", + "model.layers.67.mlp.experts.220.down_proj", + "model.layers.67.mlp.experts.220.gate_proj", + "model.layers.67.mlp.experts.220.up_proj", + "model.layers.67.mlp.experts.221.down_proj", + "model.layers.67.mlp.experts.221.gate_proj", + "model.layers.67.mlp.experts.221.up_proj", + "model.layers.67.mlp.experts.222.down_proj", + "model.layers.67.mlp.experts.222.gate_proj", + "model.layers.67.mlp.experts.222.up_proj", + "model.layers.67.mlp.experts.223.down_proj", + "model.layers.67.mlp.experts.223.gate_proj", + "model.layers.67.mlp.experts.223.up_proj", + "model.layers.67.mlp.experts.224.down_proj", + "model.layers.67.mlp.experts.224.gate_proj", + "model.layers.67.mlp.experts.224.up_proj", + "model.layers.67.mlp.experts.225.down_proj", + "model.layers.67.mlp.experts.225.gate_proj", + "model.layers.67.mlp.experts.225.up_proj", + "model.layers.67.mlp.experts.226.down_proj", + "model.layers.67.mlp.experts.226.gate_proj", + "model.layers.67.mlp.experts.226.up_proj", + "model.layers.67.mlp.experts.227.down_proj", + "model.layers.67.mlp.experts.227.gate_proj", + "model.layers.67.mlp.experts.227.up_proj", + "model.layers.67.mlp.experts.228.down_proj", + "model.layers.67.mlp.experts.228.gate_proj", + "model.layers.67.mlp.experts.228.up_proj", + "model.layers.67.mlp.experts.229.down_proj", + "model.layers.67.mlp.experts.229.gate_proj", + "model.layers.67.mlp.experts.229.up_proj", + "model.layers.67.mlp.experts.23.down_proj", + "model.layers.67.mlp.experts.23.gate_proj", + "model.layers.67.mlp.experts.23.up_proj", + "model.layers.67.mlp.experts.230.down_proj", + "model.layers.67.mlp.experts.230.gate_proj", + "model.layers.67.mlp.experts.230.up_proj", + "model.layers.67.mlp.experts.231.down_proj", + "model.layers.67.mlp.experts.231.gate_proj", + "model.layers.67.mlp.experts.231.up_proj", + "model.layers.67.mlp.experts.232.down_proj", + "model.layers.67.mlp.experts.232.gate_proj", + "model.layers.67.mlp.experts.232.up_proj", + "model.layers.67.mlp.experts.233.down_proj", + "model.layers.67.mlp.experts.233.gate_proj", + "model.layers.67.mlp.experts.233.up_proj", + "model.layers.67.mlp.experts.234.down_proj", + "model.layers.67.mlp.experts.234.gate_proj", + "model.layers.67.mlp.experts.234.up_proj", + "model.layers.67.mlp.experts.235.down_proj", + "model.layers.67.mlp.experts.235.gate_proj", + "model.layers.67.mlp.experts.235.up_proj", + "model.layers.67.mlp.experts.236.down_proj", + "model.layers.67.mlp.experts.236.gate_proj", + "model.layers.67.mlp.experts.236.up_proj", + "model.layers.67.mlp.experts.237.down_proj", + "model.layers.67.mlp.experts.237.gate_proj", + "model.layers.67.mlp.experts.237.up_proj", + "model.layers.67.mlp.experts.238.down_proj", + "model.layers.67.mlp.experts.238.gate_proj", + "model.layers.67.mlp.experts.238.up_proj", + "model.layers.67.mlp.experts.239.down_proj", + "model.layers.67.mlp.experts.239.gate_proj", + "model.layers.67.mlp.experts.239.up_proj", + "model.layers.67.mlp.experts.24.down_proj", + "model.layers.67.mlp.experts.24.gate_proj", + "model.layers.67.mlp.experts.24.up_proj", + "model.layers.67.mlp.experts.240.down_proj", + "model.layers.67.mlp.experts.240.gate_proj", + "model.layers.67.mlp.experts.240.up_proj", + "model.layers.67.mlp.experts.241.down_proj", + "model.layers.67.mlp.experts.241.gate_proj", + "model.layers.67.mlp.experts.241.up_proj", + "model.layers.67.mlp.experts.242.down_proj", + "model.layers.67.mlp.experts.242.gate_proj", + "model.layers.67.mlp.experts.242.up_proj", + "model.layers.67.mlp.experts.243.down_proj", + "model.layers.67.mlp.experts.243.gate_proj", + "model.layers.67.mlp.experts.243.up_proj", + "model.layers.67.mlp.experts.244.down_proj", + "model.layers.67.mlp.experts.244.gate_proj", + "model.layers.67.mlp.experts.244.up_proj", + "model.layers.67.mlp.experts.245.down_proj", + "model.layers.67.mlp.experts.245.gate_proj", + "model.layers.67.mlp.experts.245.up_proj", + "model.layers.67.mlp.experts.246.down_proj", + "model.layers.67.mlp.experts.246.gate_proj", + "model.layers.67.mlp.experts.246.up_proj", + "model.layers.67.mlp.experts.247.down_proj", + "model.layers.67.mlp.experts.247.gate_proj", + "model.layers.67.mlp.experts.247.up_proj", + "model.layers.67.mlp.experts.248.down_proj", + "model.layers.67.mlp.experts.248.gate_proj", + "model.layers.67.mlp.experts.248.up_proj", + "model.layers.67.mlp.experts.249.down_proj", + "model.layers.67.mlp.experts.249.gate_proj", + "model.layers.67.mlp.experts.249.up_proj", + "model.layers.67.mlp.experts.25.down_proj", + "model.layers.67.mlp.experts.25.gate_proj", + "model.layers.67.mlp.experts.25.up_proj", + "model.layers.67.mlp.experts.250.down_proj", + "model.layers.67.mlp.experts.250.gate_proj", + "model.layers.67.mlp.experts.250.up_proj", + "model.layers.67.mlp.experts.251.down_proj", + "model.layers.67.mlp.experts.251.gate_proj", + "model.layers.67.mlp.experts.251.up_proj", + "model.layers.67.mlp.experts.252.down_proj", + "model.layers.67.mlp.experts.252.gate_proj", + "model.layers.67.mlp.experts.252.up_proj", + "model.layers.67.mlp.experts.253.down_proj", + "model.layers.67.mlp.experts.253.gate_proj", + "model.layers.67.mlp.experts.253.up_proj", + "model.layers.67.mlp.experts.254.down_proj", + "model.layers.67.mlp.experts.254.gate_proj", + "model.layers.67.mlp.experts.254.up_proj", + "model.layers.67.mlp.experts.255.down_proj", + "model.layers.67.mlp.experts.255.gate_proj", + "model.layers.67.mlp.experts.255.up_proj", + "model.layers.67.mlp.experts.26.down_proj", + "model.layers.67.mlp.experts.26.gate_proj", + "model.layers.67.mlp.experts.26.up_proj", + "model.layers.67.mlp.experts.27.down_proj", + "model.layers.67.mlp.experts.27.gate_proj", + "model.layers.67.mlp.experts.27.up_proj", + "model.layers.67.mlp.experts.28.down_proj", + "model.layers.67.mlp.experts.28.gate_proj", + "model.layers.67.mlp.experts.28.up_proj", + "model.layers.67.mlp.experts.29.down_proj", + "model.layers.67.mlp.experts.29.gate_proj", + "model.layers.67.mlp.experts.29.up_proj", + "model.layers.67.mlp.experts.3.down_proj", + "model.layers.67.mlp.experts.3.gate_proj", + "model.layers.67.mlp.experts.3.up_proj", + "model.layers.67.mlp.experts.30.down_proj", + "model.layers.67.mlp.experts.30.gate_proj", + "model.layers.67.mlp.experts.30.up_proj", + "model.layers.67.mlp.experts.31.down_proj", + "model.layers.67.mlp.experts.31.gate_proj", + "model.layers.67.mlp.experts.31.up_proj", + "model.layers.67.mlp.experts.32.down_proj", + "model.layers.67.mlp.experts.32.gate_proj", + "model.layers.67.mlp.experts.32.up_proj", + "model.layers.67.mlp.experts.33.down_proj", + "model.layers.67.mlp.experts.33.gate_proj", + "model.layers.67.mlp.experts.33.up_proj", + "model.layers.67.mlp.experts.34.down_proj", + "model.layers.67.mlp.experts.34.gate_proj", + "model.layers.67.mlp.experts.34.up_proj", + "model.layers.67.mlp.experts.35.down_proj", + "model.layers.67.mlp.experts.35.gate_proj", + "model.layers.67.mlp.experts.35.up_proj", + "model.layers.67.mlp.experts.36.down_proj", + "model.layers.67.mlp.experts.36.gate_proj", + "model.layers.67.mlp.experts.36.up_proj", + "model.layers.67.mlp.experts.37.down_proj", + "model.layers.67.mlp.experts.37.gate_proj", + "model.layers.67.mlp.experts.37.up_proj", + "model.layers.67.mlp.experts.38.down_proj", + "model.layers.67.mlp.experts.38.gate_proj", + "model.layers.67.mlp.experts.38.up_proj", + "model.layers.67.mlp.experts.39.down_proj", + "model.layers.67.mlp.experts.39.gate_proj", + "model.layers.67.mlp.experts.39.up_proj", + "model.layers.67.mlp.experts.4.down_proj", + "model.layers.67.mlp.experts.4.gate_proj", + "model.layers.67.mlp.experts.4.up_proj", + "model.layers.67.mlp.experts.40.down_proj", + "model.layers.67.mlp.experts.40.gate_proj", + "model.layers.67.mlp.experts.40.up_proj", + "model.layers.67.mlp.experts.41.down_proj", + "model.layers.67.mlp.experts.41.gate_proj", + "model.layers.67.mlp.experts.41.up_proj", + "model.layers.67.mlp.experts.42.down_proj", + "model.layers.67.mlp.experts.42.gate_proj", + "model.layers.67.mlp.experts.42.up_proj", + "model.layers.67.mlp.experts.43.down_proj", + "model.layers.67.mlp.experts.43.gate_proj", + "model.layers.67.mlp.experts.43.up_proj", + "model.layers.67.mlp.experts.44.down_proj", + "model.layers.67.mlp.experts.44.gate_proj", + "model.layers.67.mlp.experts.44.up_proj", + "model.layers.67.mlp.experts.45.down_proj", + "model.layers.67.mlp.experts.45.gate_proj", + "model.layers.67.mlp.experts.45.up_proj", + "model.layers.67.mlp.experts.46.down_proj", + "model.layers.67.mlp.experts.46.gate_proj", + "model.layers.67.mlp.experts.46.up_proj", + "model.layers.67.mlp.experts.47.down_proj", + "model.layers.67.mlp.experts.47.gate_proj", + "model.layers.67.mlp.experts.47.up_proj", + "model.layers.67.mlp.experts.48.down_proj", + "model.layers.67.mlp.experts.48.gate_proj", + "model.layers.67.mlp.experts.48.up_proj", + "model.layers.67.mlp.experts.49.down_proj", + "model.layers.67.mlp.experts.49.gate_proj", + "model.layers.67.mlp.experts.49.up_proj", + "model.layers.67.mlp.experts.5.down_proj", + "model.layers.67.mlp.experts.5.gate_proj", + "model.layers.67.mlp.experts.5.up_proj", + "model.layers.67.mlp.experts.50.down_proj", + "model.layers.67.mlp.experts.50.gate_proj", + "model.layers.67.mlp.experts.50.up_proj", + "model.layers.67.mlp.experts.51.down_proj", + "model.layers.67.mlp.experts.51.gate_proj", + "model.layers.67.mlp.experts.51.up_proj", + "model.layers.67.mlp.experts.52.down_proj", + "model.layers.67.mlp.experts.52.gate_proj", + "model.layers.67.mlp.experts.52.up_proj", + "model.layers.67.mlp.experts.53.down_proj", + "model.layers.67.mlp.experts.53.gate_proj", + "model.layers.67.mlp.experts.53.up_proj", + "model.layers.67.mlp.experts.54.down_proj", + "model.layers.67.mlp.experts.54.gate_proj", + "model.layers.67.mlp.experts.54.up_proj", + "model.layers.67.mlp.experts.55.down_proj", + "model.layers.67.mlp.experts.55.gate_proj", + "model.layers.67.mlp.experts.55.up_proj", + "model.layers.67.mlp.experts.56.down_proj", + "model.layers.67.mlp.experts.56.gate_proj", + "model.layers.67.mlp.experts.56.up_proj", + "model.layers.67.mlp.experts.57.down_proj", + "model.layers.67.mlp.experts.57.gate_proj", + "model.layers.67.mlp.experts.57.up_proj", + "model.layers.67.mlp.experts.58.down_proj", + "model.layers.67.mlp.experts.58.gate_proj", + "model.layers.67.mlp.experts.58.up_proj", + "model.layers.67.mlp.experts.59.down_proj", + "model.layers.67.mlp.experts.59.gate_proj", + "model.layers.67.mlp.experts.59.up_proj", + "model.layers.67.mlp.experts.6.down_proj", + "model.layers.67.mlp.experts.6.gate_proj", + "model.layers.67.mlp.experts.6.up_proj", + "model.layers.67.mlp.experts.60.down_proj", + "model.layers.67.mlp.experts.60.gate_proj", + "model.layers.67.mlp.experts.60.up_proj", + "model.layers.67.mlp.experts.61.down_proj", + "model.layers.67.mlp.experts.61.gate_proj", + "model.layers.67.mlp.experts.61.up_proj", + "model.layers.67.mlp.experts.62.down_proj", + "model.layers.67.mlp.experts.62.gate_proj", + "model.layers.67.mlp.experts.62.up_proj", + "model.layers.67.mlp.experts.63.down_proj", + "model.layers.67.mlp.experts.63.gate_proj", + "model.layers.67.mlp.experts.63.up_proj", + "model.layers.67.mlp.experts.64.down_proj", + "model.layers.67.mlp.experts.64.gate_proj", + "model.layers.67.mlp.experts.64.up_proj", + "model.layers.67.mlp.experts.65.down_proj", + "model.layers.67.mlp.experts.65.gate_proj", + "model.layers.67.mlp.experts.65.up_proj", + "model.layers.67.mlp.experts.66.down_proj", + "model.layers.67.mlp.experts.66.gate_proj", + "model.layers.67.mlp.experts.66.up_proj", + "model.layers.67.mlp.experts.67.down_proj", + "model.layers.67.mlp.experts.67.gate_proj", + "model.layers.67.mlp.experts.67.up_proj", + "model.layers.67.mlp.experts.68.down_proj", + "model.layers.67.mlp.experts.68.gate_proj", + "model.layers.67.mlp.experts.68.up_proj", + "model.layers.67.mlp.experts.69.down_proj", + "model.layers.67.mlp.experts.69.gate_proj", + "model.layers.67.mlp.experts.69.up_proj", + "model.layers.67.mlp.experts.7.down_proj", + "model.layers.67.mlp.experts.7.gate_proj", + "model.layers.67.mlp.experts.7.up_proj", + "model.layers.67.mlp.experts.70.down_proj", + "model.layers.67.mlp.experts.70.gate_proj", + "model.layers.67.mlp.experts.70.up_proj", + "model.layers.67.mlp.experts.71.down_proj", + "model.layers.67.mlp.experts.71.gate_proj", + "model.layers.67.mlp.experts.71.up_proj", + "model.layers.67.mlp.experts.72.down_proj", + "model.layers.67.mlp.experts.72.gate_proj", + "model.layers.67.mlp.experts.72.up_proj", + "model.layers.67.mlp.experts.73.down_proj", + "model.layers.67.mlp.experts.73.gate_proj", + "model.layers.67.mlp.experts.73.up_proj", + "model.layers.67.mlp.experts.74.down_proj", + "model.layers.67.mlp.experts.74.gate_proj", + "model.layers.67.mlp.experts.74.up_proj", + "model.layers.67.mlp.experts.75.down_proj", + "model.layers.67.mlp.experts.75.gate_proj", + "model.layers.67.mlp.experts.75.up_proj", + "model.layers.67.mlp.experts.76.down_proj", + "model.layers.67.mlp.experts.76.gate_proj", + "model.layers.67.mlp.experts.76.up_proj", + "model.layers.67.mlp.experts.77.down_proj", + "model.layers.67.mlp.experts.77.gate_proj", + "model.layers.67.mlp.experts.77.up_proj", + "model.layers.67.mlp.experts.78.down_proj", + "model.layers.67.mlp.experts.78.gate_proj", + "model.layers.67.mlp.experts.78.up_proj", + "model.layers.67.mlp.experts.79.down_proj", + "model.layers.67.mlp.experts.79.gate_proj", + "model.layers.67.mlp.experts.79.up_proj", + "model.layers.67.mlp.experts.8.down_proj", + "model.layers.67.mlp.experts.8.gate_proj", + "model.layers.67.mlp.experts.8.up_proj", + "model.layers.67.mlp.experts.80.down_proj", + "model.layers.67.mlp.experts.80.gate_proj", + "model.layers.67.mlp.experts.80.up_proj", + "model.layers.67.mlp.experts.81.down_proj", + "model.layers.67.mlp.experts.81.gate_proj", + "model.layers.67.mlp.experts.81.up_proj", + "model.layers.67.mlp.experts.82.down_proj", + "model.layers.67.mlp.experts.82.gate_proj", + "model.layers.67.mlp.experts.82.up_proj", + "model.layers.67.mlp.experts.83.down_proj", + "model.layers.67.mlp.experts.83.gate_proj", + "model.layers.67.mlp.experts.83.up_proj", + "model.layers.67.mlp.experts.84.down_proj", + "model.layers.67.mlp.experts.84.gate_proj", + "model.layers.67.mlp.experts.84.up_proj", + "model.layers.67.mlp.experts.85.down_proj", + "model.layers.67.mlp.experts.85.gate_proj", + "model.layers.67.mlp.experts.85.up_proj", + "model.layers.67.mlp.experts.86.down_proj", + "model.layers.67.mlp.experts.86.gate_proj", + "model.layers.67.mlp.experts.86.up_proj", + "model.layers.67.mlp.experts.87.down_proj", + "model.layers.67.mlp.experts.87.gate_proj", + "model.layers.67.mlp.experts.87.up_proj", + "model.layers.67.mlp.experts.88.down_proj", + "model.layers.67.mlp.experts.88.gate_proj", + "model.layers.67.mlp.experts.88.up_proj", + "model.layers.67.mlp.experts.89.down_proj", + "model.layers.67.mlp.experts.89.gate_proj", + "model.layers.67.mlp.experts.89.up_proj", + "model.layers.67.mlp.experts.9.down_proj", + "model.layers.67.mlp.experts.9.gate_proj", + "model.layers.67.mlp.experts.9.up_proj", + "model.layers.67.mlp.experts.90.down_proj", + "model.layers.67.mlp.experts.90.gate_proj", + "model.layers.67.mlp.experts.90.up_proj", + "model.layers.67.mlp.experts.91.down_proj", + "model.layers.67.mlp.experts.91.gate_proj", + "model.layers.67.mlp.experts.91.up_proj", + "model.layers.67.mlp.experts.92.down_proj", + "model.layers.67.mlp.experts.92.gate_proj", + "model.layers.67.mlp.experts.92.up_proj", + "model.layers.67.mlp.experts.93.down_proj", + "model.layers.67.mlp.experts.93.gate_proj", + "model.layers.67.mlp.experts.93.up_proj", + "model.layers.67.mlp.experts.94.down_proj", + "model.layers.67.mlp.experts.94.gate_proj", + "model.layers.67.mlp.experts.94.up_proj", + "model.layers.67.mlp.experts.95.down_proj", + "model.layers.67.mlp.experts.95.gate_proj", + "model.layers.67.mlp.experts.95.up_proj", + "model.layers.67.mlp.experts.96.down_proj", + "model.layers.67.mlp.experts.96.gate_proj", + "model.layers.67.mlp.experts.96.up_proj", + "model.layers.67.mlp.experts.97.down_proj", + "model.layers.67.mlp.experts.97.gate_proj", + "model.layers.67.mlp.experts.97.up_proj", + "model.layers.67.mlp.experts.98.down_proj", + "model.layers.67.mlp.experts.98.gate_proj", + "model.layers.67.mlp.experts.98.up_proj", + "model.layers.67.mlp.experts.99.down_proj", + "model.layers.67.mlp.experts.99.gate_proj", + "model.layers.67.mlp.experts.99.up_proj", + "model.layers.68.mlp.experts.0.down_proj", + "model.layers.68.mlp.experts.0.gate_proj", + "model.layers.68.mlp.experts.0.up_proj", + "model.layers.68.mlp.experts.1.down_proj", + "model.layers.68.mlp.experts.1.gate_proj", + "model.layers.68.mlp.experts.1.up_proj", + "model.layers.68.mlp.experts.10.down_proj", + "model.layers.68.mlp.experts.10.gate_proj", + "model.layers.68.mlp.experts.10.up_proj", + "model.layers.68.mlp.experts.100.down_proj", + "model.layers.68.mlp.experts.100.gate_proj", + "model.layers.68.mlp.experts.100.up_proj", + "model.layers.68.mlp.experts.101.down_proj", + "model.layers.68.mlp.experts.101.gate_proj", + "model.layers.68.mlp.experts.101.up_proj", + "model.layers.68.mlp.experts.102.down_proj", + "model.layers.68.mlp.experts.102.gate_proj", + "model.layers.68.mlp.experts.102.up_proj", + "model.layers.68.mlp.experts.103.down_proj", + "model.layers.68.mlp.experts.103.gate_proj", + "model.layers.68.mlp.experts.103.up_proj", + "model.layers.68.mlp.experts.104.down_proj", + "model.layers.68.mlp.experts.104.gate_proj", + "model.layers.68.mlp.experts.104.up_proj", + "model.layers.68.mlp.experts.105.down_proj", + "model.layers.68.mlp.experts.105.gate_proj", + "model.layers.68.mlp.experts.105.up_proj", + "model.layers.68.mlp.experts.106.down_proj", + "model.layers.68.mlp.experts.106.gate_proj", + "model.layers.68.mlp.experts.106.up_proj", + "model.layers.68.mlp.experts.107.down_proj", + "model.layers.68.mlp.experts.107.gate_proj", + "model.layers.68.mlp.experts.107.up_proj", + "model.layers.68.mlp.experts.108.down_proj", + "model.layers.68.mlp.experts.108.gate_proj", + "model.layers.68.mlp.experts.108.up_proj", + "model.layers.68.mlp.experts.109.down_proj", + "model.layers.68.mlp.experts.109.gate_proj", + "model.layers.68.mlp.experts.109.up_proj", + "model.layers.68.mlp.experts.11.down_proj", + "model.layers.68.mlp.experts.11.gate_proj", + "model.layers.68.mlp.experts.11.up_proj", + "model.layers.68.mlp.experts.110.down_proj", + "model.layers.68.mlp.experts.110.gate_proj", + "model.layers.68.mlp.experts.110.up_proj", + "model.layers.68.mlp.experts.111.down_proj", + "model.layers.68.mlp.experts.111.gate_proj", + "model.layers.68.mlp.experts.111.up_proj", + "model.layers.68.mlp.experts.112.down_proj", + "model.layers.68.mlp.experts.112.gate_proj", + "model.layers.68.mlp.experts.112.up_proj", + "model.layers.68.mlp.experts.113.down_proj", + "model.layers.68.mlp.experts.113.gate_proj", + "model.layers.68.mlp.experts.113.up_proj", + "model.layers.68.mlp.experts.114.down_proj", + "model.layers.68.mlp.experts.114.gate_proj", + "model.layers.68.mlp.experts.114.up_proj", + "model.layers.68.mlp.experts.115.down_proj", + "model.layers.68.mlp.experts.115.gate_proj", + "model.layers.68.mlp.experts.115.up_proj", + "model.layers.68.mlp.experts.116.down_proj", + "model.layers.68.mlp.experts.116.gate_proj", + "model.layers.68.mlp.experts.116.up_proj", + "model.layers.68.mlp.experts.117.down_proj", + "model.layers.68.mlp.experts.117.gate_proj", + "model.layers.68.mlp.experts.117.up_proj", + "model.layers.68.mlp.experts.118.down_proj", + "model.layers.68.mlp.experts.118.gate_proj", + "model.layers.68.mlp.experts.118.up_proj", + "model.layers.68.mlp.experts.119.down_proj", + "model.layers.68.mlp.experts.119.gate_proj", + "model.layers.68.mlp.experts.119.up_proj", + "model.layers.68.mlp.experts.12.down_proj", + "model.layers.68.mlp.experts.12.gate_proj", + "model.layers.68.mlp.experts.12.up_proj", + "model.layers.68.mlp.experts.120.down_proj", + "model.layers.68.mlp.experts.120.gate_proj", + "model.layers.68.mlp.experts.120.up_proj", + "model.layers.68.mlp.experts.121.down_proj", + "model.layers.68.mlp.experts.121.gate_proj", + "model.layers.68.mlp.experts.121.up_proj", + "model.layers.68.mlp.experts.122.down_proj", + "model.layers.68.mlp.experts.122.gate_proj", + "model.layers.68.mlp.experts.122.up_proj", + "model.layers.68.mlp.experts.123.down_proj", + "model.layers.68.mlp.experts.123.gate_proj", + "model.layers.68.mlp.experts.123.up_proj", + "model.layers.68.mlp.experts.124.down_proj", + "model.layers.68.mlp.experts.124.gate_proj", + "model.layers.68.mlp.experts.124.up_proj", + "model.layers.68.mlp.experts.125.down_proj", + "model.layers.68.mlp.experts.125.gate_proj", + "model.layers.68.mlp.experts.125.up_proj", + "model.layers.68.mlp.experts.126.down_proj", + "model.layers.68.mlp.experts.126.gate_proj", + "model.layers.68.mlp.experts.126.up_proj", + "model.layers.68.mlp.experts.127.down_proj", + "model.layers.68.mlp.experts.127.gate_proj", + "model.layers.68.mlp.experts.127.up_proj", + "model.layers.68.mlp.experts.128.down_proj", + "model.layers.68.mlp.experts.128.gate_proj", + "model.layers.68.mlp.experts.128.up_proj", + "model.layers.68.mlp.experts.129.down_proj", + "model.layers.68.mlp.experts.129.gate_proj", + "model.layers.68.mlp.experts.129.up_proj", + "model.layers.68.mlp.experts.13.down_proj", + "model.layers.68.mlp.experts.13.gate_proj", + "model.layers.68.mlp.experts.13.up_proj", + "model.layers.68.mlp.experts.130.down_proj", + "model.layers.68.mlp.experts.130.gate_proj", + "model.layers.68.mlp.experts.130.up_proj", + "model.layers.68.mlp.experts.131.down_proj", + "model.layers.68.mlp.experts.131.gate_proj", + "model.layers.68.mlp.experts.131.up_proj", + "model.layers.68.mlp.experts.132.down_proj", + "model.layers.68.mlp.experts.132.gate_proj", + "model.layers.68.mlp.experts.132.up_proj", + "model.layers.68.mlp.experts.133.down_proj", + "model.layers.68.mlp.experts.133.gate_proj", + "model.layers.68.mlp.experts.133.up_proj", + "model.layers.68.mlp.experts.134.down_proj", + "model.layers.68.mlp.experts.134.gate_proj", + "model.layers.68.mlp.experts.134.up_proj", + "model.layers.68.mlp.experts.135.down_proj", + "model.layers.68.mlp.experts.135.gate_proj", + "model.layers.68.mlp.experts.135.up_proj", + "model.layers.68.mlp.experts.136.down_proj", + "model.layers.68.mlp.experts.136.gate_proj", + "model.layers.68.mlp.experts.136.up_proj", + "model.layers.68.mlp.experts.137.down_proj", + "model.layers.68.mlp.experts.137.gate_proj", + "model.layers.68.mlp.experts.137.up_proj", + "model.layers.68.mlp.experts.138.down_proj", + "model.layers.68.mlp.experts.138.gate_proj", + "model.layers.68.mlp.experts.138.up_proj", + "model.layers.68.mlp.experts.139.down_proj", + "model.layers.68.mlp.experts.139.gate_proj", + "model.layers.68.mlp.experts.139.up_proj", + "model.layers.68.mlp.experts.14.down_proj", + "model.layers.68.mlp.experts.14.gate_proj", + "model.layers.68.mlp.experts.14.up_proj", + "model.layers.68.mlp.experts.140.down_proj", + "model.layers.68.mlp.experts.140.gate_proj", + "model.layers.68.mlp.experts.140.up_proj", + "model.layers.68.mlp.experts.141.down_proj", + "model.layers.68.mlp.experts.141.gate_proj", + "model.layers.68.mlp.experts.141.up_proj", + "model.layers.68.mlp.experts.142.down_proj", + "model.layers.68.mlp.experts.142.gate_proj", + "model.layers.68.mlp.experts.142.up_proj", + "model.layers.68.mlp.experts.143.down_proj", + "model.layers.68.mlp.experts.143.gate_proj", + "model.layers.68.mlp.experts.143.up_proj", + "model.layers.68.mlp.experts.144.down_proj", + "model.layers.68.mlp.experts.144.gate_proj", + "model.layers.68.mlp.experts.144.up_proj", + "model.layers.68.mlp.experts.145.down_proj", + "model.layers.68.mlp.experts.145.gate_proj", + "model.layers.68.mlp.experts.145.up_proj", + "model.layers.68.mlp.experts.146.down_proj", + "model.layers.68.mlp.experts.146.gate_proj", + "model.layers.68.mlp.experts.146.up_proj", + "model.layers.68.mlp.experts.147.down_proj", + "model.layers.68.mlp.experts.147.gate_proj", + "model.layers.68.mlp.experts.147.up_proj", + "model.layers.68.mlp.experts.148.down_proj", + "model.layers.68.mlp.experts.148.gate_proj", + "model.layers.68.mlp.experts.148.up_proj", + "model.layers.68.mlp.experts.149.down_proj", + "model.layers.68.mlp.experts.149.gate_proj", + "model.layers.68.mlp.experts.149.up_proj", + "model.layers.68.mlp.experts.15.down_proj", + "model.layers.68.mlp.experts.15.gate_proj", + "model.layers.68.mlp.experts.15.up_proj", + "model.layers.68.mlp.experts.150.down_proj", + "model.layers.68.mlp.experts.150.gate_proj", + "model.layers.68.mlp.experts.150.up_proj", + "model.layers.68.mlp.experts.151.down_proj", + "model.layers.68.mlp.experts.151.gate_proj", + "model.layers.68.mlp.experts.151.up_proj", + "model.layers.68.mlp.experts.152.down_proj", + "model.layers.68.mlp.experts.152.gate_proj", + "model.layers.68.mlp.experts.152.up_proj", + "model.layers.68.mlp.experts.153.down_proj", + "model.layers.68.mlp.experts.153.gate_proj", + "model.layers.68.mlp.experts.153.up_proj", + "model.layers.68.mlp.experts.154.down_proj", + "model.layers.68.mlp.experts.154.gate_proj", + "model.layers.68.mlp.experts.154.up_proj", + "model.layers.68.mlp.experts.155.down_proj", + "model.layers.68.mlp.experts.155.gate_proj", + "model.layers.68.mlp.experts.155.up_proj", + "model.layers.68.mlp.experts.156.down_proj", + "model.layers.68.mlp.experts.156.gate_proj", + "model.layers.68.mlp.experts.156.up_proj", + "model.layers.68.mlp.experts.157.down_proj", + "model.layers.68.mlp.experts.157.gate_proj", + "model.layers.68.mlp.experts.157.up_proj", + "model.layers.68.mlp.experts.158.down_proj", + "model.layers.68.mlp.experts.158.gate_proj", + "model.layers.68.mlp.experts.158.up_proj", + "model.layers.68.mlp.experts.159.down_proj", + "model.layers.68.mlp.experts.159.gate_proj", + "model.layers.68.mlp.experts.159.up_proj", + "model.layers.68.mlp.experts.16.down_proj", + "model.layers.68.mlp.experts.16.gate_proj", + "model.layers.68.mlp.experts.16.up_proj", + "model.layers.68.mlp.experts.160.down_proj", + "model.layers.68.mlp.experts.160.gate_proj", + "model.layers.68.mlp.experts.160.up_proj", + "model.layers.68.mlp.experts.161.down_proj", + "model.layers.68.mlp.experts.161.gate_proj", + "model.layers.68.mlp.experts.161.up_proj", + "model.layers.68.mlp.experts.162.down_proj", + "model.layers.68.mlp.experts.162.gate_proj", + "model.layers.68.mlp.experts.162.up_proj", + "model.layers.68.mlp.experts.163.down_proj", + "model.layers.68.mlp.experts.163.gate_proj", + "model.layers.68.mlp.experts.163.up_proj", + "model.layers.68.mlp.experts.164.down_proj", + "model.layers.68.mlp.experts.164.gate_proj", + "model.layers.68.mlp.experts.164.up_proj", + "model.layers.68.mlp.experts.165.down_proj", + "model.layers.68.mlp.experts.165.gate_proj", + "model.layers.68.mlp.experts.165.up_proj", + "model.layers.68.mlp.experts.166.down_proj", + "model.layers.68.mlp.experts.166.gate_proj", + "model.layers.68.mlp.experts.166.up_proj", + "model.layers.68.mlp.experts.167.down_proj", + "model.layers.68.mlp.experts.167.gate_proj", + "model.layers.68.mlp.experts.167.up_proj", + "model.layers.68.mlp.experts.168.down_proj", + "model.layers.68.mlp.experts.168.gate_proj", + "model.layers.68.mlp.experts.168.up_proj", + "model.layers.68.mlp.experts.169.down_proj", + "model.layers.68.mlp.experts.169.gate_proj", + "model.layers.68.mlp.experts.169.up_proj", + "model.layers.68.mlp.experts.17.down_proj", + "model.layers.68.mlp.experts.17.gate_proj", + "model.layers.68.mlp.experts.17.up_proj", + "model.layers.68.mlp.experts.170.down_proj", + "model.layers.68.mlp.experts.170.gate_proj", + "model.layers.68.mlp.experts.170.up_proj", + "model.layers.68.mlp.experts.171.down_proj", + "model.layers.68.mlp.experts.171.gate_proj", + "model.layers.68.mlp.experts.171.up_proj", + "model.layers.68.mlp.experts.172.down_proj", + "model.layers.68.mlp.experts.172.gate_proj", + "model.layers.68.mlp.experts.172.up_proj", + "model.layers.68.mlp.experts.173.down_proj", + "model.layers.68.mlp.experts.173.gate_proj", + "model.layers.68.mlp.experts.173.up_proj", + "model.layers.68.mlp.experts.174.down_proj", + "model.layers.68.mlp.experts.174.gate_proj", + "model.layers.68.mlp.experts.174.up_proj", + "model.layers.68.mlp.experts.175.down_proj", + "model.layers.68.mlp.experts.175.gate_proj", + "model.layers.68.mlp.experts.175.up_proj", + "model.layers.68.mlp.experts.176.down_proj", + "model.layers.68.mlp.experts.176.gate_proj", + "model.layers.68.mlp.experts.176.up_proj", + "model.layers.68.mlp.experts.177.down_proj", + "model.layers.68.mlp.experts.177.gate_proj", + "model.layers.68.mlp.experts.177.up_proj", + "model.layers.68.mlp.experts.178.down_proj", + "model.layers.68.mlp.experts.178.gate_proj", + "model.layers.68.mlp.experts.178.up_proj", + "model.layers.68.mlp.experts.179.down_proj", + "model.layers.68.mlp.experts.179.gate_proj", + "model.layers.68.mlp.experts.179.up_proj", + "model.layers.68.mlp.experts.18.down_proj", + "model.layers.68.mlp.experts.18.gate_proj", + "model.layers.68.mlp.experts.18.up_proj", + "model.layers.68.mlp.experts.180.down_proj", + "model.layers.68.mlp.experts.180.gate_proj", + "model.layers.68.mlp.experts.180.up_proj", + "model.layers.68.mlp.experts.181.down_proj", + "model.layers.68.mlp.experts.181.gate_proj", + "model.layers.68.mlp.experts.181.up_proj", + "model.layers.68.mlp.experts.182.down_proj", + "model.layers.68.mlp.experts.182.gate_proj", + "model.layers.68.mlp.experts.182.up_proj", + "model.layers.68.mlp.experts.183.down_proj", + "model.layers.68.mlp.experts.183.gate_proj", + "model.layers.68.mlp.experts.183.up_proj", + "model.layers.68.mlp.experts.184.down_proj", + "model.layers.68.mlp.experts.184.gate_proj", + "model.layers.68.mlp.experts.184.up_proj", + "model.layers.68.mlp.experts.185.down_proj", + "model.layers.68.mlp.experts.185.gate_proj", + "model.layers.68.mlp.experts.185.up_proj", + "model.layers.68.mlp.experts.186.down_proj", + "model.layers.68.mlp.experts.186.gate_proj", + "model.layers.68.mlp.experts.186.up_proj", + "model.layers.68.mlp.experts.187.down_proj", + "model.layers.68.mlp.experts.187.gate_proj", + "model.layers.68.mlp.experts.187.up_proj", + "model.layers.68.mlp.experts.188.down_proj", + "model.layers.68.mlp.experts.188.gate_proj", + "model.layers.68.mlp.experts.188.up_proj", + "model.layers.68.mlp.experts.189.down_proj", + "model.layers.68.mlp.experts.189.gate_proj", + "model.layers.68.mlp.experts.189.up_proj", + "model.layers.68.mlp.experts.19.down_proj", + "model.layers.68.mlp.experts.19.gate_proj", + "model.layers.68.mlp.experts.19.up_proj", + "model.layers.68.mlp.experts.190.down_proj", + "model.layers.68.mlp.experts.190.gate_proj", + "model.layers.68.mlp.experts.190.up_proj", + "model.layers.68.mlp.experts.191.down_proj", + "model.layers.68.mlp.experts.191.gate_proj", + "model.layers.68.mlp.experts.191.up_proj", + "model.layers.68.mlp.experts.192.down_proj", + "model.layers.68.mlp.experts.192.gate_proj", + "model.layers.68.mlp.experts.192.up_proj", + "model.layers.68.mlp.experts.193.down_proj", + "model.layers.68.mlp.experts.193.gate_proj", + "model.layers.68.mlp.experts.193.up_proj", + "model.layers.68.mlp.experts.194.down_proj", + "model.layers.68.mlp.experts.194.gate_proj", + "model.layers.68.mlp.experts.194.up_proj", + "model.layers.68.mlp.experts.195.down_proj", + "model.layers.68.mlp.experts.195.gate_proj", + "model.layers.68.mlp.experts.195.up_proj", + "model.layers.68.mlp.experts.196.down_proj", + "model.layers.68.mlp.experts.196.gate_proj", + "model.layers.68.mlp.experts.196.up_proj", + "model.layers.68.mlp.experts.197.down_proj", + "model.layers.68.mlp.experts.197.gate_proj", + "model.layers.68.mlp.experts.197.up_proj", + "model.layers.68.mlp.experts.198.down_proj", + "model.layers.68.mlp.experts.198.gate_proj", + "model.layers.68.mlp.experts.198.up_proj", + "model.layers.68.mlp.experts.199.down_proj", + "model.layers.68.mlp.experts.199.gate_proj", + "model.layers.68.mlp.experts.199.up_proj", + "model.layers.68.mlp.experts.2.down_proj", + "model.layers.68.mlp.experts.2.gate_proj", + "model.layers.68.mlp.experts.2.up_proj", + "model.layers.68.mlp.experts.20.down_proj", + "model.layers.68.mlp.experts.20.gate_proj", + "model.layers.68.mlp.experts.20.up_proj", + "model.layers.68.mlp.experts.200.down_proj", + "model.layers.68.mlp.experts.200.gate_proj", + "model.layers.68.mlp.experts.200.up_proj", + "model.layers.68.mlp.experts.201.down_proj", + "model.layers.68.mlp.experts.201.gate_proj", + "model.layers.68.mlp.experts.201.up_proj", + "model.layers.68.mlp.experts.202.down_proj", + "model.layers.68.mlp.experts.202.gate_proj", + "model.layers.68.mlp.experts.202.up_proj", + "model.layers.68.mlp.experts.203.down_proj", + "model.layers.68.mlp.experts.203.gate_proj", + "model.layers.68.mlp.experts.203.up_proj", + "model.layers.68.mlp.experts.204.down_proj", + "model.layers.68.mlp.experts.204.gate_proj", + "model.layers.68.mlp.experts.204.up_proj", + "model.layers.68.mlp.experts.205.down_proj", + "model.layers.68.mlp.experts.205.gate_proj", + "model.layers.68.mlp.experts.205.up_proj", + "model.layers.68.mlp.experts.206.down_proj", + "model.layers.68.mlp.experts.206.gate_proj", + "model.layers.68.mlp.experts.206.up_proj", + "model.layers.68.mlp.experts.207.down_proj", + "model.layers.68.mlp.experts.207.gate_proj", + "model.layers.68.mlp.experts.207.up_proj", + "model.layers.68.mlp.experts.208.down_proj", + "model.layers.68.mlp.experts.208.gate_proj", + "model.layers.68.mlp.experts.208.up_proj", + "model.layers.68.mlp.experts.209.down_proj", + "model.layers.68.mlp.experts.209.gate_proj", + "model.layers.68.mlp.experts.209.up_proj", + "model.layers.68.mlp.experts.21.down_proj", + "model.layers.68.mlp.experts.21.gate_proj", + "model.layers.68.mlp.experts.21.up_proj", + "model.layers.68.mlp.experts.210.down_proj", + "model.layers.68.mlp.experts.210.gate_proj", + "model.layers.68.mlp.experts.210.up_proj", + "model.layers.68.mlp.experts.211.down_proj", + "model.layers.68.mlp.experts.211.gate_proj", + "model.layers.68.mlp.experts.211.up_proj", + "model.layers.68.mlp.experts.212.down_proj", + "model.layers.68.mlp.experts.212.gate_proj", + "model.layers.68.mlp.experts.212.up_proj", + "model.layers.68.mlp.experts.213.down_proj", + "model.layers.68.mlp.experts.213.gate_proj", + "model.layers.68.mlp.experts.213.up_proj", + "model.layers.68.mlp.experts.214.down_proj", + "model.layers.68.mlp.experts.214.gate_proj", + "model.layers.68.mlp.experts.214.up_proj", + "model.layers.68.mlp.experts.215.down_proj", + "model.layers.68.mlp.experts.215.gate_proj", + "model.layers.68.mlp.experts.215.up_proj", + "model.layers.68.mlp.experts.216.down_proj", + "model.layers.68.mlp.experts.216.gate_proj", + "model.layers.68.mlp.experts.216.up_proj", + "model.layers.68.mlp.experts.217.down_proj", + "model.layers.68.mlp.experts.217.gate_proj", + "model.layers.68.mlp.experts.217.up_proj", + "model.layers.68.mlp.experts.218.down_proj", + "model.layers.68.mlp.experts.218.gate_proj", + "model.layers.68.mlp.experts.218.up_proj", + "model.layers.68.mlp.experts.219.down_proj", + "model.layers.68.mlp.experts.219.gate_proj", + "model.layers.68.mlp.experts.219.up_proj", + "model.layers.68.mlp.experts.22.down_proj", + "model.layers.68.mlp.experts.22.gate_proj", + "model.layers.68.mlp.experts.22.up_proj", + "model.layers.68.mlp.experts.220.down_proj", + "model.layers.68.mlp.experts.220.gate_proj", + "model.layers.68.mlp.experts.220.up_proj", + "model.layers.68.mlp.experts.221.down_proj", + "model.layers.68.mlp.experts.221.gate_proj", + "model.layers.68.mlp.experts.221.up_proj", + "model.layers.68.mlp.experts.222.down_proj", + "model.layers.68.mlp.experts.222.gate_proj", + "model.layers.68.mlp.experts.222.up_proj", + "model.layers.68.mlp.experts.223.down_proj", + "model.layers.68.mlp.experts.223.gate_proj", + "model.layers.68.mlp.experts.223.up_proj", + "model.layers.68.mlp.experts.224.down_proj", + "model.layers.68.mlp.experts.224.gate_proj", + "model.layers.68.mlp.experts.224.up_proj", + "model.layers.68.mlp.experts.225.down_proj", + "model.layers.68.mlp.experts.225.gate_proj", + "model.layers.68.mlp.experts.225.up_proj", + "model.layers.68.mlp.experts.226.down_proj", + "model.layers.68.mlp.experts.226.gate_proj", + "model.layers.68.mlp.experts.226.up_proj", + "model.layers.68.mlp.experts.227.down_proj", + "model.layers.68.mlp.experts.227.gate_proj", + "model.layers.68.mlp.experts.227.up_proj", + "model.layers.68.mlp.experts.228.down_proj", + "model.layers.68.mlp.experts.228.gate_proj", + "model.layers.68.mlp.experts.228.up_proj", + "model.layers.68.mlp.experts.229.down_proj", + "model.layers.68.mlp.experts.229.gate_proj", + "model.layers.68.mlp.experts.229.up_proj", + "model.layers.68.mlp.experts.23.down_proj", + "model.layers.68.mlp.experts.23.gate_proj", + "model.layers.68.mlp.experts.23.up_proj", + "model.layers.68.mlp.experts.230.down_proj", + "model.layers.68.mlp.experts.230.gate_proj", + "model.layers.68.mlp.experts.230.up_proj", + "model.layers.68.mlp.experts.231.down_proj", + "model.layers.68.mlp.experts.231.gate_proj", + "model.layers.68.mlp.experts.231.up_proj", + "model.layers.68.mlp.experts.232.down_proj", + "model.layers.68.mlp.experts.232.gate_proj", + "model.layers.68.mlp.experts.232.up_proj", + "model.layers.68.mlp.experts.233.down_proj", + "model.layers.68.mlp.experts.233.gate_proj", + "model.layers.68.mlp.experts.233.up_proj", + "model.layers.68.mlp.experts.234.down_proj", + "model.layers.68.mlp.experts.234.gate_proj", + "model.layers.68.mlp.experts.234.up_proj", + "model.layers.68.mlp.experts.235.down_proj", + "model.layers.68.mlp.experts.235.gate_proj", + "model.layers.68.mlp.experts.235.up_proj", + "model.layers.68.mlp.experts.236.down_proj", + "model.layers.68.mlp.experts.236.gate_proj", + "model.layers.68.mlp.experts.236.up_proj", + "model.layers.68.mlp.experts.237.down_proj", + "model.layers.68.mlp.experts.237.gate_proj", + "model.layers.68.mlp.experts.237.up_proj", + "model.layers.68.mlp.experts.238.down_proj", + "model.layers.68.mlp.experts.238.gate_proj", + "model.layers.68.mlp.experts.238.up_proj", + "model.layers.68.mlp.experts.239.down_proj", + "model.layers.68.mlp.experts.239.gate_proj", + "model.layers.68.mlp.experts.239.up_proj", + "model.layers.68.mlp.experts.24.down_proj", + "model.layers.68.mlp.experts.24.gate_proj", + "model.layers.68.mlp.experts.24.up_proj", + "model.layers.68.mlp.experts.240.down_proj", + "model.layers.68.mlp.experts.240.gate_proj", + "model.layers.68.mlp.experts.240.up_proj", + "model.layers.68.mlp.experts.241.down_proj", + "model.layers.68.mlp.experts.241.gate_proj", + "model.layers.68.mlp.experts.241.up_proj", + "model.layers.68.mlp.experts.242.down_proj", + "model.layers.68.mlp.experts.242.gate_proj", + "model.layers.68.mlp.experts.242.up_proj", + "model.layers.68.mlp.experts.243.down_proj", + "model.layers.68.mlp.experts.243.gate_proj", + "model.layers.68.mlp.experts.243.up_proj", + "model.layers.68.mlp.experts.244.down_proj", + "model.layers.68.mlp.experts.244.gate_proj", + "model.layers.68.mlp.experts.244.up_proj", + "model.layers.68.mlp.experts.245.down_proj", + "model.layers.68.mlp.experts.245.gate_proj", + "model.layers.68.mlp.experts.245.up_proj", + "model.layers.68.mlp.experts.246.down_proj", + "model.layers.68.mlp.experts.246.gate_proj", + "model.layers.68.mlp.experts.246.up_proj", + "model.layers.68.mlp.experts.247.down_proj", + "model.layers.68.mlp.experts.247.gate_proj", + "model.layers.68.mlp.experts.247.up_proj", + "model.layers.68.mlp.experts.248.down_proj", + "model.layers.68.mlp.experts.248.gate_proj", + "model.layers.68.mlp.experts.248.up_proj", + "model.layers.68.mlp.experts.249.down_proj", + "model.layers.68.mlp.experts.249.gate_proj", + "model.layers.68.mlp.experts.249.up_proj", + "model.layers.68.mlp.experts.25.down_proj", + "model.layers.68.mlp.experts.25.gate_proj", + "model.layers.68.mlp.experts.25.up_proj", + "model.layers.68.mlp.experts.250.down_proj", + "model.layers.68.mlp.experts.250.gate_proj", + "model.layers.68.mlp.experts.250.up_proj", + "model.layers.68.mlp.experts.251.down_proj", + "model.layers.68.mlp.experts.251.gate_proj", + "model.layers.68.mlp.experts.251.up_proj", + "model.layers.68.mlp.experts.252.down_proj", + "model.layers.68.mlp.experts.252.gate_proj", + "model.layers.68.mlp.experts.252.up_proj", + "model.layers.68.mlp.experts.253.down_proj", + "model.layers.68.mlp.experts.253.gate_proj", + "model.layers.68.mlp.experts.253.up_proj", + "model.layers.68.mlp.experts.254.down_proj", + "model.layers.68.mlp.experts.254.gate_proj", + "model.layers.68.mlp.experts.254.up_proj", + "model.layers.68.mlp.experts.255.down_proj", + "model.layers.68.mlp.experts.255.gate_proj", + "model.layers.68.mlp.experts.255.up_proj", + "model.layers.68.mlp.experts.26.down_proj", + "model.layers.68.mlp.experts.26.gate_proj", + "model.layers.68.mlp.experts.26.up_proj", + "model.layers.68.mlp.experts.27.down_proj", + "model.layers.68.mlp.experts.27.gate_proj", + "model.layers.68.mlp.experts.27.up_proj", + "model.layers.68.mlp.experts.28.down_proj", + "model.layers.68.mlp.experts.28.gate_proj", + "model.layers.68.mlp.experts.28.up_proj", + "model.layers.68.mlp.experts.29.down_proj", + "model.layers.68.mlp.experts.29.gate_proj", + "model.layers.68.mlp.experts.29.up_proj", + "model.layers.68.mlp.experts.3.down_proj", + "model.layers.68.mlp.experts.3.gate_proj", + "model.layers.68.mlp.experts.3.up_proj", + "model.layers.68.mlp.experts.30.down_proj", + "model.layers.68.mlp.experts.30.gate_proj", + "model.layers.68.mlp.experts.30.up_proj", + "model.layers.68.mlp.experts.31.down_proj", + "model.layers.68.mlp.experts.31.gate_proj", + "model.layers.68.mlp.experts.31.up_proj", + "model.layers.68.mlp.experts.32.down_proj", + "model.layers.68.mlp.experts.32.gate_proj", + "model.layers.68.mlp.experts.32.up_proj", + "model.layers.68.mlp.experts.33.down_proj", + "model.layers.68.mlp.experts.33.gate_proj", + "model.layers.68.mlp.experts.33.up_proj", + "model.layers.68.mlp.experts.34.down_proj", + "model.layers.68.mlp.experts.34.gate_proj", + "model.layers.68.mlp.experts.34.up_proj", + "model.layers.68.mlp.experts.35.down_proj", + "model.layers.68.mlp.experts.35.gate_proj", + "model.layers.68.mlp.experts.35.up_proj", + "model.layers.68.mlp.experts.36.down_proj", + "model.layers.68.mlp.experts.36.gate_proj", + "model.layers.68.mlp.experts.36.up_proj", + "model.layers.68.mlp.experts.37.down_proj", + "model.layers.68.mlp.experts.37.gate_proj", + "model.layers.68.mlp.experts.37.up_proj", + "model.layers.68.mlp.experts.38.down_proj", + "model.layers.68.mlp.experts.38.gate_proj", + "model.layers.68.mlp.experts.38.up_proj", + "model.layers.68.mlp.experts.39.down_proj", + "model.layers.68.mlp.experts.39.gate_proj", + "model.layers.68.mlp.experts.39.up_proj", + "model.layers.68.mlp.experts.4.down_proj", + "model.layers.68.mlp.experts.4.gate_proj", + "model.layers.68.mlp.experts.4.up_proj", + "model.layers.68.mlp.experts.40.down_proj", + "model.layers.68.mlp.experts.40.gate_proj", + "model.layers.68.mlp.experts.40.up_proj", + "model.layers.68.mlp.experts.41.down_proj", + "model.layers.68.mlp.experts.41.gate_proj", + "model.layers.68.mlp.experts.41.up_proj", + "model.layers.68.mlp.experts.42.down_proj", + "model.layers.68.mlp.experts.42.gate_proj", + "model.layers.68.mlp.experts.42.up_proj", + "model.layers.68.mlp.experts.43.down_proj", + "model.layers.68.mlp.experts.43.gate_proj", + "model.layers.68.mlp.experts.43.up_proj", + "model.layers.68.mlp.experts.44.down_proj", + "model.layers.68.mlp.experts.44.gate_proj", + "model.layers.68.mlp.experts.44.up_proj", + "model.layers.68.mlp.experts.45.down_proj", + "model.layers.68.mlp.experts.45.gate_proj", + "model.layers.68.mlp.experts.45.up_proj", + "model.layers.68.mlp.experts.46.down_proj", + "model.layers.68.mlp.experts.46.gate_proj", + "model.layers.68.mlp.experts.46.up_proj", + "model.layers.68.mlp.experts.47.down_proj", + "model.layers.68.mlp.experts.47.gate_proj", + "model.layers.68.mlp.experts.47.up_proj", + "model.layers.68.mlp.experts.48.down_proj", + "model.layers.68.mlp.experts.48.gate_proj", + "model.layers.68.mlp.experts.48.up_proj", + "model.layers.68.mlp.experts.49.down_proj", + "model.layers.68.mlp.experts.49.gate_proj", + "model.layers.68.mlp.experts.49.up_proj", + "model.layers.68.mlp.experts.5.down_proj", + "model.layers.68.mlp.experts.5.gate_proj", + "model.layers.68.mlp.experts.5.up_proj", + "model.layers.68.mlp.experts.50.down_proj", + "model.layers.68.mlp.experts.50.gate_proj", + "model.layers.68.mlp.experts.50.up_proj", + "model.layers.68.mlp.experts.51.down_proj", + "model.layers.68.mlp.experts.51.gate_proj", + "model.layers.68.mlp.experts.51.up_proj", + "model.layers.68.mlp.experts.52.down_proj", + "model.layers.68.mlp.experts.52.gate_proj", + "model.layers.68.mlp.experts.52.up_proj", + "model.layers.68.mlp.experts.53.down_proj", + "model.layers.68.mlp.experts.53.gate_proj", + "model.layers.68.mlp.experts.53.up_proj", + "model.layers.68.mlp.experts.54.down_proj", + "model.layers.68.mlp.experts.54.gate_proj", + "model.layers.68.mlp.experts.54.up_proj", + "model.layers.68.mlp.experts.55.down_proj", + "model.layers.68.mlp.experts.55.gate_proj", + "model.layers.68.mlp.experts.55.up_proj", + "model.layers.68.mlp.experts.56.down_proj", + "model.layers.68.mlp.experts.56.gate_proj", + "model.layers.68.mlp.experts.56.up_proj", + "model.layers.68.mlp.experts.57.down_proj", + "model.layers.68.mlp.experts.57.gate_proj", + "model.layers.68.mlp.experts.57.up_proj", + "model.layers.68.mlp.experts.58.down_proj", + "model.layers.68.mlp.experts.58.gate_proj", + "model.layers.68.mlp.experts.58.up_proj", + "model.layers.68.mlp.experts.59.down_proj", + "model.layers.68.mlp.experts.59.gate_proj", + "model.layers.68.mlp.experts.59.up_proj", + "model.layers.68.mlp.experts.6.down_proj", + "model.layers.68.mlp.experts.6.gate_proj", + "model.layers.68.mlp.experts.6.up_proj", + "model.layers.68.mlp.experts.60.down_proj", + "model.layers.68.mlp.experts.60.gate_proj", + "model.layers.68.mlp.experts.60.up_proj", + "model.layers.68.mlp.experts.61.down_proj", + "model.layers.68.mlp.experts.61.gate_proj", + "model.layers.68.mlp.experts.61.up_proj", + "model.layers.68.mlp.experts.62.down_proj", + "model.layers.68.mlp.experts.62.gate_proj", + "model.layers.68.mlp.experts.62.up_proj", + "model.layers.68.mlp.experts.63.down_proj", + "model.layers.68.mlp.experts.63.gate_proj", + "model.layers.68.mlp.experts.63.up_proj", + "model.layers.68.mlp.experts.64.down_proj", + "model.layers.68.mlp.experts.64.gate_proj", + "model.layers.68.mlp.experts.64.up_proj", + "model.layers.68.mlp.experts.65.down_proj", + "model.layers.68.mlp.experts.65.gate_proj", + "model.layers.68.mlp.experts.65.up_proj", + "model.layers.68.mlp.experts.66.down_proj", + "model.layers.68.mlp.experts.66.gate_proj", + "model.layers.68.mlp.experts.66.up_proj", + "model.layers.68.mlp.experts.67.down_proj", + "model.layers.68.mlp.experts.67.gate_proj", + "model.layers.68.mlp.experts.67.up_proj", + "model.layers.68.mlp.experts.68.down_proj", + "model.layers.68.mlp.experts.68.gate_proj", + "model.layers.68.mlp.experts.68.up_proj", + "model.layers.68.mlp.experts.69.down_proj", + "model.layers.68.mlp.experts.69.gate_proj", + "model.layers.68.mlp.experts.69.up_proj", + "model.layers.68.mlp.experts.7.down_proj", + "model.layers.68.mlp.experts.7.gate_proj", + "model.layers.68.mlp.experts.7.up_proj", + "model.layers.68.mlp.experts.70.down_proj", + "model.layers.68.mlp.experts.70.gate_proj", + "model.layers.68.mlp.experts.70.up_proj", + "model.layers.68.mlp.experts.71.down_proj", + "model.layers.68.mlp.experts.71.gate_proj", + "model.layers.68.mlp.experts.71.up_proj", + "model.layers.68.mlp.experts.72.down_proj", + "model.layers.68.mlp.experts.72.gate_proj", + "model.layers.68.mlp.experts.72.up_proj", + "model.layers.68.mlp.experts.73.down_proj", + "model.layers.68.mlp.experts.73.gate_proj", + "model.layers.68.mlp.experts.73.up_proj", + "model.layers.68.mlp.experts.74.down_proj", + "model.layers.68.mlp.experts.74.gate_proj", + "model.layers.68.mlp.experts.74.up_proj", + "model.layers.68.mlp.experts.75.down_proj", + "model.layers.68.mlp.experts.75.gate_proj", + "model.layers.68.mlp.experts.75.up_proj", + "model.layers.68.mlp.experts.76.down_proj", + "model.layers.68.mlp.experts.76.gate_proj", + "model.layers.68.mlp.experts.76.up_proj", + "model.layers.68.mlp.experts.77.down_proj", + "model.layers.68.mlp.experts.77.gate_proj", + "model.layers.68.mlp.experts.77.up_proj", + "model.layers.68.mlp.experts.78.down_proj", + "model.layers.68.mlp.experts.78.gate_proj", + "model.layers.68.mlp.experts.78.up_proj", + "model.layers.68.mlp.experts.79.down_proj", + "model.layers.68.mlp.experts.79.gate_proj", + "model.layers.68.mlp.experts.79.up_proj", + "model.layers.68.mlp.experts.8.down_proj", + "model.layers.68.mlp.experts.8.gate_proj", + "model.layers.68.mlp.experts.8.up_proj", + "model.layers.68.mlp.experts.80.down_proj", + "model.layers.68.mlp.experts.80.gate_proj", + "model.layers.68.mlp.experts.80.up_proj", + "model.layers.68.mlp.experts.81.down_proj", + "model.layers.68.mlp.experts.81.gate_proj", + "model.layers.68.mlp.experts.81.up_proj", + "model.layers.68.mlp.experts.82.down_proj", + "model.layers.68.mlp.experts.82.gate_proj", + "model.layers.68.mlp.experts.82.up_proj", + "model.layers.68.mlp.experts.83.down_proj", + "model.layers.68.mlp.experts.83.gate_proj", + "model.layers.68.mlp.experts.83.up_proj", + "model.layers.68.mlp.experts.84.down_proj", + "model.layers.68.mlp.experts.84.gate_proj", + "model.layers.68.mlp.experts.84.up_proj", + "model.layers.68.mlp.experts.85.down_proj", + "model.layers.68.mlp.experts.85.gate_proj", + "model.layers.68.mlp.experts.85.up_proj", + "model.layers.68.mlp.experts.86.down_proj", + "model.layers.68.mlp.experts.86.gate_proj", + "model.layers.68.mlp.experts.86.up_proj", + "model.layers.68.mlp.experts.87.down_proj", + "model.layers.68.mlp.experts.87.gate_proj", + "model.layers.68.mlp.experts.87.up_proj", + "model.layers.68.mlp.experts.88.down_proj", + "model.layers.68.mlp.experts.88.gate_proj", + "model.layers.68.mlp.experts.88.up_proj", + "model.layers.68.mlp.experts.89.down_proj", + "model.layers.68.mlp.experts.89.gate_proj", + "model.layers.68.mlp.experts.89.up_proj", + "model.layers.68.mlp.experts.9.down_proj", + "model.layers.68.mlp.experts.9.gate_proj", + "model.layers.68.mlp.experts.9.up_proj", + "model.layers.68.mlp.experts.90.down_proj", + "model.layers.68.mlp.experts.90.gate_proj", + "model.layers.68.mlp.experts.90.up_proj", + "model.layers.68.mlp.experts.91.down_proj", + "model.layers.68.mlp.experts.91.gate_proj", + "model.layers.68.mlp.experts.91.up_proj", + "model.layers.68.mlp.experts.92.down_proj", + "model.layers.68.mlp.experts.92.gate_proj", + "model.layers.68.mlp.experts.92.up_proj", + "model.layers.68.mlp.experts.93.down_proj", + "model.layers.68.mlp.experts.93.gate_proj", + "model.layers.68.mlp.experts.93.up_proj", + "model.layers.68.mlp.experts.94.down_proj", + "model.layers.68.mlp.experts.94.gate_proj", + "model.layers.68.mlp.experts.94.up_proj", + "model.layers.68.mlp.experts.95.down_proj", + "model.layers.68.mlp.experts.95.gate_proj", + "model.layers.68.mlp.experts.95.up_proj", + "model.layers.68.mlp.experts.96.down_proj", + "model.layers.68.mlp.experts.96.gate_proj", + "model.layers.68.mlp.experts.96.up_proj", + "model.layers.68.mlp.experts.97.down_proj", + "model.layers.68.mlp.experts.97.gate_proj", + "model.layers.68.mlp.experts.97.up_proj", + "model.layers.68.mlp.experts.98.down_proj", + "model.layers.68.mlp.experts.98.gate_proj", + "model.layers.68.mlp.experts.98.up_proj", + "model.layers.68.mlp.experts.99.down_proj", + "model.layers.68.mlp.experts.99.gate_proj", + "model.layers.68.mlp.experts.99.up_proj", + "model.layers.69.mlp.experts.0.down_proj", + "model.layers.69.mlp.experts.0.gate_proj", + "model.layers.69.mlp.experts.0.up_proj", + "model.layers.69.mlp.experts.1.down_proj", + "model.layers.69.mlp.experts.1.gate_proj", + "model.layers.69.mlp.experts.1.up_proj", + "model.layers.69.mlp.experts.10.down_proj", + "model.layers.69.mlp.experts.10.gate_proj", + "model.layers.69.mlp.experts.10.up_proj", + "model.layers.69.mlp.experts.100.down_proj", + "model.layers.69.mlp.experts.100.gate_proj", + "model.layers.69.mlp.experts.100.up_proj", + "model.layers.69.mlp.experts.101.down_proj", + "model.layers.69.mlp.experts.101.gate_proj", + "model.layers.69.mlp.experts.101.up_proj", + "model.layers.69.mlp.experts.102.down_proj", + "model.layers.69.mlp.experts.102.gate_proj", + "model.layers.69.mlp.experts.102.up_proj", + "model.layers.69.mlp.experts.103.down_proj", + "model.layers.69.mlp.experts.103.gate_proj", + "model.layers.69.mlp.experts.103.up_proj", + "model.layers.69.mlp.experts.104.down_proj", + "model.layers.69.mlp.experts.104.gate_proj", + "model.layers.69.mlp.experts.104.up_proj", + "model.layers.69.mlp.experts.105.down_proj", + "model.layers.69.mlp.experts.105.gate_proj", + "model.layers.69.mlp.experts.105.up_proj", + "model.layers.69.mlp.experts.106.down_proj", + "model.layers.69.mlp.experts.106.gate_proj", + "model.layers.69.mlp.experts.106.up_proj", + "model.layers.69.mlp.experts.107.down_proj", + "model.layers.69.mlp.experts.107.gate_proj", + "model.layers.69.mlp.experts.107.up_proj", + "model.layers.69.mlp.experts.108.down_proj", + "model.layers.69.mlp.experts.108.gate_proj", + "model.layers.69.mlp.experts.108.up_proj", + "model.layers.69.mlp.experts.109.down_proj", + "model.layers.69.mlp.experts.109.gate_proj", + "model.layers.69.mlp.experts.109.up_proj", + "model.layers.69.mlp.experts.11.down_proj", + "model.layers.69.mlp.experts.11.gate_proj", + "model.layers.69.mlp.experts.11.up_proj", + "model.layers.69.mlp.experts.110.down_proj", + "model.layers.69.mlp.experts.110.gate_proj", + "model.layers.69.mlp.experts.110.up_proj", + "model.layers.69.mlp.experts.111.down_proj", + "model.layers.69.mlp.experts.111.gate_proj", + "model.layers.69.mlp.experts.111.up_proj", + "model.layers.69.mlp.experts.112.down_proj", + "model.layers.69.mlp.experts.112.gate_proj", + "model.layers.69.mlp.experts.112.up_proj", + "model.layers.69.mlp.experts.113.down_proj", + "model.layers.69.mlp.experts.113.gate_proj", + "model.layers.69.mlp.experts.113.up_proj", + "model.layers.69.mlp.experts.114.down_proj", + "model.layers.69.mlp.experts.114.gate_proj", + "model.layers.69.mlp.experts.114.up_proj", + "model.layers.69.mlp.experts.115.down_proj", + "model.layers.69.mlp.experts.115.gate_proj", + "model.layers.69.mlp.experts.115.up_proj", + "model.layers.69.mlp.experts.116.down_proj", + "model.layers.69.mlp.experts.116.gate_proj", + "model.layers.69.mlp.experts.116.up_proj", + "model.layers.69.mlp.experts.117.down_proj", + "model.layers.69.mlp.experts.117.gate_proj", + "model.layers.69.mlp.experts.117.up_proj", + "model.layers.69.mlp.experts.118.down_proj", + "model.layers.69.mlp.experts.118.gate_proj", + "model.layers.69.mlp.experts.118.up_proj", + "model.layers.69.mlp.experts.119.down_proj", + "model.layers.69.mlp.experts.119.gate_proj", + "model.layers.69.mlp.experts.119.up_proj", + "model.layers.69.mlp.experts.12.down_proj", + "model.layers.69.mlp.experts.12.gate_proj", + "model.layers.69.mlp.experts.12.up_proj", + "model.layers.69.mlp.experts.120.down_proj", + "model.layers.69.mlp.experts.120.gate_proj", + "model.layers.69.mlp.experts.120.up_proj", + "model.layers.69.mlp.experts.121.down_proj", + "model.layers.69.mlp.experts.121.gate_proj", + "model.layers.69.mlp.experts.121.up_proj", + "model.layers.69.mlp.experts.122.down_proj", + "model.layers.69.mlp.experts.122.gate_proj", + "model.layers.69.mlp.experts.122.up_proj", + "model.layers.69.mlp.experts.123.down_proj", + "model.layers.69.mlp.experts.123.gate_proj", + "model.layers.69.mlp.experts.123.up_proj", + "model.layers.69.mlp.experts.124.down_proj", + "model.layers.69.mlp.experts.124.gate_proj", + "model.layers.69.mlp.experts.124.up_proj", + "model.layers.69.mlp.experts.125.down_proj", + "model.layers.69.mlp.experts.125.gate_proj", + "model.layers.69.mlp.experts.125.up_proj", + "model.layers.69.mlp.experts.126.down_proj", + "model.layers.69.mlp.experts.126.gate_proj", + "model.layers.69.mlp.experts.126.up_proj", + "model.layers.69.mlp.experts.127.down_proj", + "model.layers.69.mlp.experts.127.gate_proj", + "model.layers.69.mlp.experts.127.up_proj", + "model.layers.69.mlp.experts.128.down_proj", + "model.layers.69.mlp.experts.128.gate_proj", + "model.layers.69.mlp.experts.128.up_proj", + "model.layers.69.mlp.experts.129.down_proj", + "model.layers.69.mlp.experts.129.gate_proj", + "model.layers.69.mlp.experts.129.up_proj", + "model.layers.69.mlp.experts.13.down_proj", + "model.layers.69.mlp.experts.13.gate_proj", + "model.layers.69.mlp.experts.13.up_proj", + "model.layers.69.mlp.experts.130.down_proj", + "model.layers.69.mlp.experts.130.gate_proj", + "model.layers.69.mlp.experts.130.up_proj", + "model.layers.69.mlp.experts.131.down_proj", + "model.layers.69.mlp.experts.131.gate_proj", + "model.layers.69.mlp.experts.131.up_proj", + "model.layers.69.mlp.experts.132.down_proj", + "model.layers.69.mlp.experts.132.gate_proj", + "model.layers.69.mlp.experts.132.up_proj", + "model.layers.69.mlp.experts.133.down_proj", + "model.layers.69.mlp.experts.133.gate_proj", + "model.layers.69.mlp.experts.133.up_proj", + "model.layers.69.mlp.experts.134.down_proj", + "model.layers.69.mlp.experts.134.gate_proj", + "model.layers.69.mlp.experts.134.up_proj", + "model.layers.69.mlp.experts.135.down_proj", + "model.layers.69.mlp.experts.135.gate_proj", + "model.layers.69.mlp.experts.135.up_proj", + "model.layers.69.mlp.experts.136.down_proj", + "model.layers.69.mlp.experts.136.gate_proj", + "model.layers.69.mlp.experts.136.up_proj", + "model.layers.69.mlp.experts.137.down_proj", + "model.layers.69.mlp.experts.137.gate_proj", + "model.layers.69.mlp.experts.137.up_proj", + "model.layers.69.mlp.experts.138.down_proj", + "model.layers.69.mlp.experts.138.gate_proj", + "model.layers.69.mlp.experts.138.up_proj", + "model.layers.69.mlp.experts.139.down_proj", + "model.layers.69.mlp.experts.139.gate_proj", + "model.layers.69.mlp.experts.139.up_proj", + "model.layers.69.mlp.experts.14.down_proj", + "model.layers.69.mlp.experts.14.gate_proj", + "model.layers.69.mlp.experts.14.up_proj", + "model.layers.69.mlp.experts.140.down_proj", + "model.layers.69.mlp.experts.140.gate_proj", + "model.layers.69.mlp.experts.140.up_proj", + "model.layers.69.mlp.experts.141.down_proj", + "model.layers.69.mlp.experts.141.gate_proj", + "model.layers.69.mlp.experts.141.up_proj", + "model.layers.69.mlp.experts.142.down_proj", + "model.layers.69.mlp.experts.142.gate_proj", + "model.layers.69.mlp.experts.142.up_proj", + "model.layers.69.mlp.experts.143.down_proj", + "model.layers.69.mlp.experts.143.gate_proj", + "model.layers.69.mlp.experts.143.up_proj", + "model.layers.69.mlp.experts.144.down_proj", + "model.layers.69.mlp.experts.144.gate_proj", + "model.layers.69.mlp.experts.144.up_proj", + "model.layers.69.mlp.experts.145.down_proj", + "model.layers.69.mlp.experts.145.gate_proj", + "model.layers.69.mlp.experts.145.up_proj", + "model.layers.69.mlp.experts.146.down_proj", + "model.layers.69.mlp.experts.146.gate_proj", + "model.layers.69.mlp.experts.146.up_proj", + "model.layers.69.mlp.experts.147.down_proj", + "model.layers.69.mlp.experts.147.gate_proj", + "model.layers.69.mlp.experts.147.up_proj", + "model.layers.69.mlp.experts.148.down_proj", + "model.layers.69.mlp.experts.148.gate_proj", + "model.layers.69.mlp.experts.148.up_proj", + "model.layers.69.mlp.experts.149.down_proj", + "model.layers.69.mlp.experts.149.gate_proj", + "model.layers.69.mlp.experts.149.up_proj", + "model.layers.69.mlp.experts.15.down_proj", + "model.layers.69.mlp.experts.15.gate_proj", + "model.layers.69.mlp.experts.15.up_proj", + "model.layers.69.mlp.experts.150.down_proj", + "model.layers.69.mlp.experts.150.gate_proj", + "model.layers.69.mlp.experts.150.up_proj", + "model.layers.69.mlp.experts.151.down_proj", + "model.layers.69.mlp.experts.151.gate_proj", + "model.layers.69.mlp.experts.151.up_proj", + "model.layers.69.mlp.experts.152.down_proj", + "model.layers.69.mlp.experts.152.gate_proj", + "model.layers.69.mlp.experts.152.up_proj", + "model.layers.69.mlp.experts.153.down_proj", + "model.layers.69.mlp.experts.153.gate_proj", + "model.layers.69.mlp.experts.153.up_proj", + "model.layers.69.mlp.experts.154.down_proj", + "model.layers.69.mlp.experts.154.gate_proj", + "model.layers.69.mlp.experts.154.up_proj", + "model.layers.69.mlp.experts.155.down_proj", + "model.layers.69.mlp.experts.155.gate_proj", + "model.layers.69.mlp.experts.155.up_proj", + "model.layers.69.mlp.experts.156.down_proj", + "model.layers.69.mlp.experts.156.gate_proj", + "model.layers.69.mlp.experts.156.up_proj", + "model.layers.69.mlp.experts.157.down_proj", + "model.layers.69.mlp.experts.157.gate_proj", + "model.layers.69.mlp.experts.157.up_proj", + "model.layers.69.mlp.experts.158.down_proj", + "model.layers.69.mlp.experts.158.gate_proj", + "model.layers.69.mlp.experts.158.up_proj", + "model.layers.69.mlp.experts.159.down_proj", + "model.layers.69.mlp.experts.159.gate_proj", + "model.layers.69.mlp.experts.159.up_proj", + "model.layers.69.mlp.experts.16.down_proj", + "model.layers.69.mlp.experts.16.gate_proj", + "model.layers.69.mlp.experts.16.up_proj", + "model.layers.69.mlp.experts.160.down_proj", + "model.layers.69.mlp.experts.160.gate_proj", + "model.layers.69.mlp.experts.160.up_proj", + "model.layers.69.mlp.experts.161.down_proj", + "model.layers.69.mlp.experts.161.gate_proj", + "model.layers.69.mlp.experts.161.up_proj", + "model.layers.69.mlp.experts.162.down_proj", + "model.layers.69.mlp.experts.162.gate_proj", + "model.layers.69.mlp.experts.162.up_proj", + "model.layers.69.mlp.experts.163.down_proj", + "model.layers.69.mlp.experts.163.gate_proj", + "model.layers.69.mlp.experts.163.up_proj", + "model.layers.69.mlp.experts.164.down_proj", + "model.layers.69.mlp.experts.164.gate_proj", + "model.layers.69.mlp.experts.164.up_proj", + "model.layers.69.mlp.experts.165.down_proj", + "model.layers.69.mlp.experts.165.gate_proj", + "model.layers.69.mlp.experts.165.up_proj", + "model.layers.69.mlp.experts.166.down_proj", + "model.layers.69.mlp.experts.166.gate_proj", + "model.layers.69.mlp.experts.166.up_proj", + "model.layers.69.mlp.experts.167.down_proj", + "model.layers.69.mlp.experts.167.gate_proj", + "model.layers.69.mlp.experts.167.up_proj", + "model.layers.69.mlp.experts.168.down_proj", + "model.layers.69.mlp.experts.168.gate_proj", + "model.layers.69.mlp.experts.168.up_proj", + "model.layers.69.mlp.experts.169.down_proj", + "model.layers.69.mlp.experts.169.gate_proj", + "model.layers.69.mlp.experts.169.up_proj", + "model.layers.69.mlp.experts.17.down_proj", + "model.layers.69.mlp.experts.17.gate_proj", + "model.layers.69.mlp.experts.17.up_proj", + "model.layers.69.mlp.experts.170.down_proj", + "model.layers.69.mlp.experts.170.gate_proj", + "model.layers.69.mlp.experts.170.up_proj", + "model.layers.69.mlp.experts.171.down_proj", + "model.layers.69.mlp.experts.171.gate_proj", + "model.layers.69.mlp.experts.171.up_proj", + "model.layers.69.mlp.experts.172.down_proj", + "model.layers.69.mlp.experts.172.gate_proj", + "model.layers.69.mlp.experts.172.up_proj", + "model.layers.69.mlp.experts.173.down_proj", + "model.layers.69.mlp.experts.173.gate_proj", + "model.layers.69.mlp.experts.173.up_proj", + "model.layers.69.mlp.experts.174.down_proj", + "model.layers.69.mlp.experts.174.gate_proj", + "model.layers.69.mlp.experts.174.up_proj", + "model.layers.69.mlp.experts.175.down_proj", + "model.layers.69.mlp.experts.175.gate_proj", + "model.layers.69.mlp.experts.175.up_proj", + "model.layers.69.mlp.experts.176.down_proj", + "model.layers.69.mlp.experts.176.gate_proj", + "model.layers.69.mlp.experts.176.up_proj", + "model.layers.69.mlp.experts.177.down_proj", + "model.layers.69.mlp.experts.177.gate_proj", + "model.layers.69.mlp.experts.177.up_proj", + "model.layers.69.mlp.experts.178.down_proj", + "model.layers.69.mlp.experts.178.gate_proj", + "model.layers.69.mlp.experts.178.up_proj", + "model.layers.69.mlp.experts.179.down_proj", + "model.layers.69.mlp.experts.179.gate_proj", + "model.layers.69.mlp.experts.179.up_proj", + "model.layers.69.mlp.experts.18.down_proj", + "model.layers.69.mlp.experts.18.gate_proj", + "model.layers.69.mlp.experts.18.up_proj", + "model.layers.69.mlp.experts.180.down_proj", + "model.layers.69.mlp.experts.180.gate_proj", + "model.layers.69.mlp.experts.180.up_proj", + "model.layers.69.mlp.experts.181.down_proj", + "model.layers.69.mlp.experts.181.gate_proj", + "model.layers.69.mlp.experts.181.up_proj", + "model.layers.69.mlp.experts.182.down_proj", + "model.layers.69.mlp.experts.182.gate_proj", + "model.layers.69.mlp.experts.182.up_proj", + "model.layers.69.mlp.experts.183.down_proj", + "model.layers.69.mlp.experts.183.gate_proj", + "model.layers.69.mlp.experts.183.up_proj", + "model.layers.69.mlp.experts.184.down_proj", + "model.layers.69.mlp.experts.184.gate_proj", + "model.layers.69.mlp.experts.184.up_proj", + "model.layers.69.mlp.experts.185.down_proj", + "model.layers.69.mlp.experts.185.gate_proj", + "model.layers.69.mlp.experts.185.up_proj", + "model.layers.69.mlp.experts.186.down_proj", + "model.layers.69.mlp.experts.186.gate_proj", + "model.layers.69.mlp.experts.186.up_proj", + "model.layers.69.mlp.experts.187.down_proj", + "model.layers.69.mlp.experts.187.gate_proj", + "model.layers.69.mlp.experts.187.up_proj", + "model.layers.69.mlp.experts.188.down_proj", + "model.layers.69.mlp.experts.188.gate_proj", + "model.layers.69.mlp.experts.188.up_proj", + "model.layers.69.mlp.experts.189.down_proj", + "model.layers.69.mlp.experts.189.gate_proj", + "model.layers.69.mlp.experts.189.up_proj", + "model.layers.69.mlp.experts.19.down_proj", + "model.layers.69.mlp.experts.19.gate_proj", + "model.layers.69.mlp.experts.19.up_proj", + "model.layers.69.mlp.experts.190.down_proj", + "model.layers.69.mlp.experts.190.gate_proj", + "model.layers.69.mlp.experts.190.up_proj", + "model.layers.69.mlp.experts.191.down_proj", + "model.layers.69.mlp.experts.191.gate_proj", + "model.layers.69.mlp.experts.191.up_proj", + "model.layers.69.mlp.experts.192.down_proj", + "model.layers.69.mlp.experts.192.gate_proj", + "model.layers.69.mlp.experts.192.up_proj", + "model.layers.69.mlp.experts.193.down_proj", + "model.layers.69.mlp.experts.193.gate_proj", + "model.layers.69.mlp.experts.193.up_proj", + "model.layers.69.mlp.experts.194.down_proj", + "model.layers.69.mlp.experts.194.gate_proj", + "model.layers.69.mlp.experts.194.up_proj", + "model.layers.69.mlp.experts.195.down_proj", + "model.layers.69.mlp.experts.195.gate_proj", + "model.layers.69.mlp.experts.195.up_proj", + "model.layers.69.mlp.experts.196.down_proj", + "model.layers.69.mlp.experts.196.gate_proj", + "model.layers.69.mlp.experts.196.up_proj", + "model.layers.69.mlp.experts.197.down_proj", + "model.layers.69.mlp.experts.197.gate_proj", + "model.layers.69.mlp.experts.197.up_proj", + "model.layers.69.mlp.experts.198.down_proj", + "model.layers.69.mlp.experts.198.gate_proj", + "model.layers.69.mlp.experts.198.up_proj", + "model.layers.69.mlp.experts.199.down_proj", + "model.layers.69.mlp.experts.199.gate_proj", + "model.layers.69.mlp.experts.199.up_proj", + "model.layers.69.mlp.experts.2.down_proj", + "model.layers.69.mlp.experts.2.gate_proj", + "model.layers.69.mlp.experts.2.up_proj", + "model.layers.69.mlp.experts.20.down_proj", + "model.layers.69.mlp.experts.20.gate_proj", + "model.layers.69.mlp.experts.20.up_proj", + "model.layers.69.mlp.experts.200.down_proj", + "model.layers.69.mlp.experts.200.gate_proj", + "model.layers.69.mlp.experts.200.up_proj", + "model.layers.69.mlp.experts.201.down_proj", + "model.layers.69.mlp.experts.201.gate_proj", + "model.layers.69.mlp.experts.201.up_proj", + "model.layers.69.mlp.experts.202.down_proj", + "model.layers.69.mlp.experts.202.gate_proj", + "model.layers.69.mlp.experts.202.up_proj", + "model.layers.69.mlp.experts.203.down_proj", + "model.layers.69.mlp.experts.203.gate_proj", + "model.layers.69.mlp.experts.203.up_proj", + "model.layers.69.mlp.experts.204.down_proj", + "model.layers.69.mlp.experts.204.gate_proj", + "model.layers.69.mlp.experts.204.up_proj", + "model.layers.69.mlp.experts.205.down_proj", + "model.layers.69.mlp.experts.205.gate_proj", + "model.layers.69.mlp.experts.205.up_proj", + "model.layers.69.mlp.experts.206.down_proj", + "model.layers.69.mlp.experts.206.gate_proj", + "model.layers.69.mlp.experts.206.up_proj", + "model.layers.69.mlp.experts.207.down_proj", + "model.layers.69.mlp.experts.207.gate_proj", + "model.layers.69.mlp.experts.207.up_proj", + "model.layers.69.mlp.experts.208.down_proj", + "model.layers.69.mlp.experts.208.gate_proj", + "model.layers.69.mlp.experts.208.up_proj", + "model.layers.69.mlp.experts.209.down_proj", + "model.layers.69.mlp.experts.209.gate_proj", + "model.layers.69.mlp.experts.209.up_proj", + "model.layers.69.mlp.experts.21.down_proj", + "model.layers.69.mlp.experts.21.gate_proj", + "model.layers.69.mlp.experts.21.up_proj", + "model.layers.69.mlp.experts.210.down_proj", + "model.layers.69.mlp.experts.210.gate_proj", + "model.layers.69.mlp.experts.210.up_proj", + "model.layers.69.mlp.experts.211.down_proj", + "model.layers.69.mlp.experts.211.gate_proj", + "model.layers.69.mlp.experts.211.up_proj", + "model.layers.69.mlp.experts.212.down_proj", + "model.layers.69.mlp.experts.212.gate_proj", + "model.layers.69.mlp.experts.212.up_proj", + "model.layers.69.mlp.experts.213.down_proj", + "model.layers.69.mlp.experts.213.gate_proj", + "model.layers.69.mlp.experts.213.up_proj", + "model.layers.69.mlp.experts.214.down_proj", + "model.layers.69.mlp.experts.214.gate_proj", + "model.layers.69.mlp.experts.214.up_proj", + "model.layers.69.mlp.experts.215.down_proj", + "model.layers.69.mlp.experts.215.gate_proj", + "model.layers.69.mlp.experts.215.up_proj", + "model.layers.69.mlp.experts.216.down_proj", + "model.layers.69.mlp.experts.216.gate_proj", + "model.layers.69.mlp.experts.216.up_proj", + "model.layers.69.mlp.experts.217.down_proj", + "model.layers.69.mlp.experts.217.gate_proj", + "model.layers.69.mlp.experts.217.up_proj", + "model.layers.69.mlp.experts.218.down_proj", + "model.layers.69.mlp.experts.218.gate_proj", + "model.layers.69.mlp.experts.218.up_proj", + "model.layers.69.mlp.experts.219.down_proj", + "model.layers.69.mlp.experts.219.gate_proj", + "model.layers.69.mlp.experts.219.up_proj", + "model.layers.69.mlp.experts.22.down_proj", + "model.layers.69.mlp.experts.22.gate_proj", + "model.layers.69.mlp.experts.22.up_proj", + "model.layers.69.mlp.experts.220.down_proj", + "model.layers.69.mlp.experts.220.gate_proj", + "model.layers.69.mlp.experts.220.up_proj", + "model.layers.69.mlp.experts.221.down_proj", + "model.layers.69.mlp.experts.221.gate_proj", + "model.layers.69.mlp.experts.221.up_proj", + "model.layers.69.mlp.experts.222.down_proj", + "model.layers.69.mlp.experts.222.gate_proj", + "model.layers.69.mlp.experts.222.up_proj", + "model.layers.69.mlp.experts.223.down_proj", + "model.layers.69.mlp.experts.223.gate_proj", + "model.layers.69.mlp.experts.223.up_proj", + "model.layers.69.mlp.experts.224.down_proj", + "model.layers.69.mlp.experts.224.gate_proj", + "model.layers.69.mlp.experts.224.up_proj", + "model.layers.69.mlp.experts.225.down_proj", + "model.layers.69.mlp.experts.225.gate_proj", + "model.layers.69.mlp.experts.225.up_proj", + "model.layers.69.mlp.experts.226.down_proj", + "model.layers.69.mlp.experts.226.gate_proj", + "model.layers.69.mlp.experts.226.up_proj", + "model.layers.69.mlp.experts.227.down_proj", + "model.layers.69.mlp.experts.227.gate_proj", + "model.layers.69.mlp.experts.227.up_proj", + "model.layers.69.mlp.experts.228.down_proj", + "model.layers.69.mlp.experts.228.gate_proj", + "model.layers.69.mlp.experts.228.up_proj", + "model.layers.69.mlp.experts.229.down_proj", + "model.layers.69.mlp.experts.229.gate_proj", + "model.layers.69.mlp.experts.229.up_proj", + "model.layers.69.mlp.experts.23.down_proj", + "model.layers.69.mlp.experts.23.gate_proj", + "model.layers.69.mlp.experts.23.up_proj", + "model.layers.69.mlp.experts.230.down_proj", + "model.layers.69.mlp.experts.230.gate_proj", + "model.layers.69.mlp.experts.230.up_proj", + "model.layers.69.mlp.experts.231.down_proj", + "model.layers.69.mlp.experts.231.gate_proj", + "model.layers.69.mlp.experts.231.up_proj", + "model.layers.69.mlp.experts.232.down_proj", + "model.layers.69.mlp.experts.232.gate_proj", + "model.layers.69.mlp.experts.232.up_proj", + "model.layers.69.mlp.experts.233.down_proj", + "model.layers.69.mlp.experts.233.gate_proj", + "model.layers.69.mlp.experts.233.up_proj", + "model.layers.69.mlp.experts.234.down_proj", + "model.layers.69.mlp.experts.234.gate_proj", + "model.layers.69.mlp.experts.234.up_proj", + "model.layers.69.mlp.experts.235.down_proj", + "model.layers.69.mlp.experts.235.gate_proj", + "model.layers.69.mlp.experts.235.up_proj", + "model.layers.69.mlp.experts.236.down_proj", + "model.layers.69.mlp.experts.236.gate_proj", + "model.layers.69.mlp.experts.236.up_proj", + "model.layers.69.mlp.experts.237.down_proj", + "model.layers.69.mlp.experts.237.gate_proj", + "model.layers.69.mlp.experts.237.up_proj", + "model.layers.69.mlp.experts.238.down_proj", + "model.layers.69.mlp.experts.238.gate_proj", + "model.layers.69.mlp.experts.238.up_proj", + "model.layers.69.mlp.experts.239.down_proj", + "model.layers.69.mlp.experts.239.gate_proj", + "model.layers.69.mlp.experts.239.up_proj", + "model.layers.69.mlp.experts.24.down_proj", + "model.layers.69.mlp.experts.24.gate_proj", + "model.layers.69.mlp.experts.24.up_proj", + "model.layers.69.mlp.experts.240.down_proj", + "model.layers.69.mlp.experts.240.gate_proj", + "model.layers.69.mlp.experts.240.up_proj", + "model.layers.69.mlp.experts.241.down_proj", + "model.layers.69.mlp.experts.241.gate_proj", + "model.layers.69.mlp.experts.241.up_proj", + "model.layers.69.mlp.experts.242.down_proj", + "model.layers.69.mlp.experts.242.gate_proj", + "model.layers.69.mlp.experts.242.up_proj", + "model.layers.69.mlp.experts.243.down_proj", + "model.layers.69.mlp.experts.243.gate_proj", + "model.layers.69.mlp.experts.243.up_proj", + "model.layers.69.mlp.experts.244.down_proj", + "model.layers.69.mlp.experts.244.gate_proj", + "model.layers.69.mlp.experts.244.up_proj", + "model.layers.69.mlp.experts.245.down_proj", + "model.layers.69.mlp.experts.245.gate_proj", + "model.layers.69.mlp.experts.245.up_proj", + "model.layers.69.mlp.experts.246.down_proj", + "model.layers.69.mlp.experts.246.gate_proj", + "model.layers.69.mlp.experts.246.up_proj", + "model.layers.69.mlp.experts.247.down_proj", + "model.layers.69.mlp.experts.247.gate_proj", + "model.layers.69.mlp.experts.247.up_proj", + "model.layers.69.mlp.experts.248.down_proj", + "model.layers.69.mlp.experts.248.gate_proj", + "model.layers.69.mlp.experts.248.up_proj", + "model.layers.69.mlp.experts.249.down_proj", + "model.layers.69.mlp.experts.249.gate_proj", + "model.layers.69.mlp.experts.249.up_proj", + "model.layers.69.mlp.experts.25.down_proj", + "model.layers.69.mlp.experts.25.gate_proj", + "model.layers.69.mlp.experts.25.up_proj", + "model.layers.69.mlp.experts.250.down_proj", + "model.layers.69.mlp.experts.250.gate_proj", + "model.layers.69.mlp.experts.250.up_proj", + "model.layers.69.mlp.experts.251.down_proj", + "model.layers.69.mlp.experts.251.gate_proj", + "model.layers.69.mlp.experts.251.up_proj", + "model.layers.69.mlp.experts.252.down_proj", + "model.layers.69.mlp.experts.252.gate_proj", + "model.layers.69.mlp.experts.252.up_proj", + "model.layers.69.mlp.experts.253.down_proj", + "model.layers.69.mlp.experts.253.gate_proj", + "model.layers.69.mlp.experts.253.up_proj", + "model.layers.69.mlp.experts.254.down_proj", + "model.layers.69.mlp.experts.254.gate_proj", + "model.layers.69.mlp.experts.254.up_proj", + "model.layers.69.mlp.experts.255.down_proj", + "model.layers.69.mlp.experts.255.gate_proj", + "model.layers.69.mlp.experts.255.up_proj", + "model.layers.69.mlp.experts.26.down_proj", + "model.layers.69.mlp.experts.26.gate_proj", + "model.layers.69.mlp.experts.26.up_proj", + "model.layers.69.mlp.experts.27.down_proj", + "model.layers.69.mlp.experts.27.gate_proj", + "model.layers.69.mlp.experts.27.up_proj", + "model.layers.69.mlp.experts.28.down_proj", + "model.layers.69.mlp.experts.28.gate_proj", + "model.layers.69.mlp.experts.28.up_proj", + "model.layers.69.mlp.experts.29.down_proj", + "model.layers.69.mlp.experts.29.gate_proj", + "model.layers.69.mlp.experts.29.up_proj", + "model.layers.69.mlp.experts.3.down_proj", + "model.layers.69.mlp.experts.3.gate_proj", + "model.layers.69.mlp.experts.3.up_proj", + "model.layers.69.mlp.experts.30.down_proj", + "model.layers.69.mlp.experts.30.gate_proj", + "model.layers.69.mlp.experts.30.up_proj", + "model.layers.69.mlp.experts.31.down_proj", + "model.layers.69.mlp.experts.31.gate_proj", + "model.layers.69.mlp.experts.31.up_proj", + "model.layers.69.mlp.experts.32.down_proj", + "model.layers.69.mlp.experts.32.gate_proj", + "model.layers.69.mlp.experts.32.up_proj", + "model.layers.69.mlp.experts.33.down_proj", + "model.layers.69.mlp.experts.33.gate_proj", + "model.layers.69.mlp.experts.33.up_proj", + "model.layers.69.mlp.experts.34.down_proj", + "model.layers.69.mlp.experts.34.gate_proj", + "model.layers.69.mlp.experts.34.up_proj", + "model.layers.69.mlp.experts.35.down_proj", + "model.layers.69.mlp.experts.35.gate_proj", + "model.layers.69.mlp.experts.35.up_proj", + "model.layers.69.mlp.experts.36.down_proj", + "model.layers.69.mlp.experts.36.gate_proj", + "model.layers.69.mlp.experts.36.up_proj", + "model.layers.69.mlp.experts.37.down_proj", + "model.layers.69.mlp.experts.37.gate_proj", + "model.layers.69.mlp.experts.37.up_proj", + "model.layers.69.mlp.experts.38.down_proj", + "model.layers.69.mlp.experts.38.gate_proj", + "model.layers.69.mlp.experts.38.up_proj", + "model.layers.69.mlp.experts.39.down_proj", + "model.layers.69.mlp.experts.39.gate_proj", + "model.layers.69.mlp.experts.39.up_proj", + "model.layers.69.mlp.experts.4.down_proj", + "model.layers.69.mlp.experts.4.gate_proj", + "model.layers.69.mlp.experts.4.up_proj", + "model.layers.69.mlp.experts.40.down_proj", + "model.layers.69.mlp.experts.40.gate_proj", + "model.layers.69.mlp.experts.40.up_proj", + "model.layers.69.mlp.experts.41.down_proj", + "model.layers.69.mlp.experts.41.gate_proj", + "model.layers.69.mlp.experts.41.up_proj", + "model.layers.69.mlp.experts.42.down_proj", + "model.layers.69.mlp.experts.42.gate_proj", + "model.layers.69.mlp.experts.42.up_proj", + "model.layers.69.mlp.experts.43.down_proj", + "model.layers.69.mlp.experts.43.gate_proj", + "model.layers.69.mlp.experts.43.up_proj", + "model.layers.69.mlp.experts.44.down_proj", + "model.layers.69.mlp.experts.44.gate_proj", + "model.layers.69.mlp.experts.44.up_proj", + "model.layers.69.mlp.experts.45.down_proj", + "model.layers.69.mlp.experts.45.gate_proj", + "model.layers.69.mlp.experts.45.up_proj", + "model.layers.69.mlp.experts.46.down_proj", + "model.layers.69.mlp.experts.46.gate_proj", + "model.layers.69.mlp.experts.46.up_proj", + "model.layers.69.mlp.experts.47.down_proj", + "model.layers.69.mlp.experts.47.gate_proj", + "model.layers.69.mlp.experts.47.up_proj", + "model.layers.69.mlp.experts.48.down_proj", + "model.layers.69.mlp.experts.48.gate_proj", + "model.layers.69.mlp.experts.48.up_proj", + "model.layers.69.mlp.experts.49.down_proj", + "model.layers.69.mlp.experts.49.gate_proj", + "model.layers.69.mlp.experts.49.up_proj", + "model.layers.69.mlp.experts.5.down_proj", + "model.layers.69.mlp.experts.5.gate_proj", + "model.layers.69.mlp.experts.5.up_proj", + "model.layers.69.mlp.experts.50.down_proj", + "model.layers.69.mlp.experts.50.gate_proj", + "model.layers.69.mlp.experts.50.up_proj", + "model.layers.69.mlp.experts.51.down_proj", + "model.layers.69.mlp.experts.51.gate_proj", + "model.layers.69.mlp.experts.51.up_proj", + "model.layers.69.mlp.experts.52.down_proj", + "model.layers.69.mlp.experts.52.gate_proj", + "model.layers.69.mlp.experts.52.up_proj", + "model.layers.69.mlp.experts.53.down_proj", + "model.layers.69.mlp.experts.53.gate_proj", + "model.layers.69.mlp.experts.53.up_proj", + "model.layers.69.mlp.experts.54.down_proj", + "model.layers.69.mlp.experts.54.gate_proj", + "model.layers.69.mlp.experts.54.up_proj", + "model.layers.69.mlp.experts.55.down_proj", + "model.layers.69.mlp.experts.55.gate_proj", + "model.layers.69.mlp.experts.55.up_proj", + "model.layers.69.mlp.experts.56.down_proj", + "model.layers.69.mlp.experts.56.gate_proj", + "model.layers.69.mlp.experts.56.up_proj", + "model.layers.69.mlp.experts.57.down_proj", + "model.layers.69.mlp.experts.57.gate_proj", + "model.layers.69.mlp.experts.57.up_proj", + "model.layers.69.mlp.experts.58.down_proj", + "model.layers.69.mlp.experts.58.gate_proj", + "model.layers.69.mlp.experts.58.up_proj", + "model.layers.69.mlp.experts.59.down_proj", + "model.layers.69.mlp.experts.59.gate_proj", + "model.layers.69.mlp.experts.59.up_proj", + "model.layers.69.mlp.experts.6.down_proj", + "model.layers.69.mlp.experts.6.gate_proj", + "model.layers.69.mlp.experts.6.up_proj", + "model.layers.69.mlp.experts.60.down_proj", + "model.layers.69.mlp.experts.60.gate_proj", + "model.layers.69.mlp.experts.60.up_proj", + "model.layers.69.mlp.experts.61.down_proj", + "model.layers.69.mlp.experts.61.gate_proj", + "model.layers.69.mlp.experts.61.up_proj", + "model.layers.69.mlp.experts.62.down_proj", + "model.layers.69.mlp.experts.62.gate_proj", + "model.layers.69.mlp.experts.62.up_proj", + "model.layers.69.mlp.experts.63.down_proj", + "model.layers.69.mlp.experts.63.gate_proj", + "model.layers.69.mlp.experts.63.up_proj", + "model.layers.69.mlp.experts.64.down_proj", + "model.layers.69.mlp.experts.64.gate_proj", + "model.layers.69.mlp.experts.64.up_proj", + "model.layers.69.mlp.experts.65.down_proj", + "model.layers.69.mlp.experts.65.gate_proj", + "model.layers.69.mlp.experts.65.up_proj", + "model.layers.69.mlp.experts.66.down_proj", + "model.layers.69.mlp.experts.66.gate_proj", + "model.layers.69.mlp.experts.66.up_proj", + "model.layers.69.mlp.experts.67.down_proj", + "model.layers.69.mlp.experts.67.gate_proj", + "model.layers.69.mlp.experts.67.up_proj", + "model.layers.69.mlp.experts.68.down_proj", + "model.layers.69.mlp.experts.68.gate_proj", + "model.layers.69.mlp.experts.68.up_proj", + "model.layers.69.mlp.experts.69.down_proj", + "model.layers.69.mlp.experts.69.gate_proj", + "model.layers.69.mlp.experts.69.up_proj", + "model.layers.69.mlp.experts.7.down_proj", + "model.layers.69.mlp.experts.7.gate_proj", + "model.layers.69.mlp.experts.7.up_proj", + "model.layers.69.mlp.experts.70.down_proj", + "model.layers.69.mlp.experts.70.gate_proj", + "model.layers.69.mlp.experts.70.up_proj", + "model.layers.69.mlp.experts.71.down_proj", + "model.layers.69.mlp.experts.71.gate_proj", + "model.layers.69.mlp.experts.71.up_proj", + "model.layers.69.mlp.experts.72.down_proj", + "model.layers.69.mlp.experts.72.gate_proj", + "model.layers.69.mlp.experts.72.up_proj", + "model.layers.69.mlp.experts.73.down_proj", + "model.layers.69.mlp.experts.73.gate_proj", + "model.layers.69.mlp.experts.73.up_proj", + "model.layers.69.mlp.experts.74.down_proj", + "model.layers.69.mlp.experts.74.gate_proj", + "model.layers.69.mlp.experts.74.up_proj", + "model.layers.69.mlp.experts.75.down_proj", + "model.layers.69.mlp.experts.75.gate_proj", + "model.layers.69.mlp.experts.75.up_proj", + "model.layers.69.mlp.experts.76.down_proj", + "model.layers.69.mlp.experts.76.gate_proj", + "model.layers.69.mlp.experts.76.up_proj", + "model.layers.69.mlp.experts.77.down_proj", + "model.layers.69.mlp.experts.77.gate_proj", + "model.layers.69.mlp.experts.77.up_proj", + "model.layers.69.mlp.experts.78.down_proj", + "model.layers.69.mlp.experts.78.gate_proj", + "model.layers.69.mlp.experts.78.up_proj", + "model.layers.69.mlp.experts.79.down_proj", + "model.layers.69.mlp.experts.79.gate_proj", + "model.layers.69.mlp.experts.79.up_proj", + "model.layers.69.mlp.experts.8.down_proj", + "model.layers.69.mlp.experts.8.gate_proj", + "model.layers.69.mlp.experts.8.up_proj", + "model.layers.69.mlp.experts.80.down_proj", + "model.layers.69.mlp.experts.80.gate_proj", + "model.layers.69.mlp.experts.80.up_proj", + "model.layers.69.mlp.experts.81.down_proj", + "model.layers.69.mlp.experts.81.gate_proj", + "model.layers.69.mlp.experts.81.up_proj", + "model.layers.69.mlp.experts.82.down_proj", + "model.layers.69.mlp.experts.82.gate_proj", + "model.layers.69.mlp.experts.82.up_proj", + "model.layers.69.mlp.experts.83.down_proj", + "model.layers.69.mlp.experts.83.gate_proj", + "model.layers.69.mlp.experts.83.up_proj", + "model.layers.69.mlp.experts.84.down_proj", + "model.layers.69.mlp.experts.84.gate_proj", + "model.layers.69.mlp.experts.84.up_proj", + "model.layers.69.mlp.experts.85.down_proj", + "model.layers.69.mlp.experts.85.gate_proj", + "model.layers.69.mlp.experts.85.up_proj", + "model.layers.69.mlp.experts.86.down_proj", + "model.layers.69.mlp.experts.86.gate_proj", + "model.layers.69.mlp.experts.86.up_proj", + "model.layers.69.mlp.experts.87.down_proj", + "model.layers.69.mlp.experts.87.gate_proj", + "model.layers.69.mlp.experts.87.up_proj", + "model.layers.69.mlp.experts.88.down_proj", + "model.layers.69.mlp.experts.88.gate_proj", + "model.layers.69.mlp.experts.88.up_proj", + "model.layers.69.mlp.experts.89.down_proj", + "model.layers.69.mlp.experts.89.gate_proj", + "model.layers.69.mlp.experts.89.up_proj", + "model.layers.69.mlp.experts.9.down_proj", + "model.layers.69.mlp.experts.9.gate_proj", + "model.layers.69.mlp.experts.9.up_proj", + "model.layers.69.mlp.experts.90.down_proj", + "model.layers.69.mlp.experts.90.gate_proj", + "model.layers.69.mlp.experts.90.up_proj", + "model.layers.69.mlp.experts.91.down_proj", + "model.layers.69.mlp.experts.91.gate_proj", + "model.layers.69.mlp.experts.91.up_proj", + "model.layers.69.mlp.experts.92.down_proj", + "model.layers.69.mlp.experts.92.gate_proj", + "model.layers.69.mlp.experts.92.up_proj", + "model.layers.69.mlp.experts.93.down_proj", + "model.layers.69.mlp.experts.93.gate_proj", + "model.layers.69.mlp.experts.93.up_proj", + "model.layers.69.mlp.experts.94.down_proj", + "model.layers.69.mlp.experts.94.gate_proj", + "model.layers.69.mlp.experts.94.up_proj", + "model.layers.69.mlp.experts.95.down_proj", + "model.layers.69.mlp.experts.95.gate_proj", + "model.layers.69.mlp.experts.95.up_proj", + "model.layers.69.mlp.experts.96.down_proj", + "model.layers.69.mlp.experts.96.gate_proj", + "model.layers.69.mlp.experts.96.up_proj", + "model.layers.69.mlp.experts.97.down_proj", + "model.layers.69.mlp.experts.97.gate_proj", + "model.layers.69.mlp.experts.97.up_proj", + "model.layers.69.mlp.experts.98.down_proj", + "model.layers.69.mlp.experts.98.gate_proj", + "model.layers.69.mlp.experts.98.up_proj", + "model.layers.69.mlp.experts.99.down_proj", + "model.layers.69.mlp.experts.99.gate_proj", + "model.layers.69.mlp.experts.99.up_proj", + "model.layers.7.mlp.experts.0.down_proj", + "model.layers.7.mlp.experts.0.gate_proj", + "model.layers.7.mlp.experts.0.up_proj", + "model.layers.7.mlp.experts.1.down_proj", + "model.layers.7.mlp.experts.1.gate_proj", + "model.layers.7.mlp.experts.1.up_proj", + "model.layers.7.mlp.experts.10.down_proj", + "model.layers.7.mlp.experts.10.gate_proj", + "model.layers.7.mlp.experts.10.up_proj", + "model.layers.7.mlp.experts.100.down_proj", + "model.layers.7.mlp.experts.100.gate_proj", + "model.layers.7.mlp.experts.100.up_proj", + "model.layers.7.mlp.experts.101.down_proj", + "model.layers.7.mlp.experts.101.gate_proj", + "model.layers.7.mlp.experts.101.up_proj", + "model.layers.7.mlp.experts.102.down_proj", + "model.layers.7.mlp.experts.102.gate_proj", + "model.layers.7.mlp.experts.102.up_proj", + "model.layers.7.mlp.experts.103.down_proj", + "model.layers.7.mlp.experts.103.gate_proj", + "model.layers.7.mlp.experts.103.up_proj", + "model.layers.7.mlp.experts.104.down_proj", + "model.layers.7.mlp.experts.104.gate_proj", + "model.layers.7.mlp.experts.104.up_proj", + "model.layers.7.mlp.experts.105.down_proj", + "model.layers.7.mlp.experts.105.gate_proj", + "model.layers.7.mlp.experts.105.up_proj", + "model.layers.7.mlp.experts.106.down_proj", + "model.layers.7.mlp.experts.106.gate_proj", + "model.layers.7.mlp.experts.106.up_proj", + "model.layers.7.mlp.experts.107.down_proj", + "model.layers.7.mlp.experts.107.gate_proj", + "model.layers.7.mlp.experts.107.up_proj", + "model.layers.7.mlp.experts.108.down_proj", + "model.layers.7.mlp.experts.108.gate_proj", + "model.layers.7.mlp.experts.108.up_proj", + "model.layers.7.mlp.experts.109.down_proj", + "model.layers.7.mlp.experts.109.gate_proj", + "model.layers.7.mlp.experts.109.up_proj", + "model.layers.7.mlp.experts.11.down_proj", + "model.layers.7.mlp.experts.11.gate_proj", + "model.layers.7.mlp.experts.11.up_proj", + "model.layers.7.mlp.experts.110.down_proj", + "model.layers.7.mlp.experts.110.gate_proj", + "model.layers.7.mlp.experts.110.up_proj", + "model.layers.7.mlp.experts.111.down_proj", + "model.layers.7.mlp.experts.111.gate_proj", + "model.layers.7.mlp.experts.111.up_proj", + "model.layers.7.mlp.experts.112.down_proj", + "model.layers.7.mlp.experts.112.gate_proj", + "model.layers.7.mlp.experts.112.up_proj", + "model.layers.7.mlp.experts.113.down_proj", + "model.layers.7.mlp.experts.113.gate_proj", + "model.layers.7.mlp.experts.113.up_proj", + "model.layers.7.mlp.experts.114.down_proj", + "model.layers.7.mlp.experts.114.gate_proj", + "model.layers.7.mlp.experts.114.up_proj", + "model.layers.7.mlp.experts.115.down_proj", + "model.layers.7.mlp.experts.115.gate_proj", + "model.layers.7.mlp.experts.115.up_proj", + "model.layers.7.mlp.experts.116.down_proj", + "model.layers.7.mlp.experts.116.gate_proj", + "model.layers.7.mlp.experts.116.up_proj", + "model.layers.7.mlp.experts.117.down_proj", + "model.layers.7.mlp.experts.117.gate_proj", + "model.layers.7.mlp.experts.117.up_proj", + "model.layers.7.mlp.experts.118.down_proj", + "model.layers.7.mlp.experts.118.gate_proj", + "model.layers.7.mlp.experts.118.up_proj", + "model.layers.7.mlp.experts.119.down_proj", + "model.layers.7.mlp.experts.119.gate_proj", + "model.layers.7.mlp.experts.119.up_proj", + "model.layers.7.mlp.experts.12.down_proj", + "model.layers.7.mlp.experts.12.gate_proj", + "model.layers.7.mlp.experts.12.up_proj", + "model.layers.7.mlp.experts.120.down_proj", + "model.layers.7.mlp.experts.120.gate_proj", + "model.layers.7.mlp.experts.120.up_proj", + "model.layers.7.mlp.experts.121.down_proj", + "model.layers.7.mlp.experts.121.gate_proj", + "model.layers.7.mlp.experts.121.up_proj", + "model.layers.7.mlp.experts.122.down_proj", + "model.layers.7.mlp.experts.122.gate_proj", + "model.layers.7.mlp.experts.122.up_proj", + "model.layers.7.mlp.experts.123.down_proj", + "model.layers.7.mlp.experts.123.gate_proj", + "model.layers.7.mlp.experts.123.up_proj", + "model.layers.7.mlp.experts.124.down_proj", + "model.layers.7.mlp.experts.124.gate_proj", + "model.layers.7.mlp.experts.124.up_proj", + "model.layers.7.mlp.experts.125.down_proj", + "model.layers.7.mlp.experts.125.gate_proj", + "model.layers.7.mlp.experts.125.up_proj", + "model.layers.7.mlp.experts.126.down_proj", + "model.layers.7.mlp.experts.126.gate_proj", + "model.layers.7.mlp.experts.126.up_proj", + "model.layers.7.mlp.experts.127.down_proj", + "model.layers.7.mlp.experts.127.gate_proj", + "model.layers.7.mlp.experts.127.up_proj", + "model.layers.7.mlp.experts.128.down_proj", + "model.layers.7.mlp.experts.128.gate_proj", + "model.layers.7.mlp.experts.128.up_proj", + "model.layers.7.mlp.experts.129.down_proj", + "model.layers.7.mlp.experts.129.gate_proj", + "model.layers.7.mlp.experts.129.up_proj", + "model.layers.7.mlp.experts.13.down_proj", + "model.layers.7.mlp.experts.13.gate_proj", + "model.layers.7.mlp.experts.13.up_proj", + "model.layers.7.mlp.experts.130.down_proj", + "model.layers.7.mlp.experts.130.gate_proj", + "model.layers.7.mlp.experts.130.up_proj", + "model.layers.7.mlp.experts.131.down_proj", + "model.layers.7.mlp.experts.131.gate_proj", + "model.layers.7.mlp.experts.131.up_proj", + "model.layers.7.mlp.experts.132.down_proj", + "model.layers.7.mlp.experts.132.gate_proj", + "model.layers.7.mlp.experts.132.up_proj", + "model.layers.7.mlp.experts.133.down_proj", + "model.layers.7.mlp.experts.133.gate_proj", + "model.layers.7.mlp.experts.133.up_proj", + "model.layers.7.mlp.experts.134.down_proj", + "model.layers.7.mlp.experts.134.gate_proj", + "model.layers.7.mlp.experts.134.up_proj", + "model.layers.7.mlp.experts.135.down_proj", + "model.layers.7.mlp.experts.135.gate_proj", + "model.layers.7.mlp.experts.135.up_proj", + "model.layers.7.mlp.experts.136.down_proj", + "model.layers.7.mlp.experts.136.gate_proj", + "model.layers.7.mlp.experts.136.up_proj", + "model.layers.7.mlp.experts.137.down_proj", + "model.layers.7.mlp.experts.137.gate_proj", + "model.layers.7.mlp.experts.137.up_proj", + "model.layers.7.mlp.experts.138.down_proj", + "model.layers.7.mlp.experts.138.gate_proj", + "model.layers.7.mlp.experts.138.up_proj", + "model.layers.7.mlp.experts.139.down_proj", + "model.layers.7.mlp.experts.139.gate_proj", + "model.layers.7.mlp.experts.139.up_proj", + "model.layers.7.mlp.experts.14.down_proj", + "model.layers.7.mlp.experts.14.gate_proj", + "model.layers.7.mlp.experts.14.up_proj", + "model.layers.7.mlp.experts.140.down_proj", + "model.layers.7.mlp.experts.140.gate_proj", + "model.layers.7.mlp.experts.140.up_proj", + "model.layers.7.mlp.experts.141.down_proj", + "model.layers.7.mlp.experts.141.gate_proj", + "model.layers.7.mlp.experts.141.up_proj", + "model.layers.7.mlp.experts.142.down_proj", + "model.layers.7.mlp.experts.142.gate_proj", + "model.layers.7.mlp.experts.142.up_proj", + "model.layers.7.mlp.experts.143.down_proj", + "model.layers.7.mlp.experts.143.gate_proj", + "model.layers.7.mlp.experts.143.up_proj", + "model.layers.7.mlp.experts.144.down_proj", + "model.layers.7.mlp.experts.144.gate_proj", + "model.layers.7.mlp.experts.144.up_proj", + "model.layers.7.mlp.experts.145.down_proj", + "model.layers.7.mlp.experts.145.gate_proj", + "model.layers.7.mlp.experts.145.up_proj", + "model.layers.7.mlp.experts.146.down_proj", + "model.layers.7.mlp.experts.146.gate_proj", + "model.layers.7.mlp.experts.146.up_proj", + "model.layers.7.mlp.experts.147.down_proj", + "model.layers.7.mlp.experts.147.gate_proj", + "model.layers.7.mlp.experts.147.up_proj", + "model.layers.7.mlp.experts.148.down_proj", + "model.layers.7.mlp.experts.148.gate_proj", + "model.layers.7.mlp.experts.148.up_proj", + "model.layers.7.mlp.experts.149.down_proj", + "model.layers.7.mlp.experts.149.gate_proj", + "model.layers.7.mlp.experts.149.up_proj", + "model.layers.7.mlp.experts.15.down_proj", + "model.layers.7.mlp.experts.15.gate_proj", + "model.layers.7.mlp.experts.15.up_proj", + "model.layers.7.mlp.experts.150.down_proj", + "model.layers.7.mlp.experts.150.gate_proj", + "model.layers.7.mlp.experts.150.up_proj", + "model.layers.7.mlp.experts.151.down_proj", + "model.layers.7.mlp.experts.151.gate_proj", + "model.layers.7.mlp.experts.151.up_proj", + "model.layers.7.mlp.experts.152.down_proj", + "model.layers.7.mlp.experts.152.gate_proj", + "model.layers.7.mlp.experts.152.up_proj", + "model.layers.7.mlp.experts.153.down_proj", + "model.layers.7.mlp.experts.153.gate_proj", + "model.layers.7.mlp.experts.153.up_proj", + "model.layers.7.mlp.experts.154.down_proj", + "model.layers.7.mlp.experts.154.gate_proj", + "model.layers.7.mlp.experts.154.up_proj", + "model.layers.7.mlp.experts.155.down_proj", + "model.layers.7.mlp.experts.155.gate_proj", + "model.layers.7.mlp.experts.155.up_proj", + "model.layers.7.mlp.experts.156.down_proj", + "model.layers.7.mlp.experts.156.gate_proj", + "model.layers.7.mlp.experts.156.up_proj", + "model.layers.7.mlp.experts.157.down_proj", + "model.layers.7.mlp.experts.157.gate_proj", + "model.layers.7.mlp.experts.157.up_proj", + "model.layers.7.mlp.experts.158.down_proj", + "model.layers.7.mlp.experts.158.gate_proj", + "model.layers.7.mlp.experts.158.up_proj", + "model.layers.7.mlp.experts.159.down_proj", + "model.layers.7.mlp.experts.159.gate_proj", + "model.layers.7.mlp.experts.159.up_proj", + "model.layers.7.mlp.experts.16.down_proj", + "model.layers.7.mlp.experts.16.gate_proj", + "model.layers.7.mlp.experts.16.up_proj", + "model.layers.7.mlp.experts.160.down_proj", + "model.layers.7.mlp.experts.160.gate_proj", + "model.layers.7.mlp.experts.160.up_proj", + "model.layers.7.mlp.experts.161.down_proj", + "model.layers.7.mlp.experts.161.gate_proj", + "model.layers.7.mlp.experts.161.up_proj", + "model.layers.7.mlp.experts.162.down_proj", + "model.layers.7.mlp.experts.162.gate_proj", + "model.layers.7.mlp.experts.162.up_proj", + "model.layers.7.mlp.experts.163.down_proj", + "model.layers.7.mlp.experts.163.gate_proj", + "model.layers.7.mlp.experts.163.up_proj", + "model.layers.7.mlp.experts.164.down_proj", + "model.layers.7.mlp.experts.164.gate_proj", + "model.layers.7.mlp.experts.164.up_proj", + "model.layers.7.mlp.experts.165.down_proj", + "model.layers.7.mlp.experts.165.gate_proj", + "model.layers.7.mlp.experts.165.up_proj", + "model.layers.7.mlp.experts.166.down_proj", + "model.layers.7.mlp.experts.166.gate_proj", + "model.layers.7.mlp.experts.166.up_proj", + "model.layers.7.mlp.experts.167.down_proj", + "model.layers.7.mlp.experts.167.gate_proj", + "model.layers.7.mlp.experts.167.up_proj", + "model.layers.7.mlp.experts.168.down_proj", + "model.layers.7.mlp.experts.168.gate_proj", + "model.layers.7.mlp.experts.168.up_proj", + "model.layers.7.mlp.experts.169.down_proj", + "model.layers.7.mlp.experts.169.gate_proj", + "model.layers.7.mlp.experts.169.up_proj", + "model.layers.7.mlp.experts.17.down_proj", + "model.layers.7.mlp.experts.17.gate_proj", + "model.layers.7.mlp.experts.17.up_proj", + "model.layers.7.mlp.experts.170.down_proj", + "model.layers.7.mlp.experts.170.gate_proj", + "model.layers.7.mlp.experts.170.up_proj", + "model.layers.7.mlp.experts.171.down_proj", + "model.layers.7.mlp.experts.171.gate_proj", + "model.layers.7.mlp.experts.171.up_proj", + "model.layers.7.mlp.experts.172.down_proj", + "model.layers.7.mlp.experts.172.gate_proj", + "model.layers.7.mlp.experts.172.up_proj", + "model.layers.7.mlp.experts.173.down_proj", + "model.layers.7.mlp.experts.173.gate_proj", + "model.layers.7.mlp.experts.173.up_proj", + "model.layers.7.mlp.experts.174.down_proj", + "model.layers.7.mlp.experts.174.gate_proj", + "model.layers.7.mlp.experts.174.up_proj", + "model.layers.7.mlp.experts.175.down_proj", + "model.layers.7.mlp.experts.175.gate_proj", + "model.layers.7.mlp.experts.175.up_proj", + "model.layers.7.mlp.experts.176.down_proj", + "model.layers.7.mlp.experts.176.gate_proj", + "model.layers.7.mlp.experts.176.up_proj", + "model.layers.7.mlp.experts.177.down_proj", + "model.layers.7.mlp.experts.177.gate_proj", + "model.layers.7.mlp.experts.177.up_proj", + "model.layers.7.mlp.experts.178.down_proj", + "model.layers.7.mlp.experts.178.gate_proj", + "model.layers.7.mlp.experts.178.up_proj", + "model.layers.7.mlp.experts.179.down_proj", + "model.layers.7.mlp.experts.179.gate_proj", + "model.layers.7.mlp.experts.179.up_proj", + "model.layers.7.mlp.experts.18.down_proj", + "model.layers.7.mlp.experts.18.gate_proj", + "model.layers.7.mlp.experts.18.up_proj", + "model.layers.7.mlp.experts.180.down_proj", + "model.layers.7.mlp.experts.180.gate_proj", + "model.layers.7.mlp.experts.180.up_proj", + "model.layers.7.mlp.experts.181.down_proj", + "model.layers.7.mlp.experts.181.gate_proj", + "model.layers.7.mlp.experts.181.up_proj", + "model.layers.7.mlp.experts.182.down_proj", + "model.layers.7.mlp.experts.182.gate_proj", + "model.layers.7.mlp.experts.182.up_proj", + "model.layers.7.mlp.experts.183.down_proj", + "model.layers.7.mlp.experts.183.gate_proj", + "model.layers.7.mlp.experts.183.up_proj", + "model.layers.7.mlp.experts.184.down_proj", + "model.layers.7.mlp.experts.184.gate_proj", + "model.layers.7.mlp.experts.184.up_proj", + "model.layers.7.mlp.experts.185.down_proj", + "model.layers.7.mlp.experts.185.gate_proj", + "model.layers.7.mlp.experts.185.up_proj", + "model.layers.7.mlp.experts.186.down_proj", + "model.layers.7.mlp.experts.186.gate_proj", + "model.layers.7.mlp.experts.186.up_proj", + "model.layers.7.mlp.experts.187.down_proj", + "model.layers.7.mlp.experts.187.gate_proj", + "model.layers.7.mlp.experts.187.up_proj", + "model.layers.7.mlp.experts.188.down_proj", + "model.layers.7.mlp.experts.188.gate_proj", + "model.layers.7.mlp.experts.188.up_proj", + "model.layers.7.mlp.experts.189.down_proj", + "model.layers.7.mlp.experts.189.gate_proj", + "model.layers.7.mlp.experts.189.up_proj", + "model.layers.7.mlp.experts.19.down_proj", + "model.layers.7.mlp.experts.19.gate_proj", + "model.layers.7.mlp.experts.19.up_proj", + "model.layers.7.mlp.experts.190.down_proj", + "model.layers.7.mlp.experts.190.gate_proj", + "model.layers.7.mlp.experts.190.up_proj", + "model.layers.7.mlp.experts.191.down_proj", + "model.layers.7.mlp.experts.191.gate_proj", + "model.layers.7.mlp.experts.191.up_proj", + "model.layers.7.mlp.experts.192.down_proj", + "model.layers.7.mlp.experts.192.gate_proj", + "model.layers.7.mlp.experts.192.up_proj", + "model.layers.7.mlp.experts.193.down_proj", + "model.layers.7.mlp.experts.193.gate_proj", + "model.layers.7.mlp.experts.193.up_proj", + "model.layers.7.mlp.experts.194.down_proj", + "model.layers.7.mlp.experts.194.gate_proj", + "model.layers.7.mlp.experts.194.up_proj", + "model.layers.7.mlp.experts.195.down_proj", + "model.layers.7.mlp.experts.195.gate_proj", + "model.layers.7.mlp.experts.195.up_proj", + "model.layers.7.mlp.experts.196.down_proj", + "model.layers.7.mlp.experts.196.gate_proj", + "model.layers.7.mlp.experts.196.up_proj", + "model.layers.7.mlp.experts.197.down_proj", + "model.layers.7.mlp.experts.197.gate_proj", + "model.layers.7.mlp.experts.197.up_proj", + "model.layers.7.mlp.experts.198.down_proj", + "model.layers.7.mlp.experts.198.gate_proj", + "model.layers.7.mlp.experts.198.up_proj", + "model.layers.7.mlp.experts.199.down_proj", + "model.layers.7.mlp.experts.199.gate_proj", + "model.layers.7.mlp.experts.199.up_proj", + "model.layers.7.mlp.experts.2.down_proj", + "model.layers.7.mlp.experts.2.gate_proj", + "model.layers.7.mlp.experts.2.up_proj", + "model.layers.7.mlp.experts.20.down_proj", + "model.layers.7.mlp.experts.20.gate_proj", + "model.layers.7.mlp.experts.20.up_proj", + "model.layers.7.mlp.experts.200.down_proj", + "model.layers.7.mlp.experts.200.gate_proj", + "model.layers.7.mlp.experts.200.up_proj", + "model.layers.7.mlp.experts.201.down_proj", + "model.layers.7.mlp.experts.201.gate_proj", + "model.layers.7.mlp.experts.201.up_proj", + "model.layers.7.mlp.experts.202.down_proj", + "model.layers.7.mlp.experts.202.gate_proj", + "model.layers.7.mlp.experts.202.up_proj", + "model.layers.7.mlp.experts.203.down_proj", + "model.layers.7.mlp.experts.203.gate_proj", + "model.layers.7.mlp.experts.203.up_proj", + "model.layers.7.mlp.experts.204.down_proj", + "model.layers.7.mlp.experts.204.gate_proj", + "model.layers.7.mlp.experts.204.up_proj", + "model.layers.7.mlp.experts.205.down_proj", + "model.layers.7.mlp.experts.205.gate_proj", + "model.layers.7.mlp.experts.205.up_proj", + "model.layers.7.mlp.experts.206.down_proj", + "model.layers.7.mlp.experts.206.gate_proj", + "model.layers.7.mlp.experts.206.up_proj", + "model.layers.7.mlp.experts.207.down_proj", + "model.layers.7.mlp.experts.207.gate_proj", + "model.layers.7.mlp.experts.207.up_proj", + "model.layers.7.mlp.experts.208.down_proj", + "model.layers.7.mlp.experts.208.gate_proj", + "model.layers.7.mlp.experts.208.up_proj", + "model.layers.7.mlp.experts.209.down_proj", + "model.layers.7.mlp.experts.209.gate_proj", + "model.layers.7.mlp.experts.209.up_proj", + "model.layers.7.mlp.experts.21.down_proj", + "model.layers.7.mlp.experts.21.gate_proj", + "model.layers.7.mlp.experts.21.up_proj", + "model.layers.7.mlp.experts.210.down_proj", + "model.layers.7.mlp.experts.210.gate_proj", + "model.layers.7.mlp.experts.210.up_proj", + "model.layers.7.mlp.experts.211.down_proj", + "model.layers.7.mlp.experts.211.gate_proj", + "model.layers.7.mlp.experts.211.up_proj", + "model.layers.7.mlp.experts.212.down_proj", + "model.layers.7.mlp.experts.212.gate_proj", + "model.layers.7.mlp.experts.212.up_proj", + "model.layers.7.mlp.experts.213.down_proj", + "model.layers.7.mlp.experts.213.gate_proj", + "model.layers.7.mlp.experts.213.up_proj", + "model.layers.7.mlp.experts.214.down_proj", + "model.layers.7.mlp.experts.214.gate_proj", + "model.layers.7.mlp.experts.214.up_proj", + "model.layers.7.mlp.experts.215.down_proj", + "model.layers.7.mlp.experts.215.gate_proj", + "model.layers.7.mlp.experts.215.up_proj", + "model.layers.7.mlp.experts.216.down_proj", + "model.layers.7.mlp.experts.216.gate_proj", + "model.layers.7.mlp.experts.216.up_proj", + "model.layers.7.mlp.experts.217.down_proj", + "model.layers.7.mlp.experts.217.gate_proj", + "model.layers.7.mlp.experts.217.up_proj", + "model.layers.7.mlp.experts.218.down_proj", + "model.layers.7.mlp.experts.218.gate_proj", + "model.layers.7.mlp.experts.218.up_proj", + "model.layers.7.mlp.experts.219.down_proj", + "model.layers.7.mlp.experts.219.gate_proj", + "model.layers.7.mlp.experts.219.up_proj", + "model.layers.7.mlp.experts.22.down_proj", + "model.layers.7.mlp.experts.22.gate_proj", + "model.layers.7.mlp.experts.22.up_proj", + "model.layers.7.mlp.experts.220.down_proj", + "model.layers.7.mlp.experts.220.gate_proj", + "model.layers.7.mlp.experts.220.up_proj", + "model.layers.7.mlp.experts.221.down_proj", + "model.layers.7.mlp.experts.221.gate_proj", + "model.layers.7.mlp.experts.221.up_proj", + "model.layers.7.mlp.experts.222.down_proj", + "model.layers.7.mlp.experts.222.gate_proj", + "model.layers.7.mlp.experts.222.up_proj", + "model.layers.7.mlp.experts.223.down_proj", + "model.layers.7.mlp.experts.223.gate_proj", + "model.layers.7.mlp.experts.223.up_proj", + "model.layers.7.mlp.experts.224.down_proj", + "model.layers.7.mlp.experts.224.gate_proj", + "model.layers.7.mlp.experts.224.up_proj", + "model.layers.7.mlp.experts.225.down_proj", + "model.layers.7.mlp.experts.225.gate_proj", + "model.layers.7.mlp.experts.225.up_proj", + "model.layers.7.mlp.experts.226.down_proj", + "model.layers.7.mlp.experts.226.gate_proj", + "model.layers.7.mlp.experts.226.up_proj", + "model.layers.7.mlp.experts.227.down_proj", + "model.layers.7.mlp.experts.227.gate_proj", + "model.layers.7.mlp.experts.227.up_proj", + "model.layers.7.mlp.experts.228.down_proj", + "model.layers.7.mlp.experts.228.gate_proj", + "model.layers.7.mlp.experts.228.up_proj", + "model.layers.7.mlp.experts.229.down_proj", + "model.layers.7.mlp.experts.229.gate_proj", + "model.layers.7.mlp.experts.229.up_proj", + "model.layers.7.mlp.experts.23.down_proj", + "model.layers.7.mlp.experts.23.gate_proj", + "model.layers.7.mlp.experts.23.up_proj", + "model.layers.7.mlp.experts.230.down_proj", + "model.layers.7.mlp.experts.230.gate_proj", + "model.layers.7.mlp.experts.230.up_proj", + "model.layers.7.mlp.experts.231.down_proj", + "model.layers.7.mlp.experts.231.gate_proj", + "model.layers.7.mlp.experts.231.up_proj", + "model.layers.7.mlp.experts.232.down_proj", + "model.layers.7.mlp.experts.232.gate_proj", + "model.layers.7.mlp.experts.232.up_proj", + "model.layers.7.mlp.experts.233.down_proj", + "model.layers.7.mlp.experts.233.gate_proj", + "model.layers.7.mlp.experts.233.up_proj", + "model.layers.7.mlp.experts.234.down_proj", + "model.layers.7.mlp.experts.234.gate_proj", + "model.layers.7.mlp.experts.234.up_proj", + "model.layers.7.mlp.experts.235.down_proj", + "model.layers.7.mlp.experts.235.gate_proj", + "model.layers.7.mlp.experts.235.up_proj", + "model.layers.7.mlp.experts.236.down_proj", + "model.layers.7.mlp.experts.236.gate_proj", + "model.layers.7.mlp.experts.236.up_proj", + "model.layers.7.mlp.experts.237.down_proj", + "model.layers.7.mlp.experts.237.gate_proj", + "model.layers.7.mlp.experts.237.up_proj", + "model.layers.7.mlp.experts.238.down_proj", + "model.layers.7.mlp.experts.238.gate_proj", + "model.layers.7.mlp.experts.238.up_proj", + "model.layers.7.mlp.experts.239.down_proj", + "model.layers.7.mlp.experts.239.gate_proj", + "model.layers.7.mlp.experts.239.up_proj", + "model.layers.7.mlp.experts.24.down_proj", + "model.layers.7.mlp.experts.24.gate_proj", + "model.layers.7.mlp.experts.24.up_proj", + "model.layers.7.mlp.experts.240.down_proj", + "model.layers.7.mlp.experts.240.gate_proj", + "model.layers.7.mlp.experts.240.up_proj", + "model.layers.7.mlp.experts.241.down_proj", + "model.layers.7.mlp.experts.241.gate_proj", + "model.layers.7.mlp.experts.241.up_proj", + "model.layers.7.mlp.experts.242.down_proj", + "model.layers.7.mlp.experts.242.gate_proj", + "model.layers.7.mlp.experts.242.up_proj", + "model.layers.7.mlp.experts.243.down_proj", + "model.layers.7.mlp.experts.243.gate_proj", + "model.layers.7.mlp.experts.243.up_proj", + "model.layers.7.mlp.experts.244.down_proj", + "model.layers.7.mlp.experts.244.gate_proj", + "model.layers.7.mlp.experts.244.up_proj", + "model.layers.7.mlp.experts.245.down_proj", + "model.layers.7.mlp.experts.245.gate_proj", + "model.layers.7.mlp.experts.245.up_proj", + "model.layers.7.mlp.experts.246.down_proj", + "model.layers.7.mlp.experts.246.gate_proj", + "model.layers.7.mlp.experts.246.up_proj", + "model.layers.7.mlp.experts.247.down_proj", + "model.layers.7.mlp.experts.247.gate_proj", + "model.layers.7.mlp.experts.247.up_proj", + "model.layers.7.mlp.experts.248.down_proj", + "model.layers.7.mlp.experts.248.gate_proj", + "model.layers.7.mlp.experts.248.up_proj", + "model.layers.7.mlp.experts.249.down_proj", + "model.layers.7.mlp.experts.249.gate_proj", + "model.layers.7.mlp.experts.249.up_proj", + "model.layers.7.mlp.experts.25.down_proj", + "model.layers.7.mlp.experts.25.gate_proj", + "model.layers.7.mlp.experts.25.up_proj", + "model.layers.7.mlp.experts.250.down_proj", + "model.layers.7.mlp.experts.250.gate_proj", + "model.layers.7.mlp.experts.250.up_proj", + "model.layers.7.mlp.experts.251.down_proj", + "model.layers.7.mlp.experts.251.gate_proj", + "model.layers.7.mlp.experts.251.up_proj", + "model.layers.7.mlp.experts.252.down_proj", + "model.layers.7.mlp.experts.252.gate_proj", + "model.layers.7.mlp.experts.252.up_proj", + "model.layers.7.mlp.experts.253.down_proj", + "model.layers.7.mlp.experts.253.gate_proj", + "model.layers.7.mlp.experts.253.up_proj", + "model.layers.7.mlp.experts.254.down_proj", + "model.layers.7.mlp.experts.254.gate_proj", + "model.layers.7.mlp.experts.254.up_proj", + "model.layers.7.mlp.experts.255.down_proj", + "model.layers.7.mlp.experts.255.gate_proj", + "model.layers.7.mlp.experts.255.up_proj", + "model.layers.7.mlp.experts.26.down_proj", + "model.layers.7.mlp.experts.26.gate_proj", + "model.layers.7.mlp.experts.26.up_proj", + "model.layers.7.mlp.experts.27.down_proj", + "model.layers.7.mlp.experts.27.gate_proj", + "model.layers.7.mlp.experts.27.up_proj", + "model.layers.7.mlp.experts.28.down_proj", + "model.layers.7.mlp.experts.28.gate_proj", + "model.layers.7.mlp.experts.28.up_proj", + "model.layers.7.mlp.experts.29.down_proj", + "model.layers.7.mlp.experts.29.gate_proj", + "model.layers.7.mlp.experts.29.up_proj", + "model.layers.7.mlp.experts.3.down_proj", + "model.layers.7.mlp.experts.3.gate_proj", + "model.layers.7.mlp.experts.3.up_proj", + "model.layers.7.mlp.experts.30.down_proj", + "model.layers.7.mlp.experts.30.gate_proj", + "model.layers.7.mlp.experts.30.up_proj", + "model.layers.7.mlp.experts.31.down_proj", + "model.layers.7.mlp.experts.31.gate_proj", + "model.layers.7.mlp.experts.31.up_proj", + "model.layers.7.mlp.experts.32.down_proj", + "model.layers.7.mlp.experts.32.gate_proj", + "model.layers.7.mlp.experts.32.up_proj", + "model.layers.7.mlp.experts.33.down_proj", + "model.layers.7.mlp.experts.33.gate_proj", + "model.layers.7.mlp.experts.33.up_proj", + "model.layers.7.mlp.experts.34.down_proj", + "model.layers.7.mlp.experts.34.gate_proj", + "model.layers.7.mlp.experts.34.up_proj", + "model.layers.7.mlp.experts.35.down_proj", + "model.layers.7.mlp.experts.35.gate_proj", + "model.layers.7.mlp.experts.35.up_proj", + "model.layers.7.mlp.experts.36.down_proj", + "model.layers.7.mlp.experts.36.gate_proj", + "model.layers.7.mlp.experts.36.up_proj", + "model.layers.7.mlp.experts.37.down_proj", + "model.layers.7.mlp.experts.37.gate_proj", + "model.layers.7.mlp.experts.37.up_proj", + "model.layers.7.mlp.experts.38.down_proj", + "model.layers.7.mlp.experts.38.gate_proj", + "model.layers.7.mlp.experts.38.up_proj", + "model.layers.7.mlp.experts.39.down_proj", + "model.layers.7.mlp.experts.39.gate_proj", + "model.layers.7.mlp.experts.39.up_proj", + "model.layers.7.mlp.experts.4.down_proj", + "model.layers.7.mlp.experts.4.gate_proj", + "model.layers.7.mlp.experts.4.up_proj", + "model.layers.7.mlp.experts.40.down_proj", + "model.layers.7.mlp.experts.40.gate_proj", + "model.layers.7.mlp.experts.40.up_proj", + "model.layers.7.mlp.experts.41.down_proj", + "model.layers.7.mlp.experts.41.gate_proj", + "model.layers.7.mlp.experts.41.up_proj", + "model.layers.7.mlp.experts.42.down_proj", + "model.layers.7.mlp.experts.42.gate_proj", + "model.layers.7.mlp.experts.42.up_proj", + "model.layers.7.mlp.experts.43.down_proj", + "model.layers.7.mlp.experts.43.gate_proj", + "model.layers.7.mlp.experts.43.up_proj", + "model.layers.7.mlp.experts.44.down_proj", + "model.layers.7.mlp.experts.44.gate_proj", + "model.layers.7.mlp.experts.44.up_proj", + "model.layers.7.mlp.experts.45.down_proj", + "model.layers.7.mlp.experts.45.gate_proj", + "model.layers.7.mlp.experts.45.up_proj", + "model.layers.7.mlp.experts.46.down_proj", + "model.layers.7.mlp.experts.46.gate_proj", + "model.layers.7.mlp.experts.46.up_proj", + "model.layers.7.mlp.experts.47.down_proj", + "model.layers.7.mlp.experts.47.gate_proj", + "model.layers.7.mlp.experts.47.up_proj", + "model.layers.7.mlp.experts.48.down_proj", + "model.layers.7.mlp.experts.48.gate_proj", + "model.layers.7.mlp.experts.48.up_proj", + "model.layers.7.mlp.experts.49.down_proj", + "model.layers.7.mlp.experts.49.gate_proj", + "model.layers.7.mlp.experts.49.up_proj", + "model.layers.7.mlp.experts.5.down_proj", + "model.layers.7.mlp.experts.5.gate_proj", + "model.layers.7.mlp.experts.5.up_proj", + "model.layers.7.mlp.experts.50.down_proj", + "model.layers.7.mlp.experts.50.gate_proj", + "model.layers.7.mlp.experts.50.up_proj", + "model.layers.7.mlp.experts.51.down_proj", + "model.layers.7.mlp.experts.51.gate_proj", + "model.layers.7.mlp.experts.51.up_proj", + "model.layers.7.mlp.experts.52.down_proj", + "model.layers.7.mlp.experts.52.gate_proj", + "model.layers.7.mlp.experts.52.up_proj", + "model.layers.7.mlp.experts.53.down_proj", + "model.layers.7.mlp.experts.53.gate_proj", + "model.layers.7.mlp.experts.53.up_proj", + "model.layers.7.mlp.experts.54.down_proj", + "model.layers.7.mlp.experts.54.gate_proj", + "model.layers.7.mlp.experts.54.up_proj", + "model.layers.7.mlp.experts.55.down_proj", + "model.layers.7.mlp.experts.55.gate_proj", + "model.layers.7.mlp.experts.55.up_proj", + "model.layers.7.mlp.experts.56.down_proj", + "model.layers.7.mlp.experts.56.gate_proj", + "model.layers.7.mlp.experts.56.up_proj", + "model.layers.7.mlp.experts.57.down_proj", + "model.layers.7.mlp.experts.57.gate_proj", + "model.layers.7.mlp.experts.57.up_proj", + "model.layers.7.mlp.experts.58.down_proj", + "model.layers.7.mlp.experts.58.gate_proj", + "model.layers.7.mlp.experts.58.up_proj", + "model.layers.7.mlp.experts.59.down_proj", + "model.layers.7.mlp.experts.59.gate_proj", + "model.layers.7.mlp.experts.59.up_proj", + "model.layers.7.mlp.experts.6.down_proj", + "model.layers.7.mlp.experts.6.gate_proj", + "model.layers.7.mlp.experts.6.up_proj", + "model.layers.7.mlp.experts.60.down_proj", + "model.layers.7.mlp.experts.60.gate_proj", + "model.layers.7.mlp.experts.60.up_proj", + "model.layers.7.mlp.experts.61.down_proj", + "model.layers.7.mlp.experts.61.gate_proj", + "model.layers.7.mlp.experts.61.up_proj", + "model.layers.7.mlp.experts.62.down_proj", + "model.layers.7.mlp.experts.62.gate_proj", + "model.layers.7.mlp.experts.62.up_proj", + "model.layers.7.mlp.experts.63.down_proj", + "model.layers.7.mlp.experts.63.gate_proj", + "model.layers.7.mlp.experts.63.up_proj", + "model.layers.7.mlp.experts.64.down_proj", + "model.layers.7.mlp.experts.64.gate_proj", + "model.layers.7.mlp.experts.64.up_proj", + "model.layers.7.mlp.experts.65.down_proj", + "model.layers.7.mlp.experts.65.gate_proj", + "model.layers.7.mlp.experts.65.up_proj", + "model.layers.7.mlp.experts.66.down_proj", + "model.layers.7.mlp.experts.66.gate_proj", + "model.layers.7.mlp.experts.66.up_proj", + "model.layers.7.mlp.experts.67.down_proj", + "model.layers.7.mlp.experts.67.gate_proj", + "model.layers.7.mlp.experts.67.up_proj", + "model.layers.7.mlp.experts.68.down_proj", + "model.layers.7.mlp.experts.68.gate_proj", + "model.layers.7.mlp.experts.68.up_proj", + "model.layers.7.mlp.experts.69.down_proj", + "model.layers.7.mlp.experts.69.gate_proj", + "model.layers.7.mlp.experts.69.up_proj", + "model.layers.7.mlp.experts.7.down_proj", + "model.layers.7.mlp.experts.7.gate_proj", + "model.layers.7.mlp.experts.7.up_proj", + "model.layers.7.mlp.experts.70.down_proj", + "model.layers.7.mlp.experts.70.gate_proj", + "model.layers.7.mlp.experts.70.up_proj", + "model.layers.7.mlp.experts.71.down_proj", + "model.layers.7.mlp.experts.71.gate_proj", + "model.layers.7.mlp.experts.71.up_proj", + "model.layers.7.mlp.experts.72.down_proj", + "model.layers.7.mlp.experts.72.gate_proj", + "model.layers.7.mlp.experts.72.up_proj", + "model.layers.7.mlp.experts.73.down_proj", + "model.layers.7.mlp.experts.73.gate_proj", + "model.layers.7.mlp.experts.73.up_proj", + "model.layers.7.mlp.experts.74.down_proj", + "model.layers.7.mlp.experts.74.gate_proj", + "model.layers.7.mlp.experts.74.up_proj", + "model.layers.7.mlp.experts.75.down_proj", + "model.layers.7.mlp.experts.75.gate_proj", + "model.layers.7.mlp.experts.75.up_proj", + "model.layers.7.mlp.experts.76.down_proj", + "model.layers.7.mlp.experts.76.gate_proj", + "model.layers.7.mlp.experts.76.up_proj", + "model.layers.7.mlp.experts.77.down_proj", + "model.layers.7.mlp.experts.77.gate_proj", + "model.layers.7.mlp.experts.77.up_proj", + "model.layers.7.mlp.experts.78.down_proj", + "model.layers.7.mlp.experts.78.gate_proj", + "model.layers.7.mlp.experts.78.up_proj", + "model.layers.7.mlp.experts.79.down_proj", + "model.layers.7.mlp.experts.79.gate_proj", + "model.layers.7.mlp.experts.79.up_proj", + "model.layers.7.mlp.experts.8.down_proj", + "model.layers.7.mlp.experts.8.gate_proj", + "model.layers.7.mlp.experts.8.up_proj", + "model.layers.7.mlp.experts.80.down_proj", + "model.layers.7.mlp.experts.80.gate_proj", + "model.layers.7.mlp.experts.80.up_proj", + "model.layers.7.mlp.experts.81.down_proj", + "model.layers.7.mlp.experts.81.gate_proj", + "model.layers.7.mlp.experts.81.up_proj", + "model.layers.7.mlp.experts.82.down_proj", + "model.layers.7.mlp.experts.82.gate_proj", + "model.layers.7.mlp.experts.82.up_proj", + "model.layers.7.mlp.experts.83.down_proj", + "model.layers.7.mlp.experts.83.gate_proj", + "model.layers.7.mlp.experts.83.up_proj", + "model.layers.7.mlp.experts.84.down_proj", + "model.layers.7.mlp.experts.84.gate_proj", + "model.layers.7.mlp.experts.84.up_proj", + "model.layers.7.mlp.experts.85.down_proj", + "model.layers.7.mlp.experts.85.gate_proj", + "model.layers.7.mlp.experts.85.up_proj", + "model.layers.7.mlp.experts.86.down_proj", + "model.layers.7.mlp.experts.86.gate_proj", + "model.layers.7.mlp.experts.86.up_proj", + "model.layers.7.mlp.experts.87.down_proj", + "model.layers.7.mlp.experts.87.gate_proj", + "model.layers.7.mlp.experts.87.up_proj", + "model.layers.7.mlp.experts.88.down_proj", + "model.layers.7.mlp.experts.88.gate_proj", + "model.layers.7.mlp.experts.88.up_proj", + "model.layers.7.mlp.experts.89.down_proj", + "model.layers.7.mlp.experts.89.gate_proj", + "model.layers.7.mlp.experts.89.up_proj", + "model.layers.7.mlp.experts.9.down_proj", + "model.layers.7.mlp.experts.9.gate_proj", + "model.layers.7.mlp.experts.9.up_proj", + "model.layers.7.mlp.experts.90.down_proj", + "model.layers.7.mlp.experts.90.gate_proj", + "model.layers.7.mlp.experts.90.up_proj", + "model.layers.7.mlp.experts.91.down_proj", + "model.layers.7.mlp.experts.91.gate_proj", + "model.layers.7.mlp.experts.91.up_proj", + "model.layers.7.mlp.experts.92.down_proj", + "model.layers.7.mlp.experts.92.gate_proj", + "model.layers.7.mlp.experts.92.up_proj", + "model.layers.7.mlp.experts.93.down_proj", + "model.layers.7.mlp.experts.93.gate_proj", + "model.layers.7.mlp.experts.93.up_proj", + "model.layers.7.mlp.experts.94.down_proj", + "model.layers.7.mlp.experts.94.gate_proj", + "model.layers.7.mlp.experts.94.up_proj", + "model.layers.7.mlp.experts.95.down_proj", + "model.layers.7.mlp.experts.95.gate_proj", + "model.layers.7.mlp.experts.95.up_proj", + "model.layers.7.mlp.experts.96.down_proj", + "model.layers.7.mlp.experts.96.gate_proj", + "model.layers.7.mlp.experts.96.up_proj", + "model.layers.7.mlp.experts.97.down_proj", + "model.layers.7.mlp.experts.97.gate_proj", + "model.layers.7.mlp.experts.97.up_proj", + "model.layers.7.mlp.experts.98.down_proj", + "model.layers.7.mlp.experts.98.gate_proj", + "model.layers.7.mlp.experts.98.up_proj", + "model.layers.7.mlp.experts.99.down_proj", + "model.layers.7.mlp.experts.99.gate_proj", + "model.layers.7.mlp.experts.99.up_proj", + "model.layers.70.mlp.experts.0.down_proj", + "model.layers.70.mlp.experts.0.gate_proj", + "model.layers.70.mlp.experts.0.up_proj", + "model.layers.70.mlp.experts.1.down_proj", + "model.layers.70.mlp.experts.1.gate_proj", + "model.layers.70.mlp.experts.1.up_proj", + "model.layers.70.mlp.experts.10.down_proj", + "model.layers.70.mlp.experts.10.gate_proj", + "model.layers.70.mlp.experts.10.up_proj", + "model.layers.70.mlp.experts.100.down_proj", + "model.layers.70.mlp.experts.100.gate_proj", + "model.layers.70.mlp.experts.100.up_proj", + "model.layers.70.mlp.experts.101.down_proj", + "model.layers.70.mlp.experts.101.gate_proj", + "model.layers.70.mlp.experts.101.up_proj", + "model.layers.70.mlp.experts.102.down_proj", + "model.layers.70.mlp.experts.102.gate_proj", + "model.layers.70.mlp.experts.102.up_proj", + "model.layers.70.mlp.experts.103.down_proj", + "model.layers.70.mlp.experts.103.gate_proj", + "model.layers.70.mlp.experts.103.up_proj", + "model.layers.70.mlp.experts.104.down_proj", + "model.layers.70.mlp.experts.104.gate_proj", + "model.layers.70.mlp.experts.104.up_proj", + "model.layers.70.mlp.experts.105.down_proj", + "model.layers.70.mlp.experts.105.gate_proj", + "model.layers.70.mlp.experts.105.up_proj", + "model.layers.70.mlp.experts.106.down_proj", + "model.layers.70.mlp.experts.106.gate_proj", + "model.layers.70.mlp.experts.106.up_proj", + "model.layers.70.mlp.experts.107.down_proj", + "model.layers.70.mlp.experts.107.gate_proj", + "model.layers.70.mlp.experts.107.up_proj", + "model.layers.70.mlp.experts.108.down_proj", + "model.layers.70.mlp.experts.108.gate_proj", + "model.layers.70.mlp.experts.108.up_proj", + "model.layers.70.mlp.experts.109.down_proj", + "model.layers.70.mlp.experts.109.gate_proj", + "model.layers.70.mlp.experts.109.up_proj", + "model.layers.70.mlp.experts.11.down_proj", + "model.layers.70.mlp.experts.11.gate_proj", + "model.layers.70.mlp.experts.11.up_proj", + "model.layers.70.mlp.experts.110.down_proj", + "model.layers.70.mlp.experts.110.gate_proj", + "model.layers.70.mlp.experts.110.up_proj", + "model.layers.70.mlp.experts.111.down_proj", + "model.layers.70.mlp.experts.111.gate_proj", + "model.layers.70.mlp.experts.111.up_proj", + "model.layers.70.mlp.experts.112.down_proj", + "model.layers.70.mlp.experts.112.gate_proj", + "model.layers.70.mlp.experts.112.up_proj", + "model.layers.70.mlp.experts.113.down_proj", + "model.layers.70.mlp.experts.113.gate_proj", + "model.layers.70.mlp.experts.113.up_proj", + "model.layers.70.mlp.experts.114.down_proj", + "model.layers.70.mlp.experts.114.gate_proj", + "model.layers.70.mlp.experts.114.up_proj", + "model.layers.70.mlp.experts.115.down_proj", + "model.layers.70.mlp.experts.115.gate_proj", + "model.layers.70.mlp.experts.115.up_proj", + "model.layers.70.mlp.experts.116.down_proj", + "model.layers.70.mlp.experts.116.gate_proj", + "model.layers.70.mlp.experts.116.up_proj", + "model.layers.70.mlp.experts.117.down_proj", + "model.layers.70.mlp.experts.117.gate_proj", + "model.layers.70.mlp.experts.117.up_proj", + "model.layers.70.mlp.experts.118.down_proj", + "model.layers.70.mlp.experts.118.gate_proj", + "model.layers.70.mlp.experts.118.up_proj", + "model.layers.70.mlp.experts.119.down_proj", + "model.layers.70.mlp.experts.119.gate_proj", + "model.layers.70.mlp.experts.119.up_proj", + "model.layers.70.mlp.experts.12.down_proj", + "model.layers.70.mlp.experts.12.gate_proj", + "model.layers.70.mlp.experts.12.up_proj", + "model.layers.70.mlp.experts.120.down_proj", + "model.layers.70.mlp.experts.120.gate_proj", + "model.layers.70.mlp.experts.120.up_proj", + "model.layers.70.mlp.experts.121.down_proj", + "model.layers.70.mlp.experts.121.gate_proj", + "model.layers.70.mlp.experts.121.up_proj", + "model.layers.70.mlp.experts.122.down_proj", + "model.layers.70.mlp.experts.122.gate_proj", + "model.layers.70.mlp.experts.122.up_proj", + "model.layers.70.mlp.experts.123.down_proj", + "model.layers.70.mlp.experts.123.gate_proj", + "model.layers.70.mlp.experts.123.up_proj", + "model.layers.70.mlp.experts.124.down_proj", + "model.layers.70.mlp.experts.124.gate_proj", + "model.layers.70.mlp.experts.124.up_proj", + "model.layers.70.mlp.experts.125.down_proj", + "model.layers.70.mlp.experts.125.gate_proj", + "model.layers.70.mlp.experts.125.up_proj", + "model.layers.70.mlp.experts.126.down_proj", + "model.layers.70.mlp.experts.126.gate_proj", + "model.layers.70.mlp.experts.126.up_proj", + "model.layers.70.mlp.experts.127.down_proj", + "model.layers.70.mlp.experts.127.gate_proj", + "model.layers.70.mlp.experts.127.up_proj", + "model.layers.70.mlp.experts.128.down_proj", + "model.layers.70.mlp.experts.128.gate_proj", + "model.layers.70.mlp.experts.128.up_proj", + "model.layers.70.mlp.experts.129.down_proj", + "model.layers.70.mlp.experts.129.gate_proj", + "model.layers.70.mlp.experts.129.up_proj", + "model.layers.70.mlp.experts.13.down_proj", + "model.layers.70.mlp.experts.13.gate_proj", + "model.layers.70.mlp.experts.13.up_proj", + "model.layers.70.mlp.experts.130.down_proj", + "model.layers.70.mlp.experts.130.gate_proj", + "model.layers.70.mlp.experts.130.up_proj", + "model.layers.70.mlp.experts.131.down_proj", + "model.layers.70.mlp.experts.131.gate_proj", + "model.layers.70.mlp.experts.131.up_proj", + "model.layers.70.mlp.experts.132.down_proj", + "model.layers.70.mlp.experts.132.gate_proj", + "model.layers.70.mlp.experts.132.up_proj", + "model.layers.70.mlp.experts.133.down_proj", + "model.layers.70.mlp.experts.133.gate_proj", + "model.layers.70.mlp.experts.133.up_proj", + "model.layers.70.mlp.experts.134.down_proj", + "model.layers.70.mlp.experts.134.gate_proj", + "model.layers.70.mlp.experts.134.up_proj", + "model.layers.70.mlp.experts.135.down_proj", + "model.layers.70.mlp.experts.135.gate_proj", + "model.layers.70.mlp.experts.135.up_proj", + "model.layers.70.mlp.experts.136.down_proj", + "model.layers.70.mlp.experts.136.gate_proj", + "model.layers.70.mlp.experts.136.up_proj", + "model.layers.70.mlp.experts.137.down_proj", + "model.layers.70.mlp.experts.137.gate_proj", + "model.layers.70.mlp.experts.137.up_proj", + "model.layers.70.mlp.experts.138.down_proj", + "model.layers.70.mlp.experts.138.gate_proj", + "model.layers.70.mlp.experts.138.up_proj", + "model.layers.70.mlp.experts.139.down_proj", + "model.layers.70.mlp.experts.139.gate_proj", + "model.layers.70.mlp.experts.139.up_proj", + "model.layers.70.mlp.experts.14.down_proj", + "model.layers.70.mlp.experts.14.gate_proj", + "model.layers.70.mlp.experts.14.up_proj", + "model.layers.70.mlp.experts.140.down_proj", + "model.layers.70.mlp.experts.140.gate_proj", + "model.layers.70.mlp.experts.140.up_proj", + "model.layers.70.mlp.experts.141.down_proj", + "model.layers.70.mlp.experts.141.gate_proj", + "model.layers.70.mlp.experts.141.up_proj", + "model.layers.70.mlp.experts.142.down_proj", + "model.layers.70.mlp.experts.142.gate_proj", + "model.layers.70.mlp.experts.142.up_proj", + "model.layers.70.mlp.experts.143.down_proj", + "model.layers.70.mlp.experts.143.gate_proj", + "model.layers.70.mlp.experts.143.up_proj", + "model.layers.70.mlp.experts.144.down_proj", + "model.layers.70.mlp.experts.144.gate_proj", + "model.layers.70.mlp.experts.144.up_proj", + "model.layers.70.mlp.experts.145.down_proj", + "model.layers.70.mlp.experts.145.gate_proj", + "model.layers.70.mlp.experts.145.up_proj", + "model.layers.70.mlp.experts.146.down_proj", + "model.layers.70.mlp.experts.146.gate_proj", + "model.layers.70.mlp.experts.146.up_proj", + "model.layers.70.mlp.experts.147.down_proj", + "model.layers.70.mlp.experts.147.gate_proj", + "model.layers.70.mlp.experts.147.up_proj", + "model.layers.70.mlp.experts.148.down_proj", + "model.layers.70.mlp.experts.148.gate_proj", + "model.layers.70.mlp.experts.148.up_proj", + "model.layers.70.mlp.experts.149.down_proj", + "model.layers.70.mlp.experts.149.gate_proj", + "model.layers.70.mlp.experts.149.up_proj", + "model.layers.70.mlp.experts.15.down_proj", + "model.layers.70.mlp.experts.15.gate_proj", + "model.layers.70.mlp.experts.15.up_proj", + "model.layers.70.mlp.experts.150.down_proj", + "model.layers.70.mlp.experts.150.gate_proj", + "model.layers.70.mlp.experts.150.up_proj", + "model.layers.70.mlp.experts.151.down_proj", + "model.layers.70.mlp.experts.151.gate_proj", + "model.layers.70.mlp.experts.151.up_proj", + "model.layers.70.mlp.experts.152.down_proj", + "model.layers.70.mlp.experts.152.gate_proj", + "model.layers.70.mlp.experts.152.up_proj", + "model.layers.70.mlp.experts.153.down_proj", + "model.layers.70.mlp.experts.153.gate_proj", + "model.layers.70.mlp.experts.153.up_proj", + "model.layers.70.mlp.experts.154.down_proj", + "model.layers.70.mlp.experts.154.gate_proj", + "model.layers.70.mlp.experts.154.up_proj", + "model.layers.70.mlp.experts.155.down_proj", + "model.layers.70.mlp.experts.155.gate_proj", + "model.layers.70.mlp.experts.155.up_proj", + "model.layers.70.mlp.experts.156.down_proj", + "model.layers.70.mlp.experts.156.gate_proj", + "model.layers.70.mlp.experts.156.up_proj", + "model.layers.70.mlp.experts.157.down_proj", + "model.layers.70.mlp.experts.157.gate_proj", + "model.layers.70.mlp.experts.157.up_proj", + "model.layers.70.mlp.experts.158.down_proj", + "model.layers.70.mlp.experts.158.gate_proj", + "model.layers.70.mlp.experts.158.up_proj", + "model.layers.70.mlp.experts.159.down_proj", + "model.layers.70.mlp.experts.159.gate_proj", + "model.layers.70.mlp.experts.159.up_proj", + "model.layers.70.mlp.experts.16.down_proj", + "model.layers.70.mlp.experts.16.gate_proj", + "model.layers.70.mlp.experts.16.up_proj", + "model.layers.70.mlp.experts.160.down_proj", + "model.layers.70.mlp.experts.160.gate_proj", + "model.layers.70.mlp.experts.160.up_proj", + "model.layers.70.mlp.experts.161.down_proj", + "model.layers.70.mlp.experts.161.gate_proj", + "model.layers.70.mlp.experts.161.up_proj", + "model.layers.70.mlp.experts.162.down_proj", + "model.layers.70.mlp.experts.162.gate_proj", + "model.layers.70.mlp.experts.162.up_proj", + "model.layers.70.mlp.experts.163.down_proj", + "model.layers.70.mlp.experts.163.gate_proj", + "model.layers.70.mlp.experts.163.up_proj", + "model.layers.70.mlp.experts.164.down_proj", + "model.layers.70.mlp.experts.164.gate_proj", + "model.layers.70.mlp.experts.164.up_proj", + "model.layers.70.mlp.experts.165.down_proj", + "model.layers.70.mlp.experts.165.gate_proj", + "model.layers.70.mlp.experts.165.up_proj", + "model.layers.70.mlp.experts.166.down_proj", + "model.layers.70.mlp.experts.166.gate_proj", + "model.layers.70.mlp.experts.166.up_proj", + "model.layers.70.mlp.experts.167.down_proj", + "model.layers.70.mlp.experts.167.gate_proj", + "model.layers.70.mlp.experts.167.up_proj", + "model.layers.70.mlp.experts.168.down_proj", + "model.layers.70.mlp.experts.168.gate_proj", + "model.layers.70.mlp.experts.168.up_proj", + "model.layers.70.mlp.experts.169.down_proj", + "model.layers.70.mlp.experts.169.gate_proj", + "model.layers.70.mlp.experts.169.up_proj", + "model.layers.70.mlp.experts.17.down_proj", + "model.layers.70.mlp.experts.17.gate_proj", + "model.layers.70.mlp.experts.17.up_proj", + "model.layers.70.mlp.experts.170.down_proj", + "model.layers.70.mlp.experts.170.gate_proj", + "model.layers.70.mlp.experts.170.up_proj", + "model.layers.70.mlp.experts.171.down_proj", + "model.layers.70.mlp.experts.171.gate_proj", + "model.layers.70.mlp.experts.171.up_proj", + "model.layers.70.mlp.experts.172.down_proj", + "model.layers.70.mlp.experts.172.gate_proj", + "model.layers.70.mlp.experts.172.up_proj", + "model.layers.70.mlp.experts.173.down_proj", + "model.layers.70.mlp.experts.173.gate_proj", + "model.layers.70.mlp.experts.173.up_proj", + "model.layers.70.mlp.experts.174.down_proj", + "model.layers.70.mlp.experts.174.gate_proj", + "model.layers.70.mlp.experts.174.up_proj", + "model.layers.70.mlp.experts.175.down_proj", + "model.layers.70.mlp.experts.175.gate_proj", + "model.layers.70.mlp.experts.175.up_proj", + "model.layers.70.mlp.experts.176.down_proj", + "model.layers.70.mlp.experts.176.gate_proj", + "model.layers.70.mlp.experts.176.up_proj", + "model.layers.70.mlp.experts.177.down_proj", + "model.layers.70.mlp.experts.177.gate_proj", + "model.layers.70.mlp.experts.177.up_proj", + "model.layers.70.mlp.experts.178.down_proj", + "model.layers.70.mlp.experts.178.gate_proj", + "model.layers.70.mlp.experts.178.up_proj", + "model.layers.70.mlp.experts.179.down_proj", + "model.layers.70.mlp.experts.179.gate_proj", + "model.layers.70.mlp.experts.179.up_proj", + "model.layers.70.mlp.experts.18.down_proj", + "model.layers.70.mlp.experts.18.gate_proj", + "model.layers.70.mlp.experts.18.up_proj", + "model.layers.70.mlp.experts.180.down_proj", + "model.layers.70.mlp.experts.180.gate_proj", + "model.layers.70.mlp.experts.180.up_proj", + "model.layers.70.mlp.experts.181.down_proj", + "model.layers.70.mlp.experts.181.gate_proj", + "model.layers.70.mlp.experts.181.up_proj", + "model.layers.70.mlp.experts.182.down_proj", + "model.layers.70.mlp.experts.182.gate_proj", + "model.layers.70.mlp.experts.182.up_proj", + "model.layers.70.mlp.experts.183.down_proj", + "model.layers.70.mlp.experts.183.gate_proj", + "model.layers.70.mlp.experts.183.up_proj", + "model.layers.70.mlp.experts.184.down_proj", + "model.layers.70.mlp.experts.184.gate_proj", + "model.layers.70.mlp.experts.184.up_proj", + "model.layers.70.mlp.experts.185.down_proj", + "model.layers.70.mlp.experts.185.gate_proj", + "model.layers.70.mlp.experts.185.up_proj", + "model.layers.70.mlp.experts.186.down_proj", + "model.layers.70.mlp.experts.186.gate_proj", + "model.layers.70.mlp.experts.186.up_proj", + "model.layers.70.mlp.experts.187.down_proj", + "model.layers.70.mlp.experts.187.gate_proj", + "model.layers.70.mlp.experts.187.up_proj", + "model.layers.70.mlp.experts.188.down_proj", + "model.layers.70.mlp.experts.188.gate_proj", + "model.layers.70.mlp.experts.188.up_proj", + "model.layers.70.mlp.experts.189.down_proj", + "model.layers.70.mlp.experts.189.gate_proj", + "model.layers.70.mlp.experts.189.up_proj", + "model.layers.70.mlp.experts.19.down_proj", + "model.layers.70.mlp.experts.19.gate_proj", + "model.layers.70.mlp.experts.19.up_proj", + "model.layers.70.mlp.experts.190.down_proj", + "model.layers.70.mlp.experts.190.gate_proj", + "model.layers.70.mlp.experts.190.up_proj", + "model.layers.70.mlp.experts.191.down_proj", + "model.layers.70.mlp.experts.191.gate_proj", + "model.layers.70.mlp.experts.191.up_proj", + "model.layers.70.mlp.experts.192.down_proj", + "model.layers.70.mlp.experts.192.gate_proj", + "model.layers.70.mlp.experts.192.up_proj", + "model.layers.70.mlp.experts.193.down_proj", + "model.layers.70.mlp.experts.193.gate_proj", + "model.layers.70.mlp.experts.193.up_proj", + "model.layers.70.mlp.experts.194.down_proj", + "model.layers.70.mlp.experts.194.gate_proj", + "model.layers.70.mlp.experts.194.up_proj", + "model.layers.70.mlp.experts.195.down_proj", + "model.layers.70.mlp.experts.195.gate_proj", + "model.layers.70.mlp.experts.195.up_proj", + "model.layers.70.mlp.experts.196.down_proj", + "model.layers.70.mlp.experts.196.gate_proj", + "model.layers.70.mlp.experts.196.up_proj", + "model.layers.70.mlp.experts.197.down_proj", + "model.layers.70.mlp.experts.197.gate_proj", + "model.layers.70.mlp.experts.197.up_proj", + "model.layers.70.mlp.experts.198.down_proj", + "model.layers.70.mlp.experts.198.gate_proj", + "model.layers.70.mlp.experts.198.up_proj", + "model.layers.70.mlp.experts.199.down_proj", + "model.layers.70.mlp.experts.199.gate_proj", + "model.layers.70.mlp.experts.199.up_proj", + "model.layers.70.mlp.experts.2.down_proj", + "model.layers.70.mlp.experts.2.gate_proj", + "model.layers.70.mlp.experts.2.up_proj", + "model.layers.70.mlp.experts.20.down_proj", + "model.layers.70.mlp.experts.20.gate_proj", + "model.layers.70.mlp.experts.20.up_proj", + "model.layers.70.mlp.experts.200.down_proj", + "model.layers.70.mlp.experts.200.gate_proj", + "model.layers.70.mlp.experts.200.up_proj", + "model.layers.70.mlp.experts.201.down_proj", + "model.layers.70.mlp.experts.201.gate_proj", + "model.layers.70.mlp.experts.201.up_proj", + "model.layers.70.mlp.experts.202.down_proj", + "model.layers.70.mlp.experts.202.gate_proj", + "model.layers.70.mlp.experts.202.up_proj", + "model.layers.70.mlp.experts.203.down_proj", + "model.layers.70.mlp.experts.203.gate_proj", + "model.layers.70.mlp.experts.203.up_proj", + "model.layers.70.mlp.experts.204.down_proj", + "model.layers.70.mlp.experts.204.gate_proj", + "model.layers.70.mlp.experts.204.up_proj", + "model.layers.70.mlp.experts.205.down_proj", + "model.layers.70.mlp.experts.205.gate_proj", + "model.layers.70.mlp.experts.205.up_proj", + "model.layers.70.mlp.experts.206.down_proj", + "model.layers.70.mlp.experts.206.gate_proj", + "model.layers.70.mlp.experts.206.up_proj", + "model.layers.70.mlp.experts.207.down_proj", + "model.layers.70.mlp.experts.207.gate_proj", + "model.layers.70.mlp.experts.207.up_proj", + "model.layers.70.mlp.experts.208.down_proj", + "model.layers.70.mlp.experts.208.gate_proj", + "model.layers.70.mlp.experts.208.up_proj", + "model.layers.70.mlp.experts.209.down_proj", + "model.layers.70.mlp.experts.209.gate_proj", + "model.layers.70.mlp.experts.209.up_proj", + "model.layers.70.mlp.experts.21.down_proj", + "model.layers.70.mlp.experts.21.gate_proj", + "model.layers.70.mlp.experts.21.up_proj", + "model.layers.70.mlp.experts.210.down_proj", + "model.layers.70.mlp.experts.210.gate_proj", + "model.layers.70.mlp.experts.210.up_proj", + "model.layers.70.mlp.experts.211.down_proj", + "model.layers.70.mlp.experts.211.gate_proj", + "model.layers.70.mlp.experts.211.up_proj", + "model.layers.70.mlp.experts.212.down_proj", + "model.layers.70.mlp.experts.212.gate_proj", + "model.layers.70.mlp.experts.212.up_proj", + "model.layers.70.mlp.experts.213.down_proj", + "model.layers.70.mlp.experts.213.gate_proj", + "model.layers.70.mlp.experts.213.up_proj", + "model.layers.70.mlp.experts.214.down_proj", + "model.layers.70.mlp.experts.214.gate_proj", + "model.layers.70.mlp.experts.214.up_proj", + "model.layers.70.mlp.experts.215.down_proj", + "model.layers.70.mlp.experts.215.gate_proj", + "model.layers.70.mlp.experts.215.up_proj", + "model.layers.70.mlp.experts.216.down_proj", + "model.layers.70.mlp.experts.216.gate_proj", + "model.layers.70.mlp.experts.216.up_proj", + "model.layers.70.mlp.experts.217.down_proj", + "model.layers.70.mlp.experts.217.gate_proj", + "model.layers.70.mlp.experts.217.up_proj", + "model.layers.70.mlp.experts.218.down_proj", + "model.layers.70.mlp.experts.218.gate_proj", + "model.layers.70.mlp.experts.218.up_proj", + "model.layers.70.mlp.experts.219.down_proj", + "model.layers.70.mlp.experts.219.gate_proj", + "model.layers.70.mlp.experts.219.up_proj", + "model.layers.70.mlp.experts.22.down_proj", + "model.layers.70.mlp.experts.22.gate_proj", + "model.layers.70.mlp.experts.22.up_proj", + "model.layers.70.mlp.experts.220.down_proj", + "model.layers.70.mlp.experts.220.gate_proj", + "model.layers.70.mlp.experts.220.up_proj", + "model.layers.70.mlp.experts.221.down_proj", + "model.layers.70.mlp.experts.221.gate_proj", + "model.layers.70.mlp.experts.221.up_proj", + "model.layers.70.mlp.experts.222.down_proj", + "model.layers.70.mlp.experts.222.gate_proj", + "model.layers.70.mlp.experts.222.up_proj", + "model.layers.70.mlp.experts.223.down_proj", + "model.layers.70.mlp.experts.223.gate_proj", + "model.layers.70.mlp.experts.223.up_proj", + "model.layers.70.mlp.experts.224.down_proj", + "model.layers.70.mlp.experts.224.gate_proj", + "model.layers.70.mlp.experts.224.up_proj", + "model.layers.70.mlp.experts.225.down_proj", + "model.layers.70.mlp.experts.225.gate_proj", + "model.layers.70.mlp.experts.225.up_proj", + "model.layers.70.mlp.experts.226.down_proj", + "model.layers.70.mlp.experts.226.gate_proj", + "model.layers.70.mlp.experts.226.up_proj", + "model.layers.70.mlp.experts.227.down_proj", + "model.layers.70.mlp.experts.227.gate_proj", + "model.layers.70.mlp.experts.227.up_proj", + "model.layers.70.mlp.experts.228.down_proj", + "model.layers.70.mlp.experts.228.gate_proj", + "model.layers.70.mlp.experts.228.up_proj", + "model.layers.70.mlp.experts.229.down_proj", + "model.layers.70.mlp.experts.229.gate_proj", + "model.layers.70.mlp.experts.229.up_proj", + "model.layers.70.mlp.experts.23.down_proj", + "model.layers.70.mlp.experts.23.gate_proj", + "model.layers.70.mlp.experts.23.up_proj", + "model.layers.70.mlp.experts.230.down_proj", + "model.layers.70.mlp.experts.230.gate_proj", + "model.layers.70.mlp.experts.230.up_proj", + "model.layers.70.mlp.experts.231.down_proj", + "model.layers.70.mlp.experts.231.gate_proj", + "model.layers.70.mlp.experts.231.up_proj", + "model.layers.70.mlp.experts.232.down_proj", + "model.layers.70.mlp.experts.232.gate_proj", + "model.layers.70.mlp.experts.232.up_proj", + "model.layers.70.mlp.experts.233.down_proj", + "model.layers.70.mlp.experts.233.gate_proj", + "model.layers.70.mlp.experts.233.up_proj", + "model.layers.70.mlp.experts.234.down_proj", + "model.layers.70.mlp.experts.234.gate_proj", + "model.layers.70.mlp.experts.234.up_proj", + "model.layers.70.mlp.experts.235.down_proj", + "model.layers.70.mlp.experts.235.gate_proj", + "model.layers.70.mlp.experts.235.up_proj", + "model.layers.70.mlp.experts.236.down_proj", + "model.layers.70.mlp.experts.236.gate_proj", + "model.layers.70.mlp.experts.236.up_proj", + "model.layers.70.mlp.experts.237.down_proj", + "model.layers.70.mlp.experts.237.gate_proj", + "model.layers.70.mlp.experts.237.up_proj", + "model.layers.70.mlp.experts.238.down_proj", + "model.layers.70.mlp.experts.238.gate_proj", + "model.layers.70.mlp.experts.238.up_proj", + "model.layers.70.mlp.experts.239.down_proj", + "model.layers.70.mlp.experts.239.gate_proj", + "model.layers.70.mlp.experts.239.up_proj", + "model.layers.70.mlp.experts.24.down_proj", + "model.layers.70.mlp.experts.24.gate_proj", + "model.layers.70.mlp.experts.24.up_proj", + "model.layers.70.mlp.experts.240.down_proj", + "model.layers.70.mlp.experts.240.gate_proj", + "model.layers.70.mlp.experts.240.up_proj", + "model.layers.70.mlp.experts.241.down_proj", + "model.layers.70.mlp.experts.241.gate_proj", + "model.layers.70.mlp.experts.241.up_proj", + "model.layers.70.mlp.experts.242.down_proj", + "model.layers.70.mlp.experts.242.gate_proj", + "model.layers.70.mlp.experts.242.up_proj", + "model.layers.70.mlp.experts.243.down_proj", + "model.layers.70.mlp.experts.243.gate_proj", + "model.layers.70.mlp.experts.243.up_proj", + "model.layers.70.mlp.experts.244.down_proj", + "model.layers.70.mlp.experts.244.gate_proj", + "model.layers.70.mlp.experts.244.up_proj", + "model.layers.70.mlp.experts.245.down_proj", + "model.layers.70.mlp.experts.245.gate_proj", + "model.layers.70.mlp.experts.245.up_proj", + "model.layers.70.mlp.experts.246.down_proj", + "model.layers.70.mlp.experts.246.gate_proj", + "model.layers.70.mlp.experts.246.up_proj", + "model.layers.70.mlp.experts.247.down_proj", + "model.layers.70.mlp.experts.247.gate_proj", + "model.layers.70.mlp.experts.247.up_proj", + "model.layers.70.mlp.experts.248.down_proj", + "model.layers.70.mlp.experts.248.gate_proj", + "model.layers.70.mlp.experts.248.up_proj", + "model.layers.70.mlp.experts.249.down_proj", + "model.layers.70.mlp.experts.249.gate_proj", + "model.layers.70.mlp.experts.249.up_proj", + "model.layers.70.mlp.experts.25.down_proj", + "model.layers.70.mlp.experts.25.gate_proj", + "model.layers.70.mlp.experts.25.up_proj", + "model.layers.70.mlp.experts.250.down_proj", + "model.layers.70.mlp.experts.250.gate_proj", + "model.layers.70.mlp.experts.250.up_proj", + "model.layers.70.mlp.experts.251.down_proj", + "model.layers.70.mlp.experts.251.gate_proj", + "model.layers.70.mlp.experts.251.up_proj", + "model.layers.70.mlp.experts.252.down_proj", + "model.layers.70.mlp.experts.252.gate_proj", + "model.layers.70.mlp.experts.252.up_proj", + "model.layers.70.mlp.experts.253.down_proj", + "model.layers.70.mlp.experts.253.gate_proj", + "model.layers.70.mlp.experts.253.up_proj", + "model.layers.70.mlp.experts.254.down_proj", + "model.layers.70.mlp.experts.254.gate_proj", + "model.layers.70.mlp.experts.254.up_proj", + "model.layers.70.mlp.experts.255.down_proj", + "model.layers.70.mlp.experts.255.gate_proj", + "model.layers.70.mlp.experts.255.up_proj", + "model.layers.70.mlp.experts.26.down_proj", + "model.layers.70.mlp.experts.26.gate_proj", + "model.layers.70.mlp.experts.26.up_proj", + "model.layers.70.mlp.experts.27.down_proj", + "model.layers.70.mlp.experts.27.gate_proj", + "model.layers.70.mlp.experts.27.up_proj", + "model.layers.70.mlp.experts.28.down_proj", + "model.layers.70.mlp.experts.28.gate_proj", + "model.layers.70.mlp.experts.28.up_proj", + "model.layers.70.mlp.experts.29.down_proj", + "model.layers.70.mlp.experts.29.gate_proj", + "model.layers.70.mlp.experts.29.up_proj", + "model.layers.70.mlp.experts.3.down_proj", + "model.layers.70.mlp.experts.3.gate_proj", + "model.layers.70.mlp.experts.3.up_proj", + "model.layers.70.mlp.experts.30.down_proj", + "model.layers.70.mlp.experts.30.gate_proj", + "model.layers.70.mlp.experts.30.up_proj", + "model.layers.70.mlp.experts.31.down_proj", + "model.layers.70.mlp.experts.31.gate_proj", + "model.layers.70.mlp.experts.31.up_proj", + "model.layers.70.mlp.experts.32.down_proj", + "model.layers.70.mlp.experts.32.gate_proj", + "model.layers.70.mlp.experts.32.up_proj", + "model.layers.70.mlp.experts.33.down_proj", + "model.layers.70.mlp.experts.33.gate_proj", + "model.layers.70.mlp.experts.33.up_proj", + "model.layers.70.mlp.experts.34.down_proj", + "model.layers.70.mlp.experts.34.gate_proj", + "model.layers.70.mlp.experts.34.up_proj", + "model.layers.70.mlp.experts.35.down_proj", + "model.layers.70.mlp.experts.35.gate_proj", + "model.layers.70.mlp.experts.35.up_proj", + "model.layers.70.mlp.experts.36.down_proj", + "model.layers.70.mlp.experts.36.gate_proj", + "model.layers.70.mlp.experts.36.up_proj", + "model.layers.70.mlp.experts.37.down_proj", + "model.layers.70.mlp.experts.37.gate_proj", + "model.layers.70.mlp.experts.37.up_proj", + "model.layers.70.mlp.experts.38.down_proj", + "model.layers.70.mlp.experts.38.gate_proj", + "model.layers.70.mlp.experts.38.up_proj", + "model.layers.70.mlp.experts.39.down_proj", + "model.layers.70.mlp.experts.39.gate_proj", + "model.layers.70.mlp.experts.39.up_proj", + "model.layers.70.mlp.experts.4.down_proj", + "model.layers.70.mlp.experts.4.gate_proj", + "model.layers.70.mlp.experts.4.up_proj", + "model.layers.70.mlp.experts.40.down_proj", + "model.layers.70.mlp.experts.40.gate_proj", + "model.layers.70.mlp.experts.40.up_proj", + "model.layers.70.mlp.experts.41.down_proj", + "model.layers.70.mlp.experts.41.gate_proj", + "model.layers.70.mlp.experts.41.up_proj", + "model.layers.70.mlp.experts.42.down_proj", + "model.layers.70.mlp.experts.42.gate_proj", + "model.layers.70.mlp.experts.42.up_proj", + "model.layers.70.mlp.experts.43.down_proj", + "model.layers.70.mlp.experts.43.gate_proj", + "model.layers.70.mlp.experts.43.up_proj", + "model.layers.70.mlp.experts.44.down_proj", + "model.layers.70.mlp.experts.44.gate_proj", + "model.layers.70.mlp.experts.44.up_proj", + "model.layers.70.mlp.experts.45.down_proj", + "model.layers.70.mlp.experts.45.gate_proj", + "model.layers.70.mlp.experts.45.up_proj", + "model.layers.70.mlp.experts.46.down_proj", + "model.layers.70.mlp.experts.46.gate_proj", + "model.layers.70.mlp.experts.46.up_proj", + "model.layers.70.mlp.experts.47.down_proj", + "model.layers.70.mlp.experts.47.gate_proj", + "model.layers.70.mlp.experts.47.up_proj", + "model.layers.70.mlp.experts.48.down_proj", + "model.layers.70.mlp.experts.48.gate_proj", + "model.layers.70.mlp.experts.48.up_proj", + "model.layers.70.mlp.experts.49.down_proj", + "model.layers.70.mlp.experts.49.gate_proj", + "model.layers.70.mlp.experts.49.up_proj", + "model.layers.70.mlp.experts.5.down_proj", + "model.layers.70.mlp.experts.5.gate_proj", + "model.layers.70.mlp.experts.5.up_proj", + "model.layers.70.mlp.experts.50.down_proj", + "model.layers.70.mlp.experts.50.gate_proj", + "model.layers.70.mlp.experts.50.up_proj", + "model.layers.70.mlp.experts.51.down_proj", + "model.layers.70.mlp.experts.51.gate_proj", + "model.layers.70.mlp.experts.51.up_proj", + "model.layers.70.mlp.experts.52.down_proj", + "model.layers.70.mlp.experts.52.gate_proj", + "model.layers.70.mlp.experts.52.up_proj", + "model.layers.70.mlp.experts.53.down_proj", + "model.layers.70.mlp.experts.53.gate_proj", + "model.layers.70.mlp.experts.53.up_proj", + "model.layers.70.mlp.experts.54.down_proj", + "model.layers.70.mlp.experts.54.gate_proj", + "model.layers.70.mlp.experts.54.up_proj", + "model.layers.70.mlp.experts.55.down_proj", + "model.layers.70.mlp.experts.55.gate_proj", + "model.layers.70.mlp.experts.55.up_proj", + "model.layers.70.mlp.experts.56.down_proj", + "model.layers.70.mlp.experts.56.gate_proj", + "model.layers.70.mlp.experts.56.up_proj", + "model.layers.70.mlp.experts.57.down_proj", + "model.layers.70.mlp.experts.57.gate_proj", + "model.layers.70.mlp.experts.57.up_proj", + "model.layers.70.mlp.experts.58.down_proj", + "model.layers.70.mlp.experts.58.gate_proj", + "model.layers.70.mlp.experts.58.up_proj", + "model.layers.70.mlp.experts.59.down_proj", + "model.layers.70.mlp.experts.59.gate_proj", + "model.layers.70.mlp.experts.59.up_proj", + "model.layers.70.mlp.experts.6.down_proj", + "model.layers.70.mlp.experts.6.gate_proj", + "model.layers.70.mlp.experts.6.up_proj", + "model.layers.70.mlp.experts.60.down_proj", + "model.layers.70.mlp.experts.60.gate_proj", + "model.layers.70.mlp.experts.60.up_proj", + "model.layers.70.mlp.experts.61.down_proj", + "model.layers.70.mlp.experts.61.gate_proj", + "model.layers.70.mlp.experts.61.up_proj", + "model.layers.70.mlp.experts.62.down_proj", + "model.layers.70.mlp.experts.62.gate_proj", + "model.layers.70.mlp.experts.62.up_proj", + "model.layers.70.mlp.experts.63.down_proj", + "model.layers.70.mlp.experts.63.gate_proj", + "model.layers.70.mlp.experts.63.up_proj", + "model.layers.70.mlp.experts.64.down_proj", + "model.layers.70.mlp.experts.64.gate_proj", + "model.layers.70.mlp.experts.64.up_proj", + "model.layers.70.mlp.experts.65.down_proj", + "model.layers.70.mlp.experts.65.gate_proj", + "model.layers.70.mlp.experts.65.up_proj", + "model.layers.70.mlp.experts.66.down_proj", + "model.layers.70.mlp.experts.66.gate_proj", + "model.layers.70.mlp.experts.66.up_proj", + "model.layers.70.mlp.experts.67.down_proj", + "model.layers.70.mlp.experts.67.gate_proj", + "model.layers.70.mlp.experts.67.up_proj", + "model.layers.70.mlp.experts.68.down_proj", + "model.layers.70.mlp.experts.68.gate_proj", + "model.layers.70.mlp.experts.68.up_proj", + "model.layers.70.mlp.experts.69.down_proj", + "model.layers.70.mlp.experts.69.gate_proj", + "model.layers.70.mlp.experts.69.up_proj", + "model.layers.70.mlp.experts.7.down_proj", + "model.layers.70.mlp.experts.7.gate_proj", + "model.layers.70.mlp.experts.7.up_proj", + "model.layers.70.mlp.experts.70.down_proj", + "model.layers.70.mlp.experts.70.gate_proj", + "model.layers.70.mlp.experts.70.up_proj", + "model.layers.70.mlp.experts.71.down_proj", + "model.layers.70.mlp.experts.71.gate_proj", + "model.layers.70.mlp.experts.71.up_proj", + "model.layers.70.mlp.experts.72.down_proj", + "model.layers.70.mlp.experts.72.gate_proj", + "model.layers.70.mlp.experts.72.up_proj", + "model.layers.70.mlp.experts.73.down_proj", + "model.layers.70.mlp.experts.73.gate_proj", + "model.layers.70.mlp.experts.73.up_proj", + "model.layers.70.mlp.experts.74.down_proj", + "model.layers.70.mlp.experts.74.gate_proj", + "model.layers.70.mlp.experts.74.up_proj", + "model.layers.70.mlp.experts.75.down_proj", + "model.layers.70.mlp.experts.75.gate_proj", + "model.layers.70.mlp.experts.75.up_proj", + "model.layers.70.mlp.experts.76.down_proj", + "model.layers.70.mlp.experts.76.gate_proj", + "model.layers.70.mlp.experts.76.up_proj", + "model.layers.70.mlp.experts.77.down_proj", + "model.layers.70.mlp.experts.77.gate_proj", + "model.layers.70.mlp.experts.77.up_proj", + "model.layers.70.mlp.experts.78.down_proj", + "model.layers.70.mlp.experts.78.gate_proj", + "model.layers.70.mlp.experts.78.up_proj", + "model.layers.70.mlp.experts.79.down_proj", + "model.layers.70.mlp.experts.79.gate_proj", + "model.layers.70.mlp.experts.79.up_proj", + "model.layers.70.mlp.experts.8.down_proj", + "model.layers.70.mlp.experts.8.gate_proj", + "model.layers.70.mlp.experts.8.up_proj", + "model.layers.70.mlp.experts.80.down_proj", + "model.layers.70.mlp.experts.80.gate_proj", + "model.layers.70.mlp.experts.80.up_proj", + "model.layers.70.mlp.experts.81.down_proj", + "model.layers.70.mlp.experts.81.gate_proj", + "model.layers.70.mlp.experts.81.up_proj", + "model.layers.70.mlp.experts.82.down_proj", + "model.layers.70.mlp.experts.82.gate_proj", + "model.layers.70.mlp.experts.82.up_proj", + "model.layers.70.mlp.experts.83.down_proj", + "model.layers.70.mlp.experts.83.gate_proj", + "model.layers.70.mlp.experts.83.up_proj", + "model.layers.70.mlp.experts.84.down_proj", + "model.layers.70.mlp.experts.84.gate_proj", + "model.layers.70.mlp.experts.84.up_proj", + "model.layers.70.mlp.experts.85.down_proj", + "model.layers.70.mlp.experts.85.gate_proj", + "model.layers.70.mlp.experts.85.up_proj", + "model.layers.70.mlp.experts.86.down_proj", + "model.layers.70.mlp.experts.86.gate_proj", + "model.layers.70.mlp.experts.86.up_proj", + "model.layers.70.mlp.experts.87.down_proj", + "model.layers.70.mlp.experts.87.gate_proj", + "model.layers.70.mlp.experts.87.up_proj", + "model.layers.70.mlp.experts.88.down_proj", + "model.layers.70.mlp.experts.88.gate_proj", + "model.layers.70.mlp.experts.88.up_proj", + "model.layers.70.mlp.experts.89.down_proj", + "model.layers.70.mlp.experts.89.gate_proj", + "model.layers.70.mlp.experts.89.up_proj", + "model.layers.70.mlp.experts.9.down_proj", + "model.layers.70.mlp.experts.9.gate_proj", + "model.layers.70.mlp.experts.9.up_proj", + "model.layers.70.mlp.experts.90.down_proj", + "model.layers.70.mlp.experts.90.gate_proj", + "model.layers.70.mlp.experts.90.up_proj", + "model.layers.70.mlp.experts.91.down_proj", + "model.layers.70.mlp.experts.91.gate_proj", + "model.layers.70.mlp.experts.91.up_proj", + "model.layers.70.mlp.experts.92.down_proj", + "model.layers.70.mlp.experts.92.gate_proj", + "model.layers.70.mlp.experts.92.up_proj", + "model.layers.70.mlp.experts.93.down_proj", + "model.layers.70.mlp.experts.93.gate_proj", + "model.layers.70.mlp.experts.93.up_proj", + "model.layers.70.mlp.experts.94.down_proj", + "model.layers.70.mlp.experts.94.gate_proj", + "model.layers.70.mlp.experts.94.up_proj", + "model.layers.70.mlp.experts.95.down_proj", + "model.layers.70.mlp.experts.95.gate_proj", + "model.layers.70.mlp.experts.95.up_proj", + "model.layers.70.mlp.experts.96.down_proj", + "model.layers.70.mlp.experts.96.gate_proj", + "model.layers.70.mlp.experts.96.up_proj", + "model.layers.70.mlp.experts.97.down_proj", + "model.layers.70.mlp.experts.97.gate_proj", + "model.layers.70.mlp.experts.97.up_proj", + "model.layers.70.mlp.experts.98.down_proj", + "model.layers.70.mlp.experts.98.gate_proj", + "model.layers.70.mlp.experts.98.up_proj", + "model.layers.70.mlp.experts.99.down_proj", + "model.layers.70.mlp.experts.99.gate_proj", + "model.layers.70.mlp.experts.99.up_proj", + "model.layers.71.mlp.experts.0.down_proj", + "model.layers.71.mlp.experts.0.gate_proj", + "model.layers.71.mlp.experts.0.up_proj", + "model.layers.71.mlp.experts.1.down_proj", + "model.layers.71.mlp.experts.1.gate_proj", + "model.layers.71.mlp.experts.1.up_proj", + "model.layers.71.mlp.experts.10.down_proj", + "model.layers.71.mlp.experts.10.gate_proj", + "model.layers.71.mlp.experts.10.up_proj", + "model.layers.71.mlp.experts.100.down_proj", + "model.layers.71.mlp.experts.100.gate_proj", + "model.layers.71.mlp.experts.100.up_proj", + "model.layers.71.mlp.experts.101.down_proj", + "model.layers.71.mlp.experts.101.gate_proj", + "model.layers.71.mlp.experts.101.up_proj", + "model.layers.71.mlp.experts.102.down_proj", + "model.layers.71.mlp.experts.102.gate_proj", + "model.layers.71.mlp.experts.102.up_proj", + "model.layers.71.mlp.experts.103.down_proj", + "model.layers.71.mlp.experts.103.gate_proj", + "model.layers.71.mlp.experts.103.up_proj", + "model.layers.71.mlp.experts.104.down_proj", + "model.layers.71.mlp.experts.104.gate_proj", + "model.layers.71.mlp.experts.104.up_proj", + "model.layers.71.mlp.experts.105.down_proj", + "model.layers.71.mlp.experts.105.gate_proj", + "model.layers.71.mlp.experts.105.up_proj", + "model.layers.71.mlp.experts.106.down_proj", + "model.layers.71.mlp.experts.106.gate_proj", + "model.layers.71.mlp.experts.106.up_proj", + "model.layers.71.mlp.experts.107.down_proj", + "model.layers.71.mlp.experts.107.gate_proj", + "model.layers.71.mlp.experts.107.up_proj", + "model.layers.71.mlp.experts.108.down_proj", + "model.layers.71.mlp.experts.108.gate_proj", + "model.layers.71.mlp.experts.108.up_proj", + "model.layers.71.mlp.experts.109.down_proj", + "model.layers.71.mlp.experts.109.gate_proj", + "model.layers.71.mlp.experts.109.up_proj", + "model.layers.71.mlp.experts.11.down_proj", + "model.layers.71.mlp.experts.11.gate_proj", + "model.layers.71.mlp.experts.11.up_proj", + "model.layers.71.mlp.experts.110.down_proj", + "model.layers.71.mlp.experts.110.gate_proj", + "model.layers.71.mlp.experts.110.up_proj", + "model.layers.71.mlp.experts.111.down_proj", + "model.layers.71.mlp.experts.111.gate_proj", + "model.layers.71.mlp.experts.111.up_proj", + "model.layers.71.mlp.experts.112.down_proj", + "model.layers.71.mlp.experts.112.gate_proj", + "model.layers.71.mlp.experts.112.up_proj", + "model.layers.71.mlp.experts.113.down_proj", + "model.layers.71.mlp.experts.113.gate_proj", + "model.layers.71.mlp.experts.113.up_proj", + "model.layers.71.mlp.experts.114.down_proj", + "model.layers.71.mlp.experts.114.gate_proj", + "model.layers.71.mlp.experts.114.up_proj", + "model.layers.71.mlp.experts.115.down_proj", + "model.layers.71.mlp.experts.115.gate_proj", + "model.layers.71.mlp.experts.115.up_proj", + "model.layers.71.mlp.experts.116.down_proj", + "model.layers.71.mlp.experts.116.gate_proj", + "model.layers.71.mlp.experts.116.up_proj", + "model.layers.71.mlp.experts.117.down_proj", + "model.layers.71.mlp.experts.117.gate_proj", + "model.layers.71.mlp.experts.117.up_proj", + "model.layers.71.mlp.experts.118.down_proj", + "model.layers.71.mlp.experts.118.gate_proj", + "model.layers.71.mlp.experts.118.up_proj", + "model.layers.71.mlp.experts.119.down_proj", + "model.layers.71.mlp.experts.119.gate_proj", + "model.layers.71.mlp.experts.119.up_proj", + "model.layers.71.mlp.experts.12.down_proj", + "model.layers.71.mlp.experts.12.gate_proj", + "model.layers.71.mlp.experts.12.up_proj", + "model.layers.71.mlp.experts.120.down_proj", + "model.layers.71.mlp.experts.120.gate_proj", + "model.layers.71.mlp.experts.120.up_proj", + "model.layers.71.mlp.experts.121.down_proj", + "model.layers.71.mlp.experts.121.gate_proj", + "model.layers.71.mlp.experts.121.up_proj", + "model.layers.71.mlp.experts.122.down_proj", + "model.layers.71.mlp.experts.122.gate_proj", + "model.layers.71.mlp.experts.122.up_proj", + "model.layers.71.mlp.experts.123.down_proj", + "model.layers.71.mlp.experts.123.gate_proj", + "model.layers.71.mlp.experts.123.up_proj", + "model.layers.71.mlp.experts.124.down_proj", + "model.layers.71.mlp.experts.124.gate_proj", + "model.layers.71.mlp.experts.124.up_proj", + "model.layers.71.mlp.experts.125.down_proj", + "model.layers.71.mlp.experts.125.gate_proj", + "model.layers.71.mlp.experts.125.up_proj", + "model.layers.71.mlp.experts.126.down_proj", + "model.layers.71.mlp.experts.126.gate_proj", + "model.layers.71.mlp.experts.126.up_proj", + "model.layers.71.mlp.experts.127.down_proj", + "model.layers.71.mlp.experts.127.gate_proj", + "model.layers.71.mlp.experts.127.up_proj", + "model.layers.71.mlp.experts.128.down_proj", + "model.layers.71.mlp.experts.128.gate_proj", + "model.layers.71.mlp.experts.128.up_proj", + "model.layers.71.mlp.experts.129.down_proj", + "model.layers.71.mlp.experts.129.gate_proj", + "model.layers.71.mlp.experts.129.up_proj", + "model.layers.71.mlp.experts.13.down_proj", + "model.layers.71.mlp.experts.13.gate_proj", + "model.layers.71.mlp.experts.13.up_proj", + "model.layers.71.mlp.experts.130.down_proj", + "model.layers.71.mlp.experts.130.gate_proj", + "model.layers.71.mlp.experts.130.up_proj", + "model.layers.71.mlp.experts.131.down_proj", + "model.layers.71.mlp.experts.131.gate_proj", + "model.layers.71.mlp.experts.131.up_proj", + "model.layers.71.mlp.experts.132.down_proj", + "model.layers.71.mlp.experts.132.gate_proj", + "model.layers.71.mlp.experts.132.up_proj", + "model.layers.71.mlp.experts.133.down_proj", + "model.layers.71.mlp.experts.133.gate_proj", + "model.layers.71.mlp.experts.133.up_proj", + "model.layers.71.mlp.experts.134.down_proj", + "model.layers.71.mlp.experts.134.gate_proj", + "model.layers.71.mlp.experts.134.up_proj", + "model.layers.71.mlp.experts.135.down_proj", + "model.layers.71.mlp.experts.135.gate_proj", + "model.layers.71.mlp.experts.135.up_proj", + "model.layers.71.mlp.experts.136.down_proj", + "model.layers.71.mlp.experts.136.gate_proj", + "model.layers.71.mlp.experts.136.up_proj", + "model.layers.71.mlp.experts.137.down_proj", + "model.layers.71.mlp.experts.137.gate_proj", + "model.layers.71.mlp.experts.137.up_proj", + "model.layers.71.mlp.experts.138.down_proj", + "model.layers.71.mlp.experts.138.gate_proj", + "model.layers.71.mlp.experts.138.up_proj", + "model.layers.71.mlp.experts.139.down_proj", + "model.layers.71.mlp.experts.139.gate_proj", + "model.layers.71.mlp.experts.139.up_proj", + "model.layers.71.mlp.experts.14.down_proj", + "model.layers.71.mlp.experts.14.gate_proj", + "model.layers.71.mlp.experts.14.up_proj", + "model.layers.71.mlp.experts.140.down_proj", + "model.layers.71.mlp.experts.140.gate_proj", + "model.layers.71.mlp.experts.140.up_proj", + "model.layers.71.mlp.experts.141.down_proj", + "model.layers.71.mlp.experts.141.gate_proj", + "model.layers.71.mlp.experts.141.up_proj", + "model.layers.71.mlp.experts.142.down_proj", + "model.layers.71.mlp.experts.142.gate_proj", + "model.layers.71.mlp.experts.142.up_proj", + "model.layers.71.mlp.experts.143.down_proj", + "model.layers.71.mlp.experts.143.gate_proj", + "model.layers.71.mlp.experts.143.up_proj", + "model.layers.71.mlp.experts.144.down_proj", + "model.layers.71.mlp.experts.144.gate_proj", + "model.layers.71.mlp.experts.144.up_proj", + "model.layers.71.mlp.experts.145.down_proj", + "model.layers.71.mlp.experts.145.gate_proj", + "model.layers.71.mlp.experts.145.up_proj", + "model.layers.71.mlp.experts.146.down_proj", + "model.layers.71.mlp.experts.146.gate_proj", + "model.layers.71.mlp.experts.146.up_proj", + "model.layers.71.mlp.experts.147.down_proj", + "model.layers.71.mlp.experts.147.gate_proj", + "model.layers.71.mlp.experts.147.up_proj", + "model.layers.71.mlp.experts.148.down_proj", + "model.layers.71.mlp.experts.148.gate_proj", + "model.layers.71.mlp.experts.148.up_proj", + "model.layers.71.mlp.experts.149.down_proj", + "model.layers.71.mlp.experts.149.gate_proj", + "model.layers.71.mlp.experts.149.up_proj", + "model.layers.71.mlp.experts.15.down_proj", + "model.layers.71.mlp.experts.15.gate_proj", + "model.layers.71.mlp.experts.15.up_proj", + "model.layers.71.mlp.experts.150.down_proj", + "model.layers.71.mlp.experts.150.gate_proj", + "model.layers.71.mlp.experts.150.up_proj", + "model.layers.71.mlp.experts.151.down_proj", + "model.layers.71.mlp.experts.151.gate_proj", + "model.layers.71.mlp.experts.151.up_proj", + "model.layers.71.mlp.experts.152.down_proj", + "model.layers.71.mlp.experts.152.gate_proj", + "model.layers.71.mlp.experts.152.up_proj", + "model.layers.71.mlp.experts.153.down_proj", + "model.layers.71.mlp.experts.153.gate_proj", + "model.layers.71.mlp.experts.153.up_proj", + "model.layers.71.mlp.experts.154.down_proj", + "model.layers.71.mlp.experts.154.gate_proj", + "model.layers.71.mlp.experts.154.up_proj", + "model.layers.71.mlp.experts.155.down_proj", + "model.layers.71.mlp.experts.155.gate_proj", + "model.layers.71.mlp.experts.155.up_proj", + "model.layers.71.mlp.experts.156.down_proj", + "model.layers.71.mlp.experts.156.gate_proj", + "model.layers.71.mlp.experts.156.up_proj", + "model.layers.71.mlp.experts.157.down_proj", + "model.layers.71.mlp.experts.157.gate_proj", + "model.layers.71.mlp.experts.157.up_proj", + "model.layers.71.mlp.experts.158.down_proj", + "model.layers.71.mlp.experts.158.gate_proj", + "model.layers.71.mlp.experts.158.up_proj", + "model.layers.71.mlp.experts.159.down_proj", + "model.layers.71.mlp.experts.159.gate_proj", + "model.layers.71.mlp.experts.159.up_proj", + "model.layers.71.mlp.experts.16.down_proj", + "model.layers.71.mlp.experts.16.gate_proj", + "model.layers.71.mlp.experts.16.up_proj", + "model.layers.71.mlp.experts.160.down_proj", + "model.layers.71.mlp.experts.160.gate_proj", + "model.layers.71.mlp.experts.160.up_proj", + "model.layers.71.mlp.experts.161.down_proj", + "model.layers.71.mlp.experts.161.gate_proj", + "model.layers.71.mlp.experts.161.up_proj", + "model.layers.71.mlp.experts.162.down_proj", + "model.layers.71.mlp.experts.162.gate_proj", + "model.layers.71.mlp.experts.162.up_proj", + "model.layers.71.mlp.experts.163.down_proj", + "model.layers.71.mlp.experts.163.gate_proj", + "model.layers.71.mlp.experts.163.up_proj", + "model.layers.71.mlp.experts.164.down_proj", + "model.layers.71.mlp.experts.164.gate_proj", + "model.layers.71.mlp.experts.164.up_proj", + "model.layers.71.mlp.experts.165.down_proj", + "model.layers.71.mlp.experts.165.gate_proj", + "model.layers.71.mlp.experts.165.up_proj", + "model.layers.71.mlp.experts.166.down_proj", + "model.layers.71.mlp.experts.166.gate_proj", + "model.layers.71.mlp.experts.166.up_proj", + "model.layers.71.mlp.experts.167.down_proj", + "model.layers.71.mlp.experts.167.gate_proj", + "model.layers.71.mlp.experts.167.up_proj", + "model.layers.71.mlp.experts.168.down_proj", + "model.layers.71.mlp.experts.168.gate_proj", + "model.layers.71.mlp.experts.168.up_proj", + "model.layers.71.mlp.experts.169.down_proj", + "model.layers.71.mlp.experts.169.gate_proj", + "model.layers.71.mlp.experts.169.up_proj", + "model.layers.71.mlp.experts.17.down_proj", + "model.layers.71.mlp.experts.17.gate_proj", + "model.layers.71.mlp.experts.17.up_proj", + "model.layers.71.mlp.experts.170.down_proj", + "model.layers.71.mlp.experts.170.gate_proj", + "model.layers.71.mlp.experts.170.up_proj", + "model.layers.71.mlp.experts.171.down_proj", + "model.layers.71.mlp.experts.171.gate_proj", + "model.layers.71.mlp.experts.171.up_proj", + "model.layers.71.mlp.experts.172.down_proj", + "model.layers.71.mlp.experts.172.gate_proj", + "model.layers.71.mlp.experts.172.up_proj", + "model.layers.71.mlp.experts.173.down_proj", + "model.layers.71.mlp.experts.173.gate_proj", + "model.layers.71.mlp.experts.173.up_proj", + "model.layers.71.mlp.experts.174.down_proj", + "model.layers.71.mlp.experts.174.gate_proj", + "model.layers.71.mlp.experts.174.up_proj", + "model.layers.71.mlp.experts.175.down_proj", + "model.layers.71.mlp.experts.175.gate_proj", + "model.layers.71.mlp.experts.175.up_proj", + "model.layers.71.mlp.experts.176.down_proj", + "model.layers.71.mlp.experts.176.gate_proj", + "model.layers.71.mlp.experts.176.up_proj", + "model.layers.71.mlp.experts.177.down_proj", + "model.layers.71.mlp.experts.177.gate_proj", + "model.layers.71.mlp.experts.177.up_proj", + "model.layers.71.mlp.experts.178.down_proj", + "model.layers.71.mlp.experts.178.gate_proj", + "model.layers.71.mlp.experts.178.up_proj", + "model.layers.71.mlp.experts.179.down_proj", + "model.layers.71.mlp.experts.179.gate_proj", + "model.layers.71.mlp.experts.179.up_proj", + "model.layers.71.mlp.experts.18.down_proj", + "model.layers.71.mlp.experts.18.gate_proj", + "model.layers.71.mlp.experts.18.up_proj", + "model.layers.71.mlp.experts.180.down_proj", + "model.layers.71.mlp.experts.180.gate_proj", + "model.layers.71.mlp.experts.180.up_proj", + "model.layers.71.mlp.experts.181.down_proj", + "model.layers.71.mlp.experts.181.gate_proj", + "model.layers.71.mlp.experts.181.up_proj", + "model.layers.71.mlp.experts.182.down_proj", + "model.layers.71.mlp.experts.182.gate_proj", + "model.layers.71.mlp.experts.182.up_proj", + "model.layers.71.mlp.experts.183.down_proj", + "model.layers.71.mlp.experts.183.gate_proj", + "model.layers.71.mlp.experts.183.up_proj", + "model.layers.71.mlp.experts.184.down_proj", + "model.layers.71.mlp.experts.184.gate_proj", + "model.layers.71.mlp.experts.184.up_proj", + "model.layers.71.mlp.experts.185.down_proj", + "model.layers.71.mlp.experts.185.gate_proj", + "model.layers.71.mlp.experts.185.up_proj", + "model.layers.71.mlp.experts.186.down_proj", + "model.layers.71.mlp.experts.186.gate_proj", + "model.layers.71.mlp.experts.186.up_proj", + "model.layers.71.mlp.experts.187.down_proj", + "model.layers.71.mlp.experts.187.gate_proj", + "model.layers.71.mlp.experts.187.up_proj", + "model.layers.71.mlp.experts.188.down_proj", + "model.layers.71.mlp.experts.188.gate_proj", + "model.layers.71.mlp.experts.188.up_proj", + "model.layers.71.mlp.experts.189.down_proj", + "model.layers.71.mlp.experts.189.gate_proj", + "model.layers.71.mlp.experts.189.up_proj", + "model.layers.71.mlp.experts.19.down_proj", + "model.layers.71.mlp.experts.19.gate_proj", + "model.layers.71.mlp.experts.19.up_proj", + "model.layers.71.mlp.experts.190.down_proj", + "model.layers.71.mlp.experts.190.gate_proj", + "model.layers.71.mlp.experts.190.up_proj", + "model.layers.71.mlp.experts.191.down_proj", + "model.layers.71.mlp.experts.191.gate_proj", + "model.layers.71.mlp.experts.191.up_proj", + "model.layers.71.mlp.experts.192.down_proj", + "model.layers.71.mlp.experts.192.gate_proj", + "model.layers.71.mlp.experts.192.up_proj", + "model.layers.71.mlp.experts.193.down_proj", + "model.layers.71.mlp.experts.193.gate_proj", + "model.layers.71.mlp.experts.193.up_proj", + "model.layers.71.mlp.experts.194.down_proj", + "model.layers.71.mlp.experts.194.gate_proj", + "model.layers.71.mlp.experts.194.up_proj", + "model.layers.71.mlp.experts.195.down_proj", + "model.layers.71.mlp.experts.195.gate_proj", + "model.layers.71.mlp.experts.195.up_proj", + "model.layers.71.mlp.experts.196.down_proj", + "model.layers.71.mlp.experts.196.gate_proj", + "model.layers.71.mlp.experts.196.up_proj", + "model.layers.71.mlp.experts.197.down_proj", + "model.layers.71.mlp.experts.197.gate_proj", + "model.layers.71.mlp.experts.197.up_proj", + "model.layers.71.mlp.experts.198.down_proj", + "model.layers.71.mlp.experts.198.gate_proj", + "model.layers.71.mlp.experts.198.up_proj", + "model.layers.71.mlp.experts.199.down_proj", + "model.layers.71.mlp.experts.199.gate_proj", + "model.layers.71.mlp.experts.199.up_proj", + "model.layers.71.mlp.experts.2.down_proj", + "model.layers.71.mlp.experts.2.gate_proj", + "model.layers.71.mlp.experts.2.up_proj", + "model.layers.71.mlp.experts.20.down_proj", + "model.layers.71.mlp.experts.20.gate_proj", + "model.layers.71.mlp.experts.20.up_proj", + "model.layers.71.mlp.experts.200.down_proj", + "model.layers.71.mlp.experts.200.gate_proj", + "model.layers.71.mlp.experts.200.up_proj", + "model.layers.71.mlp.experts.201.down_proj", + "model.layers.71.mlp.experts.201.gate_proj", + "model.layers.71.mlp.experts.201.up_proj", + "model.layers.71.mlp.experts.202.down_proj", + "model.layers.71.mlp.experts.202.gate_proj", + "model.layers.71.mlp.experts.202.up_proj", + "model.layers.71.mlp.experts.203.down_proj", + "model.layers.71.mlp.experts.203.gate_proj", + "model.layers.71.mlp.experts.203.up_proj", + "model.layers.71.mlp.experts.204.down_proj", + "model.layers.71.mlp.experts.204.gate_proj", + "model.layers.71.mlp.experts.204.up_proj", + "model.layers.71.mlp.experts.205.down_proj", + "model.layers.71.mlp.experts.205.gate_proj", + "model.layers.71.mlp.experts.205.up_proj", + "model.layers.71.mlp.experts.206.down_proj", + "model.layers.71.mlp.experts.206.gate_proj", + "model.layers.71.mlp.experts.206.up_proj", + "model.layers.71.mlp.experts.207.down_proj", + "model.layers.71.mlp.experts.207.gate_proj", + "model.layers.71.mlp.experts.207.up_proj", + "model.layers.71.mlp.experts.208.down_proj", + "model.layers.71.mlp.experts.208.gate_proj", + "model.layers.71.mlp.experts.208.up_proj", + "model.layers.71.mlp.experts.209.down_proj", + "model.layers.71.mlp.experts.209.gate_proj", + "model.layers.71.mlp.experts.209.up_proj", + "model.layers.71.mlp.experts.21.down_proj", + "model.layers.71.mlp.experts.21.gate_proj", + "model.layers.71.mlp.experts.21.up_proj", + "model.layers.71.mlp.experts.210.down_proj", + "model.layers.71.mlp.experts.210.gate_proj", + "model.layers.71.mlp.experts.210.up_proj", + "model.layers.71.mlp.experts.211.down_proj", + "model.layers.71.mlp.experts.211.gate_proj", + "model.layers.71.mlp.experts.211.up_proj", + "model.layers.71.mlp.experts.212.down_proj", + "model.layers.71.mlp.experts.212.gate_proj", + "model.layers.71.mlp.experts.212.up_proj", + "model.layers.71.mlp.experts.213.down_proj", + "model.layers.71.mlp.experts.213.gate_proj", + "model.layers.71.mlp.experts.213.up_proj", + "model.layers.71.mlp.experts.214.down_proj", + "model.layers.71.mlp.experts.214.gate_proj", + "model.layers.71.mlp.experts.214.up_proj", + "model.layers.71.mlp.experts.215.down_proj", + "model.layers.71.mlp.experts.215.gate_proj", + "model.layers.71.mlp.experts.215.up_proj", + "model.layers.71.mlp.experts.216.down_proj", + "model.layers.71.mlp.experts.216.gate_proj", + "model.layers.71.mlp.experts.216.up_proj", + "model.layers.71.mlp.experts.217.down_proj", + "model.layers.71.mlp.experts.217.gate_proj", + "model.layers.71.mlp.experts.217.up_proj", + "model.layers.71.mlp.experts.218.down_proj", + "model.layers.71.mlp.experts.218.gate_proj", + "model.layers.71.mlp.experts.218.up_proj", + "model.layers.71.mlp.experts.219.down_proj", + "model.layers.71.mlp.experts.219.gate_proj", + "model.layers.71.mlp.experts.219.up_proj", + "model.layers.71.mlp.experts.22.down_proj", + "model.layers.71.mlp.experts.22.gate_proj", + "model.layers.71.mlp.experts.22.up_proj", + "model.layers.71.mlp.experts.220.down_proj", + "model.layers.71.mlp.experts.220.gate_proj", + "model.layers.71.mlp.experts.220.up_proj", + "model.layers.71.mlp.experts.221.down_proj", + "model.layers.71.mlp.experts.221.gate_proj", + "model.layers.71.mlp.experts.221.up_proj", + "model.layers.71.mlp.experts.222.down_proj", + "model.layers.71.mlp.experts.222.gate_proj", + "model.layers.71.mlp.experts.222.up_proj", + "model.layers.71.mlp.experts.223.down_proj", + "model.layers.71.mlp.experts.223.gate_proj", + "model.layers.71.mlp.experts.223.up_proj", + "model.layers.71.mlp.experts.224.down_proj", + "model.layers.71.mlp.experts.224.gate_proj", + "model.layers.71.mlp.experts.224.up_proj", + "model.layers.71.mlp.experts.225.down_proj", + "model.layers.71.mlp.experts.225.gate_proj", + "model.layers.71.mlp.experts.225.up_proj", + "model.layers.71.mlp.experts.226.down_proj", + "model.layers.71.mlp.experts.226.gate_proj", + "model.layers.71.mlp.experts.226.up_proj", + "model.layers.71.mlp.experts.227.down_proj", + "model.layers.71.mlp.experts.227.gate_proj", + "model.layers.71.mlp.experts.227.up_proj", + "model.layers.71.mlp.experts.228.down_proj", + "model.layers.71.mlp.experts.228.gate_proj", + "model.layers.71.mlp.experts.228.up_proj", + "model.layers.71.mlp.experts.229.down_proj", + "model.layers.71.mlp.experts.229.gate_proj", + "model.layers.71.mlp.experts.229.up_proj", + "model.layers.71.mlp.experts.23.down_proj", + "model.layers.71.mlp.experts.23.gate_proj", + "model.layers.71.mlp.experts.23.up_proj", + "model.layers.71.mlp.experts.230.down_proj", + "model.layers.71.mlp.experts.230.gate_proj", + "model.layers.71.mlp.experts.230.up_proj", + "model.layers.71.mlp.experts.231.down_proj", + "model.layers.71.mlp.experts.231.gate_proj", + "model.layers.71.mlp.experts.231.up_proj", + "model.layers.71.mlp.experts.232.down_proj", + "model.layers.71.mlp.experts.232.gate_proj", + "model.layers.71.mlp.experts.232.up_proj", + "model.layers.71.mlp.experts.233.down_proj", + "model.layers.71.mlp.experts.233.gate_proj", + "model.layers.71.mlp.experts.233.up_proj", + "model.layers.71.mlp.experts.234.down_proj", + "model.layers.71.mlp.experts.234.gate_proj", + "model.layers.71.mlp.experts.234.up_proj", + "model.layers.71.mlp.experts.235.down_proj", + "model.layers.71.mlp.experts.235.gate_proj", + "model.layers.71.mlp.experts.235.up_proj", + "model.layers.71.mlp.experts.236.down_proj", + "model.layers.71.mlp.experts.236.gate_proj", + "model.layers.71.mlp.experts.236.up_proj", + "model.layers.71.mlp.experts.237.down_proj", + "model.layers.71.mlp.experts.237.gate_proj", + "model.layers.71.mlp.experts.237.up_proj", + "model.layers.71.mlp.experts.238.down_proj", + "model.layers.71.mlp.experts.238.gate_proj", + "model.layers.71.mlp.experts.238.up_proj", + "model.layers.71.mlp.experts.239.down_proj", + "model.layers.71.mlp.experts.239.gate_proj", + "model.layers.71.mlp.experts.239.up_proj", + "model.layers.71.mlp.experts.24.down_proj", + "model.layers.71.mlp.experts.24.gate_proj", + "model.layers.71.mlp.experts.24.up_proj", + "model.layers.71.mlp.experts.240.down_proj", + "model.layers.71.mlp.experts.240.gate_proj", + "model.layers.71.mlp.experts.240.up_proj", + "model.layers.71.mlp.experts.241.down_proj", + "model.layers.71.mlp.experts.241.gate_proj", + "model.layers.71.mlp.experts.241.up_proj", + "model.layers.71.mlp.experts.242.down_proj", + "model.layers.71.mlp.experts.242.gate_proj", + "model.layers.71.mlp.experts.242.up_proj", + "model.layers.71.mlp.experts.243.down_proj", + "model.layers.71.mlp.experts.243.gate_proj", + "model.layers.71.mlp.experts.243.up_proj", + "model.layers.71.mlp.experts.244.down_proj", + "model.layers.71.mlp.experts.244.gate_proj", + "model.layers.71.mlp.experts.244.up_proj", + "model.layers.71.mlp.experts.245.down_proj", + "model.layers.71.mlp.experts.245.gate_proj", + "model.layers.71.mlp.experts.245.up_proj", + "model.layers.71.mlp.experts.246.down_proj", + "model.layers.71.mlp.experts.246.gate_proj", + "model.layers.71.mlp.experts.246.up_proj", + "model.layers.71.mlp.experts.247.down_proj", + "model.layers.71.mlp.experts.247.gate_proj", + "model.layers.71.mlp.experts.247.up_proj", + "model.layers.71.mlp.experts.248.down_proj", + "model.layers.71.mlp.experts.248.gate_proj", + "model.layers.71.mlp.experts.248.up_proj", + "model.layers.71.mlp.experts.249.down_proj", + "model.layers.71.mlp.experts.249.gate_proj", + "model.layers.71.mlp.experts.249.up_proj", + "model.layers.71.mlp.experts.25.down_proj", + "model.layers.71.mlp.experts.25.gate_proj", + "model.layers.71.mlp.experts.25.up_proj", + "model.layers.71.mlp.experts.250.down_proj", + "model.layers.71.mlp.experts.250.gate_proj", + "model.layers.71.mlp.experts.250.up_proj", + "model.layers.71.mlp.experts.251.down_proj", + "model.layers.71.mlp.experts.251.gate_proj", + "model.layers.71.mlp.experts.251.up_proj", + "model.layers.71.mlp.experts.252.down_proj", + "model.layers.71.mlp.experts.252.gate_proj", + "model.layers.71.mlp.experts.252.up_proj", + "model.layers.71.mlp.experts.253.down_proj", + "model.layers.71.mlp.experts.253.gate_proj", + "model.layers.71.mlp.experts.253.up_proj", + "model.layers.71.mlp.experts.254.down_proj", + "model.layers.71.mlp.experts.254.gate_proj", + "model.layers.71.mlp.experts.254.up_proj", + "model.layers.71.mlp.experts.255.down_proj", + "model.layers.71.mlp.experts.255.gate_proj", + "model.layers.71.mlp.experts.255.up_proj", + "model.layers.71.mlp.experts.26.down_proj", + "model.layers.71.mlp.experts.26.gate_proj", + "model.layers.71.mlp.experts.26.up_proj", + "model.layers.71.mlp.experts.27.down_proj", + "model.layers.71.mlp.experts.27.gate_proj", + "model.layers.71.mlp.experts.27.up_proj", + "model.layers.71.mlp.experts.28.down_proj", + "model.layers.71.mlp.experts.28.gate_proj", + "model.layers.71.mlp.experts.28.up_proj", + "model.layers.71.mlp.experts.29.down_proj", + "model.layers.71.mlp.experts.29.gate_proj", + "model.layers.71.mlp.experts.29.up_proj", + "model.layers.71.mlp.experts.3.down_proj", + "model.layers.71.mlp.experts.3.gate_proj", + "model.layers.71.mlp.experts.3.up_proj", + "model.layers.71.mlp.experts.30.down_proj", + "model.layers.71.mlp.experts.30.gate_proj", + "model.layers.71.mlp.experts.30.up_proj", + "model.layers.71.mlp.experts.31.down_proj", + "model.layers.71.mlp.experts.31.gate_proj", + "model.layers.71.mlp.experts.31.up_proj", + "model.layers.71.mlp.experts.32.down_proj", + "model.layers.71.mlp.experts.32.gate_proj", + "model.layers.71.mlp.experts.32.up_proj", + "model.layers.71.mlp.experts.33.down_proj", + "model.layers.71.mlp.experts.33.gate_proj", + "model.layers.71.mlp.experts.33.up_proj", + "model.layers.71.mlp.experts.34.down_proj", + "model.layers.71.mlp.experts.34.gate_proj", + "model.layers.71.mlp.experts.34.up_proj", + "model.layers.71.mlp.experts.35.down_proj", + "model.layers.71.mlp.experts.35.gate_proj", + "model.layers.71.mlp.experts.35.up_proj", + "model.layers.71.mlp.experts.36.down_proj", + "model.layers.71.mlp.experts.36.gate_proj", + "model.layers.71.mlp.experts.36.up_proj", + "model.layers.71.mlp.experts.37.down_proj", + "model.layers.71.mlp.experts.37.gate_proj", + "model.layers.71.mlp.experts.37.up_proj", + "model.layers.71.mlp.experts.38.down_proj", + "model.layers.71.mlp.experts.38.gate_proj", + "model.layers.71.mlp.experts.38.up_proj", + "model.layers.71.mlp.experts.39.down_proj", + "model.layers.71.mlp.experts.39.gate_proj", + "model.layers.71.mlp.experts.39.up_proj", + "model.layers.71.mlp.experts.4.down_proj", + "model.layers.71.mlp.experts.4.gate_proj", + "model.layers.71.mlp.experts.4.up_proj", + "model.layers.71.mlp.experts.40.down_proj", + "model.layers.71.mlp.experts.40.gate_proj", + "model.layers.71.mlp.experts.40.up_proj", + "model.layers.71.mlp.experts.41.down_proj", + "model.layers.71.mlp.experts.41.gate_proj", + "model.layers.71.mlp.experts.41.up_proj", + "model.layers.71.mlp.experts.42.down_proj", + "model.layers.71.mlp.experts.42.gate_proj", + "model.layers.71.mlp.experts.42.up_proj", + "model.layers.71.mlp.experts.43.down_proj", + "model.layers.71.mlp.experts.43.gate_proj", + "model.layers.71.mlp.experts.43.up_proj", + "model.layers.71.mlp.experts.44.down_proj", + "model.layers.71.mlp.experts.44.gate_proj", + "model.layers.71.mlp.experts.44.up_proj", + "model.layers.71.mlp.experts.45.down_proj", + "model.layers.71.mlp.experts.45.gate_proj", + "model.layers.71.mlp.experts.45.up_proj", + "model.layers.71.mlp.experts.46.down_proj", + "model.layers.71.mlp.experts.46.gate_proj", + "model.layers.71.mlp.experts.46.up_proj", + "model.layers.71.mlp.experts.47.down_proj", + "model.layers.71.mlp.experts.47.gate_proj", + "model.layers.71.mlp.experts.47.up_proj", + "model.layers.71.mlp.experts.48.down_proj", + "model.layers.71.mlp.experts.48.gate_proj", + "model.layers.71.mlp.experts.48.up_proj", + "model.layers.71.mlp.experts.49.down_proj", + "model.layers.71.mlp.experts.49.gate_proj", + "model.layers.71.mlp.experts.49.up_proj", + "model.layers.71.mlp.experts.5.down_proj", + "model.layers.71.mlp.experts.5.gate_proj", + "model.layers.71.mlp.experts.5.up_proj", + "model.layers.71.mlp.experts.50.down_proj", + "model.layers.71.mlp.experts.50.gate_proj", + "model.layers.71.mlp.experts.50.up_proj", + "model.layers.71.mlp.experts.51.down_proj", + "model.layers.71.mlp.experts.51.gate_proj", + "model.layers.71.mlp.experts.51.up_proj", + "model.layers.71.mlp.experts.52.down_proj", + "model.layers.71.mlp.experts.52.gate_proj", + "model.layers.71.mlp.experts.52.up_proj", + "model.layers.71.mlp.experts.53.down_proj", + "model.layers.71.mlp.experts.53.gate_proj", + "model.layers.71.mlp.experts.53.up_proj", + "model.layers.71.mlp.experts.54.down_proj", + "model.layers.71.mlp.experts.54.gate_proj", + "model.layers.71.mlp.experts.54.up_proj", + "model.layers.71.mlp.experts.55.down_proj", + "model.layers.71.mlp.experts.55.gate_proj", + "model.layers.71.mlp.experts.55.up_proj", + "model.layers.71.mlp.experts.56.down_proj", + "model.layers.71.mlp.experts.56.gate_proj", + "model.layers.71.mlp.experts.56.up_proj", + "model.layers.71.mlp.experts.57.down_proj", + "model.layers.71.mlp.experts.57.gate_proj", + "model.layers.71.mlp.experts.57.up_proj", + "model.layers.71.mlp.experts.58.down_proj", + "model.layers.71.mlp.experts.58.gate_proj", + "model.layers.71.mlp.experts.58.up_proj", + "model.layers.71.mlp.experts.59.down_proj", + "model.layers.71.mlp.experts.59.gate_proj", + "model.layers.71.mlp.experts.59.up_proj", + "model.layers.71.mlp.experts.6.down_proj", + "model.layers.71.mlp.experts.6.gate_proj", + "model.layers.71.mlp.experts.6.up_proj", + "model.layers.71.mlp.experts.60.down_proj", + "model.layers.71.mlp.experts.60.gate_proj", + "model.layers.71.mlp.experts.60.up_proj", + "model.layers.71.mlp.experts.61.down_proj", + "model.layers.71.mlp.experts.61.gate_proj", + "model.layers.71.mlp.experts.61.up_proj", + "model.layers.71.mlp.experts.62.down_proj", + "model.layers.71.mlp.experts.62.gate_proj", + "model.layers.71.mlp.experts.62.up_proj", + "model.layers.71.mlp.experts.63.down_proj", + "model.layers.71.mlp.experts.63.gate_proj", + "model.layers.71.mlp.experts.63.up_proj", + "model.layers.71.mlp.experts.64.down_proj", + "model.layers.71.mlp.experts.64.gate_proj", + "model.layers.71.mlp.experts.64.up_proj", + "model.layers.71.mlp.experts.65.down_proj", + "model.layers.71.mlp.experts.65.gate_proj", + "model.layers.71.mlp.experts.65.up_proj", + "model.layers.71.mlp.experts.66.down_proj", + "model.layers.71.mlp.experts.66.gate_proj", + "model.layers.71.mlp.experts.66.up_proj", + "model.layers.71.mlp.experts.67.down_proj", + "model.layers.71.mlp.experts.67.gate_proj", + "model.layers.71.mlp.experts.67.up_proj", + "model.layers.71.mlp.experts.68.down_proj", + "model.layers.71.mlp.experts.68.gate_proj", + "model.layers.71.mlp.experts.68.up_proj", + "model.layers.71.mlp.experts.69.down_proj", + "model.layers.71.mlp.experts.69.gate_proj", + "model.layers.71.mlp.experts.69.up_proj", + "model.layers.71.mlp.experts.7.down_proj", + "model.layers.71.mlp.experts.7.gate_proj", + "model.layers.71.mlp.experts.7.up_proj", + "model.layers.71.mlp.experts.70.down_proj", + "model.layers.71.mlp.experts.70.gate_proj", + "model.layers.71.mlp.experts.70.up_proj", + "model.layers.71.mlp.experts.71.down_proj", + "model.layers.71.mlp.experts.71.gate_proj", + "model.layers.71.mlp.experts.71.up_proj", + "model.layers.71.mlp.experts.72.down_proj", + "model.layers.71.mlp.experts.72.gate_proj", + "model.layers.71.mlp.experts.72.up_proj", + "model.layers.71.mlp.experts.73.down_proj", + "model.layers.71.mlp.experts.73.gate_proj", + "model.layers.71.mlp.experts.73.up_proj", + "model.layers.71.mlp.experts.74.down_proj", + "model.layers.71.mlp.experts.74.gate_proj", + "model.layers.71.mlp.experts.74.up_proj", + "model.layers.71.mlp.experts.75.down_proj", + "model.layers.71.mlp.experts.75.gate_proj", + "model.layers.71.mlp.experts.75.up_proj", + "model.layers.71.mlp.experts.76.down_proj", + "model.layers.71.mlp.experts.76.gate_proj", + "model.layers.71.mlp.experts.76.up_proj", + "model.layers.71.mlp.experts.77.down_proj", + "model.layers.71.mlp.experts.77.gate_proj", + "model.layers.71.mlp.experts.77.up_proj", + "model.layers.71.mlp.experts.78.down_proj", + "model.layers.71.mlp.experts.78.gate_proj", + "model.layers.71.mlp.experts.78.up_proj", + "model.layers.71.mlp.experts.79.down_proj", + "model.layers.71.mlp.experts.79.gate_proj", + "model.layers.71.mlp.experts.79.up_proj", + "model.layers.71.mlp.experts.8.down_proj", + "model.layers.71.mlp.experts.8.gate_proj", + "model.layers.71.mlp.experts.8.up_proj", + "model.layers.71.mlp.experts.80.down_proj", + "model.layers.71.mlp.experts.80.gate_proj", + "model.layers.71.mlp.experts.80.up_proj", + "model.layers.71.mlp.experts.81.down_proj", + "model.layers.71.mlp.experts.81.gate_proj", + "model.layers.71.mlp.experts.81.up_proj", + "model.layers.71.mlp.experts.82.down_proj", + "model.layers.71.mlp.experts.82.gate_proj", + "model.layers.71.mlp.experts.82.up_proj", + "model.layers.71.mlp.experts.83.down_proj", + "model.layers.71.mlp.experts.83.gate_proj", + "model.layers.71.mlp.experts.83.up_proj", + "model.layers.71.mlp.experts.84.down_proj", + "model.layers.71.mlp.experts.84.gate_proj", + "model.layers.71.mlp.experts.84.up_proj", + "model.layers.71.mlp.experts.85.down_proj", + "model.layers.71.mlp.experts.85.gate_proj", + "model.layers.71.mlp.experts.85.up_proj", + "model.layers.71.mlp.experts.86.down_proj", + "model.layers.71.mlp.experts.86.gate_proj", + "model.layers.71.mlp.experts.86.up_proj", + "model.layers.71.mlp.experts.87.down_proj", + "model.layers.71.mlp.experts.87.gate_proj", + "model.layers.71.mlp.experts.87.up_proj", + "model.layers.71.mlp.experts.88.down_proj", + "model.layers.71.mlp.experts.88.gate_proj", + "model.layers.71.mlp.experts.88.up_proj", + "model.layers.71.mlp.experts.89.down_proj", + "model.layers.71.mlp.experts.89.gate_proj", + "model.layers.71.mlp.experts.89.up_proj", + "model.layers.71.mlp.experts.9.down_proj", + "model.layers.71.mlp.experts.9.gate_proj", + "model.layers.71.mlp.experts.9.up_proj", + "model.layers.71.mlp.experts.90.down_proj", + "model.layers.71.mlp.experts.90.gate_proj", + "model.layers.71.mlp.experts.90.up_proj", + "model.layers.71.mlp.experts.91.down_proj", + "model.layers.71.mlp.experts.91.gate_proj", + "model.layers.71.mlp.experts.91.up_proj", + "model.layers.71.mlp.experts.92.down_proj", + "model.layers.71.mlp.experts.92.gate_proj", + "model.layers.71.mlp.experts.92.up_proj", + "model.layers.71.mlp.experts.93.down_proj", + "model.layers.71.mlp.experts.93.gate_proj", + "model.layers.71.mlp.experts.93.up_proj", + "model.layers.71.mlp.experts.94.down_proj", + "model.layers.71.mlp.experts.94.gate_proj", + "model.layers.71.mlp.experts.94.up_proj", + "model.layers.71.mlp.experts.95.down_proj", + "model.layers.71.mlp.experts.95.gate_proj", + "model.layers.71.mlp.experts.95.up_proj", + "model.layers.71.mlp.experts.96.down_proj", + "model.layers.71.mlp.experts.96.gate_proj", + "model.layers.71.mlp.experts.96.up_proj", + "model.layers.71.mlp.experts.97.down_proj", + "model.layers.71.mlp.experts.97.gate_proj", + "model.layers.71.mlp.experts.97.up_proj", + "model.layers.71.mlp.experts.98.down_proj", + "model.layers.71.mlp.experts.98.gate_proj", + "model.layers.71.mlp.experts.98.up_proj", + "model.layers.71.mlp.experts.99.down_proj", + "model.layers.71.mlp.experts.99.gate_proj", + "model.layers.71.mlp.experts.99.up_proj", + "model.layers.72.mlp.experts.0.down_proj", + "model.layers.72.mlp.experts.0.gate_proj", + "model.layers.72.mlp.experts.0.up_proj", + "model.layers.72.mlp.experts.1.down_proj", + "model.layers.72.mlp.experts.1.gate_proj", + "model.layers.72.mlp.experts.1.up_proj", + "model.layers.72.mlp.experts.10.down_proj", + "model.layers.72.mlp.experts.10.gate_proj", + "model.layers.72.mlp.experts.10.up_proj", + "model.layers.72.mlp.experts.100.down_proj", + "model.layers.72.mlp.experts.100.gate_proj", + "model.layers.72.mlp.experts.100.up_proj", + "model.layers.72.mlp.experts.101.down_proj", + "model.layers.72.mlp.experts.101.gate_proj", + "model.layers.72.mlp.experts.101.up_proj", + "model.layers.72.mlp.experts.102.down_proj", + "model.layers.72.mlp.experts.102.gate_proj", + "model.layers.72.mlp.experts.102.up_proj", + "model.layers.72.mlp.experts.103.down_proj", + "model.layers.72.mlp.experts.103.gate_proj", + "model.layers.72.mlp.experts.103.up_proj", + "model.layers.72.mlp.experts.104.down_proj", + "model.layers.72.mlp.experts.104.gate_proj", + "model.layers.72.mlp.experts.104.up_proj", + "model.layers.72.mlp.experts.105.down_proj", + "model.layers.72.mlp.experts.105.gate_proj", + "model.layers.72.mlp.experts.105.up_proj", + "model.layers.72.mlp.experts.106.down_proj", + "model.layers.72.mlp.experts.106.gate_proj", + "model.layers.72.mlp.experts.106.up_proj", + "model.layers.72.mlp.experts.107.down_proj", + "model.layers.72.mlp.experts.107.gate_proj", + "model.layers.72.mlp.experts.107.up_proj", + "model.layers.72.mlp.experts.108.down_proj", + "model.layers.72.mlp.experts.108.gate_proj", + "model.layers.72.mlp.experts.108.up_proj", + "model.layers.72.mlp.experts.109.down_proj", + "model.layers.72.mlp.experts.109.gate_proj", + "model.layers.72.mlp.experts.109.up_proj", + "model.layers.72.mlp.experts.11.down_proj", + "model.layers.72.mlp.experts.11.gate_proj", + "model.layers.72.mlp.experts.11.up_proj", + "model.layers.72.mlp.experts.110.down_proj", + "model.layers.72.mlp.experts.110.gate_proj", + "model.layers.72.mlp.experts.110.up_proj", + "model.layers.72.mlp.experts.111.down_proj", + "model.layers.72.mlp.experts.111.gate_proj", + "model.layers.72.mlp.experts.111.up_proj", + "model.layers.72.mlp.experts.112.down_proj", + "model.layers.72.mlp.experts.112.gate_proj", + "model.layers.72.mlp.experts.112.up_proj", + "model.layers.72.mlp.experts.113.down_proj", + "model.layers.72.mlp.experts.113.gate_proj", + "model.layers.72.mlp.experts.113.up_proj", + "model.layers.72.mlp.experts.114.down_proj", + "model.layers.72.mlp.experts.114.gate_proj", + "model.layers.72.mlp.experts.114.up_proj", + "model.layers.72.mlp.experts.115.down_proj", + "model.layers.72.mlp.experts.115.gate_proj", + "model.layers.72.mlp.experts.115.up_proj", + "model.layers.72.mlp.experts.116.down_proj", + "model.layers.72.mlp.experts.116.gate_proj", + "model.layers.72.mlp.experts.116.up_proj", + "model.layers.72.mlp.experts.117.down_proj", + "model.layers.72.mlp.experts.117.gate_proj", + "model.layers.72.mlp.experts.117.up_proj", + "model.layers.72.mlp.experts.118.down_proj", + "model.layers.72.mlp.experts.118.gate_proj", + "model.layers.72.mlp.experts.118.up_proj", + "model.layers.72.mlp.experts.119.down_proj", + "model.layers.72.mlp.experts.119.gate_proj", + "model.layers.72.mlp.experts.119.up_proj", + "model.layers.72.mlp.experts.12.down_proj", + "model.layers.72.mlp.experts.12.gate_proj", + "model.layers.72.mlp.experts.12.up_proj", + "model.layers.72.mlp.experts.120.down_proj", + "model.layers.72.mlp.experts.120.gate_proj", + "model.layers.72.mlp.experts.120.up_proj", + "model.layers.72.mlp.experts.121.down_proj", + "model.layers.72.mlp.experts.121.gate_proj", + "model.layers.72.mlp.experts.121.up_proj", + "model.layers.72.mlp.experts.122.down_proj", + "model.layers.72.mlp.experts.122.gate_proj", + "model.layers.72.mlp.experts.122.up_proj", + "model.layers.72.mlp.experts.123.down_proj", + "model.layers.72.mlp.experts.123.gate_proj", + "model.layers.72.mlp.experts.123.up_proj", + "model.layers.72.mlp.experts.124.down_proj", + "model.layers.72.mlp.experts.124.gate_proj", + "model.layers.72.mlp.experts.124.up_proj", + "model.layers.72.mlp.experts.125.down_proj", + "model.layers.72.mlp.experts.125.gate_proj", + "model.layers.72.mlp.experts.125.up_proj", + "model.layers.72.mlp.experts.126.down_proj", + "model.layers.72.mlp.experts.126.gate_proj", + "model.layers.72.mlp.experts.126.up_proj", + "model.layers.72.mlp.experts.127.down_proj", + "model.layers.72.mlp.experts.127.gate_proj", + "model.layers.72.mlp.experts.127.up_proj", + "model.layers.72.mlp.experts.128.down_proj", + "model.layers.72.mlp.experts.128.gate_proj", + "model.layers.72.mlp.experts.128.up_proj", + "model.layers.72.mlp.experts.129.down_proj", + "model.layers.72.mlp.experts.129.gate_proj", + "model.layers.72.mlp.experts.129.up_proj", + "model.layers.72.mlp.experts.13.down_proj", + "model.layers.72.mlp.experts.13.gate_proj", + "model.layers.72.mlp.experts.13.up_proj", + "model.layers.72.mlp.experts.130.down_proj", + "model.layers.72.mlp.experts.130.gate_proj", + "model.layers.72.mlp.experts.130.up_proj", + "model.layers.72.mlp.experts.131.down_proj", + "model.layers.72.mlp.experts.131.gate_proj", + "model.layers.72.mlp.experts.131.up_proj", + "model.layers.72.mlp.experts.132.down_proj", + "model.layers.72.mlp.experts.132.gate_proj", + "model.layers.72.mlp.experts.132.up_proj", + "model.layers.72.mlp.experts.133.down_proj", + "model.layers.72.mlp.experts.133.gate_proj", + "model.layers.72.mlp.experts.133.up_proj", + "model.layers.72.mlp.experts.134.down_proj", + "model.layers.72.mlp.experts.134.gate_proj", + "model.layers.72.mlp.experts.134.up_proj", + "model.layers.72.mlp.experts.135.down_proj", + "model.layers.72.mlp.experts.135.gate_proj", + "model.layers.72.mlp.experts.135.up_proj", + "model.layers.72.mlp.experts.136.down_proj", + "model.layers.72.mlp.experts.136.gate_proj", + "model.layers.72.mlp.experts.136.up_proj", + "model.layers.72.mlp.experts.137.down_proj", + "model.layers.72.mlp.experts.137.gate_proj", + "model.layers.72.mlp.experts.137.up_proj", + "model.layers.72.mlp.experts.138.down_proj", + "model.layers.72.mlp.experts.138.gate_proj", + "model.layers.72.mlp.experts.138.up_proj", + "model.layers.72.mlp.experts.139.down_proj", + "model.layers.72.mlp.experts.139.gate_proj", + "model.layers.72.mlp.experts.139.up_proj", + "model.layers.72.mlp.experts.14.down_proj", + "model.layers.72.mlp.experts.14.gate_proj", + "model.layers.72.mlp.experts.14.up_proj", + "model.layers.72.mlp.experts.140.down_proj", + "model.layers.72.mlp.experts.140.gate_proj", + "model.layers.72.mlp.experts.140.up_proj", + "model.layers.72.mlp.experts.141.down_proj", + "model.layers.72.mlp.experts.141.gate_proj", + "model.layers.72.mlp.experts.141.up_proj", + "model.layers.72.mlp.experts.142.down_proj", + "model.layers.72.mlp.experts.142.gate_proj", + "model.layers.72.mlp.experts.142.up_proj", + "model.layers.72.mlp.experts.143.down_proj", + "model.layers.72.mlp.experts.143.gate_proj", + "model.layers.72.mlp.experts.143.up_proj", + "model.layers.72.mlp.experts.144.down_proj", + "model.layers.72.mlp.experts.144.gate_proj", + "model.layers.72.mlp.experts.144.up_proj", + "model.layers.72.mlp.experts.145.down_proj", + "model.layers.72.mlp.experts.145.gate_proj", + "model.layers.72.mlp.experts.145.up_proj", + "model.layers.72.mlp.experts.146.down_proj", + "model.layers.72.mlp.experts.146.gate_proj", + "model.layers.72.mlp.experts.146.up_proj", + "model.layers.72.mlp.experts.147.down_proj", + "model.layers.72.mlp.experts.147.gate_proj", + "model.layers.72.mlp.experts.147.up_proj", + "model.layers.72.mlp.experts.148.down_proj", + "model.layers.72.mlp.experts.148.gate_proj", + "model.layers.72.mlp.experts.148.up_proj", + "model.layers.72.mlp.experts.149.down_proj", + "model.layers.72.mlp.experts.149.gate_proj", + "model.layers.72.mlp.experts.149.up_proj", + "model.layers.72.mlp.experts.15.down_proj", + "model.layers.72.mlp.experts.15.gate_proj", + "model.layers.72.mlp.experts.15.up_proj", + "model.layers.72.mlp.experts.150.down_proj", + "model.layers.72.mlp.experts.150.gate_proj", + "model.layers.72.mlp.experts.150.up_proj", + "model.layers.72.mlp.experts.151.down_proj", + "model.layers.72.mlp.experts.151.gate_proj", + "model.layers.72.mlp.experts.151.up_proj", + "model.layers.72.mlp.experts.152.down_proj", + "model.layers.72.mlp.experts.152.gate_proj", + "model.layers.72.mlp.experts.152.up_proj", + "model.layers.72.mlp.experts.153.down_proj", + "model.layers.72.mlp.experts.153.gate_proj", + "model.layers.72.mlp.experts.153.up_proj", + "model.layers.72.mlp.experts.154.down_proj", + "model.layers.72.mlp.experts.154.gate_proj", + "model.layers.72.mlp.experts.154.up_proj", + "model.layers.72.mlp.experts.155.down_proj", + "model.layers.72.mlp.experts.155.gate_proj", + "model.layers.72.mlp.experts.155.up_proj", + "model.layers.72.mlp.experts.156.down_proj", + "model.layers.72.mlp.experts.156.gate_proj", + "model.layers.72.mlp.experts.156.up_proj", + "model.layers.72.mlp.experts.157.down_proj", + "model.layers.72.mlp.experts.157.gate_proj", + "model.layers.72.mlp.experts.157.up_proj", + "model.layers.72.mlp.experts.158.down_proj", + "model.layers.72.mlp.experts.158.gate_proj", + "model.layers.72.mlp.experts.158.up_proj", + "model.layers.72.mlp.experts.159.down_proj", + "model.layers.72.mlp.experts.159.gate_proj", + "model.layers.72.mlp.experts.159.up_proj", + "model.layers.72.mlp.experts.16.down_proj", + "model.layers.72.mlp.experts.16.gate_proj", + "model.layers.72.mlp.experts.16.up_proj", + "model.layers.72.mlp.experts.160.down_proj", + "model.layers.72.mlp.experts.160.gate_proj", + "model.layers.72.mlp.experts.160.up_proj", + "model.layers.72.mlp.experts.161.down_proj", + "model.layers.72.mlp.experts.161.gate_proj", + "model.layers.72.mlp.experts.161.up_proj", + "model.layers.72.mlp.experts.162.down_proj", + "model.layers.72.mlp.experts.162.gate_proj", + "model.layers.72.mlp.experts.162.up_proj", + "model.layers.72.mlp.experts.163.down_proj", + "model.layers.72.mlp.experts.163.gate_proj", + "model.layers.72.mlp.experts.163.up_proj", + "model.layers.72.mlp.experts.164.down_proj", + "model.layers.72.mlp.experts.164.gate_proj", + "model.layers.72.mlp.experts.164.up_proj", + "model.layers.72.mlp.experts.165.down_proj", + "model.layers.72.mlp.experts.165.gate_proj", + "model.layers.72.mlp.experts.165.up_proj", + "model.layers.72.mlp.experts.166.down_proj", + "model.layers.72.mlp.experts.166.gate_proj", + "model.layers.72.mlp.experts.166.up_proj", + "model.layers.72.mlp.experts.167.down_proj", + "model.layers.72.mlp.experts.167.gate_proj", + "model.layers.72.mlp.experts.167.up_proj", + "model.layers.72.mlp.experts.168.down_proj", + "model.layers.72.mlp.experts.168.gate_proj", + "model.layers.72.mlp.experts.168.up_proj", + "model.layers.72.mlp.experts.169.down_proj", + "model.layers.72.mlp.experts.169.gate_proj", + "model.layers.72.mlp.experts.169.up_proj", + "model.layers.72.mlp.experts.17.down_proj", + "model.layers.72.mlp.experts.17.gate_proj", + "model.layers.72.mlp.experts.17.up_proj", + "model.layers.72.mlp.experts.170.down_proj", + "model.layers.72.mlp.experts.170.gate_proj", + "model.layers.72.mlp.experts.170.up_proj", + "model.layers.72.mlp.experts.171.down_proj", + "model.layers.72.mlp.experts.171.gate_proj", + "model.layers.72.mlp.experts.171.up_proj", + "model.layers.72.mlp.experts.172.down_proj", + "model.layers.72.mlp.experts.172.gate_proj", + "model.layers.72.mlp.experts.172.up_proj", + "model.layers.72.mlp.experts.173.down_proj", + "model.layers.72.mlp.experts.173.gate_proj", + "model.layers.72.mlp.experts.173.up_proj", + "model.layers.72.mlp.experts.174.down_proj", + "model.layers.72.mlp.experts.174.gate_proj", + "model.layers.72.mlp.experts.174.up_proj", + "model.layers.72.mlp.experts.175.down_proj", + "model.layers.72.mlp.experts.175.gate_proj", + "model.layers.72.mlp.experts.175.up_proj", + "model.layers.72.mlp.experts.176.down_proj", + "model.layers.72.mlp.experts.176.gate_proj", + "model.layers.72.mlp.experts.176.up_proj", + "model.layers.72.mlp.experts.177.down_proj", + "model.layers.72.mlp.experts.177.gate_proj", + "model.layers.72.mlp.experts.177.up_proj", + "model.layers.72.mlp.experts.178.down_proj", + "model.layers.72.mlp.experts.178.gate_proj", + "model.layers.72.mlp.experts.178.up_proj", + "model.layers.72.mlp.experts.179.down_proj", + "model.layers.72.mlp.experts.179.gate_proj", + "model.layers.72.mlp.experts.179.up_proj", + "model.layers.72.mlp.experts.18.down_proj", + "model.layers.72.mlp.experts.18.gate_proj", + "model.layers.72.mlp.experts.18.up_proj", + "model.layers.72.mlp.experts.180.down_proj", + "model.layers.72.mlp.experts.180.gate_proj", + "model.layers.72.mlp.experts.180.up_proj", + "model.layers.72.mlp.experts.181.down_proj", + "model.layers.72.mlp.experts.181.gate_proj", + "model.layers.72.mlp.experts.181.up_proj", + "model.layers.72.mlp.experts.182.down_proj", + "model.layers.72.mlp.experts.182.gate_proj", + "model.layers.72.mlp.experts.182.up_proj", + "model.layers.72.mlp.experts.183.down_proj", + "model.layers.72.mlp.experts.183.gate_proj", + "model.layers.72.mlp.experts.183.up_proj", + "model.layers.72.mlp.experts.184.down_proj", + "model.layers.72.mlp.experts.184.gate_proj", + "model.layers.72.mlp.experts.184.up_proj", + "model.layers.72.mlp.experts.185.down_proj", + "model.layers.72.mlp.experts.185.gate_proj", + "model.layers.72.mlp.experts.185.up_proj", + "model.layers.72.mlp.experts.186.down_proj", + "model.layers.72.mlp.experts.186.gate_proj", + "model.layers.72.mlp.experts.186.up_proj", + "model.layers.72.mlp.experts.187.down_proj", + "model.layers.72.mlp.experts.187.gate_proj", + "model.layers.72.mlp.experts.187.up_proj", + "model.layers.72.mlp.experts.188.down_proj", + "model.layers.72.mlp.experts.188.gate_proj", + "model.layers.72.mlp.experts.188.up_proj", + "model.layers.72.mlp.experts.189.down_proj", + "model.layers.72.mlp.experts.189.gate_proj", + "model.layers.72.mlp.experts.189.up_proj", + "model.layers.72.mlp.experts.19.down_proj", + "model.layers.72.mlp.experts.19.gate_proj", + "model.layers.72.mlp.experts.19.up_proj", + "model.layers.72.mlp.experts.190.down_proj", + "model.layers.72.mlp.experts.190.gate_proj", + "model.layers.72.mlp.experts.190.up_proj", + "model.layers.72.mlp.experts.191.down_proj", + "model.layers.72.mlp.experts.191.gate_proj", + "model.layers.72.mlp.experts.191.up_proj", + "model.layers.72.mlp.experts.192.down_proj", + "model.layers.72.mlp.experts.192.gate_proj", + "model.layers.72.mlp.experts.192.up_proj", + "model.layers.72.mlp.experts.193.down_proj", + "model.layers.72.mlp.experts.193.gate_proj", + "model.layers.72.mlp.experts.193.up_proj", + "model.layers.72.mlp.experts.194.down_proj", + "model.layers.72.mlp.experts.194.gate_proj", + "model.layers.72.mlp.experts.194.up_proj", + "model.layers.72.mlp.experts.195.down_proj", + "model.layers.72.mlp.experts.195.gate_proj", + "model.layers.72.mlp.experts.195.up_proj", + "model.layers.72.mlp.experts.196.down_proj", + "model.layers.72.mlp.experts.196.gate_proj", + "model.layers.72.mlp.experts.196.up_proj", + "model.layers.72.mlp.experts.197.down_proj", + "model.layers.72.mlp.experts.197.gate_proj", + "model.layers.72.mlp.experts.197.up_proj", + "model.layers.72.mlp.experts.198.down_proj", + "model.layers.72.mlp.experts.198.gate_proj", + "model.layers.72.mlp.experts.198.up_proj", + "model.layers.72.mlp.experts.199.down_proj", + "model.layers.72.mlp.experts.199.gate_proj", + "model.layers.72.mlp.experts.199.up_proj", + "model.layers.72.mlp.experts.2.down_proj", + "model.layers.72.mlp.experts.2.gate_proj", + "model.layers.72.mlp.experts.2.up_proj", + "model.layers.72.mlp.experts.20.down_proj", + "model.layers.72.mlp.experts.20.gate_proj", + "model.layers.72.mlp.experts.20.up_proj", + "model.layers.72.mlp.experts.200.down_proj", + "model.layers.72.mlp.experts.200.gate_proj", + "model.layers.72.mlp.experts.200.up_proj", + "model.layers.72.mlp.experts.201.down_proj", + "model.layers.72.mlp.experts.201.gate_proj", + "model.layers.72.mlp.experts.201.up_proj", + "model.layers.72.mlp.experts.202.down_proj", + "model.layers.72.mlp.experts.202.gate_proj", + "model.layers.72.mlp.experts.202.up_proj", + "model.layers.72.mlp.experts.203.down_proj", + "model.layers.72.mlp.experts.203.gate_proj", + "model.layers.72.mlp.experts.203.up_proj", + "model.layers.72.mlp.experts.204.down_proj", + "model.layers.72.mlp.experts.204.gate_proj", + "model.layers.72.mlp.experts.204.up_proj", + "model.layers.72.mlp.experts.205.down_proj", + "model.layers.72.mlp.experts.205.gate_proj", + "model.layers.72.mlp.experts.205.up_proj", + "model.layers.72.mlp.experts.206.down_proj", + "model.layers.72.mlp.experts.206.gate_proj", + "model.layers.72.mlp.experts.206.up_proj", + "model.layers.72.mlp.experts.207.down_proj", + "model.layers.72.mlp.experts.207.gate_proj", + "model.layers.72.mlp.experts.207.up_proj", + "model.layers.72.mlp.experts.208.down_proj", + "model.layers.72.mlp.experts.208.gate_proj", + "model.layers.72.mlp.experts.208.up_proj", + "model.layers.72.mlp.experts.209.down_proj", + "model.layers.72.mlp.experts.209.gate_proj", + "model.layers.72.mlp.experts.209.up_proj", + "model.layers.72.mlp.experts.21.down_proj", + "model.layers.72.mlp.experts.21.gate_proj", + "model.layers.72.mlp.experts.21.up_proj", + "model.layers.72.mlp.experts.210.down_proj", + "model.layers.72.mlp.experts.210.gate_proj", + "model.layers.72.mlp.experts.210.up_proj", + "model.layers.72.mlp.experts.211.down_proj", + "model.layers.72.mlp.experts.211.gate_proj", + "model.layers.72.mlp.experts.211.up_proj", + "model.layers.72.mlp.experts.212.down_proj", + "model.layers.72.mlp.experts.212.gate_proj", + "model.layers.72.mlp.experts.212.up_proj", + "model.layers.72.mlp.experts.213.down_proj", + "model.layers.72.mlp.experts.213.gate_proj", + "model.layers.72.mlp.experts.213.up_proj", + "model.layers.72.mlp.experts.214.down_proj", + "model.layers.72.mlp.experts.214.gate_proj", + "model.layers.72.mlp.experts.214.up_proj", + "model.layers.72.mlp.experts.215.down_proj", + "model.layers.72.mlp.experts.215.gate_proj", + "model.layers.72.mlp.experts.215.up_proj", + "model.layers.72.mlp.experts.216.down_proj", + "model.layers.72.mlp.experts.216.gate_proj", + "model.layers.72.mlp.experts.216.up_proj", + "model.layers.72.mlp.experts.217.down_proj", + "model.layers.72.mlp.experts.217.gate_proj", + "model.layers.72.mlp.experts.217.up_proj", + "model.layers.72.mlp.experts.218.down_proj", + "model.layers.72.mlp.experts.218.gate_proj", + "model.layers.72.mlp.experts.218.up_proj", + "model.layers.72.mlp.experts.219.down_proj", + "model.layers.72.mlp.experts.219.gate_proj", + "model.layers.72.mlp.experts.219.up_proj", + "model.layers.72.mlp.experts.22.down_proj", + "model.layers.72.mlp.experts.22.gate_proj", + "model.layers.72.mlp.experts.22.up_proj", + "model.layers.72.mlp.experts.220.down_proj", + "model.layers.72.mlp.experts.220.gate_proj", + "model.layers.72.mlp.experts.220.up_proj", + "model.layers.72.mlp.experts.221.down_proj", + "model.layers.72.mlp.experts.221.gate_proj", + "model.layers.72.mlp.experts.221.up_proj", + "model.layers.72.mlp.experts.222.down_proj", + "model.layers.72.mlp.experts.222.gate_proj", + "model.layers.72.mlp.experts.222.up_proj", + "model.layers.72.mlp.experts.223.down_proj", + "model.layers.72.mlp.experts.223.gate_proj", + "model.layers.72.mlp.experts.223.up_proj", + "model.layers.72.mlp.experts.224.down_proj", + "model.layers.72.mlp.experts.224.gate_proj", + "model.layers.72.mlp.experts.224.up_proj", + "model.layers.72.mlp.experts.225.down_proj", + "model.layers.72.mlp.experts.225.gate_proj", + "model.layers.72.mlp.experts.225.up_proj", + "model.layers.72.mlp.experts.226.down_proj", + "model.layers.72.mlp.experts.226.gate_proj", + "model.layers.72.mlp.experts.226.up_proj", + "model.layers.72.mlp.experts.227.down_proj", + "model.layers.72.mlp.experts.227.gate_proj", + "model.layers.72.mlp.experts.227.up_proj", + "model.layers.72.mlp.experts.228.down_proj", + "model.layers.72.mlp.experts.228.gate_proj", + "model.layers.72.mlp.experts.228.up_proj", + "model.layers.72.mlp.experts.229.down_proj", + "model.layers.72.mlp.experts.229.gate_proj", + "model.layers.72.mlp.experts.229.up_proj", + "model.layers.72.mlp.experts.23.down_proj", + "model.layers.72.mlp.experts.23.gate_proj", + "model.layers.72.mlp.experts.23.up_proj", + "model.layers.72.mlp.experts.230.down_proj", + "model.layers.72.mlp.experts.230.gate_proj", + "model.layers.72.mlp.experts.230.up_proj", + "model.layers.72.mlp.experts.231.down_proj", + "model.layers.72.mlp.experts.231.gate_proj", + "model.layers.72.mlp.experts.231.up_proj", + "model.layers.72.mlp.experts.232.down_proj", + "model.layers.72.mlp.experts.232.gate_proj", + "model.layers.72.mlp.experts.232.up_proj", + "model.layers.72.mlp.experts.233.down_proj", + "model.layers.72.mlp.experts.233.gate_proj", + "model.layers.72.mlp.experts.233.up_proj", + "model.layers.72.mlp.experts.234.down_proj", + "model.layers.72.mlp.experts.234.gate_proj", + "model.layers.72.mlp.experts.234.up_proj", + "model.layers.72.mlp.experts.235.down_proj", + "model.layers.72.mlp.experts.235.gate_proj", + "model.layers.72.mlp.experts.235.up_proj", + "model.layers.72.mlp.experts.236.down_proj", + "model.layers.72.mlp.experts.236.gate_proj", + "model.layers.72.mlp.experts.236.up_proj", + "model.layers.72.mlp.experts.237.down_proj", + "model.layers.72.mlp.experts.237.gate_proj", + "model.layers.72.mlp.experts.237.up_proj", + "model.layers.72.mlp.experts.238.down_proj", + "model.layers.72.mlp.experts.238.gate_proj", + "model.layers.72.mlp.experts.238.up_proj", + "model.layers.72.mlp.experts.239.down_proj", + "model.layers.72.mlp.experts.239.gate_proj", + "model.layers.72.mlp.experts.239.up_proj", + "model.layers.72.mlp.experts.24.down_proj", + "model.layers.72.mlp.experts.24.gate_proj", + "model.layers.72.mlp.experts.24.up_proj", + "model.layers.72.mlp.experts.240.down_proj", + "model.layers.72.mlp.experts.240.gate_proj", + "model.layers.72.mlp.experts.240.up_proj", + "model.layers.72.mlp.experts.241.down_proj", + "model.layers.72.mlp.experts.241.gate_proj", + "model.layers.72.mlp.experts.241.up_proj", + "model.layers.72.mlp.experts.242.down_proj", + "model.layers.72.mlp.experts.242.gate_proj", + "model.layers.72.mlp.experts.242.up_proj", + "model.layers.72.mlp.experts.243.down_proj", + "model.layers.72.mlp.experts.243.gate_proj", + "model.layers.72.mlp.experts.243.up_proj", + "model.layers.72.mlp.experts.244.down_proj", + "model.layers.72.mlp.experts.244.gate_proj", + "model.layers.72.mlp.experts.244.up_proj", + "model.layers.72.mlp.experts.245.down_proj", + "model.layers.72.mlp.experts.245.gate_proj", + "model.layers.72.mlp.experts.245.up_proj", + "model.layers.72.mlp.experts.246.down_proj", + "model.layers.72.mlp.experts.246.gate_proj", + "model.layers.72.mlp.experts.246.up_proj", + "model.layers.72.mlp.experts.247.down_proj", + "model.layers.72.mlp.experts.247.gate_proj", + "model.layers.72.mlp.experts.247.up_proj", + "model.layers.72.mlp.experts.248.down_proj", + "model.layers.72.mlp.experts.248.gate_proj", + "model.layers.72.mlp.experts.248.up_proj", + "model.layers.72.mlp.experts.249.down_proj", + "model.layers.72.mlp.experts.249.gate_proj", + "model.layers.72.mlp.experts.249.up_proj", + "model.layers.72.mlp.experts.25.down_proj", + "model.layers.72.mlp.experts.25.gate_proj", + "model.layers.72.mlp.experts.25.up_proj", + "model.layers.72.mlp.experts.250.down_proj", + "model.layers.72.mlp.experts.250.gate_proj", + "model.layers.72.mlp.experts.250.up_proj", + "model.layers.72.mlp.experts.251.down_proj", + "model.layers.72.mlp.experts.251.gate_proj", + "model.layers.72.mlp.experts.251.up_proj", + "model.layers.72.mlp.experts.252.down_proj", + "model.layers.72.mlp.experts.252.gate_proj", + "model.layers.72.mlp.experts.252.up_proj", + "model.layers.72.mlp.experts.253.down_proj", + "model.layers.72.mlp.experts.253.gate_proj", + "model.layers.72.mlp.experts.253.up_proj", + "model.layers.72.mlp.experts.254.down_proj", + "model.layers.72.mlp.experts.254.gate_proj", + "model.layers.72.mlp.experts.254.up_proj", + "model.layers.72.mlp.experts.255.down_proj", + "model.layers.72.mlp.experts.255.gate_proj", + "model.layers.72.mlp.experts.255.up_proj", + "model.layers.72.mlp.experts.26.down_proj", + "model.layers.72.mlp.experts.26.gate_proj", + "model.layers.72.mlp.experts.26.up_proj", + "model.layers.72.mlp.experts.27.down_proj", + "model.layers.72.mlp.experts.27.gate_proj", + "model.layers.72.mlp.experts.27.up_proj", + "model.layers.72.mlp.experts.28.down_proj", + "model.layers.72.mlp.experts.28.gate_proj", + "model.layers.72.mlp.experts.28.up_proj", + "model.layers.72.mlp.experts.29.down_proj", + "model.layers.72.mlp.experts.29.gate_proj", + "model.layers.72.mlp.experts.29.up_proj", + "model.layers.72.mlp.experts.3.down_proj", + "model.layers.72.mlp.experts.3.gate_proj", + "model.layers.72.mlp.experts.3.up_proj", + "model.layers.72.mlp.experts.30.down_proj", + "model.layers.72.mlp.experts.30.gate_proj", + "model.layers.72.mlp.experts.30.up_proj", + "model.layers.72.mlp.experts.31.down_proj", + "model.layers.72.mlp.experts.31.gate_proj", + "model.layers.72.mlp.experts.31.up_proj", + "model.layers.72.mlp.experts.32.down_proj", + "model.layers.72.mlp.experts.32.gate_proj", + "model.layers.72.mlp.experts.32.up_proj", + "model.layers.72.mlp.experts.33.down_proj", + "model.layers.72.mlp.experts.33.gate_proj", + "model.layers.72.mlp.experts.33.up_proj", + "model.layers.72.mlp.experts.34.down_proj", + "model.layers.72.mlp.experts.34.gate_proj", + "model.layers.72.mlp.experts.34.up_proj", + "model.layers.72.mlp.experts.35.down_proj", + "model.layers.72.mlp.experts.35.gate_proj", + "model.layers.72.mlp.experts.35.up_proj", + "model.layers.72.mlp.experts.36.down_proj", + "model.layers.72.mlp.experts.36.gate_proj", + "model.layers.72.mlp.experts.36.up_proj", + "model.layers.72.mlp.experts.37.down_proj", + "model.layers.72.mlp.experts.37.gate_proj", + "model.layers.72.mlp.experts.37.up_proj", + "model.layers.72.mlp.experts.38.down_proj", + "model.layers.72.mlp.experts.38.gate_proj", + "model.layers.72.mlp.experts.38.up_proj", + "model.layers.72.mlp.experts.39.down_proj", + "model.layers.72.mlp.experts.39.gate_proj", + "model.layers.72.mlp.experts.39.up_proj", + "model.layers.72.mlp.experts.4.down_proj", + "model.layers.72.mlp.experts.4.gate_proj", + "model.layers.72.mlp.experts.4.up_proj", + "model.layers.72.mlp.experts.40.down_proj", + "model.layers.72.mlp.experts.40.gate_proj", + "model.layers.72.mlp.experts.40.up_proj", + "model.layers.72.mlp.experts.41.down_proj", + "model.layers.72.mlp.experts.41.gate_proj", + "model.layers.72.mlp.experts.41.up_proj", + "model.layers.72.mlp.experts.42.down_proj", + "model.layers.72.mlp.experts.42.gate_proj", + "model.layers.72.mlp.experts.42.up_proj", + "model.layers.72.mlp.experts.43.down_proj", + "model.layers.72.mlp.experts.43.gate_proj", + "model.layers.72.mlp.experts.43.up_proj", + "model.layers.72.mlp.experts.44.down_proj", + "model.layers.72.mlp.experts.44.gate_proj", + "model.layers.72.mlp.experts.44.up_proj", + "model.layers.72.mlp.experts.45.down_proj", + "model.layers.72.mlp.experts.45.gate_proj", + "model.layers.72.mlp.experts.45.up_proj", + "model.layers.72.mlp.experts.46.down_proj", + "model.layers.72.mlp.experts.46.gate_proj", + "model.layers.72.mlp.experts.46.up_proj", + "model.layers.72.mlp.experts.47.down_proj", + "model.layers.72.mlp.experts.47.gate_proj", + "model.layers.72.mlp.experts.47.up_proj", + "model.layers.72.mlp.experts.48.down_proj", + "model.layers.72.mlp.experts.48.gate_proj", + "model.layers.72.mlp.experts.48.up_proj", + "model.layers.72.mlp.experts.49.down_proj", + "model.layers.72.mlp.experts.49.gate_proj", + "model.layers.72.mlp.experts.49.up_proj", + "model.layers.72.mlp.experts.5.down_proj", + "model.layers.72.mlp.experts.5.gate_proj", + "model.layers.72.mlp.experts.5.up_proj", + "model.layers.72.mlp.experts.50.down_proj", + "model.layers.72.mlp.experts.50.gate_proj", + "model.layers.72.mlp.experts.50.up_proj", + "model.layers.72.mlp.experts.51.down_proj", + "model.layers.72.mlp.experts.51.gate_proj", + "model.layers.72.mlp.experts.51.up_proj", + "model.layers.72.mlp.experts.52.down_proj", + "model.layers.72.mlp.experts.52.gate_proj", + "model.layers.72.mlp.experts.52.up_proj", + "model.layers.72.mlp.experts.53.down_proj", + "model.layers.72.mlp.experts.53.gate_proj", + "model.layers.72.mlp.experts.53.up_proj", + "model.layers.72.mlp.experts.54.down_proj", + "model.layers.72.mlp.experts.54.gate_proj", + "model.layers.72.mlp.experts.54.up_proj", + "model.layers.72.mlp.experts.55.down_proj", + "model.layers.72.mlp.experts.55.gate_proj", + "model.layers.72.mlp.experts.55.up_proj", + "model.layers.72.mlp.experts.56.down_proj", + "model.layers.72.mlp.experts.56.gate_proj", + "model.layers.72.mlp.experts.56.up_proj", + "model.layers.72.mlp.experts.57.down_proj", + "model.layers.72.mlp.experts.57.gate_proj", + "model.layers.72.mlp.experts.57.up_proj", + "model.layers.72.mlp.experts.58.down_proj", + "model.layers.72.mlp.experts.58.gate_proj", + "model.layers.72.mlp.experts.58.up_proj", + "model.layers.72.mlp.experts.59.down_proj", + "model.layers.72.mlp.experts.59.gate_proj", + "model.layers.72.mlp.experts.59.up_proj", + "model.layers.72.mlp.experts.6.down_proj", + "model.layers.72.mlp.experts.6.gate_proj", + "model.layers.72.mlp.experts.6.up_proj", + "model.layers.72.mlp.experts.60.down_proj", + "model.layers.72.mlp.experts.60.gate_proj", + "model.layers.72.mlp.experts.60.up_proj", + "model.layers.72.mlp.experts.61.down_proj", + "model.layers.72.mlp.experts.61.gate_proj", + "model.layers.72.mlp.experts.61.up_proj", + "model.layers.72.mlp.experts.62.down_proj", + "model.layers.72.mlp.experts.62.gate_proj", + "model.layers.72.mlp.experts.62.up_proj", + "model.layers.72.mlp.experts.63.down_proj", + "model.layers.72.mlp.experts.63.gate_proj", + "model.layers.72.mlp.experts.63.up_proj", + "model.layers.72.mlp.experts.64.down_proj", + "model.layers.72.mlp.experts.64.gate_proj", + "model.layers.72.mlp.experts.64.up_proj", + "model.layers.72.mlp.experts.65.down_proj", + "model.layers.72.mlp.experts.65.gate_proj", + "model.layers.72.mlp.experts.65.up_proj", + "model.layers.72.mlp.experts.66.down_proj", + "model.layers.72.mlp.experts.66.gate_proj", + "model.layers.72.mlp.experts.66.up_proj", + "model.layers.72.mlp.experts.67.down_proj", + "model.layers.72.mlp.experts.67.gate_proj", + "model.layers.72.mlp.experts.67.up_proj", + "model.layers.72.mlp.experts.68.down_proj", + "model.layers.72.mlp.experts.68.gate_proj", + "model.layers.72.mlp.experts.68.up_proj", + "model.layers.72.mlp.experts.69.down_proj", + "model.layers.72.mlp.experts.69.gate_proj", + "model.layers.72.mlp.experts.69.up_proj", + "model.layers.72.mlp.experts.7.down_proj", + "model.layers.72.mlp.experts.7.gate_proj", + "model.layers.72.mlp.experts.7.up_proj", + "model.layers.72.mlp.experts.70.down_proj", + "model.layers.72.mlp.experts.70.gate_proj", + "model.layers.72.mlp.experts.70.up_proj", + "model.layers.72.mlp.experts.71.down_proj", + "model.layers.72.mlp.experts.71.gate_proj", + "model.layers.72.mlp.experts.71.up_proj", + "model.layers.72.mlp.experts.72.down_proj", + "model.layers.72.mlp.experts.72.gate_proj", + "model.layers.72.mlp.experts.72.up_proj", + "model.layers.72.mlp.experts.73.down_proj", + "model.layers.72.mlp.experts.73.gate_proj", + "model.layers.72.mlp.experts.73.up_proj", + "model.layers.72.mlp.experts.74.down_proj", + "model.layers.72.mlp.experts.74.gate_proj", + "model.layers.72.mlp.experts.74.up_proj", + "model.layers.72.mlp.experts.75.down_proj", + "model.layers.72.mlp.experts.75.gate_proj", + "model.layers.72.mlp.experts.75.up_proj", + "model.layers.72.mlp.experts.76.down_proj", + "model.layers.72.mlp.experts.76.gate_proj", + "model.layers.72.mlp.experts.76.up_proj", + "model.layers.72.mlp.experts.77.down_proj", + "model.layers.72.mlp.experts.77.gate_proj", + "model.layers.72.mlp.experts.77.up_proj", + "model.layers.72.mlp.experts.78.down_proj", + "model.layers.72.mlp.experts.78.gate_proj", + "model.layers.72.mlp.experts.78.up_proj", + "model.layers.72.mlp.experts.79.down_proj", + "model.layers.72.mlp.experts.79.gate_proj", + "model.layers.72.mlp.experts.79.up_proj", + "model.layers.72.mlp.experts.8.down_proj", + "model.layers.72.mlp.experts.8.gate_proj", + "model.layers.72.mlp.experts.8.up_proj", + "model.layers.72.mlp.experts.80.down_proj", + "model.layers.72.mlp.experts.80.gate_proj", + "model.layers.72.mlp.experts.80.up_proj", + "model.layers.72.mlp.experts.81.down_proj", + "model.layers.72.mlp.experts.81.gate_proj", + "model.layers.72.mlp.experts.81.up_proj", + "model.layers.72.mlp.experts.82.down_proj", + "model.layers.72.mlp.experts.82.gate_proj", + "model.layers.72.mlp.experts.82.up_proj", + "model.layers.72.mlp.experts.83.down_proj", + "model.layers.72.mlp.experts.83.gate_proj", + "model.layers.72.mlp.experts.83.up_proj", + "model.layers.72.mlp.experts.84.down_proj", + "model.layers.72.mlp.experts.84.gate_proj", + "model.layers.72.mlp.experts.84.up_proj", + "model.layers.72.mlp.experts.85.down_proj", + "model.layers.72.mlp.experts.85.gate_proj", + "model.layers.72.mlp.experts.85.up_proj", + "model.layers.72.mlp.experts.86.down_proj", + "model.layers.72.mlp.experts.86.gate_proj", + "model.layers.72.mlp.experts.86.up_proj", + "model.layers.72.mlp.experts.87.down_proj", + "model.layers.72.mlp.experts.87.gate_proj", + "model.layers.72.mlp.experts.87.up_proj", + "model.layers.72.mlp.experts.88.down_proj", + "model.layers.72.mlp.experts.88.gate_proj", + "model.layers.72.mlp.experts.88.up_proj", + "model.layers.72.mlp.experts.89.down_proj", + "model.layers.72.mlp.experts.89.gate_proj", + "model.layers.72.mlp.experts.89.up_proj", + "model.layers.72.mlp.experts.9.down_proj", + "model.layers.72.mlp.experts.9.gate_proj", + "model.layers.72.mlp.experts.9.up_proj", + "model.layers.72.mlp.experts.90.down_proj", + "model.layers.72.mlp.experts.90.gate_proj", + "model.layers.72.mlp.experts.90.up_proj", + "model.layers.72.mlp.experts.91.down_proj", + "model.layers.72.mlp.experts.91.gate_proj", + "model.layers.72.mlp.experts.91.up_proj", + "model.layers.72.mlp.experts.92.down_proj", + "model.layers.72.mlp.experts.92.gate_proj", + "model.layers.72.mlp.experts.92.up_proj", + "model.layers.72.mlp.experts.93.down_proj", + "model.layers.72.mlp.experts.93.gate_proj", + "model.layers.72.mlp.experts.93.up_proj", + "model.layers.72.mlp.experts.94.down_proj", + "model.layers.72.mlp.experts.94.gate_proj", + "model.layers.72.mlp.experts.94.up_proj", + "model.layers.72.mlp.experts.95.down_proj", + "model.layers.72.mlp.experts.95.gate_proj", + "model.layers.72.mlp.experts.95.up_proj", + "model.layers.72.mlp.experts.96.down_proj", + "model.layers.72.mlp.experts.96.gate_proj", + "model.layers.72.mlp.experts.96.up_proj", + "model.layers.72.mlp.experts.97.down_proj", + "model.layers.72.mlp.experts.97.gate_proj", + "model.layers.72.mlp.experts.97.up_proj", + "model.layers.72.mlp.experts.98.down_proj", + "model.layers.72.mlp.experts.98.gate_proj", + "model.layers.72.mlp.experts.98.up_proj", + "model.layers.72.mlp.experts.99.down_proj", + "model.layers.72.mlp.experts.99.gate_proj", + "model.layers.72.mlp.experts.99.up_proj", + "model.layers.73.mlp.experts.0.down_proj", + "model.layers.73.mlp.experts.0.gate_proj", + "model.layers.73.mlp.experts.0.up_proj", + "model.layers.73.mlp.experts.1.down_proj", + "model.layers.73.mlp.experts.1.gate_proj", + "model.layers.73.mlp.experts.1.up_proj", + "model.layers.73.mlp.experts.10.down_proj", + "model.layers.73.mlp.experts.10.gate_proj", + "model.layers.73.mlp.experts.10.up_proj", + "model.layers.73.mlp.experts.100.down_proj", + "model.layers.73.mlp.experts.100.gate_proj", + "model.layers.73.mlp.experts.100.up_proj", + "model.layers.73.mlp.experts.101.down_proj", + "model.layers.73.mlp.experts.101.gate_proj", + "model.layers.73.mlp.experts.101.up_proj", + "model.layers.73.mlp.experts.102.down_proj", + "model.layers.73.mlp.experts.102.gate_proj", + "model.layers.73.mlp.experts.102.up_proj", + "model.layers.73.mlp.experts.103.down_proj", + "model.layers.73.mlp.experts.103.gate_proj", + "model.layers.73.mlp.experts.103.up_proj", + "model.layers.73.mlp.experts.104.down_proj", + "model.layers.73.mlp.experts.104.gate_proj", + "model.layers.73.mlp.experts.104.up_proj", + "model.layers.73.mlp.experts.105.down_proj", + "model.layers.73.mlp.experts.105.gate_proj", + "model.layers.73.mlp.experts.105.up_proj", + "model.layers.73.mlp.experts.106.down_proj", + "model.layers.73.mlp.experts.106.gate_proj", + "model.layers.73.mlp.experts.106.up_proj", + "model.layers.73.mlp.experts.107.down_proj", + "model.layers.73.mlp.experts.107.gate_proj", + "model.layers.73.mlp.experts.107.up_proj", + "model.layers.73.mlp.experts.108.down_proj", + "model.layers.73.mlp.experts.108.gate_proj", + "model.layers.73.mlp.experts.108.up_proj", + "model.layers.73.mlp.experts.109.down_proj", + "model.layers.73.mlp.experts.109.gate_proj", + "model.layers.73.mlp.experts.109.up_proj", + "model.layers.73.mlp.experts.11.down_proj", + "model.layers.73.mlp.experts.11.gate_proj", + "model.layers.73.mlp.experts.11.up_proj", + "model.layers.73.mlp.experts.110.down_proj", + "model.layers.73.mlp.experts.110.gate_proj", + "model.layers.73.mlp.experts.110.up_proj", + "model.layers.73.mlp.experts.111.down_proj", + "model.layers.73.mlp.experts.111.gate_proj", + "model.layers.73.mlp.experts.111.up_proj", + "model.layers.73.mlp.experts.112.down_proj", + "model.layers.73.mlp.experts.112.gate_proj", + "model.layers.73.mlp.experts.112.up_proj", + "model.layers.73.mlp.experts.113.down_proj", + "model.layers.73.mlp.experts.113.gate_proj", + "model.layers.73.mlp.experts.113.up_proj", + "model.layers.73.mlp.experts.114.down_proj", + "model.layers.73.mlp.experts.114.gate_proj", + "model.layers.73.mlp.experts.114.up_proj", + "model.layers.73.mlp.experts.115.down_proj", + "model.layers.73.mlp.experts.115.gate_proj", + "model.layers.73.mlp.experts.115.up_proj", + "model.layers.73.mlp.experts.116.down_proj", + "model.layers.73.mlp.experts.116.gate_proj", + "model.layers.73.mlp.experts.116.up_proj", + "model.layers.73.mlp.experts.117.down_proj", + "model.layers.73.mlp.experts.117.gate_proj", + "model.layers.73.mlp.experts.117.up_proj", + "model.layers.73.mlp.experts.118.down_proj", + "model.layers.73.mlp.experts.118.gate_proj", + "model.layers.73.mlp.experts.118.up_proj", + "model.layers.73.mlp.experts.119.down_proj", + "model.layers.73.mlp.experts.119.gate_proj", + "model.layers.73.mlp.experts.119.up_proj", + "model.layers.73.mlp.experts.12.down_proj", + "model.layers.73.mlp.experts.12.gate_proj", + "model.layers.73.mlp.experts.12.up_proj", + "model.layers.73.mlp.experts.120.down_proj", + "model.layers.73.mlp.experts.120.gate_proj", + "model.layers.73.mlp.experts.120.up_proj", + "model.layers.73.mlp.experts.121.down_proj", + "model.layers.73.mlp.experts.121.gate_proj", + "model.layers.73.mlp.experts.121.up_proj", + "model.layers.73.mlp.experts.122.down_proj", + "model.layers.73.mlp.experts.122.gate_proj", + "model.layers.73.mlp.experts.122.up_proj", + "model.layers.73.mlp.experts.123.down_proj", + "model.layers.73.mlp.experts.123.gate_proj", + "model.layers.73.mlp.experts.123.up_proj", + "model.layers.73.mlp.experts.124.down_proj", + "model.layers.73.mlp.experts.124.gate_proj", + "model.layers.73.mlp.experts.124.up_proj", + "model.layers.73.mlp.experts.125.down_proj", + "model.layers.73.mlp.experts.125.gate_proj", + "model.layers.73.mlp.experts.125.up_proj", + "model.layers.73.mlp.experts.126.down_proj", + "model.layers.73.mlp.experts.126.gate_proj", + "model.layers.73.mlp.experts.126.up_proj", + "model.layers.73.mlp.experts.127.down_proj", + "model.layers.73.mlp.experts.127.gate_proj", + "model.layers.73.mlp.experts.127.up_proj", + "model.layers.73.mlp.experts.128.down_proj", + "model.layers.73.mlp.experts.128.gate_proj", + "model.layers.73.mlp.experts.128.up_proj", + "model.layers.73.mlp.experts.129.down_proj", + "model.layers.73.mlp.experts.129.gate_proj", + "model.layers.73.mlp.experts.129.up_proj", + "model.layers.73.mlp.experts.13.down_proj", + "model.layers.73.mlp.experts.13.gate_proj", + "model.layers.73.mlp.experts.13.up_proj", + "model.layers.73.mlp.experts.130.down_proj", + "model.layers.73.mlp.experts.130.gate_proj", + "model.layers.73.mlp.experts.130.up_proj", + "model.layers.73.mlp.experts.131.down_proj", + "model.layers.73.mlp.experts.131.gate_proj", + "model.layers.73.mlp.experts.131.up_proj", + "model.layers.73.mlp.experts.132.down_proj", + "model.layers.73.mlp.experts.132.gate_proj", + "model.layers.73.mlp.experts.132.up_proj", + "model.layers.73.mlp.experts.133.down_proj", + "model.layers.73.mlp.experts.133.gate_proj", + "model.layers.73.mlp.experts.133.up_proj", + "model.layers.73.mlp.experts.134.down_proj", + "model.layers.73.mlp.experts.134.gate_proj", + "model.layers.73.mlp.experts.134.up_proj", + "model.layers.73.mlp.experts.135.down_proj", + "model.layers.73.mlp.experts.135.gate_proj", + "model.layers.73.mlp.experts.135.up_proj", + "model.layers.73.mlp.experts.136.down_proj", + "model.layers.73.mlp.experts.136.gate_proj", + "model.layers.73.mlp.experts.136.up_proj", + "model.layers.73.mlp.experts.137.down_proj", + "model.layers.73.mlp.experts.137.gate_proj", + "model.layers.73.mlp.experts.137.up_proj", + "model.layers.73.mlp.experts.138.down_proj", + "model.layers.73.mlp.experts.138.gate_proj", + "model.layers.73.mlp.experts.138.up_proj", + "model.layers.73.mlp.experts.139.down_proj", + "model.layers.73.mlp.experts.139.gate_proj", + "model.layers.73.mlp.experts.139.up_proj", + "model.layers.73.mlp.experts.14.down_proj", + "model.layers.73.mlp.experts.14.gate_proj", + "model.layers.73.mlp.experts.14.up_proj", + "model.layers.73.mlp.experts.140.down_proj", + "model.layers.73.mlp.experts.140.gate_proj", + "model.layers.73.mlp.experts.140.up_proj", + "model.layers.73.mlp.experts.141.down_proj", + "model.layers.73.mlp.experts.141.gate_proj", + "model.layers.73.mlp.experts.141.up_proj", + "model.layers.73.mlp.experts.142.down_proj", + "model.layers.73.mlp.experts.142.gate_proj", + "model.layers.73.mlp.experts.142.up_proj", + "model.layers.73.mlp.experts.143.down_proj", + "model.layers.73.mlp.experts.143.gate_proj", + "model.layers.73.mlp.experts.143.up_proj", + "model.layers.73.mlp.experts.144.down_proj", + "model.layers.73.mlp.experts.144.gate_proj", + "model.layers.73.mlp.experts.144.up_proj", + "model.layers.73.mlp.experts.145.down_proj", + "model.layers.73.mlp.experts.145.gate_proj", + "model.layers.73.mlp.experts.145.up_proj", + "model.layers.73.mlp.experts.146.down_proj", + "model.layers.73.mlp.experts.146.gate_proj", + "model.layers.73.mlp.experts.146.up_proj", + "model.layers.73.mlp.experts.147.down_proj", + "model.layers.73.mlp.experts.147.gate_proj", + "model.layers.73.mlp.experts.147.up_proj", + "model.layers.73.mlp.experts.148.down_proj", + "model.layers.73.mlp.experts.148.gate_proj", + "model.layers.73.mlp.experts.148.up_proj", + "model.layers.73.mlp.experts.149.down_proj", + "model.layers.73.mlp.experts.149.gate_proj", + "model.layers.73.mlp.experts.149.up_proj", + "model.layers.73.mlp.experts.15.down_proj", + "model.layers.73.mlp.experts.15.gate_proj", + "model.layers.73.mlp.experts.15.up_proj", + "model.layers.73.mlp.experts.150.down_proj", + "model.layers.73.mlp.experts.150.gate_proj", + "model.layers.73.mlp.experts.150.up_proj", + "model.layers.73.mlp.experts.151.down_proj", + "model.layers.73.mlp.experts.151.gate_proj", + "model.layers.73.mlp.experts.151.up_proj", + "model.layers.73.mlp.experts.152.down_proj", + "model.layers.73.mlp.experts.152.gate_proj", + "model.layers.73.mlp.experts.152.up_proj", + "model.layers.73.mlp.experts.153.down_proj", + "model.layers.73.mlp.experts.153.gate_proj", + "model.layers.73.mlp.experts.153.up_proj", + "model.layers.73.mlp.experts.154.down_proj", + "model.layers.73.mlp.experts.154.gate_proj", + "model.layers.73.mlp.experts.154.up_proj", + "model.layers.73.mlp.experts.155.down_proj", + "model.layers.73.mlp.experts.155.gate_proj", + "model.layers.73.mlp.experts.155.up_proj", + "model.layers.73.mlp.experts.156.down_proj", + "model.layers.73.mlp.experts.156.gate_proj", + "model.layers.73.mlp.experts.156.up_proj", + "model.layers.73.mlp.experts.157.down_proj", + "model.layers.73.mlp.experts.157.gate_proj", + "model.layers.73.mlp.experts.157.up_proj", + "model.layers.73.mlp.experts.158.down_proj", + "model.layers.73.mlp.experts.158.gate_proj", + "model.layers.73.mlp.experts.158.up_proj", + "model.layers.73.mlp.experts.159.down_proj", + "model.layers.73.mlp.experts.159.gate_proj", + "model.layers.73.mlp.experts.159.up_proj", + "model.layers.73.mlp.experts.16.down_proj", + "model.layers.73.mlp.experts.16.gate_proj", + "model.layers.73.mlp.experts.16.up_proj", + "model.layers.73.mlp.experts.160.down_proj", + "model.layers.73.mlp.experts.160.gate_proj", + "model.layers.73.mlp.experts.160.up_proj", + "model.layers.73.mlp.experts.161.down_proj", + "model.layers.73.mlp.experts.161.gate_proj", + "model.layers.73.mlp.experts.161.up_proj", + "model.layers.73.mlp.experts.162.down_proj", + "model.layers.73.mlp.experts.162.gate_proj", + "model.layers.73.mlp.experts.162.up_proj", + "model.layers.73.mlp.experts.163.down_proj", + "model.layers.73.mlp.experts.163.gate_proj", + "model.layers.73.mlp.experts.163.up_proj", + "model.layers.73.mlp.experts.164.down_proj", + "model.layers.73.mlp.experts.164.gate_proj", + "model.layers.73.mlp.experts.164.up_proj", + "model.layers.73.mlp.experts.165.down_proj", + "model.layers.73.mlp.experts.165.gate_proj", + "model.layers.73.mlp.experts.165.up_proj", + "model.layers.73.mlp.experts.166.down_proj", + "model.layers.73.mlp.experts.166.gate_proj", + "model.layers.73.mlp.experts.166.up_proj", + "model.layers.73.mlp.experts.167.down_proj", + "model.layers.73.mlp.experts.167.gate_proj", + "model.layers.73.mlp.experts.167.up_proj", + "model.layers.73.mlp.experts.168.down_proj", + "model.layers.73.mlp.experts.168.gate_proj", + "model.layers.73.mlp.experts.168.up_proj", + "model.layers.73.mlp.experts.169.down_proj", + "model.layers.73.mlp.experts.169.gate_proj", + "model.layers.73.mlp.experts.169.up_proj", + "model.layers.73.mlp.experts.17.down_proj", + "model.layers.73.mlp.experts.17.gate_proj", + "model.layers.73.mlp.experts.17.up_proj", + "model.layers.73.mlp.experts.170.down_proj", + "model.layers.73.mlp.experts.170.gate_proj", + "model.layers.73.mlp.experts.170.up_proj", + "model.layers.73.mlp.experts.171.down_proj", + "model.layers.73.mlp.experts.171.gate_proj", + "model.layers.73.mlp.experts.171.up_proj", + "model.layers.73.mlp.experts.172.down_proj", + "model.layers.73.mlp.experts.172.gate_proj", + "model.layers.73.mlp.experts.172.up_proj", + "model.layers.73.mlp.experts.173.down_proj", + "model.layers.73.mlp.experts.173.gate_proj", + "model.layers.73.mlp.experts.173.up_proj", + "model.layers.73.mlp.experts.174.down_proj", + "model.layers.73.mlp.experts.174.gate_proj", + "model.layers.73.mlp.experts.174.up_proj", + "model.layers.73.mlp.experts.175.down_proj", + "model.layers.73.mlp.experts.175.gate_proj", + "model.layers.73.mlp.experts.175.up_proj", + "model.layers.73.mlp.experts.176.down_proj", + "model.layers.73.mlp.experts.176.gate_proj", + "model.layers.73.mlp.experts.176.up_proj", + "model.layers.73.mlp.experts.177.down_proj", + "model.layers.73.mlp.experts.177.gate_proj", + "model.layers.73.mlp.experts.177.up_proj", + "model.layers.73.mlp.experts.178.down_proj", + "model.layers.73.mlp.experts.178.gate_proj", + "model.layers.73.mlp.experts.178.up_proj", + "model.layers.73.mlp.experts.179.down_proj", + "model.layers.73.mlp.experts.179.gate_proj", + "model.layers.73.mlp.experts.179.up_proj", + "model.layers.73.mlp.experts.18.down_proj", + "model.layers.73.mlp.experts.18.gate_proj", + "model.layers.73.mlp.experts.18.up_proj", + "model.layers.73.mlp.experts.180.down_proj", + "model.layers.73.mlp.experts.180.gate_proj", + "model.layers.73.mlp.experts.180.up_proj", + "model.layers.73.mlp.experts.181.down_proj", + "model.layers.73.mlp.experts.181.gate_proj", + "model.layers.73.mlp.experts.181.up_proj", + "model.layers.73.mlp.experts.182.down_proj", + "model.layers.73.mlp.experts.182.gate_proj", + "model.layers.73.mlp.experts.182.up_proj", + "model.layers.73.mlp.experts.183.down_proj", + "model.layers.73.mlp.experts.183.gate_proj", + "model.layers.73.mlp.experts.183.up_proj", + "model.layers.73.mlp.experts.184.down_proj", + "model.layers.73.mlp.experts.184.gate_proj", + "model.layers.73.mlp.experts.184.up_proj", + "model.layers.73.mlp.experts.185.down_proj", + "model.layers.73.mlp.experts.185.gate_proj", + "model.layers.73.mlp.experts.185.up_proj", + "model.layers.73.mlp.experts.186.down_proj", + "model.layers.73.mlp.experts.186.gate_proj", + "model.layers.73.mlp.experts.186.up_proj", + "model.layers.73.mlp.experts.187.down_proj", + "model.layers.73.mlp.experts.187.gate_proj", + "model.layers.73.mlp.experts.187.up_proj", + "model.layers.73.mlp.experts.188.down_proj", + "model.layers.73.mlp.experts.188.gate_proj", + "model.layers.73.mlp.experts.188.up_proj", + "model.layers.73.mlp.experts.189.down_proj", + "model.layers.73.mlp.experts.189.gate_proj", + "model.layers.73.mlp.experts.189.up_proj", + "model.layers.73.mlp.experts.19.down_proj", + "model.layers.73.mlp.experts.19.gate_proj", + "model.layers.73.mlp.experts.19.up_proj", + "model.layers.73.mlp.experts.190.down_proj", + "model.layers.73.mlp.experts.190.gate_proj", + "model.layers.73.mlp.experts.190.up_proj", + "model.layers.73.mlp.experts.191.down_proj", + "model.layers.73.mlp.experts.191.gate_proj", + "model.layers.73.mlp.experts.191.up_proj", + "model.layers.73.mlp.experts.192.down_proj", + "model.layers.73.mlp.experts.192.gate_proj", + "model.layers.73.mlp.experts.192.up_proj", + "model.layers.73.mlp.experts.193.down_proj", + "model.layers.73.mlp.experts.193.gate_proj", + "model.layers.73.mlp.experts.193.up_proj", + "model.layers.73.mlp.experts.194.down_proj", + "model.layers.73.mlp.experts.194.gate_proj", + "model.layers.73.mlp.experts.194.up_proj", + "model.layers.73.mlp.experts.195.down_proj", + "model.layers.73.mlp.experts.195.gate_proj", + "model.layers.73.mlp.experts.195.up_proj", + "model.layers.73.mlp.experts.196.down_proj", + "model.layers.73.mlp.experts.196.gate_proj", + "model.layers.73.mlp.experts.196.up_proj", + "model.layers.73.mlp.experts.197.down_proj", + "model.layers.73.mlp.experts.197.gate_proj", + "model.layers.73.mlp.experts.197.up_proj", + "model.layers.73.mlp.experts.198.down_proj", + "model.layers.73.mlp.experts.198.gate_proj", + "model.layers.73.mlp.experts.198.up_proj", + "model.layers.73.mlp.experts.199.down_proj", + "model.layers.73.mlp.experts.199.gate_proj", + "model.layers.73.mlp.experts.199.up_proj", + "model.layers.73.mlp.experts.2.down_proj", + "model.layers.73.mlp.experts.2.gate_proj", + "model.layers.73.mlp.experts.2.up_proj", + "model.layers.73.mlp.experts.20.down_proj", + "model.layers.73.mlp.experts.20.gate_proj", + "model.layers.73.mlp.experts.20.up_proj", + "model.layers.73.mlp.experts.200.down_proj", + "model.layers.73.mlp.experts.200.gate_proj", + "model.layers.73.mlp.experts.200.up_proj", + "model.layers.73.mlp.experts.201.down_proj", + "model.layers.73.mlp.experts.201.gate_proj", + "model.layers.73.mlp.experts.201.up_proj", + "model.layers.73.mlp.experts.202.down_proj", + "model.layers.73.mlp.experts.202.gate_proj", + "model.layers.73.mlp.experts.202.up_proj", + "model.layers.73.mlp.experts.203.down_proj", + "model.layers.73.mlp.experts.203.gate_proj", + "model.layers.73.mlp.experts.203.up_proj", + "model.layers.73.mlp.experts.204.down_proj", + "model.layers.73.mlp.experts.204.gate_proj", + "model.layers.73.mlp.experts.204.up_proj", + "model.layers.73.mlp.experts.205.down_proj", + "model.layers.73.mlp.experts.205.gate_proj", + "model.layers.73.mlp.experts.205.up_proj", + "model.layers.73.mlp.experts.206.down_proj", + "model.layers.73.mlp.experts.206.gate_proj", + "model.layers.73.mlp.experts.206.up_proj", + "model.layers.73.mlp.experts.207.down_proj", + "model.layers.73.mlp.experts.207.gate_proj", + "model.layers.73.mlp.experts.207.up_proj", + "model.layers.73.mlp.experts.208.down_proj", + "model.layers.73.mlp.experts.208.gate_proj", + "model.layers.73.mlp.experts.208.up_proj", + "model.layers.73.mlp.experts.209.down_proj", + "model.layers.73.mlp.experts.209.gate_proj", + "model.layers.73.mlp.experts.209.up_proj", + "model.layers.73.mlp.experts.21.down_proj", + "model.layers.73.mlp.experts.21.gate_proj", + "model.layers.73.mlp.experts.21.up_proj", + "model.layers.73.mlp.experts.210.down_proj", + "model.layers.73.mlp.experts.210.gate_proj", + "model.layers.73.mlp.experts.210.up_proj", + "model.layers.73.mlp.experts.211.down_proj", + "model.layers.73.mlp.experts.211.gate_proj", + "model.layers.73.mlp.experts.211.up_proj", + "model.layers.73.mlp.experts.212.down_proj", + "model.layers.73.mlp.experts.212.gate_proj", + "model.layers.73.mlp.experts.212.up_proj", + "model.layers.73.mlp.experts.213.down_proj", + "model.layers.73.mlp.experts.213.gate_proj", + "model.layers.73.mlp.experts.213.up_proj", + "model.layers.73.mlp.experts.214.down_proj", + "model.layers.73.mlp.experts.214.gate_proj", + "model.layers.73.mlp.experts.214.up_proj", + "model.layers.73.mlp.experts.215.down_proj", + "model.layers.73.mlp.experts.215.gate_proj", + "model.layers.73.mlp.experts.215.up_proj", + "model.layers.73.mlp.experts.216.down_proj", + "model.layers.73.mlp.experts.216.gate_proj", + "model.layers.73.mlp.experts.216.up_proj", + "model.layers.73.mlp.experts.217.down_proj", + "model.layers.73.mlp.experts.217.gate_proj", + "model.layers.73.mlp.experts.217.up_proj", + "model.layers.73.mlp.experts.218.down_proj", + "model.layers.73.mlp.experts.218.gate_proj", + "model.layers.73.mlp.experts.218.up_proj", + "model.layers.73.mlp.experts.219.down_proj", + "model.layers.73.mlp.experts.219.gate_proj", + "model.layers.73.mlp.experts.219.up_proj", + "model.layers.73.mlp.experts.22.down_proj", + "model.layers.73.mlp.experts.22.gate_proj", + "model.layers.73.mlp.experts.22.up_proj", + "model.layers.73.mlp.experts.220.down_proj", + "model.layers.73.mlp.experts.220.gate_proj", + "model.layers.73.mlp.experts.220.up_proj", + "model.layers.73.mlp.experts.221.down_proj", + "model.layers.73.mlp.experts.221.gate_proj", + "model.layers.73.mlp.experts.221.up_proj", + "model.layers.73.mlp.experts.222.down_proj", + "model.layers.73.mlp.experts.222.gate_proj", + "model.layers.73.mlp.experts.222.up_proj", + "model.layers.73.mlp.experts.223.down_proj", + "model.layers.73.mlp.experts.223.gate_proj", + "model.layers.73.mlp.experts.223.up_proj", + "model.layers.73.mlp.experts.224.down_proj", + "model.layers.73.mlp.experts.224.gate_proj", + "model.layers.73.mlp.experts.224.up_proj", + "model.layers.73.mlp.experts.225.down_proj", + "model.layers.73.mlp.experts.225.gate_proj", + "model.layers.73.mlp.experts.225.up_proj", + "model.layers.73.mlp.experts.226.down_proj", + "model.layers.73.mlp.experts.226.gate_proj", + "model.layers.73.mlp.experts.226.up_proj", + "model.layers.73.mlp.experts.227.down_proj", + "model.layers.73.mlp.experts.227.gate_proj", + "model.layers.73.mlp.experts.227.up_proj", + "model.layers.73.mlp.experts.228.down_proj", + "model.layers.73.mlp.experts.228.gate_proj", + "model.layers.73.mlp.experts.228.up_proj", + "model.layers.73.mlp.experts.229.down_proj", + "model.layers.73.mlp.experts.229.gate_proj", + "model.layers.73.mlp.experts.229.up_proj", + "model.layers.73.mlp.experts.23.down_proj", + "model.layers.73.mlp.experts.23.gate_proj", + "model.layers.73.mlp.experts.23.up_proj", + "model.layers.73.mlp.experts.230.down_proj", + "model.layers.73.mlp.experts.230.gate_proj", + "model.layers.73.mlp.experts.230.up_proj", + "model.layers.73.mlp.experts.231.down_proj", + "model.layers.73.mlp.experts.231.gate_proj", + "model.layers.73.mlp.experts.231.up_proj", + "model.layers.73.mlp.experts.232.down_proj", + "model.layers.73.mlp.experts.232.gate_proj", + "model.layers.73.mlp.experts.232.up_proj", + "model.layers.73.mlp.experts.233.down_proj", + "model.layers.73.mlp.experts.233.gate_proj", + "model.layers.73.mlp.experts.233.up_proj", + "model.layers.73.mlp.experts.234.down_proj", + "model.layers.73.mlp.experts.234.gate_proj", + "model.layers.73.mlp.experts.234.up_proj", + "model.layers.73.mlp.experts.235.down_proj", + "model.layers.73.mlp.experts.235.gate_proj", + "model.layers.73.mlp.experts.235.up_proj", + "model.layers.73.mlp.experts.236.down_proj", + "model.layers.73.mlp.experts.236.gate_proj", + "model.layers.73.mlp.experts.236.up_proj", + "model.layers.73.mlp.experts.237.down_proj", + "model.layers.73.mlp.experts.237.gate_proj", + "model.layers.73.mlp.experts.237.up_proj", + "model.layers.73.mlp.experts.238.down_proj", + "model.layers.73.mlp.experts.238.gate_proj", + "model.layers.73.mlp.experts.238.up_proj", + "model.layers.73.mlp.experts.239.down_proj", + "model.layers.73.mlp.experts.239.gate_proj", + "model.layers.73.mlp.experts.239.up_proj", + "model.layers.73.mlp.experts.24.down_proj", + "model.layers.73.mlp.experts.24.gate_proj", + "model.layers.73.mlp.experts.24.up_proj", + "model.layers.73.mlp.experts.240.down_proj", + "model.layers.73.mlp.experts.240.gate_proj", + "model.layers.73.mlp.experts.240.up_proj", + "model.layers.73.mlp.experts.241.down_proj", + "model.layers.73.mlp.experts.241.gate_proj", + "model.layers.73.mlp.experts.241.up_proj", + "model.layers.73.mlp.experts.242.down_proj", + "model.layers.73.mlp.experts.242.gate_proj", + "model.layers.73.mlp.experts.242.up_proj", + "model.layers.73.mlp.experts.243.down_proj", + "model.layers.73.mlp.experts.243.gate_proj", + "model.layers.73.mlp.experts.243.up_proj", + "model.layers.73.mlp.experts.244.down_proj", + "model.layers.73.mlp.experts.244.gate_proj", + "model.layers.73.mlp.experts.244.up_proj", + "model.layers.73.mlp.experts.245.down_proj", + "model.layers.73.mlp.experts.245.gate_proj", + "model.layers.73.mlp.experts.245.up_proj", + "model.layers.73.mlp.experts.246.down_proj", + "model.layers.73.mlp.experts.246.gate_proj", + "model.layers.73.mlp.experts.246.up_proj", + "model.layers.73.mlp.experts.247.down_proj", + "model.layers.73.mlp.experts.247.gate_proj", + "model.layers.73.mlp.experts.247.up_proj", + "model.layers.73.mlp.experts.248.down_proj", + "model.layers.73.mlp.experts.248.gate_proj", + "model.layers.73.mlp.experts.248.up_proj", + "model.layers.73.mlp.experts.249.down_proj", + "model.layers.73.mlp.experts.249.gate_proj", + "model.layers.73.mlp.experts.249.up_proj", + "model.layers.73.mlp.experts.25.down_proj", + "model.layers.73.mlp.experts.25.gate_proj", + "model.layers.73.mlp.experts.25.up_proj", + "model.layers.73.mlp.experts.250.down_proj", + "model.layers.73.mlp.experts.250.gate_proj", + "model.layers.73.mlp.experts.250.up_proj", + "model.layers.73.mlp.experts.251.down_proj", + "model.layers.73.mlp.experts.251.gate_proj", + "model.layers.73.mlp.experts.251.up_proj", + "model.layers.73.mlp.experts.252.down_proj", + "model.layers.73.mlp.experts.252.gate_proj", + "model.layers.73.mlp.experts.252.up_proj", + "model.layers.73.mlp.experts.253.down_proj", + "model.layers.73.mlp.experts.253.gate_proj", + "model.layers.73.mlp.experts.253.up_proj", + "model.layers.73.mlp.experts.254.down_proj", + "model.layers.73.mlp.experts.254.gate_proj", + "model.layers.73.mlp.experts.254.up_proj", + "model.layers.73.mlp.experts.255.down_proj", + "model.layers.73.mlp.experts.255.gate_proj", + "model.layers.73.mlp.experts.255.up_proj", + "model.layers.73.mlp.experts.26.down_proj", + "model.layers.73.mlp.experts.26.gate_proj", + "model.layers.73.mlp.experts.26.up_proj", + "model.layers.73.mlp.experts.27.down_proj", + "model.layers.73.mlp.experts.27.gate_proj", + "model.layers.73.mlp.experts.27.up_proj", + "model.layers.73.mlp.experts.28.down_proj", + "model.layers.73.mlp.experts.28.gate_proj", + "model.layers.73.mlp.experts.28.up_proj", + "model.layers.73.mlp.experts.29.down_proj", + "model.layers.73.mlp.experts.29.gate_proj", + "model.layers.73.mlp.experts.29.up_proj", + "model.layers.73.mlp.experts.3.down_proj", + "model.layers.73.mlp.experts.3.gate_proj", + "model.layers.73.mlp.experts.3.up_proj", + "model.layers.73.mlp.experts.30.down_proj", + "model.layers.73.mlp.experts.30.gate_proj", + "model.layers.73.mlp.experts.30.up_proj", + "model.layers.73.mlp.experts.31.down_proj", + "model.layers.73.mlp.experts.31.gate_proj", + "model.layers.73.mlp.experts.31.up_proj", + "model.layers.73.mlp.experts.32.down_proj", + "model.layers.73.mlp.experts.32.gate_proj", + "model.layers.73.mlp.experts.32.up_proj", + "model.layers.73.mlp.experts.33.down_proj", + "model.layers.73.mlp.experts.33.gate_proj", + "model.layers.73.mlp.experts.33.up_proj", + "model.layers.73.mlp.experts.34.down_proj", + "model.layers.73.mlp.experts.34.gate_proj", + "model.layers.73.mlp.experts.34.up_proj", + "model.layers.73.mlp.experts.35.down_proj", + "model.layers.73.mlp.experts.35.gate_proj", + "model.layers.73.mlp.experts.35.up_proj", + "model.layers.73.mlp.experts.36.down_proj", + "model.layers.73.mlp.experts.36.gate_proj", + "model.layers.73.mlp.experts.36.up_proj", + "model.layers.73.mlp.experts.37.down_proj", + "model.layers.73.mlp.experts.37.gate_proj", + "model.layers.73.mlp.experts.37.up_proj", + "model.layers.73.mlp.experts.38.down_proj", + "model.layers.73.mlp.experts.38.gate_proj", + "model.layers.73.mlp.experts.38.up_proj", + "model.layers.73.mlp.experts.39.down_proj", + "model.layers.73.mlp.experts.39.gate_proj", + "model.layers.73.mlp.experts.39.up_proj", + "model.layers.73.mlp.experts.4.down_proj", + "model.layers.73.mlp.experts.4.gate_proj", + "model.layers.73.mlp.experts.4.up_proj", + "model.layers.73.mlp.experts.40.down_proj", + "model.layers.73.mlp.experts.40.gate_proj", + "model.layers.73.mlp.experts.40.up_proj", + "model.layers.73.mlp.experts.41.down_proj", + "model.layers.73.mlp.experts.41.gate_proj", + "model.layers.73.mlp.experts.41.up_proj", + "model.layers.73.mlp.experts.42.down_proj", + "model.layers.73.mlp.experts.42.gate_proj", + "model.layers.73.mlp.experts.42.up_proj", + "model.layers.73.mlp.experts.43.down_proj", + "model.layers.73.mlp.experts.43.gate_proj", + "model.layers.73.mlp.experts.43.up_proj", + "model.layers.73.mlp.experts.44.down_proj", + "model.layers.73.mlp.experts.44.gate_proj", + "model.layers.73.mlp.experts.44.up_proj", + "model.layers.73.mlp.experts.45.down_proj", + "model.layers.73.mlp.experts.45.gate_proj", + "model.layers.73.mlp.experts.45.up_proj", + "model.layers.73.mlp.experts.46.down_proj", + "model.layers.73.mlp.experts.46.gate_proj", + "model.layers.73.mlp.experts.46.up_proj", + "model.layers.73.mlp.experts.47.down_proj", + "model.layers.73.mlp.experts.47.gate_proj", + "model.layers.73.mlp.experts.47.up_proj", + "model.layers.73.mlp.experts.48.down_proj", + "model.layers.73.mlp.experts.48.gate_proj", + "model.layers.73.mlp.experts.48.up_proj", + "model.layers.73.mlp.experts.49.down_proj", + "model.layers.73.mlp.experts.49.gate_proj", + "model.layers.73.mlp.experts.49.up_proj", + "model.layers.73.mlp.experts.5.down_proj", + "model.layers.73.mlp.experts.5.gate_proj", + "model.layers.73.mlp.experts.5.up_proj", + "model.layers.73.mlp.experts.50.down_proj", + "model.layers.73.mlp.experts.50.gate_proj", + "model.layers.73.mlp.experts.50.up_proj", + "model.layers.73.mlp.experts.51.down_proj", + "model.layers.73.mlp.experts.51.gate_proj", + "model.layers.73.mlp.experts.51.up_proj", + "model.layers.73.mlp.experts.52.down_proj", + "model.layers.73.mlp.experts.52.gate_proj", + "model.layers.73.mlp.experts.52.up_proj", + "model.layers.73.mlp.experts.53.down_proj", + "model.layers.73.mlp.experts.53.gate_proj", + "model.layers.73.mlp.experts.53.up_proj", + "model.layers.73.mlp.experts.54.down_proj", + "model.layers.73.mlp.experts.54.gate_proj", + "model.layers.73.mlp.experts.54.up_proj", + "model.layers.73.mlp.experts.55.down_proj", + "model.layers.73.mlp.experts.55.gate_proj", + "model.layers.73.mlp.experts.55.up_proj", + "model.layers.73.mlp.experts.56.down_proj", + "model.layers.73.mlp.experts.56.gate_proj", + "model.layers.73.mlp.experts.56.up_proj", + "model.layers.73.mlp.experts.57.down_proj", + "model.layers.73.mlp.experts.57.gate_proj", + "model.layers.73.mlp.experts.57.up_proj", + "model.layers.73.mlp.experts.58.down_proj", + "model.layers.73.mlp.experts.58.gate_proj", + "model.layers.73.mlp.experts.58.up_proj", + "model.layers.73.mlp.experts.59.down_proj", + "model.layers.73.mlp.experts.59.gate_proj", + "model.layers.73.mlp.experts.59.up_proj", + "model.layers.73.mlp.experts.6.down_proj", + "model.layers.73.mlp.experts.6.gate_proj", + "model.layers.73.mlp.experts.6.up_proj", + "model.layers.73.mlp.experts.60.down_proj", + "model.layers.73.mlp.experts.60.gate_proj", + "model.layers.73.mlp.experts.60.up_proj", + "model.layers.73.mlp.experts.61.down_proj", + "model.layers.73.mlp.experts.61.gate_proj", + "model.layers.73.mlp.experts.61.up_proj", + "model.layers.73.mlp.experts.62.down_proj", + "model.layers.73.mlp.experts.62.gate_proj", + "model.layers.73.mlp.experts.62.up_proj", + "model.layers.73.mlp.experts.63.down_proj", + "model.layers.73.mlp.experts.63.gate_proj", + "model.layers.73.mlp.experts.63.up_proj", + "model.layers.73.mlp.experts.64.down_proj", + "model.layers.73.mlp.experts.64.gate_proj", + "model.layers.73.mlp.experts.64.up_proj", + "model.layers.73.mlp.experts.65.down_proj", + "model.layers.73.mlp.experts.65.gate_proj", + "model.layers.73.mlp.experts.65.up_proj", + "model.layers.73.mlp.experts.66.down_proj", + "model.layers.73.mlp.experts.66.gate_proj", + "model.layers.73.mlp.experts.66.up_proj", + "model.layers.73.mlp.experts.67.down_proj", + "model.layers.73.mlp.experts.67.gate_proj", + "model.layers.73.mlp.experts.67.up_proj", + "model.layers.73.mlp.experts.68.down_proj", + "model.layers.73.mlp.experts.68.gate_proj", + "model.layers.73.mlp.experts.68.up_proj", + "model.layers.73.mlp.experts.69.down_proj", + "model.layers.73.mlp.experts.69.gate_proj", + "model.layers.73.mlp.experts.69.up_proj", + "model.layers.73.mlp.experts.7.down_proj", + "model.layers.73.mlp.experts.7.gate_proj", + "model.layers.73.mlp.experts.7.up_proj", + "model.layers.73.mlp.experts.70.down_proj", + "model.layers.73.mlp.experts.70.gate_proj", + "model.layers.73.mlp.experts.70.up_proj", + "model.layers.73.mlp.experts.71.down_proj", + "model.layers.73.mlp.experts.71.gate_proj", + "model.layers.73.mlp.experts.71.up_proj", + "model.layers.73.mlp.experts.72.down_proj", + "model.layers.73.mlp.experts.72.gate_proj", + "model.layers.73.mlp.experts.72.up_proj", + "model.layers.73.mlp.experts.73.down_proj", + "model.layers.73.mlp.experts.73.gate_proj", + "model.layers.73.mlp.experts.73.up_proj", + "model.layers.73.mlp.experts.74.down_proj", + "model.layers.73.mlp.experts.74.gate_proj", + "model.layers.73.mlp.experts.74.up_proj", + "model.layers.73.mlp.experts.75.down_proj", + "model.layers.73.mlp.experts.75.gate_proj", + "model.layers.73.mlp.experts.75.up_proj", + "model.layers.73.mlp.experts.76.down_proj", + "model.layers.73.mlp.experts.76.gate_proj", + "model.layers.73.mlp.experts.76.up_proj", + "model.layers.73.mlp.experts.77.down_proj", + "model.layers.73.mlp.experts.77.gate_proj", + "model.layers.73.mlp.experts.77.up_proj", + "model.layers.73.mlp.experts.78.down_proj", + "model.layers.73.mlp.experts.78.gate_proj", + "model.layers.73.mlp.experts.78.up_proj", + "model.layers.73.mlp.experts.79.down_proj", + "model.layers.73.mlp.experts.79.gate_proj", + "model.layers.73.mlp.experts.79.up_proj", + "model.layers.73.mlp.experts.8.down_proj", + "model.layers.73.mlp.experts.8.gate_proj", + "model.layers.73.mlp.experts.8.up_proj", + "model.layers.73.mlp.experts.80.down_proj", + "model.layers.73.mlp.experts.80.gate_proj", + "model.layers.73.mlp.experts.80.up_proj", + "model.layers.73.mlp.experts.81.down_proj", + "model.layers.73.mlp.experts.81.gate_proj", + "model.layers.73.mlp.experts.81.up_proj", + "model.layers.73.mlp.experts.82.down_proj", + "model.layers.73.mlp.experts.82.gate_proj", + "model.layers.73.mlp.experts.82.up_proj", + "model.layers.73.mlp.experts.83.down_proj", + "model.layers.73.mlp.experts.83.gate_proj", + "model.layers.73.mlp.experts.83.up_proj", + "model.layers.73.mlp.experts.84.down_proj", + "model.layers.73.mlp.experts.84.gate_proj", + "model.layers.73.mlp.experts.84.up_proj", + "model.layers.73.mlp.experts.85.down_proj", + "model.layers.73.mlp.experts.85.gate_proj", + "model.layers.73.mlp.experts.85.up_proj", + "model.layers.73.mlp.experts.86.down_proj", + "model.layers.73.mlp.experts.86.gate_proj", + "model.layers.73.mlp.experts.86.up_proj", + "model.layers.73.mlp.experts.87.down_proj", + "model.layers.73.mlp.experts.87.gate_proj", + "model.layers.73.mlp.experts.87.up_proj", + "model.layers.73.mlp.experts.88.down_proj", + "model.layers.73.mlp.experts.88.gate_proj", + "model.layers.73.mlp.experts.88.up_proj", + "model.layers.73.mlp.experts.89.down_proj", + "model.layers.73.mlp.experts.89.gate_proj", + "model.layers.73.mlp.experts.89.up_proj", + "model.layers.73.mlp.experts.9.down_proj", + "model.layers.73.mlp.experts.9.gate_proj", + "model.layers.73.mlp.experts.9.up_proj", + "model.layers.73.mlp.experts.90.down_proj", + "model.layers.73.mlp.experts.90.gate_proj", + "model.layers.73.mlp.experts.90.up_proj", + "model.layers.73.mlp.experts.91.down_proj", + "model.layers.73.mlp.experts.91.gate_proj", + "model.layers.73.mlp.experts.91.up_proj", + "model.layers.73.mlp.experts.92.down_proj", + "model.layers.73.mlp.experts.92.gate_proj", + "model.layers.73.mlp.experts.92.up_proj", + "model.layers.73.mlp.experts.93.down_proj", + "model.layers.73.mlp.experts.93.gate_proj", + "model.layers.73.mlp.experts.93.up_proj", + "model.layers.73.mlp.experts.94.down_proj", + "model.layers.73.mlp.experts.94.gate_proj", + "model.layers.73.mlp.experts.94.up_proj", + "model.layers.73.mlp.experts.95.down_proj", + "model.layers.73.mlp.experts.95.gate_proj", + "model.layers.73.mlp.experts.95.up_proj", + "model.layers.73.mlp.experts.96.down_proj", + "model.layers.73.mlp.experts.96.gate_proj", + "model.layers.73.mlp.experts.96.up_proj", + "model.layers.73.mlp.experts.97.down_proj", + "model.layers.73.mlp.experts.97.gate_proj", + "model.layers.73.mlp.experts.97.up_proj", + "model.layers.73.mlp.experts.98.down_proj", + "model.layers.73.mlp.experts.98.gate_proj", + "model.layers.73.mlp.experts.98.up_proj", + "model.layers.73.mlp.experts.99.down_proj", + "model.layers.73.mlp.experts.99.gate_proj", + "model.layers.73.mlp.experts.99.up_proj", + "model.layers.74.mlp.experts.0.down_proj", + "model.layers.74.mlp.experts.0.gate_proj", + "model.layers.74.mlp.experts.0.up_proj", + "model.layers.74.mlp.experts.1.down_proj", + "model.layers.74.mlp.experts.1.gate_proj", + "model.layers.74.mlp.experts.1.up_proj", + "model.layers.74.mlp.experts.10.down_proj", + "model.layers.74.mlp.experts.10.gate_proj", + "model.layers.74.mlp.experts.10.up_proj", + "model.layers.74.mlp.experts.100.down_proj", + "model.layers.74.mlp.experts.100.gate_proj", + "model.layers.74.mlp.experts.100.up_proj", + "model.layers.74.mlp.experts.101.down_proj", + "model.layers.74.mlp.experts.101.gate_proj", + "model.layers.74.mlp.experts.101.up_proj", + "model.layers.74.mlp.experts.102.down_proj", + "model.layers.74.mlp.experts.102.gate_proj", + "model.layers.74.mlp.experts.102.up_proj", + "model.layers.74.mlp.experts.103.down_proj", + "model.layers.74.mlp.experts.103.gate_proj", + "model.layers.74.mlp.experts.103.up_proj", + "model.layers.74.mlp.experts.104.down_proj", + "model.layers.74.mlp.experts.104.gate_proj", + "model.layers.74.mlp.experts.104.up_proj", + "model.layers.74.mlp.experts.105.down_proj", + "model.layers.74.mlp.experts.105.gate_proj", + "model.layers.74.mlp.experts.105.up_proj", + "model.layers.74.mlp.experts.106.down_proj", + "model.layers.74.mlp.experts.106.gate_proj", + "model.layers.74.mlp.experts.106.up_proj", + "model.layers.74.mlp.experts.107.down_proj", + "model.layers.74.mlp.experts.107.gate_proj", + "model.layers.74.mlp.experts.107.up_proj", + "model.layers.74.mlp.experts.108.down_proj", + "model.layers.74.mlp.experts.108.gate_proj", + "model.layers.74.mlp.experts.108.up_proj", + "model.layers.74.mlp.experts.109.down_proj", + "model.layers.74.mlp.experts.109.gate_proj", + "model.layers.74.mlp.experts.109.up_proj", + "model.layers.74.mlp.experts.11.down_proj", + "model.layers.74.mlp.experts.11.gate_proj", + "model.layers.74.mlp.experts.11.up_proj", + "model.layers.74.mlp.experts.110.down_proj", + "model.layers.74.mlp.experts.110.gate_proj", + "model.layers.74.mlp.experts.110.up_proj", + "model.layers.74.mlp.experts.111.down_proj", + "model.layers.74.mlp.experts.111.gate_proj", + "model.layers.74.mlp.experts.111.up_proj", + "model.layers.74.mlp.experts.112.down_proj", + "model.layers.74.mlp.experts.112.gate_proj", + "model.layers.74.mlp.experts.112.up_proj", + "model.layers.74.mlp.experts.113.down_proj", + "model.layers.74.mlp.experts.113.gate_proj", + "model.layers.74.mlp.experts.113.up_proj", + "model.layers.74.mlp.experts.114.down_proj", + "model.layers.74.mlp.experts.114.gate_proj", + "model.layers.74.mlp.experts.114.up_proj", + "model.layers.74.mlp.experts.115.down_proj", + "model.layers.74.mlp.experts.115.gate_proj", + "model.layers.74.mlp.experts.115.up_proj", + "model.layers.74.mlp.experts.116.down_proj", + "model.layers.74.mlp.experts.116.gate_proj", + "model.layers.74.mlp.experts.116.up_proj", + "model.layers.74.mlp.experts.117.down_proj", + "model.layers.74.mlp.experts.117.gate_proj", + "model.layers.74.mlp.experts.117.up_proj", + "model.layers.74.mlp.experts.118.down_proj", + "model.layers.74.mlp.experts.118.gate_proj", + "model.layers.74.mlp.experts.118.up_proj", + "model.layers.74.mlp.experts.119.down_proj", + "model.layers.74.mlp.experts.119.gate_proj", + "model.layers.74.mlp.experts.119.up_proj", + "model.layers.74.mlp.experts.12.down_proj", + "model.layers.74.mlp.experts.12.gate_proj", + "model.layers.74.mlp.experts.12.up_proj", + "model.layers.74.mlp.experts.120.down_proj", + "model.layers.74.mlp.experts.120.gate_proj", + "model.layers.74.mlp.experts.120.up_proj", + "model.layers.74.mlp.experts.121.down_proj", + "model.layers.74.mlp.experts.121.gate_proj", + "model.layers.74.mlp.experts.121.up_proj", + "model.layers.74.mlp.experts.122.down_proj", + "model.layers.74.mlp.experts.122.gate_proj", + "model.layers.74.mlp.experts.122.up_proj", + "model.layers.74.mlp.experts.123.down_proj", + "model.layers.74.mlp.experts.123.gate_proj", + "model.layers.74.mlp.experts.123.up_proj", + "model.layers.74.mlp.experts.124.down_proj", + "model.layers.74.mlp.experts.124.gate_proj", + "model.layers.74.mlp.experts.124.up_proj", + "model.layers.74.mlp.experts.125.down_proj", + "model.layers.74.mlp.experts.125.gate_proj", + "model.layers.74.mlp.experts.125.up_proj", + "model.layers.74.mlp.experts.126.down_proj", + "model.layers.74.mlp.experts.126.gate_proj", + "model.layers.74.mlp.experts.126.up_proj", + "model.layers.74.mlp.experts.127.down_proj", + "model.layers.74.mlp.experts.127.gate_proj", + "model.layers.74.mlp.experts.127.up_proj", + "model.layers.74.mlp.experts.128.down_proj", + "model.layers.74.mlp.experts.128.gate_proj", + "model.layers.74.mlp.experts.128.up_proj", + "model.layers.74.mlp.experts.129.down_proj", + "model.layers.74.mlp.experts.129.gate_proj", + "model.layers.74.mlp.experts.129.up_proj", + "model.layers.74.mlp.experts.13.down_proj", + "model.layers.74.mlp.experts.13.gate_proj", + "model.layers.74.mlp.experts.13.up_proj", + "model.layers.74.mlp.experts.130.down_proj", + "model.layers.74.mlp.experts.130.gate_proj", + "model.layers.74.mlp.experts.130.up_proj", + "model.layers.74.mlp.experts.131.down_proj", + "model.layers.74.mlp.experts.131.gate_proj", + "model.layers.74.mlp.experts.131.up_proj", + "model.layers.74.mlp.experts.132.down_proj", + "model.layers.74.mlp.experts.132.gate_proj", + "model.layers.74.mlp.experts.132.up_proj", + "model.layers.74.mlp.experts.133.down_proj", + "model.layers.74.mlp.experts.133.gate_proj", + "model.layers.74.mlp.experts.133.up_proj", + "model.layers.74.mlp.experts.134.down_proj", + "model.layers.74.mlp.experts.134.gate_proj", + "model.layers.74.mlp.experts.134.up_proj", + "model.layers.74.mlp.experts.135.down_proj", + "model.layers.74.mlp.experts.135.gate_proj", + "model.layers.74.mlp.experts.135.up_proj", + "model.layers.74.mlp.experts.136.down_proj", + "model.layers.74.mlp.experts.136.gate_proj", + "model.layers.74.mlp.experts.136.up_proj", + "model.layers.74.mlp.experts.137.down_proj", + "model.layers.74.mlp.experts.137.gate_proj", + "model.layers.74.mlp.experts.137.up_proj", + "model.layers.74.mlp.experts.138.down_proj", + "model.layers.74.mlp.experts.138.gate_proj", + "model.layers.74.mlp.experts.138.up_proj", + "model.layers.74.mlp.experts.139.down_proj", + "model.layers.74.mlp.experts.139.gate_proj", + "model.layers.74.mlp.experts.139.up_proj", + "model.layers.74.mlp.experts.14.down_proj", + "model.layers.74.mlp.experts.14.gate_proj", + "model.layers.74.mlp.experts.14.up_proj", + "model.layers.74.mlp.experts.140.down_proj", + "model.layers.74.mlp.experts.140.gate_proj", + "model.layers.74.mlp.experts.140.up_proj", + "model.layers.74.mlp.experts.141.down_proj", + "model.layers.74.mlp.experts.141.gate_proj", + "model.layers.74.mlp.experts.141.up_proj", + "model.layers.74.mlp.experts.142.down_proj", + "model.layers.74.mlp.experts.142.gate_proj", + "model.layers.74.mlp.experts.142.up_proj", + "model.layers.74.mlp.experts.143.down_proj", + "model.layers.74.mlp.experts.143.gate_proj", + "model.layers.74.mlp.experts.143.up_proj", + "model.layers.74.mlp.experts.144.down_proj", + "model.layers.74.mlp.experts.144.gate_proj", + "model.layers.74.mlp.experts.144.up_proj", + "model.layers.74.mlp.experts.145.down_proj", + "model.layers.74.mlp.experts.145.gate_proj", + "model.layers.74.mlp.experts.145.up_proj", + "model.layers.74.mlp.experts.146.down_proj", + "model.layers.74.mlp.experts.146.gate_proj", + "model.layers.74.mlp.experts.146.up_proj", + "model.layers.74.mlp.experts.147.down_proj", + "model.layers.74.mlp.experts.147.gate_proj", + "model.layers.74.mlp.experts.147.up_proj", + "model.layers.74.mlp.experts.148.down_proj", + "model.layers.74.mlp.experts.148.gate_proj", + "model.layers.74.mlp.experts.148.up_proj", + "model.layers.74.mlp.experts.149.down_proj", + "model.layers.74.mlp.experts.149.gate_proj", + "model.layers.74.mlp.experts.149.up_proj", + "model.layers.74.mlp.experts.15.down_proj", + "model.layers.74.mlp.experts.15.gate_proj", + "model.layers.74.mlp.experts.15.up_proj", + "model.layers.74.mlp.experts.150.down_proj", + "model.layers.74.mlp.experts.150.gate_proj", + "model.layers.74.mlp.experts.150.up_proj", + "model.layers.74.mlp.experts.151.down_proj", + "model.layers.74.mlp.experts.151.gate_proj", + "model.layers.74.mlp.experts.151.up_proj", + "model.layers.74.mlp.experts.152.down_proj", + "model.layers.74.mlp.experts.152.gate_proj", + "model.layers.74.mlp.experts.152.up_proj", + "model.layers.74.mlp.experts.153.down_proj", + "model.layers.74.mlp.experts.153.gate_proj", + "model.layers.74.mlp.experts.153.up_proj", + "model.layers.74.mlp.experts.154.down_proj", + "model.layers.74.mlp.experts.154.gate_proj", + "model.layers.74.mlp.experts.154.up_proj", + "model.layers.74.mlp.experts.155.down_proj", + "model.layers.74.mlp.experts.155.gate_proj", + "model.layers.74.mlp.experts.155.up_proj", + "model.layers.74.mlp.experts.156.down_proj", + "model.layers.74.mlp.experts.156.gate_proj", + "model.layers.74.mlp.experts.156.up_proj", + "model.layers.74.mlp.experts.157.down_proj", + "model.layers.74.mlp.experts.157.gate_proj", + "model.layers.74.mlp.experts.157.up_proj", + "model.layers.74.mlp.experts.158.down_proj", + "model.layers.74.mlp.experts.158.gate_proj", + "model.layers.74.mlp.experts.158.up_proj", + "model.layers.74.mlp.experts.159.down_proj", + "model.layers.74.mlp.experts.159.gate_proj", + "model.layers.74.mlp.experts.159.up_proj", + "model.layers.74.mlp.experts.16.down_proj", + "model.layers.74.mlp.experts.16.gate_proj", + "model.layers.74.mlp.experts.16.up_proj", + "model.layers.74.mlp.experts.160.down_proj", + "model.layers.74.mlp.experts.160.gate_proj", + "model.layers.74.mlp.experts.160.up_proj", + "model.layers.74.mlp.experts.161.down_proj", + "model.layers.74.mlp.experts.161.gate_proj", + "model.layers.74.mlp.experts.161.up_proj", + "model.layers.74.mlp.experts.162.down_proj", + "model.layers.74.mlp.experts.162.gate_proj", + "model.layers.74.mlp.experts.162.up_proj", + "model.layers.74.mlp.experts.163.down_proj", + "model.layers.74.mlp.experts.163.gate_proj", + "model.layers.74.mlp.experts.163.up_proj", + "model.layers.74.mlp.experts.164.down_proj", + "model.layers.74.mlp.experts.164.gate_proj", + "model.layers.74.mlp.experts.164.up_proj", + "model.layers.74.mlp.experts.165.down_proj", + "model.layers.74.mlp.experts.165.gate_proj", + "model.layers.74.mlp.experts.165.up_proj", + "model.layers.74.mlp.experts.166.down_proj", + "model.layers.74.mlp.experts.166.gate_proj", + "model.layers.74.mlp.experts.166.up_proj", + "model.layers.74.mlp.experts.167.down_proj", + "model.layers.74.mlp.experts.167.gate_proj", + "model.layers.74.mlp.experts.167.up_proj", + "model.layers.74.mlp.experts.168.down_proj", + "model.layers.74.mlp.experts.168.gate_proj", + "model.layers.74.mlp.experts.168.up_proj", + "model.layers.74.mlp.experts.169.down_proj", + "model.layers.74.mlp.experts.169.gate_proj", + "model.layers.74.mlp.experts.169.up_proj", + "model.layers.74.mlp.experts.17.down_proj", + "model.layers.74.mlp.experts.17.gate_proj", + "model.layers.74.mlp.experts.17.up_proj", + "model.layers.74.mlp.experts.170.down_proj", + "model.layers.74.mlp.experts.170.gate_proj", + "model.layers.74.mlp.experts.170.up_proj", + "model.layers.74.mlp.experts.171.down_proj", + "model.layers.74.mlp.experts.171.gate_proj", + "model.layers.74.mlp.experts.171.up_proj", + "model.layers.74.mlp.experts.172.down_proj", + "model.layers.74.mlp.experts.172.gate_proj", + "model.layers.74.mlp.experts.172.up_proj", + "model.layers.74.mlp.experts.173.down_proj", + "model.layers.74.mlp.experts.173.gate_proj", + "model.layers.74.mlp.experts.173.up_proj", + "model.layers.74.mlp.experts.174.down_proj", + "model.layers.74.mlp.experts.174.gate_proj", + "model.layers.74.mlp.experts.174.up_proj", + "model.layers.74.mlp.experts.175.down_proj", + "model.layers.74.mlp.experts.175.gate_proj", + "model.layers.74.mlp.experts.175.up_proj", + "model.layers.74.mlp.experts.176.down_proj", + "model.layers.74.mlp.experts.176.gate_proj", + "model.layers.74.mlp.experts.176.up_proj", + "model.layers.74.mlp.experts.177.down_proj", + "model.layers.74.mlp.experts.177.gate_proj", + "model.layers.74.mlp.experts.177.up_proj", + "model.layers.74.mlp.experts.178.down_proj", + "model.layers.74.mlp.experts.178.gate_proj", + "model.layers.74.mlp.experts.178.up_proj", + "model.layers.74.mlp.experts.179.down_proj", + "model.layers.74.mlp.experts.179.gate_proj", + "model.layers.74.mlp.experts.179.up_proj", + "model.layers.74.mlp.experts.18.down_proj", + "model.layers.74.mlp.experts.18.gate_proj", + "model.layers.74.mlp.experts.18.up_proj", + "model.layers.74.mlp.experts.180.down_proj", + "model.layers.74.mlp.experts.180.gate_proj", + "model.layers.74.mlp.experts.180.up_proj", + "model.layers.74.mlp.experts.181.down_proj", + "model.layers.74.mlp.experts.181.gate_proj", + "model.layers.74.mlp.experts.181.up_proj", + "model.layers.74.mlp.experts.182.down_proj", + "model.layers.74.mlp.experts.182.gate_proj", + "model.layers.74.mlp.experts.182.up_proj", + "model.layers.74.mlp.experts.183.down_proj", + "model.layers.74.mlp.experts.183.gate_proj", + "model.layers.74.mlp.experts.183.up_proj", + "model.layers.74.mlp.experts.184.down_proj", + "model.layers.74.mlp.experts.184.gate_proj", + "model.layers.74.mlp.experts.184.up_proj", + "model.layers.74.mlp.experts.185.down_proj", + "model.layers.74.mlp.experts.185.gate_proj", + "model.layers.74.mlp.experts.185.up_proj", + "model.layers.74.mlp.experts.186.down_proj", + "model.layers.74.mlp.experts.186.gate_proj", + "model.layers.74.mlp.experts.186.up_proj", + "model.layers.74.mlp.experts.187.down_proj", + "model.layers.74.mlp.experts.187.gate_proj", + "model.layers.74.mlp.experts.187.up_proj", + "model.layers.74.mlp.experts.188.down_proj", + "model.layers.74.mlp.experts.188.gate_proj", + "model.layers.74.mlp.experts.188.up_proj", + "model.layers.74.mlp.experts.189.down_proj", + "model.layers.74.mlp.experts.189.gate_proj", + "model.layers.74.mlp.experts.189.up_proj", + "model.layers.74.mlp.experts.19.down_proj", + "model.layers.74.mlp.experts.19.gate_proj", + "model.layers.74.mlp.experts.19.up_proj", + "model.layers.74.mlp.experts.190.down_proj", + "model.layers.74.mlp.experts.190.gate_proj", + "model.layers.74.mlp.experts.190.up_proj", + "model.layers.74.mlp.experts.191.down_proj", + "model.layers.74.mlp.experts.191.gate_proj", + "model.layers.74.mlp.experts.191.up_proj", + "model.layers.74.mlp.experts.192.down_proj", + "model.layers.74.mlp.experts.192.gate_proj", + "model.layers.74.mlp.experts.192.up_proj", + "model.layers.74.mlp.experts.193.down_proj", + "model.layers.74.mlp.experts.193.gate_proj", + "model.layers.74.mlp.experts.193.up_proj", + "model.layers.74.mlp.experts.194.down_proj", + "model.layers.74.mlp.experts.194.gate_proj", + "model.layers.74.mlp.experts.194.up_proj", + "model.layers.74.mlp.experts.195.down_proj", + "model.layers.74.mlp.experts.195.gate_proj", + "model.layers.74.mlp.experts.195.up_proj", + "model.layers.74.mlp.experts.196.down_proj", + "model.layers.74.mlp.experts.196.gate_proj", + "model.layers.74.mlp.experts.196.up_proj", + "model.layers.74.mlp.experts.197.down_proj", + "model.layers.74.mlp.experts.197.gate_proj", + "model.layers.74.mlp.experts.197.up_proj", + "model.layers.74.mlp.experts.198.down_proj", + "model.layers.74.mlp.experts.198.gate_proj", + "model.layers.74.mlp.experts.198.up_proj", + "model.layers.74.mlp.experts.199.down_proj", + "model.layers.74.mlp.experts.199.gate_proj", + "model.layers.74.mlp.experts.199.up_proj", + "model.layers.74.mlp.experts.2.down_proj", + "model.layers.74.mlp.experts.2.gate_proj", + "model.layers.74.mlp.experts.2.up_proj", + "model.layers.74.mlp.experts.20.down_proj", + "model.layers.74.mlp.experts.20.gate_proj", + "model.layers.74.mlp.experts.20.up_proj", + "model.layers.74.mlp.experts.200.down_proj", + "model.layers.74.mlp.experts.200.gate_proj", + "model.layers.74.mlp.experts.200.up_proj", + "model.layers.74.mlp.experts.201.down_proj", + "model.layers.74.mlp.experts.201.gate_proj", + "model.layers.74.mlp.experts.201.up_proj", + "model.layers.74.mlp.experts.202.down_proj", + "model.layers.74.mlp.experts.202.gate_proj", + "model.layers.74.mlp.experts.202.up_proj", + "model.layers.74.mlp.experts.203.down_proj", + "model.layers.74.mlp.experts.203.gate_proj", + "model.layers.74.mlp.experts.203.up_proj", + "model.layers.74.mlp.experts.204.down_proj", + "model.layers.74.mlp.experts.204.gate_proj", + "model.layers.74.mlp.experts.204.up_proj", + "model.layers.74.mlp.experts.205.down_proj", + "model.layers.74.mlp.experts.205.gate_proj", + "model.layers.74.mlp.experts.205.up_proj", + "model.layers.74.mlp.experts.206.down_proj", + "model.layers.74.mlp.experts.206.gate_proj", + "model.layers.74.mlp.experts.206.up_proj", + "model.layers.74.mlp.experts.207.down_proj", + "model.layers.74.mlp.experts.207.gate_proj", + "model.layers.74.mlp.experts.207.up_proj", + "model.layers.74.mlp.experts.208.down_proj", + "model.layers.74.mlp.experts.208.gate_proj", + "model.layers.74.mlp.experts.208.up_proj", + "model.layers.74.mlp.experts.209.down_proj", + "model.layers.74.mlp.experts.209.gate_proj", + "model.layers.74.mlp.experts.209.up_proj", + "model.layers.74.mlp.experts.21.down_proj", + "model.layers.74.mlp.experts.21.gate_proj", + "model.layers.74.mlp.experts.21.up_proj", + "model.layers.74.mlp.experts.210.down_proj", + "model.layers.74.mlp.experts.210.gate_proj", + "model.layers.74.mlp.experts.210.up_proj", + "model.layers.74.mlp.experts.211.down_proj", + "model.layers.74.mlp.experts.211.gate_proj", + "model.layers.74.mlp.experts.211.up_proj", + "model.layers.74.mlp.experts.212.down_proj", + "model.layers.74.mlp.experts.212.gate_proj", + "model.layers.74.mlp.experts.212.up_proj", + "model.layers.74.mlp.experts.213.down_proj", + "model.layers.74.mlp.experts.213.gate_proj", + "model.layers.74.mlp.experts.213.up_proj", + "model.layers.74.mlp.experts.214.down_proj", + "model.layers.74.mlp.experts.214.gate_proj", + "model.layers.74.mlp.experts.214.up_proj", + "model.layers.74.mlp.experts.215.down_proj", + "model.layers.74.mlp.experts.215.gate_proj", + "model.layers.74.mlp.experts.215.up_proj", + "model.layers.74.mlp.experts.216.down_proj", + "model.layers.74.mlp.experts.216.gate_proj", + "model.layers.74.mlp.experts.216.up_proj", + "model.layers.74.mlp.experts.217.down_proj", + "model.layers.74.mlp.experts.217.gate_proj", + "model.layers.74.mlp.experts.217.up_proj", + "model.layers.74.mlp.experts.218.down_proj", + "model.layers.74.mlp.experts.218.gate_proj", + "model.layers.74.mlp.experts.218.up_proj", + "model.layers.74.mlp.experts.219.down_proj", + "model.layers.74.mlp.experts.219.gate_proj", + "model.layers.74.mlp.experts.219.up_proj", + "model.layers.74.mlp.experts.22.down_proj", + "model.layers.74.mlp.experts.22.gate_proj", + "model.layers.74.mlp.experts.22.up_proj", + "model.layers.74.mlp.experts.220.down_proj", + "model.layers.74.mlp.experts.220.gate_proj", + "model.layers.74.mlp.experts.220.up_proj", + "model.layers.74.mlp.experts.221.down_proj", + "model.layers.74.mlp.experts.221.gate_proj", + "model.layers.74.mlp.experts.221.up_proj", + "model.layers.74.mlp.experts.222.down_proj", + "model.layers.74.mlp.experts.222.gate_proj", + "model.layers.74.mlp.experts.222.up_proj", + "model.layers.74.mlp.experts.223.down_proj", + "model.layers.74.mlp.experts.223.gate_proj", + "model.layers.74.mlp.experts.223.up_proj", + "model.layers.74.mlp.experts.224.down_proj", + "model.layers.74.mlp.experts.224.gate_proj", + "model.layers.74.mlp.experts.224.up_proj", + "model.layers.74.mlp.experts.225.down_proj", + "model.layers.74.mlp.experts.225.gate_proj", + "model.layers.74.mlp.experts.225.up_proj", + "model.layers.74.mlp.experts.226.down_proj", + "model.layers.74.mlp.experts.226.gate_proj", + "model.layers.74.mlp.experts.226.up_proj", + "model.layers.74.mlp.experts.227.down_proj", + "model.layers.74.mlp.experts.227.gate_proj", + "model.layers.74.mlp.experts.227.up_proj", + "model.layers.74.mlp.experts.228.down_proj", + "model.layers.74.mlp.experts.228.gate_proj", + "model.layers.74.mlp.experts.228.up_proj", + "model.layers.74.mlp.experts.229.down_proj", + "model.layers.74.mlp.experts.229.gate_proj", + "model.layers.74.mlp.experts.229.up_proj", + "model.layers.74.mlp.experts.23.down_proj", + "model.layers.74.mlp.experts.23.gate_proj", + "model.layers.74.mlp.experts.23.up_proj", + "model.layers.74.mlp.experts.230.down_proj", + "model.layers.74.mlp.experts.230.gate_proj", + "model.layers.74.mlp.experts.230.up_proj", + "model.layers.74.mlp.experts.231.down_proj", + "model.layers.74.mlp.experts.231.gate_proj", + "model.layers.74.mlp.experts.231.up_proj", + "model.layers.74.mlp.experts.232.down_proj", + "model.layers.74.mlp.experts.232.gate_proj", + "model.layers.74.mlp.experts.232.up_proj", + "model.layers.74.mlp.experts.233.down_proj", + "model.layers.74.mlp.experts.233.gate_proj", + "model.layers.74.mlp.experts.233.up_proj", + "model.layers.74.mlp.experts.234.down_proj", + "model.layers.74.mlp.experts.234.gate_proj", + "model.layers.74.mlp.experts.234.up_proj", + "model.layers.74.mlp.experts.235.down_proj", + "model.layers.74.mlp.experts.235.gate_proj", + "model.layers.74.mlp.experts.235.up_proj", + "model.layers.74.mlp.experts.236.down_proj", + "model.layers.74.mlp.experts.236.gate_proj", + "model.layers.74.mlp.experts.236.up_proj", + "model.layers.74.mlp.experts.237.down_proj", + "model.layers.74.mlp.experts.237.gate_proj", + "model.layers.74.mlp.experts.237.up_proj", + "model.layers.74.mlp.experts.238.down_proj", + "model.layers.74.mlp.experts.238.gate_proj", + "model.layers.74.mlp.experts.238.up_proj", + "model.layers.74.mlp.experts.239.down_proj", + "model.layers.74.mlp.experts.239.gate_proj", + "model.layers.74.mlp.experts.239.up_proj", + "model.layers.74.mlp.experts.24.down_proj", + "model.layers.74.mlp.experts.24.gate_proj", + "model.layers.74.mlp.experts.24.up_proj", + "model.layers.74.mlp.experts.240.down_proj", + "model.layers.74.mlp.experts.240.gate_proj", + "model.layers.74.mlp.experts.240.up_proj", + "model.layers.74.mlp.experts.241.down_proj", + "model.layers.74.mlp.experts.241.gate_proj", + "model.layers.74.mlp.experts.241.up_proj", + "model.layers.74.mlp.experts.242.down_proj", + "model.layers.74.mlp.experts.242.gate_proj", + "model.layers.74.mlp.experts.242.up_proj", + "model.layers.74.mlp.experts.243.down_proj", + "model.layers.74.mlp.experts.243.gate_proj", + "model.layers.74.mlp.experts.243.up_proj", + "model.layers.74.mlp.experts.244.down_proj", + "model.layers.74.mlp.experts.244.gate_proj", + "model.layers.74.mlp.experts.244.up_proj", + "model.layers.74.mlp.experts.245.down_proj", + "model.layers.74.mlp.experts.245.gate_proj", + "model.layers.74.mlp.experts.245.up_proj", + "model.layers.74.mlp.experts.246.down_proj", + "model.layers.74.mlp.experts.246.gate_proj", + "model.layers.74.mlp.experts.246.up_proj", + "model.layers.74.mlp.experts.247.down_proj", + "model.layers.74.mlp.experts.247.gate_proj", + "model.layers.74.mlp.experts.247.up_proj", + "model.layers.74.mlp.experts.248.down_proj", + "model.layers.74.mlp.experts.248.gate_proj", + "model.layers.74.mlp.experts.248.up_proj", + "model.layers.74.mlp.experts.249.down_proj", + "model.layers.74.mlp.experts.249.gate_proj", + "model.layers.74.mlp.experts.249.up_proj", + "model.layers.74.mlp.experts.25.down_proj", + "model.layers.74.mlp.experts.25.gate_proj", + "model.layers.74.mlp.experts.25.up_proj", + "model.layers.74.mlp.experts.250.down_proj", + "model.layers.74.mlp.experts.250.gate_proj", + "model.layers.74.mlp.experts.250.up_proj", + "model.layers.74.mlp.experts.251.down_proj", + "model.layers.74.mlp.experts.251.gate_proj", + "model.layers.74.mlp.experts.251.up_proj", + "model.layers.74.mlp.experts.252.down_proj", + "model.layers.74.mlp.experts.252.gate_proj", + "model.layers.74.mlp.experts.252.up_proj", + "model.layers.74.mlp.experts.253.down_proj", + "model.layers.74.mlp.experts.253.gate_proj", + "model.layers.74.mlp.experts.253.up_proj", + "model.layers.74.mlp.experts.254.down_proj", + "model.layers.74.mlp.experts.254.gate_proj", + "model.layers.74.mlp.experts.254.up_proj", + "model.layers.74.mlp.experts.255.down_proj", + "model.layers.74.mlp.experts.255.gate_proj", + "model.layers.74.mlp.experts.255.up_proj", + "model.layers.74.mlp.experts.26.down_proj", + "model.layers.74.mlp.experts.26.gate_proj", + "model.layers.74.mlp.experts.26.up_proj", + "model.layers.74.mlp.experts.27.down_proj", + "model.layers.74.mlp.experts.27.gate_proj", + "model.layers.74.mlp.experts.27.up_proj", + "model.layers.74.mlp.experts.28.down_proj", + "model.layers.74.mlp.experts.28.gate_proj", + "model.layers.74.mlp.experts.28.up_proj", + "model.layers.74.mlp.experts.29.down_proj", + "model.layers.74.mlp.experts.29.gate_proj", + "model.layers.74.mlp.experts.29.up_proj", + "model.layers.74.mlp.experts.3.down_proj", + "model.layers.74.mlp.experts.3.gate_proj", + "model.layers.74.mlp.experts.3.up_proj", + "model.layers.74.mlp.experts.30.down_proj", + "model.layers.74.mlp.experts.30.gate_proj", + "model.layers.74.mlp.experts.30.up_proj", + "model.layers.74.mlp.experts.31.down_proj", + "model.layers.74.mlp.experts.31.gate_proj", + "model.layers.74.mlp.experts.31.up_proj", + "model.layers.74.mlp.experts.32.down_proj", + "model.layers.74.mlp.experts.32.gate_proj", + "model.layers.74.mlp.experts.32.up_proj", + "model.layers.74.mlp.experts.33.down_proj", + "model.layers.74.mlp.experts.33.gate_proj", + "model.layers.74.mlp.experts.33.up_proj", + "model.layers.74.mlp.experts.34.down_proj", + "model.layers.74.mlp.experts.34.gate_proj", + "model.layers.74.mlp.experts.34.up_proj", + "model.layers.74.mlp.experts.35.down_proj", + "model.layers.74.mlp.experts.35.gate_proj", + "model.layers.74.mlp.experts.35.up_proj", + "model.layers.74.mlp.experts.36.down_proj", + "model.layers.74.mlp.experts.36.gate_proj", + "model.layers.74.mlp.experts.36.up_proj", + "model.layers.74.mlp.experts.37.down_proj", + "model.layers.74.mlp.experts.37.gate_proj", + "model.layers.74.mlp.experts.37.up_proj", + "model.layers.74.mlp.experts.38.down_proj", + "model.layers.74.mlp.experts.38.gate_proj", + "model.layers.74.mlp.experts.38.up_proj", + "model.layers.74.mlp.experts.39.down_proj", + "model.layers.74.mlp.experts.39.gate_proj", + "model.layers.74.mlp.experts.39.up_proj", + "model.layers.74.mlp.experts.4.down_proj", + "model.layers.74.mlp.experts.4.gate_proj", + "model.layers.74.mlp.experts.4.up_proj", + "model.layers.74.mlp.experts.40.down_proj", + "model.layers.74.mlp.experts.40.gate_proj", + "model.layers.74.mlp.experts.40.up_proj", + "model.layers.74.mlp.experts.41.down_proj", + "model.layers.74.mlp.experts.41.gate_proj", + "model.layers.74.mlp.experts.41.up_proj", + "model.layers.74.mlp.experts.42.down_proj", + "model.layers.74.mlp.experts.42.gate_proj", + "model.layers.74.mlp.experts.42.up_proj", + "model.layers.74.mlp.experts.43.down_proj", + "model.layers.74.mlp.experts.43.gate_proj", + "model.layers.74.mlp.experts.43.up_proj", + "model.layers.74.mlp.experts.44.down_proj", + "model.layers.74.mlp.experts.44.gate_proj", + "model.layers.74.mlp.experts.44.up_proj", + "model.layers.74.mlp.experts.45.down_proj", + "model.layers.74.mlp.experts.45.gate_proj", + "model.layers.74.mlp.experts.45.up_proj", + "model.layers.74.mlp.experts.46.down_proj", + "model.layers.74.mlp.experts.46.gate_proj", + "model.layers.74.mlp.experts.46.up_proj", + "model.layers.74.mlp.experts.47.down_proj", + "model.layers.74.mlp.experts.47.gate_proj", + "model.layers.74.mlp.experts.47.up_proj", + "model.layers.74.mlp.experts.48.down_proj", + "model.layers.74.mlp.experts.48.gate_proj", + "model.layers.74.mlp.experts.48.up_proj", + "model.layers.74.mlp.experts.49.down_proj", + "model.layers.74.mlp.experts.49.gate_proj", + "model.layers.74.mlp.experts.49.up_proj", + "model.layers.74.mlp.experts.5.down_proj", + "model.layers.74.mlp.experts.5.gate_proj", + "model.layers.74.mlp.experts.5.up_proj", + "model.layers.74.mlp.experts.50.down_proj", + "model.layers.74.mlp.experts.50.gate_proj", + "model.layers.74.mlp.experts.50.up_proj", + "model.layers.74.mlp.experts.51.down_proj", + "model.layers.74.mlp.experts.51.gate_proj", + "model.layers.74.mlp.experts.51.up_proj", + "model.layers.74.mlp.experts.52.down_proj", + "model.layers.74.mlp.experts.52.gate_proj", + "model.layers.74.mlp.experts.52.up_proj", + "model.layers.74.mlp.experts.53.down_proj", + "model.layers.74.mlp.experts.53.gate_proj", + "model.layers.74.mlp.experts.53.up_proj", + "model.layers.74.mlp.experts.54.down_proj", + "model.layers.74.mlp.experts.54.gate_proj", + "model.layers.74.mlp.experts.54.up_proj", + "model.layers.74.mlp.experts.55.down_proj", + "model.layers.74.mlp.experts.55.gate_proj", + "model.layers.74.mlp.experts.55.up_proj", + "model.layers.74.mlp.experts.56.down_proj", + "model.layers.74.mlp.experts.56.gate_proj", + "model.layers.74.mlp.experts.56.up_proj", + "model.layers.74.mlp.experts.57.down_proj", + "model.layers.74.mlp.experts.57.gate_proj", + "model.layers.74.mlp.experts.57.up_proj", + "model.layers.74.mlp.experts.58.down_proj", + "model.layers.74.mlp.experts.58.gate_proj", + "model.layers.74.mlp.experts.58.up_proj", + "model.layers.74.mlp.experts.59.down_proj", + "model.layers.74.mlp.experts.59.gate_proj", + "model.layers.74.mlp.experts.59.up_proj", + "model.layers.74.mlp.experts.6.down_proj", + "model.layers.74.mlp.experts.6.gate_proj", + "model.layers.74.mlp.experts.6.up_proj", + "model.layers.74.mlp.experts.60.down_proj", + "model.layers.74.mlp.experts.60.gate_proj", + "model.layers.74.mlp.experts.60.up_proj", + "model.layers.74.mlp.experts.61.down_proj", + "model.layers.74.mlp.experts.61.gate_proj", + "model.layers.74.mlp.experts.61.up_proj", + "model.layers.74.mlp.experts.62.down_proj", + "model.layers.74.mlp.experts.62.gate_proj", + "model.layers.74.mlp.experts.62.up_proj", + "model.layers.74.mlp.experts.63.down_proj", + "model.layers.74.mlp.experts.63.gate_proj", + "model.layers.74.mlp.experts.63.up_proj", + "model.layers.74.mlp.experts.64.down_proj", + "model.layers.74.mlp.experts.64.gate_proj", + "model.layers.74.mlp.experts.64.up_proj", + "model.layers.74.mlp.experts.65.down_proj", + "model.layers.74.mlp.experts.65.gate_proj", + "model.layers.74.mlp.experts.65.up_proj", + "model.layers.74.mlp.experts.66.down_proj", + "model.layers.74.mlp.experts.66.gate_proj", + "model.layers.74.mlp.experts.66.up_proj", + "model.layers.74.mlp.experts.67.down_proj", + "model.layers.74.mlp.experts.67.gate_proj", + "model.layers.74.mlp.experts.67.up_proj", + "model.layers.74.mlp.experts.68.down_proj", + "model.layers.74.mlp.experts.68.gate_proj", + "model.layers.74.mlp.experts.68.up_proj", + "model.layers.74.mlp.experts.69.down_proj", + "model.layers.74.mlp.experts.69.gate_proj", + "model.layers.74.mlp.experts.69.up_proj", + "model.layers.74.mlp.experts.7.down_proj", + "model.layers.74.mlp.experts.7.gate_proj", + "model.layers.74.mlp.experts.7.up_proj", + "model.layers.74.mlp.experts.70.down_proj", + "model.layers.74.mlp.experts.70.gate_proj", + "model.layers.74.mlp.experts.70.up_proj", + "model.layers.74.mlp.experts.71.down_proj", + "model.layers.74.mlp.experts.71.gate_proj", + "model.layers.74.mlp.experts.71.up_proj", + "model.layers.74.mlp.experts.72.down_proj", + "model.layers.74.mlp.experts.72.gate_proj", + "model.layers.74.mlp.experts.72.up_proj", + "model.layers.74.mlp.experts.73.down_proj", + "model.layers.74.mlp.experts.73.gate_proj", + "model.layers.74.mlp.experts.73.up_proj", + "model.layers.74.mlp.experts.74.down_proj", + "model.layers.74.mlp.experts.74.gate_proj", + "model.layers.74.mlp.experts.74.up_proj", + "model.layers.74.mlp.experts.75.down_proj", + "model.layers.74.mlp.experts.75.gate_proj", + "model.layers.74.mlp.experts.75.up_proj", + "model.layers.74.mlp.experts.76.down_proj", + "model.layers.74.mlp.experts.76.gate_proj", + "model.layers.74.mlp.experts.76.up_proj", + "model.layers.74.mlp.experts.77.down_proj", + "model.layers.74.mlp.experts.77.gate_proj", + "model.layers.74.mlp.experts.77.up_proj", + "model.layers.74.mlp.experts.78.down_proj", + "model.layers.74.mlp.experts.78.gate_proj", + "model.layers.74.mlp.experts.78.up_proj", + "model.layers.74.mlp.experts.79.down_proj", + "model.layers.74.mlp.experts.79.gate_proj", + "model.layers.74.mlp.experts.79.up_proj", + "model.layers.74.mlp.experts.8.down_proj", + "model.layers.74.mlp.experts.8.gate_proj", + "model.layers.74.mlp.experts.8.up_proj", + "model.layers.74.mlp.experts.80.down_proj", + "model.layers.74.mlp.experts.80.gate_proj", + "model.layers.74.mlp.experts.80.up_proj", + "model.layers.74.mlp.experts.81.down_proj", + "model.layers.74.mlp.experts.81.gate_proj", + "model.layers.74.mlp.experts.81.up_proj", + "model.layers.74.mlp.experts.82.down_proj", + "model.layers.74.mlp.experts.82.gate_proj", + "model.layers.74.mlp.experts.82.up_proj", + "model.layers.74.mlp.experts.83.down_proj", + "model.layers.74.mlp.experts.83.gate_proj", + "model.layers.74.mlp.experts.83.up_proj", + "model.layers.74.mlp.experts.84.down_proj", + "model.layers.74.mlp.experts.84.gate_proj", + "model.layers.74.mlp.experts.84.up_proj", + "model.layers.74.mlp.experts.85.down_proj", + "model.layers.74.mlp.experts.85.gate_proj", + "model.layers.74.mlp.experts.85.up_proj", + "model.layers.74.mlp.experts.86.down_proj", + "model.layers.74.mlp.experts.86.gate_proj", + "model.layers.74.mlp.experts.86.up_proj", + "model.layers.74.mlp.experts.87.down_proj", + "model.layers.74.mlp.experts.87.gate_proj", + "model.layers.74.mlp.experts.87.up_proj", + "model.layers.74.mlp.experts.88.down_proj", + "model.layers.74.mlp.experts.88.gate_proj", + "model.layers.74.mlp.experts.88.up_proj", + "model.layers.74.mlp.experts.89.down_proj", + "model.layers.74.mlp.experts.89.gate_proj", + "model.layers.74.mlp.experts.89.up_proj", + "model.layers.74.mlp.experts.9.down_proj", + "model.layers.74.mlp.experts.9.gate_proj", + "model.layers.74.mlp.experts.9.up_proj", + "model.layers.74.mlp.experts.90.down_proj", + "model.layers.74.mlp.experts.90.gate_proj", + "model.layers.74.mlp.experts.90.up_proj", + "model.layers.74.mlp.experts.91.down_proj", + "model.layers.74.mlp.experts.91.gate_proj", + "model.layers.74.mlp.experts.91.up_proj", + "model.layers.74.mlp.experts.92.down_proj", + "model.layers.74.mlp.experts.92.gate_proj", + "model.layers.74.mlp.experts.92.up_proj", + "model.layers.74.mlp.experts.93.down_proj", + "model.layers.74.mlp.experts.93.gate_proj", + "model.layers.74.mlp.experts.93.up_proj", + "model.layers.74.mlp.experts.94.down_proj", + "model.layers.74.mlp.experts.94.gate_proj", + "model.layers.74.mlp.experts.94.up_proj", + "model.layers.74.mlp.experts.95.down_proj", + "model.layers.74.mlp.experts.95.gate_proj", + "model.layers.74.mlp.experts.95.up_proj", + "model.layers.74.mlp.experts.96.down_proj", + "model.layers.74.mlp.experts.96.gate_proj", + "model.layers.74.mlp.experts.96.up_proj", + "model.layers.74.mlp.experts.97.down_proj", + "model.layers.74.mlp.experts.97.gate_proj", + "model.layers.74.mlp.experts.97.up_proj", + "model.layers.74.mlp.experts.98.down_proj", + "model.layers.74.mlp.experts.98.gate_proj", + "model.layers.74.mlp.experts.98.up_proj", + "model.layers.74.mlp.experts.99.down_proj", + "model.layers.74.mlp.experts.99.gate_proj", + "model.layers.74.mlp.experts.99.up_proj", + "model.layers.75.mlp.experts.0.down_proj", + "model.layers.75.mlp.experts.0.gate_proj", + "model.layers.75.mlp.experts.0.up_proj", + "model.layers.75.mlp.experts.1.down_proj", + "model.layers.75.mlp.experts.1.gate_proj", + "model.layers.75.mlp.experts.1.up_proj", + "model.layers.75.mlp.experts.10.down_proj", + "model.layers.75.mlp.experts.10.gate_proj", + "model.layers.75.mlp.experts.10.up_proj", + "model.layers.75.mlp.experts.100.down_proj", + "model.layers.75.mlp.experts.100.gate_proj", + "model.layers.75.mlp.experts.100.up_proj", + "model.layers.75.mlp.experts.101.down_proj", + "model.layers.75.mlp.experts.101.gate_proj", + "model.layers.75.mlp.experts.101.up_proj", + "model.layers.75.mlp.experts.102.down_proj", + "model.layers.75.mlp.experts.102.gate_proj", + "model.layers.75.mlp.experts.102.up_proj", + "model.layers.75.mlp.experts.103.down_proj", + "model.layers.75.mlp.experts.103.gate_proj", + "model.layers.75.mlp.experts.103.up_proj", + "model.layers.75.mlp.experts.104.down_proj", + "model.layers.75.mlp.experts.104.gate_proj", + "model.layers.75.mlp.experts.104.up_proj", + "model.layers.75.mlp.experts.105.down_proj", + "model.layers.75.mlp.experts.105.gate_proj", + "model.layers.75.mlp.experts.105.up_proj", + "model.layers.75.mlp.experts.106.down_proj", + "model.layers.75.mlp.experts.106.gate_proj", + "model.layers.75.mlp.experts.106.up_proj", + "model.layers.75.mlp.experts.107.down_proj", + "model.layers.75.mlp.experts.107.gate_proj", + "model.layers.75.mlp.experts.107.up_proj", + "model.layers.75.mlp.experts.108.down_proj", + "model.layers.75.mlp.experts.108.gate_proj", + "model.layers.75.mlp.experts.108.up_proj", + "model.layers.75.mlp.experts.109.down_proj", + "model.layers.75.mlp.experts.109.gate_proj", + "model.layers.75.mlp.experts.109.up_proj", + "model.layers.75.mlp.experts.11.down_proj", + "model.layers.75.mlp.experts.11.gate_proj", + "model.layers.75.mlp.experts.11.up_proj", + "model.layers.75.mlp.experts.110.down_proj", + "model.layers.75.mlp.experts.110.gate_proj", + "model.layers.75.mlp.experts.110.up_proj", + "model.layers.75.mlp.experts.111.down_proj", + "model.layers.75.mlp.experts.111.gate_proj", + "model.layers.75.mlp.experts.111.up_proj", + "model.layers.75.mlp.experts.112.down_proj", + "model.layers.75.mlp.experts.112.gate_proj", + "model.layers.75.mlp.experts.112.up_proj", + "model.layers.75.mlp.experts.113.down_proj", + "model.layers.75.mlp.experts.113.gate_proj", + "model.layers.75.mlp.experts.113.up_proj", + "model.layers.75.mlp.experts.114.down_proj", + "model.layers.75.mlp.experts.114.gate_proj", + "model.layers.75.mlp.experts.114.up_proj", + "model.layers.75.mlp.experts.115.down_proj", + "model.layers.75.mlp.experts.115.gate_proj", + "model.layers.75.mlp.experts.115.up_proj", + "model.layers.75.mlp.experts.116.down_proj", + "model.layers.75.mlp.experts.116.gate_proj", + "model.layers.75.mlp.experts.116.up_proj", + "model.layers.75.mlp.experts.117.down_proj", + "model.layers.75.mlp.experts.117.gate_proj", + "model.layers.75.mlp.experts.117.up_proj", + "model.layers.75.mlp.experts.118.down_proj", + "model.layers.75.mlp.experts.118.gate_proj", + "model.layers.75.mlp.experts.118.up_proj", + "model.layers.75.mlp.experts.119.down_proj", + "model.layers.75.mlp.experts.119.gate_proj", + "model.layers.75.mlp.experts.119.up_proj", + "model.layers.75.mlp.experts.12.down_proj", + "model.layers.75.mlp.experts.12.gate_proj", + "model.layers.75.mlp.experts.12.up_proj", + "model.layers.75.mlp.experts.120.down_proj", + "model.layers.75.mlp.experts.120.gate_proj", + "model.layers.75.mlp.experts.120.up_proj", + "model.layers.75.mlp.experts.121.down_proj", + "model.layers.75.mlp.experts.121.gate_proj", + "model.layers.75.mlp.experts.121.up_proj", + "model.layers.75.mlp.experts.122.down_proj", + "model.layers.75.mlp.experts.122.gate_proj", + "model.layers.75.mlp.experts.122.up_proj", + "model.layers.75.mlp.experts.123.down_proj", + "model.layers.75.mlp.experts.123.gate_proj", + "model.layers.75.mlp.experts.123.up_proj", + "model.layers.75.mlp.experts.124.down_proj", + "model.layers.75.mlp.experts.124.gate_proj", + "model.layers.75.mlp.experts.124.up_proj", + "model.layers.75.mlp.experts.125.down_proj", + "model.layers.75.mlp.experts.125.gate_proj", + "model.layers.75.mlp.experts.125.up_proj", + "model.layers.75.mlp.experts.126.down_proj", + "model.layers.75.mlp.experts.126.gate_proj", + "model.layers.75.mlp.experts.126.up_proj", + "model.layers.75.mlp.experts.127.down_proj", + "model.layers.75.mlp.experts.127.gate_proj", + "model.layers.75.mlp.experts.127.up_proj", + "model.layers.75.mlp.experts.128.down_proj", + "model.layers.75.mlp.experts.128.gate_proj", + "model.layers.75.mlp.experts.128.up_proj", + "model.layers.75.mlp.experts.129.down_proj", + "model.layers.75.mlp.experts.129.gate_proj", + "model.layers.75.mlp.experts.129.up_proj", + "model.layers.75.mlp.experts.13.down_proj", + "model.layers.75.mlp.experts.13.gate_proj", + "model.layers.75.mlp.experts.13.up_proj", + "model.layers.75.mlp.experts.130.down_proj", + "model.layers.75.mlp.experts.130.gate_proj", + "model.layers.75.mlp.experts.130.up_proj", + "model.layers.75.mlp.experts.131.down_proj", + "model.layers.75.mlp.experts.131.gate_proj", + "model.layers.75.mlp.experts.131.up_proj", + "model.layers.75.mlp.experts.132.down_proj", + "model.layers.75.mlp.experts.132.gate_proj", + "model.layers.75.mlp.experts.132.up_proj", + "model.layers.75.mlp.experts.133.down_proj", + "model.layers.75.mlp.experts.133.gate_proj", + "model.layers.75.mlp.experts.133.up_proj", + "model.layers.75.mlp.experts.134.down_proj", + "model.layers.75.mlp.experts.134.gate_proj", + "model.layers.75.mlp.experts.134.up_proj", + "model.layers.75.mlp.experts.135.down_proj", + "model.layers.75.mlp.experts.135.gate_proj", + "model.layers.75.mlp.experts.135.up_proj", + "model.layers.75.mlp.experts.136.down_proj", + "model.layers.75.mlp.experts.136.gate_proj", + "model.layers.75.mlp.experts.136.up_proj", + "model.layers.75.mlp.experts.137.down_proj", + "model.layers.75.mlp.experts.137.gate_proj", + "model.layers.75.mlp.experts.137.up_proj", + "model.layers.75.mlp.experts.138.down_proj", + "model.layers.75.mlp.experts.138.gate_proj", + "model.layers.75.mlp.experts.138.up_proj", + "model.layers.75.mlp.experts.139.down_proj", + "model.layers.75.mlp.experts.139.gate_proj", + "model.layers.75.mlp.experts.139.up_proj", + "model.layers.75.mlp.experts.14.down_proj", + "model.layers.75.mlp.experts.14.gate_proj", + "model.layers.75.mlp.experts.14.up_proj", + "model.layers.75.mlp.experts.140.down_proj", + "model.layers.75.mlp.experts.140.gate_proj", + "model.layers.75.mlp.experts.140.up_proj", + "model.layers.75.mlp.experts.141.down_proj", + "model.layers.75.mlp.experts.141.gate_proj", + "model.layers.75.mlp.experts.141.up_proj", + "model.layers.75.mlp.experts.142.down_proj", + "model.layers.75.mlp.experts.142.gate_proj", + "model.layers.75.mlp.experts.142.up_proj", + "model.layers.75.mlp.experts.143.down_proj", + "model.layers.75.mlp.experts.143.gate_proj", + "model.layers.75.mlp.experts.143.up_proj", + "model.layers.75.mlp.experts.144.down_proj", + "model.layers.75.mlp.experts.144.gate_proj", + "model.layers.75.mlp.experts.144.up_proj", + "model.layers.75.mlp.experts.145.down_proj", + "model.layers.75.mlp.experts.145.gate_proj", + "model.layers.75.mlp.experts.145.up_proj", + "model.layers.75.mlp.experts.146.down_proj", + "model.layers.75.mlp.experts.146.gate_proj", + "model.layers.75.mlp.experts.146.up_proj", + "model.layers.75.mlp.experts.147.down_proj", + "model.layers.75.mlp.experts.147.gate_proj", + "model.layers.75.mlp.experts.147.up_proj", + "model.layers.75.mlp.experts.148.down_proj", + "model.layers.75.mlp.experts.148.gate_proj", + "model.layers.75.mlp.experts.148.up_proj", + "model.layers.75.mlp.experts.149.down_proj", + "model.layers.75.mlp.experts.149.gate_proj", + "model.layers.75.mlp.experts.149.up_proj", + "model.layers.75.mlp.experts.15.down_proj", + "model.layers.75.mlp.experts.15.gate_proj", + "model.layers.75.mlp.experts.15.up_proj", + "model.layers.75.mlp.experts.150.down_proj", + "model.layers.75.mlp.experts.150.gate_proj", + "model.layers.75.mlp.experts.150.up_proj", + "model.layers.75.mlp.experts.151.down_proj", + "model.layers.75.mlp.experts.151.gate_proj", + "model.layers.75.mlp.experts.151.up_proj", + "model.layers.75.mlp.experts.152.down_proj", + "model.layers.75.mlp.experts.152.gate_proj", + "model.layers.75.mlp.experts.152.up_proj", + "model.layers.75.mlp.experts.153.down_proj", + "model.layers.75.mlp.experts.153.gate_proj", + "model.layers.75.mlp.experts.153.up_proj", + "model.layers.75.mlp.experts.154.down_proj", + "model.layers.75.mlp.experts.154.gate_proj", + "model.layers.75.mlp.experts.154.up_proj", + "model.layers.75.mlp.experts.155.down_proj", + "model.layers.75.mlp.experts.155.gate_proj", + "model.layers.75.mlp.experts.155.up_proj", + "model.layers.75.mlp.experts.156.down_proj", + "model.layers.75.mlp.experts.156.gate_proj", + "model.layers.75.mlp.experts.156.up_proj", + "model.layers.75.mlp.experts.157.down_proj", + "model.layers.75.mlp.experts.157.gate_proj", + "model.layers.75.mlp.experts.157.up_proj", + "model.layers.75.mlp.experts.158.down_proj", + "model.layers.75.mlp.experts.158.gate_proj", + "model.layers.75.mlp.experts.158.up_proj", + "model.layers.75.mlp.experts.159.down_proj", + "model.layers.75.mlp.experts.159.gate_proj", + "model.layers.75.mlp.experts.159.up_proj", + "model.layers.75.mlp.experts.16.down_proj", + "model.layers.75.mlp.experts.16.gate_proj", + "model.layers.75.mlp.experts.16.up_proj", + "model.layers.75.mlp.experts.160.down_proj", + "model.layers.75.mlp.experts.160.gate_proj", + "model.layers.75.mlp.experts.160.up_proj", + "model.layers.75.mlp.experts.161.down_proj", + "model.layers.75.mlp.experts.161.gate_proj", + "model.layers.75.mlp.experts.161.up_proj", + "model.layers.75.mlp.experts.162.down_proj", + "model.layers.75.mlp.experts.162.gate_proj", + "model.layers.75.mlp.experts.162.up_proj", + "model.layers.75.mlp.experts.163.down_proj", + "model.layers.75.mlp.experts.163.gate_proj", + "model.layers.75.mlp.experts.163.up_proj", + "model.layers.75.mlp.experts.164.down_proj", + "model.layers.75.mlp.experts.164.gate_proj", + "model.layers.75.mlp.experts.164.up_proj", + "model.layers.75.mlp.experts.165.down_proj", + "model.layers.75.mlp.experts.165.gate_proj", + "model.layers.75.mlp.experts.165.up_proj", + "model.layers.75.mlp.experts.166.down_proj", + "model.layers.75.mlp.experts.166.gate_proj", + "model.layers.75.mlp.experts.166.up_proj", + "model.layers.75.mlp.experts.167.down_proj", + "model.layers.75.mlp.experts.167.gate_proj", + "model.layers.75.mlp.experts.167.up_proj", + "model.layers.75.mlp.experts.168.down_proj", + "model.layers.75.mlp.experts.168.gate_proj", + "model.layers.75.mlp.experts.168.up_proj", + "model.layers.75.mlp.experts.169.down_proj", + "model.layers.75.mlp.experts.169.gate_proj", + "model.layers.75.mlp.experts.169.up_proj", + "model.layers.75.mlp.experts.17.down_proj", + "model.layers.75.mlp.experts.17.gate_proj", + "model.layers.75.mlp.experts.17.up_proj", + "model.layers.75.mlp.experts.170.down_proj", + "model.layers.75.mlp.experts.170.gate_proj", + "model.layers.75.mlp.experts.170.up_proj", + "model.layers.75.mlp.experts.171.down_proj", + "model.layers.75.mlp.experts.171.gate_proj", + "model.layers.75.mlp.experts.171.up_proj", + "model.layers.75.mlp.experts.172.down_proj", + "model.layers.75.mlp.experts.172.gate_proj", + "model.layers.75.mlp.experts.172.up_proj", + "model.layers.75.mlp.experts.173.down_proj", + "model.layers.75.mlp.experts.173.gate_proj", + "model.layers.75.mlp.experts.173.up_proj", + "model.layers.75.mlp.experts.174.down_proj", + "model.layers.75.mlp.experts.174.gate_proj", + "model.layers.75.mlp.experts.174.up_proj", + "model.layers.75.mlp.experts.175.down_proj", + "model.layers.75.mlp.experts.175.gate_proj", + "model.layers.75.mlp.experts.175.up_proj", + "model.layers.75.mlp.experts.176.down_proj", + "model.layers.75.mlp.experts.176.gate_proj", + "model.layers.75.mlp.experts.176.up_proj", + "model.layers.75.mlp.experts.177.down_proj", + "model.layers.75.mlp.experts.177.gate_proj", + "model.layers.75.mlp.experts.177.up_proj", + "model.layers.75.mlp.experts.178.down_proj", + "model.layers.75.mlp.experts.178.gate_proj", + "model.layers.75.mlp.experts.178.up_proj", + "model.layers.75.mlp.experts.179.down_proj", + "model.layers.75.mlp.experts.179.gate_proj", + "model.layers.75.mlp.experts.179.up_proj", + "model.layers.75.mlp.experts.18.down_proj", + "model.layers.75.mlp.experts.18.gate_proj", + "model.layers.75.mlp.experts.18.up_proj", + "model.layers.75.mlp.experts.180.down_proj", + "model.layers.75.mlp.experts.180.gate_proj", + "model.layers.75.mlp.experts.180.up_proj", + "model.layers.75.mlp.experts.181.down_proj", + "model.layers.75.mlp.experts.181.gate_proj", + "model.layers.75.mlp.experts.181.up_proj", + "model.layers.75.mlp.experts.182.down_proj", + "model.layers.75.mlp.experts.182.gate_proj", + "model.layers.75.mlp.experts.182.up_proj", + "model.layers.75.mlp.experts.183.down_proj", + "model.layers.75.mlp.experts.183.gate_proj", + "model.layers.75.mlp.experts.183.up_proj", + "model.layers.75.mlp.experts.184.down_proj", + "model.layers.75.mlp.experts.184.gate_proj", + "model.layers.75.mlp.experts.184.up_proj", + "model.layers.75.mlp.experts.185.down_proj", + "model.layers.75.mlp.experts.185.gate_proj", + "model.layers.75.mlp.experts.185.up_proj", + "model.layers.75.mlp.experts.186.down_proj", + "model.layers.75.mlp.experts.186.gate_proj", + "model.layers.75.mlp.experts.186.up_proj", + "model.layers.75.mlp.experts.187.down_proj", + "model.layers.75.mlp.experts.187.gate_proj", + "model.layers.75.mlp.experts.187.up_proj", + "model.layers.75.mlp.experts.188.down_proj", + "model.layers.75.mlp.experts.188.gate_proj", + "model.layers.75.mlp.experts.188.up_proj", + "model.layers.75.mlp.experts.189.down_proj", + "model.layers.75.mlp.experts.189.gate_proj", + "model.layers.75.mlp.experts.189.up_proj", + "model.layers.75.mlp.experts.19.down_proj", + "model.layers.75.mlp.experts.19.gate_proj", + "model.layers.75.mlp.experts.19.up_proj", + "model.layers.75.mlp.experts.190.down_proj", + "model.layers.75.mlp.experts.190.gate_proj", + "model.layers.75.mlp.experts.190.up_proj", + "model.layers.75.mlp.experts.191.down_proj", + "model.layers.75.mlp.experts.191.gate_proj", + "model.layers.75.mlp.experts.191.up_proj", + "model.layers.75.mlp.experts.192.down_proj", + "model.layers.75.mlp.experts.192.gate_proj", + "model.layers.75.mlp.experts.192.up_proj", + "model.layers.75.mlp.experts.193.down_proj", + "model.layers.75.mlp.experts.193.gate_proj", + "model.layers.75.mlp.experts.193.up_proj", + "model.layers.75.mlp.experts.194.down_proj", + "model.layers.75.mlp.experts.194.gate_proj", + "model.layers.75.mlp.experts.194.up_proj", + "model.layers.75.mlp.experts.195.down_proj", + "model.layers.75.mlp.experts.195.gate_proj", + "model.layers.75.mlp.experts.195.up_proj", + "model.layers.75.mlp.experts.196.down_proj", + "model.layers.75.mlp.experts.196.gate_proj", + "model.layers.75.mlp.experts.196.up_proj", + "model.layers.75.mlp.experts.197.down_proj", + "model.layers.75.mlp.experts.197.gate_proj", + "model.layers.75.mlp.experts.197.up_proj", + "model.layers.75.mlp.experts.198.down_proj", + "model.layers.75.mlp.experts.198.gate_proj", + "model.layers.75.mlp.experts.198.up_proj", + "model.layers.75.mlp.experts.199.down_proj", + "model.layers.75.mlp.experts.199.gate_proj", + "model.layers.75.mlp.experts.199.up_proj", + "model.layers.75.mlp.experts.2.down_proj", + "model.layers.75.mlp.experts.2.gate_proj", + "model.layers.75.mlp.experts.2.up_proj", + "model.layers.75.mlp.experts.20.down_proj", + "model.layers.75.mlp.experts.20.gate_proj", + "model.layers.75.mlp.experts.20.up_proj", + "model.layers.75.mlp.experts.200.down_proj", + "model.layers.75.mlp.experts.200.gate_proj", + "model.layers.75.mlp.experts.200.up_proj", + "model.layers.75.mlp.experts.201.down_proj", + "model.layers.75.mlp.experts.201.gate_proj", + "model.layers.75.mlp.experts.201.up_proj", + "model.layers.75.mlp.experts.202.down_proj", + "model.layers.75.mlp.experts.202.gate_proj", + "model.layers.75.mlp.experts.202.up_proj", + "model.layers.75.mlp.experts.203.down_proj", + "model.layers.75.mlp.experts.203.gate_proj", + "model.layers.75.mlp.experts.203.up_proj", + "model.layers.75.mlp.experts.204.down_proj", + "model.layers.75.mlp.experts.204.gate_proj", + "model.layers.75.mlp.experts.204.up_proj", + "model.layers.75.mlp.experts.205.down_proj", + "model.layers.75.mlp.experts.205.gate_proj", + "model.layers.75.mlp.experts.205.up_proj", + "model.layers.75.mlp.experts.206.down_proj", + "model.layers.75.mlp.experts.206.gate_proj", + "model.layers.75.mlp.experts.206.up_proj", + "model.layers.75.mlp.experts.207.down_proj", + "model.layers.75.mlp.experts.207.gate_proj", + "model.layers.75.mlp.experts.207.up_proj", + "model.layers.75.mlp.experts.208.down_proj", + "model.layers.75.mlp.experts.208.gate_proj", + "model.layers.75.mlp.experts.208.up_proj", + "model.layers.75.mlp.experts.209.down_proj", + "model.layers.75.mlp.experts.209.gate_proj", + "model.layers.75.mlp.experts.209.up_proj", + "model.layers.75.mlp.experts.21.down_proj", + "model.layers.75.mlp.experts.21.gate_proj", + "model.layers.75.mlp.experts.21.up_proj", + "model.layers.75.mlp.experts.210.down_proj", + "model.layers.75.mlp.experts.210.gate_proj", + "model.layers.75.mlp.experts.210.up_proj", + "model.layers.75.mlp.experts.211.down_proj", + "model.layers.75.mlp.experts.211.gate_proj", + "model.layers.75.mlp.experts.211.up_proj", + "model.layers.75.mlp.experts.212.down_proj", + "model.layers.75.mlp.experts.212.gate_proj", + "model.layers.75.mlp.experts.212.up_proj", + "model.layers.75.mlp.experts.213.down_proj", + "model.layers.75.mlp.experts.213.gate_proj", + "model.layers.75.mlp.experts.213.up_proj", + "model.layers.75.mlp.experts.214.down_proj", + "model.layers.75.mlp.experts.214.gate_proj", + "model.layers.75.mlp.experts.214.up_proj", + "model.layers.75.mlp.experts.215.down_proj", + "model.layers.75.mlp.experts.215.gate_proj", + "model.layers.75.mlp.experts.215.up_proj", + "model.layers.75.mlp.experts.216.down_proj", + "model.layers.75.mlp.experts.216.gate_proj", + "model.layers.75.mlp.experts.216.up_proj", + "model.layers.75.mlp.experts.217.down_proj", + "model.layers.75.mlp.experts.217.gate_proj", + "model.layers.75.mlp.experts.217.up_proj", + "model.layers.75.mlp.experts.218.down_proj", + "model.layers.75.mlp.experts.218.gate_proj", + "model.layers.75.mlp.experts.218.up_proj", + "model.layers.75.mlp.experts.219.down_proj", + "model.layers.75.mlp.experts.219.gate_proj", + "model.layers.75.mlp.experts.219.up_proj", + "model.layers.75.mlp.experts.22.down_proj", + "model.layers.75.mlp.experts.22.gate_proj", + "model.layers.75.mlp.experts.22.up_proj", + "model.layers.75.mlp.experts.220.down_proj", + "model.layers.75.mlp.experts.220.gate_proj", + "model.layers.75.mlp.experts.220.up_proj", + "model.layers.75.mlp.experts.221.down_proj", + "model.layers.75.mlp.experts.221.gate_proj", + "model.layers.75.mlp.experts.221.up_proj", + "model.layers.75.mlp.experts.222.down_proj", + "model.layers.75.mlp.experts.222.gate_proj", + "model.layers.75.mlp.experts.222.up_proj", + "model.layers.75.mlp.experts.223.down_proj", + "model.layers.75.mlp.experts.223.gate_proj", + "model.layers.75.mlp.experts.223.up_proj", + "model.layers.75.mlp.experts.224.down_proj", + "model.layers.75.mlp.experts.224.gate_proj", + "model.layers.75.mlp.experts.224.up_proj", + "model.layers.75.mlp.experts.225.down_proj", + "model.layers.75.mlp.experts.225.gate_proj", + "model.layers.75.mlp.experts.225.up_proj", + "model.layers.75.mlp.experts.226.down_proj", + "model.layers.75.mlp.experts.226.gate_proj", + "model.layers.75.mlp.experts.226.up_proj", + "model.layers.75.mlp.experts.227.down_proj", + "model.layers.75.mlp.experts.227.gate_proj", + "model.layers.75.mlp.experts.227.up_proj", + "model.layers.75.mlp.experts.228.down_proj", + "model.layers.75.mlp.experts.228.gate_proj", + "model.layers.75.mlp.experts.228.up_proj", + "model.layers.75.mlp.experts.229.down_proj", + "model.layers.75.mlp.experts.229.gate_proj", + "model.layers.75.mlp.experts.229.up_proj", + "model.layers.75.mlp.experts.23.down_proj", + "model.layers.75.mlp.experts.23.gate_proj", + "model.layers.75.mlp.experts.23.up_proj", + "model.layers.75.mlp.experts.230.down_proj", + "model.layers.75.mlp.experts.230.gate_proj", + "model.layers.75.mlp.experts.230.up_proj", + "model.layers.75.mlp.experts.231.down_proj", + "model.layers.75.mlp.experts.231.gate_proj", + "model.layers.75.mlp.experts.231.up_proj", + "model.layers.75.mlp.experts.232.down_proj", + "model.layers.75.mlp.experts.232.gate_proj", + "model.layers.75.mlp.experts.232.up_proj", + "model.layers.75.mlp.experts.233.down_proj", + "model.layers.75.mlp.experts.233.gate_proj", + "model.layers.75.mlp.experts.233.up_proj", + "model.layers.75.mlp.experts.234.down_proj", + "model.layers.75.mlp.experts.234.gate_proj", + "model.layers.75.mlp.experts.234.up_proj", + "model.layers.75.mlp.experts.235.down_proj", + "model.layers.75.mlp.experts.235.gate_proj", + "model.layers.75.mlp.experts.235.up_proj", + "model.layers.75.mlp.experts.236.down_proj", + "model.layers.75.mlp.experts.236.gate_proj", + "model.layers.75.mlp.experts.236.up_proj", + "model.layers.75.mlp.experts.237.down_proj", + "model.layers.75.mlp.experts.237.gate_proj", + "model.layers.75.mlp.experts.237.up_proj", + "model.layers.75.mlp.experts.238.down_proj", + "model.layers.75.mlp.experts.238.gate_proj", + "model.layers.75.mlp.experts.238.up_proj", + "model.layers.75.mlp.experts.239.down_proj", + "model.layers.75.mlp.experts.239.gate_proj", + "model.layers.75.mlp.experts.239.up_proj", + "model.layers.75.mlp.experts.24.down_proj", + "model.layers.75.mlp.experts.24.gate_proj", + "model.layers.75.mlp.experts.24.up_proj", + "model.layers.75.mlp.experts.240.down_proj", + "model.layers.75.mlp.experts.240.gate_proj", + "model.layers.75.mlp.experts.240.up_proj", + "model.layers.75.mlp.experts.241.down_proj", + "model.layers.75.mlp.experts.241.gate_proj", + "model.layers.75.mlp.experts.241.up_proj", + "model.layers.75.mlp.experts.242.down_proj", + "model.layers.75.mlp.experts.242.gate_proj", + "model.layers.75.mlp.experts.242.up_proj", + "model.layers.75.mlp.experts.243.down_proj", + "model.layers.75.mlp.experts.243.gate_proj", + "model.layers.75.mlp.experts.243.up_proj", + "model.layers.75.mlp.experts.244.down_proj", + "model.layers.75.mlp.experts.244.gate_proj", + "model.layers.75.mlp.experts.244.up_proj", + "model.layers.75.mlp.experts.245.down_proj", + "model.layers.75.mlp.experts.245.gate_proj", + "model.layers.75.mlp.experts.245.up_proj", + "model.layers.75.mlp.experts.246.down_proj", + "model.layers.75.mlp.experts.246.gate_proj", + "model.layers.75.mlp.experts.246.up_proj", + "model.layers.75.mlp.experts.247.down_proj", + "model.layers.75.mlp.experts.247.gate_proj", + "model.layers.75.mlp.experts.247.up_proj", + "model.layers.75.mlp.experts.248.down_proj", + "model.layers.75.mlp.experts.248.gate_proj", + "model.layers.75.mlp.experts.248.up_proj", + "model.layers.75.mlp.experts.249.down_proj", + "model.layers.75.mlp.experts.249.gate_proj", + "model.layers.75.mlp.experts.249.up_proj", + "model.layers.75.mlp.experts.25.down_proj", + "model.layers.75.mlp.experts.25.gate_proj", + "model.layers.75.mlp.experts.25.up_proj", + "model.layers.75.mlp.experts.250.down_proj", + "model.layers.75.mlp.experts.250.gate_proj", + "model.layers.75.mlp.experts.250.up_proj", + "model.layers.75.mlp.experts.251.down_proj", + "model.layers.75.mlp.experts.251.gate_proj", + "model.layers.75.mlp.experts.251.up_proj", + "model.layers.75.mlp.experts.252.down_proj", + "model.layers.75.mlp.experts.252.gate_proj", + "model.layers.75.mlp.experts.252.up_proj", + "model.layers.75.mlp.experts.253.down_proj", + "model.layers.75.mlp.experts.253.gate_proj", + "model.layers.75.mlp.experts.253.up_proj", + "model.layers.75.mlp.experts.254.down_proj", + "model.layers.75.mlp.experts.254.gate_proj", + "model.layers.75.mlp.experts.254.up_proj", + "model.layers.75.mlp.experts.255.down_proj", + "model.layers.75.mlp.experts.255.gate_proj", + "model.layers.75.mlp.experts.255.up_proj", + "model.layers.75.mlp.experts.26.down_proj", + "model.layers.75.mlp.experts.26.gate_proj", + "model.layers.75.mlp.experts.26.up_proj", + "model.layers.75.mlp.experts.27.down_proj", + "model.layers.75.mlp.experts.27.gate_proj", + "model.layers.75.mlp.experts.27.up_proj", + "model.layers.75.mlp.experts.28.down_proj", + "model.layers.75.mlp.experts.28.gate_proj", + "model.layers.75.mlp.experts.28.up_proj", + "model.layers.75.mlp.experts.29.down_proj", + "model.layers.75.mlp.experts.29.gate_proj", + "model.layers.75.mlp.experts.29.up_proj", + "model.layers.75.mlp.experts.3.down_proj", + "model.layers.75.mlp.experts.3.gate_proj", + "model.layers.75.mlp.experts.3.up_proj", + "model.layers.75.mlp.experts.30.down_proj", + "model.layers.75.mlp.experts.30.gate_proj", + "model.layers.75.mlp.experts.30.up_proj", + "model.layers.75.mlp.experts.31.down_proj", + "model.layers.75.mlp.experts.31.gate_proj", + "model.layers.75.mlp.experts.31.up_proj", + "model.layers.75.mlp.experts.32.down_proj", + "model.layers.75.mlp.experts.32.gate_proj", + "model.layers.75.mlp.experts.32.up_proj", + "model.layers.75.mlp.experts.33.down_proj", + "model.layers.75.mlp.experts.33.gate_proj", + "model.layers.75.mlp.experts.33.up_proj", + "model.layers.75.mlp.experts.34.down_proj", + "model.layers.75.mlp.experts.34.gate_proj", + "model.layers.75.mlp.experts.34.up_proj", + "model.layers.75.mlp.experts.35.down_proj", + "model.layers.75.mlp.experts.35.gate_proj", + "model.layers.75.mlp.experts.35.up_proj", + "model.layers.75.mlp.experts.36.down_proj", + "model.layers.75.mlp.experts.36.gate_proj", + "model.layers.75.mlp.experts.36.up_proj", + "model.layers.75.mlp.experts.37.down_proj", + "model.layers.75.mlp.experts.37.gate_proj", + "model.layers.75.mlp.experts.37.up_proj", + "model.layers.75.mlp.experts.38.down_proj", + "model.layers.75.mlp.experts.38.gate_proj", + "model.layers.75.mlp.experts.38.up_proj", + "model.layers.75.mlp.experts.39.down_proj", + "model.layers.75.mlp.experts.39.gate_proj", + "model.layers.75.mlp.experts.39.up_proj", + "model.layers.75.mlp.experts.4.down_proj", + "model.layers.75.mlp.experts.4.gate_proj", + "model.layers.75.mlp.experts.4.up_proj", + "model.layers.75.mlp.experts.40.down_proj", + "model.layers.75.mlp.experts.40.gate_proj", + "model.layers.75.mlp.experts.40.up_proj", + "model.layers.75.mlp.experts.41.down_proj", + "model.layers.75.mlp.experts.41.gate_proj", + "model.layers.75.mlp.experts.41.up_proj", + "model.layers.75.mlp.experts.42.down_proj", + "model.layers.75.mlp.experts.42.gate_proj", + "model.layers.75.mlp.experts.42.up_proj", + "model.layers.75.mlp.experts.43.down_proj", + "model.layers.75.mlp.experts.43.gate_proj", + "model.layers.75.mlp.experts.43.up_proj", + "model.layers.75.mlp.experts.44.down_proj", + "model.layers.75.mlp.experts.44.gate_proj", + "model.layers.75.mlp.experts.44.up_proj", + "model.layers.75.mlp.experts.45.down_proj", + "model.layers.75.mlp.experts.45.gate_proj", + "model.layers.75.mlp.experts.45.up_proj", + "model.layers.75.mlp.experts.46.down_proj", + "model.layers.75.mlp.experts.46.gate_proj", + "model.layers.75.mlp.experts.46.up_proj", + "model.layers.75.mlp.experts.47.down_proj", + "model.layers.75.mlp.experts.47.gate_proj", + "model.layers.75.mlp.experts.47.up_proj", + "model.layers.75.mlp.experts.48.down_proj", + "model.layers.75.mlp.experts.48.gate_proj", + "model.layers.75.mlp.experts.48.up_proj", + "model.layers.75.mlp.experts.49.down_proj", + "model.layers.75.mlp.experts.49.gate_proj", + "model.layers.75.mlp.experts.49.up_proj", + "model.layers.75.mlp.experts.5.down_proj", + "model.layers.75.mlp.experts.5.gate_proj", + "model.layers.75.mlp.experts.5.up_proj", + "model.layers.75.mlp.experts.50.down_proj", + "model.layers.75.mlp.experts.50.gate_proj", + "model.layers.75.mlp.experts.50.up_proj", + "model.layers.75.mlp.experts.51.down_proj", + "model.layers.75.mlp.experts.51.gate_proj", + "model.layers.75.mlp.experts.51.up_proj", + "model.layers.75.mlp.experts.52.down_proj", + "model.layers.75.mlp.experts.52.gate_proj", + "model.layers.75.mlp.experts.52.up_proj", + "model.layers.75.mlp.experts.53.down_proj", + "model.layers.75.mlp.experts.53.gate_proj", + "model.layers.75.mlp.experts.53.up_proj", + "model.layers.75.mlp.experts.54.down_proj", + "model.layers.75.mlp.experts.54.gate_proj", + "model.layers.75.mlp.experts.54.up_proj", + "model.layers.75.mlp.experts.55.down_proj", + "model.layers.75.mlp.experts.55.gate_proj", + "model.layers.75.mlp.experts.55.up_proj", + "model.layers.75.mlp.experts.56.down_proj", + "model.layers.75.mlp.experts.56.gate_proj", + "model.layers.75.mlp.experts.56.up_proj", + "model.layers.75.mlp.experts.57.down_proj", + "model.layers.75.mlp.experts.57.gate_proj", + "model.layers.75.mlp.experts.57.up_proj", + "model.layers.75.mlp.experts.58.down_proj", + "model.layers.75.mlp.experts.58.gate_proj", + "model.layers.75.mlp.experts.58.up_proj", + "model.layers.75.mlp.experts.59.down_proj", + "model.layers.75.mlp.experts.59.gate_proj", + "model.layers.75.mlp.experts.59.up_proj", + "model.layers.75.mlp.experts.6.down_proj", + "model.layers.75.mlp.experts.6.gate_proj", + "model.layers.75.mlp.experts.6.up_proj", + "model.layers.75.mlp.experts.60.down_proj", + "model.layers.75.mlp.experts.60.gate_proj", + "model.layers.75.mlp.experts.60.up_proj", + "model.layers.75.mlp.experts.61.down_proj", + "model.layers.75.mlp.experts.61.gate_proj", + "model.layers.75.mlp.experts.61.up_proj", + "model.layers.75.mlp.experts.62.down_proj", + "model.layers.75.mlp.experts.62.gate_proj", + "model.layers.75.mlp.experts.62.up_proj", + "model.layers.75.mlp.experts.63.down_proj", + "model.layers.75.mlp.experts.63.gate_proj", + "model.layers.75.mlp.experts.63.up_proj", + "model.layers.75.mlp.experts.64.down_proj", + "model.layers.75.mlp.experts.64.gate_proj", + "model.layers.75.mlp.experts.64.up_proj", + "model.layers.75.mlp.experts.65.down_proj", + "model.layers.75.mlp.experts.65.gate_proj", + "model.layers.75.mlp.experts.65.up_proj", + "model.layers.75.mlp.experts.66.down_proj", + "model.layers.75.mlp.experts.66.gate_proj", + "model.layers.75.mlp.experts.66.up_proj", + "model.layers.75.mlp.experts.67.down_proj", + "model.layers.75.mlp.experts.67.gate_proj", + "model.layers.75.mlp.experts.67.up_proj", + "model.layers.75.mlp.experts.68.down_proj", + "model.layers.75.mlp.experts.68.gate_proj", + "model.layers.75.mlp.experts.68.up_proj", + "model.layers.75.mlp.experts.69.down_proj", + "model.layers.75.mlp.experts.69.gate_proj", + "model.layers.75.mlp.experts.69.up_proj", + "model.layers.75.mlp.experts.7.down_proj", + "model.layers.75.mlp.experts.7.gate_proj", + "model.layers.75.mlp.experts.7.up_proj", + "model.layers.75.mlp.experts.70.down_proj", + "model.layers.75.mlp.experts.70.gate_proj", + "model.layers.75.mlp.experts.70.up_proj", + "model.layers.75.mlp.experts.71.down_proj", + "model.layers.75.mlp.experts.71.gate_proj", + "model.layers.75.mlp.experts.71.up_proj", + "model.layers.75.mlp.experts.72.down_proj", + "model.layers.75.mlp.experts.72.gate_proj", + "model.layers.75.mlp.experts.72.up_proj", + "model.layers.75.mlp.experts.73.down_proj", + "model.layers.75.mlp.experts.73.gate_proj", + "model.layers.75.mlp.experts.73.up_proj", + "model.layers.75.mlp.experts.74.down_proj", + "model.layers.75.mlp.experts.74.gate_proj", + "model.layers.75.mlp.experts.74.up_proj", + "model.layers.75.mlp.experts.75.down_proj", + "model.layers.75.mlp.experts.75.gate_proj", + "model.layers.75.mlp.experts.75.up_proj", + "model.layers.75.mlp.experts.76.down_proj", + "model.layers.75.mlp.experts.76.gate_proj", + "model.layers.75.mlp.experts.76.up_proj", + "model.layers.75.mlp.experts.77.down_proj", + "model.layers.75.mlp.experts.77.gate_proj", + "model.layers.75.mlp.experts.77.up_proj", + "model.layers.75.mlp.experts.78.down_proj", + "model.layers.75.mlp.experts.78.gate_proj", + "model.layers.75.mlp.experts.78.up_proj", + "model.layers.75.mlp.experts.79.down_proj", + "model.layers.75.mlp.experts.79.gate_proj", + "model.layers.75.mlp.experts.79.up_proj", + "model.layers.75.mlp.experts.8.down_proj", + "model.layers.75.mlp.experts.8.gate_proj", + "model.layers.75.mlp.experts.8.up_proj", + "model.layers.75.mlp.experts.80.down_proj", + "model.layers.75.mlp.experts.80.gate_proj", + "model.layers.75.mlp.experts.80.up_proj", + "model.layers.75.mlp.experts.81.down_proj", + "model.layers.75.mlp.experts.81.gate_proj", + "model.layers.75.mlp.experts.81.up_proj", + "model.layers.75.mlp.experts.82.down_proj", + "model.layers.75.mlp.experts.82.gate_proj", + "model.layers.75.mlp.experts.82.up_proj", + "model.layers.75.mlp.experts.83.down_proj", + "model.layers.75.mlp.experts.83.gate_proj", + "model.layers.75.mlp.experts.83.up_proj", + "model.layers.75.mlp.experts.84.down_proj", + "model.layers.75.mlp.experts.84.gate_proj", + "model.layers.75.mlp.experts.84.up_proj", + "model.layers.75.mlp.experts.85.down_proj", + "model.layers.75.mlp.experts.85.gate_proj", + "model.layers.75.mlp.experts.85.up_proj", + "model.layers.75.mlp.experts.86.down_proj", + "model.layers.75.mlp.experts.86.gate_proj", + "model.layers.75.mlp.experts.86.up_proj", + "model.layers.75.mlp.experts.87.down_proj", + "model.layers.75.mlp.experts.87.gate_proj", + "model.layers.75.mlp.experts.87.up_proj", + "model.layers.75.mlp.experts.88.down_proj", + "model.layers.75.mlp.experts.88.gate_proj", + "model.layers.75.mlp.experts.88.up_proj", + "model.layers.75.mlp.experts.89.down_proj", + "model.layers.75.mlp.experts.89.gate_proj", + "model.layers.75.mlp.experts.89.up_proj", + "model.layers.75.mlp.experts.9.down_proj", + "model.layers.75.mlp.experts.9.gate_proj", + "model.layers.75.mlp.experts.9.up_proj", + "model.layers.75.mlp.experts.90.down_proj", + "model.layers.75.mlp.experts.90.gate_proj", + "model.layers.75.mlp.experts.90.up_proj", + "model.layers.75.mlp.experts.91.down_proj", + "model.layers.75.mlp.experts.91.gate_proj", + "model.layers.75.mlp.experts.91.up_proj", + "model.layers.75.mlp.experts.92.down_proj", + "model.layers.75.mlp.experts.92.gate_proj", + "model.layers.75.mlp.experts.92.up_proj", + "model.layers.75.mlp.experts.93.down_proj", + "model.layers.75.mlp.experts.93.gate_proj", + "model.layers.75.mlp.experts.93.up_proj", + "model.layers.75.mlp.experts.94.down_proj", + "model.layers.75.mlp.experts.94.gate_proj", + "model.layers.75.mlp.experts.94.up_proj", + "model.layers.75.mlp.experts.95.down_proj", + "model.layers.75.mlp.experts.95.gate_proj", + "model.layers.75.mlp.experts.95.up_proj", + "model.layers.75.mlp.experts.96.down_proj", + "model.layers.75.mlp.experts.96.gate_proj", + "model.layers.75.mlp.experts.96.up_proj", + "model.layers.75.mlp.experts.97.down_proj", + "model.layers.75.mlp.experts.97.gate_proj", + "model.layers.75.mlp.experts.97.up_proj", + "model.layers.75.mlp.experts.98.down_proj", + "model.layers.75.mlp.experts.98.gate_proj", + "model.layers.75.mlp.experts.98.up_proj", + "model.layers.75.mlp.experts.99.down_proj", + "model.layers.75.mlp.experts.99.gate_proj", + "model.layers.75.mlp.experts.99.up_proj", + "model.layers.76.mlp.experts.0.down_proj", + "model.layers.76.mlp.experts.0.gate_proj", + "model.layers.76.mlp.experts.0.up_proj", + "model.layers.76.mlp.experts.1.down_proj", + "model.layers.76.mlp.experts.1.gate_proj", + "model.layers.76.mlp.experts.1.up_proj", + "model.layers.76.mlp.experts.10.down_proj", + "model.layers.76.mlp.experts.10.gate_proj", + "model.layers.76.mlp.experts.10.up_proj", + "model.layers.76.mlp.experts.100.down_proj", + "model.layers.76.mlp.experts.100.gate_proj", + "model.layers.76.mlp.experts.100.up_proj", + "model.layers.76.mlp.experts.101.down_proj", + "model.layers.76.mlp.experts.101.gate_proj", + "model.layers.76.mlp.experts.101.up_proj", + "model.layers.76.mlp.experts.102.down_proj", + "model.layers.76.mlp.experts.102.gate_proj", + "model.layers.76.mlp.experts.102.up_proj", + "model.layers.76.mlp.experts.103.down_proj", + "model.layers.76.mlp.experts.103.gate_proj", + "model.layers.76.mlp.experts.103.up_proj", + "model.layers.76.mlp.experts.104.down_proj", + "model.layers.76.mlp.experts.104.gate_proj", + "model.layers.76.mlp.experts.104.up_proj", + "model.layers.76.mlp.experts.105.down_proj", + "model.layers.76.mlp.experts.105.gate_proj", + "model.layers.76.mlp.experts.105.up_proj", + "model.layers.76.mlp.experts.106.down_proj", + "model.layers.76.mlp.experts.106.gate_proj", + "model.layers.76.mlp.experts.106.up_proj", + "model.layers.76.mlp.experts.107.down_proj", + "model.layers.76.mlp.experts.107.gate_proj", + "model.layers.76.mlp.experts.107.up_proj", + "model.layers.76.mlp.experts.108.down_proj", + "model.layers.76.mlp.experts.108.gate_proj", + "model.layers.76.mlp.experts.108.up_proj", + "model.layers.76.mlp.experts.109.down_proj", + "model.layers.76.mlp.experts.109.gate_proj", + "model.layers.76.mlp.experts.109.up_proj", + "model.layers.76.mlp.experts.11.down_proj", + "model.layers.76.mlp.experts.11.gate_proj", + "model.layers.76.mlp.experts.11.up_proj", + "model.layers.76.mlp.experts.110.down_proj", + "model.layers.76.mlp.experts.110.gate_proj", + "model.layers.76.mlp.experts.110.up_proj", + "model.layers.76.mlp.experts.111.down_proj", + "model.layers.76.mlp.experts.111.gate_proj", + "model.layers.76.mlp.experts.111.up_proj", + "model.layers.76.mlp.experts.112.down_proj", + "model.layers.76.mlp.experts.112.gate_proj", + "model.layers.76.mlp.experts.112.up_proj", + "model.layers.76.mlp.experts.113.down_proj", + "model.layers.76.mlp.experts.113.gate_proj", + "model.layers.76.mlp.experts.113.up_proj", + "model.layers.76.mlp.experts.114.down_proj", + "model.layers.76.mlp.experts.114.gate_proj", + "model.layers.76.mlp.experts.114.up_proj", + "model.layers.76.mlp.experts.115.down_proj", + "model.layers.76.mlp.experts.115.gate_proj", + "model.layers.76.mlp.experts.115.up_proj", + "model.layers.76.mlp.experts.116.down_proj", + "model.layers.76.mlp.experts.116.gate_proj", + "model.layers.76.mlp.experts.116.up_proj", + "model.layers.76.mlp.experts.117.down_proj", + "model.layers.76.mlp.experts.117.gate_proj", + "model.layers.76.mlp.experts.117.up_proj", + "model.layers.76.mlp.experts.118.down_proj", + "model.layers.76.mlp.experts.118.gate_proj", + "model.layers.76.mlp.experts.118.up_proj", + "model.layers.76.mlp.experts.119.down_proj", + "model.layers.76.mlp.experts.119.gate_proj", + "model.layers.76.mlp.experts.119.up_proj", + "model.layers.76.mlp.experts.12.down_proj", + "model.layers.76.mlp.experts.12.gate_proj", + "model.layers.76.mlp.experts.12.up_proj", + "model.layers.76.mlp.experts.120.down_proj", + "model.layers.76.mlp.experts.120.gate_proj", + "model.layers.76.mlp.experts.120.up_proj", + "model.layers.76.mlp.experts.121.down_proj", + "model.layers.76.mlp.experts.121.gate_proj", + "model.layers.76.mlp.experts.121.up_proj", + "model.layers.76.mlp.experts.122.down_proj", + "model.layers.76.mlp.experts.122.gate_proj", + "model.layers.76.mlp.experts.122.up_proj", + "model.layers.76.mlp.experts.123.down_proj", + "model.layers.76.mlp.experts.123.gate_proj", + "model.layers.76.mlp.experts.123.up_proj", + "model.layers.76.mlp.experts.124.down_proj", + "model.layers.76.mlp.experts.124.gate_proj", + "model.layers.76.mlp.experts.124.up_proj", + "model.layers.76.mlp.experts.125.down_proj", + "model.layers.76.mlp.experts.125.gate_proj", + "model.layers.76.mlp.experts.125.up_proj", + "model.layers.76.mlp.experts.126.down_proj", + "model.layers.76.mlp.experts.126.gate_proj", + "model.layers.76.mlp.experts.126.up_proj", + "model.layers.76.mlp.experts.127.down_proj", + "model.layers.76.mlp.experts.127.gate_proj", + "model.layers.76.mlp.experts.127.up_proj", + "model.layers.76.mlp.experts.128.down_proj", + "model.layers.76.mlp.experts.128.gate_proj", + "model.layers.76.mlp.experts.128.up_proj", + "model.layers.76.mlp.experts.129.down_proj", + "model.layers.76.mlp.experts.129.gate_proj", + "model.layers.76.mlp.experts.129.up_proj", + "model.layers.76.mlp.experts.13.down_proj", + "model.layers.76.mlp.experts.13.gate_proj", + "model.layers.76.mlp.experts.13.up_proj", + "model.layers.76.mlp.experts.130.down_proj", + "model.layers.76.mlp.experts.130.gate_proj", + "model.layers.76.mlp.experts.130.up_proj", + "model.layers.76.mlp.experts.131.down_proj", + "model.layers.76.mlp.experts.131.gate_proj", + "model.layers.76.mlp.experts.131.up_proj", + "model.layers.76.mlp.experts.132.down_proj", + "model.layers.76.mlp.experts.132.gate_proj", + "model.layers.76.mlp.experts.132.up_proj", + "model.layers.76.mlp.experts.133.down_proj", + "model.layers.76.mlp.experts.133.gate_proj", + "model.layers.76.mlp.experts.133.up_proj", + "model.layers.76.mlp.experts.134.down_proj", + "model.layers.76.mlp.experts.134.gate_proj", + "model.layers.76.mlp.experts.134.up_proj", + "model.layers.76.mlp.experts.135.down_proj", + "model.layers.76.mlp.experts.135.gate_proj", + "model.layers.76.mlp.experts.135.up_proj", + "model.layers.76.mlp.experts.136.down_proj", + "model.layers.76.mlp.experts.136.gate_proj", + "model.layers.76.mlp.experts.136.up_proj", + "model.layers.76.mlp.experts.137.down_proj", + "model.layers.76.mlp.experts.137.gate_proj", + "model.layers.76.mlp.experts.137.up_proj", + "model.layers.76.mlp.experts.138.down_proj", + "model.layers.76.mlp.experts.138.gate_proj", + "model.layers.76.mlp.experts.138.up_proj", + "model.layers.76.mlp.experts.139.down_proj", + "model.layers.76.mlp.experts.139.gate_proj", + "model.layers.76.mlp.experts.139.up_proj", + "model.layers.76.mlp.experts.14.down_proj", + "model.layers.76.mlp.experts.14.gate_proj", + "model.layers.76.mlp.experts.14.up_proj", + "model.layers.76.mlp.experts.140.down_proj", + "model.layers.76.mlp.experts.140.gate_proj", + "model.layers.76.mlp.experts.140.up_proj", + "model.layers.76.mlp.experts.141.down_proj", + "model.layers.76.mlp.experts.141.gate_proj", + "model.layers.76.mlp.experts.141.up_proj", + "model.layers.76.mlp.experts.142.down_proj", + "model.layers.76.mlp.experts.142.gate_proj", + "model.layers.76.mlp.experts.142.up_proj", + "model.layers.76.mlp.experts.143.down_proj", + "model.layers.76.mlp.experts.143.gate_proj", + "model.layers.76.mlp.experts.143.up_proj", + "model.layers.76.mlp.experts.144.down_proj", + "model.layers.76.mlp.experts.144.gate_proj", + "model.layers.76.mlp.experts.144.up_proj", + "model.layers.76.mlp.experts.145.down_proj", + "model.layers.76.mlp.experts.145.gate_proj", + "model.layers.76.mlp.experts.145.up_proj", + "model.layers.76.mlp.experts.146.down_proj", + "model.layers.76.mlp.experts.146.gate_proj", + "model.layers.76.mlp.experts.146.up_proj", + "model.layers.76.mlp.experts.147.down_proj", + "model.layers.76.mlp.experts.147.gate_proj", + "model.layers.76.mlp.experts.147.up_proj", + "model.layers.76.mlp.experts.148.down_proj", + "model.layers.76.mlp.experts.148.gate_proj", + "model.layers.76.mlp.experts.148.up_proj", + "model.layers.76.mlp.experts.149.down_proj", + "model.layers.76.mlp.experts.149.gate_proj", + "model.layers.76.mlp.experts.149.up_proj", + "model.layers.76.mlp.experts.15.down_proj", + "model.layers.76.mlp.experts.15.gate_proj", + "model.layers.76.mlp.experts.15.up_proj", + "model.layers.76.mlp.experts.150.down_proj", + "model.layers.76.mlp.experts.150.gate_proj", + "model.layers.76.mlp.experts.150.up_proj", + "model.layers.76.mlp.experts.151.down_proj", + "model.layers.76.mlp.experts.151.gate_proj", + "model.layers.76.mlp.experts.151.up_proj", + "model.layers.76.mlp.experts.152.down_proj", + "model.layers.76.mlp.experts.152.gate_proj", + "model.layers.76.mlp.experts.152.up_proj", + "model.layers.76.mlp.experts.153.down_proj", + "model.layers.76.mlp.experts.153.gate_proj", + "model.layers.76.mlp.experts.153.up_proj", + "model.layers.76.mlp.experts.154.down_proj", + "model.layers.76.mlp.experts.154.gate_proj", + "model.layers.76.mlp.experts.154.up_proj", + "model.layers.76.mlp.experts.155.down_proj", + "model.layers.76.mlp.experts.155.gate_proj", + "model.layers.76.mlp.experts.155.up_proj", + "model.layers.76.mlp.experts.156.down_proj", + "model.layers.76.mlp.experts.156.gate_proj", + "model.layers.76.mlp.experts.156.up_proj", + "model.layers.76.mlp.experts.157.down_proj", + "model.layers.76.mlp.experts.157.gate_proj", + "model.layers.76.mlp.experts.157.up_proj", + "model.layers.76.mlp.experts.158.down_proj", + "model.layers.76.mlp.experts.158.gate_proj", + "model.layers.76.mlp.experts.158.up_proj", + "model.layers.76.mlp.experts.159.down_proj", + "model.layers.76.mlp.experts.159.gate_proj", + "model.layers.76.mlp.experts.159.up_proj", + "model.layers.76.mlp.experts.16.down_proj", + "model.layers.76.mlp.experts.16.gate_proj", + "model.layers.76.mlp.experts.16.up_proj", + "model.layers.76.mlp.experts.160.down_proj", + "model.layers.76.mlp.experts.160.gate_proj", + "model.layers.76.mlp.experts.160.up_proj", + "model.layers.76.mlp.experts.161.down_proj", + "model.layers.76.mlp.experts.161.gate_proj", + "model.layers.76.mlp.experts.161.up_proj", + "model.layers.76.mlp.experts.162.down_proj", + "model.layers.76.mlp.experts.162.gate_proj", + "model.layers.76.mlp.experts.162.up_proj", + "model.layers.76.mlp.experts.163.down_proj", + "model.layers.76.mlp.experts.163.gate_proj", + "model.layers.76.mlp.experts.163.up_proj", + "model.layers.76.mlp.experts.164.down_proj", + "model.layers.76.mlp.experts.164.gate_proj", + "model.layers.76.mlp.experts.164.up_proj", + "model.layers.76.mlp.experts.165.down_proj", + "model.layers.76.mlp.experts.165.gate_proj", + "model.layers.76.mlp.experts.165.up_proj", + "model.layers.76.mlp.experts.166.down_proj", + "model.layers.76.mlp.experts.166.gate_proj", + "model.layers.76.mlp.experts.166.up_proj", + "model.layers.76.mlp.experts.167.down_proj", + "model.layers.76.mlp.experts.167.gate_proj", + "model.layers.76.mlp.experts.167.up_proj", + "model.layers.76.mlp.experts.168.down_proj", + "model.layers.76.mlp.experts.168.gate_proj", + "model.layers.76.mlp.experts.168.up_proj", + "model.layers.76.mlp.experts.169.down_proj", + "model.layers.76.mlp.experts.169.gate_proj", + "model.layers.76.mlp.experts.169.up_proj", + "model.layers.76.mlp.experts.17.down_proj", + "model.layers.76.mlp.experts.17.gate_proj", + "model.layers.76.mlp.experts.17.up_proj", + "model.layers.76.mlp.experts.170.down_proj", + "model.layers.76.mlp.experts.170.gate_proj", + "model.layers.76.mlp.experts.170.up_proj", + "model.layers.76.mlp.experts.171.down_proj", + "model.layers.76.mlp.experts.171.gate_proj", + "model.layers.76.mlp.experts.171.up_proj", + "model.layers.76.mlp.experts.172.down_proj", + "model.layers.76.mlp.experts.172.gate_proj", + "model.layers.76.mlp.experts.172.up_proj", + "model.layers.76.mlp.experts.173.down_proj", + "model.layers.76.mlp.experts.173.gate_proj", + "model.layers.76.mlp.experts.173.up_proj", + "model.layers.76.mlp.experts.174.down_proj", + "model.layers.76.mlp.experts.174.gate_proj", + "model.layers.76.mlp.experts.174.up_proj", + "model.layers.76.mlp.experts.175.down_proj", + "model.layers.76.mlp.experts.175.gate_proj", + "model.layers.76.mlp.experts.175.up_proj", + "model.layers.76.mlp.experts.176.down_proj", + "model.layers.76.mlp.experts.176.gate_proj", + "model.layers.76.mlp.experts.176.up_proj", + "model.layers.76.mlp.experts.177.down_proj", + "model.layers.76.mlp.experts.177.gate_proj", + "model.layers.76.mlp.experts.177.up_proj", + "model.layers.76.mlp.experts.178.down_proj", + "model.layers.76.mlp.experts.178.gate_proj", + "model.layers.76.mlp.experts.178.up_proj", + "model.layers.76.mlp.experts.179.down_proj", + "model.layers.76.mlp.experts.179.gate_proj", + "model.layers.76.mlp.experts.179.up_proj", + "model.layers.76.mlp.experts.18.down_proj", + "model.layers.76.mlp.experts.18.gate_proj", + "model.layers.76.mlp.experts.18.up_proj", + "model.layers.76.mlp.experts.180.down_proj", + "model.layers.76.mlp.experts.180.gate_proj", + "model.layers.76.mlp.experts.180.up_proj", + "model.layers.76.mlp.experts.181.down_proj", + "model.layers.76.mlp.experts.181.gate_proj", + "model.layers.76.mlp.experts.181.up_proj", + "model.layers.76.mlp.experts.182.down_proj", + "model.layers.76.mlp.experts.182.gate_proj", + "model.layers.76.mlp.experts.182.up_proj", + "model.layers.76.mlp.experts.183.down_proj", + "model.layers.76.mlp.experts.183.gate_proj", + "model.layers.76.mlp.experts.183.up_proj", + "model.layers.76.mlp.experts.184.down_proj", + "model.layers.76.mlp.experts.184.gate_proj", + "model.layers.76.mlp.experts.184.up_proj", + "model.layers.76.mlp.experts.185.down_proj", + "model.layers.76.mlp.experts.185.gate_proj", + "model.layers.76.mlp.experts.185.up_proj", + "model.layers.76.mlp.experts.186.down_proj", + "model.layers.76.mlp.experts.186.gate_proj", + "model.layers.76.mlp.experts.186.up_proj", + "model.layers.76.mlp.experts.187.down_proj", + "model.layers.76.mlp.experts.187.gate_proj", + "model.layers.76.mlp.experts.187.up_proj", + "model.layers.76.mlp.experts.188.down_proj", + "model.layers.76.mlp.experts.188.gate_proj", + "model.layers.76.mlp.experts.188.up_proj", + "model.layers.76.mlp.experts.189.down_proj", + "model.layers.76.mlp.experts.189.gate_proj", + "model.layers.76.mlp.experts.189.up_proj", + "model.layers.76.mlp.experts.19.down_proj", + "model.layers.76.mlp.experts.19.gate_proj", + "model.layers.76.mlp.experts.19.up_proj", + "model.layers.76.mlp.experts.190.down_proj", + "model.layers.76.mlp.experts.190.gate_proj", + "model.layers.76.mlp.experts.190.up_proj", + "model.layers.76.mlp.experts.191.down_proj", + "model.layers.76.mlp.experts.191.gate_proj", + "model.layers.76.mlp.experts.191.up_proj", + "model.layers.76.mlp.experts.192.down_proj", + "model.layers.76.mlp.experts.192.gate_proj", + "model.layers.76.mlp.experts.192.up_proj", + "model.layers.76.mlp.experts.193.down_proj", + "model.layers.76.mlp.experts.193.gate_proj", + "model.layers.76.mlp.experts.193.up_proj", + "model.layers.76.mlp.experts.194.down_proj", + "model.layers.76.mlp.experts.194.gate_proj", + "model.layers.76.mlp.experts.194.up_proj", + "model.layers.76.mlp.experts.195.down_proj", + "model.layers.76.mlp.experts.195.gate_proj", + "model.layers.76.mlp.experts.195.up_proj", + "model.layers.76.mlp.experts.196.down_proj", + "model.layers.76.mlp.experts.196.gate_proj", + "model.layers.76.mlp.experts.196.up_proj", + "model.layers.76.mlp.experts.197.down_proj", + "model.layers.76.mlp.experts.197.gate_proj", + "model.layers.76.mlp.experts.197.up_proj", + "model.layers.76.mlp.experts.198.down_proj", + "model.layers.76.mlp.experts.198.gate_proj", + "model.layers.76.mlp.experts.198.up_proj", + "model.layers.76.mlp.experts.199.down_proj", + "model.layers.76.mlp.experts.199.gate_proj", + "model.layers.76.mlp.experts.199.up_proj", + "model.layers.76.mlp.experts.2.down_proj", + "model.layers.76.mlp.experts.2.gate_proj", + "model.layers.76.mlp.experts.2.up_proj", + "model.layers.76.mlp.experts.20.down_proj", + "model.layers.76.mlp.experts.20.gate_proj", + "model.layers.76.mlp.experts.20.up_proj", + "model.layers.76.mlp.experts.200.down_proj", + "model.layers.76.mlp.experts.200.gate_proj", + "model.layers.76.mlp.experts.200.up_proj", + "model.layers.76.mlp.experts.201.down_proj", + "model.layers.76.mlp.experts.201.gate_proj", + "model.layers.76.mlp.experts.201.up_proj", + "model.layers.76.mlp.experts.202.down_proj", + "model.layers.76.mlp.experts.202.gate_proj", + "model.layers.76.mlp.experts.202.up_proj", + "model.layers.76.mlp.experts.203.down_proj", + "model.layers.76.mlp.experts.203.gate_proj", + "model.layers.76.mlp.experts.203.up_proj", + "model.layers.76.mlp.experts.204.down_proj", + "model.layers.76.mlp.experts.204.gate_proj", + "model.layers.76.mlp.experts.204.up_proj", + "model.layers.76.mlp.experts.205.down_proj", + "model.layers.76.mlp.experts.205.gate_proj", + "model.layers.76.mlp.experts.205.up_proj", + "model.layers.76.mlp.experts.206.down_proj", + "model.layers.76.mlp.experts.206.gate_proj", + "model.layers.76.mlp.experts.206.up_proj", + "model.layers.76.mlp.experts.207.down_proj", + "model.layers.76.mlp.experts.207.gate_proj", + "model.layers.76.mlp.experts.207.up_proj", + "model.layers.76.mlp.experts.208.down_proj", + "model.layers.76.mlp.experts.208.gate_proj", + "model.layers.76.mlp.experts.208.up_proj", + "model.layers.76.mlp.experts.209.down_proj", + "model.layers.76.mlp.experts.209.gate_proj", + "model.layers.76.mlp.experts.209.up_proj", + "model.layers.76.mlp.experts.21.down_proj", + "model.layers.76.mlp.experts.21.gate_proj", + "model.layers.76.mlp.experts.21.up_proj", + "model.layers.76.mlp.experts.210.down_proj", + "model.layers.76.mlp.experts.210.gate_proj", + "model.layers.76.mlp.experts.210.up_proj", + "model.layers.76.mlp.experts.211.down_proj", + "model.layers.76.mlp.experts.211.gate_proj", + "model.layers.76.mlp.experts.211.up_proj", + "model.layers.76.mlp.experts.212.down_proj", + "model.layers.76.mlp.experts.212.gate_proj", + "model.layers.76.mlp.experts.212.up_proj", + "model.layers.76.mlp.experts.213.down_proj", + "model.layers.76.mlp.experts.213.gate_proj", + "model.layers.76.mlp.experts.213.up_proj", + "model.layers.76.mlp.experts.214.down_proj", + "model.layers.76.mlp.experts.214.gate_proj", + "model.layers.76.mlp.experts.214.up_proj", + "model.layers.76.mlp.experts.215.down_proj", + "model.layers.76.mlp.experts.215.gate_proj", + "model.layers.76.mlp.experts.215.up_proj", + "model.layers.76.mlp.experts.216.down_proj", + "model.layers.76.mlp.experts.216.gate_proj", + "model.layers.76.mlp.experts.216.up_proj", + "model.layers.76.mlp.experts.217.down_proj", + "model.layers.76.mlp.experts.217.gate_proj", + "model.layers.76.mlp.experts.217.up_proj", + "model.layers.76.mlp.experts.218.down_proj", + "model.layers.76.mlp.experts.218.gate_proj", + "model.layers.76.mlp.experts.218.up_proj", + "model.layers.76.mlp.experts.219.down_proj", + "model.layers.76.mlp.experts.219.gate_proj", + "model.layers.76.mlp.experts.219.up_proj", + "model.layers.76.mlp.experts.22.down_proj", + "model.layers.76.mlp.experts.22.gate_proj", + "model.layers.76.mlp.experts.22.up_proj", + "model.layers.76.mlp.experts.220.down_proj", + "model.layers.76.mlp.experts.220.gate_proj", + "model.layers.76.mlp.experts.220.up_proj", + "model.layers.76.mlp.experts.221.down_proj", + "model.layers.76.mlp.experts.221.gate_proj", + "model.layers.76.mlp.experts.221.up_proj", + "model.layers.76.mlp.experts.222.down_proj", + "model.layers.76.mlp.experts.222.gate_proj", + "model.layers.76.mlp.experts.222.up_proj", + "model.layers.76.mlp.experts.223.down_proj", + "model.layers.76.mlp.experts.223.gate_proj", + "model.layers.76.mlp.experts.223.up_proj", + "model.layers.76.mlp.experts.224.down_proj", + "model.layers.76.mlp.experts.224.gate_proj", + "model.layers.76.mlp.experts.224.up_proj", + "model.layers.76.mlp.experts.225.down_proj", + "model.layers.76.mlp.experts.225.gate_proj", + "model.layers.76.mlp.experts.225.up_proj", + "model.layers.76.mlp.experts.226.down_proj", + "model.layers.76.mlp.experts.226.gate_proj", + "model.layers.76.mlp.experts.226.up_proj", + "model.layers.76.mlp.experts.227.down_proj", + "model.layers.76.mlp.experts.227.gate_proj", + "model.layers.76.mlp.experts.227.up_proj", + "model.layers.76.mlp.experts.228.down_proj", + "model.layers.76.mlp.experts.228.gate_proj", + "model.layers.76.mlp.experts.228.up_proj", + "model.layers.76.mlp.experts.229.down_proj", + "model.layers.76.mlp.experts.229.gate_proj", + "model.layers.76.mlp.experts.229.up_proj", + "model.layers.76.mlp.experts.23.down_proj", + "model.layers.76.mlp.experts.23.gate_proj", + "model.layers.76.mlp.experts.23.up_proj", + "model.layers.76.mlp.experts.230.down_proj", + "model.layers.76.mlp.experts.230.gate_proj", + "model.layers.76.mlp.experts.230.up_proj", + "model.layers.76.mlp.experts.231.down_proj", + "model.layers.76.mlp.experts.231.gate_proj", + "model.layers.76.mlp.experts.231.up_proj", + "model.layers.76.mlp.experts.232.down_proj", + "model.layers.76.mlp.experts.232.gate_proj", + "model.layers.76.mlp.experts.232.up_proj", + "model.layers.76.mlp.experts.233.down_proj", + "model.layers.76.mlp.experts.233.gate_proj", + "model.layers.76.mlp.experts.233.up_proj", + "model.layers.76.mlp.experts.234.down_proj", + "model.layers.76.mlp.experts.234.gate_proj", + "model.layers.76.mlp.experts.234.up_proj", + "model.layers.76.mlp.experts.235.down_proj", + "model.layers.76.mlp.experts.235.gate_proj", + "model.layers.76.mlp.experts.235.up_proj", + "model.layers.76.mlp.experts.236.down_proj", + "model.layers.76.mlp.experts.236.gate_proj", + "model.layers.76.mlp.experts.236.up_proj", + "model.layers.76.mlp.experts.237.down_proj", + "model.layers.76.mlp.experts.237.gate_proj", + "model.layers.76.mlp.experts.237.up_proj", + "model.layers.76.mlp.experts.238.down_proj", + "model.layers.76.mlp.experts.238.gate_proj", + "model.layers.76.mlp.experts.238.up_proj", + "model.layers.76.mlp.experts.239.down_proj", + "model.layers.76.mlp.experts.239.gate_proj", + "model.layers.76.mlp.experts.239.up_proj", + "model.layers.76.mlp.experts.24.down_proj", + "model.layers.76.mlp.experts.24.gate_proj", + "model.layers.76.mlp.experts.24.up_proj", + "model.layers.76.mlp.experts.240.down_proj", + "model.layers.76.mlp.experts.240.gate_proj", + "model.layers.76.mlp.experts.240.up_proj", + "model.layers.76.mlp.experts.241.down_proj", + "model.layers.76.mlp.experts.241.gate_proj", + "model.layers.76.mlp.experts.241.up_proj", + "model.layers.76.mlp.experts.242.down_proj", + "model.layers.76.mlp.experts.242.gate_proj", + "model.layers.76.mlp.experts.242.up_proj", + "model.layers.76.mlp.experts.243.down_proj", + "model.layers.76.mlp.experts.243.gate_proj", + "model.layers.76.mlp.experts.243.up_proj", + "model.layers.76.mlp.experts.244.down_proj", + "model.layers.76.mlp.experts.244.gate_proj", + "model.layers.76.mlp.experts.244.up_proj", + "model.layers.76.mlp.experts.245.down_proj", + "model.layers.76.mlp.experts.245.gate_proj", + "model.layers.76.mlp.experts.245.up_proj", + "model.layers.76.mlp.experts.246.down_proj", + "model.layers.76.mlp.experts.246.gate_proj", + "model.layers.76.mlp.experts.246.up_proj", + "model.layers.76.mlp.experts.247.down_proj", + "model.layers.76.mlp.experts.247.gate_proj", + "model.layers.76.mlp.experts.247.up_proj", + "model.layers.76.mlp.experts.248.down_proj", + "model.layers.76.mlp.experts.248.gate_proj", + "model.layers.76.mlp.experts.248.up_proj", + "model.layers.76.mlp.experts.249.down_proj", + "model.layers.76.mlp.experts.249.gate_proj", + "model.layers.76.mlp.experts.249.up_proj", + "model.layers.76.mlp.experts.25.down_proj", + "model.layers.76.mlp.experts.25.gate_proj", + "model.layers.76.mlp.experts.25.up_proj", + "model.layers.76.mlp.experts.250.down_proj", + "model.layers.76.mlp.experts.250.gate_proj", + "model.layers.76.mlp.experts.250.up_proj", + "model.layers.76.mlp.experts.251.down_proj", + "model.layers.76.mlp.experts.251.gate_proj", + "model.layers.76.mlp.experts.251.up_proj", + "model.layers.76.mlp.experts.252.down_proj", + "model.layers.76.mlp.experts.252.gate_proj", + "model.layers.76.mlp.experts.252.up_proj", + "model.layers.76.mlp.experts.253.down_proj", + "model.layers.76.mlp.experts.253.gate_proj", + "model.layers.76.mlp.experts.253.up_proj", + "model.layers.76.mlp.experts.254.down_proj", + "model.layers.76.mlp.experts.254.gate_proj", + "model.layers.76.mlp.experts.254.up_proj", + "model.layers.76.mlp.experts.255.down_proj", + "model.layers.76.mlp.experts.255.gate_proj", + "model.layers.76.mlp.experts.255.up_proj", + "model.layers.76.mlp.experts.26.down_proj", + "model.layers.76.mlp.experts.26.gate_proj", + "model.layers.76.mlp.experts.26.up_proj", + "model.layers.76.mlp.experts.27.down_proj", + "model.layers.76.mlp.experts.27.gate_proj", + "model.layers.76.mlp.experts.27.up_proj", + "model.layers.76.mlp.experts.28.down_proj", + "model.layers.76.mlp.experts.28.gate_proj", + "model.layers.76.mlp.experts.28.up_proj", + "model.layers.76.mlp.experts.29.down_proj", + "model.layers.76.mlp.experts.29.gate_proj", + "model.layers.76.mlp.experts.29.up_proj", + "model.layers.76.mlp.experts.3.down_proj", + "model.layers.76.mlp.experts.3.gate_proj", + "model.layers.76.mlp.experts.3.up_proj", + "model.layers.76.mlp.experts.30.down_proj", + "model.layers.76.mlp.experts.30.gate_proj", + "model.layers.76.mlp.experts.30.up_proj", + "model.layers.76.mlp.experts.31.down_proj", + "model.layers.76.mlp.experts.31.gate_proj", + "model.layers.76.mlp.experts.31.up_proj", + "model.layers.76.mlp.experts.32.down_proj", + "model.layers.76.mlp.experts.32.gate_proj", + "model.layers.76.mlp.experts.32.up_proj", + "model.layers.76.mlp.experts.33.down_proj", + "model.layers.76.mlp.experts.33.gate_proj", + "model.layers.76.mlp.experts.33.up_proj", + "model.layers.76.mlp.experts.34.down_proj", + "model.layers.76.mlp.experts.34.gate_proj", + "model.layers.76.mlp.experts.34.up_proj", + "model.layers.76.mlp.experts.35.down_proj", + "model.layers.76.mlp.experts.35.gate_proj", + "model.layers.76.mlp.experts.35.up_proj", + "model.layers.76.mlp.experts.36.down_proj", + "model.layers.76.mlp.experts.36.gate_proj", + "model.layers.76.mlp.experts.36.up_proj", + "model.layers.76.mlp.experts.37.down_proj", + "model.layers.76.mlp.experts.37.gate_proj", + "model.layers.76.mlp.experts.37.up_proj", + "model.layers.76.mlp.experts.38.down_proj", + "model.layers.76.mlp.experts.38.gate_proj", + "model.layers.76.mlp.experts.38.up_proj", + "model.layers.76.mlp.experts.39.down_proj", + "model.layers.76.mlp.experts.39.gate_proj", + "model.layers.76.mlp.experts.39.up_proj", + "model.layers.76.mlp.experts.4.down_proj", + "model.layers.76.mlp.experts.4.gate_proj", + "model.layers.76.mlp.experts.4.up_proj", + "model.layers.76.mlp.experts.40.down_proj", + "model.layers.76.mlp.experts.40.gate_proj", + "model.layers.76.mlp.experts.40.up_proj", + "model.layers.76.mlp.experts.41.down_proj", + "model.layers.76.mlp.experts.41.gate_proj", + "model.layers.76.mlp.experts.41.up_proj", + "model.layers.76.mlp.experts.42.down_proj", + "model.layers.76.mlp.experts.42.gate_proj", + "model.layers.76.mlp.experts.42.up_proj", + "model.layers.76.mlp.experts.43.down_proj", + "model.layers.76.mlp.experts.43.gate_proj", + "model.layers.76.mlp.experts.43.up_proj", + "model.layers.76.mlp.experts.44.down_proj", + "model.layers.76.mlp.experts.44.gate_proj", + "model.layers.76.mlp.experts.44.up_proj", + "model.layers.76.mlp.experts.45.down_proj", + "model.layers.76.mlp.experts.45.gate_proj", + "model.layers.76.mlp.experts.45.up_proj", + "model.layers.76.mlp.experts.46.down_proj", + "model.layers.76.mlp.experts.46.gate_proj", + "model.layers.76.mlp.experts.46.up_proj", + "model.layers.76.mlp.experts.47.down_proj", + "model.layers.76.mlp.experts.47.gate_proj", + "model.layers.76.mlp.experts.47.up_proj", + "model.layers.76.mlp.experts.48.down_proj", + "model.layers.76.mlp.experts.48.gate_proj", + "model.layers.76.mlp.experts.48.up_proj", + "model.layers.76.mlp.experts.49.down_proj", + "model.layers.76.mlp.experts.49.gate_proj", + "model.layers.76.mlp.experts.49.up_proj", + "model.layers.76.mlp.experts.5.down_proj", + "model.layers.76.mlp.experts.5.gate_proj", + "model.layers.76.mlp.experts.5.up_proj", + "model.layers.76.mlp.experts.50.down_proj", + "model.layers.76.mlp.experts.50.gate_proj", + "model.layers.76.mlp.experts.50.up_proj", + "model.layers.76.mlp.experts.51.down_proj", + "model.layers.76.mlp.experts.51.gate_proj", + "model.layers.76.mlp.experts.51.up_proj", + "model.layers.76.mlp.experts.52.down_proj", + "model.layers.76.mlp.experts.52.gate_proj", + "model.layers.76.mlp.experts.52.up_proj", + "model.layers.76.mlp.experts.53.down_proj", + "model.layers.76.mlp.experts.53.gate_proj", + "model.layers.76.mlp.experts.53.up_proj", + "model.layers.76.mlp.experts.54.down_proj", + "model.layers.76.mlp.experts.54.gate_proj", + "model.layers.76.mlp.experts.54.up_proj", + "model.layers.76.mlp.experts.55.down_proj", + "model.layers.76.mlp.experts.55.gate_proj", + "model.layers.76.mlp.experts.55.up_proj", + "model.layers.76.mlp.experts.56.down_proj", + "model.layers.76.mlp.experts.56.gate_proj", + "model.layers.76.mlp.experts.56.up_proj", + "model.layers.76.mlp.experts.57.down_proj", + "model.layers.76.mlp.experts.57.gate_proj", + "model.layers.76.mlp.experts.57.up_proj", + "model.layers.76.mlp.experts.58.down_proj", + "model.layers.76.mlp.experts.58.gate_proj", + "model.layers.76.mlp.experts.58.up_proj", + "model.layers.76.mlp.experts.59.down_proj", + "model.layers.76.mlp.experts.59.gate_proj", + "model.layers.76.mlp.experts.59.up_proj", + "model.layers.76.mlp.experts.6.down_proj", + "model.layers.76.mlp.experts.6.gate_proj", + "model.layers.76.mlp.experts.6.up_proj", + "model.layers.76.mlp.experts.60.down_proj", + "model.layers.76.mlp.experts.60.gate_proj", + "model.layers.76.mlp.experts.60.up_proj", + "model.layers.76.mlp.experts.61.down_proj", + "model.layers.76.mlp.experts.61.gate_proj", + "model.layers.76.mlp.experts.61.up_proj", + "model.layers.76.mlp.experts.62.down_proj", + "model.layers.76.mlp.experts.62.gate_proj", + "model.layers.76.mlp.experts.62.up_proj", + "model.layers.76.mlp.experts.63.down_proj", + "model.layers.76.mlp.experts.63.gate_proj", + "model.layers.76.mlp.experts.63.up_proj", + "model.layers.76.mlp.experts.64.down_proj", + "model.layers.76.mlp.experts.64.gate_proj", + "model.layers.76.mlp.experts.64.up_proj", + "model.layers.76.mlp.experts.65.down_proj", + "model.layers.76.mlp.experts.65.gate_proj", + "model.layers.76.mlp.experts.65.up_proj", + "model.layers.76.mlp.experts.66.down_proj", + "model.layers.76.mlp.experts.66.gate_proj", + "model.layers.76.mlp.experts.66.up_proj", + "model.layers.76.mlp.experts.67.down_proj", + "model.layers.76.mlp.experts.67.gate_proj", + "model.layers.76.mlp.experts.67.up_proj", + "model.layers.76.mlp.experts.68.down_proj", + "model.layers.76.mlp.experts.68.gate_proj", + "model.layers.76.mlp.experts.68.up_proj", + "model.layers.76.mlp.experts.69.down_proj", + "model.layers.76.mlp.experts.69.gate_proj", + "model.layers.76.mlp.experts.69.up_proj", + "model.layers.76.mlp.experts.7.down_proj", + "model.layers.76.mlp.experts.7.gate_proj", + "model.layers.76.mlp.experts.7.up_proj", + "model.layers.76.mlp.experts.70.down_proj", + "model.layers.76.mlp.experts.70.gate_proj", + "model.layers.76.mlp.experts.70.up_proj", + "model.layers.76.mlp.experts.71.down_proj", + "model.layers.76.mlp.experts.71.gate_proj", + "model.layers.76.mlp.experts.71.up_proj", + "model.layers.76.mlp.experts.72.down_proj", + "model.layers.76.mlp.experts.72.gate_proj", + "model.layers.76.mlp.experts.72.up_proj", + "model.layers.76.mlp.experts.73.down_proj", + "model.layers.76.mlp.experts.73.gate_proj", + "model.layers.76.mlp.experts.73.up_proj", + "model.layers.76.mlp.experts.74.down_proj", + "model.layers.76.mlp.experts.74.gate_proj", + "model.layers.76.mlp.experts.74.up_proj", + "model.layers.76.mlp.experts.75.down_proj", + "model.layers.76.mlp.experts.75.gate_proj", + "model.layers.76.mlp.experts.75.up_proj", + "model.layers.76.mlp.experts.76.down_proj", + "model.layers.76.mlp.experts.76.gate_proj", + "model.layers.76.mlp.experts.76.up_proj", + "model.layers.76.mlp.experts.77.down_proj", + "model.layers.76.mlp.experts.77.gate_proj", + "model.layers.76.mlp.experts.77.up_proj", + "model.layers.76.mlp.experts.78.down_proj", + "model.layers.76.mlp.experts.78.gate_proj", + "model.layers.76.mlp.experts.78.up_proj", + "model.layers.76.mlp.experts.79.down_proj", + "model.layers.76.mlp.experts.79.gate_proj", + "model.layers.76.mlp.experts.79.up_proj", + "model.layers.76.mlp.experts.8.down_proj", + "model.layers.76.mlp.experts.8.gate_proj", + "model.layers.76.mlp.experts.8.up_proj", + "model.layers.76.mlp.experts.80.down_proj", + "model.layers.76.mlp.experts.80.gate_proj", + "model.layers.76.mlp.experts.80.up_proj", + "model.layers.76.mlp.experts.81.down_proj", + "model.layers.76.mlp.experts.81.gate_proj", + "model.layers.76.mlp.experts.81.up_proj", + "model.layers.76.mlp.experts.82.down_proj", + "model.layers.76.mlp.experts.82.gate_proj", + "model.layers.76.mlp.experts.82.up_proj", + "model.layers.76.mlp.experts.83.down_proj", + "model.layers.76.mlp.experts.83.gate_proj", + "model.layers.76.mlp.experts.83.up_proj", + "model.layers.76.mlp.experts.84.down_proj", + "model.layers.76.mlp.experts.84.gate_proj", + "model.layers.76.mlp.experts.84.up_proj", + "model.layers.76.mlp.experts.85.down_proj", + "model.layers.76.mlp.experts.85.gate_proj", + "model.layers.76.mlp.experts.85.up_proj", + "model.layers.76.mlp.experts.86.down_proj", + "model.layers.76.mlp.experts.86.gate_proj", + "model.layers.76.mlp.experts.86.up_proj", + "model.layers.76.mlp.experts.87.down_proj", + "model.layers.76.mlp.experts.87.gate_proj", + "model.layers.76.mlp.experts.87.up_proj", + "model.layers.76.mlp.experts.88.down_proj", + "model.layers.76.mlp.experts.88.gate_proj", + "model.layers.76.mlp.experts.88.up_proj", + "model.layers.76.mlp.experts.89.down_proj", + "model.layers.76.mlp.experts.89.gate_proj", + "model.layers.76.mlp.experts.89.up_proj", + "model.layers.76.mlp.experts.9.down_proj", + "model.layers.76.mlp.experts.9.gate_proj", + "model.layers.76.mlp.experts.9.up_proj", + "model.layers.76.mlp.experts.90.down_proj", + "model.layers.76.mlp.experts.90.gate_proj", + "model.layers.76.mlp.experts.90.up_proj", + "model.layers.76.mlp.experts.91.down_proj", + "model.layers.76.mlp.experts.91.gate_proj", + "model.layers.76.mlp.experts.91.up_proj", + "model.layers.76.mlp.experts.92.down_proj", + "model.layers.76.mlp.experts.92.gate_proj", + "model.layers.76.mlp.experts.92.up_proj", + "model.layers.76.mlp.experts.93.down_proj", + "model.layers.76.mlp.experts.93.gate_proj", + "model.layers.76.mlp.experts.93.up_proj", + "model.layers.76.mlp.experts.94.down_proj", + "model.layers.76.mlp.experts.94.gate_proj", + "model.layers.76.mlp.experts.94.up_proj", + "model.layers.76.mlp.experts.95.down_proj", + "model.layers.76.mlp.experts.95.gate_proj", + "model.layers.76.mlp.experts.95.up_proj", + "model.layers.76.mlp.experts.96.down_proj", + "model.layers.76.mlp.experts.96.gate_proj", + "model.layers.76.mlp.experts.96.up_proj", + "model.layers.76.mlp.experts.97.down_proj", + "model.layers.76.mlp.experts.97.gate_proj", + "model.layers.76.mlp.experts.97.up_proj", + "model.layers.76.mlp.experts.98.down_proj", + "model.layers.76.mlp.experts.98.gate_proj", + "model.layers.76.mlp.experts.98.up_proj", + "model.layers.76.mlp.experts.99.down_proj", + "model.layers.76.mlp.experts.99.gate_proj", + "model.layers.76.mlp.experts.99.up_proj", + "model.layers.77.mlp.experts.0.down_proj", + "model.layers.77.mlp.experts.0.gate_proj", + "model.layers.77.mlp.experts.0.up_proj", + "model.layers.77.mlp.experts.1.down_proj", + "model.layers.77.mlp.experts.1.gate_proj", + "model.layers.77.mlp.experts.1.up_proj", + "model.layers.77.mlp.experts.10.down_proj", + "model.layers.77.mlp.experts.10.gate_proj", + "model.layers.77.mlp.experts.10.up_proj", + "model.layers.77.mlp.experts.100.down_proj", + "model.layers.77.mlp.experts.100.gate_proj", + "model.layers.77.mlp.experts.100.up_proj", + "model.layers.77.mlp.experts.101.down_proj", + "model.layers.77.mlp.experts.101.gate_proj", + "model.layers.77.mlp.experts.101.up_proj", + "model.layers.77.mlp.experts.102.down_proj", + "model.layers.77.mlp.experts.102.gate_proj", + "model.layers.77.mlp.experts.102.up_proj", + "model.layers.77.mlp.experts.103.down_proj", + "model.layers.77.mlp.experts.103.gate_proj", + "model.layers.77.mlp.experts.103.up_proj", + "model.layers.77.mlp.experts.104.down_proj", + "model.layers.77.mlp.experts.104.gate_proj", + "model.layers.77.mlp.experts.104.up_proj", + "model.layers.77.mlp.experts.105.down_proj", + "model.layers.77.mlp.experts.105.gate_proj", + "model.layers.77.mlp.experts.105.up_proj", + "model.layers.77.mlp.experts.106.down_proj", + "model.layers.77.mlp.experts.106.gate_proj", + "model.layers.77.mlp.experts.106.up_proj", + "model.layers.77.mlp.experts.107.down_proj", + "model.layers.77.mlp.experts.107.gate_proj", + "model.layers.77.mlp.experts.107.up_proj", + "model.layers.77.mlp.experts.108.down_proj", + "model.layers.77.mlp.experts.108.gate_proj", + "model.layers.77.mlp.experts.108.up_proj", + "model.layers.77.mlp.experts.109.down_proj", + "model.layers.77.mlp.experts.109.gate_proj", + "model.layers.77.mlp.experts.109.up_proj", + "model.layers.77.mlp.experts.11.down_proj", + "model.layers.77.mlp.experts.11.gate_proj", + "model.layers.77.mlp.experts.11.up_proj", + "model.layers.77.mlp.experts.110.down_proj", + "model.layers.77.mlp.experts.110.gate_proj", + "model.layers.77.mlp.experts.110.up_proj", + "model.layers.77.mlp.experts.111.down_proj", + "model.layers.77.mlp.experts.111.gate_proj", + "model.layers.77.mlp.experts.111.up_proj", + "model.layers.77.mlp.experts.112.down_proj", + "model.layers.77.mlp.experts.112.gate_proj", + "model.layers.77.mlp.experts.112.up_proj", + "model.layers.77.mlp.experts.113.down_proj", + "model.layers.77.mlp.experts.113.gate_proj", + "model.layers.77.mlp.experts.113.up_proj", + "model.layers.77.mlp.experts.114.down_proj", + "model.layers.77.mlp.experts.114.gate_proj", + "model.layers.77.mlp.experts.114.up_proj", + "model.layers.77.mlp.experts.115.down_proj", + "model.layers.77.mlp.experts.115.gate_proj", + "model.layers.77.mlp.experts.115.up_proj", + "model.layers.77.mlp.experts.116.down_proj", + "model.layers.77.mlp.experts.116.gate_proj", + "model.layers.77.mlp.experts.116.up_proj", + "model.layers.77.mlp.experts.117.down_proj", + "model.layers.77.mlp.experts.117.gate_proj", + "model.layers.77.mlp.experts.117.up_proj", + "model.layers.77.mlp.experts.118.down_proj", + "model.layers.77.mlp.experts.118.gate_proj", + "model.layers.77.mlp.experts.118.up_proj", + "model.layers.77.mlp.experts.119.down_proj", + "model.layers.77.mlp.experts.119.gate_proj", + "model.layers.77.mlp.experts.119.up_proj", + "model.layers.77.mlp.experts.12.down_proj", + "model.layers.77.mlp.experts.12.gate_proj", + "model.layers.77.mlp.experts.12.up_proj", + "model.layers.77.mlp.experts.120.down_proj", + "model.layers.77.mlp.experts.120.gate_proj", + "model.layers.77.mlp.experts.120.up_proj", + "model.layers.77.mlp.experts.121.down_proj", + "model.layers.77.mlp.experts.121.gate_proj", + "model.layers.77.mlp.experts.121.up_proj", + "model.layers.77.mlp.experts.122.down_proj", + "model.layers.77.mlp.experts.122.gate_proj", + "model.layers.77.mlp.experts.122.up_proj", + "model.layers.77.mlp.experts.123.down_proj", + "model.layers.77.mlp.experts.123.gate_proj", + "model.layers.77.mlp.experts.123.up_proj", + "model.layers.77.mlp.experts.124.down_proj", + "model.layers.77.mlp.experts.124.gate_proj", + "model.layers.77.mlp.experts.124.up_proj", + "model.layers.77.mlp.experts.125.down_proj", + "model.layers.77.mlp.experts.125.gate_proj", + "model.layers.77.mlp.experts.125.up_proj", + "model.layers.77.mlp.experts.126.down_proj", + "model.layers.77.mlp.experts.126.gate_proj", + "model.layers.77.mlp.experts.126.up_proj", + "model.layers.77.mlp.experts.127.down_proj", + "model.layers.77.mlp.experts.127.gate_proj", + "model.layers.77.mlp.experts.127.up_proj", + "model.layers.77.mlp.experts.128.down_proj", + "model.layers.77.mlp.experts.128.gate_proj", + "model.layers.77.mlp.experts.128.up_proj", + "model.layers.77.mlp.experts.129.down_proj", + "model.layers.77.mlp.experts.129.gate_proj", + "model.layers.77.mlp.experts.129.up_proj", + "model.layers.77.mlp.experts.13.down_proj", + "model.layers.77.mlp.experts.13.gate_proj", + "model.layers.77.mlp.experts.13.up_proj", + "model.layers.77.mlp.experts.130.down_proj", + "model.layers.77.mlp.experts.130.gate_proj", + "model.layers.77.mlp.experts.130.up_proj", + "model.layers.77.mlp.experts.131.down_proj", + "model.layers.77.mlp.experts.131.gate_proj", + "model.layers.77.mlp.experts.131.up_proj", + "model.layers.77.mlp.experts.132.down_proj", + "model.layers.77.mlp.experts.132.gate_proj", + "model.layers.77.mlp.experts.132.up_proj", + "model.layers.77.mlp.experts.133.down_proj", + "model.layers.77.mlp.experts.133.gate_proj", + "model.layers.77.mlp.experts.133.up_proj", + "model.layers.77.mlp.experts.134.down_proj", + "model.layers.77.mlp.experts.134.gate_proj", + "model.layers.77.mlp.experts.134.up_proj", + "model.layers.77.mlp.experts.135.down_proj", + "model.layers.77.mlp.experts.135.gate_proj", + "model.layers.77.mlp.experts.135.up_proj", + "model.layers.77.mlp.experts.136.down_proj", + "model.layers.77.mlp.experts.136.gate_proj", + "model.layers.77.mlp.experts.136.up_proj", + "model.layers.77.mlp.experts.137.down_proj", + "model.layers.77.mlp.experts.137.gate_proj", + "model.layers.77.mlp.experts.137.up_proj", + "model.layers.77.mlp.experts.138.down_proj", + "model.layers.77.mlp.experts.138.gate_proj", + "model.layers.77.mlp.experts.138.up_proj", + "model.layers.77.mlp.experts.139.down_proj", + "model.layers.77.mlp.experts.139.gate_proj", + "model.layers.77.mlp.experts.139.up_proj", + "model.layers.77.mlp.experts.14.down_proj", + "model.layers.77.mlp.experts.14.gate_proj", + "model.layers.77.mlp.experts.14.up_proj", + "model.layers.77.mlp.experts.140.down_proj", + "model.layers.77.mlp.experts.140.gate_proj", + "model.layers.77.mlp.experts.140.up_proj", + "model.layers.77.mlp.experts.141.down_proj", + "model.layers.77.mlp.experts.141.gate_proj", + "model.layers.77.mlp.experts.141.up_proj", + "model.layers.77.mlp.experts.142.down_proj", + "model.layers.77.mlp.experts.142.gate_proj", + "model.layers.77.mlp.experts.142.up_proj", + "model.layers.77.mlp.experts.143.down_proj", + "model.layers.77.mlp.experts.143.gate_proj", + "model.layers.77.mlp.experts.143.up_proj", + "model.layers.77.mlp.experts.144.down_proj", + "model.layers.77.mlp.experts.144.gate_proj", + "model.layers.77.mlp.experts.144.up_proj", + "model.layers.77.mlp.experts.145.down_proj", + "model.layers.77.mlp.experts.145.gate_proj", + "model.layers.77.mlp.experts.145.up_proj", + "model.layers.77.mlp.experts.146.down_proj", + "model.layers.77.mlp.experts.146.gate_proj", + "model.layers.77.mlp.experts.146.up_proj", + "model.layers.77.mlp.experts.147.down_proj", + "model.layers.77.mlp.experts.147.gate_proj", + "model.layers.77.mlp.experts.147.up_proj", + "model.layers.77.mlp.experts.148.down_proj", + "model.layers.77.mlp.experts.148.gate_proj", + "model.layers.77.mlp.experts.148.up_proj", + "model.layers.77.mlp.experts.149.down_proj", + "model.layers.77.mlp.experts.149.gate_proj", + "model.layers.77.mlp.experts.149.up_proj", + "model.layers.77.mlp.experts.15.down_proj", + "model.layers.77.mlp.experts.15.gate_proj", + "model.layers.77.mlp.experts.15.up_proj", + "model.layers.77.mlp.experts.150.down_proj", + "model.layers.77.mlp.experts.150.gate_proj", + "model.layers.77.mlp.experts.150.up_proj", + "model.layers.77.mlp.experts.151.down_proj", + "model.layers.77.mlp.experts.151.gate_proj", + "model.layers.77.mlp.experts.151.up_proj", + "model.layers.77.mlp.experts.152.down_proj", + "model.layers.77.mlp.experts.152.gate_proj", + "model.layers.77.mlp.experts.152.up_proj", + "model.layers.77.mlp.experts.153.down_proj", + "model.layers.77.mlp.experts.153.gate_proj", + "model.layers.77.mlp.experts.153.up_proj", + "model.layers.77.mlp.experts.154.down_proj", + "model.layers.77.mlp.experts.154.gate_proj", + "model.layers.77.mlp.experts.154.up_proj", + "model.layers.77.mlp.experts.155.down_proj", + "model.layers.77.mlp.experts.155.gate_proj", + "model.layers.77.mlp.experts.155.up_proj", + "model.layers.77.mlp.experts.156.down_proj", + "model.layers.77.mlp.experts.156.gate_proj", + "model.layers.77.mlp.experts.156.up_proj", + "model.layers.77.mlp.experts.157.down_proj", + "model.layers.77.mlp.experts.157.gate_proj", + "model.layers.77.mlp.experts.157.up_proj", + "model.layers.77.mlp.experts.158.down_proj", + "model.layers.77.mlp.experts.158.gate_proj", + "model.layers.77.mlp.experts.158.up_proj", + "model.layers.77.mlp.experts.159.down_proj", + "model.layers.77.mlp.experts.159.gate_proj", + "model.layers.77.mlp.experts.159.up_proj", + "model.layers.77.mlp.experts.16.down_proj", + "model.layers.77.mlp.experts.16.gate_proj", + "model.layers.77.mlp.experts.16.up_proj", + "model.layers.77.mlp.experts.160.down_proj", + "model.layers.77.mlp.experts.160.gate_proj", + "model.layers.77.mlp.experts.160.up_proj", + "model.layers.77.mlp.experts.161.down_proj", + "model.layers.77.mlp.experts.161.gate_proj", + "model.layers.77.mlp.experts.161.up_proj", + "model.layers.77.mlp.experts.162.down_proj", + "model.layers.77.mlp.experts.162.gate_proj", + "model.layers.77.mlp.experts.162.up_proj", + "model.layers.77.mlp.experts.163.down_proj", + "model.layers.77.mlp.experts.163.gate_proj", + "model.layers.77.mlp.experts.163.up_proj", + "model.layers.77.mlp.experts.164.down_proj", + "model.layers.77.mlp.experts.164.gate_proj", + "model.layers.77.mlp.experts.164.up_proj", + "model.layers.77.mlp.experts.165.down_proj", + "model.layers.77.mlp.experts.165.gate_proj", + "model.layers.77.mlp.experts.165.up_proj", + "model.layers.77.mlp.experts.166.down_proj", + "model.layers.77.mlp.experts.166.gate_proj", + "model.layers.77.mlp.experts.166.up_proj", + "model.layers.77.mlp.experts.167.down_proj", + "model.layers.77.mlp.experts.167.gate_proj", + "model.layers.77.mlp.experts.167.up_proj", + "model.layers.77.mlp.experts.168.down_proj", + "model.layers.77.mlp.experts.168.gate_proj", + "model.layers.77.mlp.experts.168.up_proj", + "model.layers.77.mlp.experts.169.down_proj", + "model.layers.77.mlp.experts.169.gate_proj", + "model.layers.77.mlp.experts.169.up_proj", + "model.layers.77.mlp.experts.17.down_proj", + "model.layers.77.mlp.experts.17.gate_proj", + "model.layers.77.mlp.experts.17.up_proj", + "model.layers.77.mlp.experts.170.down_proj", + "model.layers.77.mlp.experts.170.gate_proj", + "model.layers.77.mlp.experts.170.up_proj", + "model.layers.77.mlp.experts.171.down_proj", + "model.layers.77.mlp.experts.171.gate_proj", + "model.layers.77.mlp.experts.171.up_proj", + "model.layers.77.mlp.experts.172.down_proj", + "model.layers.77.mlp.experts.172.gate_proj", + "model.layers.77.mlp.experts.172.up_proj", + "model.layers.77.mlp.experts.173.down_proj", + "model.layers.77.mlp.experts.173.gate_proj", + "model.layers.77.mlp.experts.173.up_proj", + "model.layers.77.mlp.experts.174.down_proj", + "model.layers.77.mlp.experts.174.gate_proj", + "model.layers.77.mlp.experts.174.up_proj", + "model.layers.77.mlp.experts.175.down_proj", + "model.layers.77.mlp.experts.175.gate_proj", + "model.layers.77.mlp.experts.175.up_proj", + "model.layers.77.mlp.experts.176.down_proj", + "model.layers.77.mlp.experts.176.gate_proj", + "model.layers.77.mlp.experts.176.up_proj", + "model.layers.77.mlp.experts.177.down_proj", + "model.layers.77.mlp.experts.177.gate_proj", + "model.layers.77.mlp.experts.177.up_proj", + "model.layers.77.mlp.experts.178.down_proj", + "model.layers.77.mlp.experts.178.gate_proj", + "model.layers.77.mlp.experts.178.up_proj", + "model.layers.77.mlp.experts.179.down_proj", + "model.layers.77.mlp.experts.179.gate_proj", + "model.layers.77.mlp.experts.179.up_proj", + "model.layers.77.mlp.experts.18.down_proj", + "model.layers.77.mlp.experts.18.gate_proj", + "model.layers.77.mlp.experts.18.up_proj", + "model.layers.77.mlp.experts.180.down_proj", + "model.layers.77.mlp.experts.180.gate_proj", + "model.layers.77.mlp.experts.180.up_proj", + "model.layers.77.mlp.experts.181.down_proj", + "model.layers.77.mlp.experts.181.gate_proj", + "model.layers.77.mlp.experts.181.up_proj", + "model.layers.77.mlp.experts.182.down_proj", + "model.layers.77.mlp.experts.182.gate_proj", + "model.layers.77.mlp.experts.182.up_proj", + "model.layers.77.mlp.experts.183.down_proj", + "model.layers.77.mlp.experts.183.gate_proj", + "model.layers.77.mlp.experts.183.up_proj", + "model.layers.77.mlp.experts.184.down_proj", + "model.layers.77.mlp.experts.184.gate_proj", + "model.layers.77.mlp.experts.184.up_proj", + "model.layers.77.mlp.experts.185.down_proj", + "model.layers.77.mlp.experts.185.gate_proj", + "model.layers.77.mlp.experts.185.up_proj", + "model.layers.77.mlp.experts.186.down_proj", + "model.layers.77.mlp.experts.186.gate_proj", + "model.layers.77.mlp.experts.186.up_proj", + "model.layers.77.mlp.experts.187.down_proj", + "model.layers.77.mlp.experts.187.gate_proj", + "model.layers.77.mlp.experts.187.up_proj", + "model.layers.77.mlp.experts.188.down_proj", + "model.layers.77.mlp.experts.188.gate_proj", + "model.layers.77.mlp.experts.188.up_proj", + "model.layers.77.mlp.experts.189.down_proj", + "model.layers.77.mlp.experts.189.gate_proj", + "model.layers.77.mlp.experts.189.up_proj", + "model.layers.77.mlp.experts.19.down_proj", + "model.layers.77.mlp.experts.19.gate_proj", + "model.layers.77.mlp.experts.19.up_proj", + "model.layers.77.mlp.experts.190.down_proj", + "model.layers.77.mlp.experts.190.gate_proj", + "model.layers.77.mlp.experts.190.up_proj", + "model.layers.77.mlp.experts.191.down_proj", + "model.layers.77.mlp.experts.191.gate_proj", + "model.layers.77.mlp.experts.191.up_proj", + "model.layers.77.mlp.experts.192.down_proj", + "model.layers.77.mlp.experts.192.gate_proj", + "model.layers.77.mlp.experts.192.up_proj", + "model.layers.77.mlp.experts.193.down_proj", + "model.layers.77.mlp.experts.193.gate_proj", + "model.layers.77.mlp.experts.193.up_proj", + "model.layers.77.mlp.experts.194.down_proj", + "model.layers.77.mlp.experts.194.gate_proj", + "model.layers.77.mlp.experts.194.up_proj", + "model.layers.77.mlp.experts.195.down_proj", + "model.layers.77.mlp.experts.195.gate_proj", + "model.layers.77.mlp.experts.195.up_proj", + "model.layers.77.mlp.experts.196.down_proj", + "model.layers.77.mlp.experts.196.gate_proj", + "model.layers.77.mlp.experts.196.up_proj", + "model.layers.77.mlp.experts.197.down_proj", + "model.layers.77.mlp.experts.197.gate_proj", + "model.layers.77.mlp.experts.197.up_proj", + "model.layers.77.mlp.experts.198.down_proj", + "model.layers.77.mlp.experts.198.gate_proj", + "model.layers.77.mlp.experts.198.up_proj", + "model.layers.77.mlp.experts.199.down_proj", + "model.layers.77.mlp.experts.199.gate_proj", + "model.layers.77.mlp.experts.199.up_proj", + "model.layers.77.mlp.experts.2.down_proj", + "model.layers.77.mlp.experts.2.gate_proj", + "model.layers.77.mlp.experts.2.up_proj", + "model.layers.77.mlp.experts.20.down_proj", + "model.layers.77.mlp.experts.20.gate_proj", + "model.layers.77.mlp.experts.20.up_proj", + "model.layers.77.mlp.experts.200.down_proj", + "model.layers.77.mlp.experts.200.gate_proj", + "model.layers.77.mlp.experts.200.up_proj", + "model.layers.77.mlp.experts.201.down_proj", + "model.layers.77.mlp.experts.201.gate_proj", + "model.layers.77.mlp.experts.201.up_proj", + "model.layers.77.mlp.experts.202.down_proj", + "model.layers.77.mlp.experts.202.gate_proj", + "model.layers.77.mlp.experts.202.up_proj", + "model.layers.77.mlp.experts.203.down_proj", + "model.layers.77.mlp.experts.203.gate_proj", + "model.layers.77.mlp.experts.203.up_proj", + "model.layers.77.mlp.experts.204.down_proj", + "model.layers.77.mlp.experts.204.gate_proj", + "model.layers.77.mlp.experts.204.up_proj", + "model.layers.77.mlp.experts.205.down_proj", + "model.layers.77.mlp.experts.205.gate_proj", + "model.layers.77.mlp.experts.205.up_proj", + "model.layers.77.mlp.experts.206.down_proj", + "model.layers.77.mlp.experts.206.gate_proj", + "model.layers.77.mlp.experts.206.up_proj", + "model.layers.77.mlp.experts.207.down_proj", + "model.layers.77.mlp.experts.207.gate_proj", + "model.layers.77.mlp.experts.207.up_proj", + "model.layers.77.mlp.experts.208.down_proj", + "model.layers.77.mlp.experts.208.gate_proj", + "model.layers.77.mlp.experts.208.up_proj", + "model.layers.77.mlp.experts.209.down_proj", + "model.layers.77.mlp.experts.209.gate_proj", + "model.layers.77.mlp.experts.209.up_proj", + "model.layers.77.mlp.experts.21.down_proj", + "model.layers.77.mlp.experts.21.gate_proj", + "model.layers.77.mlp.experts.21.up_proj", + "model.layers.77.mlp.experts.210.down_proj", + "model.layers.77.mlp.experts.210.gate_proj", + "model.layers.77.mlp.experts.210.up_proj", + "model.layers.77.mlp.experts.211.down_proj", + "model.layers.77.mlp.experts.211.gate_proj", + "model.layers.77.mlp.experts.211.up_proj", + "model.layers.77.mlp.experts.212.down_proj", + "model.layers.77.mlp.experts.212.gate_proj", + "model.layers.77.mlp.experts.212.up_proj", + "model.layers.77.mlp.experts.213.down_proj", + "model.layers.77.mlp.experts.213.gate_proj", + "model.layers.77.mlp.experts.213.up_proj", + "model.layers.77.mlp.experts.214.down_proj", + "model.layers.77.mlp.experts.214.gate_proj", + "model.layers.77.mlp.experts.214.up_proj", + "model.layers.77.mlp.experts.215.down_proj", + "model.layers.77.mlp.experts.215.gate_proj", + "model.layers.77.mlp.experts.215.up_proj", + "model.layers.77.mlp.experts.216.down_proj", + "model.layers.77.mlp.experts.216.gate_proj", + "model.layers.77.mlp.experts.216.up_proj", + "model.layers.77.mlp.experts.217.down_proj", + "model.layers.77.mlp.experts.217.gate_proj", + "model.layers.77.mlp.experts.217.up_proj", + "model.layers.77.mlp.experts.218.down_proj", + "model.layers.77.mlp.experts.218.gate_proj", + "model.layers.77.mlp.experts.218.up_proj", + "model.layers.77.mlp.experts.219.down_proj", + "model.layers.77.mlp.experts.219.gate_proj", + "model.layers.77.mlp.experts.219.up_proj", + "model.layers.77.mlp.experts.22.down_proj", + "model.layers.77.mlp.experts.22.gate_proj", + "model.layers.77.mlp.experts.22.up_proj", + "model.layers.77.mlp.experts.220.down_proj", + "model.layers.77.mlp.experts.220.gate_proj", + "model.layers.77.mlp.experts.220.up_proj", + "model.layers.77.mlp.experts.221.down_proj", + "model.layers.77.mlp.experts.221.gate_proj", + "model.layers.77.mlp.experts.221.up_proj", + "model.layers.77.mlp.experts.222.down_proj", + "model.layers.77.mlp.experts.222.gate_proj", + "model.layers.77.mlp.experts.222.up_proj", + "model.layers.77.mlp.experts.223.down_proj", + "model.layers.77.mlp.experts.223.gate_proj", + "model.layers.77.mlp.experts.223.up_proj", + "model.layers.77.mlp.experts.224.down_proj", + "model.layers.77.mlp.experts.224.gate_proj", + "model.layers.77.mlp.experts.224.up_proj", + "model.layers.77.mlp.experts.225.down_proj", + "model.layers.77.mlp.experts.225.gate_proj", + "model.layers.77.mlp.experts.225.up_proj", + "model.layers.77.mlp.experts.226.down_proj", + "model.layers.77.mlp.experts.226.gate_proj", + "model.layers.77.mlp.experts.226.up_proj", + "model.layers.77.mlp.experts.227.down_proj", + "model.layers.77.mlp.experts.227.gate_proj", + "model.layers.77.mlp.experts.227.up_proj", + "model.layers.77.mlp.experts.228.down_proj", + "model.layers.77.mlp.experts.228.gate_proj", + "model.layers.77.mlp.experts.228.up_proj", + "model.layers.77.mlp.experts.229.down_proj", + "model.layers.77.mlp.experts.229.gate_proj", + "model.layers.77.mlp.experts.229.up_proj", + "model.layers.77.mlp.experts.23.down_proj", + "model.layers.77.mlp.experts.23.gate_proj", + "model.layers.77.mlp.experts.23.up_proj", + "model.layers.77.mlp.experts.230.down_proj", + "model.layers.77.mlp.experts.230.gate_proj", + "model.layers.77.mlp.experts.230.up_proj", + "model.layers.77.mlp.experts.231.down_proj", + "model.layers.77.mlp.experts.231.gate_proj", + "model.layers.77.mlp.experts.231.up_proj", + "model.layers.77.mlp.experts.232.down_proj", + "model.layers.77.mlp.experts.232.gate_proj", + "model.layers.77.mlp.experts.232.up_proj", + "model.layers.77.mlp.experts.233.down_proj", + "model.layers.77.mlp.experts.233.gate_proj", + "model.layers.77.mlp.experts.233.up_proj", + "model.layers.77.mlp.experts.234.down_proj", + "model.layers.77.mlp.experts.234.gate_proj", + "model.layers.77.mlp.experts.234.up_proj", + "model.layers.77.mlp.experts.235.down_proj", + "model.layers.77.mlp.experts.235.gate_proj", + "model.layers.77.mlp.experts.235.up_proj", + "model.layers.77.mlp.experts.236.down_proj", + "model.layers.77.mlp.experts.236.gate_proj", + "model.layers.77.mlp.experts.236.up_proj", + "model.layers.77.mlp.experts.237.down_proj", + "model.layers.77.mlp.experts.237.gate_proj", + "model.layers.77.mlp.experts.237.up_proj", + "model.layers.77.mlp.experts.238.down_proj", + "model.layers.77.mlp.experts.238.gate_proj", + "model.layers.77.mlp.experts.238.up_proj", + "model.layers.77.mlp.experts.239.down_proj", + "model.layers.77.mlp.experts.239.gate_proj", + "model.layers.77.mlp.experts.239.up_proj", + "model.layers.77.mlp.experts.24.down_proj", + "model.layers.77.mlp.experts.24.gate_proj", + "model.layers.77.mlp.experts.24.up_proj", + "model.layers.77.mlp.experts.240.down_proj", + "model.layers.77.mlp.experts.240.gate_proj", + "model.layers.77.mlp.experts.240.up_proj", + "model.layers.77.mlp.experts.241.down_proj", + "model.layers.77.mlp.experts.241.gate_proj", + "model.layers.77.mlp.experts.241.up_proj", + "model.layers.77.mlp.experts.242.down_proj", + "model.layers.77.mlp.experts.242.gate_proj", + "model.layers.77.mlp.experts.242.up_proj", + "model.layers.77.mlp.experts.243.down_proj", + "model.layers.77.mlp.experts.243.gate_proj", + "model.layers.77.mlp.experts.243.up_proj", + "model.layers.77.mlp.experts.244.down_proj", + "model.layers.77.mlp.experts.244.gate_proj", + "model.layers.77.mlp.experts.244.up_proj", + "model.layers.77.mlp.experts.245.down_proj", + "model.layers.77.mlp.experts.245.gate_proj", + "model.layers.77.mlp.experts.245.up_proj", + "model.layers.77.mlp.experts.246.down_proj", + "model.layers.77.mlp.experts.246.gate_proj", + "model.layers.77.mlp.experts.246.up_proj", + "model.layers.77.mlp.experts.247.down_proj", + "model.layers.77.mlp.experts.247.gate_proj", + "model.layers.77.mlp.experts.247.up_proj", + "model.layers.77.mlp.experts.248.down_proj", + "model.layers.77.mlp.experts.248.gate_proj", + "model.layers.77.mlp.experts.248.up_proj", + "model.layers.77.mlp.experts.249.down_proj", + "model.layers.77.mlp.experts.249.gate_proj", + "model.layers.77.mlp.experts.249.up_proj", + "model.layers.77.mlp.experts.25.down_proj", + "model.layers.77.mlp.experts.25.gate_proj", + "model.layers.77.mlp.experts.25.up_proj", + "model.layers.77.mlp.experts.250.down_proj", + "model.layers.77.mlp.experts.250.gate_proj", + "model.layers.77.mlp.experts.250.up_proj", + "model.layers.77.mlp.experts.251.down_proj", + "model.layers.77.mlp.experts.251.gate_proj", + "model.layers.77.mlp.experts.251.up_proj", + "model.layers.77.mlp.experts.252.down_proj", + "model.layers.77.mlp.experts.252.gate_proj", + "model.layers.77.mlp.experts.252.up_proj", + "model.layers.77.mlp.experts.253.down_proj", + "model.layers.77.mlp.experts.253.gate_proj", + "model.layers.77.mlp.experts.253.up_proj", + "model.layers.77.mlp.experts.254.down_proj", + "model.layers.77.mlp.experts.254.gate_proj", + "model.layers.77.mlp.experts.254.up_proj", + "model.layers.77.mlp.experts.255.down_proj", + "model.layers.77.mlp.experts.255.gate_proj", + "model.layers.77.mlp.experts.255.up_proj", + "model.layers.77.mlp.experts.26.down_proj", + "model.layers.77.mlp.experts.26.gate_proj", + "model.layers.77.mlp.experts.26.up_proj", + "model.layers.77.mlp.experts.27.down_proj", + "model.layers.77.mlp.experts.27.gate_proj", + "model.layers.77.mlp.experts.27.up_proj", + "model.layers.77.mlp.experts.28.down_proj", + "model.layers.77.mlp.experts.28.gate_proj", + "model.layers.77.mlp.experts.28.up_proj", + "model.layers.77.mlp.experts.29.down_proj", + "model.layers.77.mlp.experts.29.gate_proj", + "model.layers.77.mlp.experts.29.up_proj", + "model.layers.77.mlp.experts.3.down_proj", + "model.layers.77.mlp.experts.3.gate_proj", + "model.layers.77.mlp.experts.3.up_proj", + "model.layers.77.mlp.experts.30.down_proj", + "model.layers.77.mlp.experts.30.gate_proj", + "model.layers.77.mlp.experts.30.up_proj", + "model.layers.77.mlp.experts.31.down_proj", + "model.layers.77.mlp.experts.31.gate_proj", + "model.layers.77.mlp.experts.31.up_proj", + "model.layers.77.mlp.experts.32.down_proj", + "model.layers.77.mlp.experts.32.gate_proj", + "model.layers.77.mlp.experts.32.up_proj", + "model.layers.77.mlp.experts.33.down_proj", + "model.layers.77.mlp.experts.33.gate_proj", + "model.layers.77.mlp.experts.33.up_proj", + "model.layers.77.mlp.experts.34.down_proj", + "model.layers.77.mlp.experts.34.gate_proj", + "model.layers.77.mlp.experts.34.up_proj", + "model.layers.77.mlp.experts.35.down_proj", + "model.layers.77.mlp.experts.35.gate_proj", + "model.layers.77.mlp.experts.35.up_proj", + "model.layers.77.mlp.experts.36.down_proj", + "model.layers.77.mlp.experts.36.gate_proj", + "model.layers.77.mlp.experts.36.up_proj", + "model.layers.77.mlp.experts.37.down_proj", + "model.layers.77.mlp.experts.37.gate_proj", + "model.layers.77.mlp.experts.37.up_proj", + "model.layers.77.mlp.experts.38.down_proj", + "model.layers.77.mlp.experts.38.gate_proj", + "model.layers.77.mlp.experts.38.up_proj", + "model.layers.77.mlp.experts.39.down_proj", + "model.layers.77.mlp.experts.39.gate_proj", + "model.layers.77.mlp.experts.39.up_proj", + "model.layers.77.mlp.experts.4.down_proj", + "model.layers.77.mlp.experts.4.gate_proj", + "model.layers.77.mlp.experts.4.up_proj", + "model.layers.77.mlp.experts.40.down_proj", + "model.layers.77.mlp.experts.40.gate_proj", + "model.layers.77.mlp.experts.40.up_proj", + "model.layers.77.mlp.experts.41.down_proj", + "model.layers.77.mlp.experts.41.gate_proj", + "model.layers.77.mlp.experts.41.up_proj", + "model.layers.77.mlp.experts.42.down_proj", + "model.layers.77.mlp.experts.42.gate_proj", + "model.layers.77.mlp.experts.42.up_proj", + "model.layers.77.mlp.experts.43.down_proj", + "model.layers.77.mlp.experts.43.gate_proj", + "model.layers.77.mlp.experts.43.up_proj", + "model.layers.77.mlp.experts.44.down_proj", + "model.layers.77.mlp.experts.44.gate_proj", + "model.layers.77.mlp.experts.44.up_proj", + "model.layers.77.mlp.experts.45.down_proj", + "model.layers.77.mlp.experts.45.gate_proj", + "model.layers.77.mlp.experts.45.up_proj", + "model.layers.77.mlp.experts.46.down_proj", + "model.layers.77.mlp.experts.46.gate_proj", + "model.layers.77.mlp.experts.46.up_proj", + "model.layers.77.mlp.experts.47.down_proj", + "model.layers.77.mlp.experts.47.gate_proj", + "model.layers.77.mlp.experts.47.up_proj", + "model.layers.77.mlp.experts.48.down_proj", + "model.layers.77.mlp.experts.48.gate_proj", + "model.layers.77.mlp.experts.48.up_proj", + "model.layers.77.mlp.experts.49.down_proj", + "model.layers.77.mlp.experts.49.gate_proj", + "model.layers.77.mlp.experts.49.up_proj", + "model.layers.77.mlp.experts.5.down_proj", + "model.layers.77.mlp.experts.5.gate_proj", + "model.layers.77.mlp.experts.5.up_proj", + "model.layers.77.mlp.experts.50.down_proj", + "model.layers.77.mlp.experts.50.gate_proj", + "model.layers.77.mlp.experts.50.up_proj", + "model.layers.77.mlp.experts.51.down_proj", + "model.layers.77.mlp.experts.51.gate_proj", + "model.layers.77.mlp.experts.51.up_proj", + "model.layers.77.mlp.experts.52.down_proj", + "model.layers.77.mlp.experts.52.gate_proj", + "model.layers.77.mlp.experts.52.up_proj", + "model.layers.77.mlp.experts.53.down_proj", + "model.layers.77.mlp.experts.53.gate_proj", + "model.layers.77.mlp.experts.53.up_proj", + "model.layers.77.mlp.experts.54.down_proj", + "model.layers.77.mlp.experts.54.gate_proj", + "model.layers.77.mlp.experts.54.up_proj", + "model.layers.77.mlp.experts.55.down_proj", + "model.layers.77.mlp.experts.55.gate_proj", + "model.layers.77.mlp.experts.55.up_proj", + "model.layers.77.mlp.experts.56.down_proj", + "model.layers.77.mlp.experts.56.gate_proj", + "model.layers.77.mlp.experts.56.up_proj", + "model.layers.77.mlp.experts.57.down_proj", + "model.layers.77.mlp.experts.57.gate_proj", + "model.layers.77.mlp.experts.57.up_proj", + "model.layers.77.mlp.experts.58.down_proj", + "model.layers.77.mlp.experts.58.gate_proj", + "model.layers.77.mlp.experts.58.up_proj", + "model.layers.77.mlp.experts.59.down_proj", + "model.layers.77.mlp.experts.59.gate_proj", + "model.layers.77.mlp.experts.59.up_proj", + "model.layers.77.mlp.experts.6.down_proj", + "model.layers.77.mlp.experts.6.gate_proj", + "model.layers.77.mlp.experts.6.up_proj", + "model.layers.77.mlp.experts.60.down_proj", + "model.layers.77.mlp.experts.60.gate_proj", + "model.layers.77.mlp.experts.60.up_proj", + "model.layers.77.mlp.experts.61.down_proj", + "model.layers.77.mlp.experts.61.gate_proj", + "model.layers.77.mlp.experts.61.up_proj", + "model.layers.77.mlp.experts.62.down_proj", + "model.layers.77.mlp.experts.62.gate_proj", + "model.layers.77.mlp.experts.62.up_proj", + "model.layers.77.mlp.experts.63.down_proj", + "model.layers.77.mlp.experts.63.gate_proj", + "model.layers.77.mlp.experts.63.up_proj", + "model.layers.77.mlp.experts.64.down_proj", + "model.layers.77.mlp.experts.64.gate_proj", + "model.layers.77.mlp.experts.64.up_proj", + "model.layers.77.mlp.experts.65.down_proj", + "model.layers.77.mlp.experts.65.gate_proj", + "model.layers.77.mlp.experts.65.up_proj", + "model.layers.77.mlp.experts.66.down_proj", + "model.layers.77.mlp.experts.66.gate_proj", + "model.layers.77.mlp.experts.66.up_proj", + "model.layers.77.mlp.experts.67.down_proj", + "model.layers.77.mlp.experts.67.gate_proj", + "model.layers.77.mlp.experts.67.up_proj", + "model.layers.77.mlp.experts.68.down_proj", + "model.layers.77.mlp.experts.68.gate_proj", + "model.layers.77.mlp.experts.68.up_proj", + "model.layers.77.mlp.experts.69.down_proj", + "model.layers.77.mlp.experts.69.gate_proj", + "model.layers.77.mlp.experts.69.up_proj", + "model.layers.77.mlp.experts.7.down_proj", + "model.layers.77.mlp.experts.7.gate_proj", + "model.layers.77.mlp.experts.7.up_proj", + "model.layers.77.mlp.experts.70.down_proj", + "model.layers.77.mlp.experts.70.gate_proj", + "model.layers.77.mlp.experts.70.up_proj", + "model.layers.77.mlp.experts.71.down_proj", + "model.layers.77.mlp.experts.71.gate_proj", + "model.layers.77.mlp.experts.71.up_proj", + "model.layers.77.mlp.experts.72.down_proj", + "model.layers.77.mlp.experts.72.gate_proj", + "model.layers.77.mlp.experts.72.up_proj", + "model.layers.77.mlp.experts.73.down_proj", + "model.layers.77.mlp.experts.73.gate_proj", + "model.layers.77.mlp.experts.73.up_proj", + "model.layers.77.mlp.experts.74.down_proj", + "model.layers.77.mlp.experts.74.gate_proj", + "model.layers.77.mlp.experts.74.up_proj", + "model.layers.77.mlp.experts.75.down_proj", + "model.layers.77.mlp.experts.75.gate_proj", + "model.layers.77.mlp.experts.75.up_proj", + "model.layers.77.mlp.experts.76.down_proj", + "model.layers.77.mlp.experts.76.gate_proj", + "model.layers.77.mlp.experts.76.up_proj", + "model.layers.77.mlp.experts.77.down_proj", + "model.layers.77.mlp.experts.77.gate_proj", + "model.layers.77.mlp.experts.77.up_proj", + "model.layers.77.mlp.experts.78.down_proj", + "model.layers.77.mlp.experts.78.gate_proj", + "model.layers.77.mlp.experts.78.up_proj", + "model.layers.77.mlp.experts.79.down_proj", + "model.layers.77.mlp.experts.79.gate_proj", + "model.layers.77.mlp.experts.79.up_proj", + "model.layers.77.mlp.experts.8.down_proj", + "model.layers.77.mlp.experts.8.gate_proj", + "model.layers.77.mlp.experts.8.up_proj", + "model.layers.77.mlp.experts.80.down_proj", + "model.layers.77.mlp.experts.80.gate_proj", + "model.layers.77.mlp.experts.80.up_proj", + "model.layers.77.mlp.experts.81.down_proj", + "model.layers.77.mlp.experts.81.gate_proj", + "model.layers.77.mlp.experts.81.up_proj", + "model.layers.77.mlp.experts.82.down_proj", + "model.layers.77.mlp.experts.82.gate_proj", + "model.layers.77.mlp.experts.82.up_proj", + "model.layers.77.mlp.experts.83.down_proj", + "model.layers.77.mlp.experts.83.gate_proj", + "model.layers.77.mlp.experts.83.up_proj", + "model.layers.77.mlp.experts.84.down_proj", + "model.layers.77.mlp.experts.84.gate_proj", + "model.layers.77.mlp.experts.84.up_proj", + "model.layers.77.mlp.experts.85.down_proj", + "model.layers.77.mlp.experts.85.gate_proj", + "model.layers.77.mlp.experts.85.up_proj", + "model.layers.77.mlp.experts.86.down_proj", + "model.layers.77.mlp.experts.86.gate_proj", + "model.layers.77.mlp.experts.86.up_proj", + "model.layers.77.mlp.experts.87.down_proj", + "model.layers.77.mlp.experts.87.gate_proj", + "model.layers.77.mlp.experts.87.up_proj", + "model.layers.77.mlp.experts.88.down_proj", + "model.layers.77.mlp.experts.88.gate_proj", + "model.layers.77.mlp.experts.88.up_proj", + "model.layers.77.mlp.experts.89.down_proj", + "model.layers.77.mlp.experts.89.gate_proj", + "model.layers.77.mlp.experts.89.up_proj", + "model.layers.77.mlp.experts.9.down_proj", + "model.layers.77.mlp.experts.9.gate_proj", + "model.layers.77.mlp.experts.9.up_proj", + "model.layers.77.mlp.experts.90.down_proj", + "model.layers.77.mlp.experts.90.gate_proj", + "model.layers.77.mlp.experts.90.up_proj", + "model.layers.77.mlp.experts.91.down_proj", + "model.layers.77.mlp.experts.91.gate_proj", + "model.layers.77.mlp.experts.91.up_proj", + "model.layers.77.mlp.experts.92.down_proj", + "model.layers.77.mlp.experts.92.gate_proj", + "model.layers.77.mlp.experts.92.up_proj", + "model.layers.77.mlp.experts.93.down_proj", + "model.layers.77.mlp.experts.93.gate_proj", + "model.layers.77.mlp.experts.93.up_proj", + "model.layers.77.mlp.experts.94.down_proj", + "model.layers.77.mlp.experts.94.gate_proj", + "model.layers.77.mlp.experts.94.up_proj", + "model.layers.77.mlp.experts.95.down_proj", + "model.layers.77.mlp.experts.95.gate_proj", + "model.layers.77.mlp.experts.95.up_proj", + "model.layers.77.mlp.experts.96.down_proj", + "model.layers.77.mlp.experts.96.gate_proj", + "model.layers.77.mlp.experts.96.up_proj", + "model.layers.77.mlp.experts.97.down_proj", + "model.layers.77.mlp.experts.97.gate_proj", + "model.layers.77.mlp.experts.97.up_proj", + "model.layers.77.mlp.experts.98.down_proj", + "model.layers.77.mlp.experts.98.gate_proj", + "model.layers.77.mlp.experts.98.up_proj", + "model.layers.77.mlp.experts.99.down_proj", + "model.layers.77.mlp.experts.99.gate_proj", + "model.layers.77.mlp.experts.99.up_proj", + "model.layers.78.mlp.experts.0.down_proj", + "model.layers.78.mlp.experts.0.gate_proj", + "model.layers.78.mlp.experts.0.up_proj", + "model.layers.78.mlp.experts.1.down_proj", + "model.layers.78.mlp.experts.1.gate_proj", + "model.layers.78.mlp.experts.1.up_proj", + "model.layers.78.mlp.experts.10.down_proj", + "model.layers.78.mlp.experts.10.gate_proj", + "model.layers.78.mlp.experts.10.up_proj", + "model.layers.78.mlp.experts.100.down_proj", + "model.layers.78.mlp.experts.100.gate_proj", + "model.layers.78.mlp.experts.100.up_proj", + "model.layers.78.mlp.experts.101.down_proj", + "model.layers.78.mlp.experts.101.gate_proj", + "model.layers.78.mlp.experts.101.up_proj", + "model.layers.78.mlp.experts.102.down_proj", + "model.layers.78.mlp.experts.102.gate_proj", + "model.layers.78.mlp.experts.102.up_proj", + "model.layers.78.mlp.experts.103.down_proj", + "model.layers.78.mlp.experts.103.gate_proj", + "model.layers.78.mlp.experts.103.up_proj", + "model.layers.78.mlp.experts.104.down_proj", + "model.layers.78.mlp.experts.104.gate_proj", + "model.layers.78.mlp.experts.104.up_proj", + "model.layers.78.mlp.experts.105.down_proj", + "model.layers.78.mlp.experts.105.gate_proj", + "model.layers.78.mlp.experts.105.up_proj", + "model.layers.78.mlp.experts.106.down_proj", + "model.layers.78.mlp.experts.106.gate_proj", + "model.layers.78.mlp.experts.106.up_proj", + "model.layers.78.mlp.experts.107.down_proj", + "model.layers.78.mlp.experts.107.gate_proj", + "model.layers.78.mlp.experts.107.up_proj", + "model.layers.78.mlp.experts.108.down_proj", + "model.layers.78.mlp.experts.108.gate_proj", + "model.layers.78.mlp.experts.108.up_proj", + "model.layers.78.mlp.experts.109.down_proj", + "model.layers.78.mlp.experts.109.gate_proj", + "model.layers.78.mlp.experts.109.up_proj", + "model.layers.78.mlp.experts.11.down_proj", + "model.layers.78.mlp.experts.11.gate_proj", + "model.layers.78.mlp.experts.11.up_proj", + "model.layers.78.mlp.experts.110.down_proj", + "model.layers.78.mlp.experts.110.gate_proj", + "model.layers.78.mlp.experts.110.up_proj", + "model.layers.78.mlp.experts.111.down_proj", + "model.layers.78.mlp.experts.111.gate_proj", + "model.layers.78.mlp.experts.111.up_proj", + "model.layers.78.mlp.experts.112.down_proj", + "model.layers.78.mlp.experts.112.gate_proj", + "model.layers.78.mlp.experts.112.up_proj", + "model.layers.78.mlp.experts.113.down_proj", + "model.layers.78.mlp.experts.113.gate_proj", + "model.layers.78.mlp.experts.113.up_proj", + "model.layers.78.mlp.experts.114.down_proj", + "model.layers.78.mlp.experts.114.gate_proj", + "model.layers.78.mlp.experts.114.up_proj", + "model.layers.78.mlp.experts.115.down_proj", + "model.layers.78.mlp.experts.115.gate_proj", + "model.layers.78.mlp.experts.115.up_proj", + "model.layers.78.mlp.experts.116.down_proj", + "model.layers.78.mlp.experts.116.gate_proj", + "model.layers.78.mlp.experts.116.up_proj", + "model.layers.78.mlp.experts.117.down_proj", + "model.layers.78.mlp.experts.117.gate_proj", + "model.layers.78.mlp.experts.117.up_proj", + "model.layers.78.mlp.experts.118.down_proj", + "model.layers.78.mlp.experts.118.gate_proj", + "model.layers.78.mlp.experts.118.up_proj", + "model.layers.78.mlp.experts.119.down_proj", + "model.layers.78.mlp.experts.119.gate_proj", + "model.layers.78.mlp.experts.119.up_proj", + "model.layers.78.mlp.experts.12.down_proj", + "model.layers.78.mlp.experts.12.gate_proj", + "model.layers.78.mlp.experts.12.up_proj", + "model.layers.78.mlp.experts.120.down_proj", + "model.layers.78.mlp.experts.120.gate_proj", + "model.layers.78.mlp.experts.120.up_proj", + "model.layers.78.mlp.experts.121.down_proj", + "model.layers.78.mlp.experts.121.gate_proj", + "model.layers.78.mlp.experts.121.up_proj", + "model.layers.78.mlp.experts.122.down_proj", + "model.layers.78.mlp.experts.122.gate_proj", + "model.layers.78.mlp.experts.122.up_proj", + "model.layers.78.mlp.experts.123.down_proj", + "model.layers.78.mlp.experts.123.gate_proj", + "model.layers.78.mlp.experts.123.up_proj", + "model.layers.78.mlp.experts.124.down_proj", + "model.layers.78.mlp.experts.124.gate_proj", + "model.layers.78.mlp.experts.124.up_proj", + "model.layers.78.mlp.experts.125.down_proj", + "model.layers.78.mlp.experts.125.gate_proj", + "model.layers.78.mlp.experts.125.up_proj", + "model.layers.78.mlp.experts.126.down_proj", + "model.layers.78.mlp.experts.126.gate_proj", + "model.layers.78.mlp.experts.126.up_proj", + "model.layers.78.mlp.experts.127.down_proj", + "model.layers.78.mlp.experts.127.gate_proj", + "model.layers.78.mlp.experts.127.up_proj", + "model.layers.78.mlp.experts.128.down_proj", + "model.layers.78.mlp.experts.128.gate_proj", + "model.layers.78.mlp.experts.128.up_proj", + "model.layers.78.mlp.experts.129.down_proj", + "model.layers.78.mlp.experts.129.gate_proj", + "model.layers.78.mlp.experts.129.up_proj", + "model.layers.78.mlp.experts.13.down_proj", + "model.layers.78.mlp.experts.13.gate_proj", + "model.layers.78.mlp.experts.13.up_proj", + "model.layers.78.mlp.experts.130.down_proj", + "model.layers.78.mlp.experts.130.gate_proj", + "model.layers.78.mlp.experts.130.up_proj", + "model.layers.78.mlp.experts.131.down_proj", + "model.layers.78.mlp.experts.131.gate_proj", + "model.layers.78.mlp.experts.131.up_proj", + "model.layers.78.mlp.experts.132.down_proj", + "model.layers.78.mlp.experts.132.gate_proj", + "model.layers.78.mlp.experts.132.up_proj", + "model.layers.78.mlp.experts.133.down_proj", + "model.layers.78.mlp.experts.133.gate_proj", + "model.layers.78.mlp.experts.133.up_proj", + "model.layers.78.mlp.experts.134.down_proj", + "model.layers.78.mlp.experts.134.gate_proj", + "model.layers.78.mlp.experts.134.up_proj", + "model.layers.78.mlp.experts.135.down_proj", + "model.layers.78.mlp.experts.135.gate_proj", + "model.layers.78.mlp.experts.135.up_proj", + "model.layers.78.mlp.experts.136.down_proj", + "model.layers.78.mlp.experts.136.gate_proj", + "model.layers.78.mlp.experts.136.up_proj", + "model.layers.78.mlp.experts.137.down_proj", + "model.layers.78.mlp.experts.137.gate_proj", + "model.layers.78.mlp.experts.137.up_proj", + "model.layers.78.mlp.experts.138.down_proj", + "model.layers.78.mlp.experts.138.gate_proj", + "model.layers.78.mlp.experts.138.up_proj", + "model.layers.78.mlp.experts.139.down_proj", + "model.layers.78.mlp.experts.139.gate_proj", + "model.layers.78.mlp.experts.139.up_proj", + "model.layers.78.mlp.experts.14.down_proj", + "model.layers.78.mlp.experts.14.gate_proj", + "model.layers.78.mlp.experts.14.up_proj", + "model.layers.78.mlp.experts.140.down_proj", + "model.layers.78.mlp.experts.140.gate_proj", + "model.layers.78.mlp.experts.140.up_proj", + "model.layers.78.mlp.experts.141.down_proj", + "model.layers.78.mlp.experts.141.gate_proj", + "model.layers.78.mlp.experts.141.up_proj", + "model.layers.78.mlp.experts.142.down_proj", + "model.layers.78.mlp.experts.142.gate_proj", + "model.layers.78.mlp.experts.142.up_proj", + "model.layers.78.mlp.experts.143.down_proj", + "model.layers.78.mlp.experts.143.gate_proj", + "model.layers.78.mlp.experts.143.up_proj", + "model.layers.78.mlp.experts.144.down_proj", + "model.layers.78.mlp.experts.144.gate_proj", + "model.layers.78.mlp.experts.144.up_proj", + "model.layers.78.mlp.experts.145.down_proj", + "model.layers.78.mlp.experts.145.gate_proj", + "model.layers.78.mlp.experts.145.up_proj", + "model.layers.78.mlp.experts.146.down_proj", + "model.layers.78.mlp.experts.146.gate_proj", + "model.layers.78.mlp.experts.146.up_proj", + "model.layers.78.mlp.experts.147.down_proj", + "model.layers.78.mlp.experts.147.gate_proj", + "model.layers.78.mlp.experts.147.up_proj", + "model.layers.78.mlp.experts.148.down_proj", + "model.layers.78.mlp.experts.148.gate_proj", + "model.layers.78.mlp.experts.148.up_proj", + "model.layers.78.mlp.experts.149.down_proj", + "model.layers.78.mlp.experts.149.gate_proj", + "model.layers.78.mlp.experts.149.up_proj", + "model.layers.78.mlp.experts.15.down_proj", + "model.layers.78.mlp.experts.15.gate_proj", + "model.layers.78.mlp.experts.15.up_proj", + "model.layers.78.mlp.experts.150.down_proj", + "model.layers.78.mlp.experts.150.gate_proj", + "model.layers.78.mlp.experts.150.up_proj", + "model.layers.78.mlp.experts.151.down_proj", + "model.layers.78.mlp.experts.151.gate_proj", + "model.layers.78.mlp.experts.151.up_proj", + "model.layers.78.mlp.experts.152.down_proj", + "model.layers.78.mlp.experts.152.gate_proj", + "model.layers.78.mlp.experts.152.up_proj", + "model.layers.78.mlp.experts.153.down_proj", + "model.layers.78.mlp.experts.153.gate_proj", + "model.layers.78.mlp.experts.153.up_proj", + "model.layers.78.mlp.experts.154.down_proj", + "model.layers.78.mlp.experts.154.gate_proj", + "model.layers.78.mlp.experts.154.up_proj", + "model.layers.78.mlp.experts.155.down_proj", + "model.layers.78.mlp.experts.155.gate_proj", + "model.layers.78.mlp.experts.155.up_proj", + "model.layers.78.mlp.experts.156.down_proj", + "model.layers.78.mlp.experts.156.gate_proj", + "model.layers.78.mlp.experts.156.up_proj", + "model.layers.78.mlp.experts.157.down_proj", + "model.layers.78.mlp.experts.157.gate_proj", + "model.layers.78.mlp.experts.157.up_proj", + "model.layers.78.mlp.experts.158.down_proj", + "model.layers.78.mlp.experts.158.gate_proj", + "model.layers.78.mlp.experts.158.up_proj", + "model.layers.78.mlp.experts.159.down_proj", + "model.layers.78.mlp.experts.159.gate_proj", + "model.layers.78.mlp.experts.159.up_proj", + "model.layers.78.mlp.experts.16.down_proj", + "model.layers.78.mlp.experts.16.gate_proj", + "model.layers.78.mlp.experts.16.up_proj", + "model.layers.78.mlp.experts.160.down_proj", + "model.layers.78.mlp.experts.160.gate_proj", + "model.layers.78.mlp.experts.160.up_proj", + "model.layers.78.mlp.experts.161.down_proj", + "model.layers.78.mlp.experts.161.gate_proj", + "model.layers.78.mlp.experts.161.up_proj", + "model.layers.78.mlp.experts.162.down_proj", + "model.layers.78.mlp.experts.162.gate_proj", + "model.layers.78.mlp.experts.162.up_proj", + "model.layers.78.mlp.experts.163.down_proj", + "model.layers.78.mlp.experts.163.gate_proj", + "model.layers.78.mlp.experts.163.up_proj", + "model.layers.78.mlp.experts.164.down_proj", + "model.layers.78.mlp.experts.164.gate_proj", + "model.layers.78.mlp.experts.164.up_proj", + "model.layers.78.mlp.experts.165.down_proj", + "model.layers.78.mlp.experts.165.gate_proj", + "model.layers.78.mlp.experts.165.up_proj", + "model.layers.78.mlp.experts.166.down_proj", + "model.layers.78.mlp.experts.166.gate_proj", + "model.layers.78.mlp.experts.166.up_proj", + "model.layers.78.mlp.experts.167.down_proj", + "model.layers.78.mlp.experts.167.gate_proj", + "model.layers.78.mlp.experts.167.up_proj", + "model.layers.78.mlp.experts.168.down_proj", + "model.layers.78.mlp.experts.168.gate_proj", + "model.layers.78.mlp.experts.168.up_proj", + "model.layers.78.mlp.experts.169.down_proj", + "model.layers.78.mlp.experts.169.gate_proj", + "model.layers.78.mlp.experts.169.up_proj", + "model.layers.78.mlp.experts.17.down_proj", + "model.layers.78.mlp.experts.17.gate_proj", + "model.layers.78.mlp.experts.17.up_proj", + "model.layers.78.mlp.experts.170.down_proj", + "model.layers.78.mlp.experts.170.gate_proj", + "model.layers.78.mlp.experts.170.up_proj", + "model.layers.78.mlp.experts.171.down_proj", + "model.layers.78.mlp.experts.171.gate_proj", + "model.layers.78.mlp.experts.171.up_proj", + "model.layers.78.mlp.experts.172.down_proj", + "model.layers.78.mlp.experts.172.gate_proj", + "model.layers.78.mlp.experts.172.up_proj", + "model.layers.78.mlp.experts.173.down_proj", + "model.layers.78.mlp.experts.173.gate_proj", + "model.layers.78.mlp.experts.173.up_proj", + "model.layers.78.mlp.experts.174.down_proj", + "model.layers.78.mlp.experts.174.gate_proj", + "model.layers.78.mlp.experts.174.up_proj", + "model.layers.78.mlp.experts.175.down_proj", + "model.layers.78.mlp.experts.175.gate_proj", + "model.layers.78.mlp.experts.175.up_proj", + "model.layers.78.mlp.experts.176.down_proj", + "model.layers.78.mlp.experts.176.gate_proj", + "model.layers.78.mlp.experts.176.up_proj", + "model.layers.78.mlp.experts.177.down_proj", + "model.layers.78.mlp.experts.177.gate_proj", + "model.layers.78.mlp.experts.177.up_proj", + "model.layers.78.mlp.experts.178.down_proj", + "model.layers.78.mlp.experts.178.gate_proj", + "model.layers.78.mlp.experts.178.up_proj", + "model.layers.78.mlp.experts.179.down_proj", + "model.layers.78.mlp.experts.179.gate_proj", + "model.layers.78.mlp.experts.179.up_proj", + "model.layers.78.mlp.experts.18.down_proj", + "model.layers.78.mlp.experts.18.gate_proj", + "model.layers.78.mlp.experts.18.up_proj", + "model.layers.78.mlp.experts.180.down_proj", + "model.layers.78.mlp.experts.180.gate_proj", + "model.layers.78.mlp.experts.180.up_proj", + "model.layers.78.mlp.experts.181.down_proj", + "model.layers.78.mlp.experts.181.gate_proj", + "model.layers.78.mlp.experts.181.up_proj", + "model.layers.78.mlp.experts.182.down_proj", + "model.layers.78.mlp.experts.182.gate_proj", + "model.layers.78.mlp.experts.182.up_proj", + "model.layers.78.mlp.experts.183.down_proj", + "model.layers.78.mlp.experts.183.gate_proj", + "model.layers.78.mlp.experts.183.up_proj", + "model.layers.78.mlp.experts.184.down_proj", + "model.layers.78.mlp.experts.184.gate_proj", + "model.layers.78.mlp.experts.184.up_proj", + "model.layers.78.mlp.experts.185.down_proj", + "model.layers.78.mlp.experts.185.gate_proj", + "model.layers.78.mlp.experts.185.up_proj", + "model.layers.78.mlp.experts.186.down_proj", + "model.layers.78.mlp.experts.186.gate_proj", + "model.layers.78.mlp.experts.186.up_proj", + "model.layers.78.mlp.experts.187.down_proj", + "model.layers.78.mlp.experts.187.gate_proj", + "model.layers.78.mlp.experts.187.up_proj", + "model.layers.78.mlp.experts.188.down_proj", + "model.layers.78.mlp.experts.188.gate_proj", + "model.layers.78.mlp.experts.188.up_proj", + "model.layers.78.mlp.experts.189.down_proj", + "model.layers.78.mlp.experts.189.gate_proj", + "model.layers.78.mlp.experts.189.up_proj", + "model.layers.78.mlp.experts.19.down_proj", + "model.layers.78.mlp.experts.19.gate_proj", + "model.layers.78.mlp.experts.19.up_proj", + "model.layers.78.mlp.experts.190.down_proj", + "model.layers.78.mlp.experts.190.gate_proj", + "model.layers.78.mlp.experts.190.up_proj", + "model.layers.78.mlp.experts.191.down_proj", + "model.layers.78.mlp.experts.191.gate_proj", + "model.layers.78.mlp.experts.191.up_proj", + "model.layers.78.mlp.experts.192.down_proj", + "model.layers.78.mlp.experts.192.gate_proj", + "model.layers.78.mlp.experts.192.up_proj", + "model.layers.78.mlp.experts.193.down_proj", + "model.layers.78.mlp.experts.193.gate_proj", + "model.layers.78.mlp.experts.193.up_proj", + "model.layers.78.mlp.experts.194.down_proj", + "model.layers.78.mlp.experts.194.gate_proj", + "model.layers.78.mlp.experts.194.up_proj", + "model.layers.78.mlp.experts.195.down_proj", + "model.layers.78.mlp.experts.195.gate_proj", + "model.layers.78.mlp.experts.195.up_proj", + "model.layers.78.mlp.experts.196.down_proj", + "model.layers.78.mlp.experts.196.gate_proj", + "model.layers.78.mlp.experts.196.up_proj", + "model.layers.78.mlp.experts.197.down_proj", + "model.layers.78.mlp.experts.197.gate_proj", + "model.layers.78.mlp.experts.197.up_proj", + "model.layers.78.mlp.experts.198.down_proj", + "model.layers.78.mlp.experts.198.gate_proj", + "model.layers.78.mlp.experts.198.up_proj", + "model.layers.78.mlp.experts.199.down_proj", + "model.layers.78.mlp.experts.199.gate_proj", + "model.layers.78.mlp.experts.199.up_proj", + "model.layers.78.mlp.experts.2.down_proj", + "model.layers.78.mlp.experts.2.gate_proj", + "model.layers.78.mlp.experts.2.up_proj", + "model.layers.78.mlp.experts.20.down_proj", + "model.layers.78.mlp.experts.20.gate_proj", + "model.layers.78.mlp.experts.20.up_proj", + "model.layers.78.mlp.experts.200.down_proj", + "model.layers.78.mlp.experts.200.gate_proj", + "model.layers.78.mlp.experts.200.up_proj", + "model.layers.78.mlp.experts.201.down_proj", + "model.layers.78.mlp.experts.201.gate_proj", + "model.layers.78.mlp.experts.201.up_proj", + "model.layers.78.mlp.experts.202.down_proj", + "model.layers.78.mlp.experts.202.gate_proj", + "model.layers.78.mlp.experts.202.up_proj", + "model.layers.78.mlp.experts.203.down_proj", + "model.layers.78.mlp.experts.203.gate_proj", + "model.layers.78.mlp.experts.203.up_proj", + "model.layers.78.mlp.experts.204.down_proj", + "model.layers.78.mlp.experts.204.gate_proj", + "model.layers.78.mlp.experts.204.up_proj", + "model.layers.78.mlp.experts.205.down_proj", + "model.layers.78.mlp.experts.205.gate_proj", + "model.layers.78.mlp.experts.205.up_proj", + "model.layers.78.mlp.experts.206.down_proj", + "model.layers.78.mlp.experts.206.gate_proj", + "model.layers.78.mlp.experts.206.up_proj", + "model.layers.78.mlp.experts.207.down_proj", + "model.layers.78.mlp.experts.207.gate_proj", + "model.layers.78.mlp.experts.207.up_proj", + "model.layers.78.mlp.experts.208.down_proj", + "model.layers.78.mlp.experts.208.gate_proj", + "model.layers.78.mlp.experts.208.up_proj", + "model.layers.78.mlp.experts.209.down_proj", + "model.layers.78.mlp.experts.209.gate_proj", + "model.layers.78.mlp.experts.209.up_proj", + "model.layers.78.mlp.experts.21.down_proj", + "model.layers.78.mlp.experts.21.gate_proj", + "model.layers.78.mlp.experts.21.up_proj", + "model.layers.78.mlp.experts.210.down_proj", + "model.layers.78.mlp.experts.210.gate_proj", + "model.layers.78.mlp.experts.210.up_proj", + "model.layers.78.mlp.experts.211.down_proj", + "model.layers.78.mlp.experts.211.gate_proj", + "model.layers.78.mlp.experts.211.up_proj", + "model.layers.78.mlp.experts.212.down_proj", + "model.layers.78.mlp.experts.212.gate_proj", + "model.layers.78.mlp.experts.212.up_proj", + "model.layers.78.mlp.experts.213.down_proj", + "model.layers.78.mlp.experts.213.gate_proj", + "model.layers.78.mlp.experts.213.up_proj", + "model.layers.78.mlp.experts.214.down_proj", + "model.layers.78.mlp.experts.214.gate_proj", + "model.layers.78.mlp.experts.214.up_proj", + "model.layers.78.mlp.experts.215.down_proj", + "model.layers.78.mlp.experts.215.gate_proj", + "model.layers.78.mlp.experts.215.up_proj", + "model.layers.78.mlp.experts.216.down_proj", + "model.layers.78.mlp.experts.216.gate_proj", + "model.layers.78.mlp.experts.216.up_proj", + "model.layers.78.mlp.experts.217.down_proj", + "model.layers.78.mlp.experts.217.gate_proj", + "model.layers.78.mlp.experts.217.up_proj", + "model.layers.78.mlp.experts.218.down_proj", + "model.layers.78.mlp.experts.218.gate_proj", + "model.layers.78.mlp.experts.218.up_proj", + "model.layers.78.mlp.experts.219.down_proj", + "model.layers.78.mlp.experts.219.gate_proj", + "model.layers.78.mlp.experts.219.up_proj", + "model.layers.78.mlp.experts.22.down_proj", + "model.layers.78.mlp.experts.22.gate_proj", + "model.layers.78.mlp.experts.22.up_proj", + "model.layers.78.mlp.experts.220.down_proj", + "model.layers.78.mlp.experts.220.gate_proj", + "model.layers.78.mlp.experts.220.up_proj", + "model.layers.78.mlp.experts.221.down_proj", + "model.layers.78.mlp.experts.221.gate_proj", + "model.layers.78.mlp.experts.221.up_proj", + "model.layers.78.mlp.experts.222.down_proj", + "model.layers.78.mlp.experts.222.gate_proj", + "model.layers.78.mlp.experts.222.up_proj", + "model.layers.78.mlp.experts.223.down_proj", + "model.layers.78.mlp.experts.223.gate_proj", + "model.layers.78.mlp.experts.223.up_proj", + "model.layers.78.mlp.experts.224.down_proj", + "model.layers.78.mlp.experts.224.gate_proj", + "model.layers.78.mlp.experts.224.up_proj", + "model.layers.78.mlp.experts.225.down_proj", + "model.layers.78.mlp.experts.225.gate_proj", + "model.layers.78.mlp.experts.225.up_proj", + "model.layers.78.mlp.experts.226.down_proj", + "model.layers.78.mlp.experts.226.gate_proj", + "model.layers.78.mlp.experts.226.up_proj", + "model.layers.78.mlp.experts.227.down_proj", + "model.layers.78.mlp.experts.227.gate_proj", + "model.layers.78.mlp.experts.227.up_proj", + "model.layers.78.mlp.experts.228.down_proj", + "model.layers.78.mlp.experts.228.gate_proj", + "model.layers.78.mlp.experts.228.up_proj", + "model.layers.78.mlp.experts.229.down_proj", + "model.layers.78.mlp.experts.229.gate_proj", + "model.layers.78.mlp.experts.229.up_proj", + "model.layers.78.mlp.experts.23.down_proj", + "model.layers.78.mlp.experts.23.gate_proj", + "model.layers.78.mlp.experts.23.up_proj", + "model.layers.78.mlp.experts.230.down_proj", + "model.layers.78.mlp.experts.230.gate_proj", + "model.layers.78.mlp.experts.230.up_proj", + "model.layers.78.mlp.experts.231.down_proj", + "model.layers.78.mlp.experts.231.gate_proj", + "model.layers.78.mlp.experts.231.up_proj", + "model.layers.78.mlp.experts.232.down_proj", + "model.layers.78.mlp.experts.232.gate_proj", + "model.layers.78.mlp.experts.232.up_proj", + "model.layers.78.mlp.experts.233.down_proj", + "model.layers.78.mlp.experts.233.gate_proj", + "model.layers.78.mlp.experts.233.up_proj", + "model.layers.78.mlp.experts.234.down_proj", + "model.layers.78.mlp.experts.234.gate_proj", + "model.layers.78.mlp.experts.234.up_proj", + "model.layers.78.mlp.experts.235.down_proj", + "model.layers.78.mlp.experts.235.gate_proj", + "model.layers.78.mlp.experts.235.up_proj", + "model.layers.78.mlp.experts.236.down_proj", + "model.layers.78.mlp.experts.236.gate_proj", + "model.layers.78.mlp.experts.236.up_proj", + "model.layers.78.mlp.experts.237.down_proj", + "model.layers.78.mlp.experts.237.gate_proj", + "model.layers.78.mlp.experts.237.up_proj", + "model.layers.78.mlp.experts.238.down_proj", + "model.layers.78.mlp.experts.238.gate_proj", + "model.layers.78.mlp.experts.238.up_proj", + "model.layers.78.mlp.experts.239.down_proj", + "model.layers.78.mlp.experts.239.gate_proj", + "model.layers.78.mlp.experts.239.up_proj", + "model.layers.78.mlp.experts.24.down_proj", + "model.layers.78.mlp.experts.24.gate_proj", + "model.layers.78.mlp.experts.24.up_proj", + "model.layers.78.mlp.experts.240.down_proj", + "model.layers.78.mlp.experts.240.gate_proj", + "model.layers.78.mlp.experts.240.up_proj", + "model.layers.78.mlp.experts.241.down_proj", + "model.layers.78.mlp.experts.241.gate_proj", + "model.layers.78.mlp.experts.241.up_proj", + "model.layers.78.mlp.experts.242.down_proj", + "model.layers.78.mlp.experts.242.gate_proj", + "model.layers.78.mlp.experts.242.up_proj", + "model.layers.78.mlp.experts.243.down_proj", + "model.layers.78.mlp.experts.243.gate_proj", + "model.layers.78.mlp.experts.243.up_proj", + "model.layers.78.mlp.experts.244.down_proj", + "model.layers.78.mlp.experts.244.gate_proj", + "model.layers.78.mlp.experts.244.up_proj", + "model.layers.78.mlp.experts.245.down_proj", + "model.layers.78.mlp.experts.245.gate_proj", + "model.layers.78.mlp.experts.245.up_proj", + "model.layers.78.mlp.experts.246.down_proj", + "model.layers.78.mlp.experts.246.gate_proj", + "model.layers.78.mlp.experts.246.up_proj", + "model.layers.78.mlp.experts.247.down_proj", + "model.layers.78.mlp.experts.247.gate_proj", + "model.layers.78.mlp.experts.247.up_proj", + "model.layers.78.mlp.experts.248.down_proj", + "model.layers.78.mlp.experts.248.gate_proj", + "model.layers.78.mlp.experts.248.up_proj", + "model.layers.78.mlp.experts.249.down_proj", + "model.layers.78.mlp.experts.249.gate_proj", + "model.layers.78.mlp.experts.249.up_proj", + "model.layers.78.mlp.experts.25.down_proj", + "model.layers.78.mlp.experts.25.gate_proj", + "model.layers.78.mlp.experts.25.up_proj", + "model.layers.78.mlp.experts.250.down_proj", + "model.layers.78.mlp.experts.250.gate_proj", + "model.layers.78.mlp.experts.250.up_proj", + "model.layers.78.mlp.experts.251.down_proj", + "model.layers.78.mlp.experts.251.gate_proj", + "model.layers.78.mlp.experts.251.up_proj", + "model.layers.78.mlp.experts.252.down_proj", + "model.layers.78.mlp.experts.252.gate_proj", + "model.layers.78.mlp.experts.252.up_proj", + "model.layers.78.mlp.experts.253.down_proj", + "model.layers.78.mlp.experts.253.gate_proj", + "model.layers.78.mlp.experts.253.up_proj", + "model.layers.78.mlp.experts.254.down_proj", + "model.layers.78.mlp.experts.254.gate_proj", + "model.layers.78.mlp.experts.254.up_proj", + "model.layers.78.mlp.experts.255.down_proj", + "model.layers.78.mlp.experts.255.gate_proj", + "model.layers.78.mlp.experts.255.up_proj", + "model.layers.78.mlp.experts.26.down_proj", + "model.layers.78.mlp.experts.26.gate_proj", + "model.layers.78.mlp.experts.26.up_proj", + "model.layers.78.mlp.experts.27.down_proj", + "model.layers.78.mlp.experts.27.gate_proj", + "model.layers.78.mlp.experts.27.up_proj", + "model.layers.78.mlp.experts.28.down_proj", + "model.layers.78.mlp.experts.28.gate_proj", + "model.layers.78.mlp.experts.28.up_proj", + "model.layers.78.mlp.experts.29.down_proj", + "model.layers.78.mlp.experts.29.gate_proj", + "model.layers.78.mlp.experts.29.up_proj", + "model.layers.78.mlp.experts.3.down_proj", + "model.layers.78.mlp.experts.3.gate_proj", + "model.layers.78.mlp.experts.3.up_proj", + "model.layers.78.mlp.experts.30.down_proj", + "model.layers.78.mlp.experts.30.gate_proj", + "model.layers.78.mlp.experts.30.up_proj", + "model.layers.78.mlp.experts.31.down_proj", + "model.layers.78.mlp.experts.31.gate_proj", + "model.layers.78.mlp.experts.31.up_proj", + "model.layers.78.mlp.experts.32.down_proj", + "model.layers.78.mlp.experts.32.gate_proj", + "model.layers.78.mlp.experts.32.up_proj", + "model.layers.78.mlp.experts.33.down_proj", + "model.layers.78.mlp.experts.33.gate_proj", + "model.layers.78.mlp.experts.33.up_proj", + "model.layers.78.mlp.experts.34.down_proj", + "model.layers.78.mlp.experts.34.gate_proj", + "model.layers.78.mlp.experts.34.up_proj", + "model.layers.78.mlp.experts.35.down_proj", + "model.layers.78.mlp.experts.35.gate_proj", + "model.layers.78.mlp.experts.35.up_proj", + "model.layers.78.mlp.experts.36.down_proj", + "model.layers.78.mlp.experts.36.gate_proj", + "model.layers.78.mlp.experts.36.up_proj", + "model.layers.78.mlp.experts.37.down_proj", + "model.layers.78.mlp.experts.37.gate_proj", + "model.layers.78.mlp.experts.37.up_proj", + "model.layers.78.mlp.experts.38.down_proj", + "model.layers.78.mlp.experts.38.gate_proj", + "model.layers.78.mlp.experts.38.up_proj", + "model.layers.78.mlp.experts.39.down_proj", + "model.layers.78.mlp.experts.39.gate_proj", + "model.layers.78.mlp.experts.39.up_proj", + "model.layers.78.mlp.experts.4.down_proj", + "model.layers.78.mlp.experts.4.gate_proj", + "model.layers.78.mlp.experts.4.up_proj", + "model.layers.78.mlp.experts.40.down_proj", + "model.layers.78.mlp.experts.40.gate_proj", + "model.layers.78.mlp.experts.40.up_proj", + "model.layers.78.mlp.experts.41.down_proj", + "model.layers.78.mlp.experts.41.gate_proj", + "model.layers.78.mlp.experts.41.up_proj", + "model.layers.78.mlp.experts.42.down_proj", + "model.layers.78.mlp.experts.42.gate_proj", + "model.layers.78.mlp.experts.42.up_proj", + "model.layers.78.mlp.experts.43.down_proj", + "model.layers.78.mlp.experts.43.gate_proj", + "model.layers.78.mlp.experts.43.up_proj", + "model.layers.78.mlp.experts.44.down_proj", + "model.layers.78.mlp.experts.44.gate_proj", + "model.layers.78.mlp.experts.44.up_proj", + "model.layers.78.mlp.experts.45.down_proj", + "model.layers.78.mlp.experts.45.gate_proj", + "model.layers.78.mlp.experts.45.up_proj", + "model.layers.78.mlp.experts.46.down_proj", + "model.layers.78.mlp.experts.46.gate_proj", + "model.layers.78.mlp.experts.46.up_proj", + "model.layers.78.mlp.experts.47.down_proj", + "model.layers.78.mlp.experts.47.gate_proj", + "model.layers.78.mlp.experts.47.up_proj", + "model.layers.78.mlp.experts.48.down_proj", + "model.layers.78.mlp.experts.48.gate_proj", + "model.layers.78.mlp.experts.48.up_proj", + "model.layers.78.mlp.experts.49.down_proj", + "model.layers.78.mlp.experts.49.gate_proj", + "model.layers.78.mlp.experts.49.up_proj", + "model.layers.78.mlp.experts.5.down_proj", + "model.layers.78.mlp.experts.5.gate_proj", + "model.layers.78.mlp.experts.5.up_proj", + "model.layers.78.mlp.experts.50.down_proj", + "model.layers.78.mlp.experts.50.gate_proj", + "model.layers.78.mlp.experts.50.up_proj", + "model.layers.78.mlp.experts.51.down_proj", + "model.layers.78.mlp.experts.51.gate_proj", + "model.layers.78.mlp.experts.51.up_proj", + "model.layers.78.mlp.experts.52.down_proj", + "model.layers.78.mlp.experts.52.gate_proj", + "model.layers.78.mlp.experts.52.up_proj", + "model.layers.78.mlp.experts.53.down_proj", + "model.layers.78.mlp.experts.53.gate_proj", + "model.layers.78.mlp.experts.53.up_proj", + "model.layers.78.mlp.experts.54.down_proj", + "model.layers.78.mlp.experts.54.gate_proj", + "model.layers.78.mlp.experts.54.up_proj", + "model.layers.78.mlp.experts.55.down_proj", + "model.layers.78.mlp.experts.55.gate_proj", + "model.layers.78.mlp.experts.55.up_proj", + "model.layers.78.mlp.experts.56.down_proj", + "model.layers.78.mlp.experts.56.gate_proj", + "model.layers.78.mlp.experts.56.up_proj", + "model.layers.78.mlp.experts.57.down_proj", + "model.layers.78.mlp.experts.57.gate_proj", + "model.layers.78.mlp.experts.57.up_proj", + "model.layers.78.mlp.experts.58.down_proj", + "model.layers.78.mlp.experts.58.gate_proj", + "model.layers.78.mlp.experts.58.up_proj", + "model.layers.78.mlp.experts.59.down_proj", + "model.layers.78.mlp.experts.59.gate_proj", + "model.layers.78.mlp.experts.59.up_proj", + "model.layers.78.mlp.experts.6.down_proj", + "model.layers.78.mlp.experts.6.gate_proj", + "model.layers.78.mlp.experts.6.up_proj", + "model.layers.78.mlp.experts.60.down_proj", + "model.layers.78.mlp.experts.60.gate_proj", + "model.layers.78.mlp.experts.60.up_proj", + "model.layers.78.mlp.experts.61.down_proj", + "model.layers.78.mlp.experts.61.gate_proj", + "model.layers.78.mlp.experts.61.up_proj", + "model.layers.78.mlp.experts.62.down_proj", + "model.layers.78.mlp.experts.62.gate_proj", + "model.layers.78.mlp.experts.62.up_proj", + "model.layers.78.mlp.experts.63.down_proj", + "model.layers.78.mlp.experts.63.gate_proj", + "model.layers.78.mlp.experts.63.up_proj", + "model.layers.78.mlp.experts.64.down_proj", + "model.layers.78.mlp.experts.64.gate_proj", + "model.layers.78.mlp.experts.64.up_proj", + "model.layers.78.mlp.experts.65.down_proj", + "model.layers.78.mlp.experts.65.gate_proj", + "model.layers.78.mlp.experts.65.up_proj", + "model.layers.78.mlp.experts.66.down_proj", + "model.layers.78.mlp.experts.66.gate_proj", + "model.layers.78.mlp.experts.66.up_proj", + "model.layers.78.mlp.experts.67.down_proj", + "model.layers.78.mlp.experts.67.gate_proj", + "model.layers.78.mlp.experts.67.up_proj", + "model.layers.78.mlp.experts.68.down_proj", + "model.layers.78.mlp.experts.68.gate_proj", + "model.layers.78.mlp.experts.68.up_proj", + "model.layers.78.mlp.experts.69.down_proj", + "model.layers.78.mlp.experts.69.gate_proj", + "model.layers.78.mlp.experts.69.up_proj", + "model.layers.78.mlp.experts.7.down_proj", + "model.layers.78.mlp.experts.7.gate_proj", + "model.layers.78.mlp.experts.7.up_proj", + "model.layers.78.mlp.experts.70.down_proj", + "model.layers.78.mlp.experts.70.gate_proj", + "model.layers.78.mlp.experts.70.up_proj", + "model.layers.78.mlp.experts.71.down_proj", + "model.layers.78.mlp.experts.71.gate_proj", + "model.layers.78.mlp.experts.71.up_proj", + "model.layers.78.mlp.experts.72.down_proj", + "model.layers.78.mlp.experts.72.gate_proj", + "model.layers.78.mlp.experts.72.up_proj", + "model.layers.78.mlp.experts.73.down_proj", + "model.layers.78.mlp.experts.73.gate_proj", + "model.layers.78.mlp.experts.73.up_proj", + "model.layers.78.mlp.experts.74.down_proj", + "model.layers.78.mlp.experts.74.gate_proj", + "model.layers.78.mlp.experts.74.up_proj", + "model.layers.78.mlp.experts.75.down_proj", + "model.layers.78.mlp.experts.75.gate_proj", + "model.layers.78.mlp.experts.75.up_proj", + "model.layers.78.mlp.experts.76.down_proj", + "model.layers.78.mlp.experts.76.gate_proj", + "model.layers.78.mlp.experts.76.up_proj", + "model.layers.78.mlp.experts.77.down_proj", + "model.layers.78.mlp.experts.77.gate_proj", + "model.layers.78.mlp.experts.77.up_proj", + "model.layers.78.mlp.experts.78.down_proj", + "model.layers.78.mlp.experts.78.gate_proj", + "model.layers.78.mlp.experts.78.up_proj", + "model.layers.78.mlp.experts.79.down_proj", + "model.layers.78.mlp.experts.79.gate_proj", + "model.layers.78.mlp.experts.79.up_proj", + "model.layers.78.mlp.experts.8.down_proj", + "model.layers.78.mlp.experts.8.gate_proj", + "model.layers.78.mlp.experts.8.up_proj", + "model.layers.78.mlp.experts.80.down_proj", + "model.layers.78.mlp.experts.80.gate_proj", + "model.layers.78.mlp.experts.80.up_proj", + "model.layers.78.mlp.experts.81.down_proj", + "model.layers.78.mlp.experts.81.gate_proj", + "model.layers.78.mlp.experts.81.up_proj", + "model.layers.78.mlp.experts.82.down_proj", + "model.layers.78.mlp.experts.82.gate_proj", + "model.layers.78.mlp.experts.82.up_proj", + "model.layers.78.mlp.experts.83.down_proj", + "model.layers.78.mlp.experts.83.gate_proj", + "model.layers.78.mlp.experts.83.up_proj", + "model.layers.78.mlp.experts.84.down_proj", + "model.layers.78.mlp.experts.84.gate_proj", + "model.layers.78.mlp.experts.84.up_proj", + "model.layers.78.mlp.experts.85.down_proj", + "model.layers.78.mlp.experts.85.gate_proj", + "model.layers.78.mlp.experts.85.up_proj", + "model.layers.78.mlp.experts.86.down_proj", + "model.layers.78.mlp.experts.86.gate_proj", + "model.layers.78.mlp.experts.86.up_proj", + "model.layers.78.mlp.experts.87.down_proj", + "model.layers.78.mlp.experts.87.gate_proj", + "model.layers.78.mlp.experts.87.up_proj", + "model.layers.78.mlp.experts.88.down_proj", + "model.layers.78.mlp.experts.88.gate_proj", + "model.layers.78.mlp.experts.88.up_proj", + "model.layers.78.mlp.experts.89.down_proj", + "model.layers.78.mlp.experts.89.gate_proj", + "model.layers.78.mlp.experts.89.up_proj", + "model.layers.78.mlp.experts.9.down_proj", + "model.layers.78.mlp.experts.9.gate_proj", + "model.layers.78.mlp.experts.9.up_proj", + "model.layers.78.mlp.experts.90.down_proj", + "model.layers.78.mlp.experts.90.gate_proj", + "model.layers.78.mlp.experts.90.up_proj", + "model.layers.78.mlp.experts.91.down_proj", + "model.layers.78.mlp.experts.91.gate_proj", + "model.layers.78.mlp.experts.91.up_proj", + "model.layers.78.mlp.experts.92.down_proj", + "model.layers.78.mlp.experts.92.gate_proj", + "model.layers.78.mlp.experts.92.up_proj", + "model.layers.78.mlp.experts.93.down_proj", + "model.layers.78.mlp.experts.93.gate_proj", + "model.layers.78.mlp.experts.93.up_proj", + "model.layers.78.mlp.experts.94.down_proj", + "model.layers.78.mlp.experts.94.gate_proj", + "model.layers.78.mlp.experts.94.up_proj", + "model.layers.78.mlp.experts.95.down_proj", + "model.layers.78.mlp.experts.95.gate_proj", + "model.layers.78.mlp.experts.95.up_proj", + "model.layers.78.mlp.experts.96.down_proj", + "model.layers.78.mlp.experts.96.gate_proj", + "model.layers.78.mlp.experts.96.up_proj", + "model.layers.78.mlp.experts.97.down_proj", + "model.layers.78.mlp.experts.97.gate_proj", + "model.layers.78.mlp.experts.97.up_proj", + "model.layers.78.mlp.experts.98.down_proj", + "model.layers.78.mlp.experts.98.gate_proj", + "model.layers.78.mlp.experts.98.up_proj", + "model.layers.78.mlp.experts.99.down_proj", + "model.layers.78.mlp.experts.99.gate_proj", + "model.layers.78.mlp.experts.99.up_proj", + "model.layers.8.mlp.experts.0.down_proj", + "model.layers.8.mlp.experts.0.gate_proj", + "model.layers.8.mlp.experts.0.up_proj", + "model.layers.8.mlp.experts.1.down_proj", + "model.layers.8.mlp.experts.1.gate_proj", + "model.layers.8.mlp.experts.1.up_proj", + "model.layers.8.mlp.experts.10.down_proj", + "model.layers.8.mlp.experts.10.gate_proj", + "model.layers.8.mlp.experts.10.up_proj", + "model.layers.8.mlp.experts.100.down_proj", + "model.layers.8.mlp.experts.100.gate_proj", + "model.layers.8.mlp.experts.100.up_proj", + "model.layers.8.mlp.experts.101.down_proj", + "model.layers.8.mlp.experts.101.gate_proj", + "model.layers.8.mlp.experts.101.up_proj", + "model.layers.8.mlp.experts.102.down_proj", + "model.layers.8.mlp.experts.102.gate_proj", + "model.layers.8.mlp.experts.102.up_proj", + "model.layers.8.mlp.experts.103.down_proj", + "model.layers.8.mlp.experts.103.gate_proj", + "model.layers.8.mlp.experts.103.up_proj", + "model.layers.8.mlp.experts.104.down_proj", + "model.layers.8.mlp.experts.104.gate_proj", + "model.layers.8.mlp.experts.104.up_proj", + "model.layers.8.mlp.experts.105.down_proj", + "model.layers.8.mlp.experts.105.gate_proj", + "model.layers.8.mlp.experts.105.up_proj", + "model.layers.8.mlp.experts.106.down_proj", + "model.layers.8.mlp.experts.106.gate_proj", + "model.layers.8.mlp.experts.106.up_proj", + "model.layers.8.mlp.experts.107.down_proj", + "model.layers.8.mlp.experts.107.gate_proj", + "model.layers.8.mlp.experts.107.up_proj", + "model.layers.8.mlp.experts.108.down_proj", + "model.layers.8.mlp.experts.108.gate_proj", + "model.layers.8.mlp.experts.108.up_proj", + "model.layers.8.mlp.experts.109.down_proj", + "model.layers.8.mlp.experts.109.gate_proj", + "model.layers.8.mlp.experts.109.up_proj", + "model.layers.8.mlp.experts.11.down_proj", + "model.layers.8.mlp.experts.11.gate_proj", + "model.layers.8.mlp.experts.11.up_proj", + "model.layers.8.mlp.experts.110.down_proj", + "model.layers.8.mlp.experts.110.gate_proj", + "model.layers.8.mlp.experts.110.up_proj", + "model.layers.8.mlp.experts.111.down_proj", + "model.layers.8.mlp.experts.111.gate_proj", + "model.layers.8.mlp.experts.111.up_proj", + "model.layers.8.mlp.experts.112.down_proj", + "model.layers.8.mlp.experts.112.gate_proj", + "model.layers.8.mlp.experts.112.up_proj", + "model.layers.8.mlp.experts.113.down_proj", + "model.layers.8.mlp.experts.113.gate_proj", + "model.layers.8.mlp.experts.113.up_proj", + "model.layers.8.mlp.experts.114.down_proj", + "model.layers.8.mlp.experts.114.gate_proj", + "model.layers.8.mlp.experts.114.up_proj", + "model.layers.8.mlp.experts.115.down_proj", + "model.layers.8.mlp.experts.115.gate_proj", + "model.layers.8.mlp.experts.115.up_proj", + "model.layers.8.mlp.experts.116.down_proj", + "model.layers.8.mlp.experts.116.gate_proj", + "model.layers.8.mlp.experts.116.up_proj", + "model.layers.8.mlp.experts.117.down_proj", + "model.layers.8.mlp.experts.117.gate_proj", + "model.layers.8.mlp.experts.117.up_proj", + "model.layers.8.mlp.experts.118.down_proj", + "model.layers.8.mlp.experts.118.gate_proj", + "model.layers.8.mlp.experts.118.up_proj", + "model.layers.8.mlp.experts.119.down_proj", + "model.layers.8.mlp.experts.119.gate_proj", + "model.layers.8.mlp.experts.119.up_proj", + "model.layers.8.mlp.experts.12.down_proj", + "model.layers.8.mlp.experts.12.gate_proj", + "model.layers.8.mlp.experts.12.up_proj", + "model.layers.8.mlp.experts.120.down_proj", + "model.layers.8.mlp.experts.120.gate_proj", + "model.layers.8.mlp.experts.120.up_proj", + "model.layers.8.mlp.experts.121.down_proj", + "model.layers.8.mlp.experts.121.gate_proj", + "model.layers.8.mlp.experts.121.up_proj", + "model.layers.8.mlp.experts.122.down_proj", + "model.layers.8.mlp.experts.122.gate_proj", + "model.layers.8.mlp.experts.122.up_proj", + "model.layers.8.mlp.experts.123.down_proj", + "model.layers.8.mlp.experts.123.gate_proj", + "model.layers.8.mlp.experts.123.up_proj", + "model.layers.8.mlp.experts.124.down_proj", + "model.layers.8.mlp.experts.124.gate_proj", + "model.layers.8.mlp.experts.124.up_proj", + "model.layers.8.mlp.experts.125.down_proj", + "model.layers.8.mlp.experts.125.gate_proj", + "model.layers.8.mlp.experts.125.up_proj", + "model.layers.8.mlp.experts.126.down_proj", + "model.layers.8.mlp.experts.126.gate_proj", + "model.layers.8.mlp.experts.126.up_proj", + "model.layers.8.mlp.experts.127.down_proj", + "model.layers.8.mlp.experts.127.gate_proj", + "model.layers.8.mlp.experts.127.up_proj", + "model.layers.8.mlp.experts.128.down_proj", + "model.layers.8.mlp.experts.128.gate_proj", + "model.layers.8.mlp.experts.128.up_proj", + "model.layers.8.mlp.experts.129.down_proj", + "model.layers.8.mlp.experts.129.gate_proj", + "model.layers.8.mlp.experts.129.up_proj", + "model.layers.8.mlp.experts.13.down_proj", + "model.layers.8.mlp.experts.13.gate_proj", + "model.layers.8.mlp.experts.13.up_proj", + "model.layers.8.mlp.experts.130.down_proj", + "model.layers.8.mlp.experts.130.gate_proj", + "model.layers.8.mlp.experts.130.up_proj", + "model.layers.8.mlp.experts.131.down_proj", + "model.layers.8.mlp.experts.131.gate_proj", + "model.layers.8.mlp.experts.131.up_proj", + "model.layers.8.mlp.experts.132.down_proj", + "model.layers.8.mlp.experts.132.gate_proj", + "model.layers.8.mlp.experts.132.up_proj", + "model.layers.8.mlp.experts.133.down_proj", + "model.layers.8.mlp.experts.133.gate_proj", + "model.layers.8.mlp.experts.133.up_proj", + "model.layers.8.mlp.experts.134.down_proj", + "model.layers.8.mlp.experts.134.gate_proj", + "model.layers.8.mlp.experts.134.up_proj", + "model.layers.8.mlp.experts.135.down_proj", + "model.layers.8.mlp.experts.135.gate_proj", + "model.layers.8.mlp.experts.135.up_proj", + "model.layers.8.mlp.experts.136.down_proj", + "model.layers.8.mlp.experts.136.gate_proj", + "model.layers.8.mlp.experts.136.up_proj", + "model.layers.8.mlp.experts.137.down_proj", + "model.layers.8.mlp.experts.137.gate_proj", + "model.layers.8.mlp.experts.137.up_proj", + "model.layers.8.mlp.experts.138.down_proj", + "model.layers.8.mlp.experts.138.gate_proj", + "model.layers.8.mlp.experts.138.up_proj", + "model.layers.8.mlp.experts.139.down_proj", + "model.layers.8.mlp.experts.139.gate_proj", + "model.layers.8.mlp.experts.139.up_proj", + "model.layers.8.mlp.experts.14.down_proj", + "model.layers.8.mlp.experts.14.gate_proj", + "model.layers.8.mlp.experts.14.up_proj", + "model.layers.8.mlp.experts.140.down_proj", + "model.layers.8.mlp.experts.140.gate_proj", + "model.layers.8.mlp.experts.140.up_proj", + "model.layers.8.mlp.experts.141.down_proj", + "model.layers.8.mlp.experts.141.gate_proj", + "model.layers.8.mlp.experts.141.up_proj", + "model.layers.8.mlp.experts.142.down_proj", + "model.layers.8.mlp.experts.142.gate_proj", + "model.layers.8.mlp.experts.142.up_proj", + "model.layers.8.mlp.experts.143.down_proj", + "model.layers.8.mlp.experts.143.gate_proj", + "model.layers.8.mlp.experts.143.up_proj", + "model.layers.8.mlp.experts.144.down_proj", + "model.layers.8.mlp.experts.144.gate_proj", + "model.layers.8.mlp.experts.144.up_proj", + "model.layers.8.mlp.experts.145.down_proj", + "model.layers.8.mlp.experts.145.gate_proj", + "model.layers.8.mlp.experts.145.up_proj", + "model.layers.8.mlp.experts.146.down_proj", + "model.layers.8.mlp.experts.146.gate_proj", + "model.layers.8.mlp.experts.146.up_proj", + "model.layers.8.mlp.experts.147.down_proj", + "model.layers.8.mlp.experts.147.gate_proj", + "model.layers.8.mlp.experts.147.up_proj", + "model.layers.8.mlp.experts.148.down_proj", + "model.layers.8.mlp.experts.148.gate_proj", + "model.layers.8.mlp.experts.148.up_proj", + "model.layers.8.mlp.experts.149.down_proj", + "model.layers.8.mlp.experts.149.gate_proj", + "model.layers.8.mlp.experts.149.up_proj", + "model.layers.8.mlp.experts.15.down_proj", + "model.layers.8.mlp.experts.15.gate_proj", + "model.layers.8.mlp.experts.15.up_proj", + "model.layers.8.mlp.experts.150.down_proj", + "model.layers.8.mlp.experts.150.gate_proj", + "model.layers.8.mlp.experts.150.up_proj", + "model.layers.8.mlp.experts.151.down_proj", + "model.layers.8.mlp.experts.151.gate_proj", + "model.layers.8.mlp.experts.151.up_proj", + "model.layers.8.mlp.experts.152.down_proj", + "model.layers.8.mlp.experts.152.gate_proj", + "model.layers.8.mlp.experts.152.up_proj", + "model.layers.8.mlp.experts.153.down_proj", + "model.layers.8.mlp.experts.153.gate_proj", + "model.layers.8.mlp.experts.153.up_proj", + "model.layers.8.mlp.experts.154.down_proj", + "model.layers.8.mlp.experts.154.gate_proj", + "model.layers.8.mlp.experts.154.up_proj", + "model.layers.8.mlp.experts.155.down_proj", + "model.layers.8.mlp.experts.155.gate_proj", + "model.layers.8.mlp.experts.155.up_proj", + "model.layers.8.mlp.experts.156.down_proj", + "model.layers.8.mlp.experts.156.gate_proj", + "model.layers.8.mlp.experts.156.up_proj", + "model.layers.8.mlp.experts.157.down_proj", + "model.layers.8.mlp.experts.157.gate_proj", + "model.layers.8.mlp.experts.157.up_proj", + "model.layers.8.mlp.experts.158.down_proj", + "model.layers.8.mlp.experts.158.gate_proj", + "model.layers.8.mlp.experts.158.up_proj", + "model.layers.8.mlp.experts.159.down_proj", + "model.layers.8.mlp.experts.159.gate_proj", + "model.layers.8.mlp.experts.159.up_proj", + "model.layers.8.mlp.experts.16.down_proj", + "model.layers.8.mlp.experts.16.gate_proj", + "model.layers.8.mlp.experts.16.up_proj", + "model.layers.8.mlp.experts.160.down_proj", + "model.layers.8.mlp.experts.160.gate_proj", + "model.layers.8.mlp.experts.160.up_proj", + "model.layers.8.mlp.experts.161.down_proj", + "model.layers.8.mlp.experts.161.gate_proj", + "model.layers.8.mlp.experts.161.up_proj", + "model.layers.8.mlp.experts.162.down_proj", + "model.layers.8.mlp.experts.162.gate_proj", + "model.layers.8.mlp.experts.162.up_proj", + "model.layers.8.mlp.experts.163.down_proj", + "model.layers.8.mlp.experts.163.gate_proj", + "model.layers.8.mlp.experts.163.up_proj", + "model.layers.8.mlp.experts.164.down_proj", + "model.layers.8.mlp.experts.164.gate_proj", + "model.layers.8.mlp.experts.164.up_proj", + "model.layers.8.mlp.experts.165.down_proj", + "model.layers.8.mlp.experts.165.gate_proj", + "model.layers.8.mlp.experts.165.up_proj", + "model.layers.8.mlp.experts.166.down_proj", + "model.layers.8.mlp.experts.166.gate_proj", + "model.layers.8.mlp.experts.166.up_proj", + "model.layers.8.mlp.experts.167.down_proj", + "model.layers.8.mlp.experts.167.gate_proj", + "model.layers.8.mlp.experts.167.up_proj", + "model.layers.8.mlp.experts.168.down_proj", + "model.layers.8.mlp.experts.168.gate_proj", + "model.layers.8.mlp.experts.168.up_proj", + "model.layers.8.mlp.experts.169.down_proj", + "model.layers.8.mlp.experts.169.gate_proj", + "model.layers.8.mlp.experts.169.up_proj", + "model.layers.8.mlp.experts.17.down_proj", + "model.layers.8.mlp.experts.17.gate_proj", + "model.layers.8.mlp.experts.17.up_proj", + "model.layers.8.mlp.experts.170.down_proj", + "model.layers.8.mlp.experts.170.gate_proj", + "model.layers.8.mlp.experts.170.up_proj", + "model.layers.8.mlp.experts.171.down_proj", + "model.layers.8.mlp.experts.171.gate_proj", + "model.layers.8.mlp.experts.171.up_proj", + "model.layers.8.mlp.experts.172.down_proj", + "model.layers.8.mlp.experts.172.gate_proj", + "model.layers.8.mlp.experts.172.up_proj", + "model.layers.8.mlp.experts.173.down_proj", + "model.layers.8.mlp.experts.173.gate_proj", + "model.layers.8.mlp.experts.173.up_proj", + "model.layers.8.mlp.experts.174.down_proj", + "model.layers.8.mlp.experts.174.gate_proj", + "model.layers.8.mlp.experts.174.up_proj", + "model.layers.8.mlp.experts.175.down_proj", + "model.layers.8.mlp.experts.175.gate_proj", + "model.layers.8.mlp.experts.175.up_proj", + "model.layers.8.mlp.experts.176.down_proj", + "model.layers.8.mlp.experts.176.gate_proj", + "model.layers.8.mlp.experts.176.up_proj", + "model.layers.8.mlp.experts.177.down_proj", + "model.layers.8.mlp.experts.177.gate_proj", + "model.layers.8.mlp.experts.177.up_proj", + "model.layers.8.mlp.experts.178.down_proj", + "model.layers.8.mlp.experts.178.gate_proj", + "model.layers.8.mlp.experts.178.up_proj", + "model.layers.8.mlp.experts.179.down_proj", + "model.layers.8.mlp.experts.179.gate_proj", + "model.layers.8.mlp.experts.179.up_proj", + "model.layers.8.mlp.experts.18.down_proj", + "model.layers.8.mlp.experts.18.gate_proj", + "model.layers.8.mlp.experts.18.up_proj", + "model.layers.8.mlp.experts.180.down_proj", + "model.layers.8.mlp.experts.180.gate_proj", + "model.layers.8.mlp.experts.180.up_proj", + "model.layers.8.mlp.experts.181.down_proj", + "model.layers.8.mlp.experts.181.gate_proj", + "model.layers.8.mlp.experts.181.up_proj", + "model.layers.8.mlp.experts.182.down_proj", + "model.layers.8.mlp.experts.182.gate_proj", + "model.layers.8.mlp.experts.182.up_proj", + "model.layers.8.mlp.experts.183.down_proj", + "model.layers.8.mlp.experts.183.gate_proj", + "model.layers.8.mlp.experts.183.up_proj", + "model.layers.8.mlp.experts.184.down_proj", + "model.layers.8.mlp.experts.184.gate_proj", + "model.layers.8.mlp.experts.184.up_proj", + "model.layers.8.mlp.experts.185.down_proj", + "model.layers.8.mlp.experts.185.gate_proj", + "model.layers.8.mlp.experts.185.up_proj", + "model.layers.8.mlp.experts.186.down_proj", + "model.layers.8.mlp.experts.186.gate_proj", + "model.layers.8.mlp.experts.186.up_proj", + "model.layers.8.mlp.experts.187.down_proj", + "model.layers.8.mlp.experts.187.gate_proj", + "model.layers.8.mlp.experts.187.up_proj", + "model.layers.8.mlp.experts.188.down_proj", + "model.layers.8.mlp.experts.188.gate_proj", + "model.layers.8.mlp.experts.188.up_proj", + "model.layers.8.mlp.experts.189.down_proj", + "model.layers.8.mlp.experts.189.gate_proj", + "model.layers.8.mlp.experts.189.up_proj", + "model.layers.8.mlp.experts.19.down_proj", + "model.layers.8.mlp.experts.19.gate_proj", + "model.layers.8.mlp.experts.19.up_proj", + "model.layers.8.mlp.experts.190.down_proj", + "model.layers.8.mlp.experts.190.gate_proj", + "model.layers.8.mlp.experts.190.up_proj", + "model.layers.8.mlp.experts.191.down_proj", + "model.layers.8.mlp.experts.191.gate_proj", + "model.layers.8.mlp.experts.191.up_proj", + "model.layers.8.mlp.experts.192.down_proj", + "model.layers.8.mlp.experts.192.gate_proj", + "model.layers.8.mlp.experts.192.up_proj", + "model.layers.8.mlp.experts.193.down_proj", + "model.layers.8.mlp.experts.193.gate_proj", + "model.layers.8.mlp.experts.193.up_proj", + "model.layers.8.mlp.experts.194.down_proj", + "model.layers.8.mlp.experts.194.gate_proj", + "model.layers.8.mlp.experts.194.up_proj", + "model.layers.8.mlp.experts.195.down_proj", + "model.layers.8.mlp.experts.195.gate_proj", + "model.layers.8.mlp.experts.195.up_proj", + "model.layers.8.mlp.experts.196.down_proj", + "model.layers.8.mlp.experts.196.gate_proj", + "model.layers.8.mlp.experts.196.up_proj", + "model.layers.8.mlp.experts.197.down_proj", + "model.layers.8.mlp.experts.197.gate_proj", + "model.layers.8.mlp.experts.197.up_proj", + "model.layers.8.mlp.experts.198.down_proj", + "model.layers.8.mlp.experts.198.gate_proj", + "model.layers.8.mlp.experts.198.up_proj", + "model.layers.8.mlp.experts.199.down_proj", + "model.layers.8.mlp.experts.199.gate_proj", + "model.layers.8.mlp.experts.199.up_proj", + "model.layers.8.mlp.experts.2.down_proj", + "model.layers.8.mlp.experts.2.gate_proj", + "model.layers.8.mlp.experts.2.up_proj", + "model.layers.8.mlp.experts.20.down_proj", + "model.layers.8.mlp.experts.20.gate_proj", + "model.layers.8.mlp.experts.20.up_proj", + "model.layers.8.mlp.experts.200.down_proj", + "model.layers.8.mlp.experts.200.gate_proj", + "model.layers.8.mlp.experts.200.up_proj", + "model.layers.8.mlp.experts.201.down_proj", + "model.layers.8.mlp.experts.201.gate_proj", + "model.layers.8.mlp.experts.201.up_proj", + "model.layers.8.mlp.experts.202.down_proj", + "model.layers.8.mlp.experts.202.gate_proj", + "model.layers.8.mlp.experts.202.up_proj", + "model.layers.8.mlp.experts.203.down_proj", + "model.layers.8.mlp.experts.203.gate_proj", + "model.layers.8.mlp.experts.203.up_proj", + "model.layers.8.mlp.experts.204.down_proj", + "model.layers.8.mlp.experts.204.gate_proj", + "model.layers.8.mlp.experts.204.up_proj", + "model.layers.8.mlp.experts.205.down_proj", + "model.layers.8.mlp.experts.205.gate_proj", + "model.layers.8.mlp.experts.205.up_proj", + "model.layers.8.mlp.experts.206.down_proj", + "model.layers.8.mlp.experts.206.gate_proj", + "model.layers.8.mlp.experts.206.up_proj", + "model.layers.8.mlp.experts.207.down_proj", + "model.layers.8.mlp.experts.207.gate_proj", + "model.layers.8.mlp.experts.207.up_proj", + "model.layers.8.mlp.experts.208.down_proj", + "model.layers.8.mlp.experts.208.gate_proj", + "model.layers.8.mlp.experts.208.up_proj", + "model.layers.8.mlp.experts.209.down_proj", + "model.layers.8.mlp.experts.209.gate_proj", + "model.layers.8.mlp.experts.209.up_proj", + "model.layers.8.mlp.experts.21.down_proj", + "model.layers.8.mlp.experts.21.gate_proj", + "model.layers.8.mlp.experts.21.up_proj", + "model.layers.8.mlp.experts.210.down_proj", + "model.layers.8.mlp.experts.210.gate_proj", + "model.layers.8.mlp.experts.210.up_proj", + "model.layers.8.mlp.experts.211.down_proj", + "model.layers.8.mlp.experts.211.gate_proj", + "model.layers.8.mlp.experts.211.up_proj", + "model.layers.8.mlp.experts.212.down_proj", + "model.layers.8.mlp.experts.212.gate_proj", + "model.layers.8.mlp.experts.212.up_proj", + "model.layers.8.mlp.experts.213.down_proj", + "model.layers.8.mlp.experts.213.gate_proj", + "model.layers.8.mlp.experts.213.up_proj", + "model.layers.8.mlp.experts.214.down_proj", + "model.layers.8.mlp.experts.214.gate_proj", + "model.layers.8.mlp.experts.214.up_proj", + "model.layers.8.mlp.experts.215.down_proj", + "model.layers.8.mlp.experts.215.gate_proj", + "model.layers.8.mlp.experts.215.up_proj", + "model.layers.8.mlp.experts.216.down_proj", + "model.layers.8.mlp.experts.216.gate_proj", + "model.layers.8.mlp.experts.216.up_proj", + "model.layers.8.mlp.experts.217.down_proj", + "model.layers.8.mlp.experts.217.gate_proj", + "model.layers.8.mlp.experts.217.up_proj", + "model.layers.8.mlp.experts.218.down_proj", + "model.layers.8.mlp.experts.218.gate_proj", + "model.layers.8.mlp.experts.218.up_proj", + "model.layers.8.mlp.experts.219.down_proj", + "model.layers.8.mlp.experts.219.gate_proj", + "model.layers.8.mlp.experts.219.up_proj", + "model.layers.8.mlp.experts.22.down_proj", + "model.layers.8.mlp.experts.22.gate_proj", + "model.layers.8.mlp.experts.22.up_proj", + "model.layers.8.mlp.experts.220.down_proj", + "model.layers.8.mlp.experts.220.gate_proj", + "model.layers.8.mlp.experts.220.up_proj", + "model.layers.8.mlp.experts.221.down_proj", + "model.layers.8.mlp.experts.221.gate_proj", + "model.layers.8.mlp.experts.221.up_proj", + "model.layers.8.mlp.experts.222.down_proj", + "model.layers.8.mlp.experts.222.gate_proj", + "model.layers.8.mlp.experts.222.up_proj", + "model.layers.8.mlp.experts.223.down_proj", + "model.layers.8.mlp.experts.223.gate_proj", + "model.layers.8.mlp.experts.223.up_proj", + "model.layers.8.mlp.experts.224.down_proj", + "model.layers.8.mlp.experts.224.gate_proj", + "model.layers.8.mlp.experts.224.up_proj", + "model.layers.8.mlp.experts.225.down_proj", + "model.layers.8.mlp.experts.225.gate_proj", + "model.layers.8.mlp.experts.225.up_proj", + "model.layers.8.mlp.experts.226.down_proj", + "model.layers.8.mlp.experts.226.gate_proj", + "model.layers.8.mlp.experts.226.up_proj", + "model.layers.8.mlp.experts.227.down_proj", + "model.layers.8.mlp.experts.227.gate_proj", + "model.layers.8.mlp.experts.227.up_proj", + "model.layers.8.mlp.experts.228.down_proj", + "model.layers.8.mlp.experts.228.gate_proj", + "model.layers.8.mlp.experts.228.up_proj", + "model.layers.8.mlp.experts.229.down_proj", + "model.layers.8.mlp.experts.229.gate_proj", + "model.layers.8.mlp.experts.229.up_proj", + "model.layers.8.mlp.experts.23.down_proj", + "model.layers.8.mlp.experts.23.gate_proj", + "model.layers.8.mlp.experts.23.up_proj", + "model.layers.8.mlp.experts.230.down_proj", + "model.layers.8.mlp.experts.230.gate_proj", + "model.layers.8.mlp.experts.230.up_proj", + "model.layers.8.mlp.experts.231.down_proj", + "model.layers.8.mlp.experts.231.gate_proj", + "model.layers.8.mlp.experts.231.up_proj", + "model.layers.8.mlp.experts.232.down_proj", + "model.layers.8.mlp.experts.232.gate_proj", + "model.layers.8.mlp.experts.232.up_proj", + "model.layers.8.mlp.experts.233.down_proj", + "model.layers.8.mlp.experts.233.gate_proj", + "model.layers.8.mlp.experts.233.up_proj", + "model.layers.8.mlp.experts.234.down_proj", + "model.layers.8.mlp.experts.234.gate_proj", + "model.layers.8.mlp.experts.234.up_proj", + "model.layers.8.mlp.experts.235.down_proj", + "model.layers.8.mlp.experts.235.gate_proj", + "model.layers.8.mlp.experts.235.up_proj", + "model.layers.8.mlp.experts.236.down_proj", + "model.layers.8.mlp.experts.236.gate_proj", + "model.layers.8.mlp.experts.236.up_proj", + "model.layers.8.mlp.experts.237.down_proj", + "model.layers.8.mlp.experts.237.gate_proj", + "model.layers.8.mlp.experts.237.up_proj", + "model.layers.8.mlp.experts.238.down_proj", + "model.layers.8.mlp.experts.238.gate_proj", + "model.layers.8.mlp.experts.238.up_proj", + "model.layers.8.mlp.experts.239.down_proj", + "model.layers.8.mlp.experts.239.gate_proj", + "model.layers.8.mlp.experts.239.up_proj", + "model.layers.8.mlp.experts.24.down_proj", + "model.layers.8.mlp.experts.24.gate_proj", + "model.layers.8.mlp.experts.24.up_proj", + "model.layers.8.mlp.experts.240.down_proj", + "model.layers.8.mlp.experts.240.gate_proj", + "model.layers.8.mlp.experts.240.up_proj", + "model.layers.8.mlp.experts.241.down_proj", + "model.layers.8.mlp.experts.241.gate_proj", + "model.layers.8.mlp.experts.241.up_proj", + "model.layers.8.mlp.experts.242.down_proj", + "model.layers.8.mlp.experts.242.gate_proj", + "model.layers.8.mlp.experts.242.up_proj", + "model.layers.8.mlp.experts.243.down_proj", + "model.layers.8.mlp.experts.243.gate_proj", + "model.layers.8.mlp.experts.243.up_proj", + "model.layers.8.mlp.experts.244.down_proj", + "model.layers.8.mlp.experts.244.gate_proj", + "model.layers.8.mlp.experts.244.up_proj", + "model.layers.8.mlp.experts.245.down_proj", + "model.layers.8.mlp.experts.245.gate_proj", + "model.layers.8.mlp.experts.245.up_proj", + "model.layers.8.mlp.experts.246.down_proj", + "model.layers.8.mlp.experts.246.gate_proj", + "model.layers.8.mlp.experts.246.up_proj", + "model.layers.8.mlp.experts.247.down_proj", + "model.layers.8.mlp.experts.247.gate_proj", + "model.layers.8.mlp.experts.247.up_proj", + "model.layers.8.mlp.experts.248.down_proj", + "model.layers.8.mlp.experts.248.gate_proj", + "model.layers.8.mlp.experts.248.up_proj", + "model.layers.8.mlp.experts.249.down_proj", + "model.layers.8.mlp.experts.249.gate_proj", + "model.layers.8.mlp.experts.249.up_proj", + "model.layers.8.mlp.experts.25.down_proj", + "model.layers.8.mlp.experts.25.gate_proj", + "model.layers.8.mlp.experts.25.up_proj", + "model.layers.8.mlp.experts.250.down_proj", + "model.layers.8.mlp.experts.250.gate_proj", + "model.layers.8.mlp.experts.250.up_proj", + "model.layers.8.mlp.experts.251.down_proj", + "model.layers.8.mlp.experts.251.gate_proj", + "model.layers.8.mlp.experts.251.up_proj", + "model.layers.8.mlp.experts.252.down_proj", + "model.layers.8.mlp.experts.252.gate_proj", + "model.layers.8.mlp.experts.252.up_proj", + "model.layers.8.mlp.experts.253.down_proj", + "model.layers.8.mlp.experts.253.gate_proj", + "model.layers.8.mlp.experts.253.up_proj", + "model.layers.8.mlp.experts.254.down_proj", + "model.layers.8.mlp.experts.254.gate_proj", + "model.layers.8.mlp.experts.254.up_proj", + "model.layers.8.mlp.experts.255.down_proj", + "model.layers.8.mlp.experts.255.gate_proj", + "model.layers.8.mlp.experts.255.up_proj", + "model.layers.8.mlp.experts.26.down_proj", + "model.layers.8.mlp.experts.26.gate_proj", + "model.layers.8.mlp.experts.26.up_proj", + "model.layers.8.mlp.experts.27.down_proj", + "model.layers.8.mlp.experts.27.gate_proj", + "model.layers.8.mlp.experts.27.up_proj", + "model.layers.8.mlp.experts.28.down_proj", + "model.layers.8.mlp.experts.28.gate_proj", + "model.layers.8.mlp.experts.28.up_proj", + "model.layers.8.mlp.experts.29.down_proj", + "model.layers.8.mlp.experts.29.gate_proj", + "model.layers.8.mlp.experts.29.up_proj", + "model.layers.8.mlp.experts.3.down_proj", + "model.layers.8.mlp.experts.3.gate_proj", + "model.layers.8.mlp.experts.3.up_proj", + "model.layers.8.mlp.experts.30.down_proj", + "model.layers.8.mlp.experts.30.gate_proj", + "model.layers.8.mlp.experts.30.up_proj", + "model.layers.8.mlp.experts.31.down_proj", + "model.layers.8.mlp.experts.31.gate_proj", + "model.layers.8.mlp.experts.31.up_proj", + "model.layers.8.mlp.experts.32.down_proj", + "model.layers.8.mlp.experts.32.gate_proj", + "model.layers.8.mlp.experts.32.up_proj", + "model.layers.8.mlp.experts.33.down_proj", + "model.layers.8.mlp.experts.33.gate_proj", + "model.layers.8.mlp.experts.33.up_proj", + "model.layers.8.mlp.experts.34.down_proj", + "model.layers.8.mlp.experts.34.gate_proj", + "model.layers.8.mlp.experts.34.up_proj", + "model.layers.8.mlp.experts.35.down_proj", + "model.layers.8.mlp.experts.35.gate_proj", + "model.layers.8.mlp.experts.35.up_proj", + "model.layers.8.mlp.experts.36.down_proj", + "model.layers.8.mlp.experts.36.gate_proj", + "model.layers.8.mlp.experts.36.up_proj", + "model.layers.8.mlp.experts.37.down_proj", + "model.layers.8.mlp.experts.37.gate_proj", + "model.layers.8.mlp.experts.37.up_proj", + "model.layers.8.mlp.experts.38.down_proj", + "model.layers.8.mlp.experts.38.gate_proj", + "model.layers.8.mlp.experts.38.up_proj", + "model.layers.8.mlp.experts.39.down_proj", + "model.layers.8.mlp.experts.39.gate_proj", + "model.layers.8.mlp.experts.39.up_proj", + "model.layers.8.mlp.experts.4.down_proj", + "model.layers.8.mlp.experts.4.gate_proj", + "model.layers.8.mlp.experts.4.up_proj", + "model.layers.8.mlp.experts.40.down_proj", + "model.layers.8.mlp.experts.40.gate_proj", + "model.layers.8.mlp.experts.40.up_proj", + "model.layers.8.mlp.experts.41.down_proj", + "model.layers.8.mlp.experts.41.gate_proj", + "model.layers.8.mlp.experts.41.up_proj", + "model.layers.8.mlp.experts.42.down_proj", + "model.layers.8.mlp.experts.42.gate_proj", + "model.layers.8.mlp.experts.42.up_proj", + "model.layers.8.mlp.experts.43.down_proj", + "model.layers.8.mlp.experts.43.gate_proj", + "model.layers.8.mlp.experts.43.up_proj", + "model.layers.8.mlp.experts.44.down_proj", + "model.layers.8.mlp.experts.44.gate_proj", + "model.layers.8.mlp.experts.44.up_proj", + "model.layers.8.mlp.experts.45.down_proj", + "model.layers.8.mlp.experts.45.gate_proj", + "model.layers.8.mlp.experts.45.up_proj", + "model.layers.8.mlp.experts.46.down_proj", + "model.layers.8.mlp.experts.46.gate_proj", + "model.layers.8.mlp.experts.46.up_proj", + "model.layers.8.mlp.experts.47.down_proj", + "model.layers.8.mlp.experts.47.gate_proj", + "model.layers.8.mlp.experts.47.up_proj", + "model.layers.8.mlp.experts.48.down_proj", + "model.layers.8.mlp.experts.48.gate_proj", + "model.layers.8.mlp.experts.48.up_proj", + "model.layers.8.mlp.experts.49.down_proj", + "model.layers.8.mlp.experts.49.gate_proj", + "model.layers.8.mlp.experts.49.up_proj", + "model.layers.8.mlp.experts.5.down_proj", + "model.layers.8.mlp.experts.5.gate_proj", + "model.layers.8.mlp.experts.5.up_proj", + "model.layers.8.mlp.experts.50.down_proj", + "model.layers.8.mlp.experts.50.gate_proj", + "model.layers.8.mlp.experts.50.up_proj", + "model.layers.8.mlp.experts.51.down_proj", + "model.layers.8.mlp.experts.51.gate_proj", + "model.layers.8.mlp.experts.51.up_proj", + "model.layers.8.mlp.experts.52.down_proj", + "model.layers.8.mlp.experts.52.gate_proj", + "model.layers.8.mlp.experts.52.up_proj", + "model.layers.8.mlp.experts.53.down_proj", + "model.layers.8.mlp.experts.53.gate_proj", + "model.layers.8.mlp.experts.53.up_proj", + "model.layers.8.mlp.experts.54.down_proj", + "model.layers.8.mlp.experts.54.gate_proj", + "model.layers.8.mlp.experts.54.up_proj", + "model.layers.8.mlp.experts.55.down_proj", + "model.layers.8.mlp.experts.55.gate_proj", + "model.layers.8.mlp.experts.55.up_proj", + "model.layers.8.mlp.experts.56.down_proj", + "model.layers.8.mlp.experts.56.gate_proj", + "model.layers.8.mlp.experts.56.up_proj", + "model.layers.8.mlp.experts.57.down_proj", + "model.layers.8.mlp.experts.57.gate_proj", + "model.layers.8.mlp.experts.57.up_proj", + "model.layers.8.mlp.experts.58.down_proj", + "model.layers.8.mlp.experts.58.gate_proj", + "model.layers.8.mlp.experts.58.up_proj", + "model.layers.8.mlp.experts.59.down_proj", + "model.layers.8.mlp.experts.59.gate_proj", + "model.layers.8.mlp.experts.59.up_proj", + "model.layers.8.mlp.experts.6.down_proj", + "model.layers.8.mlp.experts.6.gate_proj", + "model.layers.8.mlp.experts.6.up_proj", + "model.layers.8.mlp.experts.60.down_proj", + "model.layers.8.mlp.experts.60.gate_proj", + "model.layers.8.mlp.experts.60.up_proj", + "model.layers.8.mlp.experts.61.down_proj", + "model.layers.8.mlp.experts.61.gate_proj", + "model.layers.8.mlp.experts.61.up_proj", + "model.layers.8.mlp.experts.62.down_proj", + "model.layers.8.mlp.experts.62.gate_proj", + "model.layers.8.mlp.experts.62.up_proj", + "model.layers.8.mlp.experts.63.down_proj", + "model.layers.8.mlp.experts.63.gate_proj", + "model.layers.8.mlp.experts.63.up_proj", + "model.layers.8.mlp.experts.64.down_proj", + "model.layers.8.mlp.experts.64.gate_proj", + "model.layers.8.mlp.experts.64.up_proj", + "model.layers.8.mlp.experts.65.down_proj", + "model.layers.8.mlp.experts.65.gate_proj", + "model.layers.8.mlp.experts.65.up_proj", + "model.layers.8.mlp.experts.66.down_proj", + "model.layers.8.mlp.experts.66.gate_proj", + "model.layers.8.mlp.experts.66.up_proj", + "model.layers.8.mlp.experts.67.down_proj", + "model.layers.8.mlp.experts.67.gate_proj", + "model.layers.8.mlp.experts.67.up_proj", + "model.layers.8.mlp.experts.68.down_proj", + "model.layers.8.mlp.experts.68.gate_proj", + "model.layers.8.mlp.experts.68.up_proj", + "model.layers.8.mlp.experts.69.down_proj", + "model.layers.8.mlp.experts.69.gate_proj", + "model.layers.8.mlp.experts.69.up_proj", + "model.layers.8.mlp.experts.7.down_proj", + "model.layers.8.mlp.experts.7.gate_proj", + "model.layers.8.mlp.experts.7.up_proj", + "model.layers.8.mlp.experts.70.down_proj", + "model.layers.8.mlp.experts.70.gate_proj", + "model.layers.8.mlp.experts.70.up_proj", + "model.layers.8.mlp.experts.71.down_proj", + "model.layers.8.mlp.experts.71.gate_proj", + "model.layers.8.mlp.experts.71.up_proj", + "model.layers.8.mlp.experts.72.down_proj", + "model.layers.8.mlp.experts.72.gate_proj", + "model.layers.8.mlp.experts.72.up_proj", + "model.layers.8.mlp.experts.73.down_proj", + "model.layers.8.mlp.experts.73.gate_proj", + "model.layers.8.mlp.experts.73.up_proj", + "model.layers.8.mlp.experts.74.down_proj", + "model.layers.8.mlp.experts.74.gate_proj", + "model.layers.8.mlp.experts.74.up_proj", + "model.layers.8.mlp.experts.75.down_proj", + "model.layers.8.mlp.experts.75.gate_proj", + "model.layers.8.mlp.experts.75.up_proj", + "model.layers.8.mlp.experts.76.down_proj", + "model.layers.8.mlp.experts.76.gate_proj", + "model.layers.8.mlp.experts.76.up_proj", + "model.layers.8.mlp.experts.77.down_proj", + "model.layers.8.mlp.experts.77.gate_proj", + "model.layers.8.mlp.experts.77.up_proj", + "model.layers.8.mlp.experts.78.down_proj", + "model.layers.8.mlp.experts.78.gate_proj", + "model.layers.8.mlp.experts.78.up_proj", + "model.layers.8.mlp.experts.79.down_proj", + "model.layers.8.mlp.experts.79.gate_proj", + "model.layers.8.mlp.experts.79.up_proj", + "model.layers.8.mlp.experts.8.down_proj", + "model.layers.8.mlp.experts.8.gate_proj", + "model.layers.8.mlp.experts.8.up_proj", + "model.layers.8.mlp.experts.80.down_proj", + "model.layers.8.mlp.experts.80.gate_proj", + "model.layers.8.mlp.experts.80.up_proj", + "model.layers.8.mlp.experts.81.down_proj", + "model.layers.8.mlp.experts.81.gate_proj", + "model.layers.8.mlp.experts.81.up_proj", + "model.layers.8.mlp.experts.82.down_proj", + "model.layers.8.mlp.experts.82.gate_proj", + "model.layers.8.mlp.experts.82.up_proj", + "model.layers.8.mlp.experts.83.down_proj", + "model.layers.8.mlp.experts.83.gate_proj", + "model.layers.8.mlp.experts.83.up_proj", + "model.layers.8.mlp.experts.84.down_proj", + "model.layers.8.mlp.experts.84.gate_proj", + "model.layers.8.mlp.experts.84.up_proj", + "model.layers.8.mlp.experts.85.down_proj", + "model.layers.8.mlp.experts.85.gate_proj", + "model.layers.8.mlp.experts.85.up_proj", + "model.layers.8.mlp.experts.86.down_proj", + "model.layers.8.mlp.experts.86.gate_proj", + "model.layers.8.mlp.experts.86.up_proj", + "model.layers.8.mlp.experts.87.down_proj", + "model.layers.8.mlp.experts.87.gate_proj", + "model.layers.8.mlp.experts.87.up_proj", + "model.layers.8.mlp.experts.88.down_proj", + "model.layers.8.mlp.experts.88.gate_proj", + "model.layers.8.mlp.experts.88.up_proj", + "model.layers.8.mlp.experts.89.down_proj", + "model.layers.8.mlp.experts.89.gate_proj", + "model.layers.8.mlp.experts.89.up_proj", + "model.layers.8.mlp.experts.9.down_proj", + "model.layers.8.mlp.experts.9.gate_proj", + "model.layers.8.mlp.experts.9.up_proj", + "model.layers.8.mlp.experts.90.down_proj", + "model.layers.8.mlp.experts.90.gate_proj", + "model.layers.8.mlp.experts.90.up_proj", + "model.layers.8.mlp.experts.91.down_proj", + "model.layers.8.mlp.experts.91.gate_proj", + "model.layers.8.mlp.experts.91.up_proj", + "model.layers.8.mlp.experts.92.down_proj", + "model.layers.8.mlp.experts.92.gate_proj", + "model.layers.8.mlp.experts.92.up_proj", + "model.layers.8.mlp.experts.93.down_proj", + "model.layers.8.mlp.experts.93.gate_proj", + "model.layers.8.mlp.experts.93.up_proj", + "model.layers.8.mlp.experts.94.down_proj", + "model.layers.8.mlp.experts.94.gate_proj", + "model.layers.8.mlp.experts.94.up_proj", + "model.layers.8.mlp.experts.95.down_proj", + "model.layers.8.mlp.experts.95.gate_proj", + "model.layers.8.mlp.experts.95.up_proj", + "model.layers.8.mlp.experts.96.down_proj", + "model.layers.8.mlp.experts.96.gate_proj", + "model.layers.8.mlp.experts.96.up_proj", + "model.layers.8.mlp.experts.97.down_proj", + "model.layers.8.mlp.experts.97.gate_proj", + "model.layers.8.mlp.experts.97.up_proj", + "model.layers.8.mlp.experts.98.down_proj", + "model.layers.8.mlp.experts.98.gate_proj", + "model.layers.8.mlp.experts.98.up_proj", + "model.layers.8.mlp.experts.99.down_proj", + "model.layers.8.mlp.experts.99.gate_proj", + "model.layers.8.mlp.experts.99.up_proj", + "model.layers.9.mlp.experts.0.down_proj", + "model.layers.9.mlp.experts.0.gate_proj", + "model.layers.9.mlp.experts.0.up_proj", + "model.layers.9.mlp.experts.1.down_proj", + "model.layers.9.mlp.experts.1.gate_proj", + "model.layers.9.mlp.experts.1.up_proj", + "model.layers.9.mlp.experts.10.down_proj", + "model.layers.9.mlp.experts.10.gate_proj", + "model.layers.9.mlp.experts.10.up_proj", + "model.layers.9.mlp.experts.100.down_proj", + "model.layers.9.mlp.experts.100.gate_proj", + "model.layers.9.mlp.experts.100.up_proj", + "model.layers.9.mlp.experts.101.down_proj", + "model.layers.9.mlp.experts.101.gate_proj", + "model.layers.9.mlp.experts.101.up_proj", + "model.layers.9.mlp.experts.102.down_proj", + "model.layers.9.mlp.experts.102.gate_proj", + "model.layers.9.mlp.experts.102.up_proj", + "model.layers.9.mlp.experts.103.down_proj", + "model.layers.9.mlp.experts.103.gate_proj", + "model.layers.9.mlp.experts.103.up_proj", + "model.layers.9.mlp.experts.104.down_proj", + "model.layers.9.mlp.experts.104.gate_proj", + "model.layers.9.mlp.experts.104.up_proj", + "model.layers.9.mlp.experts.105.down_proj", + "model.layers.9.mlp.experts.105.gate_proj", + "model.layers.9.mlp.experts.105.up_proj", + "model.layers.9.mlp.experts.106.down_proj", + "model.layers.9.mlp.experts.106.gate_proj", + "model.layers.9.mlp.experts.106.up_proj", + "model.layers.9.mlp.experts.107.down_proj", + "model.layers.9.mlp.experts.107.gate_proj", + "model.layers.9.mlp.experts.107.up_proj", + "model.layers.9.mlp.experts.108.down_proj", + "model.layers.9.mlp.experts.108.gate_proj", + "model.layers.9.mlp.experts.108.up_proj", + "model.layers.9.mlp.experts.109.down_proj", + "model.layers.9.mlp.experts.109.gate_proj", + "model.layers.9.mlp.experts.109.up_proj", + "model.layers.9.mlp.experts.11.down_proj", + "model.layers.9.mlp.experts.11.gate_proj", + "model.layers.9.mlp.experts.11.up_proj", + "model.layers.9.mlp.experts.110.down_proj", + "model.layers.9.mlp.experts.110.gate_proj", + "model.layers.9.mlp.experts.110.up_proj", + "model.layers.9.mlp.experts.111.down_proj", + "model.layers.9.mlp.experts.111.gate_proj", + "model.layers.9.mlp.experts.111.up_proj", + "model.layers.9.mlp.experts.112.down_proj", + "model.layers.9.mlp.experts.112.gate_proj", + "model.layers.9.mlp.experts.112.up_proj", + "model.layers.9.mlp.experts.113.down_proj", + "model.layers.9.mlp.experts.113.gate_proj", + "model.layers.9.mlp.experts.113.up_proj", + "model.layers.9.mlp.experts.114.down_proj", + "model.layers.9.mlp.experts.114.gate_proj", + "model.layers.9.mlp.experts.114.up_proj", + "model.layers.9.mlp.experts.115.down_proj", + "model.layers.9.mlp.experts.115.gate_proj", + "model.layers.9.mlp.experts.115.up_proj", + "model.layers.9.mlp.experts.116.down_proj", + "model.layers.9.mlp.experts.116.gate_proj", + "model.layers.9.mlp.experts.116.up_proj", + "model.layers.9.mlp.experts.117.down_proj", + "model.layers.9.mlp.experts.117.gate_proj", + "model.layers.9.mlp.experts.117.up_proj", + "model.layers.9.mlp.experts.118.down_proj", + "model.layers.9.mlp.experts.118.gate_proj", + "model.layers.9.mlp.experts.118.up_proj", + "model.layers.9.mlp.experts.119.down_proj", + "model.layers.9.mlp.experts.119.gate_proj", + "model.layers.9.mlp.experts.119.up_proj", + "model.layers.9.mlp.experts.12.down_proj", + "model.layers.9.mlp.experts.12.gate_proj", + "model.layers.9.mlp.experts.12.up_proj", + "model.layers.9.mlp.experts.120.down_proj", + "model.layers.9.mlp.experts.120.gate_proj", + "model.layers.9.mlp.experts.120.up_proj", + "model.layers.9.mlp.experts.121.down_proj", + "model.layers.9.mlp.experts.121.gate_proj", + "model.layers.9.mlp.experts.121.up_proj", + "model.layers.9.mlp.experts.122.down_proj", + "model.layers.9.mlp.experts.122.gate_proj", + "model.layers.9.mlp.experts.122.up_proj", + "model.layers.9.mlp.experts.123.down_proj", + "model.layers.9.mlp.experts.123.gate_proj", + "model.layers.9.mlp.experts.123.up_proj", + "model.layers.9.mlp.experts.124.down_proj", + "model.layers.9.mlp.experts.124.gate_proj", + "model.layers.9.mlp.experts.124.up_proj", + "model.layers.9.mlp.experts.125.down_proj", + "model.layers.9.mlp.experts.125.gate_proj", + "model.layers.9.mlp.experts.125.up_proj", + "model.layers.9.mlp.experts.126.down_proj", + "model.layers.9.mlp.experts.126.gate_proj", + "model.layers.9.mlp.experts.126.up_proj", + "model.layers.9.mlp.experts.127.down_proj", + "model.layers.9.mlp.experts.127.gate_proj", + "model.layers.9.mlp.experts.127.up_proj", + "model.layers.9.mlp.experts.128.down_proj", + "model.layers.9.mlp.experts.128.gate_proj", + "model.layers.9.mlp.experts.128.up_proj", + "model.layers.9.mlp.experts.129.down_proj", + "model.layers.9.mlp.experts.129.gate_proj", + "model.layers.9.mlp.experts.129.up_proj", + "model.layers.9.mlp.experts.13.down_proj", + "model.layers.9.mlp.experts.13.gate_proj", + "model.layers.9.mlp.experts.13.up_proj", + "model.layers.9.mlp.experts.130.down_proj", + "model.layers.9.mlp.experts.130.gate_proj", + "model.layers.9.mlp.experts.130.up_proj", + "model.layers.9.mlp.experts.131.down_proj", + "model.layers.9.mlp.experts.131.gate_proj", + "model.layers.9.mlp.experts.131.up_proj", + "model.layers.9.mlp.experts.132.down_proj", + "model.layers.9.mlp.experts.132.gate_proj", + "model.layers.9.mlp.experts.132.up_proj", + "model.layers.9.mlp.experts.133.down_proj", + "model.layers.9.mlp.experts.133.gate_proj", + "model.layers.9.mlp.experts.133.up_proj", + "model.layers.9.mlp.experts.134.down_proj", + "model.layers.9.mlp.experts.134.gate_proj", + "model.layers.9.mlp.experts.134.up_proj", + "model.layers.9.mlp.experts.135.down_proj", + "model.layers.9.mlp.experts.135.gate_proj", + "model.layers.9.mlp.experts.135.up_proj", + "model.layers.9.mlp.experts.136.down_proj", + "model.layers.9.mlp.experts.136.gate_proj", + "model.layers.9.mlp.experts.136.up_proj", + "model.layers.9.mlp.experts.137.down_proj", + "model.layers.9.mlp.experts.137.gate_proj", + "model.layers.9.mlp.experts.137.up_proj", + "model.layers.9.mlp.experts.138.down_proj", + "model.layers.9.mlp.experts.138.gate_proj", + "model.layers.9.mlp.experts.138.up_proj", + "model.layers.9.mlp.experts.139.down_proj", + "model.layers.9.mlp.experts.139.gate_proj", + "model.layers.9.mlp.experts.139.up_proj", + "model.layers.9.mlp.experts.14.down_proj", + "model.layers.9.mlp.experts.14.gate_proj", + "model.layers.9.mlp.experts.14.up_proj", + "model.layers.9.mlp.experts.140.down_proj", + "model.layers.9.mlp.experts.140.gate_proj", + "model.layers.9.mlp.experts.140.up_proj", + "model.layers.9.mlp.experts.141.down_proj", + "model.layers.9.mlp.experts.141.gate_proj", + "model.layers.9.mlp.experts.141.up_proj", + "model.layers.9.mlp.experts.142.down_proj", + "model.layers.9.mlp.experts.142.gate_proj", + "model.layers.9.mlp.experts.142.up_proj", + "model.layers.9.mlp.experts.143.down_proj", + "model.layers.9.mlp.experts.143.gate_proj", + "model.layers.9.mlp.experts.143.up_proj", + "model.layers.9.mlp.experts.144.down_proj", + "model.layers.9.mlp.experts.144.gate_proj", + "model.layers.9.mlp.experts.144.up_proj", + "model.layers.9.mlp.experts.145.down_proj", + "model.layers.9.mlp.experts.145.gate_proj", + "model.layers.9.mlp.experts.145.up_proj", + "model.layers.9.mlp.experts.146.down_proj", + "model.layers.9.mlp.experts.146.gate_proj", + "model.layers.9.mlp.experts.146.up_proj", + "model.layers.9.mlp.experts.147.down_proj", + "model.layers.9.mlp.experts.147.gate_proj", + "model.layers.9.mlp.experts.147.up_proj", + "model.layers.9.mlp.experts.148.down_proj", + "model.layers.9.mlp.experts.148.gate_proj", + "model.layers.9.mlp.experts.148.up_proj", + "model.layers.9.mlp.experts.149.down_proj", + "model.layers.9.mlp.experts.149.gate_proj", + "model.layers.9.mlp.experts.149.up_proj", + "model.layers.9.mlp.experts.15.down_proj", + "model.layers.9.mlp.experts.15.gate_proj", + "model.layers.9.mlp.experts.15.up_proj", + "model.layers.9.mlp.experts.150.down_proj", + "model.layers.9.mlp.experts.150.gate_proj", + "model.layers.9.mlp.experts.150.up_proj", + "model.layers.9.mlp.experts.151.down_proj", + "model.layers.9.mlp.experts.151.gate_proj", + "model.layers.9.mlp.experts.151.up_proj", + "model.layers.9.mlp.experts.152.down_proj", + "model.layers.9.mlp.experts.152.gate_proj", + "model.layers.9.mlp.experts.152.up_proj", + "model.layers.9.mlp.experts.153.down_proj", + "model.layers.9.mlp.experts.153.gate_proj", + "model.layers.9.mlp.experts.153.up_proj", + "model.layers.9.mlp.experts.154.down_proj", + "model.layers.9.mlp.experts.154.gate_proj", + "model.layers.9.mlp.experts.154.up_proj", + "model.layers.9.mlp.experts.155.down_proj", + "model.layers.9.mlp.experts.155.gate_proj", + "model.layers.9.mlp.experts.155.up_proj", + "model.layers.9.mlp.experts.156.down_proj", + "model.layers.9.mlp.experts.156.gate_proj", + "model.layers.9.mlp.experts.156.up_proj", + "model.layers.9.mlp.experts.157.down_proj", + "model.layers.9.mlp.experts.157.gate_proj", + "model.layers.9.mlp.experts.157.up_proj", + "model.layers.9.mlp.experts.158.down_proj", + "model.layers.9.mlp.experts.158.gate_proj", + "model.layers.9.mlp.experts.158.up_proj", + "model.layers.9.mlp.experts.159.down_proj", + "model.layers.9.mlp.experts.159.gate_proj", + "model.layers.9.mlp.experts.159.up_proj", + "model.layers.9.mlp.experts.16.down_proj", + "model.layers.9.mlp.experts.16.gate_proj", + "model.layers.9.mlp.experts.16.up_proj", + "model.layers.9.mlp.experts.160.down_proj", + "model.layers.9.mlp.experts.160.gate_proj", + "model.layers.9.mlp.experts.160.up_proj", + "model.layers.9.mlp.experts.161.down_proj", + "model.layers.9.mlp.experts.161.gate_proj", + "model.layers.9.mlp.experts.161.up_proj", + "model.layers.9.mlp.experts.162.down_proj", + "model.layers.9.mlp.experts.162.gate_proj", + "model.layers.9.mlp.experts.162.up_proj", + "model.layers.9.mlp.experts.163.down_proj", + "model.layers.9.mlp.experts.163.gate_proj", + "model.layers.9.mlp.experts.163.up_proj", + "model.layers.9.mlp.experts.164.down_proj", + "model.layers.9.mlp.experts.164.gate_proj", + "model.layers.9.mlp.experts.164.up_proj", + "model.layers.9.mlp.experts.165.down_proj", + "model.layers.9.mlp.experts.165.gate_proj", + "model.layers.9.mlp.experts.165.up_proj", + "model.layers.9.mlp.experts.166.down_proj", + "model.layers.9.mlp.experts.166.gate_proj", + "model.layers.9.mlp.experts.166.up_proj", + "model.layers.9.mlp.experts.167.down_proj", + "model.layers.9.mlp.experts.167.gate_proj", + "model.layers.9.mlp.experts.167.up_proj", + "model.layers.9.mlp.experts.168.down_proj", + "model.layers.9.mlp.experts.168.gate_proj", + "model.layers.9.mlp.experts.168.up_proj", + "model.layers.9.mlp.experts.169.down_proj", + "model.layers.9.mlp.experts.169.gate_proj", + "model.layers.9.mlp.experts.169.up_proj", + "model.layers.9.mlp.experts.17.down_proj", + "model.layers.9.mlp.experts.17.gate_proj", + "model.layers.9.mlp.experts.17.up_proj", + "model.layers.9.mlp.experts.170.down_proj", + "model.layers.9.mlp.experts.170.gate_proj", + "model.layers.9.mlp.experts.170.up_proj", + "model.layers.9.mlp.experts.171.down_proj", + "model.layers.9.mlp.experts.171.gate_proj", + "model.layers.9.mlp.experts.171.up_proj", + "model.layers.9.mlp.experts.172.down_proj", + "model.layers.9.mlp.experts.172.gate_proj", + "model.layers.9.mlp.experts.172.up_proj", + "model.layers.9.mlp.experts.173.down_proj", + "model.layers.9.mlp.experts.173.gate_proj", + "model.layers.9.mlp.experts.173.up_proj", + "model.layers.9.mlp.experts.174.down_proj", + "model.layers.9.mlp.experts.174.gate_proj", + "model.layers.9.mlp.experts.174.up_proj", + "model.layers.9.mlp.experts.175.down_proj", + "model.layers.9.mlp.experts.175.gate_proj", + "model.layers.9.mlp.experts.175.up_proj", + "model.layers.9.mlp.experts.176.down_proj", + "model.layers.9.mlp.experts.176.gate_proj", + "model.layers.9.mlp.experts.176.up_proj", + "model.layers.9.mlp.experts.177.down_proj", + "model.layers.9.mlp.experts.177.gate_proj", + "model.layers.9.mlp.experts.177.up_proj", + "model.layers.9.mlp.experts.178.down_proj", + "model.layers.9.mlp.experts.178.gate_proj", + "model.layers.9.mlp.experts.178.up_proj", + "model.layers.9.mlp.experts.179.down_proj", + "model.layers.9.mlp.experts.179.gate_proj", + "model.layers.9.mlp.experts.179.up_proj", + "model.layers.9.mlp.experts.18.down_proj", + "model.layers.9.mlp.experts.18.gate_proj", + "model.layers.9.mlp.experts.18.up_proj", + "model.layers.9.mlp.experts.180.down_proj", + "model.layers.9.mlp.experts.180.gate_proj", + "model.layers.9.mlp.experts.180.up_proj", + "model.layers.9.mlp.experts.181.down_proj", + "model.layers.9.mlp.experts.181.gate_proj", + "model.layers.9.mlp.experts.181.up_proj", + "model.layers.9.mlp.experts.182.down_proj", + "model.layers.9.mlp.experts.182.gate_proj", + "model.layers.9.mlp.experts.182.up_proj", + "model.layers.9.mlp.experts.183.down_proj", + "model.layers.9.mlp.experts.183.gate_proj", + "model.layers.9.mlp.experts.183.up_proj", + "model.layers.9.mlp.experts.184.down_proj", + "model.layers.9.mlp.experts.184.gate_proj", + "model.layers.9.mlp.experts.184.up_proj", + "model.layers.9.mlp.experts.185.down_proj", + "model.layers.9.mlp.experts.185.gate_proj", + "model.layers.9.mlp.experts.185.up_proj", + "model.layers.9.mlp.experts.186.down_proj", + "model.layers.9.mlp.experts.186.gate_proj", + "model.layers.9.mlp.experts.186.up_proj", + "model.layers.9.mlp.experts.187.down_proj", + "model.layers.9.mlp.experts.187.gate_proj", + "model.layers.9.mlp.experts.187.up_proj", + "model.layers.9.mlp.experts.188.down_proj", + "model.layers.9.mlp.experts.188.gate_proj", + "model.layers.9.mlp.experts.188.up_proj", + "model.layers.9.mlp.experts.189.down_proj", + "model.layers.9.mlp.experts.189.gate_proj", + "model.layers.9.mlp.experts.189.up_proj", + "model.layers.9.mlp.experts.19.down_proj", + "model.layers.9.mlp.experts.19.gate_proj", + "model.layers.9.mlp.experts.19.up_proj", + "model.layers.9.mlp.experts.190.down_proj", + "model.layers.9.mlp.experts.190.gate_proj", + "model.layers.9.mlp.experts.190.up_proj", + "model.layers.9.mlp.experts.191.down_proj", + "model.layers.9.mlp.experts.191.gate_proj", + "model.layers.9.mlp.experts.191.up_proj", + "model.layers.9.mlp.experts.192.down_proj", + "model.layers.9.mlp.experts.192.gate_proj", + "model.layers.9.mlp.experts.192.up_proj", + "model.layers.9.mlp.experts.193.down_proj", + "model.layers.9.mlp.experts.193.gate_proj", + "model.layers.9.mlp.experts.193.up_proj", + "model.layers.9.mlp.experts.194.down_proj", + "model.layers.9.mlp.experts.194.gate_proj", + "model.layers.9.mlp.experts.194.up_proj", + "model.layers.9.mlp.experts.195.down_proj", + "model.layers.9.mlp.experts.195.gate_proj", + "model.layers.9.mlp.experts.195.up_proj", + "model.layers.9.mlp.experts.196.down_proj", + "model.layers.9.mlp.experts.196.gate_proj", + "model.layers.9.mlp.experts.196.up_proj", + "model.layers.9.mlp.experts.197.down_proj", + "model.layers.9.mlp.experts.197.gate_proj", + "model.layers.9.mlp.experts.197.up_proj", + "model.layers.9.mlp.experts.198.down_proj", + "model.layers.9.mlp.experts.198.gate_proj", + "model.layers.9.mlp.experts.198.up_proj", + "model.layers.9.mlp.experts.199.down_proj", + "model.layers.9.mlp.experts.199.gate_proj", + "model.layers.9.mlp.experts.199.up_proj", + "model.layers.9.mlp.experts.2.down_proj", + "model.layers.9.mlp.experts.2.gate_proj", + "model.layers.9.mlp.experts.2.up_proj", + "model.layers.9.mlp.experts.20.down_proj", + "model.layers.9.mlp.experts.20.gate_proj", + "model.layers.9.mlp.experts.20.up_proj", + "model.layers.9.mlp.experts.200.down_proj", + "model.layers.9.mlp.experts.200.gate_proj", + "model.layers.9.mlp.experts.200.up_proj", + "model.layers.9.mlp.experts.201.down_proj", + "model.layers.9.mlp.experts.201.gate_proj", + "model.layers.9.mlp.experts.201.up_proj", + "model.layers.9.mlp.experts.202.down_proj", + "model.layers.9.mlp.experts.202.gate_proj", + "model.layers.9.mlp.experts.202.up_proj", + "model.layers.9.mlp.experts.203.down_proj", + "model.layers.9.mlp.experts.203.gate_proj", + "model.layers.9.mlp.experts.203.up_proj", + "model.layers.9.mlp.experts.204.down_proj", + "model.layers.9.mlp.experts.204.gate_proj", + "model.layers.9.mlp.experts.204.up_proj", + "model.layers.9.mlp.experts.205.down_proj", + "model.layers.9.mlp.experts.205.gate_proj", + "model.layers.9.mlp.experts.205.up_proj", + "model.layers.9.mlp.experts.206.down_proj", + "model.layers.9.mlp.experts.206.gate_proj", + "model.layers.9.mlp.experts.206.up_proj", + "model.layers.9.mlp.experts.207.down_proj", + "model.layers.9.mlp.experts.207.gate_proj", + "model.layers.9.mlp.experts.207.up_proj", + "model.layers.9.mlp.experts.208.down_proj", + "model.layers.9.mlp.experts.208.gate_proj", + "model.layers.9.mlp.experts.208.up_proj", + "model.layers.9.mlp.experts.209.down_proj", + "model.layers.9.mlp.experts.209.gate_proj", + "model.layers.9.mlp.experts.209.up_proj", + "model.layers.9.mlp.experts.21.down_proj", + "model.layers.9.mlp.experts.21.gate_proj", + "model.layers.9.mlp.experts.21.up_proj", + "model.layers.9.mlp.experts.210.down_proj", + "model.layers.9.mlp.experts.210.gate_proj", + "model.layers.9.mlp.experts.210.up_proj", + "model.layers.9.mlp.experts.211.down_proj", + "model.layers.9.mlp.experts.211.gate_proj", + "model.layers.9.mlp.experts.211.up_proj", + "model.layers.9.mlp.experts.212.down_proj", + "model.layers.9.mlp.experts.212.gate_proj", + "model.layers.9.mlp.experts.212.up_proj", + "model.layers.9.mlp.experts.213.down_proj", + "model.layers.9.mlp.experts.213.gate_proj", + "model.layers.9.mlp.experts.213.up_proj", + "model.layers.9.mlp.experts.214.down_proj", + "model.layers.9.mlp.experts.214.gate_proj", + "model.layers.9.mlp.experts.214.up_proj", + "model.layers.9.mlp.experts.215.down_proj", + "model.layers.9.mlp.experts.215.gate_proj", + "model.layers.9.mlp.experts.215.up_proj", + "model.layers.9.mlp.experts.216.down_proj", + "model.layers.9.mlp.experts.216.gate_proj", + "model.layers.9.mlp.experts.216.up_proj", + "model.layers.9.mlp.experts.217.down_proj", + "model.layers.9.mlp.experts.217.gate_proj", + "model.layers.9.mlp.experts.217.up_proj", + "model.layers.9.mlp.experts.218.down_proj", + "model.layers.9.mlp.experts.218.gate_proj", + "model.layers.9.mlp.experts.218.up_proj", + "model.layers.9.mlp.experts.219.down_proj", + "model.layers.9.mlp.experts.219.gate_proj", + "model.layers.9.mlp.experts.219.up_proj", + "model.layers.9.mlp.experts.22.down_proj", + "model.layers.9.mlp.experts.22.gate_proj", + "model.layers.9.mlp.experts.22.up_proj", + "model.layers.9.mlp.experts.220.down_proj", + "model.layers.9.mlp.experts.220.gate_proj", + "model.layers.9.mlp.experts.220.up_proj", + "model.layers.9.mlp.experts.221.down_proj", + "model.layers.9.mlp.experts.221.gate_proj", + "model.layers.9.mlp.experts.221.up_proj", + "model.layers.9.mlp.experts.222.down_proj", + "model.layers.9.mlp.experts.222.gate_proj", + "model.layers.9.mlp.experts.222.up_proj", + "model.layers.9.mlp.experts.223.down_proj", + "model.layers.9.mlp.experts.223.gate_proj", + "model.layers.9.mlp.experts.223.up_proj", + "model.layers.9.mlp.experts.224.down_proj", + "model.layers.9.mlp.experts.224.gate_proj", + "model.layers.9.mlp.experts.224.up_proj", + "model.layers.9.mlp.experts.225.down_proj", + "model.layers.9.mlp.experts.225.gate_proj", + "model.layers.9.mlp.experts.225.up_proj", + "model.layers.9.mlp.experts.226.down_proj", + "model.layers.9.mlp.experts.226.gate_proj", + "model.layers.9.mlp.experts.226.up_proj", + "model.layers.9.mlp.experts.227.down_proj", + "model.layers.9.mlp.experts.227.gate_proj", + "model.layers.9.mlp.experts.227.up_proj", + "model.layers.9.mlp.experts.228.down_proj", + "model.layers.9.mlp.experts.228.gate_proj", + "model.layers.9.mlp.experts.228.up_proj", + "model.layers.9.mlp.experts.229.down_proj", + "model.layers.9.mlp.experts.229.gate_proj", + "model.layers.9.mlp.experts.229.up_proj", + "model.layers.9.mlp.experts.23.down_proj", + "model.layers.9.mlp.experts.23.gate_proj", + "model.layers.9.mlp.experts.23.up_proj", + "model.layers.9.mlp.experts.230.down_proj", + "model.layers.9.mlp.experts.230.gate_proj", + "model.layers.9.mlp.experts.230.up_proj", + "model.layers.9.mlp.experts.231.down_proj", + "model.layers.9.mlp.experts.231.gate_proj", + "model.layers.9.mlp.experts.231.up_proj", + "model.layers.9.mlp.experts.232.down_proj", + "model.layers.9.mlp.experts.232.gate_proj", + "model.layers.9.mlp.experts.232.up_proj", + "model.layers.9.mlp.experts.233.down_proj", + "model.layers.9.mlp.experts.233.gate_proj", + "model.layers.9.mlp.experts.233.up_proj", + "model.layers.9.mlp.experts.234.down_proj", + "model.layers.9.mlp.experts.234.gate_proj", + "model.layers.9.mlp.experts.234.up_proj", + "model.layers.9.mlp.experts.235.down_proj", + "model.layers.9.mlp.experts.235.gate_proj", + "model.layers.9.mlp.experts.235.up_proj", + "model.layers.9.mlp.experts.236.down_proj", + "model.layers.9.mlp.experts.236.gate_proj", + "model.layers.9.mlp.experts.236.up_proj", + "model.layers.9.mlp.experts.237.down_proj", + "model.layers.9.mlp.experts.237.gate_proj", + "model.layers.9.mlp.experts.237.up_proj", + "model.layers.9.mlp.experts.238.down_proj", + "model.layers.9.mlp.experts.238.gate_proj", + "model.layers.9.mlp.experts.238.up_proj", + "model.layers.9.mlp.experts.239.down_proj", + "model.layers.9.mlp.experts.239.gate_proj", + "model.layers.9.mlp.experts.239.up_proj", + "model.layers.9.mlp.experts.24.down_proj", + "model.layers.9.mlp.experts.24.gate_proj", + "model.layers.9.mlp.experts.24.up_proj", + "model.layers.9.mlp.experts.240.down_proj", + "model.layers.9.mlp.experts.240.gate_proj", + "model.layers.9.mlp.experts.240.up_proj", + "model.layers.9.mlp.experts.241.down_proj", + "model.layers.9.mlp.experts.241.gate_proj", + "model.layers.9.mlp.experts.241.up_proj", + "model.layers.9.mlp.experts.242.down_proj", + "model.layers.9.mlp.experts.242.gate_proj", + "model.layers.9.mlp.experts.242.up_proj", + "model.layers.9.mlp.experts.243.down_proj", + "model.layers.9.mlp.experts.243.gate_proj", + "model.layers.9.mlp.experts.243.up_proj", + "model.layers.9.mlp.experts.244.down_proj", + "model.layers.9.mlp.experts.244.gate_proj", + "model.layers.9.mlp.experts.244.up_proj", + "model.layers.9.mlp.experts.245.down_proj", + "model.layers.9.mlp.experts.245.gate_proj", + "model.layers.9.mlp.experts.245.up_proj", + "model.layers.9.mlp.experts.246.down_proj", + "model.layers.9.mlp.experts.246.gate_proj", + "model.layers.9.mlp.experts.246.up_proj", + "model.layers.9.mlp.experts.247.down_proj", + "model.layers.9.mlp.experts.247.gate_proj", + "model.layers.9.mlp.experts.247.up_proj", + "model.layers.9.mlp.experts.248.down_proj", + "model.layers.9.mlp.experts.248.gate_proj", + "model.layers.9.mlp.experts.248.up_proj", + "model.layers.9.mlp.experts.249.down_proj", + "model.layers.9.mlp.experts.249.gate_proj", + "model.layers.9.mlp.experts.249.up_proj", + "model.layers.9.mlp.experts.25.down_proj", + "model.layers.9.mlp.experts.25.gate_proj", + "model.layers.9.mlp.experts.25.up_proj", + "model.layers.9.mlp.experts.250.down_proj", + "model.layers.9.mlp.experts.250.gate_proj", + "model.layers.9.mlp.experts.250.up_proj", + "model.layers.9.mlp.experts.251.down_proj", + "model.layers.9.mlp.experts.251.gate_proj", + "model.layers.9.mlp.experts.251.up_proj", + "model.layers.9.mlp.experts.252.down_proj", + "model.layers.9.mlp.experts.252.gate_proj", + "model.layers.9.mlp.experts.252.up_proj", + "model.layers.9.mlp.experts.253.down_proj", + "model.layers.9.mlp.experts.253.gate_proj", + "model.layers.9.mlp.experts.253.up_proj", + "model.layers.9.mlp.experts.254.down_proj", + "model.layers.9.mlp.experts.254.gate_proj", + "model.layers.9.mlp.experts.254.up_proj", + "model.layers.9.mlp.experts.255.down_proj", + "model.layers.9.mlp.experts.255.gate_proj", + "model.layers.9.mlp.experts.255.up_proj", + "model.layers.9.mlp.experts.26.down_proj", + "model.layers.9.mlp.experts.26.gate_proj", + "model.layers.9.mlp.experts.26.up_proj", + "model.layers.9.mlp.experts.27.down_proj", + "model.layers.9.mlp.experts.27.gate_proj", + "model.layers.9.mlp.experts.27.up_proj", + "model.layers.9.mlp.experts.28.down_proj", + "model.layers.9.mlp.experts.28.gate_proj", + "model.layers.9.mlp.experts.28.up_proj", + "model.layers.9.mlp.experts.29.down_proj", + "model.layers.9.mlp.experts.29.gate_proj", + "model.layers.9.mlp.experts.29.up_proj", + "model.layers.9.mlp.experts.3.down_proj", + "model.layers.9.mlp.experts.3.gate_proj", + "model.layers.9.mlp.experts.3.up_proj", + "model.layers.9.mlp.experts.30.down_proj", + "model.layers.9.mlp.experts.30.gate_proj", + "model.layers.9.mlp.experts.30.up_proj", + "model.layers.9.mlp.experts.31.down_proj", + "model.layers.9.mlp.experts.31.gate_proj", + "model.layers.9.mlp.experts.31.up_proj", + "model.layers.9.mlp.experts.32.down_proj", + "model.layers.9.mlp.experts.32.gate_proj", + "model.layers.9.mlp.experts.32.up_proj", + "model.layers.9.mlp.experts.33.down_proj", + "model.layers.9.mlp.experts.33.gate_proj", + "model.layers.9.mlp.experts.33.up_proj", + "model.layers.9.mlp.experts.34.down_proj", + "model.layers.9.mlp.experts.34.gate_proj", + "model.layers.9.mlp.experts.34.up_proj", + "model.layers.9.mlp.experts.35.down_proj", + "model.layers.9.mlp.experts.35.gate_proj", + "model.layers.9.mlp.experts.35.up_proj", + "model.layers.9.mlp.experts.36.down_proj", + "model.layers.9.mlp.experts.36.gate_proj", + "model.layers.9.mlp.experts.36.up_proj", + "model.layers.9.mlp.experts.37.down_proj", + "model.layers.9.mlp.experts.37.gate_proj", + "model.layers.9.mlp.experts.37.up_proj", + "model.layers.9.mlp.experts.38.down_proj", + "model.layers.9.mlp.experts.38.gate_proj", + "model.layers.9.mlp.experts.38.up_proj", + "model.layers.9.mlp.experts.39.down_proj", + "model.layers.9.mlp.experts.39.gate_proj", + "model.layers.9.mlp.experts.39.up_proj", + "model.layers.9.mlp.experts.4.down_proj", + "model.layers.9.mlp.experts.4.gate_proj", + "model.layers.9.mlp.experts.4.up_proj", + "model.layers.9.mlp.experts.40.down_proj", + "model.layers.9.mlp.experts.40.gate_proj", + "model.layers.9.mlp.experts.40.up_proj", + "model.layers.9.mlp.experts.41.down_proj", + "model.layers.9.mlp.experts.41.gate_proj", + "model.layers.9.mlp.experts.41.up_proj", + "model.layers.9.mlp.experts.42.down_proj", + "model.layers.9.mlp.experts.42.gate_proj", + "model.layers.9.mlp.experts.42.up_proj", + "model.layers.9.mlp.experts.43.down_proj", + "model.layers.9.mlp.experts.43.gate_proj", + "model.layers.9.mlp.experts.43.up_proj", + "model.layers.9.mlp.experts.44.down_proj", + "model.layers.9.mlp.experts.44.gate_proj", + "model.layers.9.mlp.experts.44.up_proj", + "model.layers.9.mlp.experts.45.down_proj", + "model.layers.9.mlp.experts.45.gate_proj", + "model.layers.9.mlp.experts.45.up_proj", + "model.layers.9.mlp.experts.46.down_proj", + "model.layers.9.mlp.experts.46.gate_proj", + "model.layers.9.mlp.experts.46.up_proj", + "model.layers.9.mlp.experts.47.down_proj", + "model.layers.9.mlp.experts.47.gate_proj", + "model.layers.9.mlp.experts.47.up_proj", + "model.layers.9.mlp.experts.48.down_proj", + "model.layers.9.mlp.experts.48.gate_proj", + "model.layers.9.mlp.experts.48.up_proj", + "model.layers.9.mlp.experts.49.down_proj", + "model.layers.9.mlp.experts.49.gate_proj", + "model.layers.9.mlp.experts.49.up_proj", + "model.layers.9.mlp.experts.5.down_proj", + "model.layers.9.mlp.experts.5.gate_proj", + "model.layers.9.mlp.experts.5.up_proj", + "model.layers.9.mlp.experts.50.down_proj", + "model.layers.9.mlp.experts.50.gate_proj", + "model.layers.9.mlp.experts.50.up_proj", + "model.layers.9.mlp.experts.51.down_proj", + "model.layers.9.mlp.experts.51.gate_proj", + "model.layers.9.mlp.experts.51.up_proj", + "model.layers.9.mlp.experts.52.down_proj", + "model.layers.9.mlp.experts.52.gate_proj", + "model.layers.9.mlp.experts.52.up_proj", + "model.layers.9.mlp.experts.53.down_proj", + "model.layers.9.mlp.experts.53.gate_proj", + "model.layers.9.mlp.experts.53.up_proj", + "model.layers.9.mlp.experts.54.down_proj", + "model.layers.9.mlp.experts.54.gate_proj", + "model.layers.9.mlp.experts.54.up_proj", + "model.layers.9.mlp.experts.55.down_proj", + "model.layers.9.mlp.experts.55.gate_proj", + "model.layers.9.mlp.experts.55.up_proj", + "model.layers.9.mlp.experts.56.down_proj", + "model.layers.9.mlp.experts.56.gate_proj", + "model.layers.9.mlp.experts.56.up_proj", + "model.layers.9.mlp.experts.57.down_proj", + "model.layers.9.mlp.experts.57.gate_proj", + "model.layers.9.mlp.experts.57.up_proj", + "model.layers.9.mlp.experts.58.down_proj", + "model.layers.9.mlp.experts.58.gate_proj", + "model.layers.9.mlp.experts.58.up_proj", + "model.layers.9.mlp.experts.59.down_proj", + "model.layers.9.mlp.experts.59.gate_proj", + "model.layers.9.mlp.experts.59.up_proj", + "model.layers.9.mlp.experts.6.down_proj", + "model.layers.9.mlp.experts.6.gate_proj", + "model.layers.9.mlp.experts.6.up_proj", + "model.layers.9.mlp.experts.60.down_proj", + "model.layers.9.mlp.experts.60.gate_proj", + "model.layers.9.mlp.experts.60.up_proj", + "model.layers.9.mlp.experts.61.down_proj", + "model.layers.9.mlp.experts.61.gate_proj", + "model.layers.9.mlp.experts.61.up_proj", + "model.layers.9.mlp.experts.62.down_proj", + "model.layers.9.mlp.experts.62.gate_proj", + "model.layers.9.mlp.experts.62.up_proj", + "model.layers.9.mlp.experts.63.down_proj", + "model.layers.9.mlp.experts.63.gate_proj", + "model.layers.9.mlp.experts.63.up_proj", + "model.layers.9.mlp.experts.64.down_proj", + "model.layers.9.mlp.experts.64.gate_proj", + "model.layers.9.mlp.experts.64.up_proj", + "model.layers.9.mlp.experts.65.down_proj", + "model.layers.9.mlp.experts.65.gate_proj", + "model.layers.9.mlp.experts.65.up_proj", + "model.layers.9.mlp.experts.66.down_proj", + "model.layers.9.mlp.experts.66.gate_proj", + "model.layers.9.mlp.experts.66.up_proj", + "model.layers.9.mlp.experts.67.down_proj", + "model.layers.9.mlp.experts.67.gate_proj", + "model.layers.9.mlp.experts.67.up_proj", + "model.layers.9.mlp.experts.68.down_proj", + "model.layers.9.mlp.experts.68.gate_proj", + "model.layers.9.mlp.experts.68.up_proj", + "model.layers.9.mlp.experts.69.down_proj", + "model.layers.9.mlp.experts.69.gate_proj", + "model.layers.9.mlp.experts.69.up_proj", + "model.layers.9.mlp.experts.7.down_proj", + "model.layers.9.mlp.experts.7.gate_proj", + "model.layers.9.mlp.experts.7.up_proj", + "model.layers.9.mlp.experts.70.down_proj", + "model.layers.9.mlp.experts.70.gate_proj", + "model.layers.9.mlp.experts.70.up_proj", + "model.layers.9.mlp.experts.71.down_proj", + "model.layers.9.mlp.experts.71.gate_proj", + "model.layers.9.mlp.experts.71.up_proj", + "model.layers.9.mlp.experts.72.down_proj", + "model.layers.9.mlp.experts.72.gate_proj", + "model.layers.9.mlp.experts.72.up_proj", + "model.layers.9.mlp.experts.73.down_proj", + "model.layers.9.mlp.experts.73.gate_proj", + "model.layers.9.mlp.experts.73.up_proj", + "model.layers.9.mlp.experts.74.down_proj", + "model.layers.9.mlp.experts.74.gate_proj", + "model.layers.9.mlp.experts.74.up_proj", + "model.layers.9.mlp.experts.75.down_proj", + "model.layers.9.mlp.experts.75.gate_proj", + "model.layers.9.mlp.experts.75.up_proj", + "model.layers.9.mlp.experts.76.down_proj", + "model.layers.9.mlp.experts.76.gate_proj", + "model.layers.9.mlp.experts.76.up_proj", + "model.layers.9.mlp.experts.77.down_proj", + "model.layers.9.mlp.experts.77.gate_proj", + "model.layers.9.mlp.experts.77.up_proj", + "model.layers.9.mlp.experts.78.down_proj", + "model.layers.9.mlp.experts.78.gate_proj", + "model.layers.9.mlp.experts.78.up_proj", + "model.layers.9.mlp.experts.79.down_proj", + "model.layers.9.mlp.experts.79.gate_proj", + "model.layers.9.mlp.experts.79.up_proj", + "model.layers.9.mlp.experts.8.down_proj", + "model.layers.9.mlp.experts.8.gate_proj", + "model.layers.9.mlp.experts.8.up_proj", + "model.layers.9.mlp.experts.80.down_proj", + "model.layers.9.mlp.experts.80.gate_proj", + "model.layers.9.mlp.experts.80.up_proj", + "model.layers.9.mlp.experts.81.down_proj", + "model.layers.9.mlp.experts.81.gate_proj", + "model.layers.9.mlp.experts.81.up_proj", + "model.layers.9.mlp.experts.82.down_proj", + "model.layers.9.mlp.experts.82.gate_proj", + "model.layers.9.mlp.experts.82.up_proj", + "model.layers.9.mlp.experts.83.down_proj", + "model.layers.9.mlp.experts.83.gate_proj", + "model.layers.9.mlp.experts.83.up_proj", + "model.layers.9.mlp.experts.84.down_proj", + "model.layers.9.mlp.experts.84.gate_proj", + "model.layers.9.mlp.experts.84.up_proj", + "model.layers.9.mlp.experts.85.down_proj", + "model.layers.9.mlp.experts.85.gate_proj", + "model.layers.9.mlp.experts.85.up_proj", + "model.layers.9.mlp.experts.86.down_proj", + "model.layers.9.mlp.experts.86.gate_proj", + "model.layers.9.mlp.experts.86.up_proj", + "model.layers.9.mlp.experts.87.down_proj", + "model.layers.9.mlp.experts.87.gate_proj", + "model.layers.9.mlp.experts.87.up_proj", + "model.layers.9.mlp.experts.88.down_proj", + "model.layers.9.mlp.experts.88.gate_proj", + "model.layers.9.mlp.experts.88.up_proj", + "model.layers.9.mlp.experts.89.down_proj", + "model.layers.9.mlp.experts.89.gate_proj", + "model.layers.9.mlp.experts.89.up_proj", + "model.layers.9.mlp.experts.9.down_proj", + "model.layers.9.mlp.experts.9.gate_proj", + "model.layers.9.mlp.experts.9.up_proj", + "model.layers.9.mlp.experts.90.down_proj", + "model.layers.9.mlp.experts.90.gate_proj", + "model.layers.9.mlp.experts.90.up_proj", + "model.layers.9.mlp.experts.91.down_proj", + "model.layers.9.mlp.experts.91.gate_proj", + "model.layers.9.mlp.experts.91.up_proj", + "model.layers.9.mlp.experts.92.down_proj", + "model.layers.9.mlp.experts.92.gate_proj", + "model.layers.9.mlp.experts.92.up_proj", + "model.layers.9.mlp.experts.93.down_proj", + "model.layers.9.mlp.experts.93.gate_proj", + "model.layers.9.mlp.experts.93.up_proj", + "model.layers.9.mlp.experts.94.down_proj", + "model.layers.9.mlp.experts.94.gate_proj", + "model.layers.9.mlp.experts.94.up_proj", + "model.layers.9.mlp.experts.95.down_proj", + "model.layers.9.mlp.experts.95.gate_proj", + "model.layers.9.mlp.experts.95.up_proj", + "model.layers.9.mlp.experts.96.down_proj", + "model.layers.9.mlp.experts.96.gate_proj", + "model.layers.9.mlp.experts.96.up_proj", + "model.layers.9.mlp.experts.97.down_proj", + "model.layers.9.mlp.experts.97.gate_proj", + "model.layers.9.mlp.experts.97.up_proj", + "model.layers.9.mlp.experts.98.down_proj", + "model.layers.9.mlp.experts.98.gate_proj", + "model.layers.9.mlp.experts.98.up_proj", + "model.layers.9.mlp.experts.99.down_proj", + "model.layers.9.mlp.experts.99.gate_proj", + "model.layers.9.mlp.experts.99.up_proj" + ], + "weights": { + "num_bits": 4, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": false, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": "memoryless_minmax", + "observer_kwargs": {} + }, + "input_activations": { + "num_bits": 4, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": true, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": null, + "observer_kwargs": {} + }, + "output_activations": null, + "format": "mxfp4-pack-quantized" + }, + "group_1": { + "targets": [ + "Linear" + ], + "weights": { + "num_bits": 8, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": false, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": "memoryless_minmax", + "observer_kwargs": {} + }, + "input_activations": { + "num_bits": 8, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": true, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": null, + "observer_kwargs": {} + }, + "output_activations": null, + "format": "mxfp8-quantized" + } + }, + "quant_method": "compressed-tensors", + "kv_cache_scheme": null, + "format": "mixed-precision", + "quantization_status": "compressed", + "global_compression_ratio": null, + "ignore": [ + "lm_head", + "model.embed_tokens", + "model.layers.0.mlp.down_proj", + "model.layers.0.mlp.gate_proj", + "model.layers.0.mlp.up_proj", + "model.layers.0.self_attn.indexer.weights_proj", + "model.layers.1.mlp.down_proj", + "model.layers.1.mlp.gate_proj", + "model.layers.1.mlp.up_proj", + "model.layers.1.self_attn.indexer.weights_proj", + "model.layers.10.mlp.gate", + "model.layers.10.self_attn.indexer.weights_proj", + "model.layers.11.mlp.gate", + "model.layers.12.mlp.gate", + "model.layers.13.mlp.gate", + "model.layers.14.mlp.gate", + "model.layers.14.self_attn.indexer.weights_proj", + "model.layers.15.mlp.gate", + "model.layers.16.mlp.gate", + "model.layers.17.mlp.gate", + "model.layers.18.mlp.gate", + "model.layers.18.self_attn.indexer.weights_proj", + "model.layers.19.mlp.gate", + "model.layers.2.mlp.down_proj", + "model.layers.2.mlp.gate_proj", + "model.layers.2.mlp.up_proj", + "model.layers.2.self_attn.indexer.weights_proj", + "model.layers.20.mlp.gate", + "model.layers.21.mlp.gate", + "model.layers.22.mlp.gate", + "model.layers.22.self_attn.indexer.weights_proj", + "model.layers.23.mlp.gate", + "model.layers.24.mlp.gate", + "model.layers.25.mlp.gate", + "model.layers.26.mlp.gate", + "model.layers.26.self_attn.indexer.weights_proj", + "model.layers.27.mlp.gate", + "model.layers.28.mlp.gate", + "model.layers.29.mlp.gate", + "model.layers.3.mlp.gate", + "model.layers.30.mlp.gate", + "model.layers.30.self_attn.indexer.weights_proj", + "model.layers.31.mlp.gate", + "model.layers.32.mlp.gate", + "model.layers.33.mlp.gate", + "model.layers.34.mlp.gate", + "model.layers.34.self_attn.indexer.weights_proj", + "model.layers.35.mlp.gate", + "model.layers.36.mlp.gate", + "model.layers.37.mlp.gate", + "model.layers.38.mlp.gate", + "model.layers.38.self_attn.indexer.weights_proj", + "model.layers.39.mlp.gate", + "model.layers.4.mlp.gate", + "model.layers.40.mlp.gate", + "model.layers.41.mlp.gate", + "model.layers.42.mlp.gate", + "model.layers.42.self_attn.indexer.weights_proj", + "model.layers.43.mlp.gate", + "model.layers.44.mlp.gate", + "model.layers.45.mlp.gate", + "model.layers.46.mlp.gate", + "model.layers.46.self_attn.indexer.weights_proj", + "model.layers.47.mlp.gate", + "model.layers.48.mlp.gate", + "model.layers.49.mlp.gate", + "model.layers.5.mlp.gate", + "model.layers.50.mlp.gate", + "model.layers.50.self_attn.indexer.weights_proj", + "model.layers.51.mlp.gate", + "model.layers.52.mlp.gate", + "model.layers.53.mlp.gate", + "model.layers.54.mlp.gate", + "model.layers.54.self_attn.indexer.weights_proj", + "model.layers.55.mlp.gate", + "model.layers.56.mlp.gate", + "model.layers.57.mlp.gate", + "model.layers.58.mlp.gate", + "model.layers.58.self_attn.indexer.weights_proj", + "model.layers.59.mlp.gate", + "model.layers.6.mlp.gate", + "model.layers.6.self_attn.indexer.weights_proj", + "model.layers.60.mlp.gate", + "model.layers.61.mlp.gate", + "model.layers.62.mlp.gate", + "model.layers.62.self_attn.indexer.weights_proj", + "model.layers.63.mlp.gate", + "model.layers.64.mlp.gate", + "model.layers.65.mlp.gate", + "model.layers.66.mlp.gate", + "model.layers.66.self_attn.indexer.weights_proj", + "model.layers.67.mlp.gate", + "model.layers.68.mlp.gate", + "model.layers.69.mlp.gate", + "model.layers.7.mlp.gate", + "model.layers.70.mlp.gate", + "model.layers.70.self_attn.indexer.weights_proj", + "model.layers.71.mlp.gate", + "model.layers.72.mlp.gate", + "model.layers.73.mlp.gate", + "model.layers.74.mlp.gate", + "model.layers.74.self_attn.indexer.weights_proj", + "model.layers.75.mlp.gate", + "model.layers.76.mlp.gate", + "model.layers.77.mlp.gate", + "model.layers.78.mlp.gate", + "model.layers.78.self_attn.indexer.weights_proj", + "model.layers.8.mlp.gate", + "model.layers.9.mlp.gate" + ], + "provider": "auto-round" + } +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..216bbb092d0df0dda1dc4cedd789b92286c4c001 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "pad_token_id": 154820, + "temperature": 1.0, + "top_p": 0.95, + "transformers_version": "5.12.0" +} diff --git a/model-00001-of-00282.safetensors b/model-00001-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c6e1311fe774b84955e9e9c779a45a89c5875fff --- /dev/null +++ b/model-00001-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a18dbe8cd31da3435bf1e04b9fa84b7dd5551d2638cdf8be0955ac027243b611 +size 5037826712 diff --git a/model-00002-of-00282.safetensors b/model-00002-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..15e29979b0c790bc98d214e2a2d406cc5f10f869 --- /dev/null +++ b/model-00002-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bc8878fc9e5854148c7a595c0a6043fe998c3a0f2503f8f9c27ea4b4f15d363 +size 1438450752 diff --git a/model-00003-of-00282.safetensors b/model-00003-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1a9d8c46620d0c6eaa601d13a1600c72bbff7a41 --- /dev/null +++ b/model-00003-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:681a8a63acf661c4a83d3f4d40aed7e658f7beda79ab2d86ee8477521ba12718 +size 1423889888 diff --git a/model-00004-of-00282.safetensors b/model-00004-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc96f5f79aef57808c46e2a418ab4f3b50ca2d4d --- /dev/null +++ b/model-00004-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e61340ec4372e6c76ff5a67aef59f02968b963c1e39f3855f82f579e896c2533 +size 1423889672 diff --git a/model-00005-of-00282.safetensors b/model-00005-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c8f10e2b61125c9219fad1e20151838cd4e44bde --- /dev/null +++ b/model-00005-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0107c7234547e23abc0bd80b7fa8076b1521ce6f92d9528defe69f122f5de87 +size 1532386296 diff --git a/model-00006-of-00282.safetensors b/model-00006-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..48dfc2e81002aad751b77319118b8493191637b9 --- /dev/null +++ b/model-00006-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb6a8e3f7788cc017f78900288ab76fdad24d35887a082e34f8eafec696621f9 +size 1423889888 diff --git a/model-00007-of-00282.safetensors b/model-00007-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c663155ab425974ca59ed3cb5c42980bddb3c534 --- /dev/null +++ b/model-00007-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d630cebc143ee90ff7dbb2f26d4696a91ccb7321344c5c73797349d3e234ad74 +size 1423889888 diff --git a/model-00008-of-00282.safetensors b/model-00008-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c7005d79ba408700c856da85ecc4efe06c74b10a --- /dev/null +++ b/model-00008-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfc77ee0edc71153c9fc320ccf6864a3c4d9cd6dcc5ba7add53eaf1aedf767a9 +size 1423889536 diff --git a/model-00009-of-00282.safetensors b/model-00009-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eb8469c44d5b05d1366344967169fd46d1f7b928 --- /dev/null +++ b/model-00009-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b98296c69dd1b81984b48d2bfef8c359dda72c79f1d1edf7890edc6f228480 +size 1529215016 diff --git a/model-00010-of-00282.safetensors b/model-00010-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39285a7f8c1a40de4efc0bf12034af769c7f9336 --- /dev/null +++ b/model-00010-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e498ca95c8d3c14cf33cef582dede235539c3891e14f972bb3ffdb6ae5e933 +size 1423889880 diff --git a/model-00011-of-00282.safetensors b/model-00011-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c5cf7ebed39c9d3170adf7015d89a91d748f950e --- /dev/null +++ b/model-00011-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d8978a9893669c0a26ee897e5dfad87d9d49dd5e73a7b4f21837bc19c5acb7e +size 1423889832 diff --git a/model-00012-of-00282.safetensors b/model-00012-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..54f3e12e5e9bd6725d631f5bca9591711c563c38 --- /dev/null +++ b/model-00012-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c233c3211fa1eb018389b129fe07a709f2466684ef23cbbe323cf1b6a24add +size 1445924400 diff --git a/model-00013-of-00282.safetensors b/model-00013-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..678fda655f0549899a0d04247aacdc7ffb8e3beb --- /dev/null +++ b/model-00013-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724cb3bb3a9705fc72a8ad3f9dbd8d9449ae3377b294f4637c26472a1745a99a +size 1507180248 diff --git a/model-00014-of-00282.safetensors b/model-00014-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d0b3b632e933b1a3b7a55a476744992e8c0faf68 --- /dev/null +++ b/model-00014-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e37f9c2a6410bef230580298963f9833ed149ba54ff9fd61a516ad1ef3587ca4 +size 1423889880 diff --git a/model-00015-of-00282.safetensors b/model-00015-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c8178b3b4afed76cf178f0228f8186df0f9b423d --- /dev/null +++ b/model-00015-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e29ab736cba915e65f8d821cf21289791c9e06fabddc55b08cd32d33a866771 +size 1423889696 diff --git a/model-00016-of-00282.safetensors b/model-00016-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..73a66e2a0b234da8eadbb8ecac2369d6a79d6386 --- /dev/null +++ b/model-00016-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:592bef5bec13adf4f15080e5d3dcae677110b0242466a28815b8052ce5155a2b +size 1529214872 diff --git a/model-00017-of-00282.safetensors b/model-00017-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d52c6ee8548c1ac96b931f6f1620a2f2e935c61 --- /dev/null +++ b/model-00017-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3050cb2db053d3a44f3207a66f06525ffedbf08d376741a1466f541205e9f81 +size 1423889896 diff --git a/model-00018-of-00282.safetensors b/model-00018-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..87475862f6c9441d85d60fc6885eb01b8a006fcb --- /dev/null +++ b/model-00018-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d7c8db3364eb59e26f1ad7774aac2bcecebfa50be08e2caa017dbfba75e0ea +size 1423889880 diff --git a/model-00019-of-00282.safetensors b/model-00019-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e718d7543facfc2d3f826128c8f16654f6dd256 --- /dev/null +++ b/model-00019-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d996211cfc03d70052b5e9dc6f1b1d108505f814286a8e300f44fbffbcaf94d +size 1423889560 diff --git a/model-00020-of-00282.safetensors b/model-00020-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1c169c55d4aa9c3bcfdf05a864c7b58cfdc6ea07 --- /dev/null +++ b/model-00020-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ee05b36408b1d2092eda4cab677704b9a36f9bd88065900d6c4eab370b2324c +size 1532386400 diff --git a/model-00021-of-00282.safetensors b/model-00021-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..89c07bdac1b6dfca453c9f53c1dd0f0dff81d1fa --- /dev/null +++ b/model-00021-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f313073a870a756a3b251c13824df7af4237f199e127290bc4f11c4c1312d307 +size 1423889896 diff --git a/model-00022-of-00282.safetensors b/model-00022-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..020d8f3a2e85148912bd97299bb3b2a283bf220a --- /dev/null +++ b/model-00022-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed2394055399dddebdefac7ca8ede9687f93b71c0e2b6f333845da38742c4751 +size 1423889832 diff --git a/model-00023-of-00282.safetensors b/model-00023-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..215b05c8e78211a26885c0af95466257473a18b7 --- /dev/null +++ b/model-00023-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edeb8601bad3511a67a9010e39504aa9168472822ba7835700e3d2175d61b24f +size 1423889472 diff --git a/model-00024-of-00282.safetensors b/model-00024-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..825ed18ff5ba2dd30efe51e64e160a9cec522464 --- /dev/null +++ b/model-00024-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a602ee7fca9a6fddf7a35fa59bd7ae49fe0bc5e839b340b785225bae0ba575c1 +size 1529215136 diff --git a/model-00025-of-00282.safetensors b/model-00025-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5425dbbbcfad05945d05d0f7b8b839d4c9463fde --- /dev/null +++ b/model-00025-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5eb4273e6c75b7bb3fd48764c6221ed5dc95497356bcc1b53dbb0d9f9047c82 +size 1423889888 diff --git a/model-00026-of-00282.safetensors b/model-00026-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..41f17db3abd70cf8fbdfe797cd28dd0844c0bc1b --- /dev/null +++ b/model-00026-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:110deaa7dc014e0e44956a742588611054e6c27ff6e3343bce17c939bd2e0428 +size 1423889712 diff --git a/model-00027-of-00282.safetensors b/model-00027-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9b05bcff3b0f91d478c0486505b53f06a04966b1 --- /dev/null +++ b/model-00027-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ba50af60fc6202a0edfac819a630b7ca34c8990b14c6cd59654b5e20652b1b0 +size 1529214856 diff --git a/model-00028-of-00282.safetensors b/model-00028-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cde75aa4501e92ff5f44e7179f5cede1658a1ac5 --- /dev/null +++ b/model-00028-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e858791cb79feb902588ba75a7187435ce01b3ed03812cbb2c291266f82bf8f2 +size 1423889896 diff --git a/model-00029-of-00282.safetensors b/model-00029-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3accafc10815474a6303e408b67e3451f6285564 --- /dev/null +++ b/model-00029-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5bb8bcf36e387bd7a2ec10c8be100a11918c61eef4d68bc4cc91a771c0f0e42 +size 1423889888 diff --git a/model-00030-of-00282.safetensors b/model-00030-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..21a8b31fc071234b2dc4b4be7481adc08b8debcd --- /dev/null +++ b/model-00030-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e465b042984251e1890e390ca9577aa281ebab5cb2f06588343ce050b461f724 +size 1423889576 diff --git a/model-00031-of-00282.safetensors b/model-00031-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..48f6f845670e615681c1db013dc34e20f918daed --- /dev/null +++ b/model-00031-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bad2413f4e5c13cf1e94ad442328ba9db69d95f094cdb6e3926fa3e00f9bda3 +size 1529214984 diff --git a/model-00032-of-00282.safetensors b/model-00032-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..962937bdf89ddb6ef5509268d2987d483fd27d8e --- /dev/null +++ b/model-00032-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1928c0b3075af277041078d17c077c897398adeff03b5a152cbce956f1a12868 +size 1423889896 diff --git a/model-00033-of-00282.safetensors b/model-00033-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a5d4ae69fb2fbbc564dc59f1e8dc580b4b06164a --- /dev/null +++ b/model-00033-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d433ffa0c741ed07e4474262d332087920d9db43097d87419f21351d05f6ff +size 1423889856 diff --git a/model-00034-of-00282.safetensors b/model-00034-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..af2a41ff398c4d840a1a55a4b8a73dc9f12216b2 --- /dev/null +++ b/model-00034-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b6a669ea1054055d47f4bc854b54561f7b711b7b8b2ca4df92e7bb42c38df17 +size 1423889472 diff --git a/model-00035-of-00282.safetensors b/model-00035-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..902bf86235369223d7686a03b45d62c3cb8b3571 --- /dev/null +++ b/model-00035-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28aa7935ddd7f2034d52110bf68de8f40ebc0faa21335228389c6ba8df7b9d0b +size 1532386512 diff --git a/model-00036-of-00282.safetensors b/model-00036-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eb1fb1a7129cf318293aa56767366190fb024196 --- /dev/null +++ b/model-00036-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9edc01ddede97c3eb80f78932a66f4833482f6994e7c5940d5652578abdb6b3a +size 1423889888 diff --git a/model-00037-of-00282.safetensors b/model-00037-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f88b0f35e38926cc365d73df4110c038b24b8e4 --- /dev/null +++ b/model-00037-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a6579cea0c8ce0a582d7f2fffc805ba6bc1e0bc1a3705b0b68165552989940c +size 1423889728 diff --git a/model-00038-of-00282.safetensors b/model-00038-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f355cacf290cee1c3b1ca6e6aa5a81d9ed81bdf --- /dev/null +++ b/model-00038-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c03adaee9f17e6d417c719a12d8a9256b176811babf7b67b3adff64ec2b879d +size 1948346816 diff --git a/model-00039-of-00282.safetensors b/model-00039-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..75b1b14a6b7e92a9a0c2efb35315f77ed1f2701d --- /dev/null +++ b/model-00039-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa965e27b57e158ac97077397ade42dae817b44ed6080074dab36d2ab392daf0 +size 1423889880 diff --git a/model-00040-of-00282.safetensors b/model-00040-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2aa53c0c52d37549ba9b80b57b8ce37c12f94516 --- /dev/null +++ b/model-00040-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a139d098df80169438d117c4a90defa09bc45e697b1665e6102cdba99ecf65 +size 1423889888 diff --git a/model-00041-of-00282.safetensors b/model-00041-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c785ec0727670d8f1b47fcff5d7f45a4a8bb7fa5 --- /dev/null +++ b/model-00041-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2c3c9fa3f1e82073ae9a7f38efc028ed720f6861d2fb4d76d03ca7f4bfe4b7d +size 1423889656 diff --git a/model-00042-of-00282.safetensors b/model-00042-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7889202c1d8bfcdff8e51aeae04321ca2bd4ac0e --- /dev/null +++ b/model-00042-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa9dffb6fc76d8204370fa21cca688846c620db83c0ac9660b0ce33d83b1ee8b +size 1529214920 diff --git a/model-00043-of-00282.safetensors b/model-00043-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a34987d9070139fba174a7d79f32a1331e91eb9a --- /dev/null +++ b/model-00043-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c1f643d2786a7c2e9ae192b94cf10978fcc3470bc96e95f33cad1c5975e0f4 +size 1423889888 diff --git a/model-00044-of-00282.safetensors b/model-00044-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c7b3b52dffc9af3dcd08663bbf3624e2c60d141d --- /dev/null +++ b/model-00044-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79ab5e160fa4a28a6417bd5331a239828b184efccdd27299af2f95ff8b0a6804 +size 1423889888 diff --git a/model-00045-of-00282.safetensors b/model-00045-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ee4a65181ffd84060e7f7b5af5e3761adda9fe24 --- /dev/null +++ b/model-00045-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c16c2fd9e2706a1137556ba23de80999c764d1706440a02073b5f1c6133457c +size 1423889520 diff --git a/model-00046-of-00282.safetensors b/model-00046-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..40fbfc86bfed69b196d2fc835a86c898c555a53a --- /dev/null +++ b/model-00046-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb618c54aa3540d1f38cc36491d394ef307a0555ebc495b51fe5d06e9cc8add +size 1529215032 diff --git a/model-00047-of-00282.safetensors b/model-00047-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b3f43ca54cb33cbfd3cac4551ffa7a97580f3db --- /dev/null +++ b/model-00047-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20da1b46c9d7ac263458fa8d5ec8c8749d60ef59b4fe551b145d3dc6ce0ee74 +size 1423889880 diff --git a/model-00048-of-00282.safetensors b/model-00048-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9672ae05a1d497749a170ce5f928c7a48336cf2 --- /dev/null +++ b/model-00048-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35310b626fa8d1da9b1e0d228d48e86928055847c9b3a6555eb0f20033523c36 +size 1423889816 diff --git a/model-00049-of-00282.safetensors b/model-00049-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2b81008cdc3d80c9e4f42cbe48120e8f3a10b0da --- /dev/null +++ b/model-00049-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a543e170f380cd45d42fc0fd43b1d9d53126283bb032f001719caf47437d75a +size 1427775112 diff --git a/model-00050-of-00282.safetensors b/model-00050-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f8f135d670e03839515b95be7d118ff5efd4847e --- /dev/null +++ b/model-00050-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6503c0a53edf18967812ffec178b60dad7d70fb523cf41ce48109f46f8aeee3 +size 1495076312 diff --git a/model-00051-of-00282.safetensors b/model-00051-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf0911a320b00757ac833eba7dcd354f83bcfbab --- /dev/null +++ b/model-00051-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eaec54cca2a384ec3c0b55a0b455e5fa60f47c993d57be985fc6d8bfbfa51b2 +size 1423889880 diff --git a/model-00052-of-00282.safetensors b/model-00052-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..174cabce44618991dc215b2bdd00b70a2d10ae0c --- /dev/null +++ b/model-00052-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2255e598e2619cd1cee695affe03567a7460f8f9ebdd0d5e8dca9b9a0e08a841 +size 1423889696 diff --git a/model-00053-of-00282.safetensors b/model-00053-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8384101176138b518c43b76164a8e7331a2305a4 --- /dev/null +++ b/model-00053-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16f3e774401ce338646976b10c965d8dff1d56d1925222d75e48d7a6f9c0a12 +size 1529214880 diff --git a/model-00054-of-00282.safetensors b/model-00054-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6862adfef0d161dbac62e83fc8d482d8fa4be6b2 --- /dev/null +++ b/model-00054-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d504bcf8bca25ff07b5da857ff380fb68d4cb3750f4ef352919ef298bb5b7c7b +size 1423889896 diff --git a/model-00055-of-00282.safetensors b/model-00055-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18b805bcb880dad30ab36a442e4cfeac1db18caa --- /dev/null +++ b/model-00055-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68788132ecfa4941abbeb96dccdf5217f878a64b91cbbd97ba9e2d3de82cd4e9 +size 1423889880 diff --git a/model-00056-of-00282.safetensors b/model-00056-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..74b0e131b7eb5c392cda077e76ffe2b61e05bcb8 --- /dev/null +++ b/model-00056-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e86eedf53d71fa5a7e9ca48a9ec8240c652960c0296062404fce55c960186b8 +size 1423889560 diff --git a/model-00057-of-00282.safetensors b/model-00057-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..345646814a8512ea925fb7c8ed500b33508b8c73 --- /dev/null +++ b/model-00057-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:046f05c058d2764441197baae805f3a9b03bdadb2e793c62e3ef0b22e2dde6f6 +size 1529215000 diff --git a/model-00058-of-00282.safetensors b/model-00058-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..55b3cade06aa3c432e21850531a7c9f9d556b879 --- /dev/null +++ b/model-00058-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279eecc21899f5b7fd5ecd2bbae4be040dfd5cb495b22d78b47f0b67dcf9681e +size 1423889896 diff --git a/model-00059-of-00282.safetensors b/model-00059-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d8253a4b1f542ba90e53955d3bfe1c77ef6a6330 --- /dev/null +++ b/model-00059-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68311d14aed9443769eed5f7c1ec9a9794068a4ac017e1dfde0215145dc81bf4 +size 1423889832 diff --git a/model-00060-of-00282.safetensors b/model-00060-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8d5b130a02afb21802ef07ae8606499802bf037f --- /dev/null +++ b/model-00060-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05131b6d4466e98fb332bf786e581b007278968baa0e4fa7752cc51658f45384 +size 1423889472 diff --git a/model-00061-of-00282.safetensors b/model-00061-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8e6fe94d954a828b5a4f4ebcddd2ff4fc495dc54 --- /dev/null +++ b/model-00061-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95de87451e2f66a4c9cf80d38bfe0d8004f390d06583038c03dba469692b66eb +size 1529215136 diff --git a/model-00062-of-00282.safetensors b/model-00062-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7124eb978dbb678156946a812aff9e0442b188d4 --- /dev/null +++ b/model-00062-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55802ecb3e11522a0edcc8b54158081054f082fb25c42fb9e35890a12a456dbd +size 1423889888 diff --git a/model-00063-of-00282.safetensors b/model-00063-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4fbb40cb7c73c1582369c638f15b39f45b22fd1d --- /dev/null +++ b/model-00063-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3e10442a57c3a7c659208a61b33534e125366706afbd35ea7b439f356a25890 +size 1423889704 diff --git a/model-00064-of-00282.safetensors b/model-00064-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ee12567e4e58ef4e5d49c6495f47be6da508c40 --- /dev/null +++ b/model-00064-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72cfd14fc452258c42eecaaa21ff6603601ebe3453f703d4f26d573681f81368 +size 1532386264 diff --git a/model-00065-of-00282.safetensors b/model-00065-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f32c495ade73a089c3348ea25ca209ed6d77e30 --- /dev/null +++ b/model-00065-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38be7a6f1bfd7754e645c2e8767009f456750e876c7130fec1860465170f657 +size 1423889896 diff --git a/model-00066-of-00282.safetensors b/model-00066-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..121ab595cd1b47a2fbe466a66b9188cd0ff877d1 --- /dev/null +++ b/model-00066-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7542463760f1caeddedecc525aaed86cd131d5152afa31a3e3ce8a2fe196de2a +size 1423889880 diff --git a/model-00067-of-00282.safetensors b/model-00067-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2dc3f328212929f737ea5c841d82c32245773d0d --- /dev/null +++ b/model-00067-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e7781fbc9a1fe431490b275519ad71182c0af520a4af6954f6c6044fc6df44f +size 1423889576 diff --git a/model-00068-of-00282.safetensors b/model-00068-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e364b9b972c95f8f8d647b0ab03c693994e645f0 --- /dev/null +++ b/model-00068-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbbbd706b2f0fe2b470db9f1ef662893b38e13b91bef2b814a985b37f847a549 +size 1529214984 diff --git a/model-00069-of-00282.safetensors b/model-00069-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ffc7f6576b44ca511c4eabd8d3a7d43db47f44c --- /dev/null +++ b/model-00069-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e68527db90f89225258e752673a77fed9406ef579c6ce8494cdb44f32038713 +size 1423889896 diff --git a/model-00070-of-00282.safetensors b/model-00070-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a412bf4cec18ba1c095cd0a070ef572e9de3e6b --- /dev/null +++ b/model-00070-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac98a033a85a764fb687250c3966b824e6eae8807c58824b330ffe99186622a2 +size 1423889848 diff --git a/model-00071-of-00282.safetensors b/model-00071-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..df61e0193378e388d8e4b508de99181a92749984 --- /dev/null +++ b/model-00071-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6fa179eac08bcbca7b10ded64cec9862c189743a32c03ea60ae0165925eb9d6 +size 1423889472 diff --git a/model-00072-of-00282.safetensors b/model-00072-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fbc4f7ea752a14b7dfa6c97480287f0808c3034d --- /dev/null +++ b/model-00072-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2d75415132595ddbcb359062d69977752676df4f54b41c9207f0cb031c47d5c +size 1529215120 diff --git a/model-00073-of-00282.safetensors b/model-00073-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eb7c84b2bb58e2efc11812e9c5205cdc4122d8c7 --- /dev/null +++ b/model-00073-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2be2f7532a62ed937a685fec50ba324e3c885d48540a8a7bc88ec369d9940bbe +size 1423889888 diff --git a/model-00074-of-00282.safetensors b/model-00074-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6de8a60ec7c97a98f39492e31d3f8d6417a508c0 --- /dev/null +++ b/model-00074-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0778cd713d6fb6271c92ee8b708d517b5b266a59385c46c68489d372f47c611b +size 1423889728 diff --git a/model-00075-of-00282.safetensors b/model-00075-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..354ee4c357f91ca27a6347f43715f876dce071d4 --- /dev/null +++ b/model-00075-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a79bf6b2878229805fc080d4db34be2c9b4834548b8ee0766f0c9d984379b6d +size 1529214768 diff --git a/model-00076-of-00282.safetensors b/model-00076-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a14dfe03ebd2aef4e1079e7cf5a72b7c4257622 --- /dev/null +++ b/model-00076-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4fd5f9b50979e8db66a3cd9a48536b6897d120d7dd58935a11a61fe940d7fd4 +size 1423889464 diff --git a/model-00077-of-00282.safetensors b/model-00077-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bfcd0667d96db3b41de66baeec470c43a5ac1d16 --- /dev/null +++ b/model-00077-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:004ca8dc797d7e03b96bb1460aea0cac94c6d506ba3b68273940cd048cb276d1 +size 1423889456 diff --git a/model-00078-of-00282.safetensors b/model-00078-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b5ca160eedf9e632ae42dd6ce1bdb5256abf0d72 --- /dev/null +++ b/model-00078-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad8ab61d68fd92955d4f5aab5fb070a1555d0e7d9bbe389fdf654dc54cd97095 +size 1423889160 diff --git a/model-00079-of-00282.safetensors b/model-00079-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7138284a56bde634c2a7906c2c01b2d62b15d2d9 --- /dev/null +++ b/model-00079-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ad1ec3f4ee90b603606014e8317e5f29d58b4eaf220549e72417cbe16cf35ab +size 1529214768 diff --git a/model-00080-of-00282.safetensors b/model-00080-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7b129a1ce0387b04b778438a1dbdf696bc8e573e --- /dev/null +++ b/model-00080-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1033cc0970c7fd769496c0d16509e473b4d0789780c043957a42553948daeebd +size 1423889888 diff --git a/model-00081-of-00282.safetensors b/model-00081-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ced5f25648b404a805409d8c496125c6c719c61 --- /dev/null +++ b/model-00081-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ed112188c8ba11db1386b5b902f1805d5dd5cafe3c40f0adb4ad49247b32666 +size 1423889872 diff --git a/model-00082-of-00282.safetensors b/model-00082-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..11d3b05947ad35f99d7fca814f0e6f77a7cc8042 --- /dev/null +++ b/model-00082-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f807731fb81def08e97291504c66c6c614b00a87e8e4f9d43a62ed477d3a13a +size 1423889464 diff --git a/model-00083-of-00282.safetensors b/model-00083-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..217d31c133804a3f5455ce1e90b93cf212683cb3 --- /dev/null +++ b/model-00083-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d766965ce415f9d22a63ca4a8547f3574f1a115cf1e0f17ae0ea955dadc87e8e +size 1532386504 diff --git a/model-00084-of-00282.safetensors b/model-00084-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0c1b33dbb5f0794d4d59a18075401502143b1bdf --- /dev/null +++ b/model-00084-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1288fb8b8ca44af9f61c39bbced0bad5a87d5b3746add0b0e135c47b29a31cf +size 1423889880 diff --git a/model-00085-of-00282.safetensors b/model-00085-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..78bba6035ad63a97e6136ee93bebf1cd01c23563 --- /dev/null +++ b/model-00085-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ce76456d0c1eb329eff43230d6a9820d8abe463ed6fa0493dfbb92e2b717781 +size 1423889752 diff --git a/model-00086-of-00282.safetensors b/model-00086-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9c35b8db3e6e02d7cf5893370b0037bd2eb1c89a --- /dev/null +++ b/model-00086-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:385e7687aae84e78b649c20836db04c6bc92a04c70f41c4b62a690d9db34a02d +size 1529214816 diff --git a/model-00087-of-00282.safetensors b/model-00087-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0574ad9e67247d19283f15b1183710a18c45008a --- /dev/null +++ b/model-00087-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c7389f3e46e1a23309b23f68a389e5c94e9b30f1c921cb189a7b5db105a2840 +size 1423889888 diff --git a/model-00088-of-00282.safetensors b/model-00088-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1cba704be408d293dd3b2b17aa6fc98210c766e4 --- /dev/null +++ b/model-00088-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47e6df818afc78957d1e794b67c1a38d14a8eeb18445182966f98783a28d2676 +size 1423889888 diff --git a/model-00089-of-00282.safetensors b/model-00089-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..523614581fab0b2b71340244020f02a688eb8e47 --- /dev/null +++ b/model-00089-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5faddbaaadd1cd4e63e9cc3f6087ded3f60d550638e714e67abd2c486ebf45d9 +size 1423889616 diff --git a/model-00090-of-00282.safetensors b/model-00090-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..31b339391a0256b9efb86216ec1db4d54ef74ff1 --- /dev/null +++ b/model-00090-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d78611f3207290bebca0ade1ec27f49ac918921eaa8dafc179388bd0902136 +size 1529214944 diff --git a/model-00091-of-00282.safetensors b/model-00091-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a1400b89a9d2558cd1b6796f4f21310e0c428175 --- /dev/null +++ b/model-00091-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e145064282e8256b2786ee7e62d1981f743be9ab5b7fc82b170e9d598bcb3a +size 1423889888 diff --git a/model-00092-of-00282.safetensors b/model-00092-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..883dfdb47f32c09b0fbdedf7512f74929ef8b00d --- /dev/null +++ b/model-00092-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67e89555f475fe15d68a8f94dac98ba124939221fc1bdf0e08896542406941b5 +size 1423889888 diff --git a/model-00093-of-00282.safetensors b/model-00093-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a4a82be1b5d9bc0440fe99c357f9bae411598983 --- /dev/null +++ b/model-00093-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b2092cae9a0700687155fcd32d47a9cc24191dfa181c11fd71a76ecd07ac2d4 +size 1423889480 diff --git a/model-00094-of-00282.safetensors b/model-00094-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab712ea80d1b1b3642414a5f6dc186364f2a2fe5 --- /dev/null +++ b/model-00094-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca1ec86a06aade6305c2022491fd0cc918db2682e8d4c8fe424ae036a138893c +size 1529215080 diff --git a/model-00095-of-00282.safetensors b/model-00095-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..420765a48d87405067ccc82b3a421289e9b716fa --- /dev/null +++ b/model-00095-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10c88270b4121d1b3e01e54386021d4fc406ab30163893bc86334c22004a3a8b +size 1423889880 diff --git a/model-00096-of-00282.safetensors b/model-00096-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..62908e452805627901e5c36ce27dd490bd5b35c4 --- /dev/null +++ b/model-00096-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9857c30fa913005f5dab4bfc41c5350529bd24be9cdb65508e44b36831c2c21a +size 1423889768 diff --git a/model-00097-of-00282.safetensors b/model-00097-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2103d94b3a84a80ec7d252c59446da213ef68658 --- /dev/null +++ b/model-00097-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6808e244374a18b5c5ddc6a9df369782b8eee90689deee1bdd99eeec27c2aa6 +size 1532386208 diff --git a/model-00098-of-00282.safetensors b/model-00098-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..be375b1d12eb3d865196467034621cbc8f1f1d44 --- /dev/null +++ b/model-00098-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d81ea00ba7c3c6967d5cef9a09188c0fa0ac2a2bf0def6a37aca88614dd80fb9 +size 1423889896 diff --git a/model-00099-of-00282.safetensors b/model-00099-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..762009c6f8e5c3c17a2ebbb64593b68a45acffb3 --- /dev/null +++ b/model-00099-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d142934ce1d11cc6ede7011fe675535a6c7bbec667f6b615f4973ee1ea4399d0 +size 1423889880 diff --git a/model-00100-of-00282.safetensors b/model-00100-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..44f90b1fcbd10c8d77a44cf6a9c920208c2b977d --- /dev/null +++ b/model-00100-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e55f560929f65cc5abcdfc839f7bbab53696e6328048ca947ba9980541a254 +size 1423889632 diff --git a/model-00101-of-00282.safetensors b/model-00101-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a90b33e3f2631a2d717365b017e620d9601f4543 --- /dev/null +++ b/model-00101-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f2a38f811faf4ac1751ab0c7d08b80284767bca76e898b5c03fd4a3ee3a601 +size 1529214920 diff --git a/model-00102-of-00282.safetensors b/model-00102-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8ce9e059baf4d99f0ff10db10a7d4136b5feb26a --- /dev/null +++ b/model-00102-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:456cc067ac96cb362c4bf69f7beffcc6ce72d3c37571a7a9f5bcb69f47f50518 +size 1423889896 diff --git a/model-00103-of-00282.safetensors b/model-00103-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0a28b45e93d73d45249d02af7eda2a57b6393da1 --- /dev/null +++ b/model-00103-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:622bcbc9cb4e8d948fb2ec4a22dbe07befd0d0dbf0bdb3ace5326e84df77896f +size 1423889880 diff --git a/model-00104-of-00282.safetensors b/model-00104-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c69cb60d3a82a1836f0755aded5a5afcde0d7715 --- /dev/null +++ b/model-00104-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2040727a284066185a6bfb4f96a8bed42b9098645b37c13b0d9bd6702be900a7 +size 1423889496 diff --git a/model-00105-of-00282.safetensors b/model-00105-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b2b6613225715441034b34f1bf18216c0e8eb3d3 --- /dev/null +++ b/model-00105-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5af9e16a2d150ff675fdc904327d95603e64cfb16abe5aebb538ced447c82756 +size 1529215056 diff --git a/model-00106-of-00282.safetensors b/model-00106-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..396c2de77cac8b6364fbcc9acf8473082c189ef4 --- /dev/null +++ b/model-00106-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:215836c1cf71684f8f771d2495d3a31ade133d86e3dbabbc2fd81386cc5f989c +size 1423889888 diff --git a/model-00107-of-00282.safetensors b/model-00107-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..347ba5114fc255da1b566bbb893c2e18dfa1ce4d --- /dev/null +++ b/model-00107-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d25d19dc2c5d9ae8148cd817ad3ee78f40aab5ed7a51412327201583756b43e +size 1423889784 diff --git a/model-00108-of-00282.safetensors b/model-00108-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..598c4dd920c09d38d13045f6936d362222d0022d --- /dev/null +++ b/model-00108-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91fb29e242a777d03460931cfc72c93b2523fe02d370241e83ffb3fa0974b2ab +size 1529214784 diff --git a/model-00109-of-00282.safetensors b/model-00109-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9bb09f9cf9982d9350680bc947b14df4505ab2c3 --- /dev/null +++ b/model-00109-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7439fe680fcefe3563b03976be0ac321fc1b137bfe4f1e1f6935c2f5c950615 +size 1423889896 diff --git a/model-00110-of-00282.safetensors b/model-00110-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8d63ca3b31493fb052f83b9950397059e0279ba3 --- /dev/null +++ b/model-00110-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b973de378f866d943584a6da955b7b49f676a8418774a9974b514065d5b30f8e +size 1423889888 diff --git a/model-00111-of-00282.safetensors b/model-00111-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8bc89546d658a8137588c9e12ae4a56448921f8e --- /dev/null +++ b/model-00111-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9818b89d425ed16f86524ead8f5348f14e0b39ab1002fdf4cacddb86b9915e73 +size 1423889648 diff --git a/model-00112-of-00282.safetensors b/model-00112-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..709ad98bcb6a5a3863b52418e7e60ac62ce51caf --- /dev/null +++ b/model-00112-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12215bf8c9e6606daa8b3b6991f0ec704afc77b621040e02129e249f5d9a556a +size 1532386320 diff --git a/model-00113-of-00282.safetensors b/model-00113-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b89bfabfde442e3592a1f26663abd47ce1620a1b --- /dev/null +++ b/model-00113-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be03dc8100570cc7df6c76a5eaff386939ec165f0247e6c160c273c6f5bec1e +size 1423889896 diff --git a/model-00114-of-00282.safetensors b/model-00114-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5f5610f15a0a61f3c509c700cb7140c1bc1ef5db --- /dev/null +++ b/model-00114-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd36753915c6481ece48abb61176801132861db97b9ac5447b4dd103e203665a +size 1423889888 diff --git a/model-00115-of-00282.safetensors b/model-00115-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ec4e8b072c5a4fe366c51276a2c8b295b05fc5d --- /dev/null +++ b/model-00115-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ade29fa65a5c31c6a412c5bbf533092af85c0e5ce42ccb528f4215000c89a5 +size 1423889512 diff --git a/model-00116-of-00282.safetensors b/model-00116-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..79477c8ebf895c29b7397c672dec8e8c6aff58a3 --- /dev/null +++ b/model-00116-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a98742893cd3de67b1ea877c12b0c0c0538eeb6109be2d27087c1e2256f91c +size 1529214752 diff --git a/model-00117-of-00282.safetensors b/model-00117-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ffe17de1850d21260a08b493ff6291a17659df33 --- /dev/null +++ b/model-00117-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d1ccfb1a70d13b980fd11a3e857b870176099a601537b18ccb4287235044567 +size 1423889456 diff --git a/model-00118-of-00282.safetensors b/model-00118-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7161914f8e77a4100be9c07dab9898f1743c8943 --- /dev/null +++ b/model-00118-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a44a35ff413f17d26ad029e99fe5ae73b163071d43ad6a7e63b18e3284d12fd6 +size 1423889384 diff --git a/model-00119-of-00282.safetensors b/model-00119-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..962009370a1010ec1dd3248e97257668951c074b --- /dev/null +++ b/model-00119-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c604f025f938527116b392317b864731ea000bf7e813bd3f343e52ce3d2409aa +size 1507968568 diff --git a/model-00120-of-00282.safetensors b/model-00120-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fac3918eed57c32d60d65699f87dd7ca3536a12e --- /dev/null +++ b/model-00120-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556ff817ab10016d9261335c977f03b64997bb9ef9a472626fdac0c0ed528c60 +size 1438450752 diff --git a/model-00121-of-00282.safetensors b/model-00121-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3fe1c8d7a213cb4aa21ad8ec1e7102b6dcfc59a1 --- /dev/null +++ b/model-00121-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c32eda4f29aa1777a7b6709bc381464a769e4f04b1e9dd973110a1dbe3ea10f +size 1423889888 diff --git a/model-00122-of-00282.safetensors b/model-00122-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9297c3702ef07aaeed12f4f0e7278dc12d4fdd2f --- /dev/null +++ b/model-00122-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ade46351c438ed78fa4f8dfa1e11c8f6943431ef9c4b25e69ec6e428c34b07e8 +size 1423889672 diff --git a/model-00123-of-00282.safetensors b/model-00123-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd892a0bd818e3ed000510b5c60e18bbc0be3b22 --- /dev/null +++ b/model-00123-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96d7a960bb2aa2ccfbbc81140ca22609563227aa78a09ab197b470e56a140585 +size 1529214896 diff --git a/model-00124-of-00282.safetensors b/model-00124-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0ed6a6580c8b8e944d1cddeedf3d05369b029ffb --- /dev/null +++ b/model-00124-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a78377e649f7cce6ea470a10ebafaa57941c2b9f60835c7e213ca749f1208f06 +size 1423889888 diff --git a/model-00125-of-00282.safetensors b/model-00125-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7ddaaacd9c3e459df5962027e06bb6049cb9e395 --- /dev/null +++ b/model-00125-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43dcb7a7aec3628432fc59dab68f7982ed1144b5fdaf55cb7248b1e2db077349 +size 1423889888 diff --git a/model-00126-of-00282.safetensors b/model-00126-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..59bdea46e22d4ea96e9a76348fac0b2d7cb678ff --- /dev/null +++ b/model-00126-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be8f40b9646c8c9577dee7bf295cd7803f4867063020cc0e0a8cf9a43d9684b3 +size 1423889536 diff --git a/model-00127-of-00282.safetensors b/model-00127-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6f9982b024823371850e28016cf9d1b359dac2f3 --- /dev/null +++ b/model-00127-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6944e34595e14913a17ec1e7a232590e2ec4112e42f349410f6113009dd393f +size 1529215024 diff --git a/model-00128-of-00282.safetensors b/model-00128-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3f825884068f7283f306cd749c760872261ed8e5 --- /dev/null +++ b/model-00128-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc1c8a00690f10920d0a8654fedcad080712b467a42b5726e291d430243e68d0 +size 1423889880 diff --git a/model-00129-of-00282.safetensors b/model-00129-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0daa1b4a492884ea81606e77fa9b77426f1fc19d --- /dev/null +++ b/model-00129-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6a1f7718c742a4885d5fcad6ef32c5d5da0af6428d04ffbe9bc421385c38fdf +size 1423889832 diff --git a/model-00130-of-00282.safetensors b/model-00130-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f31ff2ffef7e813f0c172bb5b1f4a0f5c3762853 --- /dev/null +++ b/model-00130-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fdd7ca75cf30e9e1546a543930a2e565140ddb8bbfa519ef8235466b3b6723b +size 1452745616 diff --git a/model-00131-of-00282.safetensors b/model-00131-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..adeb1875d45618a27d9cca64ce2f70764941d21e --- /dev/null +++ b/model-00131-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c9aee45bc125d68f9a933cbda7adcef55287f67004d9d265dcab354effd4bf5 +size 1503530456 diff --git a/model-00132-of-00282.safetensors b/model-00132-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9edae9dedeba5fff9bc726657cc55278be6d0c97 --- /dev/null +++ b/model-00132-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a5510a33cc0f6018e71308e537b2c0b99fca1c9bd8924854ecf6dd59fdcc57a +size 1423889880 diff --git a/model-00133-of-00282.safetensors b/model-00133-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a35223470d2237627cf2d5ccf01ac09e4d141a9c --- /dev/null +++ b/model-00133-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5de4a9ef6e5b89988dabd61b59ed24bc35f25bd0f1fcb94010beac2d0873f2a9 +size 1423889696 diff --git a/model-00134-of-00282.safetensors b/model-00134-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..969b2e625f8cbe8bc2f6797fab1a96509d12a1cd --- /dev/null +++ b/model-00134-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d9d3c74c1e6a1d99205d4007773464ae076193493b62d0a0a24e5ccfc65961 +size 1529214872 diff --git a/model-00135-of-00282.safetensors b/model-00135-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d43ec68f700033ccaecf025297e5e2e945ca64f --- /dev/null +++ b/model-00135-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f798bf74e98e69a5b504c40305fd7472bd36e9661c8f8148e990ac89f1246c14 +size 1423889896 diff --git a/model-00136-of-00282.safetensors b/model-00136-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1765f3ad0eb593d1e037bb4160db61a767094972 --- /dev/null +++ b/model-00136-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a857c2a6f526d0b4862a61a556948760627f074e298c958d35be48c9a6091d22 +size 1423889880 diff --git a/model-00137-of-00282.safetensors b/model-00137-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4c1f1112c01658127cadee099fbb946ea6a084dc --- /dev/null +++ b/model-00137-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c389dfec2f1f807d342e0801c3a86f52ae98f2d4eaf535e444d89cdc3cb8d1c +size 1423889560 diff --git a/model-00138-of-00282.safetensors b/model-00138-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9554aa133381ba3a2b75fceb24d623687186dbb --- /dev/null +++ b/model-00138-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53782237071faed248f07839f1440a49ce0ee791513891a391b8c2515085aba7 +size 1529215000 diff --git a/model-00139-of-00282.safetensors b/model-00139-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..be4f248f8db933b169b023bad563a414c488bfaf --- /dev/null +++ b/model-00139-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:875dec746727c45bd4091dd7b660924b37f6dd6f12fdef45f7f7d3b8ef815b2c +size 1423889896 diff --git a/model-00140-of-00282.safetensors b/model-00140-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3f94e924b68811e70c023fefc4b0564651d2296c --- /dev/null +++ b/model-00140-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f81fadf48ec93746f02c7d4cdf37b9d1b7fc83d7ab722f982faaf9a2853a1644 +size 1423889832 diff --git a/model-00141-of-00282.safetensors b/model-00141-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cb76a578afcd6a2dad408380d2bb18ea8d94065a --- /dev/null +++ b/model-00141-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17b9146d41cf1aa5bf305101d3012960113b8e4da97ff2f0db387a617d4deabd +size 1423889472 diff --git a/model-00142-of-00282.safetensors b/model-00142-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90259d134c45d053b9c32790f83cde0fa341ed51 --- /dev/null +++ b/model-00142-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f40ff29153338924b5e265d4dc762954b7ff5799695ed62698720e221d9ddcd7 +size 1529215136 diff --git a/model-00143-of-00282.safetensors b/model-00143-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..276af72b0e3019b137016fc392dc1ac7489325dc --- /dev/null +++ b/model-00143-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35ef4efeffa88571260a9484780d98760c256c91e9c4d17e14c5728584c30514 +size 1423889888 diff --git a/model-00144-of-00282.safetensors b/model-00144-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f7bc29f65c35f633a5495ec03c947f3c4d113c9c --- /dev/null +++ b/model-00144-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88f5350ee325251e005cddaaff009eeae759dd3592909ca6ed248a419b77a4c4 +size 1423889704 diff --git a/model-00145-of-00282.safetensors b/model-00145-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..758321325a3e386d24e2b49e6290bab02bf6c410 --- /dev/null +++ b/model-00145-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d3c97037196440aef84edec3db05ede6af1d3766b0a1052eb96d596dfad84d +size 1532386256 diff --git a/model-00146-of-00282.safetensors b/model-00146-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a3278adeaf24332c9abe507ec6561efe618caaab --- /dev/null +++ b/model-00146-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3853e100251736217d800de359411808f5cff81647af0177bf257a93a5252195 +size 1423889896 diff --git a/model-00147-of-00282.safetensors b/model-00147-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b8c124e30c68ab02845e13f122c25a8373ec5b9b --- /dev/null +++ b/model-00147-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16dd644c529be8bc05ee09f084322ed4685f6b70cfb40838d4b1aa704f1c6503 +size 1423889888 diff --git a/model-00148-of-00282.safetensors b/model-00148-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..433552a61871e10b61404fa144c0eff2a6895900 --- /dev/null +++ b/model-00148-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d10bbada24e942ccf22e83a4d817351fe5ed88958020928f960128ec747818 +size 1423889576 diff --git a/model-00149-of-00282.safetensors b/model-00149-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f413c00919be677502af9043de114928dc12a6fa --- /dev/null +++ b/model-00149-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f2b00365926e2abf375f5be9b412886f99096f028d8f67b03c02d1cc209294a +size 1529214984 diff --git a/model-00150-of-00282.safetensors b/model-00150-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..64116066d7daa3887b74e1d4445e37c2fa8f31b9 --- /dev/null +++ b/model-00150-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:552d624b235062a520e21ece97ad5db57174d7b59290db236dd1f9abdbef9af6 +size 1423889896 diff --git a/model-00151-of-00282.safetensors b/model-00151-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..02da7e2a7fcb4cb4fd5c7661d0c0c04e074e0e25 --- /dev/null +++ b/model-00151-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a9fa0c7a02eb2b11616c292cd53567f3f77319238f39435194f9206dc6720cb +size 1423889856 diff --git a/model-00152-of-00282.safetensors b/model-00152-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1a99741cee24448cd64596fc7f5325ef7a10a095 --- /dev/null +++ b/model-00152-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:777751543eac469b12a4285084dc259dec189dbc66a739e33dae21da38c8414b +size 1423889472 diff --git a/model-00153-of-00282.safetensors b/model-00153-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..92bc407efb66678db684f1e161fc5f800fefef84 --- /dev/null +++ b/model-00153-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17f3dfa58563e10d954e2d26f6a9400183d9386701cef774695e075143b577dc +size 1529215112 diff --git a/model-00154-of-00282.safetensors b/model-00154-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..04a803813c31ec836094cb6f48a51cf0ae97ab16 --- /dev/null +++ b/model-00154-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aac4339b604747ca80d1e709e13f5ded568eaab59d275421b71c3e0fbc00314 +size 1423889888 diff --git a/model-00155-of-00282.safetensors b/model-00155-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6a2c949b7a776d37c3036f92cc663c440b2855dc --- /dev/null +++ b/model-00155-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:297f456ed64fd0f9144d63bb441e7b8a13b054c99bf6d54946b114aff8aead26 +size 1423889728 diff --git a/model-00156-of-00282.safetensors b/model-00156-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..54972c3d6c564ceac31187ff3cd24f8ac797a911 --- /dev/null +++ b/model-00156-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0050e50ceb53a82fc3b94f80f1b2f92016f6b26591fdd832e97143d151b99b0 +size 1529214768 diff --git a/model-00157-of-00282.safetensors b/model-00157-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..72ef8001d598982574d65140faf1e3936050b849 --- /dev/null +++ b/model-00157-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e18098923b7b5402ea0da195e6d3f6686052c916bf5ea9af029d14cb7e1e6905 +size 1423889464 diff --git a/model-00158-of-00282.safetensors b/model-00158-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d2638c4d7a5bbf93bedc03c7dce3f5f661011cc --- /dev/null +++ b/model-00158-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3cd12259775e07b44ca2da9862c238c6f5ce616b7442a80f798e60bb87623a2 +size 1423889456 diff --git a/model-00159-of-00282.safetensors b/model-00159-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f3fff953b4b6c314c712242117374fc1afbcb283 --- /dev/null +++ b/model-00159-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eda2dc359b397a83f262a162b6106ce9a0ece8a0e299e69b188583b5ea2a5ee +size 1423889168 diff --git a/model-00160-of-00282.safetensors b/model-00160-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..45bb3083a64208f829c0cc8b8aa28b9b4f5fc627 --- /dev/null +++ b/model-00160-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eccfc498b8ecd4f4d27aa2b5c1d8c6a4cb702ebcc3cfc92b30c8d0cb3a557e5 +size 1529214760 diff --git a/model-00161-of-00282.safetensors b/model-00161-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5f48194147f3453f22b8713d94d51185e6da48ae --- /dev/null +++ b/model-00161-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:952c259324aecf3202dd32f5aeb6ff75b55a42e4dc9fab08cf0b644db221c3bf +size 1423889888 diff --git a/model-00162-of-00282.safetensors b/model-00162-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a941dbc5158355334c2861406a2c700a20dbbaa2 --- /dev/null +++ b/model-00162-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786584576c2298176c8d88bf6f774c9def34030baf3d5181c2892b070b05f1d2 +size 1423889872 diff --git a/model-00163-of-00282.safetensors b/model-00163-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ecc83e6664e8de1a0a639cf9e18c95a97847021a --- /dev/null +++ b/model-00163-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197501889d25a581f51f0f0ee2b98b4e7d48813f7bb8630ae76aa97e7178bcaf +size 1423889464 diff --git a/model-00164-of-00282.safetensors b/model-00164-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..96aea385115ff2dd37ed6b43d5e08cbf7b8ab793 --- /dev/null +++ b/model-00164-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faa490c2570ea164ffc2a0bc846ca14bbfcaaf98337137769e0728e4a4940388 +size 1532386504 diff --git a/model-00165-of-00282.safetensors b/model-00165-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a02f0783d3c3464dcb913f2da1e1151bdf77d31a --- /dev/null +++ b/model-00165-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcecb9bffcf4c6dcf0654f46a17be73ab88f57447bac552807a6c218101af41b +size 1423889880 diff --git a/model-00166-of-00282.safetensors b/model-00166-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..632e26db3a2c8b327990912064bd86cd6245312d --- /dev/null +++ b/model-00166-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c4e753dc9af7aa8bcf0371972c19b9840a202f0a387c36e88105a96a7f1a2a7 +size 1423889752 diff --git a/model-00167-of-00282.safetensors b/model-00167-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4c9837b6b1775d12d801c9639885f5fe6814f88 --- /dev/null +++ b/model-00167-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3468d3af6398d6ba9c7d0e3d52e462593cd05693e5ad19dac3ed5bb3b3a3b19a +size 1529214824 diff --git a/model-00168-of-00282.safetensors b/model-00168-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5a9410da42edf8422b6ab23afdd0a141800d3772 --- /dev/null +++ b/model-00168-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da268509274422bf694e39d942957d3c242ebde049e9adec59d337933d35ccde +size 1423889896 diff --git a/model-00169-of-00282.safetensors b/model-00169-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d5e77ab3fb570518c18669f9f87353f18f74b62b --- /dev/null +++ b/model-00169-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a67e0012c0025747d5ea1902b4af492f451ef3c7f8ea05d8c12546153d9e1036 +size 1423889880 diff --git a/model-00170-of-00282.safetensors b/model-00170-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0950af0ca3f76bdf843842999d3cb925d0be9a7 --- /dev/null +++ b/model-00170-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c4745c8a0eaea7faf6a541cd763b2a1ea7b9aa8b426e7b6c2cb6a6b8d555f53 +size 1423889616 diff --git a/model-00171-of-00282.safetensors b/model-00171-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..46660b6d95015c9b8a6c032b964ea3f3b00db2c2 --- /dev/null +++ b/model-00171-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:318e838a2f5af95a3ddb36dac7305e1d5e6d23c57cd9f934d39b70c7f59e5701 +size 1529214944 diff --git a/model-00172-of-00282.safetensors b/model-00172-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d13871b480ee4aa1d9722f51345bc0e762c02145 --- /dev/null +++ b/model-00172-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ee370144bcd5190a7e8839dc86c9308dc0a2025e77e1029e4e4eaee06a55572 +size 1423889888 diff --git a/model-00173-of-00282.safetensors b/model-00173-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f05b8bf52ff69ca5b7c2d10765c0f53699ed2250 --- /dev/null +++ b/model-00173-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f501e456f184ccecb4578165069e0a7c2100d8d1046fb6bc6ba02c27dd3ee29d +size 1423889888 diff --git a/model-00174-of-00282.safetensors b/model-00174-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b3306b01bfdef953f1a5d63b16fbf180814e1bb --- /dev/null +++ b/model-00174-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90eae70b44a3f94d10bd2af120c1104f72c6bd974ac05cbd437e108b4cfc80b1 +size 1423889480 diff --git a/model-00175-of-00282.safetensors b/model-00175-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1bf1d7cbe624fdae0843ecfbf853536157355c2f --- /dev/null +++ b/model-00175-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be130830cdb7ea141116e25753e0bfaa437b3cedf5dbed129c56ccebc0c334bf +size 1529215080 diff --git a/model-00176-of-00282.safetensors b/model-00176-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..266ca7131f2f8ca41c1361d5e69a418246f784a7 --- /dev/null +++ b/model-00176-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1d24fe7a31d39ef1ce91dcbca58afa3a391005c9fa53092b73e3c927a0fa940 +size 1423889880 diff --git a/model-00177-of-00282.safetensors b/model-00177-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce559ee21d2eb2716a7aea15102376042cf94698 --- /dev/null +++ b/model-00177-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4908fae9fd54cf4700ab99bdcf35eff547a5bd31b1ab8ec1062a9608d1d678 +size 1423889768 diff --git a/model-00178-of-00282.safetensors b/model-00178-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1668c6b12b48517f2e84f89f9d86177bbb4a8853 --- /dev/null +++ b/model-00178-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ced9d6c7d6ab425a80dbf89c8ee807a9b0292de76c24cb5085e93b703641723e +size 1532386200 diff --git a/model-00179-of-00282.safetensors b/model-00179-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0cd696bcbf1d64b49741cbc37184322b83dda57d --- /dev/null +++ b/model-00179-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38037247e35c2064db2976b4322f9eab065f5b0373c40c2cf89b36a20ed6ddba +size 1423889896 diff --git a/model-00180-of-00282.safetensors b/model-00180-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..353e5ba8fb9f4831195a350ac33959c17c54f77c --- /dev/null +++ b/model-00180-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb309d6fb7943bacde585d4175d02f3a6f2bb4a85dac94f0b563a68e1897e20b +size 1423889880 diff --git a/model-00181-of-00282.safetensors b/model-00181-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2f817074d527823f4b244a0e9b4d63693ebd302f --- /dev/null +++ b/model-00181-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc638b19489279866a85f48bd8645973c5a6485421c50ca5b548b9445b6f46f +size 1423889640 diff --git a/model-00182-of-00282.safetensors b/model-00182-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..03850577f72443825daaa5577fed5fc30ca0e849 --- /dev/null +++ b/model-00182-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbca70190e07acb074e43f437af0c74e7b5b4aed93b06b813d2e7b13345a0119 +size 1529214920 diff --git a/model-00183-of-00282.safetensors b/model-00183-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d30a340cf4faaaf1416e1cfc828d54c2c1aab33a --- /dev/null +++ b/model-00183-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4753d88afab1e33b1f2a8b323477bca12a5a615770f14a932e6b4a401695477b +size 1423889896 diff --git a/model-00184-of-00282.safetensors b/model-00184-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3bd404e210dd16d9797b3190f066bb5191d12944 --- /dev/null +++ b/model-00184-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34cf862d7cc207261dfd19af6e095ad2ef04a74df3d301858144fd1aa190cb47 +size 1423889880 diff --git a/model-00185-of-00282.safetensors b/model-00185-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e48a444e26f3728b875e5a10aee5c7105618f313 --- /dev/null +++ b/model-00185-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd984a40d4673a9331f3f428a641153c84ca118ced2a1c06db22517f8394217 +size 1423889504 diff --git a/model-00186-of-00282.safetensors b/model-00186-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce9b861f22cbd40956a052d2b5b2b0bd62b6c1c7 --- /dev/null +++ b/model-00186-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b598baaae06c6cf943f18baaa491d2eb272052865bde16eaa28f22f660db92c4 +size 1529215064 diff --git a/model-00187-of-00282.safetensors b/model-00187-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9cee013f43a06b39fafcbcb3c6992ade8a5ff84c --- /dev/null +++ b/model-00187-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f95f2af9a8d75ef71e108657ba7042d7d9b49abf7695bde393c49e049ddd00b5 +size 1423889888 diff --git a/model-00188-of-00282.safetensors b/model-00188-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..edea2dfd9504d5e08bbefb6728253bb0772b383b --- /dev/null +++ b/model-00188-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941cdd3fe18f3c93c30b85e8104c0f4554fb088ee4fd091531081fecdf93793e +size 1423889784 diff --git a/model-00189-of-00282.safetensors b/model-00189-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..40228944ae9fa470cfffffd6ba3a1bce67c30373 --- /dev/null +++ b/model-00189-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:839a991ee5666bcd9b3a37305b0d649d1630bb2b41f0f36ef4ec32037df06d60 +size 1529214784 diff --git a/model-00190-of-00282.safetensors b/model-00190-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f976369ac1b51bfffc10ea0cd75f7f82733a582f --- /dev/null +++ b/model-00190-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b2ac1df703e0085ccc5d3d7dc823925c011e51dda4339be15b5e026b847b33 +size 1423889896 diff --git a/model-00191-of-00282.safetensors b/model-00191-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..79d7bd5a57e4e4dddbe2d4e25148dd7052fc9164 --- /dev/null +++ b/model-00191-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb2c3d323e40cd4e5bd80c9cf5982f2b98e1a5f5f5a26fd4ef1bac42152c3e6 +size 1423889888 diff --git a/model-00192-of-00282.safetensors b/model-00192-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..32963defd9dd54b05e7b360e6b6ea979fb749b8e --- /dev/null +++ b/model-00192-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c08b87ec2e9fcfe37582c28b35f9d5db158dc4f19f6e95dd20846431dbcbc3c +size 1423889648 diff --git a/model-00193-of-00282.safetensors b/model-00193-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8bd7b972c19f9f5ffcb600ab21c22d4134e74048 --- /dev/null +++ b/model-00193-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7837a329ed21666089ca5960a9528a707b6ca48b375456ee77247f271059d48 +size 1532386320 diff --git a/model-00194-of-00282.safetensors b/model-00194-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0490f8aaab3dcbf043dde2ba2e2d2fefa7cdc26d --- /dev/null +++ b/model-00194-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f14c161b8af49b9e7ef8f7995a4f3383443262b81135acad7c4379a36dea7e81 +size 1423889896 diff --git a/model-00195-of-00282.safetensors b/model-00195-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4cb12bc0354a259eb1794cb387f9e2d3ecab4e96 --- /dev/null +++ b/model-00195-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e7e4ac31d9cd74d0339ac4cd4516c97c9d36eaa4aa9d53f8b171385bbe83a0 +size 1423889888 diff --git a/model-00196-of-00282.safetensors b/model-00196-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..742cff060681706703876c2e113881375a3dacac --- /dev/null +++ b/model-00196-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e40fdaba4730e27b94af71575d277b53d943bbb76333ec6ccaec199e315791 +size 1423889512 diff --git a/model-00197-of-00282.safetensors b/model-00197-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8bbfc3e3affeee7b7133bd49bfe2a12edb9c897 --- /dev/null +++ b/model-00197-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f8f015f7e21bab46399ba7cd08cc48a6e0cda3c5d69b326c6a8ff6a0612f2bc +size 1529214752 diff --git a/model-00198-of-00282.safetensors b/model-00198-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..757965b2aa979d7be15066f667b888bafa864616 --- /dev/null +++ b/model-00198-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96bffa077b7d9f91a0294a6a3e8e14c867b91a65faf250ac13e3ace2c366c17d +size 1423889456 diff --git a/model-00199-of-00282.safetensors b/model-00199-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..63e8a6847226261148d5f018351322953aeaf114 --- /dev/null +++ b/model-00199-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17dd2407844ddba15f2e34eac9c7b0f642ffba6acb07892fb26110c3eab37322 +size 1423889384 diff --git a/model-00200-of-00282.safetensors b/model-00200-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3642b9e1f480131f5b3d8d95f525c0c3442edab7 --- /dev/null +++ b/model-00200-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b542f6e2c3f2df7ce3090b1f6e2eeb566a27e4eca2c6a51c6b7cdea852484966 +size 1511533448 diff --git a/model-00201-of-00282.safetensors b/model-00201-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..78d936262173727a75ff6c171b2d1b072ec9d2e1 --- /dev/null +++ b/model-00201-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e887442280d82145cd4272d3c3fb5f3681d939e56bbf4aa51a5f33f258a66bd +size 1444742200 diff --git a/model-00202-of-00282.safetensors b/model-00202-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..184017e5fe3085258bb569f2de758649c0fa586f --- /dev/null +++ b/model-00202-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2df14aac232548537c33ed599a42a5855345424668ca652be38a3a8f3e25567f +size 1423889888 diff --git a/model-00203-of-00282.safetensors b/model-00203-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1ed2fbaa00d5a9de8d8e866b813a382fde55ac2 --- /dev/null +++ b/model-00203-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ac937fd3fb05c5a80c628df192e84cfe8d9c2b73b8f9d6b87dafdfc4f07bd5 +size 1423889680 diff --git a/model-00204-of-00282.safetensors b/model-00204-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8487672bb3b89f2a44f554422d4da421a5186b18 --- /dev/null +++ b/model-00204-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52eeff6791805307676eefaa992c879881c97d07186d1b7dcfda9468ef9a0e24 +size 1529214896 diff --git a/model-00205-of-00282.safetensors b/model-00205-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dcf59fd3a0cd8ad4ce17e1b46a581a79a683be4e --- /dev/null +++ b/model-00205-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3d42788685eaaa7b869fbb0f89001ca889dabe8c5f2be1e517bb67de6116f8 +size 1423889888 diff --git a/model-00206-of-00282.safetensors b/model-00206-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..142af408d70e232fc3b82f7e79345a000886d03b --- /dev/null +++ b/model-00206-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc479275643fd1a631c61b808b2580f21eff79db9306c11eb0e3da2aebc8b176 +size 1423889888 diff --git a/model-00207-of-00282.safetensors b/model-00207-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d4a70a274df6e24055f0a9712a0e545bb6bc711 --- /dev/null +++ b/model-00207-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52b0876aa14ebe31af608f9d5a287057304aaa82c1e010e52de953ce07c712a4 +size 1423889536 diff --git a/model-00208-of-00282.safetensors b/model-00208-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1ee6ac970f81ffb517de40708682e3f4fb4bd85b --- /dev/null +++ b/model-00208-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a51ec84c38a8284362b96362341f42a7a47d373db5fc627dd667407456451043 +size 1529215016 diff --git a/model-00209-of-00282.safetensors b/model-00209-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..55744e2b1953715a883f20240a7d4a5d651eb522 --- /dev/null +++ b/model-00209-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba22d63c3ef3a5ceae153d3656c6243cb017e2cc9618986edd03890a09d2fb85 +size 1423889880 diff --git a/model-00210-of-00282.safetensors b/model-00210-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..713b4ebf2440d4742aa8a340286d47c40ef65456 --- /dev/null +++ b/model-00210-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c808e1bf7093966edcfcf8a756daaaaa8e0653cf4cdddf6dfe17913f9bc937d9 +size 1423889832 diff --git a/model-00211-of-00282.safetensors b/model-00211-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84999c405268392d6c4e89bca3bf49ebf7224f77 --- /dev/null +++ b/model-00211-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a589b3fa8d049c763722545adf91c91113a41c432ee30ea7ef6cd2b66796cd1a +size 1447128592 diff --git a/model-00212-of-00282.safetensors b/model-00212-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12b768322520a5949073405f8af87c47f22aeba1 --- /dev/null +++ b/model-00212-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fffd0c7ae57317fdcc8ab62bc9e132bf2e9aaa98c0b373ef8e31095d293b2b1 +size 1509147448 diff --git a/model-00213-of-00282.safetensors b/model-00213-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3479b00a322edffb8065df482dd24e9087b64e44 --- /dev/null +++ b/model-00213-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21c869d8c8d6c5567170eb6da9a92d9f48fc4e7b3f15d05be2620e943ec816e7 +size 1423889880 diff --git a/model-00214-of-00282.safetensors b/model-00214-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce66e0210dc584e3906daf29bb4b4dfc55835be6 --- /dev/null +++ b/model-00214-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27eccb6ea0ce76fcbd157e92b99206eceaa75e9a1ce71445524cdd99815f3860 +size 1423889696 diff --git a/model-00215-of-00282.safetensors b/model-00215-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..524f310ca82e4d336daf8f350b725695b8232365 --- /dev/null +++ b/model-00215-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:981b5a954c50602cc4f0f02eacca95833da2abb3e2c805a1ca13458dec314e81 +size 1529214872 diff --git a/model-00216-of-00282.safetensors b/model-00216-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..14ca2fe4ae59da3e92d6b3bf0a5d4ed83557a12b --- /dev/null +++ b/model-00216-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f8acb4c742ea99b5f57a38da627e147068fc742b62ea910fe94fa43f89a54d +size 1423889896 diff --git a/model-00217-of-00282.safetensors b/model-00217-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a930caed5b7e3a9853ba442d7c343d56be2b48de --- /dev/null +++ b/model-00217-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a0742f98035f845ba6997cdeed3fee7b1328155c56ed931ea11da3727e2094c +size 1423889880 diff --git a/model-00218-of-00282.safetensors b/model-00218-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ff1247d18ab23f526985868460bbe1ce8c238b0d --- /dev/null +++ b/model-00218-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeed8839af8dee180ad5b31ff9055460abf7adb3410debe37d69d68922e4fd56 +size 1423889560 diff --git a/model-00219-of-00282.safetensors b/model-00219-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..169bac41ff938e25e855816e0cced725a8462f16 --- /dev/null +++ b/model-00219-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cb88de394531db6f231aa450edc2193feeb5d2d4795ed281362d0c96393d5e6 +size 1529215000 diff --git a/model-00220-of-00282.safetensors b/model-00220-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..16d77b9ee966a59e8ee6fd52d704f9e8c4ba155a --- /dev/null +++ b/model-00220-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cea241aa194c4da81b5c09a2752e849075c475c4d687b189fa6e8d7c0f03c88 +size 1423889896 diff --git a/model-00221-of-00282.safetensors b/model-00221-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e01aeffde6b510baf4d7babd60c1a7d1e797aa5 --- /dev/null +++ b/model-00221-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cafc920cdddc54ab4bd912c3d30e4775fd8aabba112ae9b6f1d971fc373768d +size 1423889832 diff --git a/model-00222-of-00282.safetensors b/model-00222-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..320adfe3860f87241dc3700591280f903aab52d2 --- /dev/null +++ b/model-00222-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6395c5cb943a3d3fb38cd7935afc83141e8dfc2e038bac88763f6733e298b6c +size 1423889472 diff --git a/model-00223-of-00282.safetensors b/model-00223-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bba0842d148792d1bff1bdbc2f529ebe8268a0d3 --- /dev/null +++ b/model-00223-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07150522b4593334397ab3a9b9b0bdc55897c590d1740717f90c8addea5a58a +size 1529215136 diff --git a/model-00224-of-00282.safetensors b/model-00224-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d12cdc6a2a9458cb97d0782c29a8893f03ef4c2 --- /dev/null +++ b/model-00224-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:288b7c5b7e8f5ee4373570f59897f9ac7f19e06d3902778060daae39090d2821 +size 1423889888 diff --git a/model-00225-of-00282.safetensors b/model-00225-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dccfefa63feaf659bae7ef3093a59ebf540f3c41 --- /dev/null +++ b/model-00225-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6cb96ab46a67e52c2fb0ecc55c0d678ae61de3c66ba41f12ec2694432dd3095 +size 1423889712 diff --git a/model-00226-of-00282.safetensors b/model-00226-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9720ad5d72efde863f4aa9974a538e989b77b44 --- /dev/null +++ b/model-00226-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c62984f90618ed85b879f706d20bf33e0555b981a8de3dc56a8612083c08535 +size 1532386264 diff --git a/model-00227-of-00282.safetensors b/model-00227-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3eadc7a4f0cb94d997db9d84d3af2436ca4ba55f --- /dev/null +++ b/model-00227-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ae0288e7607ecd7dcf31583cdd1460a5fd90a7678acb37bdd6a7a3c4b3cd95 +size 1423889896 diff --git a/model-00228-of-00282.safetensors b/model-00228-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2fac5c203d264d96b3377e8a76ce7ef7157b23bb --- /dev/null +++ b/model-00228-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab13ed58b991849555d0251414cb1f21f7a3045ca94cead41f20923321332ea3 +size 1423889888 diff --git a/model-00229-of-00282.safetensors b/model-00229-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..55ee8777db506808f5bb4e938f4ce79b6928fb93 --- /dev/null +++ b/model-00229-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cecfe3321d749f9a2e332b4c6a9657193bfaf6ed79dcc8bf56e567503fafb862 +size 1423889576 diff --git a/model-00230-of-00282.safetensors b/model-00230-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc2c81f561ed3babbda8a8813b51f542a9c5d2af --- /dev/null +++ b/model-00230-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:989692c5af19812a8fbc731f2cdbe3e0eb4375434e32932c7dfea7f44a99e1c6 +size 1529214976 diff --git a/model-00231-of-00282.safetensors b/model-00231-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..22d86d5e575969ae52cc35a7e9010629dcd1bce4 --- /dev/null +++ b/model-00231-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e223fef822ff7f65860a0912decc1ca44d3921bf58c3752d98216fcb5e0568a +size 1423889896 diff --git a/model-00232-of-00282.safetensors b/model-00232-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a5cbe86feea7964dd2e3cf794ab23b364da72f4b --- /dev/null +++ b/model-00232-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bebfe47a9dda1ccf538821dea0aa18b67e85825726855a8357b4586be9d24dbd +size 1423889856 diff --git a/model-00233-of-00282.safetensors b/model-00233-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..afe5b408205433f4937d246099a880aa5cd3c0b5 --- /dev/null +++ b/model-00233-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9847747730508e078edaf0568c172fc45fdf73506162edad5cbe86faf7bb7495 +size 1423889464 diff --git a/model-00234-of-00282.safetensors b/model-00234-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b50381feb2854590669124809b74f538ac78d744 --- /dev/null +++ b/model-00234-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6beaee1a5095254ced98184f6da3f0738732d8207905d4cffdb9a630cffa1781 +size 1529215112 diff --git a/model-00235-of-00282.safetensors b/model-00235-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7b624112fbe878d0e07de64e5fd74bf3b4b8257c --- /dev/null +++ b/model-00235-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb344eefed8cda7dcb9d7b0833665872685013ddd6164e25704c6034672a3ebd +size 1423889888 diff --git a/model-00236-of-00282.safetensors b/model-00236-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..047a7c064b6efe1121d77595df4e64d549052c93 --- /dev/null +++ b/model-00236-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbbc317e1b17db699d49031f9a37a2ccbf304e041bd4154a8e6b1d4e3abb9645 +size 1423889728 diff --git a/model-00237-of-00282.safetensors b/model-00237-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d740ca545afabf1ea9a5ffd26725909a09f77f57 --- /dev/null +++ b/model-00237-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f24f862f8d860564f867aa0b03f28ea8db151de7ae492ca08b6eccba960af06 +size 1529214776 diff --git a/model-00238-of-00282.safetensors b/model-00238-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b9daf235d4d717e3cc96fc9bf763c03ebefdb066 --- /dev/null +++ b/model-00238-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f555995ccf9b1be537faf930f4a6306fb855f9b5f75a4b93fc03b2740c72bac +size 1423889464 diff --git a/model-00239-of-00282.safetensors b/model-00239-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8cda2e657d339946baa31ee02eeb9b12bde2025b --- /dev/null +++ b/model-00239-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df044a2e273b52895b153a56ee762d6aa9ca49d88dd045d2f8844cdb9b89e0e +size 1423889456 diff --git a/model-00240-of-00282.safetensors b/model-00240-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7b96edd4a1f9d5662191b0362d8715532fee010 --- /dev/null +++ b/model-00240-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cdead70a2009f431197ee6e3c684948ef21034c9ab9929ba281e55b2c9e33ac +size 1423889168 diff --git a/model-00241-of-00282.safetensors b/model-00241-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef6d78f15b32ee1a5a3e41eb2c50a773bc869ab2 --- /dev/null +++ b/model-00241-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350fdbec2b25a01b6875192a1dcae5d627b48852d2981266b5d8048b9966dfdb +size 1529214752 diff --git a/model-00242-of-00282.safetensors b/model-00242-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3f62bd7fa519ce3d0d37a42be3647b955e9ba142 --- /dev/null +++ b/model-00242-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1866a338677deaed826f16a06ca86e2dc8a511e1750cc1f1af9da3722de45608 +size 1423889888 diff --git a/model-00243-of-00282.safetensors b/model-00243-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..057873f57bf904324824498e9722331df16eb9b0 --- /dev/null +++ b/model-00243-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32b80dd86ca99946349af2bdbb77328a1baa537cc48634c5aa51f1d49b6c9886 +size 1423889880 diff --git a/model-00244-of-00282.safetensors b/model-00244-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b62eae616e97680b5d7a29aaeee6bbb47255da8d --- /dev/null +++ b/model-00244-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dd3a9d1160fbbb60ff0a2afed35fc6bcf46ee4dcdd52097eed527420640f500 +size 1423889464 diff --git a/model-00245-of-00282.safetensors b/model-00245-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..94989cc12e6312b8841cc7ea85bd91c320d69d96 --- /dev/null +++ b/model-00245-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05396c8410b500de19a9bb610083e63c33f2a2d965b6418c9344c77774fb31a0 +size 1532386504 diff --git a/model-00246-of-00282.safetensors b/model-00246-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b2b4cade6169ca08c10b337fad60858453e1894 --- /dev/null +++ b/model-00246-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e79980e82ef5c0915f3fc4a3637d9be9532b0717fff5b3410549fc9c20612654 +size 1423889880 diff --git a/model-00247-of-00282.safetensors b/model-00247-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..242a8a294de316bb6fa442a92cb2d01c509e9de0 --- /dev/null +++ b/model-00247-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:466927f1b165a0ee97efde4eddee976f884bbd5841168621da8f3f82059c8fbe +size 1423889752 diff --git a/model-00248-of-00282.safetensors b/model-00248-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..578ec50ed5aa1a3e2c7d58f027cba34cc960cddc --- /dev/null +++ b/model-00248-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e9fdfd53498b27a0ba77f7b90a5b7107c67cd0d237158f4021db01a0e7c1eb +size 1529214816 diff --git a/model-00249-of-00282.safetensors b/model-00249-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90ba6a5ac1ea85d2e2b75b6d0f2cd05792eae57f --- /dev/null +++ b/model-00249-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7caf28cf188e596fc2783c51a33341245ffb6992d676fc12f3dc67b9ab23e592 +size 1423889896 diff --git a/model-00250-of-00282.safetensors b/model-00250-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a72dc5caa8fbbceaec858b1d27547f07e7c8d1a --- /dev/null +++ b/model-00250-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6d6a4c9255a699ce0121741eea8eaf10122ef3a9801f62000ed3b7de779dbf +size 1423889880 diff --git a/model-00251-of-00282.safetensors b/model-00251-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7edcd5b5b6947d059c9fcfc543a07238501d112b --- /dev/null +++ b/model-00251-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5fecdb50857c9be4efbd4ad0951c538f370e4559cc359eb5dcfae74e74a0d79 +size 1423889616 diff --git a/model-00252-of-00282.safetensors b/model-00252-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5c5318f43901ffeba002f553a5421049dd72c89b --- /dev/null +++ b/model-00252-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a496c815b5e1168037d785870785a888731e4c3f211888f22b480a3713f52297 +size 1529214936 diff --git a/model-00253-of-00282.safetensors b/model-00253-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..abf3f7fa2294a6378ecfab4f0f1a20348257a57b --- /dev/null +++ b/model-00253-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:656a20a665b148b4a7d42b39144ea5de52b42b9d4761b5b7c7550fc7960cfc5b +size 1423889888 diff --git a/model-00254-of-00282.safetensors b/model-00254-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d6f4dc2c48537c778021ae4b2ba11cfa6b64d1ea --- /dev/null +++ b/model-00254-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7ec87875acb08566027be11b32f953c075556a6345148bcdb2475d9309126c8 +size 1423889888 diff --git a/model-00255-of-00282.safetensors b/model-00255-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..290989ddff357aacc03eb3f828e85f74ecc24be7 --- /dev/null +++ b/model-00255-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ed3d08fb0a5657bbd60667d02dffac8d12c2e1277cac3c6315a628475edccd9 +size 1423889480 diff --git a/model-00256-of-00282.safetensors b/model-00256-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ba766daa315136b26bbc6382c50aa04f551fb56e --- /dev/null +++ b/model-00256-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfbd4013e486799f0ef94b16d47f56256b4fced08b28e8c1e1be576f9019ce51 +size 1529215080 diff --git a/model-00257-of-00282.safetensors b/model-00257-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..06d173ef6ad06f72e7fca000a1d9564ea16f0f2e --- /dev/null +++ b/model-00257-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bccd4039e108feee6d6374919af79b7543de7d368caef079c3bfb123aceff7ad +size 1423889880 diff --git a/model-00258-of-00282.safetensors b/model-00258-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12c50b4ebfd3277f77a84a292c2541f734df1fe6 --- /dev/null +++ b/model-00258-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41458df89c01db28c01382662cf7561ce2f3832ffaad4e02852345c02b98905e +size 1423889768 diff --git a/model-00259-of-00282.safetensors b/model-00259-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ee0965b43cb5525699539f67e02a082dd17851c4 --- /dev/null +++ b/model-00259-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcca03b2b3d5318433845e61beb3e032fc59484bc3654693c8a35e8a3570e3fb +size 1532386200 diff --git a/model-00260-of-00282.safetensors b/model-00260-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b936c54b465781ab7710dc16d5290350739e35e --- /dev/null +++ b/model-00260-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba3376ea9aad042bddef74955615d31dd1aaddabe1d5fbd43b54d573e44c5ab5 +size 1423889896 diff --git a/model-00261-of-00282.safetensors b/model-00261-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3eaf9456951a4d6332e09c9c6546d59729226b1e --- /dev/null +++ b/model-00261-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac6a671718e026e7662875322993eaf2aaf8601ca4c030754400675c1ff909a1 +size 1423889880 diff --git a/model-00262-of-00282.safetensors b/model-00262-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b97b6458678a5c410a540d6781b2d7a5be58bd2 --- /dev/null +++ b/model-00262-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72beca34bec81b2f0f3d61ea104178db35f7920e1113b09df0d1d7ba2b8b9801 +size 1423889640 diff --git a/model-00263-of-00282.safetensors b/model-00263-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2d1c68b3c19846738644f6297a2f81bee18aecef --- /dev/null +++ b/model-00263-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f216d0b373b23db1ca1f17a846dce4cb053a6fb4b703ae65761dd82cda6dbce +size 1529214920 diff --git a/model-00264-of-00282.safetensors b/model-00264-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2617cff843f8a58c5d9c26046de2638416dacb90 --- /dev/null +++ b/model-00264-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88151dab640a62e089fcd03a23f2b3610bd0abf89e54b4d217739b7f580ebc8f +size 1423889896 diff --git a/model-00265-of-00282.safetensors b/model-00265-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b70ddd0973cb357144e4723d1465135c7d5408df --- /dev/null +++ b/model-00265-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b018507f9665eaefdf0a6226d1f426f9ecce2274979dd918a6162755458866b +size 1423889880 diff --git a/model-00266-of-00282.safetensors b/model-00266-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9d7325ebf2bdf4659ab8cc46a62594d3ed757e80 --- /dev/null +++ b/model-00266-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84069c591973d06f806114fbfd3a31d1f60f982bfd7ce4a137dfee1b584bc240 +size 1423889504 diff --git a/model-00267-of-00282.safetensors b/model-00267-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f75faca82b2835c50e30846dd7ece5b8a1777be2 --- /dev/null +++ b/model-00267-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f342149509a7f9fc22633ff71ba010bc0176f4aaee672a1f55da78cee45c1418 +size 1529215056 diff --git a/model-00268-of-00282.safetensors b/model-00268-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dbf9b6141037fa65d6ba7a5cbd551317bc7b6291 --- /dev/null +++ b/model-00268-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273b804967345bf79b0b5f2d8c7a45ae310cd1840ed336a212d68d3acd576543 +size 1423889888 diff --git a/model-00269-of-00282.safetensors b/model-00269-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eb2c18b558151bdd42201ade7d74d5c4f51c3fdb --- /dev/null +++ b/model-00269-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8686464780be51e9faaa6257f853094e1d79b8dbc0e79cb2e758546721dcb0c +size 1423889784 diff --git a/model-00270-of-00282.safetensors b/model-00270-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..59c5d66f9c85143360430a5984f0a47474c4e19b --- /dev/null +++ b/model-00270-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9022342ee66ad4316f2e1ace865c38737b46deaec1815a79f320e4ab67bc9e82 +size 1566987016 diff --git a/model-00271-of-00282.safetensors b/model-00271-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a658df9f936447c6c9be463878611ac2e30d7cc5 --- /dev/null +++ b/model-00271-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a6a74846ed6f4d1662ca081488872a55252605d88cf349656713ca324586735 +size 1423889864 diff --git a/model-00272-of-00282.safetensors b/model-00272-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a15be4c332792a142ed63ee30b69c5992357129 --- /dev/null +++ b/model-00272-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59bf2b92078c623e5cd74c0ad25916c1aaa7f6758dd305b5f2ecbd9465420704 +size 1423889888 diff --git a/model-00273-of-00282.safetensors b/model-00273-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..517eba33802aa0ac9fae44024a931b98e640152a --- /dev/null +++ b/model-00273-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60119afb5ffe18276ac5c350975a082183bd0488ccfb1c6e2316eec41c4d0299 +size 1423889664 diff --git a/model-00274-of-00282.safetensors b/model-00274-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2daa955a3dea80988e024d718bffee6dbb9dd35d --- /dev/null +++ b/model-00274-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2250e8e3413aae6650f6d31139d428e0c4eda6ba5b2b190e60e849529fd2ff4 +size 1532398568 diff --git a/model-00275-of-00282.safetensors b/model-00275-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b2ff76d08b78f15f236f2d1bc65be5b78701b93e --- /dev/null +++ b/model-00275-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e78d8023d2769abf167a6992ad10f112848b5bfee2f25277e9236a7f3d296f +size 1423889464 diff --git a/model-00276-of-00282.safetensors b/model-00276-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..46c2ec2339ae2f45ababf1cb74d9c7df61aef48f --- /dev/null +++ b/model-00276-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ae1aa6dbc6db48f323cba656debfa29d52d7587ca626c4423add0415a954383 +size 1423889456 diff --git a/model-00277-of-00282.safetensors b/model-00277-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c39b4d447b31033e0227cd14a957db7592ab6c55 --- /dev/null +++ b/model-00277-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:771cf0cec4a907a9aa8f133fe6f0ec72ba29ba68a55cd88910ac122cda664edc +size 1423889104 diff --git a/model-00278-of-00282.safetensors b/model-00278-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6175d620c6f75759cce7b9a056eae07804f2f149 --- /dev/null +++ b/model-00278-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f17e09a10560f2be29b4fdf920beabfc4efb9b79273e966938340347590839 +size 1529214616 diff --git a/model-00279-of-00282.safetensors b/model-00279-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3fc8d0c568918cb89f82f68fd247a64c1f7da5e6 --- /dev/null +++ b/model-00279-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b577afd1bd50143c9dc925bda3fc1227053101aa167c8c59ecd6b0dbe28f38c +size 1423889456 diff --git a/model-00280-of-00282.safetensors b/model-00280-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2575e9c0b23fc4a1318a8df96dd37a410c6245e4 --- /dev/null +++ b/model-00280-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:509dd73e9183a6e99332b5420263007de63fc94c3d939f80e9b6e4beafa1eb25 +size 1423889400 diff --git a/model-00281-of-00282.safetensors b/model-00281-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..19c26c4acd4dd041358e6c71f41fa43b31b95256 --- /dev/null +++ b/model-00281-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a255d6adbc3055d8dc4b24905285712d35f94137e25559454fdbc99b8163a3c2 +size 1444658056 diff --git a/model-00282-of-00282.safetensors b/model-00282-of-00282.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1ca4d0f5690da0421b68cc05157c93c4faf8bdb1 --- /dev/null +++ b/model-00282-of-00282.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e383ae424bca69bb9728ed83e5a49b2e743568f6f52d5c01ae6fd21e6e5a7cd +size 151405456 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..0c22bf8aa5475552aca9cd6e93eb109f56c2079d --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deed66c5d5102260b99056b51d21396a7977dc9a500bc481ca4f9586ef46300c +size 11530961 diff --git a/quantization_config.json b/quantization_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf30d822f4650ebfaf44c843da22a8ed1a6a348 --- /dev/null +++ b/quantization_config.json @@ -0,0 +1,58559 @@ +{ + "config_groups": { + "group_0": { + "targets": [ + "RoutedExperts", + "model.layers.10.mlp.experts.0.down_proj", + "model.layers.10.mlp.experts.0.gate_proj", + "model.layers.10.mlp.experts.0.up_proj", + "model.layers.10.mlp.experts.1.down_proj", + "model.layers.10.mlp.experts.1.gate_proj", + "model.layers.10.mlp.experts.1.up_proj", + "model.layers.10.mlp.experts.10.down_proj", + "model.layers.10.mlp.experts.10.gate_proj", + "model.layers.10.mlp.experts.10.up_proj", + "model.layers.10.mlp.experts.100.down_proj", + "model.layers.10.mlp.experts.100.gate_proj", + "model.layers.10.mlp.experts.100.up_proj", + "model.layers.10.mlp.experts.101.down_proj", + "model.layers.10.mlp.experts.101.gate_proj", + "model.layers.10.mlp.experts.101.up_proj", + "model.layers.10.mlp.experts.102.down_proj", + "model.layers.10.mlp.experts.102.gate_proj", + "model.layers.10.mlp.experts.102.up_proj", + "model.layers.10.mlp.experts.103.down_proj", + "model.layers.10.mlp.experts.103.gate_proj", + "model.layers.10.mlp.experts.103.up_proj", + "model.layers.10.mlp.experts.104.down_proj", + "model.layers.10.mlp.experts.104.gate_proj", + "model.layers.10.mlp.experts.104.up_proj", + "model.layers.10.mlp.experts.105.down_proj", + "model.layers.10.mlp.experts.105.gate_proj", + "model.layers.10.mlp.experts.105.up_proj", + "model.layers.10.mlp.experts.106.down_proj", + "model.layers.10.mlp.experts.106.gate_proj", + "model.layers.10.mlp.experts.106.up_proj", + "model.layers.10.mlp.experts.107.down_proj", + "model.layers.10.mlp.experts.107.gate_proj", + "model.layers.10.mlp.experts.107.up_proj", + "model.layers.10.mlp.experts.108.down_proj", + "model.layers.10.mlp.experts.108.gate_proj", + "model.layers.10.mlp.experts.108.up_proj", + "model.layers.10.mlp.experts.109.down_proj", + "model.layers.10.mlp.experts.109.gate_proj", + "model.layers.10.mlp.experts.109.up_proj", + "model.layers.10.mlp.experts.11.down_proj", + "model.layers.10.mlp.experts.11.gate_proj", + "model.layers.10.mlp.experts.11.up_proj", + "model.layers.10.mlp.experts.110.down_proj", + "model.layers.10.mlp.experts.110.gate_proj", + "model.layers.10.mlp.experts.110.up_proj", + "model.layers.10.mlp.experts.111.down_proj", + "model.layers.10.mlp.experts.111.gate_proj", + "model.layers.10.mlp.experts.111.up_proj", + "model.layers.10.mlp.experts.112.down_proj", + "model.layers.10.mlp.experts.112.gate_proj", + "model.layers.10.mlp.experts.112.up_proj", + "model.layers.10.mlp.experts.113.down_proj", + "model.layers.10.mlp.experts.113.gate_proj", + "model.layers.10.mlp.experts.113.up_proj", + "model.layers.10.mlp.experts.114.down_proj", + "model.layers.10.mlp.experts.114.gate_proj", + "model.layers.10.mlp.experts.114.up_proj", + "model.layers.10.mlp.experts.115.down_proj", + "model.layers.10.mlp.experts.115.gate_proj", + "model.layers.10.mlp.experts.115.up_proj", + "model.layers.10.mlp.experts.116.down_proj", + "model.layers.10.mlp.experts.116.gate_proj", + "model.layers.10.mlp.experts.116.up_proj", + "model.layers.10.mlp.experts.117.down_proj", + "model.layers.10.mlp.experts.117.gate_proj", + "model.layers.10.mlp.experts.117.up_proj", + "model.layers.10.mlp.experts.118.down_proj", + "model.layers.10.mlp.experts.118.gate_proj", + "model.layers.10.mlp.experts.118.up_proj", + "model.layers.10.mlp.experts.119.down_proj", + "model.layers.10.mlp.experts.119.gate_proj", + "model.layers.10.mlp.experts.119.up_proj", + "model.layers.10.mlp.experts.12.down_proj", + "model.layers.10.mlp.experts.12.gate_proj", + "model.layers.10.mlp.experts.12.up_proj", + "model.layers.10.mlp.experts.120.down_proj", + "model.layers.10.mlp.experts.120.gate_proj", + "model.layers.10.mlp.experts.120.up_proj", + "model.layers.10.mlp.experts.121.down_proj", + "model.layers.10.mlp.experts.121.gate_proj", + "model.layers.10.mlp.experts.121.up_proj", + "model.layers.10.mlp.experts.122.down_proj", + "model.layers.10.mlp.experts.122.gate_proj", + "model.layers.10.mlp.experts.122.up_proj", + "model.layers.10.mlp.experts.123.down_proj", + "model.layers.10.mlp.experts.123.gate_proj", + "model.layers.10.mlp.experts.123.up_proj", + "model.layers.10.mlp.experts.124.down_proj", + "model.layers.10.mlp.experts.124.gate_proj", + "model.layers.10.mlp.experts.124.up_proj", + "model.layers.10.mlp.experts.125.down_proj", + "model.layers.10.mlp.experts.125.gate_proj", + "model.layers.10.mlp.experts.125.up_proj", + "model.layers.10.mlp.experts.126.down_proj", + "model.layers.10.mlp.experts.126.gate_proj", + "model.layers.10.mlp.experts.126.up_proj", + "model.layers.10.mlp.experts.127.down_proj", + "model.layers.10.mlp.experts.127.gate_proj", + "model.layers.10.mlp.experts.127.up_proj", + "model.layers.10.mlp.experts.128.down_proj", + "model.layers.10.mlp.experts.128.gate_proj", + "model.layers.10.mlp.experts.128.up_proj", + "model.layers.10.mlp.experts.129.down_proj", + "model.layers.10.mlp.experts.129.gate_proj", + "model.layers.10.mlp.experts.129.up_proj", + "model.layers.10.mlp.experts.13.down_proj", + "model.layers.10.mlp.experts.13.gate_proj", + "model.layers.10.mlp.experts.13.up_proj", + "model.layers.10.mlp.experts.130.down_proj", + "model.layers.10.mlp.experts.130.gate_proj", + "model.layers.10.mlp.experts.130.up_proj", + "model.layers.10.mlp.experts.131.down_proj", + "model.layers.10.mlp.experts.131.gate_proj", + "model.layers.10.mlp.experts.131.up_proj", + "model.layers.10.mlp.experts.132.down_proj", + "model.layers.10.mlp.experts.132.gate_proj", + "model.layers.10.mlp.experts.132.up_proj", + "model.layers.10.mlp.experts.133.down_proj", + "model.layers.10.mlp.experts.133.gate_proj", + "model.layers.10.mlp.experts.133.up_proj", + "model.layers.10.mlp.experts.134.down_proj", + "model.layers.10.mlp.experts.134.gate_proj", + "model.layers.10.mlp.experts.134.up_proj", + "model.layers.10.mlp.experts.135.down_proj", + "model.layers.10.mlp.experts.135.gate_proj", + "model.layers.10.mlp.experts.135.up_proj", + "model.layers.10.mlp.experts.136.down_proj", + "model.layers.10.mlp.experts.136.gate_proj", + "model.layers.10.mlp.experts.136.up_proj", + "model.layers.10.mlp.experts.137.down_proj", + "model.layers.10.mlp.experts.137.gate_proj", + "model.layers.10.mlp.experts.137.up_proj", + "model.layers.10.mlp.experts.138.down_proj", + "model.layers.10.mlp.experts.138.gate_proj", + "model.layers.10.mlp.experts.138.up_proj", + "model.layers.10.mlp.experts.139.down_proj", + "model.layers.10.mlp.experts.139.gate_proj", + "model.layers.10.mlp.experts.139.up_proj", + "model.layers.10.mlp.experts.14.down_proj", + "model.layers.10.mlp.experts.14.gate_proj", + "model.layers.10.mlp.experts.14.up_proj", + "model.layers.10.mlp.experts.140.down_proj", + "model.layers.10.mlp.experts.140.gate_proj", + "model.layers.10.mlp.experts.140.up_proj", + "model.layers.10.mlp.experts.141.down_proj", + "model.layers.10.mlp.experts.141.gate_proj", + "model.layers.10.mlp.experts.141.up_proj", + "model.layers.10.mlp.experts.142.down_proj", + "model.layers.10.mlp.experts.142.gate_proj", + "model.layers.10.mlp.experts.142.up_proj", + "model.layers.10.mlp.experts.143.down_proj", + "model.layers.10.mlp.experts.143.gate_proj", + "model.layers.10.mlp.experts.143.up_proj", + "model.layers.10.mlp.experts.144.down_proj", + "model.layers.10.mlp.experts.144.gate_proj", + "model.layers.10.mlp.experts.144.up_proj", + "model.layers.10.mlp.experts.145.down_proj", + "model.layers.10.mlp.experts.145.gate_proj", + "model.layers.10.mlp.experts.145.up_proj", + "model.layers.10.mlp.experts.146.down_proj", + "model.layers.10.mlp.experts.146.gate_proj", + "model.layers.10.mlp.experts.146.up_proj", + "model.layers.10.mlp.experts.147.down_proj", + "model.layers.10.mlp.experts.147.gate_proj", + "model.layers.10.mlp.experts.147.up_proj", + "model.layers.10.mlp.experts.148.down_proj", + "model.layers.10.mlp.experts.148.gate_proj", + "model.layers.10.mlp.experts.148.up_proj", + "model.layers.10.mlp.experts.149.down_proj", + "model.layers.10.mlp.experts.149.gate_proj", + "model.layers.10.mlp.experts.149.up_proj", + "model.layers.10.mlp.experts.15.down_proj", + "model.layers.10.mlp.experts.15.gate_proj", + "model.layers.10.mlp.experts.15.up_proj", + "model.layers.10.mlp.experts.150.down_proj", + "model.layers.10.mlp.experts.150.gate_proj", + "model.layers.10.mlp.experts.150.up_proj", + "model.layers.10.mlp.experts.151.down_proj", + "model.layers.10.mlp.experts.151.gate_proj", + "model.layers.10.mlp.experts.151.up_proj", + "model.layers.10.mlp.experts.152.down_proj", + "model.layers.10.mlp.experts.152.gate_proj", + "model.layers.10.mlp.experts.152.up_proj", + "model.layers.10.mlp.experts.153.down_proj", + "model.layers.10.mlp.experts.153.gate_proj", + "model.layers.10.mlp.experts.153.up_proj", + "model.layers.10.mlp.experts.154.down_proj", + "model.layers.10.mlp.experts.154.gate_proj", + "model.layers.10.mlp.experts.154.up_proj", + "model.layers.10.mlp.experts.155.down_proj", + "model.layers.10.mlp.experts.155.gate_proj", + "model.layers.10.mlp.experts.155.up_proj", + "model.layers.10.mlp.experts.156.down_proj", + "model.layers.10.mlp.experts.156.gate_proj", + "model.layers.10.mlp.experts.156.up_proj", + "model.layers.10.mlp.experts.157.down_proj", + "model.layers.10.mlp.experts.157.gate_proj", + "model.layers.10.mlp.experts.157.up_proj", + "model.layers.10.mlp.experts.158.down_proj", + "model.layers.10.mlp.experts.158.gate_proj", + "model.layers.10.mlp.experts.158.up_proj", + "model.layers.10.mlp.experts.159.down_proj", + "model.layers.10.mlp.experts.159.gate_proj", + "model.layers.10.mlp.experts.159.up_proj", + "model.layers.10.mlp.experts.16.down_proj", + "model.layers.10.mlp.experts.16.gate_proj", + "model.layers.10.mlp.experts.16.up_proj", + "model.layers.10.mlp.experts.160.down_proj", + "model.layers.10.mlp.experts.160.gate_proj", + "model.layers.10.mlp.experts.160.up_proj", + "model.layers.10.mlp.experts.161.down_proj", + "model.layers.10.mlp.experts.161.gate_proj", + "model.layers.10.mlp.experts.161.up_proj", + "model.layers.10.mlp.experts.162.down_proj", + "model.layers.10.mlp.experts.162.gate_proj", + "model.layers.10.mlp.experts.162.up_proj", + "model.layers.10.mlp.experts.163.down_proj", + "model.layers.10.mlp.experts.163.gate_proj", + "model.layers.10.mlp.experts.163.up_proj", + "model.layers.10.mlp.experts.164.down_proj", + "model.layers.10.mlp.experts.164.gate_proj", + "model.layers.10.mlp.experts.164.up_proj", + "model.layers.10.mlp.experts.165.down_proj", + "model.layers.10.mlp.experts.165.gate_proj", + "model.layers.10.mlp.experts.165.up_proj", + "model.layers.10.mlp.experts.166.down_proj", + "model.layers.10.mlp.experts.166.gate_proj", + "model.layers.10.mlp.experts.166.up_proj", + "model.layers.10.mlp.experts.167.down_proj", + "model.layers.10.mlp.experts.167.gate_proj", + "model.layers.10.mlp.experts.167.up_proj", + "model.layers.10.mlp.experts.168.down_proj", + "model.layers.10.mlp.experts.168.gate_proj", + "model.layers.10.mlp.experts.168.up_proj", + "model.layers.10.mlp.experts.169.down_proj", + "model.layers.10.mlp.experts.169.gate_proj", + "model.layers.10.mlp.experts.169.up_proj", + "model.layers.10.mlp.experts.17.down_proj", + "model.layers.10.mlp.experts.17.gate_proj", + "model.layers.10.mlp.experts.17.up_proj", + "model.layers.10.mlp.experts.170.down_proj", + "model.layers.10.mlp.experts.170.gate_proj", + "model.layers.10.mlp.experts.170.up_proj", + "model.layers.10.mlp.experts.171.down_proj", + "model.layers.10.mlp.experts.171.gate_proj", + "model.layers.10.mlp.experts.171.up_proj", + "model.layers.10.mlp.experts.172.down_proj", + "model.layers.10.mlp.experts.172.gate_proj", + "model.layers.10.mlp.experts.172.up_proj", + "model.layers.10.mlp.experts.173.down_proj", + "model.layers.10.mlp.experts.173.gate_proj", + "model.layers.10.mlp.experts.173.up_proj", + "model.layers.10.mlp.experts.174.down_proj", + "model.layers.10.mlp.experts.174.gate_proj", + "model.layers.10.mlp.experts.174.up_proj", + "model.layers.10.mlp.experts.175.down_proj", + "model.layers.10.mlp.experts.175.gate_proj", + "model.layers.10.mlp.experts.175.up_proj", + "model.layers.10.mlp.experts.176.down_proj", + "model.layers.10.mlp.experts.176.gate_proj", + "model.layers.10.mlp.experts.176.up_proj", + "model.layers.10.mlp.experts.177.down_proj", + "model.layers.10.mlp.experts.177.gate_proj", + "model.layers.10.mlp.experts.177.up_proj", + "model.layers.10.mlp.experts.178.down_proj", + "model.layers.10.mlp.experts.178.gate_proj", + "model.layers.10.mlp.experts.178.up_proj", + "model.layers.10.mlp.experts.179.down_proj", + "model.layers.10.mlp.experts.179.gate_proj", + "model.layers.10.mlp.experts.179.up_proj", + "model.layers.10.mlp.experts.18.down_proj", + "model.layers.10.mlp.experts.18.gate_proj", + "model.layers.10.mlp.experts.18.up_proj", + "model.layers.10.mlp.experts.180.down_proj", + "model.layers.10.mlp.experts.180.gate_proj", + "model.layers.10.mlp.experts.180.up_proj", + "model.layers.10.mlp.experts.181.down_proj", + "model.layers.10.mlp.experts.181.gate_proj", + "model.layers.10.mlp.experts.181.up_proj", + "model.layers.10.mlp.experts.182.down_proj", + "model.layers.10.mlp.experts.182.gate_proj", + "model.layers.10.mlp.experts.182.up_proj", + "model.layers.10.mlp.experts.183.down_proj", + "model.layers.10.mlp.experts.183.gate_proj", + "model.layers.10.mlp.experts.183.up_proj", + "model.layers.10.mlp.experts.184.down_proj", + "model.layers.10.mlp.experts.184.gate_proj", + "model.layers.10.mlp.experts.184.up_proj", + "model.layers.10.mlp.experts.185.down_proj", + "model.layers.10.mlp.experts.185.gate_proj", + "model.layers.10.mlp.experts.185.up_proj", + "model.layers.10.mlp.experts.186.down_proj", + "model.layers.10.mlp.experts.186.gate_proj", + "model.layers.10.mlp.experts.186.up_proj", + "model.layers.10.mlp.experts.187.down_proj", + "model.layers.10.mlp.experts.187.gate_proj", + "model.layers.10.mlp.experts.187.up_proj", + "model.layers.10.mlp.experts.188.down_proj", + "model.layers.10.mlp.experts.188.gate_proj", + "model.layers.10.mlp.experts.188.up_proj", + "model.layers.10.mlp.experts.189.down_proj", + "model.layers.10.mlp.experts.189.gate_proj", + "model.layers.10.mlp.experts.189.up_proj", + "model.layers.10.mlp.experts.19.down_proj", + "model.layers.10.mlp.experts.19.gate_proj", + "model.layers.10.mlp.experts.19.up_proj", + "model.layers.10.mlp.experts.190.down_proj", + "model.layers.10.mlp.experts.190.gate_proj", + "model.layers.10.mlp.experts.190.up_proj", + "model.layers.10.mlp.experts.191.down_proj", + "model.layers.10.mlp.experts.191.gate_proj", + "model.layers.10.mlp.experts.191.up_proj", + "model.layers.10.mlp.experts.192.down_proj", + "model.layers.10.mlp.experts.192.gate_proj", + "model.layers.10.mlp.experts.192.up_proj", + "model.layers.10.mlp.experts.193.down_proj", + "model.layers.10.mlp.experts.193.gate_proj", + "model.layers.10.mlp.experts.193.up_proj", + "model.layers.10.mlp.experts.194.down_proj", + "model.layers.10.mlp.experts.194.gate_proj", + "model.layers.10.mlp.experts.194.up_proj", + "model.layers.10.mlp.experts.195.down_proj", + "model.layers.10.mlp.experts.195.gate_proj", + "model.layers.10.mlp.experts.195.up_proj", + "model.layers.10.mlp.experts.196.down_proj", + "model.layers.10.mlp.experts.196.gate_proj", + "model.layers.10.mlp.experts.196.up_proj", + "model.layers.10.mlp.experts.197.down_proj", + "model.layers.10.mlp.experts.197.gate_proj", + "model.layers.10.mlp.experts.197.up_proj", + "model.layers.10.mlp.experts.198.down_proj", + "model.layers.10.mlp.experts.198.gate_proj", + "model.layers.10.mlp.experts.198.up_proj", + "model.layers.10.mlp.experts.199.down_proj", + "model.layers.10.mlp.experts.199.gate_proj", + "model.layers.10.mlp.experts.199.up_proj", + "model.layers.10.mlp.experts.2.down_proj", + "model.layers.10.mlp.experts.2.gate_proj", + "model.layers.10.mlp.experts.2.up_proj", + "model.layers.10.mlp.experts.20.down_proj", + "model.layers.10.mlp.experts.20.gate_proj", + "model.layers.10.mlp.experts.20.up_proj", + "model.layers.10.mlp.experts.200.down_proj", + "model.layers.10.mlp.experts.200.gate_proj", + "model.layers.10.mlp.experts.200.up_proj", + "model.layers.10.mlp.experts.201.down_proj", + "model.layers.10.mlp.experts.201.gate_proj", + "model.layers.10.mlp.experts.201.up_proj", + "model.layers.10.mlp.experts.202.down_proj", + "model.layers.10.mlp.experts.202.gate_proj", + "model.layers.10.mlp.experts.202.up_proj", + "model.layers.10.mlp.experts.203.down_proj", + "model.layers.10.mlp.experts.203.gate_proj", + "model.layers.10.mlp.experts.203.up_proj", + "model.layers.10.mlp.experts.204.down_proj", + "model.layers.10.mlp.experts.204.gate_proj", + "model.layers.10.mlp.experts.204.up_proj", + "model.layers.10.mlp.experts.205.down_proj", + "model.layers.10.mlp.experts.205.gate_proj", + "model.layers.10.mlp.experts.205.up_proj", + "model.layers.10.mlp.experts.206.down_proj", + "model.layers.10.mlp.experts.206.gate_proj", + "model.layers.10.mlp.experts.206.up_proj", + "model.layers.10.mlp.experts.207.down_proj", + "model.layers.10.mlp.experts.207.gate_proj", + "model.layers.10.mlp.experts.207.up_proj", + "model.layers.10.mlp.experts.208.down_proj", + "model.layers.10.mlp.experts.208.gate_proj", + "model.layers.10.mlp.experts.208.up_proj", + "model.layers.10.mlp.experts.209.down_proj", + "model.layers.10.mlp.experts.209.gate_proj", + "model.layers.10.mlp.experts.209.up_proj", + "model.layers.10.mlp.experts.21.down_proj", + "model.layers.10.mlp.experts.21.gate_proj", + "model.layers.10.mlp.experts.21.up_proj", + "model.layers.10.mlp.experts.210.down_proj", + "model.layers.10.mlp.experts.210.gate_proj", + "model.layers.10.mlp.experts.210.up_proj", + "model.layers.10.mlp.experts.211.down_proj", + "model.layers.10.mlp.experts.211.gate_proj", + "model.layers.10.mlp.experts.211.up_proj", + "model.layers.10.mlp.experts.212.down_proj", + "model.layers.10.mlp.experts.212.gate_proj", + "model.layers.10.mlp.experts.212.up_proj", + "model.layers.10.mlp.experts.213.down_proj", + "model.layers.10.mlp.experts.213.gate_proj", + "model.layers.10.mlp.experts.213.up_proj", + "model.layers.10.mlp.experts.214.down_proj", + "model.layers.10.mlp.experts.214.gate_proj", + "model.layers.10.mlp.experts.214.up_proj", + "model.layers.10.mlp.experts.215.down_proj", + "model.layers.10.mlp.experts.215.gate_proj", + "model.layers.10.mlp.experts.215.up_proj", + "model.layers.10.mlp.experts.216.down_proj", + "model.layers.10.mlp.experts.216.gate_proj", + "model.layers.10.mlp.experts.216.up_proj", + "model.layers.10.mlp.experts.217.down_proj", + "model.layers.10.mlp.experts.217.gate_proj", + "model.layers.10.mlp.experts.217.up_proj", + "model.layers.10.mlp.experts.218.down_proj", + "model.layers.10.mlp.experts.218.gate_proj", + "model.layers.10.mlp.experts.218.up_proj", + "model.layers.10.mlp.experts.219.down_proj", + "model.layers.10.mlp.experts.219.gate_proj", + "model.layers.10.mlp.experts.219.up_proj", + "model.layers.10.mlp.experts.22.down_proj", + "model.layers.10.mlp.experts.22.gate_proj", + "model.layers.10.mlp.experts.22.up_proj", + "model.layers.10.mlp.experts.220.down_proj", + "model.layers.10.mlp.experts.220.gate_proj", + "model.layers.10.mlp.experts.220.up_proj", + "model.layers.10.mlp.experts.221.down_proj", + "model.layers.10.mlp.experts.221.gate_proj", + "model.layers.10.mlp.experts.221.up_proj", + "model.layers.10.mlp.experts.222.down_proj", + "model.layers.10.mlp.experts.222.gate_proj", + "model.layers.10.mlp.experts.222.up_proj", + "model.layers.10.mlp.experts.223.down_proj", + "model.layers.10.mlp.experts.223.gate_proj", + "model.layers.10.mlp.experts.223.up_proj", + "model.layers.10.mlp.experts.224.down_proj", + "model.layers.10.mlp.experts.224.gate_proj", + "model.layers.10.mlp.experts.224.up_proj", + "model.layers.10.mlp.experts.225.down_proj", + "model.layers.10.mlp.experts.225.gate_proj", + "model.layers.10.mlp.experts.225.up_proj", + "model.layers.10.mlp.experts.226.down_proj", + "model.layers.10.mlp.experts.226.gate_proj", + "model.layers.10.mlp.experts.226.up_proj", + "model.layers.10.mlp.experts.227.down_proj", + "model.layers.10.mlp.experts.227.gate_proj", + "model.layers.10.mlp.experts.227.up_proj", + "model.layers.10.mlp.experts.228.down_proj", + "model.layers.10.mlp.experts.228.gate_proj", + "model.layers.10.mlp.experts.228.up_proj", + "model.layers.10.mlp.experts.229.down_proj", + "model.layers.10.mlp.experts.229.gate_proj", + "model.layers.10.mlp.experts.229.up_proj", + "model.layers.10.mlp.experts.23.down_proj", + "model.layers.10.mlp.experts.23.gate_proj", + "model.layers.10.mlp.experts.23.up_proj", + "model.layers.10.mlp.experts.230.down_proj", + "model.layers.10.mlp.experts.230.gate_proj", + "model.layers.10.mlp.experts.230.up_proj", + "model.layers.10.mlp.experts.231.down_proj", + "model.layers.10.mlp.experts.231.gate_proj", + "model.layers.10.mlp.experts.231.up_proj", + "model.layers.10.mlp.experts.232.down_proj", + "model.layers.10.mlp.experts.232.gate_proj", + "model.layers.10.mlp.experts.232.up_proj", + "model.layers.10.mlp.experts.233.down_proj", + "model.layers.10.mlp.experts.233.gate_proj", + "model.layers.10.mlp.experts.233.up_proj", + "model.layers.10.mlp.experts.234.down_proj", + "model.layers.10.mlp.experts.234.gate_proj", + "model.layers.10.mlp.experts.234.up_proj", + "model.layers.10.mlp.experts.235.down_proj", + "model.layers.10.mlp.experts.235.gate_proj", + "model.layers.10.mlp.experts.235.up_proj", + "model.layers.10.mlp.experts.236.down_proj", + "model.layers.10.mlp.experts.236.gate_proj", + "model.layers.10.mlp.experts.236.up_proj", + "model.layers.10.mlp.experts.237.down_proj", + "model.layers.10.mlp.experts.237.gate_proj", + "model.layers.10.mlp.experts.237.up_proj", + "model.layers.10.mlp.experts.238.down_proj", + "model.layers.10.mlp.experts.238.gate_proj", + "model.layers.10.mlp.experts.238.up_proj", + "model.layers.10.mlp.experts.239.down_proj", + "model.layers.10.mlp.experts.239.gate_proj", + "model.layers.10.mlp.experts.239.up_proj", + "model.layers.10.mlp.experts.24.down_proj", + "model.layers.10.mlp.experts.24.gate_proj", + "model.layers.10.mlp.experts.24.up_proj", + "model.layers.10.mlp.experts.240.down_proj", + "model.layers.10.mlp.experts.240.gate_proj", + "model.layers.10.mlp.experts.240.up_proj", + "model.layers.10.mlp.experts.241.down_proj", + "model.layers.10.mlp.experts.241.gate_proj", + "model.layers.10.mlp.experts.241.up_proj", + "model.layers.10.mlp.experts.242.down_proj", + "model.layers.10.mlp.experts.242.gate_proj", + "model.layers.10.mlp.experts.242.up_proj", + "model.layers.10.mlp.experts.243.down_proj", + "model.layers.10.mlp.experts.243.gate_proj", + "model.layers.10.mlp.experts.243.up_proj", + "model.layers.10.mlp.experts.244.down_proj", + "model.layers.10.mlp.experts.244.gate_proj", + "model.layers.10.mlp.experts.244.up_proj", + "model.layers.10.mlp.experts.245.down_proj", + "model.layers.10.mlp.experts.245.gate_proj", + "model.layers.10.mlp.experts.245.up_proj", + "model.layers.10.mlp.experts.246.down_proj", + "model.layers.10.mlp.experts.246.gate_proj", + "model.layers.10.mlp.experts.246.up_proj", + "model.layers.10.mlp.experts.247.down_proj", + "model.layers.10.mlp.experts.247.gate_proj", + "model.layers.10.mlp.experts.247.up_proj", + "model.layers.10.mlp.experts.248.down_proj", + "model.layers.10.mlp.experts.248.gate_proj", + "model.layers.10.mlp.experts.248.up_proj", + "model.layers.10.mlp.experts.249.down_proj", + "model.layers.10.mlp.experts.249.gate_proj", + "model.layers.10.mlp.experts.249.up_proj", + "model.layers.10.mlp.experts.25.down_proj", + "model.layers.10.mlp.experts.25.gate_proj", + "model.layers.10.mlp.experts.25.up_proj", + "model.layers.10.mlp.experts.250.down_proj", + "model.layers.10.mlp.experts.250.gate_proj", + "model.layers.10.mlp.experts.250.up_proj", + "model.layers.10.mlp.experts.251.down_proj", + "model.layers.10.mlp.experts.251.gate_proj", + "model.layers.10.mlp.experts.251.up_proj", + "model.layers.10.mlp.experts.252.down_proj", + "model.layers.10.mlp.experts.252.gate_proj", + "model.layers.10.mlp.experts.252.up_proj", + "model.layers.10.mlp.experts.253.down_proj", + "model.layers.10.mlp.experts.253.gate_proj", + "model.layers.10.mlp.experts.253.up_proj", + "model.layers.10.mlp.experts.254.down_proj", + "model.layers.10.mlp.experts.254.gate_proj", + "model.layers.10.mlp.experts.254.up_proj", + "model.layers.10.mlp.experts.255.down_proj", + "model.layers.10.mlp.experts.255.gate_proj", + "model.layers.10.mlp.experts.255.up_proj", + "model.layers.10.mlp.experts.26.down_proj", + "model.layers.10.mlp.experts.26.gate_proj", + "model.layers.10.mlp.experts.26.up_proj", + "model.layers.10.mlp.experts.27.down_proj", + "model.layers.10.mlp.experts.27.gate_proj", + "model.layers.10.mlp.experts.27.up_proj", + "model.layers.10.mlp.experts.28.down_proj", + "model.layers.10.mlp.experts.28.gate_proj", + "model.layers.10.mlp.experts.28.up_proj", + "model.layers.10.mlp.experts.29.down_proj", + "model.layers.10.mlp.experts.29.gate_proj", + "model.layers.10.mlp.experts.29.up_proj", + "model.layers.10.mlp.experts.3.down_proj", + "model.layers.10.mlp.experts.3.gate_proj", + "model.layers.10.mlp.experts.3.up_proj", + "model.layers.10.mlp.experts.30.down_proj", + "model.layers.10.mlp.experts.30.gate_proj", + "model.layers.10.mlp.experts.30.up_proj", + "model.layers.10.mlp.experts.31.down_proj", + "model.layers.10.mlp.experts.31.gate_proj", + "model.layers.10.mlp.experts.31.up_proj", + "model.layers.10.mlp.experts.32.down_proj", + "model.layers.10.mlp.experts.32.gate_proj", + "model.layers.10.mlp.experts.32.up_proj", + "model.layers.10.mlp.experts.33.down_proj", + "model.layers.10.mlp.experts.33.gate_proj", + "model.layers.10.mlp.experts.33.up_proj", + "model.layers.10.mlp.experts.34.down_proj", + "model.layers.10.mlp.experts.34.gate_proj", + "model.layers.10.mlp.experts.34.up_proj", + "model.layers.10.mlp.experts.35.down_proj", + "model.layers.10.mlp.experts.35.gate_proj", + "model.layers.10.mlp.experts.35.up_proj", + "model.layers.10.mlp.experts.36.down_proj", + "model.layers.10.mlp.experts.36.gate_proj", + "model.layers.10.mlp.experts.36.up_proj", + "model.layers.10.mlp.experts.37.down_proj", + "model.layers.10.mlp.experts.37.gate_proj", + "model.layers.10.mlp.experts.37.up_proj", + "model.layers.10.mlp.experts.38.down_proj", + "model.layers.10.mlp.experts.38.gate_proj", + "model.layers.10.mlp.experts.38.up_proj", + "model.layers.10.mlp.experts.39.down_proj", + "model.layers.10.mlp.experts.39.gate_proj", + "model.layers.10.mlp.experts.39.up_proj", + "model.layers.10.mlp.experts.4.down_proj", + "model.layers.10.mlp.experts.4.gate_proj", + "model.layers.10.mlp.experts.4.up_proj", + "model.layers.10.mlp.experts.40.down_proj", + "model.layers.10.mlp.experts.40.gate_proj", + "model.layers.10.mlp.experts.40.up_proj", + "model.layers.10.mlp.experts.41.down_proj", + "model.layers.10.mlp.experts.41.gate_proj", + "model.layers.10.mlp.experts.41.up_proj", + "model.layers.10.mlp.experts.42.down_proj", + "model.layers.10.mlp.experts.42.gate_proj", + "model.layers.10.mlp.experts.42.up_proj", + "model.layers.10.mlp.experts.43.down_proj", + "model.layers.10.mlp.experts.43.gate_proj", + "model.layers.10.mlp.experts.43.up_proj", + "model.layers.10.mlp.experts.44.down_proj", + "model.layers.10.mlp.experts.44.gate_proj", + "model.layers.10.mlp.experts.44.up_proj", + "model.layers.10.mlp.experts.45.down_proj", + "model.layers.10.mlp.experts.45.gate_proj", + "model.layers.10.mlp.experts.45.up_proj", + "model.layers.10.mlp.experts.46.down_proj", + "model.layers.10.mlp.experts.46.gate_proj", + "model.layers.10.mlp.experts.46.up_proj", + "model.layers.10.mlp.experts.47.down_proj", + "model.layers.10.mlp.experts.47.gate_proj", + "model.layers.10.mlp.experts.47.up_proj", + "model.layers.10.mlp.experts.48.down_proj", + "model.layers.10.mlp.experts.48.gate_proj", + "model.layers.10.mlp.experts.48.up_proj", + "model.layers.10.mlp.experts.49.down_proj", + "model.layers.10.mlp.experts.49.gate_proj", + "model.layers.10.mlp.experts.49.up_proj", + "model.layers.10.mlp.experts.5.down_proj", + "model.layers.10.mlp.experts.5.gate_proj", + "model.layers.10.mlp.experts.5.up_proj", + "model.layers.10.mlp.experts.50.down_proj", + "model.layers.10.mlp.experts.50.gate_proj", + "model.layers.10.mlp.experts.50.up_proj", + "model.layers.10.mlp.experts.51.down_proj", + "model.layers.10.mlp.experts.51.gate_proj", + "model.layers.10.mlp.experts.51.up_proj", + "model.layers.10.mlp.experts.52.down_proj", + "model.layers.10.mlp.experts.52.gate_proj", + "model.layers.10.mlp.experts.52.up_proj", + "model.layers.10.mlp.experts.53.down_proj", + "model.layers.10.mlp.experts.53.gate_proj", + "model.layers.10.mlp.experts.53.up_proj", + "model.layers.10.mlp.experts.54.down_proj", + "model.layers.10.mlp.experts.54.gate_proj", + "model.layers.10.mlp.experts.54.up_proj", + "model.layers.10.mlp.experts.55.down_proj", + "model.layers.10.mlp.experts.55.gate_proj", + "model.layers.10.mlp.experts.55.up_proj", + "model.layers.10.mlp.experts.56.down_proj", + "model.layers.10.mlp.experts.56.gate_proj", + "model.layers.10.mlp.experts.56.up_proj", + "model.layers.10.mlp.experts.57.down_proj", + "model.layers.10.mlp.experts.57.gate_proj", + "model.layers.10.mlp.experts.57.up_proj", + "model.layers.10.mlp.experts.58.down_proj", + "model.layers.10.mlp.experts.58.gate_proj", + "model.layers.10.mlp.experts.58.up_proj", + "model.layers.10.mlp.experts.59.down_proj", + "model.layers.10.mlp.experts.59.gate_proj", + "model.layers.10.mlp.experts.59.up_proj", + "model.layers.10.mlp.experts.6.down_proj", + "model.layers.10.mlp.experts.6.gate_proj", + "model.layers.10.mlp.experts.6.up_proj", + "model.layers.10.mlp.experts.60.down_proj", + "model.layers.10.mlp.experts.60.gate_proj", + "model.layers.10.mlp.experts.60.up_proj", + "model.layers.10.mlp.experts.61.down_proj", + "model.layers.10.mlp.experts.61.gate_proj", + "model.layers.10.mlp.experts.61.up_proj", + "model.layers.10.mlp.experts.62.down_proj", + "model.layers.10.mlp.experts.62.gate_proj", + "model.layers.10.mlp.experts.62.up_proj", + "model.layers.10.mlp.experts.63.down_proj", + "model.layers.10.mlp.experts.63.gate_proj", + "model.layers.10.mlp.experts.63.up_proj", + "model.layers.10.mlp.experts.64.down_proj", + "model.layers.10.mlp.experts.64.gate_proj", + "model.layers.10.mlp.experts.64.up_proj", + "model.layers.10.mlp.experts.65.down_proj", + "model.layers.10.mlp.experts.65.gate_proj", + "model.layers.10.mlp.experts.65.up_proj", + "model.layers.10.mlp.experts.66.down_proj", + "model.layers.10.mlp.experts.66.gate_proj", + "model.layers.10.mlp.experts.66.up_proj", + "model.layers.10.mlp.experts.67.down_proj", + "model.layers.10.mlp.experts.67.gate_proj", + "model.layers.10.mlp.experts.67.up_proj", + "model.layers.10.mlp.experts.68.down_proj", + "model.layers.10.mlp.experts.68.gate_proj", + "model.layers.10.mlp.experts.68.up_proj", + "model.layers.10.mlp.experts.69.down_proj", + "model.layers.10.mlp.experts.69.gate_proj", + "model.layers.10.mlp.experts.69.up_proj", + "model.layers.10.mlp.experts.7.down_proj", + "model.layers.10.mlp.experts.7.gate_proj", + "model.layers.10.mlp.experts.7.up_proj", + "model.layers.10.mlp.experts.70.down_proj", + "model.layers.10.mlp.experts.70.gate_proj", + "model.layers.10.mlp.experts.70.up_proj", + "model.layers.10.mlp.experts.71.down_proj", + "model.layers.10.mlp.experts.71.gate_proj", + "model.layers.10.mlp.experts.71.up_proj", + "model.layers.10.mlp.experts.72.down_proj", + "model.layers.10.mlp.experts.72.gate_proj", + "model.layers.10.mlp.experts.72.up_proj", + "model.layers.10.mlp.experts.73.down_proj", + "model.layers.10.mlp.experts.73.gate_proj", + "model.layers.10.mlp.experts.73.up_proj", + "model.layers.10.mlp.experts.74.down_proj", + "model.layers.10.mlp.experts.74.gate_proj", + "model.layers.10.mlp.experts.74.up_proj", + "model.layers.10.mlp.experts.75.down_proj", + "model.layers.10.mlp.experts.75.gate_proj", + "model.layers.10.mlp.experts.75.up_proj", + "model.layers.10.mlp.experts.76.down_proj", + "model.layers.10.mlp.experts.76.gate_proj", + "model.layers.10.mlp.experts.76.up_proj", + "model.layers.10.mlp.experts.77.down_proj", + "model.layers.10.mlp.experts.77.gate_proj", + "model.layers.10.mlp.experts.77.up_proj", + "model.layers.10.mlp.experts.78.down_proj", + "model.layers.10.mlp.experts.78.gate_proj", + "model.layers.10.mlp.experts.78.up_proj", + "model.layers.10.mlp.experts.79.down_proj", + "model.layers.10.mlp.experts.79.gate_proj", + "model.layers.10.mlp.experts.79.up_proj", + "model.layers.10.mlp.experts.8.down_proj", + "model.layers.10.mlp.experts.8.gate_proj", + "model.layers.10.mlp.experts.8.up_proj", + "model.layers.10.mlp.experts.80.down_proj", + "model.layers.10.mlp.experts.80.gate_proj", + "model.layers.10.mlp.experts.80.up_proj", + "model.layers.10.mlp.experts.81.down_proj", + "model.layers.10.mlp.experts.81.gate_proj", + "model.layers.10.mlp.experts.81.up_proj", + "model.layers.10.mlp.experts.82.down_proj", + "model.layers.10.mlp.experts.82.gate_proj", + "model.layers.10.mlp.experts.82.up_proj", + "model.layers.10.mlp.experts.83.down_proj", + "model.layers.10.mlp.experts.83.gate_proj", + "model.layers.10.mlp.experts.83.up_proj", + "model.layers.10.mlp.experts.84.down_proj", + "model.layers.10.mlp.experts.84.gate_proj", + "model.layers.10.mlp.experts.84.up_proj", + "model.layers.10.mlp.experts.85.down_proj", + "model.layers.10.mlp.experts.85.gate_proj", + "model.layers.10.mlp.experts.85.up_proj", + "model.layers.10.mlp.experts.86.down_proj", + "model.layers.10.mlp.experts.86.gate_proj", + "model.layers.10.mlp.experts.86.up_proj", + "model.layers.10.mlp.experts.87.down_proj", + "model.layers.10.mlp.experts.87.gate_proj", + "model.layers.10.mlp.experts.87.up_proj", + "model.layers.10.mlp.experts.88.down_proj", + "model.layers.10.mlp.experts.88.gate_proj", + "model.layers.10.mlp.experts.88.up_proj", + "model.layers.10.mlp.experts.89.down_proj", + "model.layers.10.mlp.experts.89.gate_proj", + "model.layers.10.mlp.experts.89.up_proj", + "model.layers.10.mlp.experts.9.down_proj", + "model.layers.10.mlp.experts.9.gate_proj", + "model.layers.10.mlp.experts.9.up_proj", + "model.layers.10.mlp.experts.90.down_proj", + "model.layers.10.mlp.experts.90.gate_proj", + "model.layers.10.mlp.experts.90.up_proj", + "model.layers.10.mlp.experts.91.down_proj", + "model.layers.10.mlp.experts.91.gate_proj", + "model.layers.10.mlp.experts.91.up_proj", + "model.layers.10.mlp.experts.92.down_proj", + "model.layers.10.mlp.experts.92.gate_proj", + "model.layers.10.mlp.experts.92.up_proj", + "model.layers.10.mlp.experts.93.down_proj", + "model.layers.10.mlp.experts.93.gate_proj", + "model.layers.10.mlp.experts.93.up_proj", + "model.layers.10.mlp.experts.94.down_proj", + "model.layers.10.mlp.experts.94.gate_proj", + "model.layers.10.mlp.experts.94.up_proj", + "model.layers.10.mlp.experts.95.down_proj", + "model.layers.10.mlp.experts.95.gate_proj", + "model.layers.10.mlp.experts.95.up_proj", + "model.layers.10.mlp.experts.96.down_proj", + "model.layers.10.mlp.experts.96.gate_proj", + "model.layers.10.mlp.experts.96.up_proj", + "model.layers.10.mlp.experts.97.down_proj", + "model.layers.10.mlp.experts.97.gate_proj", + "model.layers.10.mlp.experts.97.up_proj", + "model.layers.10.mlp.experts.98.down_proj", + "model.layers.10.mlp.experts.98.gate_proj", + "model.layers.10.mlp.experts.98.up_proj", + "model.layers.10.mlp.experts.99.down_proj", + "model.layers.10.mlp.experts.99.gate_proj", + "model.layers.10.mlp.experts.99.up_proj", + "model.layers.11.mlp.experts.0.down_proj", + "model.layers.11.mlp.experts.0.gate_proj", + "model.layers.11.mlp.experts.0.up_proj", + "model.layers.11.mlp.experts.1.down_proj", + "model.layers.11.mlp.experts.1.gate_proj", + "model.layers.11.mlp.experts.1.up_proj", + "model.layers.11.mlp.experts.10.down_proj", + "model.layers.11.mlp.experts.10.gate_proj", + "model.layers.11.mlp.experts.10.up_proj", + "model.layers.11.mlp.experts.100.down_proj", + "model.layers.11.mlp.experts.100.gate_proj", + "model.layers.11.mlp.experts.100.up_proj", + "model.layers.11.mlp.experts.101.down_proj", + "model.layers.11.mlp.experts.101.gate_proj", + "model.layers.11.mlp.experts.101.up_proj", + "model.layers.11.mlp.experts.102.down_proj", + "model.layers.11.mlp.experts.102.gate_proj", + "model.layers.11.mlp.experts.102.up_proj", + "model.layers.11.mlp.experts.103.down_proj", + "model.layers.11.mlp.experts.103.gate_proj", + "model.layers.11.mlp.experts.103.up_proj", + "model.layers.11.mlp.experts.104.down_proj", + "model.layers.11.mlp.experts.104.gate_proj", + "model.layers.11.mlp.experts.104.up_proj", + "model.layers.11.mlp.experts.105.down_proj", + "model.layers.11.mlp.experts.105.gate_proj", + "model.layers.11.mlp.experts.105.up_proj", + "model.layers.11.mlp.experts.106.down_proj", + "model.layers.11.mlp.experts.106.gate_proj", + "model.layers.11.mlp.experts.106.up_proj", + "model.layers.11.mlp.experts.107.down_proj", + "model.layers.11.mlp.experts.107.gate_proj", + "model.layers.11.mlp.experts.107.up_proj", + "model.layers.11.mlp.experts.108.down_proj", + "model.layers.11.mlp.experts.108.gate_proj", + "model.layers.11.mlp.experts.108.up_proj", + "model.layers.11.mlp.experts.109.down_proj", + "model.layers.11.mlp.experts.109.gate_proj", + "model.layers.11.mlp.experts.109.up_proj", + "model.layers.11.mlp.experts.11.down_proj", + "model.layers.11.mlp.experts.11.gate_proj", + "model.layers.11.mlp.experts.11.up_proj", + "model.layers.11.mlp.experts.110.down_proj", + "model.layers.11.mlp.experts.110.gate_proj", + "model.layers.11.mlp.experts.110.up_proj", + "model.layers.11.mlp.experts.111.down_proj", + "model.layers.11.mlp.experts.111.gate_proj", + "model.layers.11.mlp.experts.111.up_proj", + "model.layers.11.mlp.experts.112.down_proj", + "model.layers.11.mlp.experts.112.gate_proj", + "model.layers.11.mlp.experts.112.up_proj", + "model.layers.11.mlp.experts.113.down_proj", + "model.layers.11.mlp.experts.113.gate_proj", + "model.layers.11.mlp.experts.113.up_proj", + "model.layers.11.mlp.experts.114.down_proj", + "model.layers.11.mlp.experts.114.gate_proj", + "model.layers.11.mlp.experts.114.up_proj", + "model.layers.11.mlp.experts.115.down_proj", + "model.layers.11.mlp.experts.115.gate_proj", + "model.layers.11.mlp.experts.115.up_proj", + "model.layers.11.mlp.experts.116.down_proj", + "model.layers.11.mlp.experts.116.gate_proj", + "model.layers.11.mlp.experts.116.up_proj", + "model.layers.11.mlp.experts.117.down_proj", + "model.layers.11.mlp.experts.117.gate_proj", + "model.layers.11.mlp.experts.117.up_proj", + "model.layers.11.mlp.experts.118.down_proj", + "model.layers.11.mlp.experts.118.gate_proj", + "model.layers.11.mlp.experts.118.up_proj", + "model.layers.11.mlp.experts.119.down_proj", + "model.layers.11.mlp.experts.119.gate_proj", + "model.layers.11.mlp.experts.119.up_proj", + "model.layers.11.mlp.experts.12.down_proj", + "model.layers.11.mlp.experts.12.gate_proj", + "model.layers.11.mlp.experts.12.up_proj", + "model.layers.11.mlp.experts.120.down_proj", + "model.layers.11.mlp.experts.120.gate_proj", + "model.layers.11.mlp.experts.120.up_proj", + "model.layers.11.mlp.experts.121.down_proj", + "model.layers.11.mlp.experts.121.gate_proj", + "model.layers.11.mlp.experts.121.up_proj", + "model.layers.11.mlp.experts.122.down_proj", + "model.layers.11.mlp.experts.122.gate_proj", + "model.layers.11.mlp.experts.122.up_proj", + "model.layers.11.mlp.experts.123.down_proj", + "model.layers.11.mlp.experts.123.gate_proj", + "model.layers.11.mlp.experts.123.up_proj", + "model.layers.11.mlp.experts.124.down_proj", + "model.layers.11.mlp.experts.124.gate_proj", + "model.layers.11.mlp.experts.124.up_proj", + "model.layers.11.mlp.experts.125.down_proj", + "model.layers.11.mlp.experts.125.gate_proj", + "model.layers.11.mlp.experts.125.up_proj", + "model.layers.11.mlp.experts.126.down_proj", + "model.layers.11.mlp.experts.126.gate_proj", + "model.layers.11.mlp.experts.126.up_proj", + "model.layers.11.mlp.experts.127.down_proj", + "model.layers.11.mlp.experts.127.gate_proj", + "model.layers.11.mlp.experts.127.up_proj", + "model.layers.11.mlp.experts.128.down_proj", + "model.layers.11.mlp.experts.128.gate_proj", + "model.layers.11.mlp.experts.128.up_proj", + "model.layers.11.mlp.experts.129.down_proj", + "model.layers.11.mlp.experts.129.gate_proj", + "model.layers.11.mlp.experts.129.up_proj", + "model.layers.11.mlp.experts.13.down_proj", + "model.layers.11.mlp.experts.13.gate_proj", + "model.layers.11.mlp.experts.13.up_proj", + "model.layers.11.mlp.experts.130.down_proj", + "model.layers.11.mlp.experts.130.gate_proj", + "model.layers.11.mlp.experts.130.up_proj", + "model.layers.11.mlp.experts.131.down_proj", + "model.layers.11.mlp.experts.131.gate_proj", + "model.layers.11.mlp.experts.131.up_proj", + "model.layers.11.mlp.experts.132.down_proj", + "model.layers.11.mlp.experts.132.gate_proj", + "model.layers.11.mlp.experts.132.up_proj", + "model.layers.11.mlp.experts.133.down_proj", + "model.layers.11.mlp.experts.133.gate_proj", + "model.layers.11.mlp.experts.133.up_proj", + "model.layers.11.mlp.experts.134.down_proj", + "model.layers.11.mlp.experts.134.gate_proj", + "model.layers.11.mlp.experts.134.up_proj", + "model.layers.11.mlp.experts.135.down_proj", + "model.layers.11.mlp.experts.135.gate_proj", + "model.layers.11.mlp.experts.135.up_proj", + "model.layers.11.mlp.experts.136.down_proj", + "model.layers.11.mlp.experts.136.gate_proj", + "model.layers.11.mlp.experts.136.up_proj", + "model.layers.11.mlp.experts.137.down_proj", + "model.layers.11.mlp.experts.137.gate_proj", + "model.layers.11.mlp.experts.137.up_proj", + "model.layers.11.mlp.experts.138.down_proj", + "model.layers.11.mlp.experts.138.gate_proj", + "model.layers.11.mlp.experts.138.up_proj", + "model.layers.11.mlp.experts.139.down_proj", + "model.layers.11.mlp.experts.139.gate_proj", + "model.layers.11.mlp.experts.139.up_proj", + "model.layers.11.mlp.experts.14.down_proj", + "model.layers.11.mlp.experts.14.gate_proj", + "model.layers.11.mlp.experts.14.up_proj", + "model.layers.11.mlp.experts.140.down_proj", + "model.layers.11.mlp.experts.140.gate_proj", + "model.layers.11.mlp.experts.140.up_proj", + "model.layers.11.mlp.experts.141.down_proj", + "model.layers.11.mlp.experts.141.gate_proj", + "model.layers.11.mlp.experts.141.up_proj", + "model.layers.11.mlp.experts.142.down_proj", + "model.layers.11.mlp.experts.142.gate_proj", + "model.layers.11.mlp.experts.142.up_proj", + "model.layers.11.mlp.experts.143.down_proj", + "model.layers.11.mlp.experts.143.gate_proj", + "model.layers.11.mlp.experts.143.up_proj", + "model.layers.11.mlp.experts.144.down_proj", + "model.layers.11.mlp.experts.144.gate_proj", + "model.layers.11.mlp.experts.144.up_proj", + "model.layers.11.mlp.experts.145.down_proj", + "model.layers.11.mlp.experts.145.gate_proj", + "model.layers.11.mlp.experts.145.up_proj", + "model.layers.11.mlp.experts.146.down_proj", + "model.layers.11.mlp.experts.146.gate_proj", + "model.layers.11.mlp.experts.146.up_proj", + "model.layers.11.mlp.experts.147.down_proj", + "model.layers.11.mlp.experts.147.gate_proj", + "model.layers.11.mlp.experts.147.up_proj", + "model.layers.11.mlp.experts.148.down_proj", + "model.layers.11.mlp.experts.148.gate_proj", + "model.layers.11.mlp.experts.148.up_proj", + "model.layers.11.mlp.experts.149.down_proj", + "model.layers.11.mlp.experts.149.gate_proj", + "model.layers.11.mlp.experts.149.up_proj", + "model.layers.11.mlp.experts.15.down_proj", + "model.layers.11.mlp.experts.15.gate_proj", + "model.layers.11.mlp.experts.15.up_proj", + "model.layers.11.mlp.experts.150.down_proj", + "model.layers.11.mlp.experts.150.gate_proj", + "model.layers.11.mlp.experts.150.up_proj", + "model.layers.11.mlp.experts.151.down_proj", + "model.layers.11.mlp.experts.151.gate_proj", + "model.layers.11.mlp.experts.151.up_proj", + "model.layers.11.mlp.experts.152.down_proj", + "model.layers.11.mlp.experts.152.gate_proj", + "model.layers.11.mlp.experts.152.up_proj", + "model.layers.11.mlp.experts.153.down_proj", + "model.layers.11.mlp.experts.153.gate_proj", + "model.layers.11.mlp.experts.153.up_proj", + "model.layers.11.mlp.experts.154.down_proj", + "model.layers.11.mlp.experts.154.gate_proj", + "model.layers.11.mlp.experts.154.up_proj", + "model.layers.11.mlp.experts.155.down_proj", + "model.layers.11.mlp.experts.155.gate_proj", + "model.layers.11.mlp.experts.155.up_proj", + "model.layers.11.mlp.experts.156.down_proj", + "model.layers.11.mlp.experts.156.gate_proj", + "model.layers.11.mlp.experts.156.up_proj", + "model.layers.11.mlp.experts.157.down_proj", + "model.layers.11.mlp.experts.157.gate_proj", + "model.layers.11.mlp.experts.157.up_proj", + "model.layers.11.mlp.experts.158.down_proj", + "model.layers.11.mlp.experts.158.gate_proj", + "model.layers.11.mlp.experts.158.up_proj", + "model.layers.11.mlp.experts.159.down_proj", + "model.layers.11.mlp.experts.159.gate_proj", + "model.layers.11.mlp.experts.159.up_proj", + "model.layers.11.mlp.experts.16.down_proj", + "model.layers.11.mlp.experts.16.gate_proj", + "model.layers.11.mlp.experts.16.up_proj", + "model.layers.11.mlp.experts.160.down_proj", + "model.layers.11.mlp.experts.160.gate_proj", + "model.layers.11.mlp.experts.160.up_proj", + "model.layers.11.mlp.experts.161.down_proj", + "model.layers.11.mlp.experts.161.gate_proj", + "model.layers.11.mlp.experts.161.up_proj", + "model.layers.11.mlp.experts.162.down_proj", + "model.layers.11.mlp.experts.162.gate_proj", + "model.layers.11.mlp.experts.162.up_proj", + "model.layers.11.mlp.experts.163.down_proj", + "model.layers.11.mlp.experts.163.gate_proj", + "model.layers.11.mlp.experts.163.up_proj", + "model.layers.11.mlp.experts.164.down_proj", + "model.layers.11.mlp.experts.164.gate_proj", + "model.layers.11.mlp.experts.164.up_proj", + "model.layers.11.mlp.experts.165.down_proj", + "model.layers.11.mlp.experts.165.gate_proj", + "model.layers.11.mlp.experts.165.up_proj", + "model.layers.11.mlp.experts.166.down_proj", + "model.layers.11.mlp.experts.166.gate_proj", + "model.layers.11.mlp.experts.166.up_proj", + "model.layers.11.mlp.experts.167.down_proj", + "model.layers.11.mlp.experts.167.gate_proj", + "model.layers.11.mlp.experts.167.up_proj", + "model.layers.11.mlp.experts.168.down_proj", + "model.layers.11.mlp.experts.168.gate_proj", + "model.layers.11.mlp.experts.168.up_proj", + "model.layers.11.mlp.experts.169.down_proj", + "model.layers.11.mlp.experts.169.gate_proj", + "model.layers.11.mlp.experts.169.up_proj", + "model.layers.11.mlp.experts.17.down_proj", + "model.layers.11.mlp.experts.17.gate_proj", + "model.layers.11.mlp.experts.17.up_proj", + "model.layers.11.mlp.experts.170.down_proj", + "model.layers.11.mlp.experts.170.gate_proj", + "model.layers.11.mlp.experts.170.up_proj", + "model.layers.11.mlp.experts.171.down_proj", + "model.layers.11.mlp.experts.171.gate_proj", + "model.layers.11.mlp.experts.171.up_proj", + "model.layers.11.mlp.experts.172.down_proj", + "model.layers.11.mlp.experts.172.gate_proj", + "model.layers.11.mlp.experts.172.up_proj", + "model.layers.11.mlp.experts.173.down_proj", + "model.layers.11.mlp.experts.173.gate_proj", + "model.layers.11.mlp.experts.173.up_proj", + "model.layers.11.mlp.experts.174.down_proj", + "model.layers.11.mlp.experts.174.gate_proj", + "model.layers.11.mlp.experts.174.up_proj", + "model.layers.11.mlp.experts.175.down_proj", + "model.layers.11.mlp.experts.175.gate_proj", + "model.layers.11.mlp.experts.175.up_proj", + "model.layers.11.mlp.experts.176.down_proj", + "model.layers.11.mlp.experts.176.gate_proj", + "model.layers.11.mlp.experts.176.up_proj", + "model.layers.11.mlp.experts.177.down_proj", + "model.layers.11.mlp.experts.177.gate_proj", + "model.layers.11.mlp.experts.177.up_proj", + "model.layers.11.mlp.experts.178.down_proj", + "model.layers.11.mlp.experts.178.gate_proj", + "model.layers.11.mlp.experts.178.up_proj", + "model.layers.11.mlp.experts.179.down_proj", + "model.layers.11.mlp.experts.179.gate_proj", + "model.layers.11.mlp.experts.179.up_proj", + "model.layers.11.mlp.experts.18.down_proj", + "model.layers.11.mlp.experts.18.gate_proj", + "model.layers.11.mlp.experts.18.up_proj", + "model.layers.11.mlp.experts.180.down_proj", + "model.layers.11.mlp.experts.180.gate_proj", + "model.layers.11.mlp.experts.180.up_proj", + "model.layers.11.mlp.experts.181.down_proj", + "model.layers.11.mlp.experts.181.gate_proj", + "model.layers.11.mlp.experts.181.up_proj", + "model.layers.11.mlp.experts.182.down_proj", + "model.layers.11.mlp.experts.182.gate_proj", + "model.layers.11.mlp.experts.182.up_proj", + "model.layers.11.mlp.experts.183.down_proj", + "model.layers.11.mlp.experts.183.gate_proj", + "model.layers.11.mlp.experts.183.up_proj", + "model.layers.11.mlp.experts.184.down_proj", + "model.layers.11.mlp.experts.184.gate_proj", + "model.layers.11.mlp.experts.184.up_proj", + "model.layers.11.mlp.experts.185.down_proj", + "model.layers.11.mlp.experts.185.gate_proj", + "model.layers.11.mlp.experts.185.up_proj", + "model.layers.11.mlp.experts.186.down_proj", + "model.layers.11.mlp.experts.186.gate_proj", + "model.layers.11.mlp.experts.186.up_proj", + "model.layers.11.mlp.experts.187.down_proj", + "model.layers.11.mlp.experts.187.gate_proj", + "model.layers.11.mlp.experts.187.up_proj", + "model.layers.11.mlp.experts.188.down_proj", + "model.layers.11.mlp.experts.188.gate_proj", + "model.layers.11.mlp.experts.188.up_proj", + "model.layers.11.mlp.experts.189.down_proj", + "model.layers.11.mlp.experts.189.gate_proj", + "model.layers.11.mlp.experts.189.up_proj", + "model.layers.11.mlp.experts.19.down_proj", + "model.layers.11.mlp.experts.19.gate_proj", + "model.layers.11.mlp.experts.19.up_proj", + "model.layers.11.mlp.experts.190.down_proj", + "model.layers.11.mlp.experts.190.gate_proj", + "model.layers.11.mlp.experts.190.up_proj", + "model.layers.11.mlp.experts.191.down_proj", + "model.layers.11.mlp.experts.191.gate_proj", + "model.layers.11.mlp.experts.191.up_proj", + "model.layers.11.mlp.experts.192.down_proj", + "model.layers.11.mlp.experts.192.gate_proj", + "model.layers.11.mlp.experts.192.up_proj", + "model.layers.11.mlp.experts.193.down_proj", + "model.layers.11.mlp.experts.193.gate_proj", + "model.layers.11.mlp.experts.193.up_proj", + "model.layers.11.mlp.experts.194.down_proj", + "model.layers.11.mlp.experts.194.gate_proj", + "model.layers.11.mlp.experts.194.up_proj", + "model.layers.11.mlp.experts.195.down_proj", + "model.layers.11.mlp.experts.195.gate_proj", + "model.layers.11.mlp.experts.195.up_proj", + "model.layers.11.mlp.experts.196.down_proj", + "model.layers.11.mlp.experts.196.gate_proj", + "model.layers.11.mlp.experts.196.up_proj", + "model.layers.11.mlp.experts.197.down_proj", + "model.layers.11.mlp.experts.197.gate_proj", + "model.layers.11.mlp.experts.197.up_proj", + "model.layers.11.mlp.experts.198.down_proj", + "model.layers.11.mlp.experts.198.gate_proj", + "model.layers.11.mlp.experts.198.up_proj", + "model.layers.11.mlp.experts.199.down_proj", + "model.layers.11.mlp.experts.199.gate_proj", + "model.layers.11.mlp.experts.199.up_proj", + "model.layers.11.mlp.experts.2.down_proj", + "model.layers.11.mlp.experts.2.gate_proj", + "model.layers.11.mlp.experts.2.up_proj", + "model.layers.11.mlp.experts.20.down_proj", + "model.layers.11.mlp.experts.20.gate_proj", + "model.layers.11.mlp.experts.20.up_proj", + "model.layers.11.mlp.experts.200.down_proj", + "model.layers.11.mlp.experts.200.gate_proj", + "model.layers.11.mlp.experts.200.up_proj", + "model.layers.11.mlp.experts.201.down_proj", + "model.layers.11.mlp.experts.201.gate_proj", + "model.layers.11.mlp.experts.201.up_proj", + "model.layers.11.mlp.experts.202.down_proj", + "model.layers.11.mlp.experts.202.gate_proj", + "model.layers.11.mlp.experts.202.up_proj", + "model.layers.11.mlp.experts.203.down_proj", + "model.layers.11.mlp.experts.203.gate_proj", + "model.layers.11.mlp.experts.203.up_proj", + "model.layers.11.mlp.experts.204.down_proj", + "model.layers.11.mlp.experts.204.gate_proj", + "model.layers.11.mlp.experts.204.up_proj", + "model.layers.11.mlp.experts.205.down_proj", + "model.layers.11.mlp.experts.205.gate_proj", + "model.layers.11.mlp.experts.205.up_proj", + "model.layers.11.mlp.experts.206.down_proj", + "model.layers.11.mlp.experts.206.gate_proj", + "model.layers.11.mlp.experts.206.up_proj", + "model.layers.11.mlp.experts.207.down_proj", + "model.layers.11.mlp.experts.207.gate_proj", + "model.layers.11.mlp.experts.207.up_proj", + "model.layers.11.mlp.experts.208.down_proj", + "model.layers.11.mlp.experts.208.gate_proj", + "model.layers.11.mlp.experts.208.up_proj", + "model.layers.11.mlp.experts.209.down_proj", + "model.layers.11.mlp.experts.209.gate_proj", + "model.layers.11.mlp.experts.209.up_proj", + "model.layers.11.mlp.experts.21.down_proj", + "model.layers.11.mlp.experts.21.gate_proj", + "model.layers.11.mlp.experts.21.up_proj", + "model.layers.11.mlp.experts.210.down_proj", + "model.layers.11.mlp.experts.210.gate_proj", + "model.layers.11.mlp.experts.210.up_proj", + "model.layers.11.mlp.experts.211.down_proj", + "model.layers.11.mlp.experts.211.gate_proj", + "model.layers.11.mlp.experts.211.up_proj", + "model.layers.11.mlp.experts.212.down_proj", + "model.layers.11.mlp.experts.212.gate_proj", + "model.layers.11.mlp.experts.212.up_proj", + "model.layers.11.mlp.experts.213.down_proj", + "model.layers.11.mlp.experts.213.gate_proj", + "model.layers.11.mlp.experts.213.up_proj", + "model.layers.11.mlp.experts.214.down_proj", + "model.layers.11.mlp.experts.214.gate_proj", + "model.layers.11.mlp.experts.214.up_proj", + "model.layers.11.mlp.experts.215.down_proj", + "model.layers.11.mlp.experts.215.gate_proj", + "model.layers.11.mlp.experts.215.up_proj", + "model.layers.11.mlp.experts.216.down_proj", + "model.layers.11.mlp.experts.216.gate_proj", + "model.layers.11.mlp.experts.216.up_proj", + "model.layers.11.mlp.experts.217.down_proj", + "model.layers.11.mlp.experts.217.gate_proj", + "model.layers.11.mlp.experts.217.up_proj", + "model.layers.11.mlp.experts.218.down_proj", + "model.layers.11.mlp.experts.218.gate_proj", + "model.layers.11.mlp.experts.218.up_proj", + "model.layers.11.mlp.experts.219.down_proj", + "model.layers.11.mlp.experts.219.gate_proj", + "model.layers.11.mlp.experts.219.up_proj", + "model.layers.11.mlp.experts.22.down_proj", + "model.layers.11.mlp.experts.22.gate_proj", + "model.layers.11.mlp.experts.22.up_proj", + "model.layers.11.mlp.experts.220.down_proj", + "model.layers.11.mlp.experts.220.gate_proj", + "model.layers.11.mlp.experts.220.up_proj", + "model.layers.11.mlp.experts.221.down_proj", + "model.layers.11.mlp.experts.221.gate_proj", + "model.layers.11.mlp.experts.221.up_proj", + "model.layers.11.mlp.experts.222.down_proj", + "model.layers.11.mlp.experts.222.gate_proj", + "model.layers.11.mlp.experts.222.up_proj", + "model.layers.11.mlp.experts.223.down_proj", + "model.layers.11.mlp.experts.223.gate_proj", + "model.layers.11.mlp.experts.223.up_proj", + "model.layers.11.mlp.experts.224.down_proj", + "model.layers.11.mlp.experts.224.gate_proj", + "model.layers.11.mlp.experts.224.up_proj", + "model.layers.11.mlp.experts.225.down_proj", + "model.layers.11.mlp.experts.225.gate_proj", + "model.layers.11.mlp.experts.225.up_proj", + "model.layers.11.mlp.experts.226.down_proj", + "model.layers.11.mlp.experts.226.gate_proj", + "model.layers.11.mlp.experts.226.up_proj", + "model.layers.11.mlp.experts.227.down_proj", + "model.layers.11.mlp.experts.227.gate_proj", + "model.layers.11.mlp.experts.227.up_proj", + "model.layers.11.mlp.experts.228.down_proj", + "model.layers.11.mlp.experts.228.gate_proj", + "model.layers.11.mlp.experts.228.up_proj", + "model.layers.11.mlp.experts.229.down_proj", + "model.layers.11.mlp.experts.229.gate_proj", + "model.layers.11.mlp.experts.229.up_proj", + "model.layers.11.mlp.experts.23.down_proj", + "model.layers.11.mlp.experts.23.gate_proj", + "model.layers.11.mlp.experts.23.up_proj", + "model.layers.11.mlp.experts.230.down_proj", + "model.layers.11.mlp.experts.230.gate_proj", + "model.layers.11.mlp.experts.230.up_proj", + "model.layers.11.mlp.experts.231.down_proj", + "model.layers.11.mlp.experts.231.gate_proj", + "model.layers.11.mlp.experts.231.up_proj", + "model.layers.11.mlp.experts.232.down_proj", + "model.layers.11.mlp.experts.232.gate_proj", + "model.layers.11.mlp.experts.232.up_proj", + "model.layers.11.mlp.experts.233.down_proj", + "model.layers.11.mlp.experts.233.gate_proj", + "model.layers.11.mlp.experts.233.up_proj", + "model.layers.11.mlp.experts.234.down_proj", + "model.layers.11.mlp.experts.234.gate_proj", + "model.layers.11.mlp.experts.234.up_proj", + "model.layers.11.mlp.experts.235.down_proj", + "model.layers.11.mlp.experts.235.gate_proj", + "model.layers.11.mlp.experts.235.up_proj", + "model.layers.11.mlp.experts.236.down_proj", + "model.layers.11.mlp.experts.236.gate_proj", + "model.layers.11.mlp.experts.236.up_proj", + "model.layers.11.mlp.experts.237.down_proj", + "model.layers.11.mlp.experts.237.gate_proj", + "model.layers.11.mlp.experts.237.up_proj", + "model.layers.11.mlp.experts.238.down_proj", + "model.layers.11.mlp.experts.238.gate_proj", + "model.layers.11.mlp.experts.238.up_proj", + "model.layers.11.mlp.experts.239.down_proj", + "model.layers.11.mlp.experts.239.gate_proj", + "model.layers.11.mlp.experts.239.up_proj", + "model.layers.11.mlp.experts.24.down_proj", + "model.layers.11.mlp.experts.24.gate_proj", + "model.layers.11.mlp.experts.24.up_proj", + "model.layers.11.mlp.experts.240.down_proj", + "model.layers.11.mlp.experts.240.gate_proj", + "model.layers.11.mlp.experts.240.up_proj", + "model.layers.11.mlp.experts.241.down_proj", + "model.layers.11.mlp.experts.241.gate_proj", + "model.layers.11.mlp.experts.241.up_proj", + "model.layers.11.mlp.experts.242.down_proj", + "model.layers.11.mlp.experts.242.gate_proj", + "model.layers.11.mlp.experts.242.up_proj", + "model.layers.11.mlp.experts.243.down_proj", + "model.layers.11.mlp.experts.243.gate_proj", + "model.layers.11.mlp.experts.243.up_proj", + "model.layers.11.mlp.experts.244.down_proj", + "model.layers.11.mlp.experts.244.gate_proj", + "model.layers.11.mlp.experts.244.up_proj", + "model.layers.11.mlp.experts.245.down_proj", + "model.layers.11.mlp.experts.245.gate_proj", + "model.layers.11.mlp.experts.245.up_proj", + "model.layers.11.mlp.experts.246.down_proj", + "model.layers.11.mlp.experts.246.gate_proj", + "model.layers.11.mlp.experts.246.up_proj", + "model.layers.11.mlp.experts.247.down_proj", + "model.layers.11.mlp.experts.247.gate_proj", + "model.layers.11.mlp.experts.247.up_proj", + "model.layers.11.mlp.experts.248.down_proj", + "model.layers.11.mlp.experts.248.gate_proj", + "model.layers.11.mlp.experts.248.up_proj", + "model.layers.11.mlp.experts.249.down_proj", + "model.layers.11.mlp.experts.249.gate_proj", + "model.layers.11.mlp.experts.249.up_proj", + "model.layers.11.mlp.experts.25.down_proj", + "model.layers.11.mlp.experts.25.gate_proj", + "model.layers.11.mlp.experts.25.up_proj", + "model.layers.11.mlp.experts.250.down_proj", + "model.layers.11.mlp.experts.250.gate_proj", + "model.layers.11.mlp.experts.250.up_proj", + "model.layers.11.mlp.experts.251.down_proj", + "model.layers.11.mlp.experts.251.gate_proj", + "model.layers.11.mlp.experts.251.up_proj", + "model.layers.11.mlp.experts.252.down_proj", + "model.layers.11.mlp.experts.252.gate_proj", + "model.layers.11.mlp.experts.252.up_proj", + "model.layers.11.mlp.experts.253.down_proj", + "model.layers.11.mlp.experts.253.gate_proj", + "model.layers.11.mlp.experts.253.up_proj", + "model.layers.11.mlp.experts.254.down_proj", + "model.layers.11.mlp.experts.254.gate_proj", + "model.layers.11.mlp.experts.254.up_proj", + "model.layers.11.mlp.experts.255.down_proj", + "model.layers.11.mlp.experts.255.gate_proj", + "model.layers.11.mlp.experts.255.up_proj", + "model.layers.11.mlp.experts.26.down_proj", + "model.layers.11.mlp.experts.26.gate_proj", + "model.layers.11.mlp.experts.26.up_proj", + "model.layers.11.mlp.experts.27.down_proj", + "model.layers.11.mlp.experts.27.gate_proj", + "model.layers.11.mlp.experts.27.up_proj", + "model.layers.11.mlp.experts.28.down_proj", + "model.layers.11.mlp.experts.28.gate_proj", + "model.layers.11.mlp.experts.28.up_proj", + "model.layers.11.mlp.experts.29.down_proj", + "model.layers.11.mlp.experts.29.gate_proj", + "model.layers.11.mlp.experts.29.up_proj", + "model.layers.11.mlp.experts.3.down_proj", + "model.layers.11.mlp.experts.3.gate_proj", + "model.layers.11.mlp.experts.3.up_proj", + "model.layers.11.mlp.experts.30.down_proj", + "model.layers.11.mlp.experts.30.gate_proj", + "model.layers.11.mlp.experts.30.up_proj", + "model.layers.11.mlp.experts.31.down_proj", + "model.layers.11.mlp.experts.31.gate_proj", + "model.layers.11.mlp.experts.31.up_proj", + "model.layers.11.mlp.experts.32.down_proj", + "model.layers.11.mlp.experts.32.gate_proj", + "model.layers.11.mlp.experts.32.up_proj", + "model.layers.11.mlp.experts.33.down_proj", + "model.layers.11.mlp.experts.33.gate_proj", + "model.layers.11.mlp.experts.33.up_proj", + "model.layers.11.mlp.experts.34.down_proj", + "model.layers.11.mlp.experts.34.gate_proj", + "model.layers.11.mlp.experts.34.up_proj", + "model.layers.11.mlp.experts.35.down_proj", + "model.layers.11.mlp.experts.35.gate_proj", + "model.layers.11.mlp.experts.35.up_proj", + "model.layers.11.mlp.experts.36.down_proj", + "model.layers.11.mlp.experts.36.gate_proj", + "model.layers.11.mlp.experts.36.up_proj", + "model.layers.11.mlp.experts.37.down_proj", + "model.layers.11.mlp.experts.37.gate_proj", + "model.layers.11.mlp.experts.37.up_proj", + "model.layers.11.mlp.experts.38.down_proj", + "model.layers.11.mlp.experts.38.gate_proj", + "model.layers.11.mlp.experts.38.up_proj", + "model.layers.11.mlp.experts.39.down_proj", + "model.layers.11.mlp.experts.39.gate_proj", + "model.layers.11.mlp.experts.39.up_proj", + "model.layers.11.mlp.experts.4.down_proj", + "model.layers.11.mlp.experts.4.gate_proj", + "model.layers.11.mlp.experts.4.up_proj", + "model.layers.11.mlp.experts.40.down_proj", + "model.layers.11.mlp.experts.40.gate_proj", + "model.layers.11.mlp.experts.40.up_proj", + "model.layers.11.mlp.experts.41.down_proj", + "model.layers.11.mlp.experts.41.gate_proj", + "model.layers.11.mlp.experts.41.up_proj", + "model.layers.11.mlp.experts.42.down_proj", + "model.layers.11.mlp.experts.42.gate_proj", + "model.layers.11.mlp.experts.42.up_proj", + "model.layers.11.mlp.experts.43.down_proj", + "model.layers.11.mlp.experts.43.gate_proj", + "model.layers.11.mlp.experts.43.up_proj", + "model.layers.11.mlp.experts.44.down_proj", + "model.layers.11.mlp.experts.44.gate_proj", + "model.layers.11.mlp.experts.44.up_proj", + "model.layers.11.mlp.experts.45.down_proj", + "model.layers.11.mlp.experts.45.gate_proj", + "model.layers.11.mlp.experts.45.up_proj", + "model.layers.11.mlp.experts.46.down_proj", + "model.layers.11.mlp.experts.46.gate_proj", + "model.layers.11.mlp.experts.46.up_proj", + "model.layers.11.mlp.experts.47.down_proj", + "model.layers.11.mlp.experts.47.gate_proj", + "model.layers.11.mlp.experts.47.up_proj", + "model.layers.11.mlp.experts.48.down_proj", + "model.layers.11.mlp.experts.48.gate_proj", + "model.layers.11.mlp.experts.48.up_proj", + "model.layers.11.mlp.experts.49.down_proj", + "model.layers.11.mlp.experts.49.gate_proj", + "model.layers.11.mlp.experts.49.up_proj", + "model.layers.11.mlp.experts.5.down_proj", + "model.layers.11.mlp.experts.5.gate_proj", + "model.layers.11.mlp.experts.5.up_proj", + "model.layers.11.mlp.experts.50.down_proj", + "model.layers.11.mlp.experts.50.gate_proj", + "model.layers.11.mlp.experts.50.up_proj", + "model.layers.11.mlp.experts.51.down_proj", + "model.layers.11.mlp.experts.51.gate_proj", + "model.layers.11.mlp.experts.51.up_proj", + "model.layers.11.mlp.experts.52.down_proj", + "model.layers.11.mlp.experts.52.gate_proj", + "model.layers.11.mlp.experts.52.up_proj", + "model.layers.11.mlp.experts.53.down_proj", + "model.layers.11.mlp.experts.53.gate_proj", + "model.layers.11.mlp.experts.53.up_proj", + "model.layers.11.mlp.experts.54.down_proj", + "model.layers.11.mlp.experts.54.gate_proj", + "model.layers.11.mlp.experts.54.up_proj", + "model.layers.11.mlp.experts.55.down_proj", + "model.layers.11.mlp.experts.55.gate_proj", + "model.layers.11.mlp.experts.55.up_proj", + "model.layers.11.mlp.experts.56.down_proj", + "model.layers.11.mlp.experts.56.gate_proj", + "model.layers.11.mlp.experts.56.up_proj", + "model.layers.11.mlp.experts.57.down_proj", + "model.layers.11.mlp.experts.57.gate_proj", + "model.layers.11.mlp.experts.57.up_proj", + "model.layers.11.mlp.experts.58.down_proj", + "model.layers.11.mlp.experts.58.gate_proj", + "model.layers.11.mlp.experts.58.up_proj", + "model.layers.11.mlp.experts.59.down_proj", + "model.layers.11.mlp.experts.59.gate_proj", + "model.layers.11.mlp.experts.59.up_proj", + "model.layers.11.mlp.experts.6.down_proj", + "model.layers.11.mlp.experts.6.gate_proj", + "model.layers.11.mlp.experts.6.up_proj", + "model.layers.11.mlp.experts.60.down_proj", + "model.layers.11.mlp.experts.60.gate_proj", + "model.layers.11.mlp.experts.60.up_proj", + "model.layers.11.mlp.experts.61.down_proj", + "model.layers.11.mlp.experts.61.gate_proj", + "model.layers.11.mlp.experts.61.up_proj", + "model.layers.11.mlp.experts.62.down_proj", + "model.layers.11.mlp.experts.62.gate_proj", + "model.layers.11.mlp.experts.62.up_proj", + "model.layers.11.mlp.experts.63.down_proj", + "model.layers.11.mlp.experts.63.gate_proj", + "model.layers.11.mlp.experts.63.up_proj", + "model.layers.11.mlp.experts.64.down_proj", + "model.layers.11.mlp.experts.64.gate_proj", + "model.layers.11.mlp.experts.64.up_proj", + "model.layers.11.mlp.experts.65.down_proj", + "model.layers.11.mlp.experts.65.gate_proj", + "model.layers.11.mlp.experts.65.up_proj", + "model.layers.11.mlp.experts.66.down_proj", + "model.layers.11.mlp.experts.66.gate_proj", + "model.layers.11.mlp.experts.66.up_proj", + "model.layers.11.mlp.experts.67.down_proj", + "model.layers.11.mlp.experts.67.gate_proj", + "model.layers.11.mlp.experts.67.up_proj", + "model.layers.11.mlp.experts.68.down_proj", + "model.layers.11.mlp.experts.68.gate_proj", + "model.layers.11.mlp.experts.68.up_proj", + "model.layers.11.mlp.experts.69.down_proj", + "model.layers.11.mlp.experts.69.gate_proj", + "model.layers.11.mlp.experts.69.up_proj", + "model.layers.11.mlp.experts.7.down_proj", + "model.layers.11.mlp.experts.7.gate_proj", + "model.layers.11.mlp.experts.7.up_proj", + "model.layers.11.mlp.experts.70.down_proj", + "model.layers.11.mlp.experts.70.gate_proj", + "model.layers.11.mlp.experts.70.up_proj", + "model.layers.11.mlp.experts.71.down_proj", + "model.layers.11.mlp.experts.71.gate_proj", + "model.layers.11.mlp.experts.71.up_proj", + "model.layers.11.mlp.experts.72.down_proj", + "model.layers.11.mlp.experts.72.gate_proj", + "model.layers.11.mlp.experts.72.up_proj", + "model.layers.11.mlp.experts.73.down_proj", + "model.layers.11.mlp.experts.73.gate_proj", + "model.layers.11.mlp.experts.73.up_proj", + "model.layers.11.mlp.experts.74.down_proj", + "model.layers.11.mlp.experts.74.gate_proj", + "model.layers.11.mlp.experts.74.up_proj", + "model.layers.11.mlp.experts.75.down_proj", + "model.layers.11.mlp.experts.75.gate_proj", + "model.layers.11.mlp.experts.75.up_proj", + "model.layers.11.mlp.experts.76.down_proj", + "model.layers.11.mlp.experts.76.gate_proj", + "model.layers.11.mlp.experts.76.up_proj", + "model.layers.11.mlp.experts.77.down_proj", + "model.layers.11.mlp.experts.77.gate_proj", + "model.layers.11.mlp.experts.77.up_proj", + "model.layers.11.mlp.experts.78.down_proj", + "model.layers.11.mlp.experts.78.gate_proj", + "model.layers.11.mlp.experts.78.up_proj", + "model.layers.11.mlp.experts.79.down_proj", + "model.layers.11.mlp.experts.79.gate_proj", + "model.layers.11.mlp.experts.79.up_proj", + "model.layers.11.mlp.experts.8.down_proj", + "model.layers.11.mlp.experts.8.gate_proj", + "model.layers.11.mlp.experts.8.up_proj", + "model.layers.11.mlp.experts.80.down_proj", + "model.layers.11.mlp.experts.80.gate_proj", + "model.layers.11.mlp.experts.80.up_proj", + "model.layers.11.mlp.experts.81.down_proj", + "model.layers.11.mlp.experts.81.gate_proj", + "model.layers.11.mlp.experts.81.up_proj", + "model.layers.11.mlp.experts.82.down_proj", + "model.layers.11.mlp.experts.82.gate_proj", + "model.layers.11.mlp.experts.82.up_proj", + "model.layers.11.mlp.experts.83.down_proj", + "model.layers.11.mlp.experts.83.gate_proj", + "model.layers.11.mlp.experts.83.up_proj", + "model.layers.11.mlp.experts.84.down_proj", + "model.layers.11.mlp.experts.84.gate_proj", + "model.layers.11.mlp.experts.84.up_proj", + "model.layers.11.mlp.experts.85.down_proj", + "model.layers.11.mlp.experts.85.gate_proj", + "model.layers.11.mlp.experts.85.up_proj", + "model.layers.11.mlp.experts.86.down_proj", + "model.layers.11.mlp.experts.86.gate_proj", + "model.layers.11.mlp.experts.86.up_proj", + "model.layers.11.mlp.experts.87.down_proj", + "model.layers.11.mlp.experts.87.gate_proj", + "model.layers.11.mlp.experts.87.up_proj", + "model.layers.11.mlp.experts.88.down_proj", + "model.layers.11.mlp.experts.88.gate_proj", + "model.layers.11.mlp.experts.88.up_proj", + "model.layers.11.mlp.experts.89.down_proj", + "model.layers.11.mlp.experts.89.gate_proj", + "model.layers.11.mlp.experts.89.up_proj", + "model.layers.11.mlp.experts.9.down_proj", + "model.layers.11.mlp.experts.9.gate_proj", + "model.layers.11.mlp.experts.9.up_proj", + "model.layers.11.mlp.experts.90.down_proj", + "model.layers.11.mlp.experts.90.gate_proj", + "model.layers.11.mlp.experts.90.up_proj", + "model.layers.11.mlp.experts.91.down_proj", + "model.layers.11.mlp.experts.91.gate_proj", + "model.layers.11.mlp.experts.91.up_proj", + "model.layers.11.mlp.experts.92.down_proj", + "model.layers.11.mlp.experts.92.gate_proj", + "model.layers.11.mlp.experts.92.up_proj", + "model.layers.11.mlp.experts.93.down_proj", + "model.layers.11.mlp.experts.93.gate_proj", + "model.layers.11.mlp.experts.93.up_proj", + "model.layers.11.mlp.experts.94.down_proj", + "model.layers.11.mlp.experts.94.gate_proj", + "model.layers.11.mlp.experts.94.up_proj", + "model.layers.11.mlp.experts.95.down_proj", + "model.layers.11.mlp.experts.95.gate_proj", + "model.layers.11.mlp.experts.95.up_proj", + "model.layers.11.mlp.experts.96.down_proj", + "model.layers.11.mlp.experts.96.gate_proj", + "model.layers.11.mlp.experts.96.up_proj", + "model.layers.11.mlp.experts.97.down_proj", + "model.layers.11.mlp.experts.97.gate_proj", + "model.layers.11.mlp.experts.97.up_proj", + "model.layers.11.mlp.experts.98.down_proj", + "model.layers.11.mlp.experts.98.gate_proj", + "model.layers.11.mlp.experts.98.up_proj", + "model.layers.11.mlp.experts.99.down_proj", + "model.layers.11.mlp.experts.99.gate_proj", + "model.layers.11.mlp.experts.99.up_proj", + "model.layers.12.mlp.experts.0.down_proj", + "model.layers.12.mlp.experts.0.gate_proj", + "model.layers.12.mlp.experts.0.up_proj", + "model.layers.12.mlp.experts.1.down_proj", + "model.layers.12.mlp.experts.1.gate_proj", + "model.layers.12.mlp.experts.1.up_proj", + "model.layers.12.mlp.experts.10.down_proj", + "model.layers.12.mlp.experts.10.gate_proj", + "model.layers.12.mlp.experts.10.up_proj", + "model.layers.12.mlp.experts.100.down_proj", + "model.layers.12.mlp.experts.100.gate_proj", + "model.layers.12.mlp.experts.100.up_proj", + "model.layers.12.mlp.experts.101.down_proj", + "model.layers.12.mlp.experts.101.gate_proj", + "model.layers.12.mlp.experts.101.up_proj", + "model.layers.12.mlp.experts.102.down_proj", + "model.layers.12.mlp.experts.102.gate_proj", + "model.layers.12.mlp.experts.102.up_proj", + "model.layers.12.mlp.experts.103.down_proj", + "model.layers.12.mlp.experts.103.gate_proj", + "model.layers.12.mlp.experts.103.up_proj", + "model.layers.12.mlp.experts.104.down_proj", + "model.layers.12.mlp.experts.104.gate_proj", + "model.layers.12.mlp.experts.104.up_proj", + "model.layers.12.mlp.experts.105.down_proj", + "model.layers.12.mlp.experts.105.gate_proj", + "model.layers.12.mlp.experts.105.up_proj", + "model.layers.12.mlp.experts.106.down_proj", + "model.layers.12.mlp.experts.106.gate_proj", + "model.layers.12.mlp.experts.106.up_proj", + "model.layers.12.mlp.experts.107.down_proj", + "model.layers.12.mlp.experts.107.gate_proj", + "model.layers.12.mlp.experts.107.up_proj", + "model.layers.12.mlp.experts.108.down_proj", + "model.layers.12.mlp.experts.108.gate_proj", + "model.layers.12.mlp.experts.108.up_proj", + "model.layers.12.mlp.experts.109.down_proj", + "model.layers.12.mlp.experts.109.gate_proj", + "model.layers.12.mlp.experts.109.up_proj", + "model.layers.12.mlp.experts.11.down_proj", + "model.layers.12.mlp.experts.11.gate_proj", + "model.layers.12.mlp.experts.11.up_proj", + "model.layers.12.mlp.experts.110.down_proj", + "model.layers.12.mlp.experts.110.gate_proj", + "model.layers.12.mlp.experts.110.up_proj", + "model.layers.12.mlp.experts.111.down_proj", + "model.layers.12.mlp.experts.111.gate_proj", + "model.layers.12.mlp.experts.111.up_proj", + "model.layers.12.mlp.experts.112.down_proj", + "model.layers.12.mlp.experts.112.gate_proj", + "model.layers.12.mlp.experts.112.up_proj", + "model.layers.12.mlp.experts.113.down_proj", + "model.layers.12.mlp.experts.113.gate_proj", + "model.layers.12.mlp.experts.113.up_proj", + "model.layers.12.mlp.experts.114.down_proj", + "model.layers.12.mlp.experts.114.gate_proj", + "model.layers.12.mlp.experts.114.up_proj", + "model.layers.12.mlp.experts.115.down_proj", + "model.layers.12.mlp.experts.115.gate_proj", + "model.layers.12.mlp.experts.115.up_proj", + "model.layers.12.mlp.experts.116.down_proj", + "model.layers.12.mlp.experts.116.gate_proj", + "model.layers.12.mlp.experts.116.up_proj", + "model.layers.12.mlp.experts.117.down_proj", + "model.layers.12.mlp.experts.117.gate_proj", + "model.layers.12.mlp.experts.117.up_proj", + "model.layers.12.mlp.experts.118.down_proj", + "model.layers.12.mlp.experts.118.gate_proj", + "model.layers.12.mlp.experts.118.up_proj", + "model.layers.12.mlp.experts.119.down_proj", + "model.layers.12.mlp.experts.119.gate_proj", + "model.layers.12.mlp.experts.119.up_proj", + "model.layers.12.mlp.experts.12.down_proj", + "model.layers.12.mlp.experts.12.gate_proj", + "model.layers.12.mlp.experts.12.up_proj", + "model.layers.12.mlp.experts.120.down_proj", + "model.layers.12.mlp.experts.120.gate_proj", + "model.layers.12.mlp.experts.120.up_proj", + "model.layers.12.mlp.experts.121.down_proj", + "model.layers.12.mlp.experts.121.gate_proj", + "model.layers.12.mlp.experts.121.up_proj", + "model.layers.12.mlp.experts.122.down_proj", + "model.layers.12.mlp.experts.122.gate_proj", + "model.layers.12.mlp.experts.122.up_proj", + "model.layers.12.mlp.experts.123.down_proj", + "model.layers.12.mlp.experts.123.gate_proj", + "model.layers.12.mlp.experts.123.up_proj", + "model.layers.12.mlp.experts.124.down_proj", + "model.layers.12.mlp.experts.124.gate_proj", + "model.layers.12.mlp.experts.124.up_proj", + "model.layers.12.mlp.experts.125.down_proj", + "model.layers.12.mlp.experts.125.gate_proj", + "model.layers.12.mlp.experts.125.up_proj", + "model.layers.12.mlp.experts.126.down_proj", + "model.layers.12.mlp.experts.126.gate_proj", + "model.layers.12.mlp.experts.126.up_proj", + "model.layers.12.mlp.experts.127.down_proj", + "model.layers.12.mlp.experts.127.gate_proj", + "model.layers.12.mlp.experts.127.up_proj", + "model.layers.12.mlp.experts.128.down_proj", + "model.layers.12.mlp.experts.128.gate_proj", + "model.layers.12.mlp.experts.128.up_proj", + "model.layers.12.mlp.experts.129.down_proj", + "model.layers.12.mlp.experts.129.gate_proj", + "model.layers.12.mlp.experts.129.up_proj", + "model.layers.12.mlp.experts.13.down_proj", + "model.layers.12.mlp.experts.13.gate_proj", + "model.layers.12.mlp.experts.13.up_proj", + "model.layers.12.mlp.experts.130.down_proj", + "model.layers.12.mlp.experts.130.gate_proj", + "model.layers.12.mlp.experts.130.up_proj", + "model.layers.12.mlp.experts.131.down_proj", + "model.layers.12.mlp.experts.131.gate_proj", + "model.layers.12.mlp.experts.131.up_proj", + "model.layers.12.mlp.experts.132.down_proj", + "model.layers.12.mlp.experts.132.gate_proj", + "model.layers.12.mlp.experts.132.up_proj", + "model.layers.12.mlp.experts.133.down_proj", + "model.layers.12.mlp.experts.133.gate_proj", + "model.layers.12.mlp.experts.133.up_proj", + "model.layers.12.mlp.experts.134.down_proj", + "model.layers.12.mlp.experts.134.gate_proj", + "model.layers.12.mlp.experts.134.up_proj", + "model.layers.12.mlp.experts.135.down_proj", + "model.layers.12.mlp.experts.135.gate_proj", + "model.layers.12.mlp.experts.135.up_proj", + "model.layers.12.mlp.experts.136.down_proj", + "model.layers.12.mlp.experts.136.gate_proj", + "model.layers.12.mlp.experts.136.up_proj", + "model.layers.12.mlp.experts.137.down_proj", + "model.layers.12.mlp.experts.137.gate_proj", + "model.layers.12.mlp.experts.137.up_proj", + "model.layers.12.mlp.experts.138.down_proj", + "model.layers.12.mlp.experts.138.gate_proj", + "model.layers.12.mlp.experts.138.up_proj", + "model.layers.12.mlp.experts.139.down_proj", + "model.layers.12.mlp.experts.139.gate_proj", + "model.layers.12.mlp.experts.139.up_proj", + "model.layers.12.mlp.experts.14.down_proj", + "model.layers.12.mlp.experts.14.gate_proj", + "model.layers.12.mlp.experts.14.up_proj", + "model.layers.12.mlp.experts.140.down_proj", + "model.layers.12.mlp.experts.140.gate_proj", + "model.layers.12.mlp.experts.140.up_proj", + "model.layers.12.mlp.experts.141.down_proj", + "model.layers.12.mlp.experts.141.gate_proj", + "model.layers.12.mlp.experts.141.up_proj", + "model.layers.12.mlp.experts.142.down_proj", + "model.layers.12.mlp.experts.142.gate_proj", + "model.layers.12.mlp.experts.142.up_proj", + "model.layers.12.mlp.experts.143.down_proj", + "model.layers.12.mlp.experts.143.gate_proj", + "model.layers.12.mlp.experts.143.up_proj", + "model.layers.12.mlp.experts.144.down_proj", + "model.layers.12.mlp.experts.144.gate_proj", + "model.layers.12.mlp.experts.144.up_proj", + "model.layers.12.mlp.experts.145.down_proj", + "model.layers.12.mlp.experts.145.gate_proj", + "model.layers.12.mlp.experts.145.up_proj", + "model.layers.12.mlp.experts.146.down_proj", + "model.layers.12.mlp.experts.146.gate_proj", + "model.layers.12.mlp.experts.146.up_proj", + "model.layers.12.mlp.experts.147.down_proj", + "model.layers.12.mlp.experts.147.gate_proj", + "model.layers.12.mlp.experts.147.up_proj", + "model.layers.12.mlp.experts.148.down_proj", + "model.layers.12.mlp.experts.148.gate_proj", + "model.layers.12.mlp.experts.148.up_proj", + "model.layers.12.mlp.experts.149.down_proj", + "model.layers.12.mlp.experts.149.gate_proj", + "model.layers.12.mlp.experts.149.up_proj", + "model.layers.12.mlp.experts.15.down_proj", + "model.layers.12.mlp.experts.15.gate_proj", + "model.layers.12.mlp.experts.15.up_proj", + "model.layers.12.mlp.experts.150.down_proj", + "model.layers.12.mlp.experts.150.gate_proj", + "model.layers.12.mlp.experts.150.up_proj", + "model.layers.12.mlp.experts.151.down_proj", + "model.layers.12.mlp.experts.151.gate_proj", + "model.layers.12.mlp.experts.151.up_proj", + "model.layers.12.mlp.experts.152.down_proj", + "model.layers.12.mlp.experts.152.gate_proj", + "model.layers.12.mlp.experts.152.up_proj", + "model.layers.12.mlp.experts.153.down_proj", + "model.layers.12.mlp.experts.153.gate_proj", + "model.layers.12.mlp.experts.153.up_proj", + "model.layers.12.mlp.experts.154.down_proj", + "model.layers.12.mlp.experts.154.gate_proj", + "model.layers.12.mlp.experts.154.up_proj", + "model.layers.12.mlp.experts.155.down_proj", + "model.layers.12.mlp.experts.155.gate_proj", + "model.layers.12.mlp.experts.155.up_proj", + "model.layers.12.mlp.experts.156.down_proj", + "model.layers.12.mlp.experts.156.gate_proj", + "model.layers.12.mlp.experts.156.up_proj", + "model.layers.12.mlp.experts.157.down_proj", + "model.layers.12.mlp.experts.157.gate_proj", + "model.layers.12.mlp.experts.157.up_proj", + "model.layers.12.mlp.experts.158.down_proj", + "model.layers.12.mlp.experts.158.gate_proj", + "model.layers.12.mlp.experts.158.up_proj", + "model.layers.12.mlp.experts.159.down_proj", + "model.layers.12.mlp.experts.159.gate_proj", + "model.layers.12.mlp.experts.159.up_proj", + "model.layers.12.mlp.experts.16.down_proj", + "model.layers.12.mlp.experts.16.gate_proj", + "model.layers.12.mlp.experts.16.up_proj", + "model.layers.12.mlp.experts.160.down_proj", + "model.layers.12.mlp.experts.160.gate_proj", + "model.layers.12.mlp.experts.160.up_proj", + "model.layers.12.mlp.experts.161.down_proj", + "model.layers.12.mlp.experts.161.gate_proj", + "model.layers.12.mlp.experts.161.up_proj", + "model.layers.12.mlp.experts.162.down_proj", + "model.layers.12.mlp.experts.162.gate_proj", + "model.layers.12.mlp.experts.162.up_proj", + "model.layers.12.mlp.experts.163.down_proj", + "model.layers.12.mlp.experts.163.gate_proj", + "model.layers.12.mlp.experts.163.up_proj", + "model.layers.12.mlp.experts.164.down_proj", + "model.layers.12.mlp.experts.164.gate_proj", + "model.layers.12.mlp.experts.164.up_proj", + "model.layers.12.mlp.experts.165.down_proj", + "model.layers.12.mlp.experts.165.gate_proj", + "model.layers.12.mlp.experts.165.up_proj", + "model.layers.12.mlp.experts.166.down_proj", + "model.layers.12.mlp.experts.166.gate_proj", + "model.layers.12.mlp.experts.166.up_proj", + "model.layers.12.mlp.experts.167.down_proj", + "model.layers.12.mlp.experts.167.gate_proj", + "model.layers.12.mlp.experts.167.up_proj", + "model.layers.12.mlp.experts.168.down_proj", + "model.layers.12.mlp.experts.168.gate_proj", + "model.layers.12.mlp.experts.168.up_proj", + "model.layers.12.mlp.experts.169.down_proj", + "model.layers.12.mlp.experts.169.gate_proj", + "model.layers.12.mlp.experts.169.up_proj", + "model.layers.12.mlp.experts.17.down_proj", + "model.layers.12.mlp.experts.17.gate_proj", + "model.layers.12.mlp.experts.17.up_proj", + "model.layers.12.mlp.experts.170.down_proj", + "model.layers.12.mlp.experts.170.gate_proj", + "model.layers.12.mlp.experts.170.up_proj", + "model.layers.12.mlp.experts.171.down_proj", + "model.layers.12.mlp.experts.171.gate_proj", + "model.layers.12.mlp.experts.171.up_proj", + "model.layers.12.mlp.experts.172.down_proj", + "model.layers.12.mlp.experts.172.gate_proj", + "model.layers.12.mlp.experts.172.up_proj", + "model.layers.12.mlp.experts.173.down_proj", + "model.layers.12.mlp.experts.173.gate_proj", + "model.layers.12.mlp.experts.173.up_proj", + "model.layers.12.mlp.experts.174.down_proj", + "model.layers.12.mlp.experts.174.gate_proj", + "model.layers.12.mlp.experts.174.up_proj", + "model.layers.12.mlp.experts.175.down_proj", + "model.layers.12.mlp.experts.175.gate_proj", + "model.layers.12.mlp.experts.175.up_proj", + "model.layers.12.mlp.experts.176.down_proj", + "model.layers.12.mlp.experts.176.gate_proj", + "model.layers.12.mlp.experts.176.up_proj", + "model.layers.12.mlp.experts.177.down_proj", + "model.layers.12.mlp.experts.177.gate_proj", + "model.layers.12.mlp.experts.177.up_proj", + "model.layers.12.mlp.experts.178.down_proj", + "model.layers.12.mlp.experts.178.gate_proj", + "model.layers.12.mlp.experts.178.up_proj", + "model.layers.12.mlp.experts.179.down_proj", + "model.layers.12.mlp.experts.179.gate_proj", + "model.layers.12.mlp.experts.179.up_proj", + "model.layers.12.mlp.experts.18.down_proj", + "model.layers.12.mlp.experts.18.gate_proj", + "model.layers.12.mlp.experts.18.up_proj", + "model.layers.12.mlp.experts.180.down_proj", + "model.layers.12.mlp.experts.180.gate_proj", + "model.layers.12.mlp.experts.180.up_proj", + "model.layers.12.mlp.experts.181.down_proj", + "model.layers.12.mlp.experts.181.gate_proj", + "model.layers.12.mlp.experts.181.up_proj", + "model.layers.12.mlp.experts.182.down_proj", + "model.layers.12.mlp.experts.182.gate_proj", + "model.layers.12.mlp.experts.182.up_proj", + "model.layers.12.mlp.experts.183.down_proj", + "model.layers.12.mlp.experts.183.gate_proj", + "model.layers.12.mlp.experts.183.up_proj", + "model.layers.12.mlp.experts.184.down_proj", + "model.layers.12.mlp.experts.184.gate_proj", + "model.layers.12.mlp.experts.184.up_proj", + "model.layers.12.mlp.experts.185.down_proj", + "model.layers.12.mlp.experts.185.gate_proj", + "model.layers.12.mlp.experts.185.up_proj", + "model.layers.12.mlp.experts.186.down_proj", + "model.layers.12.mlp.experts.186.gate_proj", + "model.layers.12.mlp.experts.186.up_proj", + "model.layers.12.mlp.experts.187.down_proj", + "model.layers.12.mlp.experts.187.gate_proj", + "model.layers.12.mlp.experts.187.up_proj", + "model.layers.12.mlp.experts.188.down_proj", + "model.layers.12.mlp.experts.188.gate_proj", + "model.layers.12.mlp.experts.188.up_proj", + "model.layers.12.mlp.experts.189.down_proj", + "model.layers.12.mlp.experts.189.gate_proj", + "model.layers.12.mlp.experts.189.up_proj", + "model.layers.12.mlp.experts.19.down_proj", + "model.layers.12.mlp.experts.19.gate_proj", + "model.layers.12.mlp.experts.19.up_proj", + "model.layers.12.mlp.experts.190.down_proj", + "model.layers.12.mlp.experts.190.gate_proj", + "model.layers.12.mlp.experts.190.up_proj", + "model.layers.12.mlp.experts.191.down_proj", + "model.layers.12.mlp.experts.191.gate_proj", + "model.layers.12.mlp.experts.191.up_proj", + "model.layers.12.mlp.experts.192.down_proj", + "model.layers.12.mlp.experts.192.gate_proj", + "model.layers.12.mlp.experts.192.up_proj", + "model.layers.12.mlp.experts.193.down_proj", + "model.layers.12.mlp.experts.193.gate_proj", + "model.layers.12.mlp.experts.193.up_proj", + "model.layers.12.mlp.experts.194.down_proj", + "model.layers.12.mlp.experts.194.gate_proj", + "model.layers.12.mlp.experts.194.up_proj", + "model.layers.12.mlp.experts.195.down_proj", + "model.layers.12.mlp.experts.195.gate_proj", + "model.layers.12.mlp.experts.195.up_proj", + "model.layers.12.mlp.experts.196.down_proj", + "model.layers.12.mlp.experts.196.gate_proj", + "model.layers.12.mlp.experts.196.up_proj", + "model.layers.12.mlp.experts.197.down_proj", + "model.layers.12.mlp.experts.197.gate_proj", + "model.layers.12.mlp.experts.197.up_proj", + "model.layers.12.mlp.experts.198.down_proj", + "model.layers.12.mlp.experts.198.gate_proj", + "model.layers.12.mlp.experts.198.up_proj", + "model.layers.12.mlp.experts.199.down_proj", + "model.layers.12.mlp.experts.199.gate_proj", + "model.layers.12.mlp.experts.199.up_proj", + "model.layers.12.mlp.experts.2.down_proj", + "model.layers.12.mlp.experts.2.gate_proj", + "model.layers.12.mlp.experts.2.up_proj", + "model.layers.12.mlp.experts.20.down_proj", + "model.layers.12.mlp.experts.20.gate_proj", + "model.layers.12.mlp.experts.20.up_proj", + "model.layers.12.mlp.experts.200.down_proj", + "model.layers.12.mlp.experts.200.gate_proj", + "model.layers.12.mlp.experts.200.up_proj", + "model.layers.12.mlp.experts.201.down_proj", + "model.layers.12.mlp.experts.201.gate_proj", + "model.layers.12.mlp.experts.201.up_proj", + "model.layers.12.mlp.experts.202.down_proj", + "model.layers.12.mlp.experts.202.gate_proj", + "model.layers.12.mlp.experts.202.up_proj", + "model.layers.12.mlp.experts.203.down_proj", + "model.layers.12.mlp.experts.203.gate_proj", + "model.layers.12.mlp.experts.203.up_proj", + "model.layers.12.mlp.experts.204.down_proj", + "model.layers.12.mlp.experts.204.gate_proj", + "model.layers.12.mlp.experts.204.up_proj", + "model.layers.12.mlp.experts.205.down_proj", + "model.layers.12.mlp.experts.205.gate_proj", + "model.layers.12.mlp.experts.205.up_proj", + "model.layers.12.mlp.experts.206.down_proj", + "model.layers.12.mlp.experts.206.gate_proj", + "model.layers.12.mlp.experts.206.up_proj", + "model.layers.12.mlp.experts.207.down_proj", + "model.layers.12.mlp.experts.207.gate_proj", + "model.layers.12.mlp.experts.207.up_proj", + "model.layers.12.mlp.experts.208.down_proj", + "model.layers.12.mlp.experts.208.gate_proj", + "model.layers.12.mlp.experts.208.up_proj", + "model.layers.12.mlp.experts.209.down_proj", + "model.layers.12.mlp.experts.209.gate_proj", + "model.layers.12.mlp.experts.209.up_proj", + "model.layers.12.mlp.experts.21.down_proj", + "model.layers.12.mlp.experts.21.gate_proj", + "model.layers.12.mlp.experts.21.up_proj", + "model.layers.12.mlp.experts.210.down_proj", + "model.layers.12.mlp.experts.210.gate_proj", + "model.layers.12.mlp.experts.210.up_proj", + "model.layers.12.mlp.experts.211.down_proj", + "model.layers.12.mlp.experts.211.gate_proj", + "model.layers.12.mlp.experts.211.up_proj", + "model.layers.12.mlp.experts.212.down_proj", + "model.layers.12.mlp.experts.212.gate_proj", + "model.layers.12.mlp.experts.212.up_proj", + "model.layers.12.mlp.experts.213.down_proj", + "model.layers.12.mlp.experts.213.gate_proj", + "model.layers.12.mlp.experts.213.up_proj", + "model.layers.12.mlp.experts.214.down_proj", + "model.layers.12.mlp.experts.214.gate_proj", + "model.layers.12.mlp.experts.214.up_proj", + "model.layers.12.mlp.experts.215.down_proj", + "model.layers.12.mlp.experts.215.gate_proj", + "model.layers.12.mlp.experts.215.up_proj", + "model.layers.12.mlp.experts.216.down_proj", + "model.layers.12.mlp.experts.216.gate_proj", + "model.layers.12.mlp.experts.216.up_proj", + "model.layers.12.mlp.experts.217.down_proj", + "model.layers.12.mlp.experts.217.gate_proj", + "model.layers.12.mlp.experts.217.up_proj", + "model.layers.12.mlp.experts.218.down_proj", + "model.layers.12.mlp.experts.218.gate_proj", + "model.layers.12.mlp.experts.218.up_proj", + "model.layers.12.mlp.experts.219.down_proj", + "model.layers.12.mlp.experts.219.gate_proj", + "model.layers.12.mlp.experts.219.up_proj", + "model.layers.12.mlp.experts.22.down_proj", + "model.layers.12.mlp.experts.22.gate_proj", + "model.layers.12.mlp.experts.22.up_proj", + "model.layers.12.mlp.experts.220.down_proj", + "model.layers.12.mlp.experts.220.gate_proj", + "model.layers.12.mlp.experts.220.up_proj", + "model.layers.12.mlp.experts.221.down_proj", + "model.layers.12.mlp.experts.221.gate_proj", + "model.layers.12.mlp.experts.221.up_proj", + "model.layers.12.mlp.experts.222.down_proj", + "model.layers.12.mlp.experts.222.gate_proj", + "model.layers.12.mlp.experts.222.up_proj", + "model.layers.12.mlp.experts.223.down_proj", + "model.layers.12.mlp.experts.223.gate_proj", + "model.layers.12.mlp.experts.223.up_proj", + "model.layers.12.mlp.experts.224.down_proj", + "model.layers.12.mlp.experts.224.gate_proj", + "model.layers.12.mlp.experts.224.up_proj", + "model.layers.12.mlp.experts.225.down_proj", + "model.layers.12.mlp.experts.225.gate_proj", + "model.layers.12.mlp.experts.225.up_proj", + "model.layers.12.mlp.experts.226.down_proj", + "model.layers.12.mlp.experts.226.gate_proj", + "model.layers.12.mlp.experts.226.up_proj", + "model.layers.12.mlp.experts.227.down_proj", + "model.layers.12.mlp.experts.227.gate_proj", + "model.layers.12.mlp.experts.227.up_proj", + "model.layers.12.mlp.experts.228.down_proj", + "model.layers.12.mlp.experts.228.gate_proj", + "model.layers.12.mlp.experts.228.up_proj", + "model.layers.12.mlp.experts.229.down_proj", + "model.layers.12.mlp.experts.229.gate_proj", + "model.layers.12.mlp.experts.229.up_proj", + "model.layers.12.mlp.experts.23.down_proj", + "model.layers.12.mlp.experts.23.gate_proj", + "model.layers.12.mlp.experts.23.up_proj", + "model.layers.12.mlp.experts.230.down_proj", + "model.layers.12.mlp.experts.230.gate_proj", + "model.layers.12.mlp.experts.230.up_proj", + "model.layers.12.mlp.experts.231.down_proj", + "model.layers.12.mlp.experts.231.gate_proj", + "model.layers.12.mlp.experts.231.up_proj", + "model.layers.12.mlp.experts.232.down_proj", + "model.layers.12.mlp.experts.232.gate_proj", + "model.layers.12.mlp.experts.232.up_proj", + "model.layers.12.mlp.experts.233.down_proj", + "model.layers.12.mlp.experts.233.gate_proj", + "model.layers.12.mlp.experts.233.up_proj", + "model.layers.12.mlp.experts.234.down_proj", + "model.layers.12.mlp.experts.234.gate_proj", + "model.layers.12.mlp.experts.234.up_proj", + "model.layers.12.mlp.experts.235.down_proj", + "model.layers.12.mlp.experts.235.gate_proj", + "model.layers.12.mlp.experts.235.up_proj", + "model.layers.12.mlp.experts.236.down_proj", + "model.layers.12.mlp.experts.236.gate_proj", + "model.layers.12.mlp.experts.236.up_proj", + "model.layers.12.mlp.experts.237.down_proj", + "model.layers.12.mlp.experts.237.gate_proj", + "model.layers.12.mlp.experts.237.up_proj", + "model.layers.12.mlp.experts.238.down_proj", + "model.layers.12.mlp.experts.238.gate_proj", + "model.layers.12.mlp.experts.238.up_proj", + "model.layers.12.mlp.experts.239.down_proj", + "model.layers.12.mlp.experts.239.gate_proj", + "model.layers.12.mlp.experts.239.up_proj", + "model.layers.12.mlp.experts.24.down_proj", + "model.layers.12.mlp.experts.24.gate_proj", + "model.layers.12.mlp.experts.24.up_proj", + "model.layers.12.mlp.experts.240.down_proj", + "model.layers.12.mlp.experts.240.gate_proj", + "model.layers.12.mlp.experts.240.up_proj", + "model.layers.12.mlp.experts.241.down_proj", + "model.layers.12.mlp.experts.241.gate_proj", + "model.layers.12.mlp.experts.241.up_proj", + "model.layers.12.mlp.experts.242.down_proj", + "model.layers.12.mlp.experts.242.gate_proj", + "model.layers.12.mlp.experts.242.up_proj", + "model.layers.12.mlp.experts.243.down_proj", + "model.layers.12.mlp.experts.243.gate_proj", + "model.layers.12.mlp.experts.243.up_proj", + "model.layers.12.mlp.experts.244.down_proj", + "model.layers.12.mlp.experts.244.gate_proj", + "model.layers.12.mlp.experts.244.up_proj", + "model.layers.12.mlp.experts.245.down_proj", + "model.layers.12.mlp.experts.245.gate_proj", + "model.layers.12.mlp.experts.245.up_proj", + "model.layers.12.mlp.experts.246.down_proj", + "model.layers.12.mlp.experts.246.gate_proj", + "model.layers.12.mlp.experts.246.up_proj", + "model.layers.12.mlp.experts.247.down_proj", + "model.layers.12.mlp.experts.247.gate_proj", + "model.layers.12.mlp.experts.247.up_proj", + "model.layers.12.mlp.experts.248.down_proj", + "model.layers.12.mlp.experts.248.gate_proj", + "model.layers.12.mlp.experts.248.up_proj", + "model.layers.12.mlp.experts.249.down_proj", + "model.layers.12.mlp.experts.249.gate_proj", + "model.layers.12.mlp.experts.249.up_proj", + "model.layers.12.mlp.experts.25.down_proj", + "model.layers.12.mlp.experts.25.gate_proj", + "model.layers.12.mlp.experts.25.up_proj", + "model.layers.12.mlp.experts.250.down_proj", + "model.layers.12.mlp.experts.250.gate_proj", + "model.layers.12.mlp.experts.250.up_proj", + "model.layers.12.mlp.experts.251.down_proj", + "model.layers.12.mlp.experts.251.gate_proj", + "model.layers.12.mlp.experts.251.up_proj", + "model.layers.12.mlp.experts.252.down_proj", + "model.layers.12.mlp.experts.252.gate_proj", + "model.layers.12.mlp.experts.252.up_proj", + "model.layers.12.mlp.experts.253.down_proj", + "model.layers.12.mlp.experts.253.gate_proj", + "model.layers.12.mlp.experts.253.up_proj", + "model.layers.12.mlp.experts.254.down_proj", + "model.layers.12.mlp.experts.254.gate_proj", + "model.layers.12.mlp.experts.254.up_proj", + "model.layers.12.mlp.experts.255.down_proj", + "model.layers.12.mlp.experts.255.gate_proj", + "model.layers.12.mlp.experts.255.up_proj", + "model.layers.12.mlp.experts.26.down_proj", + "model.layers.12.mlp.experts.26.gate_proj", + "model.layers.12.mlp.experts.26.up_proj", + "model.layers.12.mlp.experts.27.down_proj", + "model.layers.12.mlp.experts.27.gate_proj", + "model.layers.12.mlp.experts.27.up_proj", + "model.layers.12.mlp.experts.28.down_proj", + "model.layers.12.mlp.experts.28.gate_proj", + "model.layers.12.mlp.experts.28.up_proj", + "model.layers.12.mlp.experts.29.down_proj", + "model.layers.12.mlp.experts.29.gate_proj", + "model.layers.12.mlp.experts.29.up_proj", + "model.layers.12.mlp.experts.3.down_proj", + "model.layers.12.mlp.experts.3.gate_proj", + "model.layers.12.mlp.experts.3.up_proj", + "model.layers.12.mlp.experts.30.down_proj", + "model.layers.12.mlp.experts.30.gate_proj", + "model.layers.12.mlp.experts.30.up_proj", + "model.layers.12.mlp.experts.31.down_proj", + "model.layers.12.mlp.experts.31.gate_proj", + "model.layers.12.mlp.experts.31.up_proj", + "model.layers.12.mlp.experts.32.down_proj", + "model.layers.12.mlp.experts.32.gate_proj", + "model.layers.12.mlp.experts.32.up_proj", + "model.layers.12.mlp.experts.33.down_proj", + "model.layers.12.mlp.experts.33.gate_proj", + "model.layers.12.mlp.experts.33.up_proj", + "model.layers.12.mlp.experts.34.down_proj", + "model.layers.12.mlp.experts.34.gate_proj", + "model.layers.12.mlp.experts.34.up_proj", + "model.layers.12.mlp.experts.35.down_proj", + "model.layers.12.mlp.experts.35.gate_proj", + "model.layers.12.mlp.experts.35.up_proj", + "model.layers.12.mlp.experts.36.down_proj", + "model.layers.12.mlp.experts.36.gate_proj", + "model.layers.12.mlp.experts.36.up_proj", + "model.layers.12.mlp.experts.37.down_proj", + "model.layers.12.mlp.experts.37.gate_proj", + "model.layers.12.mlp.experts.37.up_proj", + "model.layers.12.mlp.experts.38.down_proj", + "model.layers.12.mlp.experts.38.gate_proj", + "model.layers.12.mlp.experts.38.up_proj", + "model.layers.12.mlp.experts.39.down_proj", + "model.layers.12.mlp.experts.39.gate_proj", + "model.layers.12.mlp.experts.39.up_proj", + "model.layers.12.mlp.experts.4.down_proj", + "model.layers.12.mlp.experts.4.gate_proj", + "model.layers.12.mlp.experts.4.up_proj", + "model.layers.12.mlp.experts.40.down_proj", + "model.layers.12.mlp.experts.40.gate_proj", + "model.layers.12.mlp.experts.40.up_proj", + "model.layers.12.mlp.experts.41.down_proj", + "model.layers.12.mlp.experts.41.gate_proj", + "model.layers.12.mlp.experts.41.up_proj", + "model.layers.12.mlp.experts.42.down_proj", + "model.layers.12.mlp.experts.42.gate_proj", + "model.layers.12.mlp.experts.42.up_proj", + "model.layers.12.mlp.experts.43.down_proj", + "model.layers.12.mlp.experts.43.gate_proj", + "model.layers.12.mlp.experts.43.up_proj", + "model.layers.12.mlp.experts.44.down_proj", + "model.layers.12.mlp.experts.44.gate_proj", + "model.layers.12.mlp.experts.44.up_proj", + "model.layers.12.mlp.experts.45.down_proj", + "model.layers.12.mlp.experts.45.gate_proj", + "model.layers.12.mlp.experts.45.up_proj", + "model.layers.12.mlp.experts.46.down_proj", + "model.layers.12.mlp.experts.46.gate_proj", + "model.layers.12.mlp.experts.46.up_proj", + "model.layers.12.mlp.experts.47.down_proj", + "model.layers.12.mlp.experts.47.gate_proj", + "model.layers.12.mlp.experts.47.up_proj", + "model.layers.12.mlp.experts.48.down_proj", + "model.layers.12.mlp.experts.48.gate_proj", + "model.layers.12.mlp.experts.48.up_proj", + "model.layers.12.mlp.experts.49.down_proj", + "model.layers.12.mlp.experts.49.gate_proj", + "model.layers.12.mlp.experts.49.up_proj", + "model.layers.12.mlp.experts.5.down_proj", + "model.layers.12.mlp.experts.5.gate_proj", + "model.layers.12.mlp.experts.5.up_proj", + "model.layers.12.mlp.experts.50.down_proj", + "model.layers.12.mlp.experts.50.gate_proj", + "model.layers.12.mlp.experts.50.up_proj", + "model.layers.12.mlp.experts.51.down_proj", + "model.layers.12.mlp.experts.51.gate_proj", + "model.layers.12.mlp.experts.51.up_proj", + "model.layers.12.mlp.experts.52.down_proj", + "model.layers.12.mlp.experts.52.gate_proj", + "model.layers.12.mlp.experts.52.up_proj", + "model.layers.12.mlp.experts.53.down_proj", + "model.layers.12.mlp.experts.53.gate_proj", + "model.layers.12.mlp.experts.53.up_proj", + "model.layers.12.mlp.experts.54.down_proj", + "model.layers.12.mlp.experts.54.gate_proj", + "model.layers.12.mlp.experts.54.up_proj", + "model.layers.12.mlp.experts.55.down_proj", + "model.layers.12.mlp.experts.55.gate_proj", + "model.layers.12.mlp.experts.55.up_proj", + "model.layers.12.mlp.experts.56.down_proj", + "model.layers.12.mlp.experts.56.gate_proj", + "model.layers.12.mlp.experts.56.up_proj", + "model.layers.12.mlp.experts.57.down_proj", + "model.layers.12.mlp.experts.57.gate_proj", + "model.layers.12.mlp.experts.57.up_proj", + "model.layers.12.mlp.experts.58.down_proj", + "model.layers.12.mlp.experts.58.gate_proj", + "model.layers.12.mlp.experts.58.up_proj", + "model.layers.12.mlp.experts.59.down_proj", + "model.layers.12.mlp.experts.59.gate_proj", + "model.layers.12.mlp.experts.59.up_proj", + "model.layers.12.mlp.experts.6.down_proj", + "model.layers.12.mlp.experts.6.gate_proj", + "model.layers.12.mlp.experts.6.up_proj", + "model.layers.12.mlp.experts.60.down_proj", + "model.layers.12.mlp.experts.60.gate_proj", + "model.layers.12.mlp.experts.60.up_proj", + "model.layers.12.mlp.experts.61.down_proj", + "model.layers.12.mlp.experts.61.gate_proj", + "model.layers.12.mlp.experts.61.up_proj", + "model.layers.12.mlp.experts.62.down_proj", + "model.layers.12.mlp.experts.62.gate_proj", + "model.layers.12.mlp.experts.62.up_proj", + "model.layers.12.mlp.experts.63.down_proj", + "model.layers.12.mlp.experts.63.gate_proj", + "model.layers.12.mlp.experts.63.up_proj", + "model.layers.12.mlp.experts.64.down_proj", + "model.layers.12.mlp.experts.64.gate_proj", + "model.layers.12.mlp.experts.64.up_proj", + "model.layers.12.mlp.experts.65.down_proj", + "model.layers.12.mlp.experts.65.gate_proj", + "model.layers.12.mlp.experts.65.up_proj", + "model.layers.12.mlp.experts.66.down_proj", + "model.layers.12.mlp.experts.66.gate_proj", + "model.layers.12.mlp.experts.66.up_proj", + "model.layers.12.mlp.experts.67.down_proj", + "model.layers.12.mlp.experts.67.gate_proj", + "model.layers.12.mlp.experts.67.up_proj", + "model.layers.12.mlp.experts.68.down_proj", + "model.layers.12.mlp.experts.68.gate_proj", + "model.layers.12.mlp.experts.68.up_proj", + "model.layers.12.mlp.experts.69.down_proj", + "model.layers.12.mlp.experts.69.gate_proj", + "model.layers.12.mlp.experts.69.up_proj", + "model.layers.12.mlp.experts.7.down_proj", + "model.layers.12.mlp.experts.7.gate_proj", + "model.layers.12.mlp.experts.7.up_proj", + "model.layers.12.mlp.experts.70.down_proj", + "model.layers.12.mlp.experts.70.gate_proj", + "model.layers.12.mlp.experts.70.up_proj", + "model.layers.12.mlp.experts.71.down_proj", + "model.layers.12.mlp.experts.71.gate_proj", + "model.layers.12.mlp.experts.71.up_proj", + "model.layers.12.mlp.experts.72.down_proj", + "model.layers.12.mlp.experts.72.gate_proj", + "model.layers.12.mlp.experts.72.up_proj", + "model.layers.12.mlp.experts.73.down_proj", + "model.layers.12.mlp.experts.73.gate_proj", + "model.layers.12.mlp.experts.73.up_proj", + "model.layers.12.mlp.experts.74.down_proj", + "model.layers.12.mlp.experts.74.gate_proj", + "model.layers.12.mlp.experts.74.up_proj", + "model.layers.12.mlp.experts.75.down_proj", + "model.layers.12.mlp.experts.75.gate_proj", + "model.layers.12.mlp.experts.75.up_proj", + "model.layers.12.mlp.experts.76.down_proj", + "model.layers.12.mlp.experts.76.gate_proj", + "model.layers.12.mlp.experts.76.up_proj", + "model.layers.12.mlp.experts.77.down_proj", + "model.layers.12.mlp.experts.77.gate_proj", + "model.layers.12.mlp.experts.77.up_proj", + "model.layers.12.mlp.experts.78.down_proj", + "model.layers.12.mlp.experts.78.gate_proj", + "model.layers.12.mlp.experts.78.up_proj", + "model.layers.12.mlp.experts.79.down_proj", + "model.layers.12.mlp.experts.79.gate_proj", + "model.layers.12.mlp.experts.79.up_proj", + "model.layers.12.mlp.experts.8.down_proj", + "model.layers.12.mlp.experts.8.gate_proj", + "model.layers.12.mlp.experts.8.up_proj", + "model.layers.12.mlp.experts.80.down_proj", + "model.layers.12.mlp.experts.80.gate_proj", + "model.layers.12.mlp.experts.80.up_proj", + "model.layers.12.mlp.experts.81.down_proj", + "model.layers.12.mlp.experts.81.gate_proj", + "model.layers.12.mlp.experts.81.up_proj", + "model.layers.12.mlp.experts.82.down_proj", + "model.layers.12.mlp.experts.82.gate_proj", + "model.layers.12.mlp.experts.82.up_proj", + "model.layers.12.mlp.experts.83.down_proj", + "model.layers.12.mlp.experts.83.gate_proj", + "model.layers.12.mlp.experts.83.up_proj", + "model.layers.12.mlp.experts.84.down_proj", + "model.layers.12.mlp.experts.84.gate_proj", + "model.layers.12.mlp.experts.84.up_proj", + "model.layers.12.mlp.experts.85.down_proj", + "model.layers.12.mlp.experts.85.gate_proj", + "model.layers.12.mlp.experts.85.up_proj", + "model.layers.12.mlp.experts.86.down_proj", + "model.layers.12.mlp.experts.86.gate_proj", + "model.layers.12.mlp.experts.86.up_proj", + "model.layers.12.mlp.experts.87.down_proj", + "model.layers.12.mlp.experts.87.gate_proj", + "model.layers.12.mlp.experts.87.up_proj", + "model.layers.12.mlp.experts.88.down_proj", + "model.layers.12.mlp.experts.88.gate_proj", + "model.layers.12.mlp.experts.88.up_proj", + "model.layers.12.mlp.experts.89.down_proj", + "model.layers.12.mlp.experts.89.gate_proj", + "model.layers.12.mlp.experts.89.up_proj", + "model.layers.12.mlp.experts.9.down_proj", + "model.layers.12.mlp.experts.9.gate_proj", + "model.layers.12.mlp.experts.9.up_proj", + "model.layers.12.mlp.experts.90.down_proj", + "model.layers.12.mlp.experts.90.gate_proj", + "model.layers.12.mlp.experts.90.up_proj", + "model.layers.12.mlp.experts.91.down_proj", + "model.layers.12.mlp.experts.91.gate_proj", + "model.layers.12.mlp.experts.91.up_proj", + "model.layers.12.mlp.experts.92.down_proj", + "model.layers.12.mlp.experts.92.gate_proj", + "model.layers.12.mlp.experts.92.up_proj", + "model.layers.12.mlp.experts.93.down_proj", + "model.layers.12.mlp.experts.93.gate_proj", + "model.layers.12.mlp.experts.93.up_proj", + "model.layers.12.mlp.experts.94.down_proj", + "model.layers.12.mlp.experts.94.gate_proj", + "model.layers.12.mlp.experts.94.up_proj", + "model.layers.12.mlp.experts.95.down_proj", + "model.layers.12.mlp.experts.95.gate_proj", + "model.layers.12.mlp.experts.95.up_proj", + "model.layers.12.mlp.experts.96.down_proj", + "model.layers.12.mlp.experts.96.gate_proj", + "model.layers.12.mlp.experts.96.up_proj", + "model.layers.12.mlp.experts.97.down_proj", + "model.layers.12.mlp.experts.97.gate_proj", + "model.layers.12.mlp.experts.97.up_proj", + "model.layers.12.mlp.experts.98.down_proj", + "model.layers.12.mlp.experts.98.gate_proj", + "model.layers.12.mlp.experts.98.up_proj", + "model.layers.12.mlp.experts.99.down_proj", + "model.layers.12.mlp.experts.99.gate_proj", + "model.layers.12.mlp.experts.99.up_proj", + "model.layers.13.mlp.experts.0.down_proj", + "model.layers.13.mlp.experts.0.gate_proj", + "model.layers.13.mlp.experts.0.up_proj", + "model.layers.13.mlp.experts.1.down_proj", + "model.layers.13.mlp.experts.1.gate_proj", + "model.layers.13.mlp.experts.1.up_proj", + "model.layers.13.mlp.experts.10.down_proj", + "model.layers.13.mlp.experts.10.gate_proj", + "model.layers.13.mlp.experts.10.up_proj", + "model.layers.13.mlp.experts.100.down_proj", + "model.layers.13.mlp.experts.100.gate_proj", + "model.layers.13.mlp.experts.100.up_proj", + "model.layers.13.mlp.experts.101.down_proj", + "model.layers.13.mlp.experts.101.gate_proj", + "model.layers.13.mlp.experts.101.up_proj", + "model.layers.13.mlp.experts.102.down_proj", + "model.layers.13.mlp.experts.102.gate_proj", + "model.layers.13.mlp.experts.102.up_proj", + "model.layers.13.mlp.experts.103.down_proj", + "model.layers.13.mlp.experts.103.gate_proj", + "model.layers.13.mlp.experts.103.up_proj", + "model.layers.13.mlp.experts.104.down_proj", + "model.layers.13.mlp.experts.104.gate_proj", + "model.layers.13.mlp.experts.104.up_proj", + "model.layers.13.mlp.experts.105.down_proj", + "model.layers.13.mlp.experts.105.gate_proj", + "model.layers.13.mlp.experts.105.up_proj", + "model.layers.13.mlp.experts.106.down_proj", + "model.layers.13.mlp.experts.106.gate_proj", + "model.layers.13.mlp.experts.106.up_proj", + "model.layers.13.mlp.experts.107.down_proj", + "model.layers.13.mlp.experts.107.gate_proj", + "model.layers.13.mlp.experts.107.up_proj", + "model.layers.13.mlp.experts.108.down_proj", + "model.layers.13.mlp.experts.108.gate_proj", + "model.layers.13.mlp.experts.108.up_proj", + "model.layers.13.mlp.experts.109.down_proj", + "model.layers.13.mlp.experts.109.gate_proj", + "model.layers.13.mlp.experts.109.up_proj", + "model.layers.13.mlp.experts.11.down_proj", + "model.layers.13.mlp.experts.11.gate_proj", + "model.layers.13.mlp.experts.11.up_proj", + "model.layers.13.mlp.experts.110.down_proj", + "model.layers.13.mlp.experts.110.gate_proj", + "model.layers.13.mlp.experts.110.up_proj", + "model.layers.13.mlp.experts.111.down_proj", + "model.layers.13.mlp.experts.111.gate_proj", + "model.layers.13.mlp.experts.111.up_proj", + "model.layers.13.mlp.experts.112.down_proj", + "model.layers.13.mlp.experts.112.gate_proj", + "model.layers.13.mlp.experts.112.up_proj", + "model.layers.13.mlp.experts.113.down_proj", + "model.layers.13.mlp.experts.113.gate_proj", + "model.layers.13.mlp.experts.113.up_proj", + "model.layers.13.mlp.experts.114.down_proj", + "model.layers.13.mlp.experts.114.gate_proj", + "model.layers.13.mlp.experts.114.up_proj", + "model.layers.13.mlp.experts.115.down_proj", + "model.layers.13.mlp.experts.115.gate_proj", + "model.layers.13.mlp.experts.115.up_proj", + "model.layers.13.mlp.experts.116.down_proj", + "model.layers.13.mlp.experts.116.gate_proj", + "model.layers.13.mlp.experts.116.up_proj", + "model.layers.13.mlp.experts.117.down_proj", + "model.layers.13.mlp.experts.117.gate_proj", + "model.layers.13.mlp.experts.117.up_proj", + "model.layers.13.mlp.experts.118.down_proj", + "model.layers.13.mlp.experts.118.gate_proj", + "model.layers.13.mlp.experts.118.up_proj", + "model.layers.13.mlp.experts.119.down_proj", + "model.layers.13.mlp.experts.119.gate_proj", + "model.layers.13.mlp.experts.119.up_proj", + "model.layers.13.mlp.experts.12.down_proj", + "model.layers.13.mlp.experts.12.gate_proj", + "model.layers.13.mlp.experts.12.up_proj", + "model.layers.13.mlp.experts.120.down_proj", + "model.layers.13.mlp.experts.120.gate_proj", + "model.layers.13.mlp.experts.120.up_proj", + "model.layers.13.mlp.experts.121.down_proj", + "model.layers.13.mlp.experts.121.gate_proj", + "model.layers.13.mlp.experts.121.up_proj", + "model.layers.13.mlp.experts.122.down_proj", + "model.layers.13.mlp.experts.122.gate_proj", + "model.layers.13.mlp.experts.122.up_proj", + "model.layers.13.mlp.experts.123.down_proj", + "model.layers.13.mlp.experts.123.gate_proj", + "model.layers.13.mlp.experts.123.up_proj", + "model.layers.13.mlp.experts.124.down_proj", + "model.layers.13.mlp.experts.124.gate_proj", + "model.layers.13.mlp.experts.124.up_proj", + "model.layers.13.mlp.experts.125.down_proj", + "model.layers.13.mlp.experts.125.gate_proj", + "model.layers.13.mlp.experts.125.up_proj", + "model.layers.13.mlp.experts.126.down_proj", + "model.layers.13.mlp.experts.126.gate_proj", + "model.layers.13.mlp.experts.126.up_proj", + "model.layers.13.mlp.experts.127.down_proj", + "model.layers.13.mlp.experts.127.gate_proj", + "model.layers.13.mlp.experts.127.up_proj", + "model.layers.13.mlp.experts.128.down_proj", + "model.layers.13.mlp.experts.128.gate_proj", + "model.layers.13.mlp.experts.128.up_proj", + "model.layers.13.mlp.experts.129.down_proj", + "model.layers.13.mlp.experts.129.gate_proj", + "model.layers.13.mlp.experts.129.up_proj", + "model.layers.13.mlp.experts.13.down_proj", + "model.layers.13.mlp.experts.13.gate_proj", + "model.layers.13.mlp.experts.13.up_proj", + "model.layers.13.mlp.experts.130.down_proj", + "model.layers.13.mlp.experts.130.gate_proj", + "model.layers.13.mlp.experts.130.up_proj", + "model.layers.13.mlp.experts.131.down_proj", + "model.layers.13.mlp.experts.131.gate_proj", + "model.layers.13.mlp.experts.131.up_proj", + "model.layers.13.mlp.experts.132.down_proj", + "model.layers.13.mlp.experts.132.gate_proj", + "model.layers.13.mlp.experts.132.up_proj", + "model.layers.13.mlp.experts.133.down_proj", + "model.layers.13.mlp.experts.133.gate_proj", + "model.layers.13.mlp.experts.133.up_proj", + "model.layers.13.mlp.experts.134.down_proj", + "model.layers.13.mlp.experts.134.gate_proj", + "model.layers.13.mlp.experts.134.up_proj", + "model.layers.13.mlp.experts.135.down_proj", + "model.layers.13.mlp.experts.135.gate_proj", + "model.layers.13.mlp.experts.135.up_proj", + "model.layers.13.mlp.experts.136.down_proj", + "model.layers.13.mlp.experts.136.gate_proj", + "model.layers.13.mlp.experts.136.up_proj", + "model.layers.13.mlp.experts.137.down_proj", + "model.layers.13.mlp.experts.137.gate_proj", + "model.layers.13.mlp.experts.137.up_proj", + "model.layers.13.mlp.experts.138.down_proj", + "model.layers.13.mlp.experts.138.gate_proj", + "model.layers.13.mlp.experts.138.up_proj", + "model.layers.13.mlp.experts.139.down_proj", + "model.layers.13.mlp.experts.139.gate_proj", + "model.layers.13.mlp.experts.139.up_proj", + "model.layers.13.mlp.experts.14.down_proj", + "model.layers.13.mlp.experts.14.gate_proj", + "model.layers.13.mlp.experts.14.up_proj", + "model.layers.13.mlp.experts.140.down_proj", + "model.layers.13.mlp.experts.140.gate_proj", + "model.layers.13.mlp.experts.140.up_proj", + "model.layers.13.mlp.experts.141.down_proj", + "model.layers.13.mlp.experts.141.gate_proj", + "model.layers.13.mlp.experts.141.up_proj", + "model.layers.13.mlp.experts.142.down_proj", + "model.layers.13.mlp.experts.142.gate_proj", + "model.layers.13.mlp.experts.142.up_proj", + "model.layers.13.mlp.experts.143.down_proj", + "model.layers.13.mlp.experts.143.gate_proj", + "model.layers.13.mlp.experts.143.up_proj", + "model.layers.13.mlp.experts.144.down_proj", + "model.layers.13.mlp.experts.144.gate_proj", + "model.layers.13.mlp.experts.144.up_proj", + "model.layers.13.mlp.experts.145.down_proj", + "model.layers.13.mlp.experts.145.gate_proj", + "model.layers.13.mlp.experts.145.up_proj", + "model.layers.13.mlp.experts.146.down_proj", + "model.layers.13.mlp.experts.146.gate_proj", + "model.layers.13.mlp.experts.146.up_proj", + "model.layers.13.mlp.experts.147.down_proj", + "model.layers.13.mlp.experts.147.gate_proj", + "model.layers.13.mlp.experts.147.up_proj", + "model.layers.13.mlp.experts.148.down_proj", + "model.layers.13.mlp.experts.148.gate_proj", + "model.layers.13.mlp.experts.148.up_proj", + "model.layers.13.mlp.experts.149.down_proj", + "model.layers.13.mlp.experts.149.gate_proj", + "model.layers.13.mlp.experts.149.up_proj", + "model.layers.13.mlp.experts.15.down_proj", + "model.layers.13.mlp.experts.15.gate_proj", + "model.layers.13.mlp.experts.15.up_proj", + "model.layers.13.mlp.experts.150.down_proj", + "model.layers.13.mlp.experts.150.gate_proj", + "model.layers.13.mlp.experts.150.up_proj", + "model.layers.13.mlp.experts.151.down_proj", + "model.layers.13.mlp.experts.151.gate_proj", + "model.layers.13.mlp.experts.151.up_proj", + "model.layers.13.mlp.experts.152.down_proj", + "model.layers.13.mlp.experts.152.gate_proj", + "model.layers.13.mlp.experts.152.up_proj", + "model.layers.13.mlp.experts.153.down_proj", + "model.layers.13.mlp.experts.153.gate_proj", + "model.layers.13.mlp.experts.153.up_proj", + "model.layers.13.mlp.experts.154.down_proj", + "model.layers.13.mlp.experts.154.gate_proj", + "model.layers.13.mlp.experts.154.up_proj", + "model.layers.13.mlp.experts.155.down_proj", + "model.layers.13.mlp.experts.155.gate_proj", + "model.layers.13.mlp.experts.155.up_proj", + "model.layers.13.mlp.experts.156.down_proj", + "model.layers.13.mlp.experts.156.gate_proj", + "model.layers.13.mlp.experts.156.up_proj", + "model.layers.13.mlp.experts.157.down_proj", + "model.layers.13.mlp.experts.157.gate_proj", + "model.layers.13.mlp.experts.157.up_proj", + "model.layers.13.mlp.experts.158.down_proj", + "model.layers.13.mlp.experts.158.gate_proj", + "model.layers.13.mlp.experts.158.up_proj", + "model.layers.13.mlp.experts.159.down_proj", + "model.layers.13.mlp.experts.159.gate_proj", + "model.layers.13.mlp.experts.159.up_proj", + "model.layers.13.mlp.experts.16.down_proj", + "model.layers.13.mlp.experts.16.gate_proj", + "model.layers.13.mlp.experts.16.up_proj", + "model.layers.13.mlp.experts.160.down_proj", + "model.layers.13.mlp.experts.160.gate_proj", + "model.layers.13.mlp.experts.160.up_proj", + "model.layers.13.mlp.experts.161.down_proj", + "model.layers.13.mlp.experts.161.gate_proj", + "model.layers.13.mlp.experts.161.up_proj", + "model.layers.13.mlp.experts.162.down_proj", + "model.layers.13.mlp.experts.162.gate_proj", + "model.layers.13.mlp.experts.162.up_proj", + "model.layers.13.mlp.experts.163.down_proj", + "model.layers.13.mlp.experts.163.gate_proj", + "model.layers.13.mlp.experts.163.up_proj", + "model.layers.13.mlp.experts.164.down_proj", + "model.layers.13.mlp.experts.164.gate_proj", + "model.layers.13.mlp.experts.164.up_proj", + "model.layers.13.mlp.experts.165.down_proj", + "model.layers.13.mlp.experts.165.gate_proj", + "model.layers.13.mlp.experts.165.up_proj", + "model.layers.13.mlp.experts.166.down_proj", + "model.layers.13.mlp.experts.166.gate_proj", + "model.layers.13.mlp.experts.166.up_proj", + "model.layers.13.mlp.experts.167.down_proj", + "model.layers.13.mlp.experts.167.gate_proj", + "model.layers.13.mlp.experts.167.up_proj", + "model.layers.13.mlp.experts.168.down_proj", + "model.layers.13.mlp.experts.168.gate_proj", + "model.layers.13.mlp.experts.168.up_proj", + "model.layers.13.mlp.experts.169.down_proj", + "model.layers.13.mlp.experts.169.gate_proj", + "model.layers.13.mlp.experts.169.up_proj", + "model.layers.13.mlp.experts.17.down_proj", + "model.layers.13.mlp.experts.17.gate_proj", + "model.layers.13.mlp.experts.17.up_proj", + "model.layers.13.mlp.experts.170.down_proj", + "model.layers.13.mlp.experts.170.gate_proj", + "model.layers.13.mlp.experts.170.up_proj", + "model.layers.13.mlp.experts.171.down_proj", + "model.layers.13.mlp.experts.171.gate_proj", + "model.layers.13.mlp.experts.171.up_proj", + "model.layers.13.mlp.experts.172.down_proj", + "model.layers.13.mlp.experts.172.gate_proj", + "model.layers.13.mlp.experts.172.up_proj", + "model.layers.13.mlp.experts.173.down_proj", + "model.layers.13.mlp.experts.173.gate_proj", + "model.layers.13.mlp.experts.173.up_proj", + "model.layers.13.mlp.experts.174.down_proj", + "model.layers.13.mlp.experts.174.gate_proj", + "model.layers.13.mlp.experts.174.up_proj", + "model.layers.13.mlp.experts.175.down_proj", + "model.layers.13.mlp.experts.175.gate_proj", + "model.layers.13.mlp.experts.175.up_proj", + "model.layers.13.mlp.experts.176.down_proj", + "model.layers.13.mlp.experts.176.gate_proj", + "model.layers.13.mlp.experts.176.up_proj", + "model.layers.13.mlp.experts.177.down_proj", + "model.layers.13.mlp.experts.177.gate_proj", + "model.layers.13.mlp.experts.177.up_proj", + "model.layers.13.mlp.experts.178.down_proj", + "model.layers.13.mlp.experts.178.gate_proj", + "model.layers.13.mlp.experts.178.up_proj", + "model.layers.13.mlp.experts.179.down_proj", + "model.layers.13.mlp.experts.179.gate_proj", + "model.layers.13.mlp.experts.179.up_proj", + "model.layers.13.mlp.experts.18.down_proj", + "model.layers.13.mlp.experts.18.gate_proj", + "model.layers.13.mlp.experts.18.up_proj", + "model.layers.13.mlp.experts.180.down_proj", + "model.layers.13.mlp.experts.180.gate_proj", + "model.layers.13.mlp.experts.180.up_proj", + "model.layers.13.mlp.experts.181.down_proj", + "model.layers.13.mlp.experts.181.gate_proj", + "model.layers.13.mlp.experts.181.up_proj", + "model.layers.13.mlp.experts.182.down_proj", + "model.layers.13.mlp.experts.182.gate_proj", + "model.layers.13.mlp.experts.182.up_proj", + "model.layers.13.mlp.experts.183.down_proj", + "model.layers.13.mlp.experts.183.gate_proj", + "model.layers.13.mlp.experts.183.up_proj", + "model.layers.13.mlp.experts.184.down_proj", + "model.layers.13.mlp.experts.184.gate_proj", + "model.layers.13.mlp.experts.184.up_proj", + "model.layers.13.mlp.experts.185.down_proj", + "model.layers.13.mlp.experts.185.gate_proj", + "model.layers.13.mlp.experts.185.up_proj", + "model.layers.13.mlp.experts.186.down_proj", + "model.layers.13.mlp.experts.186.gate_proj", + "model.layers.13.mlp.experts.186.up_proj", + "model.layers.13.mlp.experts.187.down_proj", + "model.layers.13.mlp.experts.187.gate_proj", + "model.layers.13.mlp.experts.187.up_proj", + "model.layers.13.mlp.experts.188.down_proj", + "model.layers.13.mlp.experts.188.gate_proj", + "model.layers.13.mlp.experts.188.up_proj", + "model.layers.13.mlp.experts.189.down_proj", + "model.layers.13.mlp.experts.189.gate_proj", + "model.layers.13.mlp.experts.189.up_proj", + "model.layers.13.mlp.experts.19.down_proj", + "model.layers.13.mlp.experts.19.gate_proj", + "model.layers.13.mlp.experts.19.up_proj", + "model.layers.13.mlp.experts.190.down_proj", + "model.layers.13.mlp.experts.190.gate_proj", + "model.layers.13.mlp.experts.190.up_proj", + "model.layers.13.mlp.experts.191.down_proj", + "model.layers.13.mlp.experts.191.gate_proj", + "model.layers.13.mlp.experts.191.up_proj", + "model.layers.13.mlp.experts.192.down_proj", + "model.layers.13.mlp.experts.192.gate_proj", + "model.layers.13.mlp.experts.192.up_proj", + "model.layers.13.mlp.experts.193.down_proj", + "model.layers.13.mlp.experts.193.gate_proj", + "model.layers.13.mlp.experts.193.up_proj", + "model.layers.13.mlp.experts.194.down_proj", + "model.layers.13.mlp.experts.194.gate_proj", + "model.layers.13.mlp.experts.194.up_proj", + "model.layers.13.mlp.experts.195.down_proj", + "model.layers.13.mlp.experts.195.gate_proj", + "model.layers.13.mlp.experts.195.up_proj", + "model.layers.13.mlp.experts.196.down_proj", + "model.layers.13.mlp.experts.196.gate_proj", + "model.layers.13.mlp.experts.196.up_proj", + "model.layers.13.mlp.experts.197.down_proj", + "model.layers.13.mlp.experts.197.gate_proj", + "model.layers.13.mlp.experts.197.up_proj", + "model.layers.13.mlp.experts.198.down_proj", + "model.layers.13.mlp.experts.198.gate_proj", + "model.layers.13.mlp.experts.198.up_proj", + "model.layers.13.mlp.experts.199.down_proj", + "model.layers.13.mlp.experts.199.gate_proj", + "model.layers.13.mlp.experts.199.up_proj", + "model.layers.13.mlp.experts.2.down_proj", + "model.layers.13.mlp.experts.2.gate_proj", + "model.layers.13.mlp.experts.2.up_proj", + "model.layers.13.mlp.experts.20.down_proj", + "model.layers.13.mlp.experts.20.gate_proj", + "model.layers.13.mlp.experts.20.up_proj", + "model.layers.13.mlp.experts.200.down_proj", + "model.layers.13.mlp.experts.200.gate_proj", + "model.layers.13.mlp.experts.200.up_proj", + "model.layers.13.mlp.experts.201.down_proj", + "model.layers.13.mlp.experts.201.gate_proj", + "model.layers.13.mlp.experts.201.up_proj", + "model.layers.13.mlp.experts.202.down_proj", + "model.layers.13.mlp.experts.202.gate_proj", + "model.layers.13.mlp.experts.202.up_proj", + "model.layers.13.mlp.experts.203.down_proj", + "model.layers.13.mlp.experts.203.gate_proj", + "model.layers.13.mlp.experts.203.up_proj", + "model.layers.13.mlp.experts.204.down_proj", + "model.layers.13.mlp.experts.204.gate_proj", + "model.layers.13.mlp.experts.204.up_proj", + "model.layers.13.mlp.experts.205.down_proj", + "model.layers.13.mlp.experts.205.gate_proj", + "model.layers.13.mlp.experts.205.up_proj", + "model.layers.13.mlp.experts.206.down_proj", + "model.layers.13.mlp.experts.206.gate_proj", + "model.layers.13.mlp.experts.206.up_proj", + "model.layers.13.mlp.experts.207.down_proj", + "model.layers.13.mlp.experts.207.gate_proj", + "model.layers.13.mlp.experts.207.up_proj", + "model.layers.13.mlp.experts.208.down_proj", + "model.layers.13.mlp.experts.208.gate_proj", + "model.layers.13.mlp.experts.208.up_proj", + "model.layers.13.mlp.experts.209.down_proj", + "model.layers.13.mlp.experts.209.gate_proj", + "model.layers.13.mlp.experts.209.up_proj", + "model.layers.13.mlp.experts.21.down_proj", + "model.layers.13.mlp.experts.21.gate_proj", + "model.layers.13.mlp.experts.21.up_proj", + "model.layers.13.mlp.experts.210.down_proj", + "model.layers.13.mlp.experts.210.gate_proj", + "model.layers.13.mlp.experts.210.up_proj", + "model.layers.13.mlp.experts.211.down_proj", + "model.layers.13.mlp.experts.211.gate_proj", + "model.layers.13.mlp.experts.211.up_proj", + "model.layers.13.mlp.experts.212.down_proj", + "model.layers.13.mlp.experts.212.gate_proj", + "model.layers.13.mlp.experts.212.up_proj", + "model.layers.13.mlp.experts.213.down_proj", + "model.layers.13.mlp.experts.213.gate_proj", + "model.layers.13.mlp.experts.213.up_proj", + "model.layers.13.mlp.experts.214.down_proj", + "model.layers.13.mlp.experts.214.gate_proj", + "model.layers.13.mlp.experts.214.up_proj", + "model.layers.13.mlp.experts.215.down_proj", + "model.layers.13.mlp.experts.215.gate_proj", + "model.layers.13.mlp.experts.215.up_proj", + "model.layers.13.mlp.experts.216.down_proj", + "model.layers.13.mlp.experts.216.gate_proj", + "model.layers.13.mlp.experts.216.up_proj", + "model.layers.13.mlp.experts.217.down_proj", + "model.layers.13.mlp.experts.217.gate_proj", + "model.layers.13.mlp.experts.217.up_proj", + "model.layers.13.mlp.experts.218.down_proj", + "model.layers.13.mlp.experts.218.gate_proj", + "model.layers.13.mlp.experts.218.up_proj", + "model.layers.13.mlp.experts.219.down_proj", + "model.layers.13.mlp.experts.219.gate_proj", + "model.layers.13.mlp.experts.219.up_proj", + "model.layers.13.mlp.experts.22.down_proj", + "model.layers.13.mlp.experts.22.gate_proj", + "model.layers.13.mlp.experts.22.up_proj", + "model.layers.13.mlp.experts.220.down_proj", + "model.layers.13.mlp.experts.220.gate_proj", + "model.layers.13.mlp.experts.220.up_proj", + "model.layers.13.mlp.experts.221.down_proj", + "model.layers.13.mlp.experts.221.gate_proj", + "model.layers.13.mlp.experts.221.up_proj", + "model.layers.13.mlp.experts.222.down_proj", + "model.layers.13.mlp.experts.222.gate_proj", + "model.layers.13.mlp.experts.222.up_proj", + "model.layers.13.mlp.experts.223.down_proj", + "model.layers.13.mlp.experts.223.gate_proj", + "model.layers.13.mlp.experts.223.up_proj", + "model.layers.13.mlp.experts.224.down_proj", + "model.layers.13.mlp.experts.224.gate_proj", + "model.layers.13.mlp.experts.224.up_proj", + "model.layers.13.mlp.experts.225.down_proj", + "model.layers.13.mlp.experts.225.gate_proj", + "model.layers.13.mlp.experts.225.up_proj", + "model.layers.13.mlp.experts.226.down_proj", + "model.layers.13.mlp.experts.226.gate_proj", + "model.layers.13.mlp.experts.226.up_proj", + "model.layers.13.mlp.experts.227.down_proj", + "model.layers.13.mlp.experts.227.gate_proj", + "model.layers.13.mlp.experts.227.up_proj", + "model.layers.13.mlp.experts.228.down_proj", + "model.layers.13.mlp.experts.228.gate_proj", + "model.layers.13.mlp.experts.228.up_proj", + "model.layers.13.mlp.experts.229.down_proj", + "model.layers.13.mlp.experts.229.gate_proj", + "model.layers.13.mlp.experts.229.up_proj", + "model.layers.13.mlp.experts.23.down_proj", + "model.layers.13.mlp.experts.23.gate_proj", + "model.layers.13.mlp.experts.23.up_proj", + "model.layers.13.mlp.experts.230.down_proj", + "model.layers.13.mlp.experts.230.gate_proj", + "model.layers.13.mlp.experts.230.up_proj", + "model.layers.13.mlp.experts.231.down_proj", + "model.layers.13.mlp.experts.231.gate_proj", + "model.layers.13.mlp.experts.231.up_proj", + "model.layers.13.mlp.experts.232.down_proj", + "model.layers.13.mlp.experts.232.gate_proj", + "model.layers.13.mlp.experts.232.up_proj", + "model.layers.13.mlp.experts.233.down_proj", + "model.layers.13.mlp.experts.233.gate_proj", + "model.layers.13.mlp.experts.233.up_proj", + "model.layers.13.mlp.experts.234.down_proj", + "model.layers.13.mlp.experts.234.gate_proj", + "model.layers.13.mlp.experts.234.up_proj", + "model.layers.13.mlp.experts.235.down_proj", + "model.layers.13.mlp.experts.235.gate_proj", + "model.layers.13.mlp.experts.235.up_proj", + "model.layers.13.mlp.experts.236.down_proj", + "model.layers.13.mlp.experts.236.gate_proj", + "model.layers.13.mlp.experts.236.up_proj", + "model.layers.13.mlp.experts.237.down_proj", + "model.layers.13.mlp.experts.237.gate_proj", + "model.layers.13.mlp.experts.237.up_proj", + "model.layers.13.mlp.experts.238.down_proj", + "model.layers.13.mlp.experts.238.gate_proj", + "model.layers.13.mlp.experts.238.up_proj", + "model.layers.13.mlp.experts.239.down_proj", + "model.layers.13.mlp.experts.239.gate_proj", + "model.layers.13.mlp.experts.239.up_proj", + "model.layers.13.mlp.experts.24.down_proj", + "model.layers.13.mlp.experts.24.gate_proj", + "model.layers.13.mlp.experts.24.up_proj", + "model.layers.13.mlp.experts.240.down_proj", + "model.layers.13.mlp.experts.240.gate_proj", + "model.layers.13.mlp.experts.240.up_proj", + "model.layers.13.mlp.experts.241.down_proj", + "model.layers.13.mlp.experts.241.gate_proj", + "model.layers.13.mlp.experts.241.up_proj", + "model.layers.13.mlp.experts.242.down_proj", + "model.layers.13.mlp.experts.242.gate_proj", + "model.layers.13.mlp.experts.242.up_proj", + "model.layers.13.mlp.experts.243.down_proj", + "model.layers.13.mlp.experts.243.gate_proj", + "model.layers.13.mlp.experts.243.up_proj", + "model.layers.13.mlp.experts.244.down_proj", + "model.layers.13.mlp.experts.244.gate_proj", + "model.layers.13.mlp.experts.244.up_proj", + "model.layers.13.mlp.experts.245.down_proj", + "model.layers.13.mlp.experts.245.gate_proj", + "model.layers.13.mlp.experts.245.up_proj", + "model.layers.13.mlp.experts.246.down_proj", + "model.layers.13.mlp.experts.246.gate_proj", + "model.layers.13.mlp.experts.246.up_proj", + "model.layers.13.mlp.experts.247.down_proj", + "model.layers.13.mlp.experts.247.gate_proj", + "model.layers.13.mlp.experts.247.up_proj", + "model.layers.13.mlp.experts.248.down_proj", + "model.layers.13.mlp.experts.248.gate_proj", + "model.layers.13.mlp.experts.248.up_proj", + "model.layers.13.mlp.experts.249.down_proj", + "model.layers.13.mlp.experts.249.gate_proj", + "model.layers.13.mlp.experts.249.up_proj", + "model.layers.13.mlp.experts.25.down_proj", + "model.layers.13.mlp.experts.25.gate_proj", + "model.layers.13.mlp.experts.25.up_proj", + "model.layers.13.mlp.experts.250.down_proj", + "model.layers.13.mlp.experts.250.gate_proj", + "model.layers.13.mlp.experts.250.up_proj", + "model.layers.13.mlp.experts.251.down_proj", + "model.layers.13.mlp.experts.251.gate_proj", + "model.layers.13.mlp.experts.251.up_proj", + "model.layers.13.mlp.experts.252.down_proj", + "model.layers.13.mlp.experts.252.gate_proj", + "model.layers.13.mlp.experts.252.up_proj", + "model.layers.13.mlp.experts.253.down_proj", + "model.layers.13.mlp.experts.253.gate_proj", + "model.layers.13.mlp.experts.253.up_proj", + "model.layers.13.mlp.experts.254.down_proj", + "model.layers.13.mlp.experts.254.gate_proj", + "model.layers.13.mlp.experts.254.up_proj", + "model.layers.13.mlp.experts.255.down_proj", + "model.layers.13.mlp.experts.255.gate_proj", + "model.layers.13.mlp.experts.255.up_proj", + "model.layers.13.mlp.experts.26.down_proj", + "model.layers.13.mlp.experts.26.gate_proj", + "model.layers.13.mlp.experts.26.up_proj", + "model.layers.13.mlp.experts.27.down_proj", + "model.layers.13.mlp.experts.27.gate_proj", + "model.layers.13.mlp.experts.27.up_proj", + "model.layers.13.mlp.experts.28.down_proj", + "model.layers.13.mlp.experts.28.gate_proj", + "model.layers.13.mlp.experts.28.up_proj", + "model.layers.13.mlp.experts.29.down_proj", + "model.layers.13.mlp.experts.29.gate_proj", + "model.layers.13.mlp.experts.29.up_proj", + "model.layers.13.mlp.experts.3.down_proj", + "model.layers.13.mlp.experts.3.gate_proj", + "model.layers.13.mlp.experts.3.up_proj", + "model.layers.13.mlp.experts.30.down_proj", + "model.layers.13.mlp.experts.30.gate_proj", + "model.layers.13.mlp.experts.30.up_proj", + "model.layers.13.mlp.experts.31.down_proj", + "model.layers.13.mlp.experts.31.gate_proj", + "model.layers.13.mlp.experts.31.up_proj", + "model.layers.13.mlp.experts.32.down_proj", + "model.layers.13.mlp.experts.32.gate_proj", + "model.layers.13.mlp.experts.32.up_proj", + "model.layers.13.mlp.experts.33.down_proj", + "model.layers.13.mlp.experts.33.gate_proj", + "model.layers.13.mlp.experts.33.up_proj", + "model.layers.13.mlp.experts.34.down_proj", + "model.layers.13.mlp.experts.34.gate_proj", + "model.layers.13.mlp.experts.34.up_proj", + "model.layers.13.mlp.experts.35.down_proj", + "model.layers.13.mlp.experts.35.gate_proj", + "model.layers.13.mlp.experts.35.up_proj", + "model.layers.13.mlp.experts.36.down_proj", + "model.layers.13.mlp.experts.36.gate_proj", + "model.layers.13.mlp.experts.36.up_proj", + "model.layers.13.mlp.experts.37.down_proj", + "model.layers.13.mlp.experts.37.gate_proj", + "model.layers.13.mlp.experts.37.up_proj", + "model.layers.13.mlp.experts.38.down_proj", + "model.layers.13.mlp.experts.38.gate_proj", + "model.layers.13.mlp.experts.38.up_proj", + "model.layers.13.mlp.experts.39.down_proj", + "model.layers.13.mlp.experts.39.gate_proj", + "model.layers.13.mlp.experts.39.up_proj", + "model.layers.13.mlp.experts.4.down_proj", + "model.layers.13.mlp.experts.4.gate_proj", + "model.layers.13.mlp.experts.4.up_proj", + "model.layers.13.mlp.experts.40.down_proj", + "model.layers.13.mlp.experts.40.gate_proj", + "model.layers.13.mlp.experts.40.up_proj", + "model.layers.13.mlp.experts.41.down_proj", + "model.layers.13.mlp.experts.41.gate_proj", + "model.layers.13.mlp.experts.41.up_proj", + "model.layers.13.mlp.experts.42.down_proj", + "model.layers.13.mlp.experts.42.gate_proj", + "model.layers.13.mlp.experts.42.up_proj", + "model.layers.13.mlp.experts.43.down_proj", + "model.layers.13.mlp.experts.43.gate_proj", + "model.layers.13.mlp.experts.43.up_proj", + "model.layers.13.mlp.experts.44.down_proj", + "model.layers.13.mlp.experts.44.gate_proj", + "model.layers.13.mlp.experts.44.up_proj", + "model.layers.13.mlp.experts.45.down_proj", + "model.layers.13.mlp.experts.45.gate_proj", + "model.layers.13.mlp.experts.45.up_proj", + "model.layers.13.mlp.experts.46.down_proj", + "model.layers.13.mlp.experts.46.gate_proj", + "model.layers.13.mlp.experts.46.up_proj", + "model.layers.13.mlp.experts.47.down_proj", + "model.layers.13.mlp.experts.47.gate_proj", + "model.layers.13.mlp.experts.47.up_proj", + "model.layers.13.mlp.experts.48.down_proj", + "model.layers.13.mlp.experts.48.gate_proj", + "model.layers.13.mlp.experts.48.up_proj", + "model.layers.13.mlp.experts.49.down_proj", + "model.layers.13.mlp.experts.49.gate_proj", + "model.layers.13.mlp.experts.49.up_proj", + "model.layers.13.mlp.experts.5.down_proj", + "model.layers.13.mlp.experts.5.gate_proj", + "model.layers.13.mlp.experts.5.up_proj", + "model.layers.13.mlp.experts.50.down_proj", + "model.layers.13.mlp.experts.50.gate_proj", + "model.layers.13.mlp.experts.50.up_proj", + "model.layers.13.mlp.experts.51.down_proj", + "model.layers.13.mlp.experts.51.gate_proj", + "model.layers.13.mlp.experts.51.up_proj", + "model.layers.13.mlp.experts.52.down_proj", + "model.layers.13.mlp.experts.52.gate_proj", + "model.layers.13.mlp.experts.52.up_proj", + "model.layers.13.mlp.experts.53.down_proj", + "model.layers.13.mlp.experts.53.gate_proj", + "model.layers.13.mlp.experts.53.up_proj", + "model.layers.13.mlp.experts.54.down_proj", + "model.layers.13.mlp.experts.54.gate_proj", + "model.layers.13.mlp.experts.54.up_proj", + "model.layers.13.mlp.experts.55.down_proj", + "model.layers.13.mlp.experts.55.gate_proj", + "model.layers.13.mlp.experts.55.up_proj", + "model.layers.13.mlp.experts.56.down_proj", + "model.layers.13.mlp.experts.56.gate_proj", + "model.layers.13.mlp.experts.56.up_proj", + "model.layers.13.mlp.experts.57.down_proj", + "model.layers.13.mlp.experts.57.gate_proj", + "model.layers.13.mlp.experts.57.up_proj", + "model.layers.13.mlp.experts.58.down_proj", + "model.layers.13.mlp.experts.58.gate_proj", + "model.layers.13.mlp.experts.58.up_proj", + "model.layers.13.mlp.experts.59.down_proj", + "model.layers.13.mlp.experts.59.gate_proj", + "model.layers.13.mlp.experts.59.up_proj", + "model.layers.13.mlp.experts.6.down_proj", + "model.layers.13.mlp.experts.6.gate_proj", + "model.layers.13.mlp.experts.6.up_proj", + "model.layers.13.mlp.experts.60.down_proj", + "model.layers.13.mlp.experts.60.gate_proj", + "model.layers.13.mlp.experts.60.up_proj", + "model.layers.13.mlp.experts.61.down_proj", + "model.layers.13.mlp.experts.61.gate_proj", + "model.layers.13.mlp.experts.61.up_proj", + "model.layers.13.mlp.experts.62.down_proj", + "model.layers.13.mlp.experts.62.gate_proj", + "model.layers.13.mlp.experts.62.up_proj", + "model.layers.13.mlp.experts.63.down_proj", + "model.layers.13.mlp.experts.63.gate_proj", + "model.layers.13.mlp.experts.63.up_proj", + "model.layers.13.mlp.experts.64.down_proj", + "model.layers.13.mlp.experts.64.gate_proj", + "model.layers.13.mlp.experts.64.up_proj", + "model.layers.13.mlp.experts.65.down_proj", + "model.layers.13.mlp.experts.65.gate_proj", + "model.layers.13.mlp.experts.65.up_proj", + "model.layers.13.mlp.experts.66.down_proj", + "model.layers.13.mlp.experts.66.gate_proj", + "model.layers.13.mlp.experts.66.up_proj", + "model.layers.13.mlp.experts.67.down_proj", + "model.layers.13.mlp.experts.67.gate_proj", + "model.layers.13.mlp.experts.67.up_proj", + "model.layers.13.mlp.experts.68.down_proj", + "model.layers.13.mlp.experts.68.gate_proj", + "model.layers.13.mlp.experts.68.up_proj", + "model.layers.13.mlp.experts.69.down_proj", + "model.layers.13.mlp.experts.69.gate_proj", + "model.layers.13.mlp.experts.69.up_proj", + "model.layers.13.mlp.experts.7.down_proj", + "model.layers.13.mlp.experts.7.gate_proj", + "model.layers.13.mlp.experts.7.up_proj", + "model.layers.13.mlp.experts.70.down_proj", + "model.layers.13.mlp.experts.70.gate_proj", + "model.layers.13.mlp.experts.70.up_proj", + "model.layers.13.mlp.experts.71.down_proj", + "model.layers.13.mlp.experts.71.gate_proj", + "model.layers.13.mlp.experts.71.up_proj", + "model.layers.13.mlp.experts.72.down_proj", + "model.layers.13.mlp.experts.72.gate_proj", + "model.layers.13.mlp.experts.72.up_proj", + "model.layers.13.mlp.experts.73.down_proj", + "model.layers.13.mlp.experts.73.gate_proj", + "model.layers.13.mlp.experts.73.up_proj", + "model.layers.13.mlp.experts.74.down_proj", + "model.layers.13.mlp.experts.74.gate_proj", + "model.layers.13.mlp.experts.74.up_proj", + "model.layers.13.mlp.experts.75.down_proj", + "model.layers.13.mlp.experts.75.gate_proj", + "model.layers.13.mlp.experts.75.up_proj", + "model.layers.13.mlp.experts.76.down_proj", + "model.layers.13.mlp.experts.76.gate_proj", + "model.layers.13.mlp.experts.76.up_proj", + "model.layers.13.mlp.experts.77.down_proj", + "model.layers.13.mlp.experts.77.gate_proj", + "model.layers.13.mlp.experts.77.up_proj", + "model.layers.13.mlp.experts.78.down_proj", + "model.layers.13.mlp.experts.78.gate_proj", + "model.layers.13.mlp.experts.78.up_proj", + "model.layers.13.mlp.experts.79.down_proj", + "model.layers.13.mlp.experts.79.gate_proj", + "model.layers.13.mlp.experts.79.up_proj", + "model.layers.13.mlp.experts.8.down_proj", + "model.layers.13.mlp.experts.8.gate_proj", + "model.layers.13.mlp.experts.8.up_proj", + "model.layers.13.mlp.experts.80.down_proj", + "model.layers.13.mlp.experts.80.gate_proj", + "model.layers.13.mlp.experts.80.up_proj", + "model.layers.13.mlp.experts.81.down_proj", + "model.layers.13.mlp.experts.81.gate_proj", + "model.layers.13.mlp.experts.81.up_proj", + "model.layers.13.mlp.experts.82.down_proj", + "model.layers.13.mlp.experts.82.gate_proj", + "model.layers.13.mlp.experts.82.up_proj", + "model.layers.13.mlp.experts.83.down_proj", + "model.layers.13.mlp.experts.83.gate_proj", + "model.layers.13.mlp.experts.83.up_proj", + "model.layers.13.mlp.experts.84.down_proj", + "model.layers.13.mlp.experts.84.gate_proj", + "model.layers.13.mlp.experts.84.up_proj", + "model.layers.13.mlp.experts.85.down_proj", + "model.layers.13.mlp.experts.85.gate_proj", + "model.layers.13.mlp.experts.85.up_proj", + "model.layers.13.mlp.experts.86.down_proj", + "model.layers.13.mlp.experts.86.gate_proj", + "model.layers.13.mlp.experts.86.up_proj", + "model.layers.13.mlp.experts.87.down_proj", + "model.layers.13.mlp.experts.87.gate_proj", + "model.layers.13.mlp.experts.87.up_proj", + "model.layers.13.mlp.experts.88.down_proj", + "model.layers.13.mlp.experts.88.gate_proj", + "model.layers.13.mlp.experts.88.up_proj", + "model.layers.13.mlp.experts.89.down_proj", + "model.layers.13.mlp.experts.89.gate_proj", + "model.layers.13.mlp.experts.89.up_proj", + "model.layers.13.mlp.experts.9.down_proj", + "model.layers.13.mlp.experts.9.gate_proj", + "model.layers.13.mlp.experts.9.up_proj", + "model.layers.13.mlp.experts.90.down_proj", + "model.layers.13.mlp.experts.90.gate_proj", + "model.layers.13.mlp.experts.90.up_proj", + "model.layers.13.mlp.experts.91.down_proj", + "model.layers.13.mlp.experts.91.gate_proj", + "model.layers.13.mlp.experts.91.up_proj", + "model.layers.13.mlp.experts.92.down_proj", + "model.layers.13.mlp.experts.92.gate_proj", + "model.layers.13.mlp.experts.92.up_proj", + "model.layers.13.mlp.experts.93.down_proj", + "model.layers.13.mlp.experts.93.gate_proj", + "model.layers.13.mlp.experts.93.up_proj", + "model.layers.13.mlp.experts.94.down_proj", + "model.layers.13.mlp.experts.94.gate_proj", + "model.layers.13.mlp.experts.94.up_proj", + "model.layers.13.mlp.experts.95.down_proj", + "model.layers.13.mlp.experts.95.gate_proj", + "model.layers.13.mlp.experts.95.up_proj", + "model.layers.13.mlp.experts.96.down_proj", + "model.layers.13.mlp.experts.96.gate_proj", + "model.layers.13.mlp.experts.96.up_proj", + "model.layers.13.mlp.experts.97.down_proj", + "model.layers.13.mlp.experts.97.gate_proj", + "model.layers.13.mlp.experts.97.up_proj", + "model.layers.13.mlp.experts.98.down_proj", + "model.layers.13.mlp.experts.98.gate_proj", + "model.layers.13.mlp.experts.98.up_proj", + "model.layers.13.mlp.experts.99.down_proj", + "model.layers.13.mlp.experts.99.gate_proj", + "model.layers.13.mlp.experts.99.up_proj", + "model.layers.14.mlp.experts.0.down_proj", + "model.layers.14.mlp.experts.0.gate_proj", + "model.layers.14.mlp.experts.0.up_proj", + "model.layers.14.mlp.experts.1.down_proj", + "model.layers.14.mlp.experts.1.gate_proj", + "model.layers.14.mlp.experts.1.up_proj", + "model.layers.14.mlp.experts.10.down_proj", + "model.layers.14.mlp.experts.10.gate_proj", + "model.layers.14.mlp.experts.10.up_proj", + "model.layers.14.mlp.experts.100.down_proj", + "model.layers.14.mlp.experts.100.gate_proj", + "model.layers.14.mlp.experts.100.up_proj", + "model.layers.14.mlp.experts.101.down_proj", + "model.layers.14.mlp.experts.101.gate_proj", + "model.layers.14.mlp.experts.101.up_proj", + "model.layers.14.mlp.experts.102.down_proj", + "model.layers.14.mlp.experts.102.gate_proj", + "model.layers.14.mlp.experts.102.up_proj", + "model.layers.14.mlp.experts.103.down_proj", + "model.layers.14.mlp.experts.103.gate_proj", + "model.layers.14.mlp.experts.103.up_proj", + "model.layers.14.mlp.experts.104.down_proj", + "model.layers.14.mlp.experts.104.gate_proj", + "model.layers.14.mlp.experts.104.up_proj", + "model.layers.14.mlp.experts.105.down_proj", + "model.layers.14.mlp.experts.105.gate_proj", + "model.layers.14.mlp.experts.105.up_proj", + "model.layers.14.mlp.experts.106.down_proj", + "model.layers.14.mlp.experts.106.gate_proj", + "model.layers.14.mlp.experts.106.up_proj", + "model.layers.14.mlp.experts.107.down_proj", + "model.layers.14.mlp.experts.107.gate_proj", + "model.layers.14.mlp.experts.107.up_proj", + "model.layers.14.mlp.experts.108.down_proj", + "model.layers.14.mlp.experts.108.gate_proj", + "model.layers.14.mlp.experts.108.up_proj", + "model.layers.14.mlp.experts.109.down_proj", + "model.layers.14.mlp.experts.109.gate_proj", + "model.layers.14.mlp.experts.109.up_proj", + "model.layers.14.mlp.experts.11.down_proj", + "model.layers.14.mlp.experts.11.gate_proj", + "model.layers.14.mlp.experts.11.up_proj", + "model.layers.14.mlp.experts.110.down_proj", + "model.layers.14.mlp.experts.110.gate_proj", + "model.layers.14.mlp.experts.110.up_proj", + "model.layers.14.mlp.experts.111.down_proj", + "model.layers.14.mlp.experts.111.gate_proj", + "model.layers.14.mlp.experts.111.up_proj", + "model.layers.14.mlp.experts.112.down_proj", + "model.layers.14.mlp.experts.112.gate_proj", + "model.layers.14.mlp.experts.112.up_proj", + "model.layers.14.mlp.experts.113.down_proj", + "model.layers.14.mlp.experts.113.gate_proj", + "model.layers.14.mlp.experts.113.up_proj", + "model.layers.14.mlp.experts.114.down_proj", + "model.layers.14.mlp.experts.114.gate_proj", + "model.layers.14.mlp.experts.114.up_proj", + "model.layers.14.mlp.experts.115.down_proj", + "model.layers.14.mlp.experts.115.gate_proj", + "model.layers.14.mlp.experts.115.up_proj", + "model.layers.14.mlp.experts.116.down_proj", + "model.layers.14.mlp.experts.116.gate_proj", + "model.layers.14.mlp.experts.116.up_proj", + "model.layers.14.mlp.experts.117.down_proj", + "model.layers.14.mlp.experts.117.gate_proj", + "model.layers.14.mlp.experts.117.up_proj", + "model.layers.14.mlp.experts.118.down_proj", + "model.layers.14.mlp.experts.118.gate_proj", + "model.layers.14.mlp.experts.118.up_proj", + "model.layers.14.mlp.experts.119.down_proj", + "model.layers.14.mlp.experts.119.gate_proj", + "model.layers.14.mlp.experts.119.up_proj", + "model.layers.14.mlp.experts.12.down_proj", + "model.layers.14.mlp.experts.12.gate_proj", + "model.layers.14.mlp.experts.12.up_proj", + "model.layers.14.mlp.experts.120.down_proj", + "model.layers.14.mlp.experts.120.gate_proj", + "model.layers.14.mlp.experts.120.up_proj", + "model.layers.14.mlp.experts.121.down_proj", + "model.layers.14.mlp.experts.121.gate_proj", + "model.layers.14.mlp.experts.121.up_proj", + "model.layers.14.mlp.experts.122.down_proj", + "model.layers.14.mlp.experts.122.gate_proj", + "model.layers.14.mlp.experts.122.up_proj", + "model.layers.14.mlp.experts.123.down_proj", + "model.layers.14.mlp.experts.123.gate_proj", + "model.layers.14.mlp.experts.123.up_proj", + "model.layers.14.mlp.experts.124.down_proj", + "model.layers.14.mlp.experts.124.gate_proj", + "model.layers.14.mlp.experts.124.up_proj", + "model.layers.14.mlp.experts.125.down_proj", + "model.layers.14.mlp.experts.125.gate_proj", + "model.layers.14.mlp.experts.125.up_proj", + "model.layers.14.mlp.experts.126.down_proj", + "model.layers.14.mlp.experts.126.gate_proj", + "model.layers.14.mlp.experts.126.up_proj", + "model.layers.14.mlp.experts.127.down_proj", + "model.layers.14.mlp.experts.127.gate_proj", + "model.layers.14.mlp.experts.127.up_proj", + "model.layers.14.mlp.experts.128.down_proj", + "model.layers.14.mlp.experts.128.gate_proj", + "model.layers.14.mlp.experts.128.up_proj", + "model.layers.14.mlp.experts.129.down_proj", + "model.layers.14.mlp.experts.129.gate_proj", + "model.layers.14.mlp.experts.129.up_proj", + "model.layers.14.mlp.experts.13.down_proj", + "model.layers.14.mlp.experts.13.gate_proj", + "model.layers.14.mlp.experts.13.up_proj", + "model.layers.14.mlp.experts.130.down_proj", + "model.layers.14.mlp.experts.130.gate_proj", + "model.layers.14.mlp.experts.130.up_proj", + "model.layers.14.mlp.experts.131.down_proj", + "model.layers.14.mlp.experts.131.gate_proj", + "model.layers.14.mlp.experts.131.up_proj", + "model.layers.14.mlp.experts.132.down_proj", + "model.layers.14.mlp.experts.132.gate_proj", + "model.layers.14.mlp.experts.132.up_proj", + "model.layers.14.mlp.experts.133.down_proj", + "model.layers.14.mlp.experts.133.gate_proj", + "model.layers.14.mlp.experts.133.up_proj", + "model.layers.14.mlp.experts.134.down_proj", + "model.layers.14.mlp.experts.134.gate_proj", + "model.layers.14.mlp.experts.134.up_proj", + "model.layers.14.mlp.experts.135.down_proj", + "model.layers.14.mlp.experts.135.gate_proj", + "model.layers.14.mlp.experts.135.up_proj", + "model.layers.14.mlp.experts.136.down_proj", + "model.layers.14.mlp.experts.136.gate_proj", + "model.layers.14.mlp.experts.136.up_proj", + "model.layers.14.mlp.experts.137.down_proj", + "model.layers.14.mlp.experts.137.gate_proj", + "model.layers.14.mlp.experts.137.up_proj", + "model.layers.14.mlp.experts.138.down_proj", + "model.layers.14.mlp.experts.138.gate_proj", + "model.layers.14.mlp.experts.138.up_proj", + "model.layers.14.mlp.experts.139.down_proj", + "model.layers.14.mlp.experts.139.gate_proj", + "model.layers.14.mlp.experts.139.up_proj", + "model.layers.14.mlp.experts.14.down_proj", + "model.layers.14.mlp.experts.14.gate_proj", + "model.layers.14.mlp.experts.14.up_proj", + "model.layers.14.mlp.experts.140.down_proj", + "model.layers.14.mlp.experts.140.gate_proj", + "model.layers.14.mlp.experts.140.up_proj", + "model.layers.14.mlp.experts.141.down_proj", + "model.layers.14.mlp.experts.141.gate_proj", + "model.layers.14.mlp.experts.141.up_proj", + "model.layers.14.mlp.experts.142.down_proj", + "model.layers.14.mlp.experts.142.gate_proj", + "model.layers.14.mlp.experts.142.up_proj", + "model.layers.14.mlp.experts.143.down_proj", + "model.layers.14.mlp.experts.143.gate_proj", + "model.layers.14.mlp.experts.143.up_proj", + "model.layers.14.mlp.experts.144.down_proj", + "model.layers.14.mlp.experts.144.gate_proj", + "model.layers.14.mlp.experts.144.up_proj", + "model.layers.14.mlp.experts.145.down_proj", + "model.layers.14.mlp.experts.145.gate_proj", + "model.layers.14.mlp.experts.145.up_proj", + "model.layers.14.mlp.experts.146.down_proj", + "model.layers.14.mlp.experts.146.gate_proj", + "model.layers.14.mlp.experts.146.up_proj", + "model.layers.14.mlp.experts.147.down_proj", + "model.layers.14.mlp.experts.147.gate_proj", + "model.layers.14.mlp.experts.147.up_proj", + "model.layers.14.mlp.experts.148.down_proj", + "model.layers.14.mlp.experts.148.gate_proj", + "model.layers.14.mlp.experts.148.up_proj", + "model.layers.14.mlp.experts.149.down_proj", + "model.layers.14.mlp.experts.149.gate_proj", + "model.layers.14.mlp.experts.149.up_proj", + "model.layers.14.mlp.experts.15.down_proj", + "model.layers.14.mlp.experts.15.gate_proj", + "model.layers.14.mlp.experts.15.up_proj", + "model.layers.14.mlp.experts.150.down_proj", + "model.layers.14.mlp.experts.150.gate_proj", + "model.layers.14.mlp.experts.150.up_proj", + "model.layers.14.mlp.experts.151.down_proj", + "model.layers.14.mlp.experts.151.gate_proj", + "model.layers.14.mlp.experts.151.up_proj", + "model.layers.14.mlp.experts.152.down_proj", + "model.layers.14.mlp.experts.152.gate_proj", + "model.layers.14.mlp.experts.152.up_proj", + "model.layers.14.mlp.experts.153.down_proj", + "model.layers.14.mlp.experts.153.gate_proj", + "model.layers.14.mlp.experts.153.up_proj", + "model.layers.14.mlp.experts.154.down_proj", + "model.layers.14.mlp.experts.154.gate_proj", + "model.layers.14.mlp.experts.154.up_proj", + "model.layers.14.mlp.experts.155.down_proj", + "model.layers.14.mlp.experts.155.gate_proj", + "model.layers.14.mlp.experts.155.up_proj", + "model.layers.14.mlp.experts.156.down_proj", + "model.layers.14.mlp.experts.156.gate_proj", + "model.layers.14.mlp.experts.156.up_proj", + "model.layers.14.mlp.experts.157.down_proj", + "model.layers.14.mlp.experts.157.gate_proj", + "model.layers.14.mlp.experts.157.up_proj", + "model.layers.14.mlp.experts.158.down_proj", + "model.layers.14.mlp.experts.158.gate_proj", + "model.layers.14.mlp.experts.158.up_proj", + "model.layers.14.mlp.experts.159.down_proj", + "model.layers.14.mlp.experts.159.gate_proj", + "model.layers.14.mlp.experts.159.up_proj", + "model.layers.14.mlp.experts.16.down_proj", + "model.layers.14.mlp.experts.16.gate_proj", + "model.layers.14.mlp.experts.16.up_proj", + "model.layers.14.mlp.experts.160.down_proj", + "model.layers.14.mlp.experts.160.gate_proj", + "model.layers.14.mlp.experts.160.up_proj", + "model.layers.14.mlp.experts.161.down_proj", + "model.layers.14.mlp.experts.161.gate_proj", + "model.layers.14.mlp.experts.161.up_proj", + "model.layers.14.mlp.experts.162.down_proj", + "model.layers.14.mlp.experts.162.gate_proj", + "model.layers.14.mlp.experts.162.up_proj", + "model.layers.14.mlp.experts.163.down_proj", + "model.layers.14.mlp.experts.163.gate_proj", + "model.layers.14.mlp.experts.163.up_proj", + "model.layers.14.mlp.experts.164.down_proj", + "model.layers.14.mlp.experts.164.gate_proj", + "model.layers.14.mlp.experts.164.up_proj", + "model.layers.14.mlp.experts.165.down_proj", + "model.layers.14.mlp.experts.165.gate_proj", + "model.layers.14.mlp.experts.165.up_proj", + "model.layers.14.mlp.experts.166.down_proj", + "model.layers.14.mlp.experts.166.gate_proj", + "model.layers.14.mlp.experts.166.up_proj", + "model.layers.14.mlp.experts.167.down_proj", + "model.layers.14.mlp.experts.167.gate_proj", + "model.layers.14.mlp.experts.167.up_proj", + "model.layers.14.mlp.experts.168.down_proj", + "model.layers.14.mlp.experts.168.gate_proj", + "model.layers.14.mlp.experts.168.up_proj", + "model.layers.14.mlp.experts.169.down_proj", + "model.layers.14.mlp.experts.169.gate_proj", + "model.layers.14.mlp.experts.169.up_proj", + "model.layers.14.mlp.experts.17.down_proj", + "model.layers.14.mlp.experts.17.gate_proj", + "model.layers.14.mlp.experts.17.up_proj", + "model.layers.14.mlp.experts.170.down_proj", + "model.layers.14.mlp.experts.170.gate_proj", + "model.layers.14.mlp.experts.170.up_proj", + "model.layers.14.mlp.experts.171.down_proj", + "model.layers.14.mlp.experts.171.gate_proj", + "model.layers.14.mlp.experts.171.up_proj", + "model.layers.14.mlp.experts.172.down_proj", + "model.layers.14.mlp.experts.172.gate_proj", + "model.layers.14.mlp.experts.172.up_proj", + "model.layers.14.mlp.experts.173.down_proj", + "model.layers.14.mlp.experts.173.gate_proj", + "model.layers.14.mlp.experts.173.up_proj", + "model.layers.14.mlp.experts.174.down_proj", + "model.layers.14.mlp.experts.174.gate_proj", + "model.layers.14.mlp.experts.174.up_proj", + "model.layers.14.mlp.experts.175.down_proj", + "model.layers.14.mlp.experts.175.gate_proj", + "model.layers.14.mlp.experts.175.up_proj", + "model.layers.14.mlp.experts.176.down_proj", + "model.layers.14.mlp.experts.176.gate_proj", + "model.layers.14.mlp.experts.176.up_proj", + "model.layers.14.mlp.experts.177.down_proj", + "model.layers.14.mlp.experts.177.gate_proj", + "model.layers.14.mlp.experts.177.up_proj", + "model.layers.14.mlp.experts.178.down_proj", + "model.layers.14.mlp.experts.178.gate_proj", + "model.layers.14.mlp.experts.178.up_proj", + "model.layers.14.mlp.experts.179.down_proj", + "model.layers.14.mlp.experts.179.gate_proj", + "model.layers.14.mlp.experts.179.up_proj", + "model.layers.14.mlp.experts.18.down_proj", + "model.layers.14.mlp.experts.18.gate_proj", + "model.layers.14.mlp.experts.18.up_proj", + "model.layers.14.mlp.experts.180.down_proj", + "model.layers.14.mlp.experts.180.gate_proj", + "model.layers.14.mlp.experts.180.up_proj", + "model.layers.14.mlp.experts.181.down_proj", + "model.layers.14.mlp.experts.181.gate_proj", + "model.layers.14.mlp.experts.181.up_proj", + "model.layers.14.mlp.experts.182.down_proj", + "model.layers.14.mlp.experts.182.gate_proj", + "model.layers.14.mlp.experts.182.up_proj", + "model.layers.14.mlp.experts.183.down_proj", + "model.layers.14.mlp.experts.183.gate_proj", + "model.layers.14.mlp.experts.183.up_proj", + "model.layers.14.mlp.experts.184.down_proj", + "model.layers.14.mlp.experts.184.gate_proj", + "model.layers.14.mlp.experts.184.up_proj", + "model.layers.14.mlp.experts.185.down_proj", + "model.layers.14.mlp.experts.185.gate_proj", + "model.layers.14.mlp.experts.185.up_proj", + "model.layers.14.mlp.experts.186.down_proj", + "model.layers.14.mlp.experts.186.gate_proj", + "model.layers.14.mlp.experts.186.up_proj", + "model.layers.14.mlp.experts.187.down_proj", + "model.layers.14.mlp.experts.187.gate_proj", + "model.layers.14.mlp.experts.187.up_proj", + "model.layers.14.mlp.experts.188.down_proj", + "model.layers.14.mlp.experts.188.gate_proj", + "model.layers.14.mlp.experts.188.up_proj", + "model.layers.14.mlp.experts.189.down_proj", + "model.layers.14.mlp.experts.189.gate_proj", + "model.layers.14.mlp.experts.189.up_proj", + "model.layers.14.mlp.experts.19.down_proj", + "model.layers.14.mlp.experts.19.gate_proj", + "model.layers.14.mlp.experts.19.up_proj", + "model.layers.14.mlp.experts.190.down_proj", + "model.layers.14.mlp.experts.190.gate_proj", + "model.layers.14.mlp.experts.190.up_proj", + "model.layers.14.mlp.experts.191.down_proj", + "model.layers.14.mlp.experts.191.gate_proj", + "model.layers.14.mlp.experts.191.up_proj", + "model.layers.14.mlp.experts.192.down_proj", + "model.layers.14.mlp.experts.192.gate_proj", + "model.layers.14.mlp.experts.192.up_proj", + "model.layers.14.mlp.experts.193.down_proj", + "model.layers.14.mlp.experts.193.gate_proj", + "model.layers.14.mlp.experts.193.up_proj", + "model.layers.14.mlp.experts.194.down_proj", + "model.layers.14.mlp.experts.194.gate_proj", + "model.layers.14.mlp.experts.194.up_proj", + "model.layers.14.mlp.experts.195.down_proj", + "model.layers.14.mlp.experts.195.gate_proj", + "model.layers.14.mlp.experts.195.up_proj", + "model.layers.14.mlp.experts.196.down_proj", + "model.layers.14.mlp.experts.196.gate_proj", + "model.layers.14.mlp.experts.196.up_proj", + "model.layers.14.mlp.experts.197.down_proj", + "model.layers.14.mlp.experts.197.gate_proj", + "model.layers.14.mlp.experts.197.up_proj", + "model.layers.14.mlp.experts.198.down_proj", + "model.layers.14.mlp.experts.198.gate_proj", + "model.layers.14.mlp.experts.198.up_proj", + "model.layers.14.mlp.experts.199.down_proj", + "model.layers.14.mlp.experts.199.gate_proj", + "model.layers.14.mlp.experts.199.up_proj", + "model.layers.14.mlp.experts.2.down_proj", + "model.layers.14.mlp.experts.2.gate_proj", + "model.layers.14.mlp.experts.2.up_proj", + "model.layers.14.mlp.experts.20.down_proj", + "model.layers.14.mlp.experts.20.gate_proj", + "model.layers.14.mlp.experts.20.up_proj", + "model.layers.14.mlp.experts.200.down_proj", + "model.layers.14.mlp.experts.200.gate_proj", + "model.layers.14.mlp.experts.200.up_proj", + "model.layers.14.mlp.experts.201.down_proj", + "model.layers.14.mlp.experts.201.gate_proj", + "model.layers.14.mlp.experts.201.up_proj", + "model.layers.14.mlp.experts.202.down_proj", + "model.layers.14.mlp.experts.202.gate_proj", + "model.layers.14.mlp.experts.202.up_proj", + "model.layers.14.mlp.experts.203.down_proj", + "model.layers.14.mlp.experts.203.gate_proj", + "model.layers.14.mlp.experts.203.up_proj", + "model.layers.14.mlp.experts.204.down_proj", + "model.layers.14.mlp.experts.204.gate_proj", + "model.layers.14.mlp.experts.204.up_proj", + "model.layers.14.mlp.experts.205.down_proj", + "model.layers.14.mlp.experts.205.gate_proj", + "model.layers.14.mlp.experts.205.up_proj", + "model.layers.14.mlp.experts.206.down_proj", + "model.layers.14.mlp.experts.206.gate_proj", + "model.layers.14.mlp.experts.206.up_proj", + "model.layers.14.mlp.experts.207.down_proj", + "model.layers.14.mlp.experts.207.gate_proj", + "model.layers.14.mlp.experts.207.up_proj", + "model.layers.14.mlp.experts.208.down_proj", + "model.layers.14.mlp.experts.208.gate_proj", + "model.layers.14.mlp.experts.208.up_proj", + "model.layers.14.mlp.experts.209.down_proj", + "model.layers.14.mlp.experts.209.gate_proj", + "model.layers.14.mlp.experts.209.up_proj", + "model.layers.14.mlp.experts.21.down_proj", + "model.layers.14.mlp.experts.21.gate_proj", + "model.layers.14.mlp.experts.21.up_proj", + "model.layers.14.mlp.experts.210.down_proj", + "model.layers.14.mlp.experts.210.gate_proj", + "model.layers.14.mlp.experts.210.up_proj", + "model.layers.14.mlp.experts.211.down_proj", + "model.layers.14.mlp.experts.211.gate_proj", + "model.layers.14.mlp.experts.211.up_proj", + "model.layers.14.mlp.experts.212.down_proj", + "model.layers.14.mlp.experts.212.gate_proj", + "model.layers.14.mlp.experts.212.up_proj", + "model.layers.14.mlp.experts.213.down_proj", + "model.layers.14.mlp.experts.213.gate_proj", + "model.layers.14.mlp.experts.213.up_proj", + "model.layers.14.mlp.experts.214.down_proj", + "model.layers.14.mlp.experts.214.gate_proj", + "model.layers.14.mlp.experts.214.up_proj", + "model.layers.14.mlp.experts.215.down_proj", + "model.layers.14.mlp.experts.215.gate_proj", + "model.layers.14.mlp.experts.215.up_proj", + "model.layers.14.mlp.experts.216.down_proj", + "model.layers.14.mlp.experts.216.gate_proj", + "model.layers.14.mlp.experts.216.up_proj", + "model.layers.14.mlp.experts.217.down_proj", + "model.layers.14.mlp.experts.217.gate_proj", + "model.layers.14.mlp.experts.217.up_proj", + "model.layers.14.mlp.experts.218.down_proj", + "model.layers.14.mlp.experts.218.gate_proj", + "model.layers.14.mlp.experts.218.up_proj", + "model.layers.14.mlp.experts.219.down_proj", + "model.layers.14.mlp.experts.219.gate_proj", + "model.layers.14.mlp.experts.219.up_proj", + "model.layers.14.mlp.experts.22.down_proj", + "model.layers.14.mlp.experts.22.gate_proj", + "model.layers.14.mlp.experts.22.up_proj", + "model.layers.14.mlp.experts.220.down_proj", + "model.layers.14.mlp.experts.220.gate_proj", + "model.layers.14.mlp.experts.220.up_proj", + "model.layers.14.mlp.experts.221.down_proj", + "model.layers.14.mlp.experts.221.gate_proj", + "model.layers.14.mlp.experts.221.up_proj", + "model.layers.14.mlp.experts.222.down_proj", + "model.layers.14.mlp.experts.222.gate_proj", + "model.layers.14.mlp.experts.222.up_proj", + "model.layers.14.mlp.experts.223.down_proj", + "model.layers.14.mlp.experts.223.gate_proj", + "model.layers.14.mlp.experts.223.up_proj", + "model.layers.14.mlp.experts.224.down_proj", + "model.layers.14.mlp.experts.224.gate_proj", + "model.layers.14.mlp.experts.224.up_proj", + "model.layers.14.mlp.experts.225.down_proj", + "model.layers.14.mlp.experts.225.gate_proj", + "model.layers.14.mlp.experts.225.up_proj", + "model.layers.14.mlp.experts.226.down_proj", + "model.layers.14.mlp.experts.226.gate_proj", + "model.layers.14.mlp.experts.226.up_proj", + "model.layers.14.mlp.experts.227.down_proj", + "model.layers.14.mlp.experts.227.gate_proj", + "model.layers.14.mlp.experts.227.up_proj", + "model.layers.14.mlp.experts.228.down_proj", + "model.layers.14.mlp.experts.228.gate_proj", + "model.layers.14.mlp.experts.228.up_proj", + "model.layers.14.mlp.experts.229.down_proj", + "model.layers.14.mlp.experts.229.gate_proj", + "model.layers.14.mlp.experts.229.up_proj", + "model.layers.14.mlp.experts.23.down_proj", + "model.layers.14.mlp.experts.23.gate_proj", + "model.layers.14.mlp.experts.23.up_proj", + "model.layers.14.mlp.experts.230.down_proj", + "model.layers.14.mlp.experts.230.gate_proj", + "model.layers.14.mlp.experts.230.up_proj", + "model.layers.14.mlp.experts.231.down_proj", + "model.layers.14.mlp.experts.231.gate_proj", + "model.layers.14.mlp.experts.231.up_proj", + "model.layers.14.mlp.experts.232.down_proj", + "model.layers.14.mlp.experts.232.gate_proj", + "model.layers.14.mlp.experts.232.up_proj", + "model.layers.14.mlp.experts.233.down_proj", + "model.layers.14.mlp.experts.233.gate_proj", + "model.layers.14.mlp.experts.233.up_proj", + "model.layers.14.mlp.experts.234.down_proj", + "model.layers.14.mlp.experts.234.gate_proj", + "model.layers.14.mlp.experts.234.up_proj", + "model.layers.14.mlp.experts.235.down_proj", + "model.layers.14.mlp.experts.235.gate_proj", + "model.layers.14.mlp.experts.235.up_proj", + "model.layers.14.mlp.experts.236.down_proj", + "model.layers.14.mlp.experts.236.gate_proj", + "model.layers.14.mlp.experts.236.up_proj", + "model.layers.14.mlp.experts.237.down_proj", + "model.layers.14.mlp.experts.237.gate_proj", + "model.layers.14.mlp.experts.237.up_proj", + "model.layers.14.mlp.experts.238.down_proj", + "model.layers.14.mlp.experts.238.gate_proj", + "model.layers.14.mlp.experts.238.up_proj", + "model.layers.14.mlp.experts.239.down_proj", + "model.layers.14.mlp.experts.239.gate_proj", + "model.layers.14.mlp.experts.239.up_proj", + "model.layers.14.mlp.experts.24.down_proj", + "model.layers.14.mlp.experts.24.gate_proj", + "model.layers.14.mlp.experts.24.up_proj", + "model.layers.14.mlp.experts.240.down_proj", + "model.layers.14.mlp.experts.240.gate_proj", + "model.layers.14.mlp.experts.240.up_proj", + "model.layers.14.mlp.experts.241.down_proj", + "model.layers.14.mlp.experts.241.gate_proj", + "model.layers.14.mlp.experts.241.up_proj", + "model.layers.14.mlp.experts.242.down_proj", + "model.layers.14.mlp.experts.242.gate_proj", + "model.layers.14.mlp.experts.242.up_proj", + "model.layers.14.mlp.experts.243.down_proj", + "model.layers.14.mlp.experts.243.gate_proj", + "model.layers.14.mlp.experts.243.up_proj", + "model.layers.14.mlp.experts.244.down_proj", + "model.layers.14.mlp.experts.244.gate_proj", + "model.layers.14.mlp.experts.244.up_proj", + "model.layers.14.mlp.experts.245.down_proj", + "model.layers.14.mlp.experts.245.gate_proj", + "model.layers.14.mlp.experts.245.up_proj", + "model.layers.14.mlp.experts.246.down_proj", + "model.layers.14.mlp.experts.246.gate_proj", + "model.layers.14.mlp.experts.246.up_proj", + "model.layers.14.mlp.experts.247.down_proj", + "model.layers.14.mlp.experts.247.gate_proj", + "model.layers.14.mlp.experts.247.up_proj", + "model.layers.14.mlp.experts.248.down_proj", + "model.layers.14.mlp.experts.248.gate_proj", + "model.layers.14.mlp.experts.248.up_proj", + "model.layers.14.mlp.experts.249.down_proj", + "model.layers.14.mlp.experts.249.gate_proj", + "model.layers.14.mlp.experts.249.up_proj", + "model.layers.14.mlp.experts.25.down_proj", + "model.layers.14.mlp.experts.25.gate_proj", + "model.layers.14.mlp.experts.25.up_proj", + "model.layers.14.mlp.experts.250.down_proj", + "model.layers.14.mlp.experts.250.gate_proj", + "model.layers.14.mlp.experts.250.up_proj", + "model.layers.14.mlp.experts.251.down_proj", + "model.layers.14.mlp.experts.251.gate_proj", + "model.layers.14.mlp.experts.251.up_proj", + "model.layers.14.mlp.experts.252.down_proj", + "model.layers.14.mlp.experts.252.gate_proj", + "model.layers.14.mlp.experts.252.up_proj", + "model.layers.14.mlp.experts.253.down_proj", + "model.layers.14.mlp.experts.253.gate_proj", + "model.layers.14.mlp.experts.253.up_proj", + "model.layers.14.mlp.experts.254.down_proj", + "model.layers.14.mlp.experts.254.gate_proj", + "model.layers.14.mlp.experts.254.up_proj", + "model.layers.14.mlp.experts.255.down_proj", + "model.layers.14.mlp.experts.255.gate_proj", + "model.layers.14.mlp.experts.255.up_proj", + "model.layers.14.mlp.experts.26.down_proj", + "model.layers.14.mlp.experts.26.gate_proj", + "model.layers.14.mlp.experts.26.up_proj", + "model.layers.14.mlp.experts.27.down_proj", + "model.layers.14.mlp.experts.27.gate_proj", + "model.layers.14.mlp.experts.27.up_proj", + "model.layers.14.mlp.experts.28.down_proj", + "model.layers.14.mlp.experts.28.gate_proj", + "model.layers.14.mlp.experts.28.up_proj", + "model.layers.14.mlp.experts.29.down_proj", + "model.layers.14.mlp.experts.29.gate_proj", + "model.layers.14.mlp.experts.29.up_proj", + "model.layers.14.mlp.experts.3.down_proj", + "model.layers.14.mlp.experts.3.gate_proj", + "model.layers.14.mlp.experts.3.up_proj", + "model.layers.14.mlp.experts.30.down_proj", + "model.layers.14.mlp.experts.30.gate_proj", + "model.layers.14.mlp.experts.30.up_proj", + "model.layers.14.mlp.experts.31.down_proj", + "model.layers.14.mlp.experts.31.gate_proj", + "model.layers.14.mlp.experts.31.up_proj", + "model.layers.14.mlp.experts.32.down_proj", + "model.layers.14.mlp.experts.32.gate_proj", + "model.layers.14.mlp.experts.32.up_proj", + "model.layers.14.mlp.experts.33.down_proj", + "model.layers.14.mlp.experts.33.gate_proj", + "model.layers.14.mlp.experts.33.up_proj", + "model.layers.14.mlp.experts.34.down_proj", + "model.layers.14.mlp.experts.34.gate_proj", + "model.layers.14.mlp.experts.34.up_proj", + "model.layers.14.mlp.experts.35.down_proj", + "model.layers.14.mlp.experts.35.gate_proj", + "model.layers.14.mlp.experts.35.up_proj", + "model.layers.14.mlp.experts.36.down_proj", + "model.layers.14.mlp.experts.36.gate_proj", + "model.layers.14.mlp.experts.36.up_proj", + "model.layers.14.mlp.experts.37.down_proj", + "model.layers.14.mlp.experts.37.gate_proj", + "model.layers.14.mlp.experts.37.up_proj", + "model.layers.14.mlp.experts.38.down_proj", + "model.layers.14.mlp.experts.38.gate_proj", + "model.layers.14.mlp.experts.38.up_proj", + "model.layers.14.mlp.experts.39.down_proj", + "model.layers.14.mlp.experts.39.gate_proj", + "model.layers.14.mlp.experts.39.up_proj", + "model.layers.14.mlp.experts.4.down_proj", + "model.layers.14.mlp.experts.4.gate_proj", + "model.layers.14.mlp.experts.4.up_proj", + "model.layers.14.mlp.experts.40.down_proj", + "model.layers.14.mlp.experts.40.gate_proj", + "model.layers.14.mlp.experts.40.up_proj", + "model.layers.14.mlp.experts.41.down_proj", + "model.layers.14.mlp.experts.41.gate_proj", + "model.layers.14.mlp.experts.41.up_proj", + "model.layers.14.mlp.experts.42.down_proj", + "model.layers.14.mlp.experts.42.gate_proj", + "model.layers.14.mlp.experts.42.up_proj", + "model.layers.14.mlp.experts.43.down_proj", + "model.layers.14.mlp.experts.43.gate_proj", + "model.layers.14.mlp.experts.43.up_proj", + "model.layers.14.mlp.experts.44.down_proj", + "model.layers.14.mlp.experts.44.gate_proj", + "model.layers.14.mlp.experts.44.up_proj", + "model.layers.14.mlp.experts.45.down_proj", + "model.layers.14.mlp.experts.45.gate_proj", + "model.layers.14.mlp.experts.45.up_proj", + "model.layers.14.mlp.experts.46.down_proj", + "model.layers.14.mlp.experts.46.gate_proj", + "model.layers.14.mlp.experts.46.up_proj", + "model.layers.14.mlp.experts.47.down_proj", + "model.layers.14.mlp.experts.47.gate_proj", + "model.layers.14.mlp.experts.47.up_proj", + "model.layers.14.mlp.experts.48.down_proj", + "model.layers.14.mlp.experts.48.gate_proj", + "model.layers.14.mlp.experts.48.up_proj", + "model.layers.14.mlp.experts.49.down_proj", + "model.layers.14.mlp.experts.49.gate_proj", + "model.layers.14.mlp.experts.49.up_proj", + "model.layers.14.mlp.experts.5.down_proj", + "model.layers.14.mlp.experts.5.gate_proj", + "model.layers.14.mlp.experts.5.up_proj", + "model.layers.14.mlp.experts.50.down_proj", + "model.layers.14.mlp.experts.50.gate_proj", + "model.layers.14.mlp.experts.50.up_proj", + "model.layers.14.mlp.experts.51.down_proj", + "model.layers.14.mlp.experts.51.gate_proj", + "model.layers.14.mlp.experts.51.up_proj", + "model.layers.14.mlp.experts.52.down_proj", + "model.layers.14.mlp.experts.52.gate_proj", + "model.layers.14.mlp.experts.52.up_proj", + "model.layers.14.mlp.experts.53.down_proj", + "model.layers.14.mlp.experts.53.gate_proj", + "model.layers.14.mlp.experts.53.up_proj", + "model.layers.14.mlp.experts.54.down_proj", + "model.layers.14.mlp.experts.54.gate_proj", + "model.layers.14.mlp.experts.54.up_proj", + "model.layers.14.mlp.experts.55.down_proj", + "model.layers.14.mlp.experts.55.gate_proj", + "model.layers.14.mlp.experts.55.up_proj", + "model.layers.14.mlp.experts.56.down_proj", + "model.layers.14.mlp.experts.56.gate_proj", + "model.layers.14.mlp.experts.56.up_proj", + "model.layers.14.mlp.experts.57.down_proj", + "model.layers.14.mlp.experts.57.gate_proj", + "model.layers.14.mlp.experts.57.up_proj", + "model.layers.14.mlp.experts.58.down_proj", + "model.layers.14.mlp.experts.58.gate_proj", + "model.layers.14.mlp.experts.58.up_proj", + "model.layers.14.mlp.experts.59.down_proj", + "model.layers.14.mlp.experts.59.gate_proj", + "model.layers.14.mlp.experts.59.up_proj", + "model.layers.14.mlp.experts.6.down_proj", + "model.layers.14.mlp.experts.6.gate_proj", + "model.layers.14.mlp.experts.6.up_proj", + "model.layers.14.mlp.experts.60.down_proj", + "model.layers.14.mlp.experts.60.gate_proj", + "model.layers.14.mlp.experts.60.up_proj", + "model.layers.14.mlp.experts.61.down_proj", + "model.layers.14.mlp.experts.61.gate_proj", + "model.layers.14.mlp.experts.61.up_proj", + "model.layers.14.mlp.experts.62.down_proj", + "model.layers.14.mlp.experts.62.gate_proj", + "model.layers.14.mlp.experts.62.up_proj", + "model.layers.14.mlp.experts.63.down_proj", + "model.layers.14.mlp.experts.63.gate_proj", + "model.layers.14.mlp.experts.63.up_proj", + "model.layers.14.mlp.experts.64.down_proj", + "model.layers.14.mlp.experts.64.gate_proj", + "model.layers.14.mlp.experts.64.up_proj", + "model.layers.14.mlp.experts.65.down_proj", + "model.layers.14.mlp.experts.65.gate_proj", + "model.layers.14.mlp.experts.65.up_proj", + "model.layers.14.mlp.experts.66.down_proj", + "model.layers.14.mlp.experts.66.gate_proj", + "model.layers.14.mlp.experts.66.up_proj", + "model.layers.14.mlp.experts.67.down_proj", + "model.layers.14.mlp.experts.67.gate_proj", + "model.layers.14.mlp.experts.67.up_proj", + "model.layers.14.mlp.experts.68.down_proj", + "model.layers.14.mlp.experts.68.gate_proj", + "model.layers.14.mlp.experts.68.up_proj", + "model.layers.14.mlp.experts.69.down_proj", + "model.layers.14.mlp.experts.69.gate_proj", + "model.layers.14.mlp.experts.69.up_proj", + "model.layers.14.mlp.experts.7.down_proj", + "model.layers.14.mlp.experts.7.gate_proj", + "model.layers.14.mlp.experts.7.up_proj", + "model.layers.14.mlp.experts.70.down_proj", + "model.layers.14.mlp.experts.70.gate_proj", + "model.layers.14.mlp.experts.70.up_proj", + "model.layers.14.mlp.experts.71.down_proj", + "model.layers.14.mlp.experts.71.gate_proj", + "model.layers.14.mlp.experts.71.up_proj", + "model.layers.14.mlp.experts.72.down_proj", + "model.layers.14.mlp.experts.72.gate_proj", + "model.layers.14.mlp.experts.72.up_proj", + "model.layers.14.mlp.experts.73.down_proj", + "model.layers.14.mlp.experts.73.gate_proj", + "model.layers.14.mlp.experts.73.up_proj", + "model.layers.14.mlp.experts.74.down_proj", + "model.layers.14.mlp.experts.74.gate_proj", + "model.layers.14.mlp.experts.74.up_proj", + "model.layers.14.mlp.experts.75.down_proj", + "model.layers.14.mlp.experts.75.gate_proj", + "model.layers.14.mlp.experts.75.up_proj", + "model.layers.14.mlp.experts.76.down_proj", + "model.layers.14.mlp.experts.76.gate_proj", + "model.layers.14.mlp.experts.76.up_proj", + "model.layers.14.mlp.experts.77.down_proj", + "model.layers.14.mlp.experts.77.gate_proj", + "model.layers.14.mlp.experts.77.up_proj", + "model.layers.14.mlp.experts.78.down_proj", + "model.layers.14.mlp.experts.78.gate_proj", + "model.layers.14.mlp.experts.78.up_proj", + "model.layers.14.mlp.experts.79.down_proj", + "model.layers.14.mlp.experts.79.gate_proj", + "model.layers.14.mlp.experts.79.up_proj", + "model.layers.14.mlp.experts.8.down_proj", + "model.layers.14.mlp.experts.8.gate_proj", + "model.layers.14.mlp.experts.8.up_proj", + "model.layers.14.mlp.experts.80.down_proj", + "model.layers.14.mlp.experts.80.gate_proj", + "model.layers.14.mlp.experts.80.up_proj", + "model.layers.14.mlp.experts.81.down_proj", + "model.layers.14.mlp.experts.81.gate_proj", + "model.layers.14.mlp.experts.81.up_proj", + "model.layers.14.mlp.experts.82.down_proj", + "model.layers.14.mlp.experts.82.gate_proj", + "model.layers.14.mlp.experts.82.up_proj", + "model.layers.14.mlp.experts.83.down_proj", + "model.layers.14.mlp.experts.83.gate_proj", + "model.layers.14.mlp.experts.83.up_proj", + "model.layers.14.mlp.experts.84.down_proj", + "model.layers.14.mlp.experts.84.gate_proj", + "model.layers.14.mlp.experts.84.up_proj", + "model.layers.14.mlp.experts.85.down_proj", + "model.layers.14.mlp.experts.85.gate_proj", + "model.layers.14.mlp.experts.85.up_proj", + "model.layers.14.mlp.experts.86.down_proj", + "model.layers.14.mlp.experts.86.gate_proj", + "model.layers.14.mlp.experts.86.up_proj", + "model.layers.14.mlp.experts.87.down_proj", + "model.layers.14.mlp.experts.87.gate_proj", + "model.layers.14.mlp.experts.87.up_proj", + "model.layers.14.mlp.experts.88.down_proj", + "model.layers.14.mlp.experts.88.gate_proj", + "model.layers.14.mlp.experts.88.up_proj", + "model.layers.14.mlp.experts.89.down_proj", + "model.layers.14.mlp.experts.89.gate_proj", + "model.layers.14.mlp.experts.89.up_proj", + "model.layers.14.mlp.experts.9.down_proj", + "model.layers.14.mlp.experts.9.gate_proj", + "model.layers.14.mlp.experts.9.up_proj", + "model.layers.14.mlp.experts.90.down_proj", + "model.layers.14.mlp.experts.90.gate_proj", + "model.layers.14.mlp.experts.90.up_proj", + "model.layers.14.mlp.experts.91.down_proj", + "model.layers.14.mlp.experts.91.gate_proj", + "model.layers.14.mlp.experts.91.up_proj", + "model.layers.14.mlp.experts.92.down_proj", + "model.layers.14.mlp.experts.92.gate_proj", + "model.layers.14.mlp.experts.92.up_proj", + "model.layers.14.mlp.experts.93.down_proj", + "model.layers.14.mlp.experts.93.gate_proj", + "model.layers.14.mlp.experts.93.up_proj", + "model.layers.14.mlp.experts.94.down_proj", + "model.layers.14.mlp.experts.94.gate_proj", + "model.layers.14.mlp.experts.94.up_proj", + "model.layers.14.mlp.experts.95.down_proj", + "model.layers.14.mlp.experts.95.gate_proj", + "model.layers.14.mlp.experts.95.up_proj", + "model.layers.14.mlp.experts.96.down_proj", + "model.layers.14.mlp.experts.96.gate_proj", + "model.layers.14.mlp.experts.96.up_proj", + "model.layers.14.mlp.experts.97.down_proj", + "model.layers.14.mlp.experts.97.gate_proj", + "model.layers.14.mlp.experts.97.up_proj", + "model.layers.14.mlp.experts.98.down_proj", + "model.layers.14.mlp.experts.98.gate_proj", + "model.layers.14.mlp.experts.98.up_proj", + "model.layers.14.mlp.experts.99.down_proj", + "model.layers.14.mlp.experts.99.gate_proj", + "model.layers.14.mlp.experts.99.up_proj", + "model.layers.15.mlp.experts.0.down_proj", + "model.layers.15.mlp.experts.0.gate_proj", + "model.layers.15.mlp.experts.0.up_proj", + "model.layers.15.mlp.experts.1.down_proj", + "model.layers.15.mlp.experts.1.gate_proj", + "model.layers.15.mlp.experts.1.up_proj", + "model.layers.15.mlp.experts.10.down_proj", + "model.layers.15.mlp.experts.10.gate_proj", + "model.layers.15.mlp.experts.10.up_proj", + "model.layers.15.mlp.experts.100.down_proj", + "model.layers.15.mlp.experts.100.gate_proj", + "model.layers.15.mlp.experts.100.up_proj", + "model.layers.15.mlp.experts.101.down_proj", + "model.layers.15.mlp.experts.101.gate_proj", + "model.layers.15.mlp.experts.101.up_proj", + "model.layers.15.mlp.experts.102.down_proj", + "model.layers.15.mlp.experts.102.gate_proj", + "model.layers.15.mlp.experts.102.up_proj", + "model.layers.15.mlp.experts.103.down_proj", + "model.layers.15.mlp.experts.103.gate_proj", + "model.layers.15.mlp.experts.103.up_proj", + "model.layers.15.mlp.experts.104.down_proj", + "model.layers.15.mlp.experts.104.gate_proj", + "model.layers.15.mlp.experts.104.up_proj", + "model.layers.15.mlp.experts.105.down_proj", + "model.layers.15.mlp.experts.105.gate_proj", + "model.layers.15.mlp.experts.105.up_proj", + "model.layers.15.mlp.experts.106.down_proj", + "model.layers.15.mlp.experts.106.gate_proj", + "model.layers.15.mlp.experts.106.up_proj", + "model.layers.15.mlp.experts.107.down_proj", + "model.layers.15.mlp.experts.107.gate_proj", + "model.layers.15.mlp.experts.107.up_proj", + "model.layers.15.mlp.experts.108.down_proj", + "model.layers.15.mlp.experts.108.gate_proj", + "model.layers.15.mlp.experts.108.up_proj", + "model.layers.15.mlp.experts.109.down_proj", + "model.layers.15.mlp.experts.109.gate_proj", + "model.layers.15.mlp.experts.109.up_proj", + "model.layers.15.mlp.experts.11.down_proj", + "model.layers.15.mlp.experts.11.gate_proj", + "model.layers.15.mlp.experts.11.up_proj", + "model.layers.15.mlp.experts.110.down_proj", + "model.layers.15.mlp.experts.110.gate_proj", + "model.layers.15.mlp.experts.110.up_proj", + "model.layers.15.mlp.experts.111.down_proj", + "model.layers.15.mlp.experts.111.gate_proj", + "model.layers.15.mlp.experts.111.up_proj", + "model.layers.15.mlp.experts.112.down_proj", + "model.layers.15.mlp.experts.112.gate_proj", + "model.layers.15.mlp.experts.112.up_proj", + "model.layers.15.mlp.experts.113.down_proj", + "model.layers.15.mlp.experts.113.gate_proj", + "model.layers.15.mlp.experts.113.up_proj", + "model.layers.15.mlp.experts.114.down_proj", + "model.layers.15.mlp.experts.114.gate_proj", + "model.layers.15.mlp.experts.114.up_proj", + "model.layers.15.mlp.experts.115.down_proj", + "model.layers.15.mlp.experts.115.gate_proj", + "model.layers.15.mlp.experts.115.up_proj", + "model.layers.15.mlp.experts.116.down_proj", + "model.layers.15.mlp.experts.116.gate_proj", + "model.layers.15.mlp.experts.116.up_proj", + "model.layers.15.mlp.experts.117.down_proj", + "model.layers.15.mlp.experts.117.gate_proj", + "model.layers.15.mlp.experts.117.up_proj", + "model.layers.15.mlp.experts.118.down_proj", + "model.layers.15.mlp.experts.118.gate_proj", + "model.layers.15.mlp.experts.118.up_proj", + "model.layers.15.mlp.experts.119.down_proj", + "model.layers.15.mlp.experts.119.gate_proj", + "model.layers.15.mlp.experts.119.up_proj", + "model.layers.15.mlp.experts.12.down_proj", + "model.layers.15.mlp.experts.12.gate_proj", + "model.layers.15.mlp.experts.12.up_proj", + "model.layers.15.mlp.experts.120.down_proj", + "model.layers.15.mlp.experts.120.gate_proj", + "model.layers.15.mlp.experts.120.up_proj", + "model.layers.15.mlp.experts.121.down_proj", + "model.layers.15.mlp.experts.121.gate_proj", + "model.layers.15.mlp.experts.121.up_proj", + "model.layers.15.mlp.experts.122.down_proj", + "model.layers.15.mlp.experts.122.gate_proj", + "model.layers.15.mlp.experts.122.up_proj", + "model.layers.15.mlp.experts.123.down_proj", + "model.layers.15.mlp.experts.123.gate_proj", + "model.layers.15.mlp.experts.123.up_proj", + "model.layers.15.mlp.experts.124.down_proj", + "model.layers.15.mlp.experts.124.gate_proj", + "model.layers.15.mlp.experts.124.up_proj", + "model.layers.15.mlp.experts.125.down_proj", + "model.layers.15.mlp.experts.125.gate_proj", + "model.layers.15.mlp.experts.125.up_proj", + "model.layers.15.mlp.experts.126.down_proj", + "model.layers.15.mlp.experts.126.gate_proj", + "model.layers.15.mlp.experts.126.up_proj", + "model.layers.15.mlp.experts.127.down_proj", + "model.layers.15.mlp.experts.127.gate_proj", + "model.layers.15.mlp.experts.127.up_proj", + "model.layers.15.mlp.experts.128.down_proj", + "model.layers.15.mlp.experts.128.gate_proj", + "model.layers.15.mlp.experts.128.up_proj", + "model.layers.15.mlp.experts.129.down_proj", + "model.layers.15.mlp.experts.129.gate_proj", + "model.layers.15.mlp.experts.129.up_proj", + "model.layers.15.mlp.experts.13.down_proj", + "model.layers.15.mlp.experts.13.gate_proj", + "model.layers.15.mlp.experts.13.up_proj", + "model.layers.15.mlp.experts.130.down_proj", + "model.layers.15.mlp.experts.130.gate_proj", + "model.layers.15.mlp.experts.130.up_proj", + "model.layers.15.mlp.experts.131.down_proj", + "model.layers.15.mlp.experts.131.gate_proj", + "model.layers.15.mlp.experts.131.up_proj", + "model.layers.15.mlp.experts.132.down_proj", + "model.layers.15.mlp.experts.132.gate_proj", + "model.layers.15.mlp.experts.132.up_proj", + "model.layers.15.mlp.experts.133.down_proj", + "model.layers.15.mlp.experts.133.gate_proj", + "model.layers.15.mlp.experts.133.up_proj", + "model.layers.15.mlp.experts.134.down_proj", + "model.layers.15.mlp.experts.134.gate_proj", + "model.layers.15.mlp.experts.134.up_proj", + "model.layers.15.mlp.experts.135.down_proj", + "model.layers.15.mlp.experts.135.gate_proj", + "model.layers.15.mlp.experts.135.up_proj", + "model.layers.15.mlp.experts.136.down_proj", + "model.layers.15.mlp.experts.136.gate_proj", + "model.layers.15.mlp.experts.136.up_proj", + "model.layers.15.mlp.experts.137.down_proj", + "model.layers.15.mlp.experts.137.gate_proj", + "model.layers.15.mlp.experts.137.up_proj", + "model.layers.15.mlp.experts.138.down_proj", + "model.layers.15.mlp.experts.138.gate_proj", + "model.layers.15.mlp.experts.138.up_proj", + "model.layers.15.mlp.experts.139.down_proj", + "model.layers.15.mlp.experts.139.gate_proj", + "model.layers.15.mlp.experts.139.up_proj", + "model.layers.15.mlp.experts.14.down_proj", + "model.layers.15.mlp.experts.14.gate_proj", + "model.layers.15.mlp.experts.14.up_proj", + "model.layers.15.mlp.experts.140.down_proj", + "model.layers.15.mlp.experts.140.gate_proj", + "model.layers.15.mlp.experts.140.up_proj", + "model.layers.15.mlp.experts.141.down_proj", + "model.layers.15.mlp.experts.141.gate_proj", + "model.layers.15.mlp.experts.141.up_proj", + "model.layers.15.mlp.experts.142.down_proj", + "model.layers.15.mlp.experts.142.gate_proj", + "model.layers.15.mlp.experts.142.up_proj", + "model.layers.15.mlp.experts.143.down_proj", + "model.layers.15.mlp.experts.143.gate_proj", + "model.layers.15.mlp.experts.143.up_proj", + "model.layers.15.mlp.experts.144.down_proj", + "model.layers.15.mlp.experts.144.gate_proj", + "model.layers.15.mlp.experts.144.up_proj", + "model.layers.15.mlp.experts.145.down_proj", + "model.layers.15.mlp.experts.145.gate_proj", + "model.layers.15.mlp.experts.145.up_proj", + "model.layers.15.mlp.experts.146.down_proj", + "model.layers.15.mlp.experts.146.gate_proj", + "model.layers.15.mlp.experts.146.up_proj", + "model.layers.15.mlp.experts.147.down_proj", + "model.layers.15.mlp.experts.147.gate_proj", + "model.layers.15.mlp.experts.147.up_proj", + "model.layers.15.mlp.experts.148.down_proj", + "model.layers.15.mlp.experts.148.gate_proj", + "model.layers.15.mlp.experts.148.up_proj", + "model.layers.15.mlp.experts.149.down_proj", + "model.layers.15.mlp.experts.149.gate_proj", + "model.layers.15.mlp.experts.149.up_proj", + "model.layers.15.mlp.experts.15.down_proj", + "model.layers.15.mlp.experts.15.gate_proj", + "model.layers.15.mlp.experts.15.up_proj", + "model.layers.15.mlp.experts.150.down_proj", + "model.layers.15.mlp.experts.150.gate_proj", + "model.layers.15.mlp.experts.150.up_proj", + "model.layers.15.mlp.experts.151.down_proj", + "model.layers.15.mlp.experts.151.gate_proj", + "model.layers.15.mlp.experts.151.up_proj", + "model.layers.15.mlp.experts.152.down_proj", + "model.layers.15.mlp.experts.152.gate_proj", + "model.layers.15.mlp.experts.152.up_proj", + "model.layers.15.mlp.experts.153.down_proj", + "model.layers.15.mlp.experts.153.gate_proj", + "model.layers.15.mlp.experts.153.up_proj", + "model.layers.15.mlp.experts.154.down_proj", + "model.layers.15.mlp.experts.154.gate_proj", + "model.layers.15.mlp.experts.154.up_proj", + "model.layers.15.mlp.experts.155.down_proj", + "model.layers.15.mlp.experts.155.gate_proj", + "model.layers.15.mlp.experts.155.up_proj", + "model.layers.15.mlp.experts.156.down_proj", + "model.layers.15.mlp.experts.156.gate_proj", + "model.layers.15.mlp.experts.156.up_proj", + "model.layers.15.mlp.experts.157.down_proj", + "model.layers.15.mlp.experts.157.gate_proj", + "model.layers.15.mlp.experts.157.up_proj", + "model.layers.15.mlp.experts.158.down_proj", + "model.layers.15.mlp.experts.158.gate_proj", + "model.layers.15.mlp.experts.158.up_proj", + "model.layers.15.mlp.experts.159.down_proj", + "model.layers.15.mlp.experts.159.gate_proj", + "model.layers.15.mlp.experts.159.up_proj", + "model.layers.15.mlp.experts.16.down_proj", + "model.layers.15.mlp.experts.16.gate_proj", + "model.layers.15.mlp.experts.16.up_proj", + "model.layers.15.mlp.experts.160.down_proj", + "model.layers.15.mlp.experts.160.gate_proj", + "model.layers.15.mlp.experts.160.up_proj", + "model.layers.15.mlp.experts.161.down_proj", + "model.layers.15.mlp.experts.161.gate_proj", + "model.layers.15.mlp.experts.161.up_proj", + "model.layers.15.mlp.experts.162.down_proj", + "model.layers.15.mlp.experts.162.gate_proj", + "model.layers.15.mlp.experts.162.up_proj", + "model.layers.15.mlp.experts.163.down_proj", + "model.layers.15.mlp.experts.163.gate_proj", + "model.layers.15.mlp.experts.163.up_proj", + "model.layers.15.mlp.experts.164.down_proj", + "model.layers.15.mlp.experts.164.gate_proj", + "model.layers.15.mlp.experts.164.up_proj", + "model.layers.15.mlp.experts.165.down_proj", + "model.layers.15.mlp.experts.165.gate_proj", + "model.layers.15.mlp.experts.165.up_proj", + "model.layers.15.mlp.experts.166.down_proj", + "model.layers.15.mlp.experts.166.gate_proj", + "model.layers.15.mlp.experts.166.up_proj", + "model.layers.15.mlp.experts.167.down_proj", + "model.layers.15.mlp.experts.167.gate_proj", + "model.layers.15.mlp.experts.167.up_proj", + "model.layers.15.mlp.experts.168.down_proj", + "model.layers.15.mlp.experts.168.gate_proj", + "model.layers.15.mlp.experts.168.up_proj", + "model.layers.15.mlp.experts.169.down_proj", + "model.layers.15.mlp.experts.169.gate_proj", + "model.layers.15.mlp.experts.169.up_proj", + "model.layers.15.mlp.experts.17.down_proj", + "model.layers.15.mlp.experts.17.gate_proj", + "model.layers.15.mlp.experts.17.up_proj", + "model.layers.15.mlp.experts.170.down_proj", + "model.layers.15.mlp.experts.170.gate_proj", + "model.layers.15.mlp.experts.170.up_proj", + "model.layers.15.mlp.experts.171.down_proj", + "model.layers.15.mlp.experts.171.gate_proj", + "model.layers.15.mlp.experts.171.up_proj", + "model.layers.15.mlp.experts.172.down_proj", + "model.layers.15.mlp.experts.172.gate_proj", + "model.layers.15.mlp.experts.172.up_proj", + "model.layers.15.mlp.experts.173.down_proj", + "model.layers.15.mlp.experts.173.gate_proj", + "model.layers.15.mlp.experts.173.up_proj", + "model.layers.15.mlp.experts.174.down_proj", + "model.layers.15.mlp.experts.174.gate_proj", + "model.layers.15.mlp.experts.174.up_proj", + "model.layers.15.mlp.experts.175.down_proj", + "model.layers.15.mlp.experts.175.gate_proj", + "model.layers.15.mlp.experts.175.up_proj", + "model.layers.15.mlp.experts.176.down_proj", + "model.layers.15.mlp.experts.176.gate_proj", + "model.layers.15.mlp.experts.176.up_proj", + "model.layers.15.mlp.experts.177.down_proj", + "model.layers.15.mlp.experts.177.gate_proj", + "model.layers.15.mlp.experts.177.up_proj", + "model.layers.15.mlp.experts.178.down_proj", + "model.layers.15.mlp.experts.178.gate_proj", + "model.layers.15.mlp.experts.178.up_proj", + "model.layers.15.mlp.experts.179.down_proj", + "model.layers.15.mlp.experts.179.gate_proj", + "model.layers.15.mlp.experts.179.up_proj", + "model.layers.15.mlp.experts.18.down_proj", + "model.layers.15.mlp.experts.18.gate_proj", + "model.layers.15.mlp.experts.18.up_proj", + "model.layers.15.mlp.experts.180.down_proj", + "model.layers.15.mlp.experts.180.gate_proj", + "model.layers.15.mlp.experts.180.up_proj", + "model.layers.15.mlp.experts.181.down_proj", + "model.layers.15.mlp.experts.181.gate_proj", + "model.layers.15.mlp.experts.181.up_proj", + "model.layers.15.mlp.experts.182.down_proj", + "model.layers.15.mlp.experts.182.gate_proj", + "model.layers.15.mlp.experts.182.up_proj", + "model.layers.15.mlp.experts.183.down_proj", + "model.layers.15.mlp.experts.183.gate_proj", + "model.layers.15.mlp.experts.183.up_proj", + "model.layers.15.mlp.experts.184.down_proj", + "model.layers.15.mlp.experts.184.gate_proj", + "model.layers.15.mlp.experts.184.up_proj", + "model.layers.15.mlp.experts.185.down_proj", + "model.layers.15.mlp.experts.185.gate_proj", + "model.layers.15.mlp.experts.185.up_proj", + "model.layers.15.mlp.experts.186.down_proj", + "model.layers.15.mlp.experts.186.gate_proj", + "model.layers.15.mlp.experts.186.up_proj", + "model.layers.15.mlp.experts.187.down_proj", + "model.layers.15.mlp.experts.187.gate_proj", + "model.layers.15.mlp.experts.187.up_proj", + "model.layers.15.mlp.experts.188.down_proj", + "model.layers.15.mlp.experts.188.gate_proj", + "model.layers.15.mlp.experts.188.up_proj", + "model.layers.15.mlp.experts.189.down_proj", + "model.layers.15.mlp.experts.189.gate_proj", + "model.layers.15.mlp.experts.189.up_proj", + "model.layers.15.mlp.experts.19.down_proj", + "model.layers.15.mlp.experts.19.gate_proj", + "model.layers.15.mlp.experts.19.up_proj", + "model.layers.15.mlp.experts.190.down_proj", + "model.layers.15.mlp.experts.190.gate_proj", + "model.layers.15.mlp.experts.190.up_proj", + "model.layers.15.mlp.experts.191.down_proj", + "model.layers.15.mlp.experts.191.gate_proj", + "model.layers.15.mlp.experts.191.up_proj", + "model.layers.15.mlp.experts.192.down_proj", + "model.layers.15.mlp.experts.192.gate_proj", + "model.layers.15.mlp.experts.192.up_proj", + "model.layers.15.mlp.experts.193.down_proj", + "model.layers.15.mlp.experts.193.gate_proj", + "model.layers.15.mlp.experts.193.up_proj", + "model.layers.15.mlp.experts.194.down_proj", + "model.layers.15.mlp.experts.194.gate_proj", + "model.layers.15.mlp.experts.194.up_proj", + "model.layers.15.mlp.experts.195.down_proj", + "model.layers.15.mlp.experts.195.gate_proj", + "model.layers.15.mlp.experts.195.up_proj", + "model.layers.15.mlp.experts.196.down_proj", + "model.layers.15.mlp.experts.196.gate_proj", + "model.layers.15.mlp.experts.196.up_proj", + "model.layers.15.mlp.experts.197.down_proj", + "model.layers.15.mlp.experts.197.gate_proj", + "model.layers.15.mlp.experts.197.up_proj", + "model.layers.15.mlp.experts.198.down_proj", + "model.layers.15.mlp.experts.198.gate_proj", + "model.layers.15.mlp.experts.198.up_proj", + "model.layers.15.mlp.experts.199.down_proj", + "model.layers.15.mlp.experts.199.gate_proj", + "model.layers.15.mlp.experts.199.up_proj", + "model.layers.15.mlp.experts.2.down_proj", + "model.layers.15.mlp.experts.2.gate_proj", + "model.layers.15.mlp.experts.2.up_proj", + "model.layers.15.mlp.experts.20.down_proj", + "model.layers.15.mlp.experts.20.gate_proj", + "model.layers.15.mlp.experts.20.up_proj", + "model.layers.15.mlp.experts.200.down_proj", + "model.layers.15.mlp.experts.200.gate_proj", + "model.layers.15.mlp.experts.200.up_proj", + "model.layers.15.mlp.experts.201.down_proj", + "model.layers.15.mlp.experts.201.gate_proj", + "model.layers.15.mlp.experts.201.up_proj", + "model.layers.15.mlp.experts.202.down_proj", + "model.layers.15.mlp.experts.202.gate_proj", + "model.layers.15.mlp.experts.202.up_proj", + "model.layers.15.mlp.experts.203.down_proj", + "model.layers.15.mlp.experts.203.gate_proj", + "model.layers.15.mlp.experts.203.up_proj", + "model.layers.15.mlp.experts.204.down_proj", + "model.layers.15.mlp.experts.204.gate_proj", + "model.layers.15.mlp.experts.204.up_proj", + "model.layers.15.mlp.experts.205.down_proj", + "model.layers.15.mlp.experts.205.gate_proj", + "model.layers.15.mlp.experts.205.up_proj", + "model.layers.15.mlp.experts.206.down_proj", + "model.layers.15.mlp.experts.206.gate_proj", + "model.layers.15.mlp.experts.206.up_proj", + "model.layers.15.mlp.experts.207.down_proj", + "model.layers.15.mlp.experts.207.gate_proj", + "model.layers.15.mlp.experts.207.up_proj", + "model.layers.15.mlp.experts.208.down_proj", + "model.layers.15.mlp.experts.208.gate_proj", + "model.layers.15.mlp.experts.208.up_proj", + "model.layers.15.mlp.experts.209.down_proj", + "model.layers.15.mlp.experts.209.gate_proj", + "model.layers.15.mlp.experts.209.up_proj", + "model.layers.15.mlp.experts.21.down_proj", + "model.layers.15.mlp.experts.21.gate_proj", + "model.layers.15.mlp.experts.21.up_proj", + "model.layers.15.mlp.experts.210.down_proj", + "model.layers.15.mlp.experts.210.gate_proj", + "model.layers.15.mlp.experts.210.up_proj", + "model.layers.15.mlp.experts.211.down_proj", + "model.layers.15.mlp.experts.211.gate_proj", + "model.layers.15.mlp.experts.211.up_proj", + "model.layers.15.mlp.experts.212.down_proj", + "model.layers.15.mlp.experts.212.gate_proj", + "model.layers.15.mlp.experts.212.up_proj", + "model.layers.15.mlp.experts.213.down_proj", + "model.layers.15.mlp.experts.213.gate_proj", + "model.layers.15.mlp.experts.213.up_proj", + "model.layers.15.mlp.experts.214.down_proj", + "model.layers.15.mlp.experts.214.gate_proj", + "model.layers.15.mlp.experts.214.up_proj", + "model.layers.15.mlp.experts.215.down_proj", + "model.layers.15.mlp.experts.215.gate_proj", + "model.layers.15.mlp.experts.215.up_proj", + "model.layers.15.mlp.experts.216.down_proj", + "model.layers.15.mlp.experts.216.gate_proj", + "model.layers.15.mlp.experts.216.up_proj", + "model.layers.15.mlp.experts.217.down_proj", + "model.layers.15.mlp.experts.217.gate_proj", + "model.layers.15.mlp.experts.217.up_proj", + "model.layers.15.mlp.experts.218.down_proj", + "model.layers.15.mlp.experts.218.gate_proj", + "model.layers.15.mlp.experts.218.up_proj", + "model.layers.15.mlp.experts.219.down_proj", + "model.layers.15.mlp.experts.219.gate_proj", + "model.layers.15.mlp.experts.219.up_proj", + "model.layers.15.mlp.experts.22.down_proj", + "model.layers.15.mlp.experts.22.gate_proj", + "model.layers.15.mlp.experts.22.up_proj", + "model.layers.15.mlp.experts.220.down_proj", + "model.layers.15.mlp.experts.220.gate_proj", + "model.layers.15.mlp.experts.220.up_proj", + "model.layers.15.mlp.experts.221.down_proj", + "model.layers.15.mlp.experts.221.gate_proj", + "model.layers.15.mlp.experts.221.up_proj", + "model.layers.15.mlp.experts.222.down_proj", + "model.layers.15.mlp.experts.222.gate_proj", + "model.layers.15.mlp.experts.222.up_proj", + "model.layers.15.mlp.experts.223.down_proj", + "model.layers.15.mlp.experts.223.gate_proj", + "model.layers.15.mlp.experts.223.up_proj", + "model.layers.15.mlp.experts.224.down_proj", + "model.layers.15.mlp.experts.224.gate_proj", + "model.layers.15.mlp.experts.224.up_proj", + "model.layers.15.mlp.experts.225.down_proj", + "model.layers.15.mlp.experts.225.gate_proj", + "model.layers.15.mlp.experts.225.up_proj", + "model.layers.15.mlp.experts.226.down_proj", + "model.layers.15.mlp.experts.226.gate_proj", + "model.layers.15.mlp.experts.226.up_proj", + "model.layers.15.mlp.experts.227.down_proj", + "model.layers.15.mlp.experts.227.gate_proj", + "model.layers.15.mlp.experts.227.up_proj", + "model.layers.15.mlp.experts.228.down_proj", + "model.layers.15.mlp.experts.228.gate_proj", + "model.layers.15.mlp.experts.228.up_proj", + "model.layers.15.mlp.experts.229.down_proj", + "model.layers.15.mlp.experts.229.gate_proj", + "model.layers.15.mlp.experts.229.up_proj", + "model.layers.15.mlp.experts.23.down_proj", + "model.layers.15.mlp.experts.23.gate_proj", + "model.layers.15.mlp.experts.23.up_proj", + "model.layers.15.mlp.experts.230.down_proj", + "model.layers.15.mlp.experts.230.gate_proj", + "model.layers.15.mlp.experts.230.up_proj", + "model.layers.15.mlp.experts.231.down_proj", + "model.layers.15.mlp.experts.231.gate_proj", + "model.layers.15.mlp.experts.231.up_proj", + "model.layers.15.mlp.experts.232.down_proj", + "model.layers.15.mlp.experts.232.gate_proj", + "model.layers.15.mlp.experts.232.up_proj", + "model.layers.15.mlp.experts.233.down_proj", + "model.layers.15.mlp.experts.233.gate_proj", + "model.layers.15.mlp.experts.233.up_proj", + "model.layers.15.mlp.experts.234.down_proj", + "model.layers.15.mlp.experts.234.gate_proj", + "model.layers.15.mlp.experts.234.up_proj", + "model.layers.15.mlp.experts.235.down_proj", + "model.layers.15.mlp.experts.235.gate_proj", + "model.layers.15.mlp.experts.235.up_proj", + "model.layers.15.mlp.experts.236.down_proj", + "model.layers.15.mlp.experts.236.gate_proj", + "model.layers.15.mlp.experts.236.up_proj", + "model.layers.15.mlp.experts.237.down_proj", + "model.layers.15.mlp.experts.237.gate_proj", + "model.layers.15.mlp.experts.237.up_proj", + "model.layers.15.mlp.experts.238.down_proj", + "model.layers.15.mlp.experts.238.gate_proj", + "model.layers.15.mlp.experts.238.up_proj", + "model.layers.15.mlp.experts.239.down_proj", + "model.layers.15.mlp.experts.239.gate_proj", + "model.layers.15.mlp.experts.239.up_proj", + "model.layers.15.mlp.experts.24.down_proj", + "model.layers.15.mlp.experts.24.gate_proj", + "model.layers.15.mlp.experts.24.up_proj", + "model.layers.15.mlp.experts.240.down_proj", + "model.layers.15.mlp.experts.240.gate_proj", + "model.layers.15.mlp.experts.240.up_proj", + "model.layers.15.mlp.experts.241.down_proj", + "model.layers.15.mlp.experts.241.gate_proj", + "model.layers.15.mlp.experts.241.up_proj", + "model.layers.15.mlp.experts.242.down_proj", + "model.layers.15.mlp.experts.242.gate_proj", + "model.layers.15.mlp.experts.242.up_proj", + "model.layers.15.mlp.experts.243.down_proj", + "model.layers.15.mlp.experts.243.gate_proj", + "model.layers.15.mlp.experts.243.up_proj", + "model.layers.15.mlp.experts.244.down_proj", + "model.layers.15.mlp.experts.244.gate_proj", + "model.layers.15.mlp.experts.244.up_proj", + "model.layers.15.mlp.experts.245.down_proj", + "model.layers.15.mlp.experts.245.gate_proj", + "model.layers.15.mlp.experts.245.up_proj", + "model.layers.15.mlp.experts.246.down_proj", + "model.layers.15.mlp.experts.246.gate_proj", + "model.layers.15.mlp.experts.246.up_proj", + "model.layers.15.mlp.experts.247.down_proj", + "model.layers.15.mlp.experts.247.gate_proj", + "model.layers.15.mlp.experts.247.up_proj", + "model.layers.15.mlp.experts.248.down_proj", + "model.layers.15.mlp.experts.248.gate_proj", + "model.layers.15.mlp.experts.248.up_proj", + "model.layers.15.mlp.experts.249.down_proj", + "model.layers.15.mlp.experts.249.gate_proj", + "model.layers.15.mlp.experts.249.up_proj", + "model.layers.15.mlp.experts.25.down_proj", + "model.layers.15.mlp.experts.25.gate_proj", + "model.layers.15.mlp.experts.25.up_proj", + "model.layers.15.mlp.experts.250.down_proj", + "model.layers.15.mlp.experts.250.gate_proj", + "model.layers.15.mlp.experts.250.up_proj", + "model.layers.15.mlp.experts.251.down_proj", + "model.layers.15.mlp.experts.251.gate_proj", + "model.layers.15.mlp.experts.251.up_proj", + "model.layers.15.mlp.experts.252.down_proj", + "model.layers.15.mlp.experts.252.gate_proj", + "model.layers.15.mlp.experts.252.up_proj", + "model.layers.15.mlp.experts.253.down_proj", + "model.layers.15.mlp.experts.253.gate_proj", + "model.layers.15.mlp.experts.253.up_proj", + "model.layers.15.mlp.experts.254.down_proj", + "model.layers.15.mlp.experts.254.gate_proj", + "model.layers.15.mlp.experts.254.up_proj", + "model.layers.15.mlp.experts.255.down_proj", + "model.layers.15.mlp.experts.255.gate_proj", + "model.layers.15.mlp.experts.255.up_proj", + "model.layers.15.mlp.experts.26.down_proj", + "model.layers.15.mlp.experts.26.gate_proj", + "model.layers.15.mlp.experts.26.up_proj", + "model.layers.15.mlp.experts.27.down_proj", + "model.layers.15.mlp.experts.27.gate_proj", + "model.layers.15.mlp.experts.27.up_proj", + "model.layers.15.mlp.experts.28.down_proj", + "model.layers.15.mlp.experts.28.gate_proj", + "model.layers.15.mlp.experts.28.up_proj", + "model.layers.15.mlp.experts.29.down_proj", + "model.layers.15.mlp.experts.29.gate_proj", + "model.layers.15.mlp.experts.29.up_proj", + "model.layers.15.mlp.experts.3.down_proj", + "model.layers.15.mlp.experts.3.gate_proj", + "model.layers.15.mlp.experts.3.up_proj", + "model.layers.15.mlp.experts.30.down_proj", + "model.layers.15.mlp.experts.30.gate_proj", + "model.layers.15.mlp.experts.30.up_proj", + "model.layers.15.mlp.experts.31.down_proj", + "model.layers.15.mlp.experts.31.gate_proj", + "model.layers.15.mlp.experts.31.up_proj", + "model.layers.15.mlp.experts.32.down_proj", + "model.layers.15.mlp.experts.32.gate_proj", + "model.layers.15.mlp.experts.32.up_proj", + "model.layers.15.mlp.experts.33.down_proj", + "model.layers.15.mlp.experts.33.gate_proj", + "model.layers.15.mlp.experts.33.up_proj", + "model.layers.15.mlp.experts.34.down_proj", + "model.layers.15.mlp.experts.34.gate_proj", + "model.layers.15.mlp.experts.34.up_proj", + "model.layers.15.mlp.experts.35.down_proj", + "model.layers.15.mlp.experts.35.gate_proj", + "model.layers.15.mlp.experts.35.up_proj", + "model.layers.15.mlp.experts.36.down_proj", + "model.layers.15.mlp.experts.36.gate_proj", + "model.layers.15.mlp.experts.36.up_proj", + "model.layers.15.mlp.experts.37.down_proj", + "model.layers.15.mlp.experts.37.gate_proj", + "model.layers.15.mlp.experts.37.up_proj", + "model.layers.15.mlp.experts.38.down_proj", + "model.layers.15.mlp.experts.38.gate_proj", + "model.layers.15.mlp.experts.38.up_proj", + "model.layers.15.mlp.experts.39.down_proj", + "model.layers.15.mlp.experts.39.gate_proj", + "model.layers.15.mlp.experts.39.up_proj", + "model.layers.15.mlp.experts.4.down_proj", + "model.layers.15.mlp.experts.4.gate_proj", + "model.layers.15.mlp.experts.4.up_proj", + "model.layers.15.mlp.experts.40.down_proj", + "model.layers.15.mlp.experts.40.gate_proj", + "model.layers.15.mlp.experts.40.up_proj", + "model.layers.15.mlp.experts.41.down_proj", + "model.layers.15.mlp.experts.41.gate_proj", + "model.layers.15.mlp.experts.41.up_proj", + "model.layers.15.mlp.experts.42.down_proj", + "model.layers.15.mlp.experts.42.gate_proj", + "model.layers.15.mlp.experts.42.up_proj", + "model.layers.15.mlp.experts.43.down_proj", + "model.layers.15.mlp.experts.43.gate_proj", + "model.layers.15.mlp.experts.43.up_proj", + "model.layers.15.mlp.experts.44.down_proj", + "model.layers.15.mlp.experts.44.gate_proj", + "model.layers.15.mlp.experts.44.up_proj", + "model.layers.15.mlp.experts.45.down_proj", + "model.layers.15.mlp.experts.45.gate_proj", + "model.layers.15.mlp.experts.45.up_proj", + "model.layers.15.mlp.experts.46.down_proj", + "model.layers.15.mlp.experts.46.gate_proj", + "model.layers.15.mlp.experts.46.up_proj", + "model.layers.15.mlp.experts.47.down_proj", + "model.layers.15.mlp.experts.47.gate_proj", + "model.layers.15.mlp.experts.47.up_proj", + "model.layers.15.mlp.experts.48.down_proj", + "model.layers.15.mlp.experts.48.gate_proj", + "model.layers.15.mlp.experts.48.up_proj", + "model.layers.15.mlp.experts.49.down_proj", + "model.layers.15.mlp.experts.49.gate_proj", + "model.layers.15.mlp.experts.49.up_proj", + "model.layers.15.mlp.experts.5.down_proj", + "model.layers.15.mlp.experts.5.gate_proj", + "model.layers.15.mlp.experts.5.up_proj", + "model.layers.15.mlp.experts.50.down_proj", + "model.layers.15.mlp.experts.50.gate_proj", + "model.layers.15.mlp.experts.50.up_proj", + "model.layers.15.mlp.experts.51.down_proj", + "model.layers.15.mlp.experts.51.gate_proj", + "model.layers.15.mlp.experts.51.up_proj", + "model.layers.15.mlp.experts.52.down_proj", + "model.layers.15.mlp.experts.52.gate_proj", + "model.layers.15.mlp.experts.52.up_proj", + "model.layers.15.mlp.experts.53.down_proj", + "model.layers.15.mlp.experts.53.gate_proj", + "model.layers.15.mlp.experts.53.up_proj", + "model.layers.15.mlp.experts.54.down_proj", + "model.layers.15.mlp.experts.54.gate_proj", + "model.layers.15.mlp.experts.54.up_proj", + "model.layers.15.mlp.experts.55.down_proj", + "model.layers.15.mlp.experts.55.gate_proj", + "model.layers.15.mlp.experts.55.up_proj", + "model.layers.15.mlp.experts.56.down_proj", + "model.layers.15.mlp.experts.56.gate_proj", + "model.layers.15.mlp.experts.56.up_proj", + "model.layers.15.mlp.experts.57.down_proj", + "model.layers.15.mlp.experts.57.gate_proj", + "model.layers.15.mlp.experts.57.up_proj", + "model.layers.15.mlp.experts.58.down_proj", + "model.layers.15.mlp.experts.58.gate_proj", + "model.layers.15.mlp.experts.58.up_proj", + "model.layers.15.mlp.experts.59.down_proj", + "model.layers.15.mlp.experts.59.gate_proj", + "model.layers.15.mlp.experts.59.up_proj", + "model.layers.15.mlp.experts.6.down_proj", + "model.layers.15.mlp.experts.6.gate_proj", + "model.layers.15.mlp.experts.6.up_proj", + "model.layers.15.mlp.experts.60.down_proj", + "model.layers.15.mlp.experts.60.gate_proj", + "model.layers.15.mlp.experts.60.up_proj", + "model.layers.15.mlp.experts.61.down_proj", + "model.layers.15.mlp.experts.61.gate_proj", + "model.layers.15.mlp.experts.61.up_proj", + "model.layers.15.mlp.experts.62.down_proj", + "model.layers.15.mlp.experts.62.gate_proj", + "model.layers.15.mlp.experts.62.up_proj", + "model.layers.15.mlp.experts.63.down_proj", + "model.layers.15.mlp.experts.63.gate_proj", + "model.layers.15.mlp.experts.63.up_proj", + "model.layers.15.mlp.experts.64.down_proj", + "model.layers.15.mlp.experts.64.gate_proj", + "model.layers.15.mlp.experts.64.up_proj", + "model.layers.15.mlp.experts.65.down_proj", + "model.layers.15.mlp.experts.65.gate_proj", + "model.layers.15.mlp.experts.65.up_proj", + "model.layers.15.mlp.experts.66.down_proj", + "model.layers.15.mlp.experts.66.gate_proj", + "model.layers.15.mlp.experts.66.up_proj", + "model.layers.15.mlp.experts.67.down_proj", + "model.layers.15.mlp.experts.67.gate_proj", + "model.layers.15.mlp.experts.67.up_proj", + "model.layers.15.mlp.experts.68.down_proj", + "model.layers.15.mlp.experts.68.gate_proj", + "model.layers.15.mlp.experts.68.up_proj", + "model.layers.15.mlp.experts.69.down_proj", + "model.layers.15.mlp.experts.69.gate_proj", + "model.layers.15.mlp.experts.69.up_proj", + "model.layers.15.mlp.experts.7.down_proj", + "model.layers.15.mlp.experts.7.gate_proj", + "model.layers.15.mlp.experts.7.up_proj", + "model.layers.15.mlp.experts.70.down_proj", + "model.layers.15.mlp.experts.70.gate_proj", + "model.layers.15.mlp.experts.70.up_proj", + "model.layers.15.mlp.experts.71.down_proj", + "model.layers.15.mlp.experts.71.gate_proj", + "model.layers.15.mlp.experts.71.up_proj", + "model.layers.15.mlp.experts.72.down_proj", + "model.layers.15.mlp.experts.72.gate_proj", + "model.layers.15.mlp.experts.72.up_proj", + "model.layers.15.mlp.experts.73.down_proj", + "model.layers.15.mlp.experts.73.gate_proj", + "model.layers.15.mlp.experts.73.up_proj", + "model.layers.15.mlp.experts.74.down_proj", + "model.layers.15.mlp.experts.74.gate_proj", + "model.layers.15.mlp.experts.74.up_proj", + "model.layers.15.mlp.experts.75.down_proj", + "model.layers.15.mlp.experts.75.gate_proj", + "model.layers.15.mlp.experts.75.up_proj", + "model.layers.15.mlp.experts.76.down_proj", + "model.layers.15.mlp.experts.76.gate_proj", + "model.layers.15.mlp.experts.76.up_proj", + "model.layers.15.mlp.experts.77.down_proj", + "model.layers.15.mlp.experts.77.gate_proj", + "model.layers.15.mlp.experts.77.up_proj", + "model.layers.15.mlp.experts.78.down_proj", + "model.layers.15.mlp.experts.78.gate_proj", + "model.layers.15.mlp.experts.78.up_proj", + "model.layers.15.mlp.experts.79.down_proj", + "model.layers.15.mlp.experts.79.gate_proj", + "model.layers.15.mlp.experts.79.up_proj", + "model.layers.15.mlp.experts.8.down_proj", + "model.layers.15.mlp.experts.8.gate_proj", + "model.layers.15.mlp.experts.8.up_proj", + "model.layers.15.mlp.experts.80.down_proj", + "model.layers.15.mlp.experts.80.gate_proj", + "model.layers.15.mlp.experts.80.up_proj", + "model.layers.15.mlp.experts.81.down_proj", + "model.layers.15.mlp.experts.81.gate_proj", + "model.layers.15.mlp.experts.81.up_proj", + "model.layers.15.mlp.experts.82.down_proj", + "model.layers.15.mlp.experts.82.gate_proj", + "model.layers.15.mlp.experts.82.up_proj", + "model.layers.15.mlp.experts.83.down_proj", + "model.layers.15.mlp.experts.83.gate_proj", + "model.layers.15.mlp.experts.83.up_proj", + "model.layers.15.mlp.experts.84.down_proj", + "model.layers.15.mlp.experts.84.gate_proj", + "model.layers.15.mlp.experts.84.up_proj", + "model.layers.15.mlp.experts.85.down_proj", + "model.layers.15.mlp.experts.85.gate_proj", + "model.layers.15.mlp.experts.85.up_proj", + "model.layers.15.mlp.experts.86.down_proj", + "model.layers.15.mlp.experts.86.gate_proj", + "model.layers.15.mlp.experts.86.up_proj", + "model.layers.15.mlp.experts.87.down_proj", + "model.layers.15.mlp.experts.87.gate_proj", + "model.layers.15.mlp.experts.87.up_proj", + "model.layers.15.mlp.experts.88.down_proj", + "model.layers.15.mlp.experts.88.gate_proj", + "model.layers.15.mlp.experts.88.up_proj", + "model.layers.15.mlp.experts.89.down_proj", + "model.layers.15.mlp.experts.89.gate_proj", + "model.layers.15.mlp.experts.89.up_proj", + "model.layers.15.mlp.experts.9.down_proj", + "model.layers.15.mlp.experts.9.gate_proj", + "model.layers.15.mlp.experts.9.up_proj", + "model.layers.15.mlp.experts.90.down_proj", + "model.layers.15.mlp.experts.90.gate_proj", + "model.layers.15.mlp.experts.90.up_proj", + "model.layers.15.mlp.experts.91.down_proj", + "model.layers.15.mlp.experts.91.gate_proj", + "model.layers.15.mlp.experts.91.up_proj", + "model.layers.15.mlp.experts.92.down_proj", + "model.layers.15.mlp.experts.92.gate_proj", + "model.layers.15.mlp.experts.92.up_proj", + "model.layers.15.mlp.experts.93.down_proj", + "model.layers.15.mlp.experts.93.gate_proj", + "model.layers.15.mlp.experts.93.up_proj", + "model.layers.15.mlp.experts.94.down_proj", + "model.layers.15.mlp.experts.94.gate_proj", + "model.layers.15.mlp.experts.94.up_proj", + "model.layers.15.mlp.experts.95.down_proj", + "model.layers.15.mlp.experts.95.gate_proj", + "model.layers.15.mlp.experts.95.up_proj", + "model.layers.15.mlp.experts.96.down_proj", + "model.layers.15.mlp.experts.96.gate_proj", + "model.layers.15.mlp.experts.96.up_proj", + "model.layers.15.mlp.experts.97.down_proj", + "model.layers.15.mlp.experts.97.gate_proj", + "model.layers.15.mlp.experts.97.up_proj", + "model.layers.15.mlp.experts.98.down_proj", + "model.layers.15.mlp.experts.98.gate_proj", + "model.layers.15.mlp.experts.98.up_proj", + "model.layers.15.mlp.experts.99.down_proj", + "model.layers.15.mlp.experts.99.gate_proj", + "model.layers.15.mlp.experts.99.up_proj", + "model.layers.16.mlp.experts.0.down_proj", + "model.layers.16.mlp.experts.0.gate_proj", + "model.layers.16.mlp.experts.0.up_proj", + "model.layers.16.mlp.experts.1.down_proj", + "model.layers.16.mlp.experts.1.gate_proj", + "model.layers.16.mlp.experts.1.up_proj", + "model.layers.16.mlp.experts.10.down_proj", + "model.layers.16.mlp.experts.10.gate_proj", + "model.layers.16.mlp.experts.10.up_proj", + "model.layers.16.mlp.experts.100.down_proj", + "model.layers.16.mlp.experts.100.gate_proj", + "model.layers.16.mlp.experts.100.up_proj", + "model.layers.16.mlp.experts.101.down_proj", + "model.layers.16.mlp.experts.101.gate_proj", + "model.layers.16.mlp.experts.101.up_proj", + "model.layers.16.mlp.experts.102.down_proj", + "model.layers.16.mlp.experts.102.gate_proj", + "model.layers.16.mlp.experts.102.up_proj", + "model.layers.16.mlp.experts.103.down_proj", + "model.layers.16.mlp.experts.103.gate_proj", + "model.layers.16.mlp.experts.103.up_proj", + "model.layers.16.mlp.experts.104.down_proj", + "model.layers.16.mlp.experts.104.gate_proj", + "model.layers.16.mlp.experts.104.up_proj", + "model.layers.16.mlp.experts.105.down_proj", + "model.layers.16.mlp.experts.105.gate_proj", + "model.layers.16.mlp.experts.105.up_proj", + "model.layers.16.mlp.experts.106.down_proj", + "model.layers.16.mlp.experts.106.gate_proj", + "model.layers.16.mlp.experts.106.up_proj", + "model.layers.16.mlp.experts.107.down_proj", + "model.layers.16.mlp.experts.107.gate_proj", + "model.layers.16.mlp.experts.107.up_proj", + "model.layers.16.mlp.experts.108.down_proj", + "model.layers.16.mlp.experts.108.gate_proj", + "model.layers.16.mlp.experts.108.up_proj", + "model.layers.16.mlp.experts.109.down_proj", + "model.layers.16.mlp.experts.109.gate_proj", + "model.layers.16.mlp.experts.109.up_proj", + "model.layers.16.mlp.experts.11.down_proj", + "model.layers.16.mlp.experts.11.gate_proj", + "model.layers.16.mlp.experts.11.up_proj", + "model.layers.16.mlp.experts.110.down_proj", + "model.layers.16.mlp.experts.110.gate_proj", + "model.layers.16.mlp.experts.110.up_proj", + "model.layers.16.mlp.experts.111.down_proj", + "model.layers.16.mlp.experts.111.gate_proj", + "model.layers.16.mlp.experts.111.up_proj", + "model.layers.16.mlp.experts.112.down_proj", + "model.layers.16.mlp.experts.112.gate_proj", + "model.layers.16.mlp.experts.112.up_proj", + "model.layers.16.mlp.experts.113.down_proj", + "model.layers.16.mlp.experts.113.gate_proj", + "model.layers.16.mlp.experts.113.up_proj", + "model.layers.16.mlp.experts.114.down_proj", + "model.layers.16.mlp.experts.114.gate_proj", + "model.layers.16.mlp.experts.114.up_proj", + "model.layers.16.mlp.experts.115.down_proj", + "model.layers.16.mlp.experts.115.gate_proj", + "model.layers.16.mlp.experts.115.up_proj", + "model.layers.16.mlp.experts.116.down_proj", + "model.layers.16.mlp.experts.116.gate_proj", + "model.layers.16.mlp.experts.116.up_proj", + "model.layers.16.mlp.experts.117.down_proj", + "model.layers.16.mlp.experts.117.gate_proj", + "model.layers.16.mlp.experts.117.up_proj", + "model.layers.16.mlp.experts.118.down_proj", + "model.layers.16.mlp.experts.118.gate_proj", + "model.layers.16.mlp.experts.118.up_proj", + "model.layers.16.mlp.experts.119.down_proj", + "model.layers.16.mlp.experts.119.gate_proj", + "model.layers.16.mlp.experts.119.up_proj", + "model.layers.16.mlp.experts.12.down_proj", + "model.layers.16.mlp.experts.12.gate_proj", + "model.layers.16.mlp.experts.12.up_proj", + "model.layers.16.mlp.experts.120.down_proj", + "model.layers.16.mlp.experts.120.gate_proj", + "model.layers.16.mlp.experts.120.up_proj", + "model.layers.16.mlp.experts.121.down_proj", + "model.layers.16.mlp.experts.121.gate_proj", + "model.layers.16.mlp.experts.121.up_proj", + "model.layers.16.mlp.experts.122.down_proj", + "model.layers.16.mlp.experts.122.gate_proj", + "model.layers.16.mlp.experts.122.up_proj", + "model.layers.16.mlp.experts.123.down_proj", + "model.layers.16.mlp.experts.123.gate_proj", + "model.layers.16.mlp.experts.123.up_proj", + "model.layers.16.mlp.experts.124.down_proj", + "model.layers.16.mlp.experts.124.gate_proj", + "model.layers.16.mlp.experts.124.up_proj", + "model.layers.16.mlp.experts.125.down_proj", + "model.layers.16.mlp.experts.125.gate_proj", + "model.layers.16.mlp.experts.125.up_proj", + "model.layers.16.mlp.experts.126.down_proj", + "model.layers.16.mlp.experts.126.gate_proj", + "model.layers.16.mlp.experts.126.up_proj", + "model.layers.16.mlp.experts.127.down_proj", + "model.layers.16.mlp.experts.127.gate_proj", + "model.layers.16.mlp.experts.127.up_proj", + "model.layers.16.mlp.experts.128.down_proj", + "model.layers.16.mlp.experts.128.gate_proj", + "model.layers.16.mlp.experts.128.up_proj", + "model.layers.16.mlp.experts.129.down_proj", + "model.layers.16.mlp.experts.129.gate_proj", + "model.layers.16.mlp.experts.129.up_proj", + "model.layers.16.mlp.experts.13.down_proj", + "model.layers.16.mlp.experts.13.gate_proj", + "model.layers.16.mlp.experts.13.up_proj", + "model.layers.16.mlp.experts.130.down_proj", + "model.layers.16.mlp.experts.130.gate_proj", + "model.layers.16.mlp.experts.130.up_proj", + "model.layers.16.mlp.experts.131.down_proj", + "model.layers.16.mlp.experts.131.gate_proj", + "model.layers.16.mlp.experts.131.up_proj", + "model.layers.16.mlp.experts.132.down_proj", + "model.layers.16.mlp.experts.132.gate_proj", + "model.layers.16.mlp.experts.132.up_proj", + "model.layers.16.mlp.experts.133.down_proj", + "model.layers.16.mlp.experts.133.gate_proj", + "model.layers.16.mlp.experts.133.up_proj", + "model.layers.16.mlp.experts.134.down_proj", + "model.layers.16.mlp.experts.134.gate_proj", + "model.layers.16.mlp.experts.134.up_proj", + "model.layers.16.mlp.experts.135.down_proj", + "model.layers.16.mlp.experts.135.gate_proj", + "model.layers.16.mlp.experts.135.up_proj", + "model.layers.16.mlp.experts.136.down_proj", + "model.layers.16.mlp.experts.136.gate_proj", + "model.layers.16.mlp.experts.136.up_proj", + "model.layers.16.mlp.experts.137.down_proj", + "model.layers.16.mlp.experts.137.gate_proj", + "model.layers.16.mlp.experts.137.up_proj", + "model.layers.16.mlp.experts.138.down_proj", + "model.layers.16.mlp.experts.138.gate_proj", + "model.layers.16.mlp.experts.138.up_proj", + "model.layers.16.mlp.experts.139.down_proj", + "model.layers.16.mlp.experts.139.gate_proj", + "model.layers.16.mlp.experts.139.up_proj", + "model.layers.16.mlp.experts.14.down_proj", + "model.layers.16.mlp.experts.14.gate_proj", + "model.layers.16.mlp.experts.14.up_proj", + "model.layers.16.mlp.experts.140.down_proj", + "model.layers.16.mlp.experts.140.gate_proj", + "model.layers.16.mlp.experts.140.up_proj", + "model.layers.16.mlp.experts.141.down_proj", + "model.layers.16.mlp.experts.141.gate_proj", + "model.layers.16.mlp.experts.141.up_proj", + "model.layers.16.mlp.experts.142.down_proj", + "model.layers.16.mlp.experts.142.gate_proj", + "model.layers.16.mlp.experts.142.up_proj", + "model.layers.16.mlp.experts.143.down_proj", + "model.layers.16.mlp.experts.143.gate_proj", + "model.layers.16.mlp.experts.143.up_proj", + "model.layers.16.mlp.experts.144.down_proj", + "model.layers.16.mlp.experts.144.gate_proj", + "model.layers.16.mlp.experts.144.up_proj", + "model.layers.16.mlp.experts.145.down_proj", + "model.layers.16.mlp.experts.145.gate_proj", + "model.layers.16.mlp.experts.145.up_proj", + "model.layers.16.mlp.experts.146.down_proj", + "model.layers.16.mlp.experts.146.gate_proj", + "model.layers.16.mlp.experts.146.up_proj", + "model.layers.16.mlp.experts.147.down_proj", + "model.layers.16.mlp.experts.147.gate_proj", + "model.layers.16.mlp.experts.147.up_proj", + "model.layers.16.mlp.experts.148.down_proj", + "model.layers.16.mlp.experts.148.gate_proj", + "model.layers.16.mlp.experts.148.up_proj", + "model.layers.16.mlp.experts.149.down_proj", + "model.layers.16.mlp.experts.149.gate_proj", + "model.layers.16.mlp.experts.149.up_proj", + "model.layers.16.mlp.experts.15.down_proj", + "model.layers.16.mlp.experts.15.gate_proj", + "model.layers.16.mlp.experts.15.up_proj", + "model.layers.16.mlp.experts.150.down_proj", + "model.layers.16.mlp.experts.150.gate_proj", + "model.layers.16.mlp.experts.150.up_proj", + "model.layers.16.mlp.experts.151.down_proj", + "model.layers.16.mlp.experts.151.gate_proj", + "model.layers.16.mlp.experts.151.up_proj", + "model.layers.16.mlp.experts.152.down_proj", + "model.layers.16.mlp.experts.152.gate_proj", + "model.layers.16.mlp.experts.152.up_proj", + "model.layers.16.mlp.experts.153.down_proj", + "model.layers.16.mlp.experts.153.gate_proj", + "model.layers.16.mlp.experts.153.up_proj", + "model.layers.16.mlp.experts.154.down_proj", + "model.layers.16.mlp.experts.154.gate_proj", + "model.layers.16.mlp.experts.154.up_proj", + "model.layers.16.mlp.experts.155.down_proj", + "model.layers.16.mlp.experts.155.gate_proj", + "model.layers.16.mlp.experts.155.up_proj", + "model.layers.16.mlp.experts.156.down_proj", + "model.layers.16.mlp.experts.156.gate_proj", + "model.layers.16.mlp.experts.156.up_proj", + "model.layers.16.mlp.experts.157.down_proj", + "model.layers.16.mlp.experts.157.gate_proj", + "model.layers.16.mlp.experts.157.up_proj", + "model.layers.16.mlp.experts.158.down_proj", + "model.layers.16.mlp.experts.158.gate_proj", + "model.layers.16.mlp.experts.158.up_proj", + "model.layers.16.mlp.experts.159.down_proj", + "model.layers.16.mlp.experts.159.gate_proj", + "model.layers.16.mlp.experts.159.up_proj", + "model.layers.16.mlp.experts.16.down_proj", + "model.layers.16.mlp.experts.16.gate_proj", + "model.layers.16.mlp.experts.16.up_proj", + "model.layers.16.mlp.experts.160.down_proj", + "model.layers.16.mlp.experts.160.gate_proj", + "model.layers.16.mlp.experts.160.up_proj", + "model.layers.16.mlp.experts.161.down_proj", + "model.layers.16.mlp.experts.161.gate_proj", + "model.layers.16.mlp.experts.161.up_proj", + "model.layers.16.mlp.experts.162.down_proj", + "model.layers.16.mlp.experts.162.gate_proj", + "model.layers.16.mlp.experts.162.up_proj", + "model.layers.16.mlp.experts.163.down_proj", + "model.layers.16.mlp.experts.163.gate_proj", + "model.layers.16.mlp.experts.163.up_proj", + "model.layers.16.mlp.experts.164.down_proj", + "model.layers.16.mlp.experts.164.gate_proj", + "model.layers.16.mlp.experts.164.up_proj", + "model.layers.16.mlp.experts.165.down_proj", + "model.layers.16.mlp.experts.165.gate_proj", + "model.layers.16.mlp.experts.165.up_proj", + "model.layers.16.mlp.experts.166.down_proj", + "model.layers.16.mlp.experts.166.gate_proj", + "model.layers.16.mlp.experts.166.up_proj", + "model.layers.16.mlp.experts.167.down_proj", + "model.layers.16.mlp.experts.167.gate_proj", + "model.layers.16.mlp.experts.167.up_proj", + "model.layers.16.mlp.experts.168.down_proj", + "model.layers.16.mlp.experts.168.gate_proj", + "model.layers.16.mlp.experts.168.up_proj", + "model.layers.16.mlp.experts.169.down_proj", + "model.layers.16.mlp.experts.169.gate_proj", + "model.layers.16.mlp.experts.169.up_proj", + "model.layers.16.mlp.experts.17.down_proj", + "model.layers.16.mlp.experts.17.gate_proj", + "model.layers.16.mlp.experts.17.up_proj", + "model.layers.16.mlp.experts.170.down_proj", + "model.layers.16.mlp.experts.170.gate_proj", + "model.layers.16.mlp.experts.170.up_proj", + "model.layers.16.mlp.experts.171.down_proj", + "model.layers.16.mlp.experts.171.gate_proj", + "model.layers.16.mlp.experts.171.up_proj", + "model.layers.16.mlp.experts.172.down_proj", + "model.layers.16.mlp.experts.172.gate_proj", + "model.layers.16.mlp.experts.172.up_proj", + "model.layers.16.mlp.experts.173.down_proj", + "model.layers.16.mlp.experts.173.gate_proj", + "model.layers.16.mlp.experts.173.up_proj", + "model.layers.16.mlp.experts.174.down_proj", + "model.layers.16.mlp.experts.174.gate_proj", + "model.layers.16.mlp.experts.174.up_proj", + "model.layers.16.mlp.experts.175.down_proj", + "model.layers.16.mlp.experts.175.gate_proj", + "model.layers.16.mlp.experts.175.up_proj", + "model.layers.16.mlp.experts.176.down_proj", + "model.layers.16.mlp.experts.176.gate_proj", + "model.layers.16.mlp.experts.176.up_proj", + "model.layers.16.mlp.experts.177.down_proj", + "model.layers.16.mlp.experts.177.gate_proj", + "model.layers.16.mlp.experts.177.up_proj", + "model.layers.16.mlp.experts.178.down_proj", + "model.layers.16.mlp.experts.178.gate_proj", + "model.layers.16.mlp.experts.178.up_proj", + "model.layers.16.mlp.experts.179.down_proj", + "model.layers.16.mlp.experts.179.gate_proj", + "model.layers.16.mlp.experts.179.up_proj", + "model.layers.16.mlp.experts.18.down_proj", + "model.layers.16.mlp.experts.18.gate_proj", + "model.layers.16.mlp.experts.18.up_proj", + "model.layers.16.mlp.experts.180.down_proj", + "model.layers.16.mlp.experts.180.gate_proj", + "model.layers.16.mlp.experts.180.up_proj", + "model.layers.16.mlp.experts.181.down_proj", + "model.layers.16.mlp.experts.181.gate_proj", + "model.layers.16.mlp.experts.181.up_proj", + "model.layers.16.mlp.experts.182.down_proj", + "model.layers.16.mlp.experts.182.gate_proj", + "model.layers.16.mlp.experts.182.up_proj", + "model.layers.16.mlp.experts.183.down_proj", + "model.layers.16.mlp.experts.183.gate_proj", + "model.layers.16.mlp.experts.183.up_proj", + "model.layers.16.mlp.experts.184.down_proj", + "model.layers.16.mlp.experts.184.gate_proj", + "model.layers.16.mlp.experts.184.up_proj", + "model.layers.16.mlp.experts.185.down_proj", + "model.layers.16.mlp.experts.185.gate_proj", + "model.layers.16.mlp.experts.185.up_proj", + "model.layers.16.mlp.experts.186.down_proj", + "model.layers.16.mlp.experts.186.gate_proj", + "model.layers.16.mlp.experts.186.up_proj", + "model.layers.16.mlp.experts.187.down_proj", + "model.layers.16.mlp.experts.187.gate_proj", + "model.layers.16.mlp.experts.187.up_proj", + "model.layers.16.mlp.experts.188.down_proj", + "model.layers.16.mlp.experts.188.gate_proj", + "model.layers.16.mlp.experts.188.up_proj", + "model.layers.16.mlp.experts.189.down_proj", + "model.layers.16.mlp.experts.189.gate_proj", + "model.layers.16.mlp.experts.189.up_proj", + "model.layers.16.mlp.experts.19.down_proj", + "model.layers.16.mlp.experts.19.gate_proj", + "model.layers.16.mlp.experts.19.up_proj", + "model.layers.16.mlp.experts.190.down_proj", + "model.layers.16.mlp.experts.190.gate_proj", + "model.layers.16.mlp.experts.190.up_proj", + "model.layers.16.mlp.experts.191.down_proj", + "model.layers.16.mlp.experts.191.gate_proj", + "model.layers.16.mlp.experts.191.up_proj", + "model.layers.16.mlp.experts.192.down_proj", + "model.layers.16.mlp.experts.192.gate_proj", + "model.layers.16.mlp.experts.192.up_proj", + "model.layers.16.mlp.experts.193.down_proj", + "model.layers.16.mlp.experts.193.gate_proj", + "model.layers.16.mlp.experts.193.up_proj", + "model.layers.16.mlp.experts.194.down_proj", + "model.layers.16.mlp.experts.194.gate_proj", + "model.layers.16.mlp.experts.194.up_proj", + "model.layers.16.mlp.experts.195.down_proj", + "model.layers.16.mlp.experts.195.gate_proj", + "model.layers.16.mlp.experts.195.up_proj", + "model.layers.16.mlp.experts.196.down_proj", + "model.layers.16.mlp.experts.196.gate_proj", + "model.layers.16.mlp.experts.196.up_proj", + "model.layers.16.mlp.experts.197.down_proj", + "model.layers.16.mlp.experts.197.gate_proj", + "model.layers.16.mlp.experts.197.up_proj", + "model.layers.16.mlp.experts.198.down_proj", + "model.layers.16.mlp.experts.198.gate_proj", + "model.layers.16.mlp.experts.198.up_proj", + "model.layers.16.mlp.experts.199.down_proj", + "model.layers.16.mlp.experts.199.gate_proj", + "model.layers.16.mlp.experts.199.up_proj", + "model.layers.16.mlp.experts.2.down_proj", + "model.layers.16.mlp.experts.2.gate_proj", + "model.layers.16.mlp.experts.2.up_proj", + "model.layers.16.mlp.experts.20.down_proj", + "model.layers.16.mlp.experts.20.gate_proj", + "model.layers.16.mlp.experts.20.up_proj", + "model.layers.16.mlp.experts.200.down_proj", + "model.layers.16.mlp.experts.200.gate_proj", + "model.layers.16.mlp.experts.200.up_proj", + "model.layers.16.mlp.experts.201.down_proj", + "model.layers.16.mlp.experts.201.gate_proj", + "model.layers.16.mlp.experts.201.up_proj", + "model.layers.16.mlp.experts.202.down_proj", + "model.layers.16.mlp.experts.202.gate_proj", + "model.layers.16.mlp.experts.202.up_proj", + "model.layers.16.mlp.experts.203.down_proj", + "model.layers.16.mlp.experts.203.gate_proj", + "model.layers.16.mlp.experts.203.up_proj", + "model.layers.16.mlp.experts.204.down_proj", + "model.layers.16.mlp.experts.204.gate_proj", + "model.layers.16.mlp.experts.204.up_proj", + "model.layers.16.mlp.experts.205.down_proj", + "model.layers.16.mlp.experts.205.gate_proj", + "model.layers.16.mlp.experts.205.up_proj", + "model.layers.16.mlp.experts.206.down_proj", + "model.layers.16.mlp.experts.206.gate_proj", + "model.layers.16.mlp.experts.206.up_proj", + "model.layers.16.mlp.experts.207.down_proj", + "model.layers.16.mlp.experts.207.gate_proj", + "model.layers.16.mlp.experts.207.up_proj", + "model.layers.16.mlp.experts.208.down_proj", + "model.layers.16.mlp.experts.208.gate_proj", + "model.layers.16.mlp.experts.208.up_proj", + "model.layers.16.mlp.experts.209.down_proj", + "model.layers.16.mlp.experts.209.gate_proj", + "model.layers.16.mlp.experts.209.up_proj", + "model.layers.16.mlp.experts.21.down_proj", + "model.layers.16.mlp.experts.21.gate_proj", + "model.layers.16.mlp.experts.21.up_proj", + "model.layers.16.mlp.experts.210.down_proj", + "model.layers.16.mlp.experts.210.gate_proj", + "model.layers.16.mlp.experts.210.up_proj", + "model.layers.16.mlp.experts.211.down_proj", + "model.layers.16.mlp.experts.211.gate_proj", + "model.layers.16.mlp.experts.211.up_proj", + "model.layers.16.mlp.experts.212.down_proj", + "model.layers.16.mlp.experts.212.gate_proj", + "model.layers.16.mlp.experts.212.up_proj", + "model.layers.16.mlp.experts.213.down_proj", + "model.layers.16.mlp.experts.213.gate_proj", + "model.layers.16.mlp.experts.213.up_proj", + "model.layers.16.mlp.experts.214.down_proj", + "model.layers.16.mlp.experts.214.gate_proj", + "model.layers.16.mlp.experts.214.up_proj", + "model.layers.16.mlp.experts.215.down_proj", + "model.layers.16.mlp.experts.215.gate_proj", + "model.layers.16.mlp.experts.215.up_proj", + "model.layers.16.mlp.experts.216.down_proj", + "model.layers.16.mlp.experts.216.gate_proj", + "model.layers.16.mlp.experts.216.up_proj", + "model.layers.16.mlp.experts.217.down_proj", + "model.layers.16.mlp.experts.217.gate_proj", + "model.layers.16.mlp.experts.217.up_proj", + "model.layers.16.mlp.experts.218.down_proj", + "model.layers.16.mlp.experts.218.gate_proj", + "model.layers.16.mlp.experts.218.up_proj", + "model.layers.16.mlp.experts.219.down_proj", + "model.layers.16.mlp.experts.219.gate_proj", + "model.layers.16.mlp.experts.219.up_proj", + "model.layers.16.mlp.experts.22.down_proj", + "model.layers.16.mlp.experts.22.gate_proj", + "model.layers.16.mlp.experts.22.up_proj", + "model.layers.16.mlp.experts.220.down_proj", + "model.layers.16.mlp.experts.220.gate_proj", + "model.layers.16.mlp.experts.220.up_proj", + "model.layers.16.mlp.experts.221.down_proj", + "model.layers.16.mlp.experts.221.gate_proj", + "model.layers.16.mlp.experts.221.up_proj", + "model.layers.16.mlp.experts.222.down_proj", + "model.layers.16.mlp.experts.222.gate_proj", + "model.layers.16.mlp.experts.222.up_proj", + "model.layers.16.mlp.experts.223.down_proj", + "model.layers.16.mlp.experts.223.gate_proj", + "model.layers.16.mlp.experts.223.up_proj", + "model.layers.16.mlp.experts.224.down_proj", + "model.layers.16.mlp.experts.224.gate_proj", + "model.layers.16.mlp.experts.224.up_proj", + "model.layers.16.mlp.experts.225.down_proj", + "model.layers.16.mlp.experts.225.gate_proj", + "model.layers.16.mlp.experts.225.up_proj", + "model.layers.16.mlp.experts.226.down_proj", + "model.layers.16.mlp.experts.226.gate_proj", + "model.layers.16.mlp.experts.226.up_proj", + "model.layers.16.mlp.experts.227.down_proj", + "model.layers.16.mlp.experts.227.gate_proj", + "model.layers.16.mlp.experts.227.up_proj", + "model.layers.16.mlp.experts.228.down_proj", + "model.layers.16.mlp.experts.228.gate_proj", + "model.layers.16.mlp.experts.228.up_proj", + "model.layers.16.mlp.experts.229.down_proj", + "model.layers.16.mlp.experts.229.gate_proj", + "model.layers.16.mlp.experts.229.up_proj", + "model.layers.16.mlp.experts.23.down_proj", + "model.layers.16.mlp.experts.23.gate_proj", + "model.layers.16.mlp.experts.23.up_proj", + "model.layers.16.mlp.experts.230.down_proj", + "model.layers.16.mlp.experts.230.gate_proj", + "model.layers.16.mlp.experts.230.up_proj", + "model.layers.16.mlp.experts.231.down_proj", + "model.layers.16.mlp.experts.231.gate_proj", + "model.layers.16.mlp.experts.231.up_proj", + "model.layers.16.mlp.experts.232.down_proj", + "model.layers.16.mlp.experts.232.gate_proj", + "model.layers.16.mlp.experts.232.up_proj", + "model.layers.16.mlp.experts.233.down_proj", + "model.layers.16.mlp.experts.233.gate_proj", + "model.layers.16.mlp.experts.233.up_proj", + "model.layers.16.mlp.experts.234.down_proj", + "model.layers.16.mlp.experts.234.gate_proj", + "model.layers.16.mlp.experts.234.up_proj", + "model.layers.16.mlp.experts.235.down_proj", + "model.layers.16.mlp.experts.235.gate_proj", + "model.layers.16.mlp.experts.235.up_proj", + "model.layers.16.mlp.experts.236.down_proj", + "model.layers.16.mlp.experts.236.gate_proj", + "model.layers.16.mlp.experts.236.up_proj", + "model.layers.16.mlp.experts.237.down_proj", + "model.layers.16.mlp.experts.237.gate_proj", + "model.layers.16.mlp.experts.237.up_proj", + "model.layers.16.mlp.experts.238.down_proj", + "model.layers.16.mlp.experts.238.gate_proj", + "model.layers.16.mlp.experts.238.up_proj", + "model.layers.16.mlp.experts.239.down_proj", + "model.layers.16.mlp.experts.239.gate_proj", + "model.layers.16.mlp.experts.239.up_proj", + "model.layers.16.mlp.experts.24.down_proj", + "model.layers.16.mlp.experts.24.gate_proj", + "model.layers.16.mlp.experts.24.up_proj", + "model.layers.16.mlp.experts.240.down_proj", + "model.layers.16.mlp.experts.240.gate_proj", + "model.layers.16.mlp.experts.240.up_proj", + "model.layers.16.mlp.experts.241.down_proj", + "model.layers.16.mlp.experts.241.gate_proj", + "model.layers.16.mlp.experts.241.up_proj", + "model.layers.16.mlp.experts.242.down_proj", + "model.layers.16.mlp.experts.242.gate_proj", + "model.layers.16.mlp.experts.242.up_proj", + "model.layers.16.mlp.experts.243.down_proj", + "model.layers.16.mlp.experts.243.gate_proj", + "model.layers.16.mlp.experts.243.up_proj", + "model.layers.16.mlp.experts.244.down_proj", + "model.layers.16.mlp.experts.244.gate_proj", + "model.layers.16.mlp.experts.244.up_proj", + "model.layers.16.mlp.experts.245.down_proj", + "model.layers.16.mlp.experts.245.gate_proj", + "model.layers.16.mlp.experts.245.up_proj", + "model.layers.16.mlp.experts.246.down_proj", + "model.layers.16.mlp.experts.246.gate_proj", + "model.layers.16.mlp.experts.246.up_proj", + "model.layers.16.mlp.experts.247.down_proj", + "model.layers.16.mlp.experts.247.gate_proj", + "model.layers.16.mlp.experts.247.up_proj", + "model.layers.16.mlp.experts.248.down_proj", + "model.layers.16.mlp.experts.248.gate_proj", + "model.layers.16.mlp.experts.248.up_proj", + "model.layers.16.mlp.experts.249.down_proj", + "model.layers.16.mlp.experts.249.gate_proj", + "model.layers.16.mlp.experts.249.up_proj", + "model.layers.16.mlp.experts.25.down_proj", + "model.layers.16.mlp.experts.25.gate_proj", + "model.layers.16.mlp.experts.25.up_proj", + "model.layers.16.mlp.experts.250.down_proj", + "model.layers.16.mlp.experts.250.gate_proj", + "model.layers.16.mlp.experts.250.up_proj", + "model.layers.16.mlp.experts.251.down_proj", + "model.layers.16.mlp.experts.251.gate_proj", + "model.layers.16.mlp.experts.251.up_proj", + "model.layers.16.mlp.experts.252.down_proj", + "model.layers.16.mlp.experts.252.gate_proj", + "model.layers.16.mlp.experts.252.up_proj", + "model.layers.16.mlp.experts.253.down_proj", + "model.layers.16.mlp.experts.253.gate_proj", + "model.layers.16.mlp.experts.253.up_proj", + "model.layers.16.mlp.experts.254.down_proj", + "model.layers.16.mlp.experts.254.gate_proj", + "model.layers.16.mlp.experts.254.up_proj", + "model.layers.16.mlp.experts.255.down_proj", + "model.layers.16.mlp.experts.255.gate_proj", + "model.layers.16.mlp.experts.255.up_proj", + "model.layers.16.mlp.experts.26.down_proj", + "model.layers.16.mlp.experts.26.gate_proj", + "model.layers.16.mlp.experts.26.up_proj", + "model.layers.16.mlp.experts.27.down_proj", + "model.layers.16.mlp.experts.27.gate_proj", + "model.layers.16.mlp.experts.27.up_proj", + "model.layers.16.mlp.experts.28.down_proj", + "model.layers.16.mlp.experts.28.gate_proj", + "model.layers.16.mlp.experts.28.up_proj", + "model.layers.16.mlp.experts.29.down_proj", + "model.layers.16.mlp.experts.29.gate_proj", + "model.layers.16.mlp.experts.29.up_proj", + "model.layers.16.mlp.experts.3.down_proj", + "model.layers.16.mlp.experts.3.gate_proj", + "model.layers.16.mlp.experts.3.up_proj", + "model.layers.16.mlp.experts.30.down_proj", + "model.layers.16.mlp.experts.30.gate_proj", + "model.layers.16.mlp.experts.30.up_proj", + "model.layers.16.mlp.experts.31.down_proj", + "model.layers.16.mlp.experts.31.gate_proj", + "model.layers.16.mlp.experts.31.up_proj", + "model.layers.16.mlp.experts.32.down_proj", + "model.layers.16.mlp.experts.32.gate_proj", + "model.layers.16.mlp.experts.32.up_proj", + "model.layers.16.mlp.experts.33.down_proj", + "model.layers.16.mlp.experts.33.gate_proj", + "model.layers.16.mlp.experts.33.up_proj", + "model.layers.16.mlp.experts.34.down_proj", + "model.layers.16.mlp.experts.34.gate_proj", + "model.layers.16.mlp.experts.34.up_proj", + "model.layers.16.mlp.experts.35.down_proj", + "model.layers.16.mlp.experts.35.gate_proj", + "model.layers.16.mlp.experts.35.up_proj", + "model.layers.16.mlp.experts.36.down_proj", + "model.layers.16.mlp.experts.36.gate_proj", + "model.layers.16.mlp.experts.36.up_proj", + "model.layers.16.mlp.experts.37.down_proj", + "model.layers.16.mlp.experts.37.gate_proj", + "model.layers.16.mlp.experts.37.up_proj", + "model.layers.16.mlp.experts.38.down_proj", + "model.layers.16.mlp.experts.38.gate_proj", + "model.layers.16.mlp.experts.38.up_proj", + "model.layers.16.mlp.experts.39.down_proj", + "model.layers.16.mlp.experts.39.gate_proj", + "model.layers.16.mlp.experts.39.up_proj", + "model.layers.16.mlp.experts.4.down_proj", + "model.layers.16.mlp.experts.4.gate_proj", + "model.layers.16.mlp.experts.4.up_proj", + "model.layers.16.mlp.experts.40.down_proj", + "model.layers.16.mlp.experts.40.gate_proj", + "model.layers.16.mlp.experts.40.up_proj", + "model.layers.16.mlp.experts.41.down_proj", + "model.layers.16.mlp.experts.41.gate_proj", + "model.layers.16.mlp.experts.41.up_proj", + "model.layers.16.mlp.experts.42.down_proj", + "model.layers.16.mlp.experts.42.gate_proj", + "model.layers.16.mlp.experts.42.up_proj", + "model.layers.16.mlp.experts.43.down_proj", + "model.layers.16.mlp.experts.43.gate_proj", + "model.layers.16.mlp.experts.43.up_proj", + "model.layers.16.mlp.experts.44.down_proj", + "model.layers.16.mlp.experts.44.gate_proj", + "model.layers.16.mlp.experts.44.up_proj", + "model.layers.16.mlp.experts.45.down_proj", + "model.layers.16.mlp.experts.45.gate_proj", + "model.layers.16.mlp.experts.45.up_proj", + "model.layers.16.mlp.experts.46.down_proj", + "model.layers.16.mlp.experts.46.gate_proj", + "model.layers.16.mlp.experts.46.up_proj", + "model.layers.16.mlp.experts.47.down_proj", + "model.layers.16.mlp.experts.47.gate_proj", + "model.layers.16.mlp.experts.47.up_proj", + "model.layers.16.mlp.experts.48.down_proj", + "model.layers.16.mlp.experts.48.gate_proj", + "model.layers.16.mlp.experts.48.up_proj", + "model.layers.16.mlp.experts.49.down_proj", + "model.layers.16.mlp.experts.49.gate_proj", + "model.layers.16.mlp.experts.49.up_proj", + "model.layers.16.mlp.experts.5.down_proj", + "model.layers.16.mlp.experts.5.gate_proj", + "model.layers.16.mlp.experts.5.up_proj", + "model.layers.16.mlp.experts.50.down_proj", + "model.layers.16.mlp.experts.50.gate_proj", + "model.layers.16.mlp.experts.50.up_proj", + "model.layers.16.mlp.experts.51.down_proj", + "model.layers.16.mlp.experts.51.gate_proj", + "model.layers.16.mlp.experts.51.up_proj", + "model.layers.16.mlp.experts.52.down_proj", + "model.layers.16.mlp.experts.52.gate_proj", + "model.layers.16.mlp.experts.52.up_proj", + "model.layers.16.mlp.experts.53.down_proj", + "model.layers.16.mlp.experts.53.gate_proj", + "model.layers.16.mlp.experts.53.up_proj", + "model.layers.16.mlp.experts.54.down_proj", + "model.layers.16.mlp.experts.54.gate_proj", + "model.layers.16.mlp.experts.54.up_proj", + "model.layers.16.mlp.experts.55.down_proj", + "model.layers.16.mlp.experts.55.gate_proj", + "model.layers.16.mlp.experts.55.up_proj", + "model.layers.16.mlp.experts.56.down_proj", + "model.layers.16.mlp.experts.56.gate_proj", + "model.layers.16.mlp.experts.56.up_proj", + "model.layers.16.mlp.experts.57.down_proj", + "model.layers.16.mlp.experts.57.gate_proj", + "model.layers.16.mlp.experts.57.up_proj", + "model.layers.16.mlp.experts.58.down_proj", + "model.layers.16.mlp.experts.58.gate_proj", + "model.layers.16.mlp.experts.58.up_proj", + "model.layers.16.mlp.experts.59.down_proj", + "model.layers.16.mlp.experts.59.gate_proj", + "model.layers.16.mlp.experts.59.up_proj", + "model.layers.16.mlp.experts.6.down_proj", + "model.layers.16.mlp.experts.6.gate_proj", + "model.layers.16.mlp.experts.6.up_proj", + "model.layers.16.mlp.experts.60.down_proj", + "model.layers.16.mlp.experts.60.gate_proj", + "model.layers.16.mlp.experts.60.up_proj", + "model.layers.16.mlp.experts.61.down_proj", + "model.layers.16.mlp.experts.61.gate_proj", + "model.layers.16.mlp.experts.61.up_proj", + "model.layers.16.mlp.experts.62.down_proj", + "model.layers.16.mlp.experts.62.gate_proj", + "model.layers.16.mlp.experts.62.up_proj", + "model.layers.16.mlp.experts.63.down_proj", + "model.layers.16.mlp.experts.63.gate_proj", + "model.layers.16.mlp.experts.63.up_proj", + "model.layers.16.mlp.experts.64.down_proj", + "model.layers.16.mlp.experts.64.gate_proj", + "model.layers.16.mlp.experts.64.up_proj", + "model.layers.16.mlp.experts.65.down_proj", + "model.layers.16.mlp.experts.65.gate_proj", + "model.layers.16.mlp.experts.65.up_proj", + "model.layers.16.mlp.experts.66.down_proj", + "model.layers.16.mlp.experts.66.gate_proj", + "model.layers.16.mlp.experts.66.up_proj", + "model.layers.16.mlp.experts.67.down_proj", + "model.layers.16.mlp.experts.67.gate_proj", + "model.layers.16.mlp.experts.67.up_proj", + "model.layers.16.mlp.experts.68.down_proj", + "model.layers.16.mlp.experts.68.gate_proj", + "model.layers.16.mlp.experts.68.up_proj", + "model.layers.16.mlp.experts.69.down_proj", + "model.layers.16.mlp.experts.69.gate_proj", + "model.layers.16.mlp.experts.69.up_proj", + "model.layers.16.mlp.experts.7.down_proj", + "model.layers.16.mlp.experts.7.gate_proj", + "model.layers.16.mlp.experts.7.up_proj", + "model.layers.16.mlp.experts.70.down_proj", + "model.layers.16.mlp.experts.70.gate_proj", + "model.layers.16.mlp.experts.70.up_proj", + "model.layers.16.mlp.experts.71.down_proj", + "model.layers.16.mlp.experts.71.gate_proj", + "model.layers.16.mlp.experts.71.up_proj", + "model.layers.16.mlp.experts.72.down_proj", + "model.layers.16.mlp.experts.72.gate_proj", + "model.layers.16.mlp.experts.72.up_proj", + "model.layers.16.mlp.experts.73.down_proj", + "model.layers.16.mlp.experts.73.gate_proj", + "model.layers.16.mlp.experts.73.up_proj", + "model.layers.16.mlp.experts.74.down_proj", + "model.layers.16.mlp.experts.74.gate_proj", + "model.layers.16.mlp.experts.74.up_proj", + "model.layers.16.mlp.experts.75.down_proj", + "model.layers.16.mlp.experts.75.gate_proj", + "model.layers.16.mlp.experts.75.up_proj", + "model.layers.16.mlp.experts.76.down_proj", + "model.layers.16.mlp.experts.76.gate_proj", + "model.layers.16.mlp.experts.76.up_proj", + "model.layers.16.mlp.experts.77.down_proj", + "model.layers.16.mlp.experts.77.gate_proj", + "model.layers.16.mlp.experts.77.up_proj", + "model.layers.16.mlp.experts.78.down_proj", + "model.layers.16.mlp.experts.78.gate_proj", + "model.layers.16.mlp.experts.78.up_proj", + "model.layers.16.mlp.experts.79.down_proj", + "model.layers.16.mlp.experts.79.gate_proj", + "model.layers.16.mlp.experts.79.up_proj", + "model.layers.16.mlp.experts.8.down_proj", + "model.layers.16.mlp.experts.8.gate_proj", + "model.layers.16.mlp.experts.8.up_proj", + "model.layers.16.mlp.experts.80.down_proj", + "model.layers.16.mlp.experts.80.gate_proj", + "model.layers.16.mlp.experts.80.up_proj", + "model.layers.16.mlp.experts.81.down_proj", + "model.layers.16.mlp.experts.81.gate_proj", + "model.layers.16.mlp.experts.81.up_proj", + "model.layers.16.mlp.experts.82.down_proj", + "model.layers.16.mlp.experts.82.gate_proj", + "model.layers.16.mlp.experts.82.up_proj", + "model.layers.16.mlp.experts.83.down_proj", + "model.layers.16.mlp.experts.83.gate_proj", + "model.layers.16.mlp.experts.83.up_proj", + "model.layers.16.mlp.experts.84.down_proj", + "model.layers.16.mlp.experts.84.gate_proj", + "model.layers.16.mlp.experts.84.up_proj", + "model.layers.16.mlp.experts.85.down_proj", + "model.layers.16.mlp.experts.85.gate_proj", + "model.layers.16.mlp.experts.85.up_proj", + "model.layers.16.mlp.experts.86.down_proj", + "model.layers.16.mlp.experts.86.gate_proj", + "model.layers.16.mlp.experts.86.up_proj", + "model.layers.16.mlp.experts.87.down_proj", + "model.layers.16.mlp.experts.87.gate_proj", + "model.layers.16.mlp.experts.87.up_proj", + "model.layers.16.mlp.experts.88.down_proj", + "model.layers.16.mlp.experts.88.gate_proj", + "model.layers.16.mlp.experts.88.up_proj", + "model.layers.16.mlp.experts.89.down_proj", + "model.layers.16.mlp.experts.89.gate_proj", + "model.layers.16.mlp.experts.89.up_proj", + "model.layers.16.mlp.experts.9.down_proj", + "model.layers.16.mlp.experts.9.gate_proj", + "model.layers.16.mlp.experts.9.up_proj", + "model.layers.16.mlp.experts.90.down_proj", + "model.layers.16.mlp.experts.90.gate_proj", + "model.layers.16.mlp.experts.90.up_proj", + "model.layers.16.mlp.experts.91.down_proj", + "model.layers.16.mlp.experts.91.gate_proj", + "model.layers.16.mlp.experts.91.up_proj", + "model.layers.16.mlp.experts.92.down_proj", + "model.layers.16.mlp.experts.92.gate_proj", + "model.layers.16.mlp.experts.92.up_proj", + "model.layers.16.mlp.experts.93.down_proj", + "model.layers.16.mlp.experts.93.gate_proj", + "model.layers.16.mlp.experts.93.up_proj", + "model.layers.16.mlp.experts.94.down_proj", + "model.layers.16.mlp.experts.94.gate_proj", + "model.layers.16.mlp.experts.94.up_proj", + "model.layers.16.mlp.experts.95.down_proj", + "model.layers.16.mlp.experts.95.gate_proj", + "model.layers.16.mlp.experts.95.up_proj", + "model.layers.16.mlp.experts.96.down_proj", + "model.layers.16.mlp.experts.96.gate_proj", + "model.layers.16.mlp.experts.96.up_proj", + "model.layers.16.mlp.experts.97.down_proj", + "model.layers.16.mlp.experts.97.gate_proj", + "model.layers.16.mlp.experts.97.up_proj", + "model.layers.16.mlp.experts.98.down_proj", + "model.layers.16.mlp.experts.98.gate_proj", + "model.layers.16.mlp.experts.98.up_proj", + "model.layers.16.mlp.experts.99.down_proj", + "model.layers.16.mlp.experts.99.gate_proj", + "model.layers.16.mlp.experts.99.up_proj", + "model.layers.17.mlp.experts.0.down_proj", + "model.layers.17.mlp.experts.0.gate_proj", + "model.layers.17.mlp.experts.0.up_proj", + "model.layers.17.mlp.experts.1.down_proj", + "model.layers.17.mlp.experts.1.gate_proj", + "model.layers.17.mlp.experts.1.up_proj", + "model.layers.17.mlp.experts.10.down_proj", + "model.layers.17.mlp.experts.10.gate_proj", + "model.layers.17.mlp.experts.10.up_proj", + "model.layers.17.mlp.experts.100.down_proj", + "model.layers.17.mlp.experts.100.gate_proj", + "model.layers.17.mlp.experts.100.up_proj", + "model.layers.17.mlp.experts.101.down_proj", + "model.layers.17.mlp.experts.101.gate_proj", + "model.layers.17.mlp.experts.101.up_proj", + "model.layers.17.mlp.experts.102.down_proj", + "model.layers.17.mlp.experts.102.gate_proj", + "model.layers.17.mlp.experts.102.up_proj", + "model.layers.17.mlp.experts.103.down_proj", + "model.layers.17.mlp.experts.103.gate_proj", + "model.layers.17.mlp.experts.103.up_proj", + "model.layers.17.mlp.experts.104.down_proj", + "model.layers.17.mlp.experts.104.gate_proj", + "model.layers.17.mlp.experts.104.up_proj", + "model.layers.17.mlp.experts.105.down_proj", + "model.layers.17.mlp.experts.105.gate_proj", + "model.layers.17.mlp.experts.105.up_proj", + "model.layers.17.mlp.experts.106.down_proj", + "model.layers.17.mlp.experts.106.gate_proj", + "model.layers.17.mlp.experts.106.up_proj", + "model.layers.17.mlp.experts.107.down_proj", + "model.layers.17.mlp.experts.107.gate_proj", + "model.layers.17.mlp.experts.107.up_proj", + "model.layers.17.mlp.experts.108.down_proj", + "model.layers.17.mlp.experts.108.gate_proj", + "model.layers.17.mlp.experts.108.up_proj", + "model.layers.17.mlp.experts.109.down_proj", + "model.layers.17.mlp.experts.109.gate_proj", + "model.layers.17.mlp.experts.109.up_proj", + "model.layers.17.mlp.experts.11.down_proj", + "model.layers.17.mlp.experts.11.gate_proj", + "model.layers.17.mlp.experts.11.up_proj", + "model.layers.17.mlp.experts.110.down_proj", + "model.layers.17.mlp.experts.110.gate_proj", + "model.layers.17.mlp.experts.110.up_proj", + "model.layers.17.mlp.experts.111.down_proj", + "model.layers.17.mlp.experts.111.gate_proj", + "model.layers.17.mlp.experts.111.up_proj", + "model.layers.17.mlp.experts.112.down_proj", + "model.layers.17.mlp.experts.112.gate_proj", + "model.layers.17.mlp.experts.112.up_proj", + "model.layers.17.mlp.experts.113.down_proj", + "model.layers.17.mlp.experts.113.gate_proj", + "model.layers.17.mlp.experts.113.up_proj", + "model.layers.17.mlp.experts.114.down_proj", + "model.layers.17.mlp.experts.114.gate_proj", + "model.layers.17.mlp.experts.114.up_proj", + "model.layers.17.mlp.experts.115.down_proj", + "model.layers.17.mlp.experts.115.gate_proj", + "model.layers.17.mlp.experts.115.up_proj", + "model.layers.17.mlp.experts.116.down_proj", + "model.layers.17.mlp.experts.116.gate_proj", + "model.layers.17.mlp.experts.116.up_proj", + "model.layers.17.mlp.experts.117.down_proj", + "model.layers.17.mlp.experts.117.gate_proj", + "model.layers.17.mlp.experts.117.up_proj", + "model.layers.17.mlp.experts.118.down_proj", + "model.layers.17.mlp.experts.118.gate_proj", + "model.layers.17.mlp.experts.118.up_proj", + "model.layers.17.mlp.experts.119.down_proj", + "model.layers.17.mlp.experts.119.gate_proj", + "model.layers.17.mlp.experts.119.up_proj", + "model.layers.17.mlp.experts.12.down_proj", + "model.layers.17.mlp.experts.12.gate_proj", + "model.layers.17.mlp.experts.12.up_proj", + "model.layers.17.mlp.experts.120.down_proj", + "model.layers.17.mlp.experts.120.gate_proj", + "model.layers.17.mlp.experts.120.up_proj", + "model.layers.17.mlp.experts.121.down_proj", + "model.layers.17.mlp.experts.121.gate_proj", + "model.layers.17.mlp.experts.121.up_proj", + "model.layers.17.mlp.experts.122.down_proj", + "model.layers.17.mlp.experts.122.gate_proj", + "model.layers.17.mlp.experts.122.up_proj", + "model.layers.17.mlp.experts.123.down_proj", + "model.layers.17.mlp.experts.123.gate_proj", + "model.layers.17.mlp.experts.123.up_proj", + "model.layers.17.mlp.experts.124.down_proj", + "model.layers.17.mlp.experts.124.gate_proj", + "model.layers.17.mlp.experts.124.up_proj", + "model.layers.17.mlp.experts.125.down_proj", + "model.layers.17.mlp.experts.125.gate_proj", + "model.layers.17.mlp.experts.125.up_proj", + "model.layers.17.mlp.experts.126.down_proj", + "model.layers.17.mlp.experts.126.gate_proj", + "model.layers.17.mlp.experts.126.up_proj", + "model.layers.17.mlp.experts.127.down_proj", + "model.layers.17.mlp.experts.127.gate_proj", + "model.layers.17.mlp.experts.127.up_proj", + "model.layers.17.mlp.experts.128.down_proj", + "model.layers.17.mlp.experts.128.gate_proj", + "model.layers.17.mlp.experts.128.up_proj", + "model.layers.17.mlp.experts.129.down_proj", + "model.layers.17.mlp.experts.129.gate_proj", + "model.layers.17.mlp.experts.129.up_proj", + "model.layers.17.mlp.experts.13.down_proj", + "model.layers.17.mlp.experts.13.gate_proj", + "model.layers.17.mlp.experts.13.up_proj", + "model.layers.17.mlp.experts.130.down_proj", + "model.layers.17.mlp.experts.130.gate_proj", + "model.layers.17.mlp.experts.130.up_proj", + "model.layers.17.mlp.experts.131.down_proj", + "model.layers.17.mlp.experts.131.gate_proj", + "model.layers.17.mlp.experts.131.up_proj", + "model.layers.17.mlp.experts.132.down_proj", + "model.layers.17.mlp.experts.132.gate_proj", + "model.layers.17.mlp.experts.132.up_proj", + "model.layers.17.mlp.experts.133.down_proj", + "model.layers.17.mlp.experts.133.gate_proj", + "model.layers.17.mlp.experts.133.up_proj", + "model.layers.17.mlp.experts.134.down_proj", + "model.layers.17.mlp.experts.134.gate_proj", + "model.layers.17.mlp.experts.134.up_proj", + "model.layers.17.mlp.experts.135.down_proj", + "model.layers.17.mlp.experts.135.gate_proj", + "model.layers.17.mlp.experts.135.up_proj", + "model.layers.17.mlp.experts.136.down_proj", + "model.layers.17.mlp.experts.136.gate_proj", + "model.layers.17.mlp.experts.136.up_proj", + "model.layers.17.mlp.experts.137.down_proj", + "model.layers.17.mlp.experts.137.gate_proj", + "model.layers.17.mlp.experts.137.up_proj", + "model.layers.17.mlp.experts.138.down_proj", + "model.layers.17.mlp.experts.138.gate_proj", + "model.layers.17.mlp.experts.138.up_proj", + "model.layers.17.mlp.experts.139.down_proj", + "model.layers.17.mlp.experts.139.gate_proj", + "model.layers.17.mlp.experts.139.up_proj", + "model.layers.17.mlp.experts.14.down_proj", + "model.layers.17.mlp.experts.14.gate_proj", + "model.layers.17.mlp.experts.14.up_proj", + "model.layers.17.mlp.experts.140.down_proj", + "model.layers.17.mlp.experts.140.gate_proj", + "model.layers.17.mlp.experts.140.up_proj", + "model.layers.17.mlp.experts.141.down_proj", + "model.layers.17.mlp.experts.141.gate_proj", + "model.layers.17.mlp.experts.141.up_proj", + "model.layers.17.mlp.experts.142.down_proj", + "model.layers.17.mlp.experts.142.gate_proj", + "model.layers.17.mlp.experts.142.up_proj", + "model.layers.17.mlp.experts.143.down_proj", + "model.layers.17.mlp.experts.143.gate_proj", + "model.layers.17.mlp.experts.143.up_proj", + "model.layers.17.mlp.experts.144.down_proj", + "model.layers.17.mlp.experts.144.gate_proj", + "model.layers.17.mlp.experts.144.up_proj", + "model.layers.17.mlp.experts.145.down_proj", + "model.layers.17.mlp.experts.145.gate_proj", + "model.layers.17.mlp.experts.145.up_proj", + "model.layers.17.mlp.experts.146.down_proj", + "model.layers.17.mlp.experts.146.gate_proj", + "model.layers.17.mlp.experts.146.up_proj", + "model.layers.17.mlp.experts.147.down_proj", + "model.layers.17.mlp.experts.147.gate_proj", + "model.layers.17.mlp.experts.147.up_proj", + "model.layers.17.mlp.experts.148.down_proj", + "model.layers.17.mlp.experts.148.gate_proj", + "model.layers.17.mlp.experts.148.up_proj", + "model.layers.17.mlp.experts.149.down_proj", + "model.layers.17.mlp.experts.149.gate_proj", + "model.layers.17.mlp.experts.149.up_proj", + "model.layers.17.mlp.experts.15.down_proj", + "model.layers.17.mlp.experts.15.gate_proj", + "model.layers.17.mlp.experts.15.up_proj", + "model.layers.17.mlp.experts.150.down_proj", + "model.layers.17.mlp.experts.150.gate_proj", + "model.layers.17.mlp.experts.150.up_proj", + "model.layers.17.mlp.experts.151.down_proj", + "model.layers.17.mlp.experts.151.gate_proj", + "model.layers.17.mlp.experts.151.up_proj", + "model.layers.17.mlp.experts.152.down_proj", + "model.layers.17.mlp.experts.152.gate_proj", + "model.layers.17.mlp.experts.152.up_proj", + "model.layers.17.mlp.experts.153.down_proj", + "model.layers.17.mlp.experts.153.gate_proj", + "model.layers.17.mlp.experts.153.up_proj", + "model.layers.17.mlp.experts.154.down_proj", + "model.layers.17.mlp.experts.154.gate_proj", + "model.layers.17.mlp.experts.154.up_proj", + "model.layers.17.mlp.experts.155.down_proj", + "model.layers.17.mlp.experts.155.gate_proj", + "model.layers.17.mlp.experts.155.up_proj", + "model.layers.17.mlp.experts.156.down_proj", + "model.layers.17.mlp.experts.156.gate_proj", + "model.layers.17.mlp.experts.156.up_proj", + "model.layers.17.mlp.experts.157.down_proj", + "model.layers.17.mlp.experts.157.gate_proj", + "model.layers.17.mlp.experts.157.up_proj", + "model.layers.17.mlp.experts.158.down_proj", + "model.layers.17.mlp.experts.158.gate_proj", + "model.layers.17.mlp.experts.158.up_proj", + "model.layers.17.mlp.experts.159.down_proj", + "model.layers.17.mlp.experts.159.gate_proj", + "model.layers.17.mlp.experts.159.up_proj", + "model.layers.17.mlp.experts.16.down_proj", + "model.layers.17.mlp.experts.16.gate_proj", + "model.layers.17.mlp.experts.16.up_proj", + "model.layers.17.mlp.experts.160.down_proj", + "model.layers.17.mlp.experts.160.gate_proj", + "model.layers.17.mlp.experts.160.up_proj", + "model.layers.17.mlp.experts.161.down_proj", + "model.layers.17.mlp.experts.161.gate_proj", + "model.layers.17.mlp.experts.161.up_proj", + "model.layers.17.mlp.experts.162.down_proj", + "model.layers.17.mlp.experts.162.gate_proj", + "model.layers.17.mlp.experts.162.up_proj", + "model.layers.17.mlp.experts.163.down_proj", + "model.layers.17.mlp.experts.163.gate_proj", + "model.layers.17.mlp.experts.163.up_proj", + "model.layers.17.mlp.experts.164.down_proj", + "model.layers.17.mlp.experts.164.gate_proj", + "model.layers.17.mlp.experts.164.up_proj", + "model.layers.17.mlp.experts.165.down_proj", + "model.layers.17.mlp.experts.165.gate_proj", + "model.layers.17.mlp.experts.165.up_proj", + "model.layers.17.mlp.experts.166.down_proj", + "model.layers.17.mlp.experts.166.gate_proj", + "model.layers.17.mlp.experts.166.up_proj", + "model.layers.17.mlp.experts.167.down_proj", + "model.layers.17.mlp.experts.167.gate_proj", + "model.layers.17.mlp.experts.167.up_proj", + "model.layers.17.mlp.experts.168.down_proj", + "model.layers.17.mlp.experts.168.gate_proj", + "model.layers.17.mlp.experts.168.up_proj", + "model.layers.17.mlp.experts.169.down_proj", + "model.layers.17.mlp.experts.169.gate_proj", + "model.layers.17.mlp.experts.169.up_proj", + "model.layers.17.mlp.experts.17.down_proj", + "model.layers.17.mlp.experts.17.gate_proj", + "model.layers.17.mlp.experts.17.up_proj", + "model.layers.17.mlp.experts.170.down_proj", + "model.layers.17.mlp.experts.170.gate_proj", + "model.layers.17.mlp.experts.170.up_proj", + "model.layers.17.mlp.experts.171.down_proj", + "model.layers.17.mlp.experts.171.gate_proj", + "model.layers.17.mlp.experts.171.up_proj", + "model.layers.17.mlp.experts.172.down_proj", + "model.layers.17.mlp.experts.172.gate_proj", + "model.layers.17.mlp.experts.172.up_proj", + "model.layers.17.mlp.experts.173.down_proj", + "model.layers.17.mlp.experts.173.gate_proj", + "model.layers.17.mlp.experts.173.up_proj", + "model.layers.17.mlp.experts.174.down_proj", + "model.layers.17.mlp.experts.174.gate_proj", + "model.layers.17.mlp.experts.174.up_proj", + "model.layers.17.mlp.experts.175.down_proj", + "model.layers.17.mlp.experts.175.gate_proj", + "model.layers.17.mlp.experts.175.up_proj", + "model.layers.17.mlp.experts.176.down_proj", + "model.layers.17.mlp.experts.176.gate_proj", + "model.layers.17.mlp.experts.176.up_proj", + "model.layers.17.mlp.experts.177.down_proj", + "model.layers.17.mlp.experts.177.gate_proj", + "model.layers.17.mlp.experts.177.up_proj", + "model.layers.17.mlp.experts.178.down_proj", + "model.layers.17.mlp.experts.178.gate_proj", + "model.layers.17.mlp.experts.178.up_proj", + "model.layers.17.mlp.experts.179.down_proj", + "model.layers.17.mlp.experts.179.gate_proj", + "model.layers.17.mlp.experts.179.up_proj", + "model.layers.17.mlp.experts.18.down_proj", + "model.layers.17.mlp.experts.18.gate_proj", + "model.layers.17.mlp.experts.18.up_proj", + "model.layers.17.mlp.experts.180.down_proj", + "model.layers.17.mlp.experts.180.gate_proj", + "model.layers.17.mlp.experts.180.up_proj", + "model.layers.17.mlp.experts.181.down_proj", + "model.layers.17.mlp.experts.181.gate_proj", + "model.layers.17.mlp.experts.181.up_proj", + "model.layers.17.mlp.experts.182.down_proj", + "model.layers.17.mlp.experts.182.gate_proj", + "model.layers.17.mlp.experts.182.up_proj", + "model.layers.17.mlp.experts.183.down_proj", + "model.layers.17.mlp.experts.183.gate_proj", + "model.layers.17.mlp.experts.183.up_proj", + "model.layers.17.mlp.experts.184.down_proj", + "model.layers.17.mlp.experts.184.gate_proj", + "model.layers.17.mlp.experts.184.up_proj", + "model.layers.17.mlp.experts.185.down_proj", + "model.layers.17.mlp.experts.185.gate_proj", + "model.layers.17.mlp.experts.185.up_proj", + "model.layers.17.mlp.experts.186.down_proj", + "model.layers.17.mlp.experts.186.gate_proj", + "model.layers.17.mlp.experts.186.up_proj", + "model.layers.17.mlp.experts.187.down_proj", + "model.layers.17.mlp.experts.187.gate_proj", + "model.layers.17.mlp.experts.187.up_proj", + "model.layers.17.mlp.experts.188.down_proj", + "model.layers.17.mlp.experts.188.gate_proj", + "model.layers.17.mlp.experts.188.up_proj", + "model.layers.17.mlp.experts.189.down_proj", + "model.layers.17.mlp.experts.189.gate_proj", + "model.layers.17.mlp.experts.189.up_proj", + "model.layers.17.mlp.experts.19.down_proj", + "model.layers.17.mlp.experts.19.gate_proj", + "model.layers.17.mlp.experts.19.up_proj", + "model.layers.17.mlp.experts.190.down_proj", + "model.layers.17.mlp.experts.190.gate_proj", + "model.layers.17.mlp.experts.190.up_proj", + "model.layers.17.mlp.experts.191.down_proj", + "model.layers.17.mlp.experts.191.gate_proj", + "model.layers.17.mlp.experts.191.up_proj", + "model.layers.17.mlp.experts.192.down_proj", + "model.layers.17.mlp.experts.192.gate_proj", + "model.layers.17.mlp.experts.192.up_proj", + "model.layers.17.mlp.experts.193.down_proj", + "model.layers.17.mlp.experts.193.gate_proj", + "model.layers.17.mlp.experts.193.up_proj", + "model.layers.17.mlp.experts.194.down_proj", + "model.layers.17.mlp.experts.194.gate_proj", + "model.layers.17.mlp.experts.194.up_proj", + "model.layers.17.mlp.experts.195.down_proj", + "model.layers.17.mlp.experts.195.gate_proj", + "model.layers.17.mlp.experts.195.up_proj", + "model.layers.17.mlp.experts.196.down_proj", + "model.layers.17.mlp.experts.196.gate_proj", + "model.layers.17.mlp.experts.196.up_proj", + "model.layers.17.mlp.experts.197.down_proj", + "model.layers.17.mlp.experts.197.gate_proj", + "model.layers.17.mlp.experts.197.up_proj", + "model.layers.17.mlp.experts.198.down_proj", + "model.layers.17.mlp.experts.198.gate_proj", + "model.layers.17.mlp.experts.198.up_proj", + "model.layers.17.mlp.experts.199.down_proj", + "model.layers.17.mlp.experts.199.gate_proj", + "model.layers.17.mlp.experts.199.up_proj", + "model.layers.17.mlp.experts.2.down_proj", + "model.layers.17.mlp.experts.2.gate_proj", + "model.layers.17.mlp.experts.2.up_proj", + "model.layers.17.mlp.experts.20.down_proj", + "model.layers.17.mlp.experts.20.gate_proj", + "model.layers.17.mlp.experts.20.up_proj", + "model.layers.17.mlp.experts.200.down_proj", + "model.layers.17.mlp.experts.200.gate_proj", + "model.layers.17.mlp.experts.200.up_proj", + "model.layers.17.mlp.experts.201.down_proj", + "model.layers.17.mlp.experts.201.gate_proj", + "model.layers.17.mlp.experts.201.up_proj", + "model.layers.17.mlp.experts.202.down_proj", + "model.layers.17.mlp.experts.202.gate_proj", + "model.layers.17.mlp.experts.202.up_proj", + "model.layers.17.mlp.experts.203.down_proj", + "model.layers.17.mlp.experts.203.gate_proj", + "model.layers.17.mlp.experts.203.up_proj", + "model.layers.17.mlp.experts.204.down_proj", + "model.layers.17.mlp.experts.204.gate_proj", + "model.layers.17.mlp.experts.204.up_proj", + "model.layers.17.mlp.experts.205.down_proj", + "model.layers.17.mlp.experts.205.gate_proj", + "model.layers.17.mlp.experts.205.up_proj", + "model.layers.17.mlp.experts.206.down_proj", + "model.layers.17.mlp.experts.206.gate_proj", + "model.layers.17.mlp.experts.206.up_proj", + "model.layers.17.mlp.experts.207.down_proj", + "model.layers.17.mlp.experts.207.gate_proj", + "model.layers.17.mlp.experts.207.up_proj", + "model.layers.17.mlp.experts.208.down_proj", + "model.layers.17.mlp.experts.208.gate_proj", + "model.layers.17.mlp.experts.208.up_proj", + "model.layers.17.mlp.experts.209.down_proj", + "model.layers.17.mlp.experts.209.gate_proj", + "model.layers.17.mlp.experts.209.up_proj", + "model.layers.17.mlp.experts.21.down_proj", + "model.layers.17.mlp.experts.21.gate_proj", + "model.layers.17.mlp.experts.21.up_proj", + "model.layers.17.mlp.experts.210.down_proj", + "model.layers.17.mlp.experts.210.gate_proj", + "model.layers.17.mlp.experts.210.up_proj", + "model.layers.17.mlp.experts.211.down_proj", + "model.layers.17.mlp.experts.211.gate_proj", + "model.layers.17.mlp.experts.211.up_proj", + "model.layers.17.mlp.experts.212.down_proj", + "model.layers.17.mlp.experts.212.gate_proj", + "model.layers.17.mlp.experts.212.up_proj", + "model.layers.17.mlp.experts.213.down_proj", + "model.layers.17.mlp.experts.213.gate_proj", + "model.layers.17.mlp.experts.213.up_proj", + "model.layers.17.mlp.experts.214.down_proj", + "model.layers.17.mlp.experts.214.gate_proj", + "model.layers.17.mlp.experts.214.up_proj", + "model.layers.17.mlp.experts.215.down_proj", + "model.layers.17.mlp.experts.215.gate_proj", + "model.layers.17.mlp.experts.215.up_proj", + "model.layers.17.mlp.experts.216.down_proj", + "model.layers.17.mlp.experts.216.gate_proj", + "model.layers.17.mlp.experts.216.up_proj", + "model.layers.17.mlp.experts.217.down_proj", + "model.layers.17.mlp.experts.217.gate_proj", + "model.layers.17.mlp.experts.217.up_proj", + "model.layers.17.mlp.experts.218.down_proj", + "model.layers.17.mlp.experts.218.gate_proj", + "model.layers.17.mlp.experts.218.up_proj", + "model.layers.17.mlp.experts.219.down_proj", + "model.layers.17.mlp.experts.219.gate_proj", + "model.layers.17.mlp.experts.219.up_proj", + "model.layers.17.mlp.experts.22.down_proj", + "model.layers.17.mlp.experts.22.gate_proj", + "model.layers.17.mlp.experts.22.up_proj", + "model.layers.17.mlp.experts.220.down_proj", + "model.layers.17.mlp.experts.220.gate_proj", + "model.layers.17.mlp.experts.220.up_proj", + "model.layers.17.mlp.experts.221.down_proj", + "model.layers.17.mlp.experts.221.gate_proj", + "model.layers.17.mlp.experts.221.up_proj", + "model.layers.17.mlp.experts.222.down_proj", + "model.layers.17.mlp.experts.222.gate_proj", + "model.layers.17.mlp.experts.222.up_proj", + "model.layers.17.mlp.experts.223.down_proj", + "model.layers.17.mlp.experts.223.gate_proj", + "model.layers.17.mlp.experts.223.up_proj", + "model.layers.17.mlp.experts.224.down_proj", + "model.layers.17.mlp.experts.224.gate_proj", + "model.layers.17.mlp.experts.224.up_proj", + "model.layers.17.mlp.experts.225.down_proj", + "model.layers.17.mlp.experts.225.gate_proj", + "model.layers.17.mlp.experts.225.up_proj", + "model.layers.17.mlp.experts.226.down_proj", + "model.layers.17.mlp.experts.226.gate_proj", + "model.layers.17.mlp.experts.226.up_proj", + "model.layers.17.mlp.experts.227.down_proj", + "model.layers.17.mlp.experts.227.gate_proj", + "model.layers.17.mlp.experts.227.up_proj", + "model.layers.17.mlp.experts.228.down_proj", + "model.layers.17.mlp.experts.228.gate_proj", + "model.layers.17.mlp.experts.228.up_proj", + "model.layers.17.mlp.experts.229.down_proj", + "model.layers.17.mlp.experts.229.gate_proj", + "model.layers.17.mlp.experts.229.up_proj", + "model.layers.17.mlp.experts.23.down_proj", + "model.layers.17.mlp.experts.23.gate_proj", + "model.layers.17.mlp.experts.23.up_proj", + "model.layers.17.mlp.experts.230.down_proj", + "model.layers.17.mlp.experts.230.gate_proj", + "model.layers.17.mlp.experts.230.up_proj", + "model.layers.17.mlp.experts.231.down_proj", + "model.layers.17.mlp.experts.231.gate_proj", + "model.layers.17.mlp.experts.231.up_proj", + "model.layers.17.mlp.experts.232.down_proj", + "model.layers.17.mlp.experts.232.gate_proj", + "model.layers.17.mlp.experts.232.up_proj", + "model.layers.17.mlp.experts.233.down_proj", + "model.layers.17.mlp.experts.233.gate_proj", + "model.layers.17.mlp.experts.233.up_proj", + "model.layers.17.mlp.experts.234.down_proj", + "model.layers.17.mlp.experts.234.gate_proj", + "model.layers.17.mlp.experts.234.up_proj", + "model.layers.17.mlp.experts.235.down_proj", + "model.layers.17.mlp.experts.235.gate_proj", + "model.layers.17.mlp.experts.235.up_proj", + "model.layers.17.mlp.experts.236.down_proj", + "model.layers.17.mlp.experts.236.gate_proj", + "model.layers.17.mlp.experts.236.up_proj", + "model.layers.17.mlp.experts.237.down_proj", + "model.layers.17.mlp.experts.237.gate_proj", + "model.layers.17.mlp.experts.237.up_proj", + "model.layers.17.mlp.experts.238.down_proj", + "model.layers.17.mlp.experts.238.gate_proj", + "model.layers.17.mlp.experts.238.up_proj", + "model.layers.17.mlp.experts.239.down_proj", + "model.layers.17.mlp.experts.239.gate_proj", + "model.layers.17.mlp.experts.239.up_proj", + "model.layers.17.mlp.experts.24.down_proj", + "model.layers.17.mlp.experts.24.gate_proj", + "model.layers.17.mlp.experts.24.up_proj", + "model.layers.17.mlp.experts.240.down_proj", + "model.layers.17.mlp.experts.240.gate_proj", + "model.layers.17.mlp.experts.240.up_proj", + "model.layers.17.mlp.experts.241.down_proj", + "model.layers.17.mlp.experts.241.gate_proj", + "model.layers.17.mlp.experts.241.up_proj", + "model.layers.17.mlp.experts.242.down_proj", + "model.layers.17.mlp.experts.242.gate_proj", + "model.layers.17.mlp.experts.242.up_proj", + "model.layers.17.mlp.experts.243.down_proj", + "model.layers.17.mlp.experts.243.gate_proj", + "model.layers.17.mlp.experts.243.up_proj", + "model.layers.17.mlp.experts.244.down_proj", + "model.layers.17.mlp.experts.244.gate_proj", + "model.layers.17.mlp.experts.244.up_proj", + "model.layers.17.mlp.experts.245.down_proj", + "model.layers.17.mlp.experts.245.gate_proj", + "model.layers.17.mlp.experts.245.up_proj", + "model.layers.17.mlp.experts.246.down_proj", + "model.layers.17.mlp.experts.246.gate_proj", + "model.layers.17.mlp.experts.246.up_proj", + "model.layers.17.mlp.experts.247.down_proj", + "model.layers.17.mlp.experts.247.gate_proj", + "model.layers.17.mlp.experts.247.up_proj", + "model.layers.17.mlp.experts.248.down_proj", + "model.layers.17.mlp.experts.248.gate_proj", + "model.layers.17.mlp.experts.248.up_proj", + "model.layers.17.mlp.experts.249.down_proj", + "model.layers.17.mlp.experts.249.gate_proj", + "model.layers.17.mlp.experts.249.up_proj", + "model.layers.17.mlp.experts.25.down_proj", + "model.layers.17.mlp.experts.25.gate_proj", + "model.layers.17.mlp.experts.25.up_proj", + "model.layers.17.mlp.experts.250.down_proj", + "model.layers.17.mlp.experts.250.gate_proj", + "model.layers.17.mlp.experts.250.up_proj", + "model.layers.17.mlp.experts.251.down_proj", + "model.layers.17.mlp.experts.251.gate_proj", + "model.layers.17.mlp.experts.251.up_proj", + "model.layers.17.mlp.experts.252.down_proj", + "model.layers.17.mlp.experts.252.gate_proj", + "model.layers.17.mlp.experts.252.up_proj", + "model.layers.17.mlp.experts.253.down_proj", + "model.layers.17.mlp.experts.253.gate_proj", + "model.layers.17.mlp.experts.253.up_proj", + "model.layers.17.mlp.experts.254.down_proj", + "model.layers.17.mlp.experts.254.gate_proj", + "model.layers.17.mlp.experts.254.up_proj", + "model.layers.17.mlp.experts.255.down_proj", + "model.layers.17.mlp.experts.255.gate_proj", + "model.layers.17.mlp.experts.255.up_proj", + "model.layers.17.mlp.experts.26.down_proj", + "model.layers.17.mlp.experts.26.gate_proj", + "model.layers.17.mlp.experts.26.up_proj", + "model.layers.17.mlp.experts.27.down_proj", + "model.layers.17.mlp.experts.27.gate_proj", + "model.layers.17.mlp.experts.27.up_proj", + "model.layers.17.mlp.experts.28.down_proj", + "model.layers.17.mlp.experts.28.gate_proj", + "model.layers.17.mlp.experts.28.up_proj", + "model.layers.17.mlp.experts.29.down_proj", + "model.layers.17.mlp.experts.29.gate_proj", + "model.layers.17.mlp.experts.29.up_proj", + "model.layers.17.mlp.experts.3.down_proj", + "model.layers.17.mlp.experts.3.gate_proj", + "model.layers.17.mlp.experts.3.up_proj", + "model.layers.17.mlp.experts.30.down_proj", + "model.layers.17.mlp.experts.30.gate_proj", + "model.layers.17.mlp.experts.30.up_proj", + "model.layers.17.mlp.experts.31.down_proj", + "model.layers.17.mlp.experts.31.gate_proj", + "model.layers.17.mlp.experts.31.up_proj", + "model.layers.17.mlp.experts.32.down_proj", + "model.layers.17.mlp.experts.32.gate_proj", + "model.layers.17.mlp.experts.32.up_proj", + "model.layers.17.mlp.experts.33.down_proj", + "model.layers.17.mlp.experts.33.gate_proj", + "model.layers.17.mlp.experts.33.up_proj", + "model.layers.17.mlp.experts.34.down_proj", + "model.layers.17.mlp.experts.34.gate_proj", + "model.layers.17.mlp.experts.34.up_proj", + "model.layers.17.mlp.experts.35.down_proj", + "model.layers.17.mlp.experts.35.gate_proj", + "model.layers.17.mlp.experts.35.up_proj", + "model.layers.17.mlp.experts.36.down_proj", + "model.layers.17.mlp.experts.36.gate_proj", + "model.layers.17.mlp.experts.36.up_proj", + "model.layers.17.mlp.experts.37.down_proj", + "model.layers.17.mlp.experts.37.gate_proj", + "model.layers.17.mlp.experts.37.up_proj", + "model.layers.17.mlp.experts.38.down_proj", + "model.layers.17.mlp.experts.38.gate_proj", + "model.layers.17.mlp.experts.38.up_proj", + "model.layers.17.mlp.experts.39.down_proj", + "model.layers.17.mlp.experts.39.gate_proj", + "model.layers.17.mlp.experts.39.up_proj", + "model.layers.17.mlp.experts.4.down_proj", + "model.layers.17.mlp.experts.4.gate_proj", + "model.layers.17.mlp.experts.4.up_proj", + "model.layers.17.mlp.experts.40.down_proj", + "model.layers.17.mlp.experts.40.gate_proj", + "model.layers.17.mlp.experts.40.up_proj", + "model.layers.17.mlp.experts.41.down_proj", + "model.layers.17.mlp.experts.41.gate_proj", + "model.layers.17.mlp.experts.41.up_proj", + "model.layers.17.mlp.experts.42.down_proj", + "model.layers.17.mlp.experts.42.gate_proj", + "model.layers.17.mlp.experts.42.up_proj", + "model.layers.17.mlp.experts.43.down_proj", + "model.layers.17.mlp.experts.43.gate_proj", + "model.layers.17.mlp.experts.43.up_proj", + "model.layers.17.mlp.experts.44.down_proj", + "model.layers.17.mlp.experts.44.gate_proj", + "model.layers.17.mlp.experts.44.up_proj", + "model.layers.17.mlp.experts.45.down_proj", + "model.layers.17.mlp.experts.45.gate_proj", + "model.layers.17.mlp.experts.45.up_proj", + "model.layers.17.mlp.experts.46.down_proj", + "model.layers.17.mlp.experts.46.gate_proj", + "model.layers.17.mlp.experts.46.up_proj", + "model.layers.17.mlp.experts.47.down_proj", + "model.layers.17.mlp.experts.47.gate_proj", + "model.layers.17.mlp.experts.47.up_proj", + "model.layers.17.mlp.experts.48.down_proj", + "model.layers.17.mlp.experts.48.gate_proj", + "model.layers.17.mlp.experts.48.up_proj", + "model.layers.17.mlp.experts.49.down_proj", + "model.layers.17.mlp.experts.49.gate_proj", + "model.layers.17.mlp.experts.49.up_proj", + "model.layers.17.mlp.experts.5.down_proj", + "model.layers.17.mlp.experts.5.gate_proj", + "model.layers.17.mlp.experts.5.up_proj", + "model.layers.17.mlp.experts.50.down_proj", + "model.layers.17.mlp.experts.50.gate_proj", + "model.layers.17.mlp.experts.50.up_proj", + "model.layers.17.mlp.experts.51.down_proj", + "model.layers.17.mlp.experts.51.gate_proj", + "model.layers.17.mlp.experts.51.up_proj", + "model.layers.17.mlp.experts.52.down_proj", + "model.layers.17.mlp.experts.52.gate_proj", + "model.layers.17.mlp.experts.52.up_proj", + "model.layers.17.mlp.experts.53.down_proj", + "model.layers.17.mlp.experts.53.gate_proj", + "model.layers.17.mlp.experts.53.up_proj", + "model.layers.17.mlp.experts.54.down_proj", + "model.layers.17.mlp.experts.54.gate_proj", + "model.layers.17.mlp.experts.54.up_proj", + "model.layers.17.mlp.experts.55.down_proj", + "model.layers.17.mlp.experts.55.gate_proj", + "model.layers.17.mlp.experts.55.up_proj", + "model.layers.17.mlp.experts.56.down_proj", + "model.layers.17.mlp.experts.56.gate_proj", + "model.layers.17.mlp.experts.56.up_proj", + "model.layers.17.mlp.experts.57.down_proj", + "model.layers.17.mlp.experts.57.gate_proj", + "model.layers.17.mlp.experts.57.up_proj", + "model.layers.17.mlp.experts.58.down_proj", + "model.layers.17.mlp.experts.58.gate_proj", + "model.layers.17.mlp.experts.58.up_proj", + "model.layers.17.mlp.experts.59.down_proj", + "model.layers.17.mlp.experts.59.gate_proj", + "model.layers.17.mlp.experts.59.up_proj", + "model.layers.17.mlp.experts.6.down_proj", + "model.layers.17.mlp.experts.6.gate_proj", + "model.layers.17.mlp.experts.6.up_proj", + "model.layers.17.mlp.experts.60.down_proj", + "model.layers.17.mlp.experts.60.gate_proj", + "model.layers.17.mlp.experts.60.up_proj", + "model.layers.17.mlp.experts.61.down_proj", + "model.layers.17.mlp.experts.61.gate_proj", + "model.layers.17.mlp.experts.61.up_proj", + "model.layers.17.mlp.experts.62.down_proj", + "model.layers.17.mlp.experts.62.gate_proj", + "model.layers.17.mlp.experts.62.up_proj", + "model.layers.17.mlp.experts.63.down_proj", + "model.layers.17.mlp.experts.63.gate_proj", + "model.layers.17.mlp.experts.63.up_proj", + "model.layers.17.mlp.experts.64.down_proj", + "model.layers.17.mlp.experts.64.gate_proj", + "model.layers.17.mlp.experts.64.up_proj", + "model.layers.17.mlp.experts.65.down_proj", + "model.layers.17.mlp.experts.65.gate_proj", + "model.layers.17.mlp.experts.65.up_proj", + "model.layers.17.mlp.experts.66.down_proj", + "model.layers.17.mlp.experts.66.gate_proj", + "model.layers.17.mlp.experts.66.up_proj", + "model.layers.17.mlp.experts.67.down_proj", + "model.layers.17.mlp.experts.67.gate_proj", + "model.layers.17.mlp.experts.67.up_proj", + "model.layers.17.mlp.experts.68.down_proj", + "model.layers.17.mlp.experts.68.gate_proj", + "model.layers.17.mlp.experts.68.up_proj", + "model.layers.17.mlp.experts.69.down_proj", + "model.layers.17.mlp.experts.69.gate_proj", + "model.layers.17.mlp.experts.69.up_proj", + "model.layers.17.mlp.experts.7.down_proj", + "model.layers.17.mlp.experts.7.gate_proj", + "model.layers.17.mlp.experts.7.up_proj", + "model.layers.17.mlp.experts.70.down_proj", + "model.layers.17.mlp.experts.70.gate_proj", + "model.layers.17.mlp.experts.70.up_proj", + "model.layers.17.mlp.experts.71.down_proj", + "model.layers.17.mlp.experts.71.gate_proj", + "model.layers.17.mlp.experts.71.up_proj", + "model.layers.17.mlp.experts.72.down_proj", + "model.layers.17.mlp.experts.72.gate_proj", + "model.layers.17.mlp.experts.72.up_proj", + "model.layers.17.mlp.experts.73.down_proj", + "model.layers.17.mlp.experts.73.gate_proj", + "model.layers.17.mlp.experts.73.up_proj", + "model.layers.17.mlp.experts.74.down_proj", + "model.layers.17.mlp.experts.74.gate_proj", + "model.layers.17.mlp.experts.74.up_proj", + "model.layers.17.mlp.experts.75.down_proj", + "model.layers.17.mlp.experts.75.gate_proj", + "model.layers.17.mlp.experts.75.up_proj", + "model.layers.17.mlp.experts.76.down_proj", + "model.layers.17.mlp.experts.76.gate_proj", + "model.layers.17.mlp.experts.76.up_proj", + "model.layers.17.mlp.experts.77.down_proj", + "model.layers.17.mlp.experts.77.gate_proj", + "model.layers.17.mlp.experts.77.up_proj", + "model.layers.17.mlp.experts.78.down_proj", + "model.layers.17.mlp.experts.78.gate_proj", + "model.layers.17.mlp.experts.78.up_proj", + "model.layers.17.mlp.experts.79.down_proj", + "model.layers.17.mlp.experts.79.gate_proj", + "model.layers.17.mlp.experts.79.up_proj", + "model.layers.17.mlp.experts.8.down_proj", + "model.layers.17.mlp.experts.8.gate_proj", + "model.layers.17.mlp.experts.8.up_proj", + "model.layers.17.mlp.experts.80.down_proj", + "model.layers.17.mlp.experts.80.gate_proj", + "model.layers.17.mlp.experts.80.up_proj", + "model.layers.17.mlp.experts.81.down_proj", + "model.layers.17.mlp.experts.81.gate_proj", + "model.layers.17.mlp.experts.81.up_proj", + "model.layers.17.mlp.experts.82.down_proj", + "model.layers.17.mlp.experts.82.gate_proj", + "model.layers.17.mlp.experts.82.up_proj", + "model.layers.17.mlp.experts.83.down_proj", + "model.layers.17.mlp.experts.83.gate_proj", + "model.layers.17.mlp.experts.83.up_proj", + "model.layers.17.mlp.experts.84.down_proj", + "model.layers.17.mlp.experts.84.gate_proj", + "model.layers.17.mlp.experts.84.up_proj", + "model.layers.17.mlp.experts.85.down_proj", + "model.layers.17.mlp.experts.85.gate_proj", + "model.layers.17.mlp.experts.85.up_proj", + "model.layers.17.mlp.experts.86.down_proj", + "model.layers.17.mlp.experts.86.gate_proj", + "model.layers.17.mlp.experts.86.up_proj", + "model.layers.17.mlp.experts.87.down_proj", + "model.layers.17.mlp.experts.87.gate_proj", + "model.layers.17.mlp.experts.87.up_proj", + "model.layers.17.mlp.experts.88.down_proj", + "model.layers.17.mlp.experts.88.gate_proj", + "model.layers.17.mlp.experts.88.up_proj", + "model.layers.17.mlp.experts.89.down_proj", + "model.layers.17.mlp.experts.89.gate_proj", + "model.layers.17.mlp.experts.89.up_proj", + "model.layers.17.mlp.experts.9.down_proj", + "model.layers.17.mlp.experts.9.gate_proj", + "model.layers.17.mlp.experts.9.up_proj", + "model.layers.17.mlp.experts.90.down_proj", + "model.layers.17.mlp.experts.90.gate_proj", + "model.layers.17.mlp.experts.90.up_proj", + "model.layers.17.mlp.experts.91.down_proj", + "model.layers.17.mlp.experts.91.gate_proj", + "model.layers.17.mlp.experts.91.up_proj", + "model.layers.17.mlp.experts.92.down_proj", + "model.layers.17.mlp.experts.92.gate_proj", + "model.layers.17.mlp.experts.92.up_proj", + "model.layers.17.mlp.experts.93.down_proj", + "model.layers.17.mlp.experts.93.gate_proj", + "model.layers.17.mlp.experts.93.up_proj", + "model.layers.17.mlp.experts.94.down_proj", + "model.layers.17.mlp.experts.94.gate_proj", + "model.layers.17.mlp.experts.94.up_proj", + "model.layers.17.mlp.experts.95.down_proj", + "model.layers.17.mlp.experts.95.gate_proj", + "model.layers.17.mlp.experts.95.up_proj", + "model.layers.17.mlp.experts.96.down_proj", + "model.layers.17.mlp.experts.96.gate_proj", + "model.layers.17.mlp.experts.96.up_proj", + "model.layers.17.mlp.experts.97.down_proj", + "model.layers.17.mlp.experts.97.gate_proj", + "model.layers.17.mlp.experts.97.up_proj", + "model.layers.17.mlp.experts.98.down_proj", + "model.layers.17.mlp.experts.98.gate_proj", + "model.layers.17.mlp.experts.98.up_proj", + "model.layers.17.mlp.experts.99.down_proj", + "model.layers.17.mlp.experts.99.gate_proj", + "model.layers.17.mlp.experts.99.up_proj", + "model.layers.18.mlp.experts.0.down_proj", + "model.layers.18.mlp.experts.0.gate_proj", + "model.layers.18.mlp.experts.0.up_proj", + "model.layers.18.mlp.experts.1.down_proj", + "model.layers.18.mlp.experts.1.gate_proj", + "model.layers.18.mlp.experts.1.up_proj", + "model.layers.18.mlp.experts.10.down_proj", + "model.layers.18.mlp.experts.10.gate_proj", + "model.layers.18.mlp.experts.10.up_proj", + "model.layers.18.mlp.experts.100.down_proj", + "model.layers.18.mlp.experts.100.gate_proj", + "model.layers.18.mlp.experts.100.up_proj", + "model.layers.18.mlp.experts.101.down_proj", + "model.layers.18.mlp.experts.101.gate_proj", + "model.layers.18.mlp.experts.101.up_proj", + "model.layers.18.mlp.experts.102.down_proj", + "model.layers.18.mlp.experts.102.gate_proj", + "model.layers.18.mlp.experts.102.up_proj", + "model.layers.18.mlp.experts.103.down_proj", + "model.layers.18.mlp.experts.103.gate_proj", + "model.layers.18.mlp.experts.103.up_proj", + "model.layers.18.mlp.experts.104.down_proj", + "model.layers.18.mlp.experts.104.gate_proj", + "model.layers.18.mlp.experts.104.up_proj", + "model.layers.18.mlp.experts.105.down_proj", + "model.layers.18.mlp.experts.105.gate_proj", + "model.layers.18.mlp.experts.105.up_proj", + "model.layers.18.mlp.experts.106.down_proj", + "model.layers.18.mlp.experts.106.gate_proj", + "model.layers.18.mlp.experts.106.up_proj", + "model.layers.18.mlp.experts.107.down_proj", + "model.layers.18.mlp.experts.107.gate_proj", + "model.layers.18.mlp.experts.107.up_proj", + "model.layers.18.mlp.experts.108.down_proj", + "model.layers.18.mlp.experts.108.gate_proj", + "model.layers.18.mlp.experts.108.up_proj", + "model.layers.18.mlp.experts.109.down_proj", + "model.layers.18.mlp.experts.109.gate_proj", + "model.layers.18.mlp.experts.109.up_proj", + "model.layers.18.mlp.experts.11.down_proj", + "model.layers.18.mlp.experts.11.gate_proj", + "model.layers.18.mlp.experts.11.up_proj", + "model.layers.18.mlp.experts.110.down_proj", + "model.layers.18.mlp.experts.110.gate_proj", + "model.layers.18.mlp.experts.110.up_proj", + "model.layers.18.mlp.experts.111.down_proj", + "model.layers.18.mlp.experts.111.gate_proj", + "model.layers.18.mlp.experts.111.up_proj", + "model.layers.18.mlp.experts.112.down_proj", + "model.layers.18.mlp.experts.112.gate_proj", + "model.layers.18.mlp.experts.112.up_proj", + "model.layers.18.mlp.experts.113.down_proj", + "model.layers.18.mlp.experts.113.gate_proj", + "model.layers.18.mlp.experts.113.up_proj", + "model.layers.18.mlp.experts.114.down_proj", + "model.layers.18.mlp.experts.114.gate_proj", + "model.layers.18.mlp.experts.114.up_proj", + "model.layers.18.mlp.experts.115.down_proj", + "model.layers.18.mlp.experts.115.gate_proj", + "model.layers.18.mlp.experts.115.up_proj", + "model.layers.18.mlp.experts.116.down_proj", + "model.layers.18.mlp.experts.116.gate_proj", + "model.layers.18.mlp.experts.116.up_proj", + "model.layers.18.mlp.experts.117.down_proj", + "model.layers.18.mlp.experts.117.gate_proj", + "model.layers.18.mlp.experts.117.up_proj", + "model.layers.18.mlp.experts.118.down_proj", + "model.layers.18.mlp.experts.118.gate_proj", + "model.layers.18.mlp.experts.118.up_proj", + "model.layers.18.mlp.experts.119.down_proj", + "model.layers.18.mlp.experts.119.gate_proj", + "model.layers.18.mlp.experts.119.up_proj", + "model.layers.18.mlp.experts.12.down_proj", + "model.layers.18.mlp.experts.12.gate_proj", + "model.layers.18.mlp.experts.12.up_proj", + "model.layers.18.mlp.experts.120.down_proj", + "model.layers.18.mlp.experts.120.gate_proj", + "model.layers.18.mlp.experts.120.up_proj", + "model.layers.18.mlp.experts.121.down_proj", + "model.layers.18.mlp.experts.121.gate_proj", + "model.layers.18.mlp.experts.121.up_proj", + "model.layers.18.mlp.experts.122.down_proj", + "model.layers.18.mlp.experts.122.gate_proj", + "model.layers.18.mlp.experts.122.up_proj", + "model.layers.18.mlp.experts.123.down_proj", + "model.layers.18.mlp.experts.123.gate_proj", + "model.layers.18.mlp.experts.123.up_proj", + "model.layers.18.mlp.experts.124.down_proj", + "model.layers.18.mlp.experts.124.gate_proj", + "model.layers.18.mlp.experts.124.up_proj", + "model.layers.18.mlp.experts.125.down_proj", + "model.layers.18.mlp.experts.125.gate_proj", + "model.layers.18.mlp.experts.125.up_proj", + "model.layers.18.mlp.experts.126.down_proj", + "model.layers.18.mlp.experts.126.gate_proj", + "model.layers.18.mlp.experts.126.up_proj", + "model.layers.18.mlp.experts.127.down_proj", + "model.layers.18.mlp.experts.127.gate_proj", + "model.layers.18.mlp.experts.127.up_proj", + "model.layers.18.mlp.experts.128.down_proj", + "model.layers.18.mlp.experts.128.gate_proj", + "model.layers.18.mlp.experts.128.up_proj", + "model.layers.18.mlp.experts.129.down_proj", + "model.layers.18.mlp.experts.129.gate_proj", + "model.layers.18.mlp.experts.129.up_proj", + "model.layers.18.mlp.experts.13.down_proj", + "model.layers.18.mlp.experts.13.gate_proj", + "model.layers.18.mlp.experts.13.up_proj", + "model.layers.18.mlp.experts.130.down_proj", + "model.layers.18.mlp.experts.130.gate_proj", + "model.layers.18.mlp.experts.130.up_proj", + "model.layers.18.mlp.experts.131.down_proj", + "model.layers.18.mlp.experts.131.gate_proj", + "model.layers.18.mlp.experts.131.up_proj", + "model.layers.18.mlp.experts.132.down_proj", + "model.layers.18.mlp.experts.132.gate_proj", + "model.layers.18.mlp.experts.132.up_proj", + "model.layers.18.mlp.experts.133.down_proj", + "model.layers.18.mlp.experts.133.gate_proj", + "model.layers.18.mlp.experts.133.up_proj", + "model.layers.18.mlp.experts.134.down_proj", + "model.layers.18.mlp.experts.134.gate_proj", + "model.layers.18.mlp.experts.134.up_proj", + "model.layers.18.mlp.experts.135.down_proj", + "model.layers.18.mlp.experts.135.gate_proj", + "model.layers.18.mlp.experts.135.up_proj", + "model.layers.18.mlp.experts.136.down_proj", + "model.layers.18.mlp.experts.136.gate_proj", + "model.layers.18.mlp.experts.136.up_proj", + "model.layers.18.mlp.experts.137.down_proj", + "model.layers.18.mlp.experts.137.gate_proj", + "model.layers.18.mlp.experts.137.up_proj", + "model.layers.18.mlp.experts.138.down_proj", + "model.layers.18.mlp.experts.138.gate_proj", + "model.layers.18.mlp.experts.138.up_proj", + "model.layers.18.mlp.experts.139.down_proj", + "model.layers.18.mlp.experts.139.gate_proj", + "model.layers.18.mlp.experts.139.up_proj", + "model.layers.18.mlp.experts.14.down_proj", + "model.layers.18.mlp.experts.14.gate_proj", + "model.layers.18.mlp.experts.14.up_proj", + "model.layers.18.mlp.experts.140.down_proj", + "model.layers.18.mlp.experts.140.gate_proj", + "model.layers.18.mlp.experts.140.up_proj", + "model.layers.18.mlp.experts.141.down_proj", + "model.layers.18.mlp.experts.141.gate_proj", + "model.layers.18.mlp.experts.141.up_proj", + "model.layers.18.mlp.experts.142.down_proj", + "model.layers.18.mlp.experts.142.gate_proj", + "model.layers.18.mlp.experts.142.up_proj", + "model.layers.18.mlp.experts.143.down_proj", + "model.layers.18.mlp.experts.143.gate_proj", + "model.layers.18.mlp.experts.143.up_proj", + "model.layers.18.mlp.experts.144.down_proj", + "model.layers.18.mlp.experts.144.gate_proj", + "model.layers.18.mlp.experts.144.up_proj", + "model.layers.18.mlp.experts.145.down_proj", + "model.layers.18.mlp.experts.145.gate_proj", + "model.layers.18.mlp.experts.145.up_proj", + "model.layers.18.mlp.experts.146.down_proj", + "model.layers.18.mlp.experts.146.gate_proj", + "model.layers.18.mlp.experts.146.up_proj", + "model.layers.18.mlp.experts.147.down_proj", + "model.layers.18.mlp.experts.147.gate_proj", + "model.layers.18.mlp.experts.147.up_proj", + "model.layers.18.mlp.experts.148.down_proj", + "model.layers.18.mlp.experts.148.gate_proj", + "model.layers.18.mlp.experts.148.up_proj", + "model.layers.18.mlp.experts.149.down_proj", + "model.layers.18.mlp.experts.149.gate_proj", + "model.layers.18.mlp.experts.149.up_proj", + "model.layers.18.mlp.experts.15.down_proj", + "model.layers.18.mlp.experts.15.gate_proj", + "model.layers.18.mlp.experts.15.up_proj", + "model.layers.18.mlp.experts.150.down_proj", + "model.layers.18.mlp.experts.150.gate_proj", + "model.layers.18.mlp.experts.150.up_proj", + "model.layers.18.mlp.experts.151.down_proj", + "model.layers.18.mlp.experts.151.gate_proj", + "model.layers.18.mlp.experts.151.up_proj", + "model.layers.18.mlp.experts.152.down_proj", + "model.layers.18.mlp.experts.152.gate_proj", + "model.layers.18.mlp.experts.152.up_proj", + "model.layers.18.mlp.experts.153.down_proj", + "model.layers.18.mlp.experts.153.gate_proj", + "model.layers.18.mlp.experts.153.up_proj", + "model.layers.18.mlp.experts.154.down_proj", + "model.layers.18.mlp.experts.154.gate_proj", + "model.layers.18.mlp.experts.154.up_proj", + "model.layers.18.mlp.experts.155.down_proj", + "model.layers.18.mlp.experts.155.gate_proj", + "model.layers.18.mlp.experts.155.up_proj", + "model.layers.18.mlp.experts.156.down_proj", + "model.layers.18.mlp.experts.156.gate_proj", + "model.layers.18.mlp.experts.156.up_proj", + "model.layers.18.mlp.experts.157.down_proj", + "model.layers.18.mlp.experts.157.gate_proj", + "model.layers.18.mlp.experts.157.up_proj", + "model.layers.18.mlp.experts.158.down_proj", + "model.layers.18.mlp.experts.158.gate_proj", + "model.layers.18.mlp.experts.158.up_proj", + "model.layers.18.mlp.experts.159.down_proj", + "model.layers.18.mlp.experts.159.gate_proj", + "model.layers.18.mlp.experts.159.up_proj", + "model.layers.18.mlp.experts.16.down_proj", + "model.layers.18.mlp.experts.16.gate_proj", + "model.layers.18.mlp.experts.16.up_proj", + "model.layers.18.mlp.experts.160.down_proj", + "model.layers.18.mlp.experts.160.gate_proj", + "model.layers.18.mlp.experts.160.up_proj", + "model.layers.18.mlp.experts.161.down_proj", + "model.layers.18.mlp.experts.161.gate_proj", + "model.layers.18.mlp.experts.161.up_proj", + "model.layers.18.mlp.experts.162.down_proj", + "model.layers.18.mlp.experts.162.gate_proj", + "model.layers.18.mlp.experts.162.up_proj", + "model.layers.18.mlp.experts.163.down_proj", + "model.layers.18.mlp.experts.163.gate_proj", + "model.layers.18.mlp.experts.163.up_proj", + "model.layers.18.mlp.experts.164.down_proj", + "model.layers.18.mlp.experts.164.gate_proj", + "model.layers.18.mlp.experts.164.up_proj", + "model.layers.18.mlp.experts.165.down_proj", + "model.layers.18.mlp.experts.165.gate_proj", + "model.layers.18.mlp.experts.165.up_proj", + "model.layers.18.mlp.experts.166.down_proj", + "model.layers.18.mlp.experts.166.gate_proj", + "model.layers.18.mlp.experts.166.up_proj", + "model.layers.18.mlp.experts.167.down_proj", + "model.layers.18.mlp.experts.167.gate_proj", + "model.layers.18.mlp.experts.167.up_proj", + "model.layers.18.mlp.experts.168.down_proj", + "model.layers.18.mlp.experts.168.gate_proj", + "model.layers.18.mlp.experts.168.up_proj", + "model.layers.18.mlp.experts.169.down_proj", + "model.layers.18.mlp.experts.169.gate_proj", + "model.layers.18.mlp.experts.169.up_proj", + "model.layers.18.mlp.experts.17.down_proj", + "model.layers.18.mlp.experts.17.gate_proj", + "model.layers.18.mlp.experts.17.up_proj", + "model.layers.18.mlp.experts.170.down_proj", + "model.layers.18.mlp.experts.170.gate_proj", + "model.layers.18.mlp.experts.170.up_proj", + "model.layers.18.mlp.experts.171.down_proj", + "model.layers.18.mlp.experts.171.gate_proj", + "model.layers.18.mlp.experts.171.up_proj", + "model.layers.18.mlp.experts.172.down_proj", + "model.layers.18.mlp.experts.172.gate_proj", + "model.layers.18.mlp.experts.172.up_proj", + "model.layers.18.mlp.experts.173.down_proj", + "model.layers.18.mlp.experts.173.gate_proj", + "model.layers.18.mlp.experts.173.up_proj", + "model.layers.18.mlp.experts.174.down_proj", + "model.layers.18.mlp.experts.174.gate_proj", + "model.layers.18.mlp.experts.174.up_proj", + "model.layers.18.mlp.experts.175.down_proj", + "model.layers.18.mlp.experts.175.gate_proj", + "model.layers.18.mlp.experts.175.up_proj", + "model.layers.18.mlp.experts.176.down_proj", + "model.layers.18.mlp.experts.176.gate_proj", + "model.layers.18.mlp.experts.176.up_proj", + "model.layers.18.mlp.experts.177.down_proj", + "model.layers.18.mlp.experts.177.gate_proj", + "model.layers.18.mlp.experts.177.up_proj", + "model.layers.18.mlp.experts.178.down_proj", + "model.layers.18.mlp.experts.178.gate_proj", + "model.layers.18.mlp.experts.178.up_proj", + "model.layers.18.mlp.experts.179.down_proj", + "model.layers.18.mlp.experts.179.gate_proj", + "model.layers.18.mlp.experts.179.up_proj", + "model.layers.18.mlp.experts.18.down_proj", + "model.layers.18.mlp.experts.18.gate_proj", + "model.layers.18.mlp.experts.18.up_proj", + "model.layers.18.mlp.experts.180.down_proj", + "model.layers.18.mlp.experts.180.gate_proj", + "model.layers.18.mlp.experts.180.up_proj", + "model.layers.18.mlp.experts.181.down_proj", + "model.layers.18.mlp.experts.181.gate_proj", + "model.layers.18.mlp.experts.181.up_proj", + "model.layers.18.mlp.experts.182.down_proj", + "model.layers.18.mlp.experts.182.gate_proj", + "model.layers.18.mlp.experts.182.up_proj", + "model.layers.18.mlp.experts.183.down_proj", + "model.layers.18.mlp.experts.183.gate_proj", + "model.layers.18.mlp.experts.183.up_proj", + "model.layers.18.mlp.experts.184.down_proj", + "model.layers.18.mlp.experts.184.gate_proj", + "model.layers.18.mlp.experts.184.up_proj", + "model.layers.18.mlp.experts.185.down_proj", + "model.layers.18.mlp.experts.185.gate_proj", + "model.layers.18.mlp.experts.185.up_proj", + "model.layers.18.mlp.experts.186.down_proj", + "model.layers.18.mlp.experts.186.gate_proj", + "model.layers.18.mlp.experts.186.up_proj", + "model.layers.18.mlp.experts.187.down_proj", + "model.layers.18.mlp.experts.187.gate_proj", + "model.layers.18.mlp.experts.187.up_proj", + "model.layers.18.mlp.experts.188.down_proj", + "model.layers.18.mlp.experts.188.gate_proj", + "model.layers.18.mlp.experts.188.up_proj", + "model.layers.18.mlp.experts.189.down_proj", + "model.layers.18.mlp.experts.189.gate_proj", + "model.layers.18.mlp.experts.189.up_proj", + "model.layers.18.mlp.experts.19.down_proj", + "model.layers.18.mlp.experts.19.gate_proj", + "model.layers.18.mlp.experts.19.up_proj", + "model.layers.18.mlp.experts.190.down_proj", + "model.layers.18.mlp.experts.190.gate_proj", + "model.layers.18.mlp.experts.190.up_proj", + "model.layers.18.mlp.experts.191.down_proj", + "model.layers.18.mlp.experts.191.gate_proj", + "model.layers.18.mlp.experts.191.up_proj", + "model.layers.18.mlp.experts.192.down_proj", + "model.layers.18.mlp.experts.192.gate_proj", + "model.layers.18.mlp.experts.192.up_proj", + "model.layers.18.mlp.experts.193.down_proj", + "model.layers.18.mlp.experts.193.gate_proj", + "model.layers.18.mlp.experts.193.up_proj", + "model.layers.18.mlp.experts.194.down_proj", + "model.layers.18.mlp.experts.194.gate_proj", + "model.layers.18.mlp.experts.194.up_proj", + "model.layers.18.mlp.experts.195.down_proj", + "model.layers.18.mlp.experts.195.gate_proj", + "model.layers.18.mlp.experts.195.up_proj", + "model.layers.18.mlp.experts.196.down_proj", + "model.layers.18.mlp.experts.196.gate_proj", + "model.layers.18.mlp.experts.196.up_proj", + "model.layers.18.mlp.experts.197.down_proj", + "model.layers.18.mlp.experts.197.gate_proj", + "model.layers.18.mlp.experts.197.up_proj", + "model.layers.18.mlp.experts.198.down_proj", + "model.layers.18.mlp.experts.198.gate_proj", + "model.layers.18.mlp.experts.198.up_proj", + "model.layers.18.mlp.experts.199.down_proj", + "model.layers.18.mlp.experts.199.gate_proj", + "model.layers.18.mlp.experts.199.up_proj", + "model.layers.18.mlp.experts.2.down_proj", + "model.layers.18.mlp.experts.2.gate_proj", + "model.layers.18.mlp.experts.2.up_proj", + "model.layers.18.mlp.experts.20.down_proj", + "model.layers.18.mlp.experts.20.gate_proj", + "model.layers.18.mlp.experts.20.up_proj", + "model.layers.18.mlp.experts.200.down_proj", + "model.layers.18.mlp.experts.200.gate_proj", + "model.layers.18.mlp.experts.200.up_proj", + "model.layers.18.mlp.experts.201.down_proj", + "model.layers.18.mlp.experts.201.gate_proj", + "model.layers.18.mlp.experts.201.up_proj", + "model.layers.18.mlp.experts.202.down_proj", + "model.layers.18.mlp.experts.202.gate_proj", + "model.layers.18.mlp.experts.202.up_proj", + "model.layers.18.mlp.experts.203.down_proj", + "model.layers.18.mlp.experts.203.gate_proj", + "model.layers.18.mlp.experts.203.up_proj", + "model.layers.18.mlp.experts.204.down_proj", + "model.layers.18.mlp.experts.204.gate_proj", + "model.layers.18.mlp.experts.204.up_proj", + "model.layers.18.mlp.experts.205.down_proj", + "model.layers.18.mlp.experts.205.gate_proj", + "model.layers.18.mlp.experts.205.up_proj", + "model.layers.18.mlp.experts.206.down_proj", + "model.layers.18.mlp.experts.206.gate_proj", + "model.layers.18.mlp.experts.206.up_proj", + "model.layers.18.mlp.experts.207.down_proj", + "model.layers.18.mlp.experts.207.gate_proj", + "model.layers.18.mlp.experts.207.up_proj", + "model.layers.18.mlp.experts.208.down_proj", + "model.layers.18.mlp.experts.208.gate_proj", + "model.layers.18.mlp.experts.208.up_proj", + "model.layers.18.mlp.experts.209.down_proj", + "model.layers.18.mlp.experts.209.gate_proj", + "model.layers.18.mlp.experts.209.up_proj", + "model.layers.18.mlp.experts.21.down_proj", + "model.layers.18.mlp.experts.21.gate_proj", + "model.layers.18.mlp.experts.21.up_proj", + "model.layers.18.mlp.experts.210.down_proj", + "model.layers.18.mlp.experts.210.gate_proj", + "model.layers.18.mlp.experts.210.up_proj", + "model.layers.18.mlp.experts.211.down_proj", + "model.layers.18.mlp.experts.211.gate_proj", + "model.layers.18.mlp.experts.211.up_proj", + "model.layers.18.mlp.experts.212.down_proj", + "model.layers.18.mlp.experts.212.gate_proj", + "model.layers.18.mlp.experts.212.up_proj", + "model.layers.18.mlp.experts.213.down_proj", + "model.layers.18.mlp.experts.213.gate_proj", + "model.layers.18.mlp.experts.213.up_proj", + "model.layers.18.mlp.experts.214.down_proj", + "model.layers.18.mlp.experts.214.gate_proj", + "model.layers.18.mlp.experts.214.up_proj", + "model.layers.18.mlp.experts.215.down_proj", + "model.layers.18.mlp.experts.215.gate_proj", + "model.layers.18.mlp.experts.215.up_proj", + "model.layers.18.mlp.experts.216.down_proj", + "model.layers.18.mlp.experts.216.gate_proj", + "model.layers.18.mlp.experts.216.up_proj", + "model.layers.18.mlp.experts.217.down_proj", + "model.layers.18.mlp.experts.217.gate_proj", + "model.layers.18.mlp.experts.217.up_proj", + "model.layers.18.mlp.experts.218.down_proj", + "model.layers.18.mlp.experts.218.gate_proj", + "model.layers.18.mlp.experts.218.up_proj", + "model.layers.18.mlp.experts.219.down_proj", + "model.layers.18.mlp.experts.219.gate_proj", + "model.layers.18.mlp.experts.219.up_proj", + "model.layers.18.mlp.experts.22.down_proj", + "model.layers.18.mlp.experts.22.gate_proj", + "model.layers.18.mlp.experts.22.up_proj", + "model.layers.18.mlp.experts.220.down_proj", + "model.layers.18.mlp.experts.220.gate_proj", + "model.layers.18.mlp.experts.220.up_proj", + "model.layers.18.mlp.experts.221.down_proj", + "model.layers.18.mlp.experts.221.gate_proj", + "model.layers.18.mlp.experts.221.up_proj", + "model.layers.18.mlp.experts.222.down_proj", + "model.layers.18.mlp.experts.222.gate_proj", + "model.layers.18.mlp.experts.222.up_proj", + "model.layers.18.mlp.experts.223.down_proj", + "model.layers.18.mlp.experts.223.gate_proj", + "model.layers.18.mlp.experts.223.up_proj", + "model.layers.18.mlp.experts.224.down_proj", + "model.layers.18.mlp.experts.224.gate_proj", + "model.layers.18.mlp.experts.224.up_proj", + "model.layers.18.mlp.experts.225.down_proj", + "model.layers.18.mlp.experts.225.gate_proj", + "model.layers.18.mlp.experts.225.up_proj", + "model.layers.18.mlp.experts.226.down_proj", + "model.layers.18.mlp.experts.226.gate_proj", + "model.layers.18.mlp.experts.226.up_proj", + "model.layers.18.mlp.experts.227.down_proj", + "model.layers.18.mlp.experts.227.gate_proj", + "model.layers.18.mlp.experts.227.up_proj", + "model.layers.18.mlp.experts.228.down_proj", + "model.layers.18.mlp.experts.228.gate_proj", + "model.layers.18.mlp.experts.228.up_proj", + "model.layers.18.mlp.experts.229.down_proj", + "model.layers.18.mlp.experts.229.gate_proj", + "model.layers.18.mlp.experts.229.up_proj", + "model.layers.18.mlp.experts.23.down_proj", + "model.layers.18.mlp.experts.23.gate_proj", + "model.layers.18.mlp.experts.23.up_proj", + "model.layers.18.mlp.experts.230.down_proj", + "model.layers.18.mlp.experts.230.gate_proj", + "model.layers.18.mlp.experts.230.up_proj", + "model.layers.18.mlp.experts.231.down_proj", + "model.layers.18.mlp.experts.231.gate_proj", + "model.layers.18.mlp.experts.231.up_proj", + "model.layers.18.mlp.experts.232.down_proj", + "model.layers.18.mlp.experts.232.gate_proj", + "model.layers.18.mlp.experts.232.up_proj", + "model.layers.18.mlp.experts.233.down_proj", + "model.layers.18.mlp.experts.233.gate_proj", + "model.layers.18.mlp.experts.233.up_proj", + "model.layers.18.mlp.experts.234.down_proj", + "model.layers.18.mlp.experts.234.gate_proj", + "model.layers.18.mlp.experts.234.up_proj", + "model.layers.18.mlp.experts.235.down_proj", + "model.layers.18.mlp.experts.235.gate_proj", + "model.layers.18.mlp.experts.235.up_proj", + "model.layers.18.mlp.experts.236.down_proj", + "model.layers.18.mlp.experts.236.gate_proj", + "model.layers.18.mlp.experts.236.up_proj", + "model.layers.18.mlp.experts.237.down_proj", + "model.layers.18.mlp.experts.237.gate_proj", + "model.layers.18.mlp.experts.237.up_proj", + "model.layers.18.mlp.experts.238.down_proj", + "model.layers.18.mlp.experts.238.gate_proj", + "model.layers.18.mlp.experts.238.up_proj", + "model.layers.18.mlp.experts.239.down_proj", + "model.layers.18.mlp.experts.239.gate_proj", + "model.layers.18.mlp.experts.239.up_proj", + "model.layers.18.mlp.experts.24.down_proj", + "model.layers.18.mlp.experts.24.gate_proj", + "model.layers.18.mlp.experts.24.up_proj", + "model.layers.18.mlp.experts.240.down_proj", + "model.layers.18.mlp.experts.240.gate_proj", + "model.layers.18.mlp.experts.240.up_proj", + "model.layers.18.mlp.experts.241.down_proj", + "model.layers.18.mlp.experts.241.gate_proj", + "model.layers.18.mlp.experts.241.up_proj", + "model.layers.18.mlp.experts.242.down_proj", + "model.layers.18.mlp.experts.242.gate_proj", + "model.layers.18.mlp.experts.242.up_proj", + "model.layers.18.mlp.experts.243.down_proj", + "model.layers.18.mlp.experts.243.gate_proj", + "model.layers.18.mlp.experts.243.up_proj", + "model.layers.18.mlp.experts.244.down_proj", + "model.layers.18.mlp.experts.244.gate_proj", + "model.layers.18.mlp.experts.244.up_proj", + "model.layers.18.mlp.experts.245.down_proj", + "model.layers.18.mlp.experts.245.gate_proj", + "model.layers.18.mlp.experts.245.up_proj", + "model.layers.18.mlp.experts.246.down_proj", + "model.layers.18.mlp.experts.246.gate_proj", + "model.layers.18.mlp.experts.246.up_proj", + "model.layers.18.mlp.experts.247.down_proj", + "model.layers.18.mlp.experts.247.gate_proj", + "model.layers.18.mlp.experts.247.up_proj", + "model.layers.18.mlp.experts.248.down_proj", + "model.layers.18.mlp.experts.248.gate_proj", + "model.layers.18.mlp.experts.248.up_proj", + "model.layers.18.mlp.experts.249.down_proj", + "model.layers.18.mlp.experts.249.gate_proj", + "model.layers.18.mlp.experts.249.up_proj", + "model.layers.18.mlp.experts.25.down_proj", + "model.layers.18.mlp.experts.25.gate_proj", + "model.layers.18.mlp.experts.25.up_proj", + "model.layers.18.mlp.experts.250.down_proj", + "model.layers.18.mlp.experts.250.gate_proj", + "model.layers.18.mlp.experts.250.up_proj", + "model.layers.18.mlp.experts.251.down_proj", + "model.layers.18.mlp.experts.251.gate_proj", + "model.layers.18.mlp.experts.251.up_proj", + "model.layers.18.mlp.experts.252.down_proj", + "model.layers.18.mlp.experts.252.gate_proj", + "model.layers.18.mlp.experts.252.up_proj", + "model.layers.18.mlp.experts.253.down_proj", + "model.layers.18.mlp.experts.253.gate_proj", + "model.layers.18.mlp.experts.253.up_proj", + "model.layers.18.mlp.experts.254.down_proj", + "model.layers.18.mlp.experts.254.gate_proj", + "model.layers.18.mlp.experts.254.up_proj", + "model.layers.18.mlp.experts.255.down_proj", + "model.layers.18.mlp.experts.255.gate_proj", + "model.layers.18.mlp.experts.255.up_proj", + "model.layers.18.mlp.experts.26.down_proj", + "model.layers.18.mlp.experts.26.gate_proj", + "model.layers.18.mlp.experts.26.up_proj", + "model.layers.18.mlp.experts.27.down_proj", + "model.layers.18.mlp.experts.27.gate_proj", + "model.layers.18.mlp.experts.27.up_proj", + "model.layers.18.mlp.experts.28.down_proj", + "model.layers.18.mlp.experts.28.gate_proj", + "model.layers.18.mlp.experts.28.up_proj", + "model.layers.18.mlp.experts.29.down_proj", + "model.layers.18.mlp.experts.29.gate_proj", + "model.layers.18.mlp.experts.29.up_proj", + "model.layers.18.mlp.experts.3.down_proj", + "model.layers.18.mlp.experts.3.gate_proj", + "model.layers.18.mlp.experts.3.up_proj", + "model.layers.18.mlp.experts.30.down_proj", + "model.layers.18.mlp.experts.30.gate_proj", + "model.layers.18.mlp.experts.30.up_proj", + "model.layers.18.mlp.experts.31.down_proj", + "model.layers.18.mlp.experts.31.gate_proj", + "model.layers.18.mlp.experts.31.up_proj", + "model.layers.18.mlp.experts.32.down_proj", + "model.layers.18.mlp.experts.32.gate_proj", + "model.layers.18.mlp.experts.32.up_proj", + "model.layers.18.mlp.experts.33.down_proj", + "model.layers.18.mlp.experts.33.gate_proj", + "model.layers.18.mlp.experts.33.up_proj", + "model.layers.18.mlp.experts.34.down_proj", + "model.layers.18.mlp.experts.34.gate_proj", + "model.layers.18.mlp.experts.34.up_proj", + "model.layers.18.mlp.experts.35.down_proj", + "model.layers.18.mlp.experts.35.gate_proj", + "model.layers.18.mlp.experts.35.up_proj", + "model.layers.18.mlp.experts.36.down_proj", + "model.layers.18.mlp.experts.36.gate_proj", + "model.layers.18.mlp.experts.36.up_proj", + "model.layers.18.mlp.experts.37.down_proj", + "model.layers.18.mlp.experts.37.gate_proj", + "model.layers.18.mlp.experts.37.up_proj", + "model.layers.18.mlp.experts.38.down_proj", + "model.layers.18.mlp.experts.38.gate_proj", + "model.layers.18.mlp.experts.38.up_proj", + "model.layers.18.mlp.experts.39.down_proj", + "model.layers.18.mlp.experts.39.gate_proj", + "model.layers.18.mlp.experts.39.up_proj", + "model.layers.18.mlp.experts.4.down_proj", + "model.layers.18.mlp.experts.4.gate_proj", + "model.layers.18.mlp.experts.4.up_proj", + "model.layers.18.mlp.experts.40.down_proj", + "model.layers.18.mlp.experts.40.gate_proj", + "model.layers.18.mlp.experts.40.up_proj", + "model.layers.18.mlp.experts.41.down_proj", + "model.layers.18.mlp.experts.41.gate_proj", + "model.layers.18.mlp.experts.41.up_proj", + "model.layers.18.mlp.experts.42.down_proj", + "model.layers.18.mlp.experts.42.gate_proj", + "model.layers.18.mlp.experts.42.up_proj", + "model.layers.18.mlp.experts.43.down_proj", + "model.layers.18.mlp.experts.43.gate_proj", + "model.layers.18.mlp.experts.43.up_proj", + "model.layers.18.mlp.experts.44.down_proj", + "model.layers.18.mlp.experts.44.gate_proj", + "model.layers.18.mlp.experts.44.up_proj", + "model.layers.18.mlp.experts.45.down_proj", + "model.layers.18.mlp.experts.45.gate_proj", + "model.layers.18.mlp.experts.45.up_proj", + "model.layers.18.mlp.experts.46.down_proj", + "model.layers.18.mlp.experts.46.gate_proj", + "model.layers.18.mlp.experts.46.up_proj", + "model.layers.18.mlp.experts.47.down_proj", + "model.layers.18.mlp.experts.47.gate_proj", + "model.layers.18.mlp.experts.47.up_proj", + "model.layers.18.mlp.experts.48.down_proj", + "model.layers.18.mlp.experts.48.gate_proj", + "model.layers.18.mlp.experts.48.up_proj", + "model.layers.18.mlp.experts.49.down_proj", + "model.layers.18.mlp.experts.49.gate_proj", + "model.layers.18.mlp.experts.49.up_proj", + "model.layers.18.mlp.experts.5.down_proj", + "model.layers.18.mlp.experts.5.gate_proj", + "model.layers.18.mlp.experts.5.up_proj", + "model.layers.18.mlp.experts.50.down_proj", + "model.layers.18.mlp.experts.50.gate_proj", + "model.layers.18.mlp.experts.50.up_proj", + "model.layers.18.mlp.experts.51.down_proj", + "model.layers.18.mlp.experts.51.gate_proj", + "model.layers.18.mlp.experts.51.up_proj", + "model.layers.18.mlp.experts.52.down_proj", + "model.layers.18.mlp.experts.52.gate_proj", + "model.layers.18.mlp.experts.52.up_proj", + "model.layers.18.mlp.experts.53.down_proj", + "model.layers.18.mlp.experts.53.gate_proj", + "model.layers.18.mlp.experts.53.up_proj", + "model.layers.18.mlp.experts.54.down_proj", + "model.layers.18.mlp.experts.54.gate_proj", + "model.layers.18.mlp.experts.54.up_proj", + "model.layers.18.mlp.experts.55.down_proj", + "model.layers.18.mlp.experts.55.gate_proj", + "model.layers.18.mlp.experts.55.up_proj", + "model.layers.18.mlp.experts.56.down_proj", + "model.layers.18.mlp.experts.56.gate_proj", + "model.layers.18.mlp.experts.56.up_proj", + "model.layers.18.mlp.experts.57.down_proj", + "model.layers.18.mlp.experts.57.gate_proj", + "model.layers.18.mlp.experts.57.up_proj", + "model.layers.18.mlp.experts.58.down_proj", + "model.layers.18.mlp.experts.58.gate_proj", + "model.layers.18.mlp.experts.58.up_proj", + "model.layers.18.mlp.experts.59.down_proj", + "model.layers.18.mlp.experts.59.gate_proj", + "model.layers.18.mlp.experts.59.up_proj", + "model.layers.18.mlp.experts.6.down_proj", + "model.layers.18.mlp.experts.6.gate_proj", + "model.layers.18.mlp.experts.6.up_proj", + "model.layers.18.mlp.experts.60.down_proj", + "model.layers.18.mlp.experts.60.gate_proj", + "model.layers.18.mlp.experts.60.up_proj", + "model.layers.18.mlp.experts.61.down_proj", + "model.layers.18.mlp.experts.61.gate_proj", + "model.layers.18.mlp.experts.61.up_proj", + "model.layers.18.mlp.experts.62.down_proj", + "model.layers.18.mlp.experts.62.gate_proj", + "model.layers.18.mlp.experts.62.up_proj", + "model.layers.18.mlp.experts.63.down_proj", + "model.layers.18.mlp.experts.63.gate_proj", + "model.layers.18.mlp.experts.63.up_proj", + "model.layers.18.mlp.experts.64.down_proj", + "model.layers.18.mlp.experts.64.gate_proj", + "model.layers.18.mlp.experts.64.up_proj", + "model.layers.18.mlp.experts.65.down_proj", + "model.layers.18.mlp.experts.65.gate_proj", + "model.layers.18.mlp.experts.65.up_proj", + "model.layers.18.mlp.experts.66.down_proj", + "model.layers.18.mlp.experts.66.gate_proj", + "model.layers.18.mlp.experts.66.up_proj", + "model.layers.18.mlp.experts.67.down_proj", + "model.layers.18.mlp.experts.67.gate_proj", + "model.layers.18.mlp.experts.67.up_proj", + "model.layers.18.mlp.experts.68.down_proj", + "model.layers.18.mlp.experts.68.gate_proj", + "model.layers.18.mlp.experts.68.up_proj", + "model.layers.18.mlp.experts.69.down_proj", + "model.layers.18.mlp.experts.69.gate_proj", + "model.layers.18.mlp.experts.69.up_proj", + "model.layers.18.mlp.experts.7.down_proj", + "model.layers.18.mlp.experts.7.gate_proj", + "model.layers.18.mlp.experts.7.up_proj", + "model.layers.18.mlp.experts.70.down_proj", + "model.layers.18.mlp.experts.70.gate_proj", + "model.layers.18.mlp.experts.70.up_proj", + "model.layers.18.mlp.experts.71.down_proj", + "model.layers.18.mlp.experts.71.gate_proj", + "model.layers.18.mlp.experts.71.up_proj", + "model.layers.18.mlp.experts.72.down_proj", + "model.layers.18.mlp.experts.72.gate_proj", + "model.layers.18.mlp.experts.72.up_proj", + "model.layers.18.mlp.experts.73.down_proj", + "model.layers.18.mlp.experts.73.gate_proj", + "model.layers.18.mlp.experts.73.up_proj", + "model.layers.18.mlp.experts.74.down_proj", + "model.layers.18.mlp.experts.74.gate_proj", + "model.layers.18.mlp.experts.74.up_proj", + "model.layers.18.mlp.experts.75.down_proj", + "model.layers.18.mlp.experts.75.gate_proj", + "model.layers.18.mlp.experts.75.up_proj", + "model.layers.18.mlp.experts.76.down_proj", + "model.layers.18.mlp.experts.76.gate_proj", + "model.layers.18.mlp.experts.76.up_proj", + "model.layers.18.mlp.experts.77.down_proj", + "model.layers.18.mlp.experts.77.gate_proj", + "model.layers.18.mlp.experts.77.up_proj", + "model.layers.18.mlp.experts.78.down_proj", + "model.layers.18.mlp.experts.78.gate_proj", + "model.layers.18.mlp.experts.78.up_proj", + "model.layers.18.mlp.experts.79.down_proj", + "model.layers.18.mlp.experts.79.gate_proj", + "model.layers.18.mlp.experts.79.up_proj", + "model.layers.18.mlp.experts.8.down_proj", + "model.layers.18.mlp.experts.8.gate_proj", + "model.layers.18.mlp.experts.8.up_proj", + "model.layers.18.mlp.experts.80.down_proj", + "model.layers.18.mlp.experts.80.gate_proj", + "model.layers.18.mlp.experts.80.up_proj", + "model.layers.18.mlp.experts.81.down_proj", + "model.layers.18.mlp.experts.81.gate_proj", + "model.layers.18.mlp.experts.81.up_proj", + "model.layers.18.mlp.experts.82.down_proj", + "model.layers.18.mlp.experts.82.gate_proj", + "model.layers.18.mlp.experts.82.up_proj", + "model.layers.18.mlp.experts.83.down_proj", + "model.layers.18.mlp.experts.83.gate_proj", + "model.layers.18.mlp.experts.83.up_proj", + "model.layers.18.mlp.experts.84.down_proj", + "model.layers.18.mlp.experts.84.gate_proj", + "model.layers.18.mlp.experts.84.up_proj", + "model.layers.18.mlp.experts.85.down_proj", + "model.layers.18.mlp.experts.85.gate_proj", + "model.layers.18.mlp.experts.85.up_proj", + "model.layers.18.mlp.experts.86.down_proj", + "model.layers.18.mlp.experts.86.gate_proj", + "model.layers.18.mlp.experts.86.up_proj", + "model.layers.18.mlp.experts.87.down_proj", + "model.layers.18.mlp.experts.87.gate_proj", + "model.layers.18.mlp.experts.87.up_proj", + "model.layers.18.mlp.experts.88.down_proj", + "model.layers.18.mlp.experts.88.gate_proj", + "model.layers.18.mlp.experts.88.up_proj", + "model.layers.18.mlp.experts.89.down_proj", + "model.layers.18.mlp.experts.89.gate_proj", + "model.layers.18.mlp.experts.89.up_proj", + "model.layers.18.mlp.experts.9.down_proj", + "model.layers.18.mlp.experts.9.gate_proj", + "model.layers.18.mlp.experts.9.up_proj", + "model.layers.18.mlp.experts.90.down_proj", + "model.layers.18.mlp.experts.90.gate_proj", + "model.layers.18.mlp.experts.90.up_proj", + "model.layers.18.mlp.experts.91.down_proj", + "model.layers.18.mlp.experts.91.gate_proj", + "model.layers.18.mlp.experts.91.up_proj", + "model.layers.18.mlp.experts.92.down_proj", + "model.layers.18.mlp.experts.92.gate_proj", + "model.layers.18.mlp.experts.92.up_proj", + "model.layers.18.mlp.experts.93.down_proj", + "model.layers.18.mlp.experts.93.gate_proj", + "model.layers.18.mlp.experts.93.up_proj", + "model.layers.18.mlp.experts.94.down_proj", + "model.layers.18.mlp.experts.94.gate_proj", + "model.layers.18.mlp.experts.94.up_proj", + "model.layers.18.mlp.experts.95.down_proj", + "model.layers.18.mlp.experts.95.gate_proj", + "model.layers.18.mlp.experts.95.up_proj", + "model.layers.18.mlp.experts.96.down_proj", + "model.layers.18.mlp.experts.96.gate_proj", + "model.layers.18.mlp.experts.96.up_proj", + "model.layers.18.mlp.experts.97.down_proj", + "model.layers.18.mlp.experts.97.gate_proj", + "model.layers.18.mlp.experts.97.up_proj", + "model.layers.18.mlp.experts.98.down_proj", + "model.layers.18.mlp.experts.98.gate_proj", + "model.layers.18.mlp.experts.98.up_proj", + "model.layers.18.mlp.experts.99.down_proj", + "model.layers.18.mlp.experts.99.gate_proj", + "model.layers.18.mlp.experts.99.up_proj", + "model.layers.19.mlp.experts.0.down_proj", + "model.layers.19.mlp.experts.0.gate_proj", + "model.layers.19.mlp.experts.0.up_proj", + "model.layers.19.mlp.experts.1.down_proj", + "model.layers.19.mlp.experts.1.gate_proj", + "model.layers.19.mlp.experts.1.up_proj", + "model.layers.19.mlp.experts.10.down_proj", + "model.layers.19.mlp.experts.10.gate_proj", + "model.layers.19.mlp.experts.10.up_proj", + "model.layers.19.mlp.experts.100.down_proj", + "model.layers.19.mlp.experts.100.gate_proj", + "model.layers.19.mlp.experts.100.up_proj", + "model.layers.19.mlp.experts.101.down_proj", + "model.layers.19.mlp.experts.101.gate_proj", + "model.layers.19.mlp.experts.101.up_proj", + "model.layers.19.mlp.experts.102.down_proj", + "model.layers.19.mlp.experts.102.gate_proj", + "model.layers.19.mlp.experts.102.up_proj", + "model.layers.19.mlp.experts.103.down_proj", + "model.layers.19.mlp.experts.103.gate_proj", + "model.layers.19.mlp.experts.103.up_proj", + "model.layers.19.mlp.experts.104.down_proj", + "model.layers.19.mlp.experts.104.gate_proj", + "model.layers.19.mlp.experts.104.up_proj", + "model.layers.19.mlp.experts.105.down_proj", + "model.layers.19.mlp.experts.105.gate_proj", + "model.layers.19.mlp.experts.105.up_proj", + "model.layers.19.mlp.experts.106.down_proj", + "model.layers.19.mlp.experts.106.gate_proj", + "model.layers.19.mlp.experts.106.up_proj", + "model.layers.19.mlp.experts.107.down_proj", + "model.layers.19.mlp.experts.107.gate_proj", + "model.layers.19.mlp.experts.107.up_proj", + "model.layers.19.mlp.experts.108.down_proj", + "model.layers.19.mlp.experts.108.gate_proj", + "model.layers.19.mlp.experts.108.up_proj", + "model.layers.19.mlp.experts.109.down_proj", + "model.layers.19.mlp.experts.109.gate_proj", + "model.layers.19.mlp.experts.109.up_proj", + "model.layers.19.mlp.experts.11.down_proj", + "model.layers.19.mlp.experts.11.gate_proj", + "model.layers.19.mlp.experts.11.up_proj", + "model.layers.19.mlp.experts.110.down_proj", + "model.layers.19.mlp.experts.110.gate_proj", + "model.layers.19.mlp.experts.110.up_proj", + "model.layers.19.mlp.experts.111.down_proj", + "model.layers.19.mlp.experts.111.gate_proj", + "model.layers.19.mlp.experts.111.up_proj", + "model.layers.19.mlp.experts.112.down_proj", + "model.layers.19.mlp.experts.112.gate_proj", + "model.layers.19.mlp.experts.112.up_proj", + "model.layers.19.mlp.experts.113.down_proj", + "model.layers.19.mlp.experts.113.gate_proj", + "model.layers.19.mlp.experts.113.up_proj", + "model.layers.19.mlp.experts.114.down_proj", + "model.layers.19.mlp.experts.114.gate_proj", + "model.layers.19.mlp.experts.114.up_proj", + "model.layers.19.mlp.experts.115.down_proj", + "model.layers.19.mlp.experts.115.gate_proj", + "model.layers.19.mlp.experts.115.up_proj", + "model.layers.19.mlp.experts.116.down_proj", + "model.layers.19.mlp.experts.116.gate_proj", + "model.layers.19.mlp.experts.116.up_proj", + "model.layers.19.mlp.experts.117.down_proj", + "model.layers.19.mlp.experts.117.gate_proj", + "model.layers.19.mlp.experts.117.up_proj", + "model.layers.19.mlp.experts.118.down_proj", + "model.layers.19.mlp.experts.118.gate_proj", + "model.layers.19.mlp.experts.118.up_proj", + "model.layers.19.mlp.experts.119.down_proj", + "model.layers.19.mlp.experts.119.gate_proj", + "model.layers.19.mlp.experts.119.up_proj", + "model.layers.19.mlp.experts.12.down_proj", + "model.layers.19.mlp.experts.12.gate_proj", + "model.layers.19.mlp.experts.12.up_proj", + "model.layers.19.mlp.experts.120.down_proj", + "model.layers.19.mlp.experts.120.gate_proj", + "model.layers.19.mlp.experts.120.up_proj", + "model.layers.19.mlp.experts.121.down_proj", + "model.layers.19.mlp.experts.121.gate_proj", + "model.layers.19.mlp.experts.121.up_proj", + "model.layers.19.mlp.experts.122.down_proj", + "model.layers.19.mlp.experts.122.gate_proj", + "model.layers.19.mlp.experts.122.up_proj", + "model.layers.19.mlp.experts.123.down_proj", + "model.layers.19.mlp.experts.123.gate_proj", + "model.layers.19.mlp.experts.123.up_proj", + "model.layers.19.mlp.experts.124.down_proj", + "model.layers.19.mlp.experts.124.gate_proj", + "model.layers.19.mlp.experts.124.up_proj", + "model.layers.19.mlp.experts.125.down_proj", + "model.layers.19.mlp.experts.125.gate_proj", + "model.layers.19.mlp.experts.125.up_proj", + "model.layers.19.mlp.experts.126.down_proj", + "model.layers.19.mlp.experts.126.gate_proj", + "model.layers.19.mlp.experts.126.up_proj", + "model.layers.19.mlp.experts.127.down_proj", + "model.layers.19.mlp.experts.127.gate_proj", + "model.layers.19.mlp.experts.127.up_proj", + "model.layers.19.mlp.experts.128.down_proj", + "model.layers.19.mlp.experts.128.gate_proj", + "model.layers.19.mlp.experts.128.up_proj", + "model.layers.19.mlp.experts.129.down_proj", + "model.layers.19.mlp.experts.129.gate_proj", + "model.layers.19.mlp.experts.129.up_proj", + "model.layers.19.mlp.experts.13.down_proj", + "model.layers.19.mlp.experts.13.gate_proj", + "model.layers.19.mlp.experts.13.up_proj", + "model.layers.19.mlp.experts.130.down_proj", + "model.layers.19.mlp.experts.130.gate_proj", + "model.layers.19.mlp.experts.130.up_proj", + "model.layers.19.mlp.experts.131.down_proj", + "model.layers.19.mlp.experts.131.gate_proj", + "model.layers.19.mlp.experts.131.up_proj", + "model.layers.19.mlp.experts.132.down_proj", + "model.layers.19.mlp.experts.132.gate_proj", + "model.layers.19.mlp.experts.132.up_proj", + "model.layers.19.mlp.experts.133.down_proj", + "model.layers.19.mlp.experts.133.gate_proj", + "model.layers.19.mlp.experts.133.up_proj", + "model.layers.19.mlp.experts.134.down_proj", + "model.layers.19.mlp.experts.134.gate_proj", + "model.layers.19.mlp.experts.134.up_proj", + "model.layers.19.mlp.experts.135.down_proj", + "model.layers.19.mlp.experts.135.gate_proj", + "model.layers.19.mlp.experts.135.up_proj", + "model.layers.19.mlp.experts.136.down_proj", + "model.layers.19.mlp.experts.136.gate_proj", + "model.layers.19.mlp.experts.136.up_proj", + "model.layers.19.mlp.experts.137.down_proj", + "model.layers.19.mlp.experts.137.gate_proj", + "model.layers.19.mlp.experts.137.up_proj", + "model.layers.19.mlp.experts.138.down_proj", + "model.layers.19.mlp.experts.138.gate_proj", + "model.layers.19.mlp.experts.138.up_proj", + "model.layers.19.mlp.experts.139.down_proj", + "model.layers.19.mlp.experts.139.gate_proj", + "model.layers.19.mlp.experts.139.up_proj", + "model.layers.19.mlp.experts.14.down_proj", + "model.layers.19.mlp.experts.14.gate_proj", + "model.layers.19.mlp.experts.14.up_proj", + "model.layers.19.mlp.experts.140.down_proj", + "model.layers.19.mlp.experts.140.gate_proj", + "model.layers.19.mlp.experts.140.up_proj", + "model.layers.19.mlp.experts.141.down_proj", + "model.layers.19.mlp.experts.141.gate_proj", + "model.layers.19.mlp.experts.141.up_proj", + "model.layers.19.mlp.experts.142.down_proj", + "model.layers.19.mlp.experts.142.gate_proj", + "model.layers.19.mlp.experts.142.up_proj", + "model.layers.19.mlp.experts.143.down_proj", + "model.layers.19.mlp.experts.143.gate_proj", + "model.layers.19.mlp.experts.143.up_proj", + "model.layers.19.mlp.experts.144.down_proj", + "model.layers.19.mlp.experts.144.gate_proj", + "model.layers.19.mlp.experts.144.up_proj", + "model.layers.19.mlp.experts.145.down_proj", + "model.layers.19.mlp.experts.145.gate_proj", + "model.layers.19.mlp.experts.145.up_proj", + "model.layers.19.mlp.experts.146.down_proj", + "model.layers.19.mlp.experts.146.gate_proj", + "model.layers.19.mlp.experts.146.up_proj", + "model.layers.19.mlp.experts.147.down_proj", + "model.layers.19.mlp.experts.147.gate_proj", + "model.layers.19.mlp.experts.147.up_proj", + "model.layers.19.mlp.experts.148.down_proj", + "model.layers.19.mlp.experts.148.gate_proj", + "model.layers.19.mlp.experts.148.up_proj", + "model.layers.19.mlp.experts.149.down_proj", + "model.layers.19.mlp.experts.149.gate_proj", + "model.layers.19.mlp.experts.149.up_proj", + "model.layers.19.mlp.experts.15.down_proj", + "model.layers.19.mlp.experts.15.gate_proj", + "model.layers.19.mlp.experts.15.up_proj", + "model.layers.19.mlp.experts.150.down_proj", + "model.layers.19.mlp.experts.150.gate_proj", + "model.layers.19.mlp.experts.150.up_proj", + "model.layers.19.mlp.experts.151.down_proj", + "model.layers.19.mlp.experts.151.gate_proj", + "model.layers.19.mlp.experts.151.up_proj", + "model.layers.19.mlp.experts.152.down_proj", + "model.layers.19.mlp.experts.152.gate_proj", + "model.layers.19.mlp.experts.152.up_proj", + "model.layers.19.mlp.experts.153.down_proj", + "model.layers.19.mlp.experts.153.gate_proj", + "model.layers.19.mlp.experts.153.up_proj", + "model.layers.19.mlp.experts.154.down_proj", + "model.layers.19.mlp.experts.154.gate_proj", + "model.layers.19.mlp.experts.154.up_proj", + "model.layers.19.mlp.experts.155.down_proj", + "model.layers.19.mlp.experts.155.gate_proj", + "model.layers.19.mlp.experts.155.up_proj", + "model.layers.19.mlp.experts.156.down_proj", + "model.layers.19.mlp.experts.156.gate_proj", + "model.layers.19.mlp.experts.156.up_proj", + "model.layers.19.mlp.experts.157.down_proj", + "model.layers.19.mlp.experts.157.gate_proj", + "model.layers.19.mlp.experts.157.up_proj", + "model.layers.19.mlp.experts.158.down_proj", + "model.layers.19.mlp.experts.158.gate_proj", + "model.layers.19.mlp.experts.158.up_proj", + "model.layers.19.mlp.experts.159.down_proj", + "model.layers.19.mlp.experts.159.gate_proj", + "model.layers.19.mlp.experts.159.up_proj", + "model.layers.19.mlp.experts.16.down_proj", + "model.layers.19.mlp.experts.16.gate_proj", + "model.layers.19.mlp.experts.16.up_proj", + "model.layers.19.mlp.experts.160.down_proj", + "model.layers.19.mlp.experts.160.gate_proj", + "model.layers.19.mlp.experts.160.up_proj", + "model.layers.19.mlp.experts.161.down_proj", + "model.layers.19.mlp.experts.161.gate_proj", + "model.layers.19.mlp.experts.161.up_proj", + "model.layers.19.mlp.experts.162.down_proj", + "model.layers.19.mlp.experts.162.gate_proj", + "model.layers.19.mlp.experts.162.up_proj", + "model.layers.19.mlp.experts.163.down_proj", + "model.layers.19.mlp.experts.163.gate_proj", + "model.layers.19.mlp.experts.163.up_proj", + "model.layers.19.mlp.experts.164.down_proj", + "model.layers.19.mlp.experts.164.gate_proj", + "model.layers.19.mlp.experts.164.up_proj", + "model.layers.19.mlp.experts.165.down_proj", + "model.layers.19.mlp.experts.165.gate_proj", + "model.layers.19.mlp.experts.165.up_proj", + "model.layers.19.mlp.experts.166.down_proj", + "model.layers.19.mlp.experts.166.gate_proj", + "model.layers.19.mlp.experts.166.up_proj", + "model.layers.19.mlp.experts.167.down_proj", + "model.layers.19.mlp.experts.167.gate_proj", + "model.layers.19.mlp.experts.167.up_proj", + "model.layers.19.mlp.experts.168.down_proj", + "model.layers.19.mlp.experts.168.gate_proj", + "model.layers.19.mlp.experts.168.up_proj", + "model.layers.19.mlp.experts.169.down_proj", + "model.layers.19.mlp.experts.169.gate_proj", + "model.layers.19.mlp.experts.169.up_proj", + "model.layers.19.mlp.experts.17.down_proj", + "model.layers.19.mlp.experts.17.gate_proj", + "model.layers.19.mlp.experts.17.up_proj", + "model.layers.19.mlp.experts.170.down_proj", + "model.layers.19.mlp.experts.170.gate_proj", + "model.layers.19.mlp.experts.170.up_proj", + "model.layers.19.mlp.experts.171.down_proj", + "model.layers.19.mlp.experts.171.gate_proj", + "model.layers.19.mlp.experts.171.up_proj", + "model.layers.19.mlp.experts.172.down_proj", + "model.layers.19.mlp.experts.172.gate_proj", + "model.layers.19.mlp.experts.172.up_proj", + "model.layers.19.mlp.experts.173.down_proj", + "model.layers.19.mlp.experts.173.gate_proj", + "model.layers.19.mlp.experts.173.up_proj", + "model.layers.19.mlp.experts.174.down_proj", + "model.layers.19.mlp.experts.174.gate_proj", + "model.layers.19.mlp.experts.174.up_proj", + "model.layers.19.mlp.experts.175.down_proj", + "model.layers.19.mlp.experts.175.gate_proj", + "model.layers.19.mlp.experts.175.up_proj", + "model.layers.19.mlp.experts.176.down_proj", + "model.layers.19.mlp.experts.176.gate_proj", + "model.layers.19.mlp.experts.176.up_proj", + "model.layers.19.mlp.experts.177.down_proj", + "model.layers.19.mlp.experts.177.gate_proj", + "model.layers.19.mlp.experts.177.up_proj", + "model.layers.19.mlp.experts.178.down_proj", + "model.layers.19.mlp.experts.178.gate_proj", + "model.layers.19.mlp.experts.178.up_proj", + "model.layers.19.mlp.experts.179.down_proj", + "model.layers.19.mlp.experts.179.gate_proj", + "model.layers.19.mlp.experts.179.up_proj", + "model.layers.19.mlp.experts.18.down_proj", + "model.layers.19.mlp.experts.18.gate_proj", + "model.layers.19.mlp.experts.18.up_proj", + "model.layers.19.mlp.experts.180.down_proj", + "model.layers.19.mlp.experts.180.gate_proj", + "model.layers.19.mlp.experts.180.up_proj", + "model.layers.19.mlp.experts.181.down_proj", + "model.layers.19.mlp.experts.181.gate_proj", + "model.layers.19.mlp.experts.181.up_proj", + "model.layers.19.mlp.experts.182.down_proj", + "model.layers.19.mlp.experts.182.gate_proj", + "model.layers.19.mlp.experts.182.up_proj", + "model.layers.19.mlp.experts.183.down_proj", + "model.layers.19.mlp.experts.183.gate_proj", + "model.layers.19.mlp.experts.183.up_proj", + "model.layers.19.mlp.experts.184.down_proj", + "model.layers.19.mlp.experts.184.gate_proj", + "model.layers.19.mlp.experts.184.up_proj", + "model.layers.19.mlp.experts.185.down_proj", + "model.layers.19.mlp.experts.185.gate_proj", + "model.layers.19.mlp.experts.185.up_proj", + "model.layers.19.mlp.experts.186.down_proj", + "model.layers.19.mlp.experts.186.gate_proj", + "model.layers.19.mlp.experts.186.up_proj", + "model.layers.19.mlp.experts.187.down_proj", + "model.layers.19.mlp.experts.187.gate_proj", + "model.layers.19.mlp.experts.187.up_proj", + "model.layers.19.mlp.experts.188.down_proj", + "model.layers.19.mlp.experts.188.gate_proj", + "model.layers.19.mlp.experts.188.up_proj", + "model.layers.19.mlp.experts.189.down_proj", + "model.layers.19.mlp.experts.189.gate_proj", + "model.layers.19.mlp.experts.189.up_proj", + "model.layers.19.mlp.experts.19.down_proj", + "model.layers.19.mlp.experts.19.gate_proj", + "model.layers.19.mlp.experts.19.up_proj", + "model.layers.19.mlp.experts.190.down_proj", + "model.layers.19.mlp.experts.190.gate_proj", + "model.layers.19.mlp.experts.190.up_proj", + "model.layers.19.mlp.experts.191.down_proj", + "model.layers.19.mlp.experts.191.gate_proj", + "model.layers.19.mlp.experts.191.up_proj", + "model.layers.19.mlp.experts.192.down_proj", + "model.layers.19.mlp.experts.192.gate_proj", + "model.layers.19.mlp.experts.192.up_proj", + "model.layers.19.mlp.experts.193.down_proj", + "model.layers.19.mlp.experts.193.gate_proj", + "model.layers.19.mlp.experts.193.up_proj", + "model.layers.19.mlp.experts.194.down_proj", + "model.layers.19.mlp.experts.194.gate_proj", + "model.layers.19.mlp.experts.194.up_proj", + "model.layers.19.mlp.experts.195.down_proj", + "model.layers.19.mlp.experts.195.gate_proj", + "model.layers.19.mlp.experts.195.up_proj", + "model.layers.19.mlp.experts.196.down_proj", + "model.layers.19.mlp.experts.196.gate_proj", + "model.layers.19.mlp.experts.196.up_proj", + "model.layers.19.mlp.experts.197.down_proj", + "model.layers.19.mlp.experts.197.gate_proj", + "model.layers.19.mlp.experts.197.up_proj", + "model.layers.19.mlp.experts.198.down_proj", + "model.layers.19.mlp.experts.198.gate_proj", + "model.layers.19.mlp.experts.198.up_proj", + "model.layers.19.mlp.experts.199.down_proj", + "model.layers.19.mlp.experts.199.gate_proj", + "model.layers.19.mlp.experts.199.up_proj", + "model.layers.19.mlp.experts.2.down_proj", + "model.layers.19.mlp.experts.2.gate_proj", + "model.layers.19.mlp.experts.2.up_proj", + "model.layers.19.mlp.experts.20.down_proj", + "model.layers.19.mlp.experts.20.gate_proj", + "model.layers.19.mlp.experts.20.up_proj", + "model.layers.19.mlp.experts.200.down_proj", + "model.layers.19.mlp.experts.200.gate_proj", + "model.layers.19.mlp.experts.200.up_proj", + "model.layers.19.mlp.experts.201.down_proj", + "model.layers.19.mlp.experts.201.gate_proj", + "model.layers.19.mlp.experts.201.up_proj", + "model.layers.19.mlp.experts.202.down_proj", + "model.layers.19.mlp.experts.202.gate_proj", + "model.layers.19.mlp.experts.202.up_proj", + "model.layers.19.mlp.experts.203.down_proj", + "model.layers.19.mlp.experts.203.gate_proj", + "model.layers.19.mlp.experts.203.up_proj", + "model.layers.19.mlp.experts.204.down_proj", + "model.layers.19.mlp.experts.204.gate_proj", + "model.layers.19.mlp.experts.204.up_proj", + "model.layers.19.mlp.experts.205.down_proj", + "model.layers.19.mlp.experts.205.gate_proj", + "model.layers.19.mlp.experts.205.up_proj", + "model.layers.19.mlp.experts.206.down_proj", + "model.layers.19.mlp.experts.206.gate_proj", + "model.layers.19.mlp.experts.206.up_proj", + "model.layers.19.mlp.experts.207.down_proj", + "model.layers.19.mlp.experts.207.gate_proj", + "model.layers.19.mlp.experts.207.up_proj", + "model.layers.19.mlp.experts.208.down_proj", + "model.layers.19.mlp.experts.208.gate_proj", + "model.layers.19.mlp.experts.208.up_proj", + "model.layers.19.mlp.experts.209.down_proj", + "model.layers.19.mlp.experts.209.gate_proj", + "model.layers.19.mlp.experts.209.up_proj", + "model.layers.19.mlp.experts.21.down_proj", + "model.layers.19.mlp.experts.21.gate_proj", + "model.layers.19.mlp.experts.21.up_proj", + "model.layers.19.mlp.experts.210.down_proj", + "model.layers.19.mlp.experts.210.gate_proj", + "model.layers.19.mlp.experts.210.up_proj", + "model.layers.19.mlp.experts.211.down_proj", + "model.layers.19.mlp.experts.211.gate_proj", + "model.layers.19.mlp.experts.211.up_proj", + "model.layers.19.mlp.experts.212.down_proj", + "model.layers.19.mlp.experts.212.gate_proj", + "model.layers.19.mlp.experts.212.up_proj", + "model.layers.19.mlp.experts.213.down_proj", + "model.layers.19.mlp.experts.213.gate_proj", + "model.layers.19.mlp.experts.213.up_proj", + "model.layers.19.mlp.experts.214.down_proj", + "model.layers.19.mlp.experts.214.gate_proj", + "model.layers.19.mlp.experts.214.up_proj", + "model.layers.19.mlp.experts.215.down_proj", + "model.layers.19.mlp.experts.215.gate_proj", + "model.layers.19.mlp.experts.215.up_proj", + "model.layers.19.mlp.experts.216.down_proj", + "model.layers.19.mlp.experts.216.gate_proj", + "model.layers.19.mlp.experts.216.up_proj", + "model.layers.19.mlp.experts.217.down_proj", + "model.layers.19.mlp.experts.217.gate_proj", + "model.layers.19.mlp.experts.217.up_proj", + "model.layers.19.mlp.experts.218.down_proj", + "model.layers.19.mlp.experts.218.gate_proj", + "model.layers.19.mlp.experts.218.up_proj", + "model.layers.19.mlp.experts.219.down_proj", + "model.layers.19.mlp.experts.219.gate_proj", + "model.layers.19.mlp.experts.219.up_proj", + "model.layers.19.mlp.experts.22.down_proj", + "model.layers.19.mlp.experts.22.gate_proj", + "model.layers.19.mlp.experts.22.up_proj", + "model.layers.19.mlp.experts.220.down_proj", + "model.layers.19.mlp.experts.220.gate_proj", + "model.layers.19.mlp.experts.220.up_proj", + "model.layers.19.mlp.experts.221.down_proj", + "model.layers.19.mlp.experts.221.gate_proj", + "model.layers.19.mlp.experts.221.up_proj", + "model.layers.19.mlp.experts.222.down_proj", + "model.layers.19.mlp.experts.222.gate_proj", + "model.layers.19.mlp.experts.222.up_proj", + "model.layers.19.mlp.experts.223.down_proj", + "model.layers.19.mlp.experts.223.gate_proj", + "model.layers.19.mlp.experts.223.up_proj", + "model.layers.19.mlp.experts.224.down_proj", + "model.layers.19.mlp.experts.224.gate_proj", + "model.layers.19.mlp.experts.224.up_proj", + "model.layers.19.mlp.experts.225.down_proj", + "model.layers.19.mlp.experts.225.gate_proj", + "model.layers.19.mlp.experts.225.up_proj", + "model.layers.19.mlp.experts.226.down_proj", + "model.layers.19.mlp.experts.226.gate_proj", + "model.layers.19.mlp.experts.226.up_proj", + "model.layers.19.mlp.experts.227.down_proj", + "model.layers.19.mlp.experts.227.gate_proj", + "model.layers.19.mlp.experts.227.up_proj", + "model.layers.19.mlp.experts.228.down_proj", + "model.layers.19.mlp.experts.228.gate_proj", + "model.layers.19.mlp.experts.228.up_proj", + "model.layers.19.mlp.experts.229.down_proj", + "model.layers.19.mlp.experts.229.gate_proj", + "model.layers.19.mlp.experts.229.up_proj", + "model.layers.19.mlp.experts.23.down_proj", + "model.layers.19.mlp.experts.23.gate_proj", + "model.layers.19.mlp.experts.23.up_proj", + "model.layers.19.mlp.experts.230.down_proj", + "model.layers.19.mlp.experts.230.gate_proj", + "model.layers.19.mlp.experts.230.up_proj", + "model.layers.19.mlp.experts.231.down_proj", + "model.layers.19.mlp.experts.231.gate_proj", + "model.layers.19.mlp.experts.231.up_proj", + "model.layers.19.mlp.experts.232.down_proj", + "model.layers.19.mlp.experts.232.gate_proj", + "model.layers.19.mlp.experts.232.up_proj", + "model.layers.19.mlp.experts.233.down_proj", + "model.layers.19.mlp.experts.233.gate_proj", + "model.layers.19.mlp.experts.233.up_proj", + "model.layers.19.mlp.experts.234.down_proj", + "model.layers.19.mlp.experts.234.gate_proj", + "model.layers.19.mlp.experts.234.up_proj", + "model.layers.19.mlp.experts.235.down_proj", + "model.layers.19.mlp.experts.235.gate_proj", + "model.layers.19.mlp.experts.235.up_proj", + "model.layers.19.mlp.experts.236.down_proj", + "model.layers.19.mlp.experts.236.gate_proj", + "model.layers.19.mlp.experts.236.up_proj", + "model.layers.19.mlp.experts.237.down_proj", + "model.layers.19.mlp.experts.237.gate_proj", + "model.layers.19.mlp.experts.237.up_proj", + "model.layers.19.mlp.experts.238.down_proj", + "model.layers.19.mlp.experts.238.gate_proj", + "model.layers.19.mlp.experts.238.up_proj", + "model.layers.19.mlp.experts.239.down_proj", + "model.layers.19.mlp.experts.239.gate_proj", + "model.layers.19.mlp.experts.239.up_proj", + "model.layers.19.mlp.experts.24.down_proj", + "model.layers.19.mlp.experts.24.gate_proj", + "model.layers.19.mlp.experts.24.up_proj", + "model.layers.19.mlp.experts.240.down_proj", + "model.layers.19.mlp.experts.240.gate_proj", + "model.layers.19.mlp.experts.240.up_proj", + "model.layers.19.mlp.experts.241.down_proj", + "model.layers.19.mlp.experts.241.gate_proj", + "model.layers.19.mlp.experts.241.up_proj", + "model.layers.19.mlp.experts.242.down_proj", + "model.layers.19.mlp.experts.242.gate_proj", + "model.layers.19.mlp.experts.242.up_proj", + "model.layers.19.mlp.experts.243.down_proj", + "model.layers.19.mlp.experts.243.gate_proj", + "model.layers.19.mlp.experts.243.up_proj", + "model.layers.19.mlp.experts.244.down_proj", + "model.layers.19.mlp.experts.244.gate_proj", + "model.layers.19.mlp.experts.244.up_proj", + "model.layers.19.mlp.experts.245.down_proj", + "model.layers.19.mlp.experts.245.gate_proj", + "model.layers.19.mlp.experts.245.up_proj", + "model.layers.19.mlp.experts.246.down_proj", + "model.layers.19.mlp.experts.246.gate_proj", + "model.layers.19.mlp.experts.246.up_proj", + "model.layers.19.mlp.experts.247.down_proj", + "model.layers.19.mlp.experts.247.gate_proj", + "model.layers.19.mlp.experts.247.up_proj", + "model.layers.19.mlp.experts.248.down_proj", + "model.layers.19.mlp.experts.248.gate_proj", + "model.layers.19.mlp.experts.248.up_proj", + "model.layers.19.mlp.experts.249.down_proj", + "model.layers.19.mlp.experts.249.gate_proj", + "model.layers.19.mlp.experts.249.up_proj", + "model.layers.19.mlp.experts.25.down_proj", + "model.layers.19.mlp.experts.25.gate_proj", + "model.layers.19.mlp.experts.25.up_proj", + "model.layers.19.mlp.experts.250.down_proj", + "model.layers.19.mlp.experts.250.gate_proj", + "model.layers.19.mlp.experts.250.up_proj", + "model.layers.19.mlp.experts.251.down_proj", + "model.layers.19.mlp.experts.251.gate_proj", + "model.layers.19.mlp.experts.251.up_proj", + "model.layers.19.mlp.experts.252.down_proj", + "model.layers.19.mlp.experts.252.gate_proj", + "model.layers.19.mlp.experts.252.up_proj", + "model.layers.19.mlp.experts.253.down_proj", + "model.layers.19.mlp.experts.253.gate_proj", + "model.layers.19.mlp.experts.253.up_proj", + "model.layers.19.mlp.experts.254.down_proj", + "model.layers.19.mlp.experts.254.gate_proj", + "model.layers.19.mlp.experts.254.up_proj", + "model.layers.19.mlp.experts.255.down_proj", + "model.layers.19.mlp.experts.255.gate_proj", + "model.layers.19.mlp.experts.255.up_proj", + "model.layers.19.mlp.experts.26.down_proj", + "model.layers.19.mlp.experts.26.gate_proj", + "model.layers.19.mlp.experts.26.up_proj", + "model.layers.19.mlp.experts.27.down_proj", + "model.layers.19.mlp.experts.27.gate_proj", + "model.layers.19.mlp.experts.27.up_proj", + "model.layers.19.mlp.experts.28.down_proj", + "model.layers.19.mlp.experts.28.gate_proj", + "model.layers.19.mlp.experts.28.up_proj", + "model.layers.19.mlp.experts.29.down_proj", + "model.layers.19.mlp.experts.29.gate_proj", + "model.layers.19.mlp.experts.29.up_proj", + "model.layers.19.mlp.experts.3.down_proj", + "model.layers.19.mlp.experts.3.gate_proj", + "model.layers.19.mlp.experts.3.up_proj", + "model.layers.19.mlp.experts.30.down_proj", + "model.layers.19.mlp.experts.30.gate_proj", + "model.layers.19.mlp.experts.30.up_proj", + "model.layers.19.mlp.experts.31.down_proj", + "model.layers.19.mlp.experts.31.gate_proj", + "model.layers.19.mlp.experts.31.up_proj", + "model.layers.19.mlp.experts.32.down_proj", + "model.layers.19.mlp.experts.32.gate_proj", + "model.layers.19.mlp.experts.32.up_proj", + "model.layers.19.mlp.experts.33.down_proj", + "model.layers.19.mlp.experts.33.gate_proj", + "model.layers.19.mlp.experts.33.up_proj", + "model.layers.19.mlp.experts.34.down_proj", + "model.layers.19.mlp.experts.34.gate_proj", + "model.layers.19.mlp.experts.34.up_proj", + "model.layers.19.mlp.experts.35.down_proj", + "model.layers.19.mlp.experts.35.gate_proj", + "model.layers.19.mlp.experts.35.up_proj", + "model.layers.19.mlp.experts.36.down_proj", + "model.layers.19.mlp.experts.36.gate_proj", + "model.layers.19.mlp.experts.36.up_proj", + "model.layers.19.mlp.experts.37.down_proj", + "model.layers.19.mlp.experts.37.gate_proj", + "model.layers.19.mlp.experts.37.up_proj", + "model.layers.19.mlp.experts.38.down_proj", + "model.layers.19.mlp.experts.38.gate_proj", + "model.layers.19.mlp.experts.38.up_proj", + "model.layers.19.mlp.experts.39.down_proj", + "model.layers.19.mlp.experts.39.gate_proj", + "model.layers.19.mlp.experts.39.up_proj", + "model.layers.19.mlp.experts.4.down_proj", + "model.layers.19.mlp.experts.4.gate_proj", + "model.layers.19.mlp.experts.4.up_proj", + "model.layers.19.mlp.experts.40.down_proj", + "model.layers.19.mlp.experts.40.gate_proj", + "model.layers.19.mlp.experts.40.up_proj", + "model.layers.19.mlp.experts.41.down_proj", + "model.layers.19.mlp.experts.41.gate_proj", + "model.layers.19.mlp.experts.41.up_proj", + "model.layers.19.mlp.experts.42.down_proj", + "model.layers.19.mlp.experts.42.gate_proj", + "model.layers.19.mlp.experts.42.up_proj", + "model.layers.19.mlp.experts.43.down_proj", + "model.layers.19.mlp.experts.43.gate_proj", + "model.layers.19.mlp.experts.43.up_proj", + "model.layers.19.mlp.experts.44.down_proj", + "model.layers.19.mlp.experts.44.gate_proj", + "model.layers.19.mlp.experts.44.up_proj", + "model.layers.19.mlp.experts.45.down_proj", + "model.layers.19.mlp.experts.45.gate_proj", + "model.layers.19.mlp.experts.45.up_proj", + "model.layers.19.mlp.experts.46.down_proj", + "model.layers.19.mlp.experts.46.gate_proj", + "model.layers.19.mlp.experts.46.up_proj", + "model.layers.19.mlp.experts.47.down_proj", + "model.layers.19.mlp.experts.47.gate_proj", + "model.layers.19.mlp.experts.47.up_proj", + "model.layers.19.mlp.experts.48.down_proj", + "model.layers.19.mlp.experts.48.gate_proj", + "model.layers.19.mlp.experts.48.up_proj", + "model.layers.19.mlp.experts.49.down_proj", + "model.layers.19.mlp.experts.49.gate_proj", + "model.layers.19.mlp.experts.49.up_proj", + "model.layers.19.mlp.experts.5.down_proj", + "model.layers.19.mlp.experts.5.gate_proj", + "model.layers.19.mlp.experts.5.up_proj", + "model.layers.19.mlp.experts.50.down_proj", + "model.layers.19.mlp.experts.50.gate_proj", + "model.layers.19.mlp.experts.50.up_proj", + "model.layers.19.mlp.experts.51.down_proj", + "model.layers.19.mlp.experts.51.gate_proj", + "model.layers.19.mlp.experts.51.up_proj", + "model.layers.19.mlp.experts.52.down_proj", + "model.layers.19.mlp.experts.52.gate_proj", + "model.layers.19.mlp.experts.52.up_proj", + "model.layers.19.mlp.experts.53.down_proj", + "model.layers.19.mlp.experts.53.gate_proj", + "model.layers.19.mlp.experts.53.up_proj", + "model.layers.19.mlp.experts.54.down_proj", + "model.layers.19.mlp.experts.54.gate_proj", + "model.layers.19.mlp.experts.54.up_proj", + "model.layers.19.mlp.experts.55.down_proj", + "model.layers.19.mlp.experts.55.gate_proj", + "model.layers.19.mlp.experts.55.up_proj", + "model.layers.19.mlp.experts.56.down_proj", + "model.layers.19.mlp.experts.56.gate_proj", + "model.layers.19.mlp.experts.56.up_proj", + "model.layers.19.mlp.experts.57.down_proj", + "model.layers.19.mlp.experts.57.gate_proj", + "model.layers.19.mlp.experts.57.up_proj", + "model.layers.19.mlp.experts.58.down_proj", + "model.layers.19.mlp.experts.58.gate_proj", + "model.layers.19.mlp.experts.58.up_proj", + "model.layers.19.mlp.experts.59.down_proj", + "model.layers.19.mlp.experts.59.gate_proj", + "model.layers.19.mlp.experts.59.up_proj", + "model.layers.19.mlp.experts.6.down_proj", + "model.layers.19.mlp.experts.6.gate_proj", + "model.layers.19.mlp.experts.6.up_proj", + "model.layers.19.mlp.experts.60.down_proj", + "model.layers.19.mlp.experts.60.gate_proj", + "model.layers.19.mlp.experts.60.up_proj", + "model.layers.19.mlp.experts.61.down_proj", + "model.layers.19.mlp.experts.61.gate_proj", + "model.layers.19.mlp.experts.61.up_proj", + "model.layers.19.mlp.experts.62.down_proj", + "model.layers.19.mlp.experts.62.gate_proj", + "model.layers.19.mlp.experts.62.up_proj", + "model.layers.19.mlp.experts.63.down_proj", + "model.layers.19.mlp.experts.63.gate_proj", + "model.layers.19.mlp.experts.63.up_proj", + "model.layers.19.mlp.experts.64.down_proj", + "model.layers.19.mlp.experts.64.gate_proj", + "model.layers.19.mlp.experts.64.up_proj", + "model.layers.19.mlp.experts.65.down_proj", + "model.layers.19.mlp.experts.65.gate_proj", + "model.layers.19.mlp.experts.65.up_proj", + "model.layers.19.mlp.experts.66.down_proj", + "model.layers.19.mlp.experts.66.gate_proj", + "model.layers.19.mlp.experts.66.up_proj", + "model.layers.19.mlp.experts.67.down_proj", + "model.layers.19.mlp.experts.67.gate_proj", + "model.layers.19.mlp.experts.67.up_proj", + "model.layers.19.mlp.experts.68.down_proj", + "model.layers.19.mlp.experts.68.gate_proj", + "model.layers.19.mlp.experts.68.up_proj", + "model.layers.19.mlp.experts.69.down_proj", + "model.layers.19.mlp.experts.69.gate_proj", + "model.layers.19.mlp.experts.69.up_proj", + "model.layers.19.mlp.experts.7.down_proj", + "model.layers.19.mlp.experts.7.gate_proj", + "model.layers.19.mlp.experts.7.up_proj", + "model.layers.19.mlp.experts.70.down_proj", + "model.layers.19.mlp.experts.70.gate_proj", + "model.layers.19.mlp.experts.70.up_proj", + "model.layers.19.mlp.experts.71.down_proj", + "model.layers.19.mlp.experts.71.gate_proj", + "model.layers.19.mlp.experts.71.up_proj", + "model.layers.19.mlp.experts.72.down_proj", + "model.layers.19.mlp.experts.72.gate_proj", + "model.layers.19.mlp.experts.72.up_proj", + "model.layers.19.mlp.experts.73.down_proj", + "model.layers.19.mlp.experts.73.gate_proj", + "model.layers.19.mlp.experts.73.up_proj", + "model.layers.19.mlp.experts.74.down_proj", + "model.layers.19.mlp.experts.74.gate_proj", + "model.layers.19.mlp.experts.74.up_proj", + "model.layers.19.mlp.experts.75.down_proj", + "model.layers.19.mlp.experts.75.gate_proj", + "model.layers.19.mlp.experts.75.up_proj", + "model.layers.19.mlp.experts.76.down_proj", + "model.layers.19.mlp.experts.76.gate_proj", + "model.layers.19.mlp.experts.76.up_proj", + "model.layers.19.mlp.experts.77.down_proj", + "model.layers.19.mlp.experts.77.gate_proj", + "model.layers.19.mlp.experts.77.up_proj", + "model.layers.19.mlp.experts.78.down_proj", + "model.layers.19.mlp.experts.78.gate_proj", + "model.layers.19.mlp.experts.78.up_proj", + "model.layers.19.mlp.experts.79.down_proj", + "model.layers.19.mlp.experts.79.gate_proj", + "model.layers.19.mlp.experts.79.up_proj", + "model.layers.19.mlp.experts.8.down_proj", + "model.layers.19.mlp.experts.8.gate_proj", + "model.layers.19.mlp.experts.8.up_proj", + "model.layers.19.mlp.experts.80.down_proj", + "model.layers.19.mlp.experts.80.gate_proj", + "model.layers.19.mlp.experts.80.up_proj", + "model.layers.19.mlp.experts.81.down_proj", + "model.layers.19.mlp.experts.81.gate_proj", + "model.layers.19.mlp.experts.81.up_proj", + "model.layers.19.mlp.experts.82.down_proj", + "model.layers.19.mlp.experts.82.gate_proj", + "model.layers.19.mlp.experts.82.up_proj", + "model.layers.19.mlp.experts.83.down_proj", + "model.layers.19.mlp.experts.83.gate_proj", + "model.layers.19.mlp.experts.83.up_proj", + "model.layers.19.mlp.experts.84.down_proj", + "model.layers.19.mlp.experts.84.gate_proj", + "model.layers.19.mlp.experts.84.up_proj", + "model.layers.19.mlp.experts.85.down_proj", + "model.layers.19.mlp.experts.85.gate_proj", + "model.layers.19.mlp.experts.85.up_proj", + "model.layers.19.mlp.experts.86.down_proj", + "model.layers.19.mlp.experts.86.gate_proj", + "model.layers.19.mlp.experts.86.up_proj", + "model.layers.19.mlp.experts.87.down_proj", + "model.layers.19.mlp.experts.87.gate_proj", + "model.layers.19.mlp.experts.87.up_proj", + "model.layers.19.mlp.experts.88.down_proj", + "model.layers.19.mlp.experts.88.gate_proj", + "model.layers.19.mlp.experts.88.up_proj", + "model.layers.19.mlp.experts.89.down_proj", + "model.layers.19.mlp.experts.89.gate_proj", + "model.layers.19.mlp.experts.89.up_proj", + "model.layers.19.mlp.experts.9.down_proj", + "model.layers.19.mlp.experts.9.gate_proj", + "model.layers.19.mlp.experts.9.up_proj", + "model.layers.19.mlp.experts.90.down_proj", + "model.layers.19.mlp.experts.90.gate_proj", + "model.layers.19.mlp.experts.90.up_proj", + "model.layers.19.mlp.experts.91.down_proj", + "model.layers.19.mlp.experts.91.gate_proj", + "model.layers.19.mlp.experts.91.up_proj", + "model.layers.19.mlp.experts.92.down_proj", + "model.layers.19.mlp.experts.92.gate_proj", + "model.layers.19.mlp.experts.92.up_proj", + "model.layers.19.mlp.experts.93.down_proj", + "model.layers.19.mlp.experts.93.gate_proj", + "model.layers.19.mlp.experts.93.up_proj", + "model.layers.19.mlp.experts.94.down_proj", + "model.layers.19.mlp.experts.94.gate_proj", + "model.layers.19.mlp.experts.94.up_proj", + "model.layers.19.mlp.experts.95.down_proj", + "model.layers.19.mlp.experts.95.gate_proj", + "model.layers.19.mlp.experts.95.up_proj", + "model.layers.19.mlp.experts.96.down_proj", + "model.layers.19.mlp.experts.96.gate_proj", + "model.layers.19.mlp.experts.96.up_proj", + "model.layers.19.mlp.experts.97.down_proj", + "model.layers.19.mlp.experts.97.gate_proj", + "model.layers.19.mlp.experts.97.up_proj", + "model.layers.19.mlp.experts.98.down_proj", + "model.layers.19.mlp.experts.98.gate_proj", + "model.layers.19.mlp.experts.98.up_proj", + "model.layers.19.mlp.experts.99.down_proj", + "model.layers.19.mlp.experts.99.gate_proj", + "model.layers.19.mlp.experts.99.up_proj", + "model.layers.20.mlp.experts.0.down_proj", + "model.layers.20.mlp.experts.0.gate_proj", + "model.layers.20.mlp.experts.0.up_proj", + "model.layers.20.mlp.experts.1.down_proj", + "model.layers.20.mlp.experts.1.gate_proj", + "model.layers.20.mlp.experts.1.up_proj", + "model.layers.20.mlp.experts.10.down_proj", + "model.layers.20.mlp.experts.10.gate_proj", + "model.layers.20.mlp.experts.10.up_proj", + "model.layers.20.mlp.experts.100.down_proj", + "model.layers.20.mlp.experts.100.gate_proj", + "model.layers.20.mlp.experts.100.up_proj", + "model.layers.20.mlp.experts.101.down_proj", + "model.layers.20.mlp.experts.101.gate_proj", + "model.layers.20.mlp.experts.101.up_proj", + "model.layers.20.mlp.experts.102.down_proj", + "model.layers.20.mlp.experts.102.gate_proj", + "model.layers.20.mlp.experts.102.up_proj", + "model.layers.20.mlp.experts.103.down_proj", + "model.layers.20.mlp.experts.103.gate_proj", + "model.layers.20.mlp.experts.103.up_proj", + "model.layers.20.mlp.experts.104.down_proj", + "model.layers.20.mlp.experts.104.gate_proj", + "model.layers.20.mlp.experts.104.up_proj", + "model.layers.20.mlp.experts.105.down_proj", + "model.layers.20.mlp.experts.105.gate_proj", + "model.layers.20.mlp.experts.105.up_proj", + "model.layers.20.mlp.experts.106.down_proj", + "model.layers.20.mlp.experts.106.gate_proj", + "model.layers.20.mlp.experts.106.up_proj", + "model.layers.20.mlp.experts.107.down_proj", + "model.layers.20.mlp.experts.107.gate_proj", + "model.layers.20.mlp.experts.107.up_proj", + "model.layers.20.mlp.experts.108.down_proj", + "model.layers.20.mlp.experts.108.gate_proj", + "model.layers.20.mlp.experts.108.up_proj", + "model.layers.20.mlp.experts.109.down_proj", + "model.layers.20.mlp.experts.109.gate_proj", + "model.layers.20.mlp.experts.109.up_proj", + "model.layers.20.mlp.experts.11.down_proj", + "model.layers.20.mlp.experts.11.gate_proj", + "model.layers.20.mlp.experts.11.up_proj", + "model.layers.20.mlp.experts.110.down_proj", + "model.layers.20.mlp.experts.110.gate_proj", + "model.layers.20.mlp.experts.110.up_proj", + "model.layers.20.mlp.experts.111.down_proj", + "model.layers.20.mlp.experts.111.gate_proj", + "model.layers.20.mlp.experts.111.up_proj", + "model.layers.20.mlp.experts.112.down_proj", + "model.layers.20.mlp.experts.112.gate_proj", + "model.layers.20.mlp.experts.112.up_proj", + "model.layers.20.mlp.experts.113.down_proj", + "model.layers.20.mlp.experts.113.gate_proj", + "model.layers.20.mlp.experts.113.up_proj", + "model.layers.20.mlp.experts.114.down_proj", + "model.layers.20.mlp.experts.114.gate_proj", + "model.layers.20.mlp.experts.114.up_proj", + "model.layers.20.mlp.experts.115.down_proj", + "model.layers.20.mlp.experts.115.gate_proj", + "model.layers.20.mlp.experts.115.up_proj", + "model.layers.20.mlp.experts.116.down_proj", + "model.layers.20.mlp.experts.116.gate_proj", + "model.layers.20.mlp.experts.116.up_proj", + "model.layers.20.mlp.experts.117.down_proj", + "model.layers.20.mlp.experts.117.gate_proj", + "model.layers.20.mlp.experts.117.up_proj", + "model.layers.20.mlp.experts.118.down_proj", + "model.layers.20.mlp.experts.118.gate_proj", + "model.layers.20.mlp.experts.118.up_proj", + "model.layers.20.mlp.experts.119.down_proj", + "model.layers.20.mlp.experts.119.gate_proj", + "model.layers.20.mlp.experts.119.up_proj", + "model.layers.20.mlp.experts.12.down_proj", + "model.layers.20.mlp.experts.12.gate_proj", + "model.layers.20.mlp.experts.12.up_proj", + "model.layers.20.mlp.experts.120.down_proj", + "model.layers.20.mlp.experts.120.gate_proj", + "model.layers.20.mlp.experts.120.up_proj", + "model.layers.20.mlp.experts.121.down_proj", + "model.layers.20.mlp.experts.121.gate_proj", + "model.layers.20.mlp.experts.121.up_proj", + "model.layers.20.mlp.experts.122.down_proj", + "model.layers.20.mlp.experts.122.gate_proj", + "model.layers.20.mlp.experts.122.up_proj", + "model.layers.20.mlp.experts.123.down_proj", + "model.layers.20.mlp.experts.123.gate_proj", + "model.layers.20.mlp.experts.123.up_proj", + "model.layers.20.mlp.experts.124.down_proj", + "model.layers.20.mlp.experts.124.gate_proj", + "model.layers.20.mlp.experts.124.up_proj", + "model.layers.20.mlp.experts.125.down_proj", + "model.layers.20.mlp.experts.125.gate_proj", + "model.layers.20.mlp.experts.125.up_proj", + "model.layers.20.mlp.experts.126.down_proj", + "model.layers.20.mlp.experts.126.gate_proj", + "model.layers.20.mlp.experts.126.up_proj", + "model.layers.20.mlp.experts.127.down_proj", + "model.layers.20.mlp.experts.127.gate_proj", + "model.layers.20.mlp.experts.127.up_proj", + "model.layers.20.mlp.experts.128.down_proj", + "model.layers.20.mlp.experts.128.gate_proj", + "model.layers.20.mlp.experts.128.up_proj", + "model.layers.20.mlp.experts.129.down_proj", + "model.layers.20.mlp.experts.129.gate_proj", + "model.layers.20.mlp.experts.129.up_proj", + "model.layers.20.mlp.experts.13.down_proj", + "model.layers.20.mlp.experts.13.gate_proj", + "model.layers.20.mlp.experts.13.up_proj", + "model.layers.20.mlp.experts.130.down_proj", + "model.layers.20.mlp.experts.130.gate_proj", + "model.layers.20.mlp.experts.130.up_proj", + "model.layers.20.mlp.experts.131.down_proj", + "model.layers.20.mlp.experts.131.gate_proj", + "model.layers.20.mlp.experts.131.up_proj", + "model.layers.20.mlp.experts.132.down_proj", + "model.layers.20.mlp.experts.132.gate_proj", + "model.layers.20.mlp.experts.132.up_proj", + "model.layers.20.mlp.experts.133.down_proj", + "model.layers.20.mlp.experts.133.gate_proj", + "model.layers.20.mlp.experts.133.up_proj", + "model.layers.20.mlp.experts.134.down_proj", + "model.layers.20.mlp.experts.134.gate_proj", + "model.layers.20.mlp.experts.134.up_proj", + "model.layers.20.mlp.experts.135.down_proj", + "model.layers.20.mlp.experts.135.gate_proj", + "model.layers.20.mlp.experts.135.up_proj", + "model.layers.20.mlp.experts.136.down_proj", + "model.layers.20.mlp.experts.136.gate_proj", + "model.layers.20.mlp.experts.136.up_proj", + "model.layers.20.mlp.experts.137.down_proj", + "model.layers.20.mlp.experts.137.gate_proj", + "model.layers.20.mlp.experts.137.up_proj", + "model.layers.20.mlp.experts.138.down_proj", + "model.layers.20.mlp.experts.138.gate_proj", + "model.layers.20.mlp.experts.138.up_proj", + "model.layers.20.mlp.experts.139.down_proj", + "model.layers.20.mlp.experts.139.gate_proj", + "model.layers.20.mlp.experts.139.up_proj", + "model.layers.20.mlp.experts.14.down_proj", + "model.layers.20.mlp.experts.14.gate_proj", + "model.layers.20.mlp.experts.14.up_proj", + "model.layers.20.mlp.experts.140.down_proj", + "model.layers.20.mlp.experts.140.gate_proj", + "model.layers.20.mlp.experts.140.up_proj", + "model.layers.20.mlp.experts.141.down_proj", + "model.layers.20.mlp.experts.141.gate_proj", + "model.layers.20.mlp.experts.141.up_proj", + "model.layers.20.mlp.experts.142.down_proj", + "model.layers.20.mlp.experts.142.gate_proj", + "model.layers.20.mlp.experts.142.up_proj", + "model.layers.20.mlp.experts.143.down_proj", + "model.layers.20.mlp.experts.143.gate_proj", + "model.layers.20.mlp.experts.143.up_proj", + "model.layers.20.mlp.experts.144.down_proj", + "model.layers.20.mlp.experts.144.gate_proj", + "model.layers.20.mlp.experts.144.up_proj", + "model.layers.20.mlp.experts.145.down_proj", + "model.layers.20.mlp.experts.145.gate_proj", + "model.layers.20.mlp.experts.145.up_proj", + "model.layers.20.mlp.experts.146.down_proj", + "model.layers.20.mlp.experts.146.gate_proj", + "model.layers.20.mlp.experts.146.up_proj", + "model.layers.20.mlp.experts.147.down_proj", + "model.layers.20.mlp.experts.147.gate_proj", + "model.layers.20.mlp.experts.147.up_proj", + "model.layers.20.mlp.experts.148.down_proj", + "model.layers.20.mlp.experts.148.gate_proj", + "model.layers.20.mlp.experts.148.up_proj", + "model.layers.20.mlp.experts.149.down_proj", + "model.layers.20.mlp.experts.149.gate_proj", + "model.layers.20.mlp.experts.149.up_proj", + "model.layers.20.mlp.experts.15.down_proj", + "model.layers.20.mlp.experts.15.gate_proj", + "model.layers.20.mlp.experts.15.up_proj", + "model.layers.20.mlp.experts.150.down_proj", + "model.layers.20.mlp.experts.150.gate_proj", + "model.layers.20.mlp.experts.150.up_proj", + "model.layers.20.mlp.experts.151.down_proj", + "model.layers.20.mlp.experts.151.gate_proj", + "model.layers.20.mlp.experts.151.up_proj", + "model.layers.20.mlp.experts.152.down_proj", + "model.layers.20.mlp.experts.152.gate_proj", + "model.layers.20.mlp.experts.152.up_proj", + "model.layers.20.mlp.experts.153.down_proj", + "model.layers.20.mlp.experts.153.gate_proj", + "model.layers.20.mlp.experts.153.up_proj", + "model.layers.20.mlp.experts.154.down_proj", + "model.layers.20.mlp.experts.154.gate_proj", + "model.layers.20.mlp.experts.154.up_proj", + "model.layers.20.mlp.experts.155.down_proj", + "model.layers.20.mlp.experts.155.gate_proj", + "model.layers.20.mlp.experts.155.up_proj", + "model.layers.20.mlp.experts.156.down_proj", + "model.layers.20.mlp.experts.156.gate_proj", + "model.layers.20.mlp.experts.156.up_proj", + "model.layers.20.mlp.experts.157.down_proj", + "model.layers.20.mlp.experts.157.gate_proj", + "model.layers.20.mlp.experts.157.up_proj", + "model.layers.20.mlp.experts.158.down_proj", + "model.layers.20.mlp.experts.158.gate_proj", + "model.layers.20.mlp.experts.158.up_proj", + "model.layers.20.mlp.experts.159.down_proj", + "model.layers.20.mlp.experts.159.gate_proj", + "model.layers.20.mlp.experts.159.up_proj", + "model.layers.20.mlp.experts.16.down_proj", + "model.layers.20.mlp.experts.16.gate_proj", + "model.layers.20.mlp.experts.16.up_proj", + "model.layers.20.mlp.experts.160.down_proj", + "model.layers.20.mlp.experts.160.gate_proj", + "model.layers.20.mlp.experts.160.up_proj", + "model.layers.20.mlp.experts.161.down_proj", + "model.layers.20.mlp.experts.161.gate_proj", + "model.layers.20.mlp.experts.161.up_proj", + "model.layers.20.mlp.experts.162.down_proj", + "model.layers.20.mlp.experts.162.gate_proj", + "model.layers.20.mlp.experts.162.up_proj", + "model.layers.20.mlp.experts.163.down_proj", + "model.layers.20.mlp.experts.163.gate_proj", + "model.layers.20.mlp.experts.163.up_proj", + "model.layers.20.mlp.experts.164.down_proj", + "model.layers.20.mlp.experts.164.gate_proj", + "model.layers.20.mlp.experts.164.up_proj", + "model.layers.20.mlp.experts.165.down_proj", + "model.layers.20.mlp.experts.165.gate_proj", + "model.layers.20.mlp.experts.165.up_proj", + "model.layers.20.mlp.experts.166.down_proj", + "model.layers.20.mlp.experts.166.gate_proj", + "model.layers.20.mlp.experts.166.up_proj", + "model.layers.20.mlp.experts.167.down_proj", + "model.layers.20.mlp.experts.167.gate_proj", + "model.layers.20.mlp.experts.167.up_proj", + "model.layers.20.mlp.experts.168.down_proj", + "model.layers.20.mlp.experts.168.gate_proj", + "model.layers.20.mlp.experts.168.up_proj", + "model.layers.20.mlp.experts.169.down_proj", + "model.layers.20.mlp.experts.169.gate_proj", + "model.layers.20.mlp.experts.169.up_proj", + "model.layers.20.mlp.experts.17.down_proj", + "model.layers.20.mlp.experts.17.gate_proj", + "model.layers.20.mlp.experts.17.up_proj", + "model.layers.20.mlp.experts.170.down_proj", + "model.layers.20.mlp.experts.170.gate_proj", + "model.layers.20.mlp.experts.170.up_proj", + "model.layers.20.mlp.experts.171.down_proj", + "model.layers.20.mlp.experts.171.gate_proj", + "model.layers.20.mlp.experts.171.up_proj", + "model.layers.20.mlp.experts.172.down_proj", + "model.layers.20.mlp.experts.172.gate_proj", + "model.layers.20.mlp.experts.172.up_proj", + "model.layers.20.mlp.experts.173.down_proj", + "model.layers.20.mlp.experts.173.gate_proj", + "model.layers.20.mlp.experts.173.up_proj", + "model.layers.20.mlp.experts.174.down_proj", + "model.layers.20.mlp.experts.174.gate_proj", + "model.layers.20.mlp.experts.174.up_proj", + "model.layers.20.mlp.experts.175.down_proj", + "model.layers.20.mlp.experts.175.gate_proj", + "model.layers.20.mlp.experts.175.up_proj", + "model.layers.20.mlp.experts.176.down_proj", + "model.layers.20.mlp.experts.176.gate_proj", + "model.layers.20.mlp.experts.176.up_proj", + "model.layers.20.mlp.experts.177.down_proj", + "model.layers.20.mlp.experts.177.gate_proj", + "model.layers.20.mlp.experts.177.up_proj", + "model.layers.20.mlp.experts.178.down_proj", + "model.layers.20.mlp.experts.178.gate_proj", + "model.layers.20.mlp.experts.178.up_proj", + "model.layers.20.mlp.experts.179.down_proj", + "model.layers.20.mlp.experts.179.gate_proj", + "model.layers.20.mlp.experts.179.up_proj", + "model.layers.20.mlp.experts.18.down_proj", + "model.layers.20.mlp.experts.18.gate_proj", + "model.layers.20.mlp.experts.18.up_proj", + "model.layers.20.mlp.experts.180.down_proj", + "model.layers.20.mlp.experts.180.gate_proj", + "model.layers.20.mlp.experts.180.up_proj", + "model.layers.20.mlp.experts.181.down_proj", + "model.layers.20.mlp.experts.181.gate_proj", + "model.layers.20.mlp.experts.181.up_proj", + "model.layers.20.mlp.experts.182.down_proj", + "model.layers.20.mlp.experts.182.gate_proj", + "model.layers.20.mlp.experts.182.up_proj", + "model.layers.20.mlp.experts.183.down_proj", + "model.layers.20.mlp.experts.183.gate_proj", + "model.layers.20.mlp.experts.183.up_proj", + "model.layers.20.mlp.experts.184.down_proj", + "model.layers.20.mlp.experts.184.gate_proj", + "model.layers.20.mlp.experts.184.up_proj", + "model.layers.20.mlp.experts.185.down_proj", + "model.layers.20.mlp.experts.185.gate_proj", + "model.layers.20.mlp.experts.185.up_proj", + "model.layers.20.mlp.experts.186.down_proj", + "model.layers.20.mlp.experts.186.gate_proj", + "model.layers.20.mlp.experts.186.up_proj", + "model.layers.20.mlp.experts.187.down_proj", + "model.layers.20.mlp.experts.187.gate_proj", + "model.layers.20.mlp.experts.187.up_proj", + "model.layers.20.mlp.experts.188.down_proj", + "model.layers.20.mlp.experts.188.gate_proj", + "model.layers.20.mlp.experts.188.up_proj", + "model.layers.20.mlp.experts.189.down_proj", + "model.layers.20.mlp.experts.189.gate_proj", + "model.layers.20.mlp.experts.189.up_proj", + "model.layers.20.mlp.experts.19.down_proj", + "model.layers.20.mlp.experts.19.gate_proj", + "model.layers.20.mlp.experts.19.up_proj", + "model.layers.20.mlp.experts.190.down_proj", + "model.layers.20.mlp.experts.190.gate_proj", + "model.layers.20.mlp.experts.190.up_proj", + "model.layers.20.mlp.experts.191.down_proj", + "model.layers.20.mlp.experts.191.gate_proj", + "model.layers.20.mlp.experts.191.up_proj", + "model.layers.20.mlp.experts.192.down_proj", + "model.layers.20.mlp.experts.192.gate_proj", + "model.layers.20.mlp.experts.192.up_proj", + "model.layers.20.mlp.experts.193.down_proj", + "model.layers.20.mlp.experts.193.gate_proj", + "model.layers.20.mlp.experts.193.up_proj", + "model.layers.20.mlp.experts.194.down_proj", + "model.layers.20.mlp.experts.194.gate_proj", + "model.layers.20.mlp.experts.194.up_proj", + "model.layers.20.mlp.experts.195.down_proj", + "model.layers.20.mlp.experts.195.gate_proj", + "model.layers.20.mlp.experts.195.up_proj", + "model.layers.20.mlp.experts.196.down_proj", + "model.layers.20.mlp.experts.196.gate_proj", + "model.layers.20.mlp.experts.196.up_proj", + "model.layers.20.mlp.experts.197.down_proj", + "model.layers.20.mlp.experts.197.gate_proj", + "model.layers.20.mlp.experts.197.up_proj", + "model.layers.20.mlp.experts.198.down_proj", + "model.layers.20.mlp.experts.198.gate_proj", + "model.layers.20.mlp.experts.198.up_proj", + "model.layers.20.mlp.experts.199.down_proj", + "model.layers.20.mlp.experts.199.gate_proj", + "model.layers.20.mlp.experts.199.up_proj", + "model.layers.20.mlp.experts.2.down_proj", + "model.layers.20.mlp.experts.2.gate_proj", + "model.layers.20.mlp.experts.2.up_proj", + "model.layers.20.mlp.experts.20.down_proj", + "model.layers.20.mlp.experts.20.gate_proj", + "model.layers.20.mlp.experts.20.up_proj", + "model.layers.20.mlp.experts.200.down_proj", + "model.layers.20.mlp.experts.200.gate_proj", + "model.layers.20.mlp.experts.200.up_proj", + "model.layers.20.mlp.experts.201.down_proj", + "model.layers.20.mlp.experts.201.gate_proj", + "model.layers.20.mlp.experts.201.up_proj", + "model.layers.20.mlp.experts.202.down_proj", + "model.layers.20.mlp.experts.202.gate_proj", + "model.layers.20.mlp.experts.202.up_proj", + "model.layers.20.mlp.experts.203.down_proj", + "model.layers.20.mlp.experts.203.gate_proj", + "model.layers.20.mlp.experts.203.up_proj", + "model.layers.20.mlp.experts.204.down_proj", + "model.layers.20.mlp.experts.204.gate_proj", + "model.layers.20.mlp.experts.204.up_proj", + "model.layers.20.mlp.experts.205.down_proj", + "model.layers.20.mlp.experts.205.gate_proj", + "model.layers.20.mlp.experts.205.up_proj", + "model.layers.20.mlp.experts.206.down_proj", + "model.layers.20.mlp.experts.206.gate_proj", + "model.layers.20.mlp.experts.206.up_proj", + "model.layers.20.mlp.experts.207.down_proj", + "model.layers.20.mlp.experts.207.gate_proj", + "model.layers.20.mlp.experts.207.up_proj", + "model.layers.20.mlp.experts.208.down_proj", + "model.layers.20.mlp.experts.208.gate_proj", + "model.layers.20.mlp.experts.208.up_proj", + "model.layers.20.mlp.experts.209.down_proj", + "model.layers.20.mlp.experts.209.gate_proj", + "model.layers.20.mlp.experts.209.up_proj", + "model.layers.20.mlp.experts.21.down_proj", + "model.layers.20.mlp.experts.21.gate_proj", + "model.layers.20.mlp.experts.21.up_proj", + "model.layers.20.mlp.experts.210.down_proj", + "model.layers.20.mlp.experts.210.gate_proj", + "model.layers.20.mlp.experts.210.up_proj", + "model.layers.20.mlp.experts.211.down_proj", + "model.layers.20.mlp.experts.211.gate_proj", + "model.layers.20.mlp.experts.211.up_proj", + "model.layers.20.mlp.experts.212.down_proj", + "model.layers.20.mlp.experts.212.gate_proj", + "model.layers.20.mlp.experts.212.up_proj", + "model.layers.20.mlp.experts.213.down_proj", + "model.layers.20.mlp.experts.213.gate_proj", + "model.layers.20.mlp.experts.213.up_proj", + "model.layers.20.mlp.experts.214.down_proj", + "model.layers.20.mlp.experts.214.gate_proj", + "model.layers.20.mlp.experts.214.up_proj", + "model.layers.20.mlp.experts.215.down_proj", + "model.layers.20.mlp.experts.215.gate_proj", + "model.layers.20.mlp.experts.215.up_proj", + "model.layers.20.mlp.experts.216.down_proj", + "model.layers.20.mlp.experts.216.gate_proj", + "model.layers.20.mlp.experts.216.up_proj", + "model.layers.20.mlp.experts.217.down_proj", + "model.layers.20.mlp.experts.217.gate_proj", + "model.layers.20.mlp.experts.217.up_proj", + "model.layers.20.mlp.experts.218.down_proj", + "model.layers.20.mlp.experts.218.gate_proj", + "model.layers.20.mlp.experts.218.up_proj", + "model.layers.20.mlp.experts.219.down_proj", + "model.layers.20.mlp.experts.219.gate_proj", + "model.layers.20.mlp.experts.219.up_proj", + "model.layers.20.mlp.experts.22.down_proj", + "model.layers.20.mlp.experts.22.gate_proj", + "model.layers.20.mlp.experts.22.up_proj", + "model.layers.20.mlp.experts.220.down_proj", + "model.layers.20.mlp.experts.220.gate_proj", + "model.layers.20.mlp.experts.220.up_proj", + "model.layers.20.mlp.experts.221.down_proj", + "model.layers.20.mlp.experts.221.gate_proj", + "model.layers.20.mlp.experts.221.up_proj", + "model.layers.20.mlp.experts.222.down_proj", + "model.layers.20.mlp.experts.222.gate_proj", + "model.layers.20.mlp.experts.222.up_proj", + "model.layers.20.mlp.experts.223.down_proj", + "model.layers.20.mlp.experts.223.gate_proj", + "model.layers.20.mlp.experts.223.up_proj", + "model.layers.20.mlp.experts.224.down_proj", + "model.layers.20.mlp.experts.224.gate_proj", + "model.layers.20.mlp.experts.224.up_proj", + "model.layers.20.mlp.experts.225.down_proj", + "model.layers.20.mlp.experts.225.gate_proj", + "model.layers.20.mlp.experts.225.up_proj", + "model.layers.20.mlp.experts.226.down_proj", + "model.layers.20.mlp.experts.226.gate_proj", + "model.layers.20.mlp.experts.226.up_proj", + "model.layers.20.mlp.experts.227.down_proj", + "model.layers.20.mlp.experts.227.gate_proj", + "model.layers.20.mlp.experts.227.up_proj", + "model.layers.20.mlp.experts.228.down_proj", + "model.layers.20.mlp.experts.228.gate_proj", + "model.layers.20.mlp.experts.228.up_proj", + "model.layers.20.mlp.experts.229.down_proj", + "model.layers.20.mlp.experts.229.gate_proj", + "model.layers.20.mlp.experts.229.up_proj", + "model.layers.20.mlp.experts.23.down_proj", + "model.layers.20.mlp.experts.23.gate_proj", + "model.layers.20.mlp.experts.23.up_proj", + "model.layers.20.mlp.experts.230.down_proj", + "model.layers.20.mlp.experts.230.gate_proj", + "model.layers.20.mlp.experts.230.up_proj", + "model.layers.20.mlp.experts.231.down_proj", + "model.layers.20.mlp.experts.231.gate_proj", + "model.layers.20.mlp.experts.231.up_proj", + "model.layers.20.mlp.experts.232.down_proj", + "model.layers.20.mlp.experts.232.gate_proj", + "model.layers.20.mlp.experts.232.up_proj", + "model.layers.20.mlp.experts.233.down_proj", + "model.layers.20.mlp.experts.233.gate_proj", + "model.layers.20.mlp.experts.233.up_proj", + "model.layers.20.mlp.experts.234.down_proj", + "model.layers.20.mlp.experts.234.gate_proj", + "model.layers.20.mlp.experts.234.up_proj", + "model.layers.20.mlp.experts.235.down_proj", + "model.layers.20.mlp.experts.235.gate_proj", + "model.layers.20.mlp.experts.235.up_proj", + "model.layers.20.mlp.experts.236.down_proj", + "model.layers.20.mlp.experts.236.gate_proj", + "model.layers.20.mlp.experts.236.up_proj", + "model.layers.20.mlp.experts.237.down_proj", + "model.layers.20.mlp.experts.237.gate_proj", + "model.layers.20.mlp.experts.237.up_proj", + "model.layers.20.mlp.experts.238.down_proj", + "model.layers.20.mlp.experts.238.gate_proj", + "model.layers.20.mlp.experts.238.up_proj", + "model.layers.20.mlp.experts.239.down_proj", + "model.layers.20.mlp.experts.239.gate_proj", + "model.layers.20.mlp.experts.239.up_proj", + "model.layers.20.mlp.experts.24.down_proj", + "model.layers.20.mlp.experts.24.gate_proj", + "model.layers.20.mlp.experts.24.up_proj", + "model.layers.20.mlp.experts.240.down_proj", + "model.layers.20.mlp.experts.240.gate_proj", + "model.layers.20.mlp.experts.240.up_proj", + "model.layers.20.mlp.experts.241.down_proj", + "model.layers.20.mlp.experts.241.gate_proj", + "model.layers.20.mlp.experts.241.up_proj", + "model.layers.20.mlp.experts.242.down_proj", + "model.layers.20.mlp.experts.242.gate_proj", + "model.layers.20.mlp.experts.242.up_proj", + "model.layers.20.mlp.experts.243.down_proj", + "model.layers.20.mlp.experts.243.gate_proj", + "model.layers.20.mlp.experts.243.up_proj", + "model.layers.20.mlp.experts.244.down_proj", + "model.layers.20.mlp.experts.244.gate_proj", + "model.layers.20.mlp.experts.244.up_proj", + "model.layers.20.mlp.experts.245.down_proj", + "model.layers.20.mlp.experts.245.gate_proj", + "model.layers.20.mlp.experts.245.up_proj", + "model.layers.20.mlp.experts.246.down_proj", + "model.layers.20.mlp.experts.246.gate_proj", + "model.layers.20.mlp.experts.246.up_proj", + "model.layers.20.mlp.experts.247.down_proj", + "model.layers.20.mlp.experts.247.gate_proj", + "model.layers.20.mlp.experts.247.up_proj", + "model.layers.20.mlp.experts.248.down_proj", + "model.layers.20.mlp.experts.248.gate_proj", + "model.layers.20.mlp.experts.248.up_proj", + "model.layers.20.mlp.experts.249.down_proj", + "model.layers.20.mlp.experts.249.gate_proj", + "model.layers.20.mlp.experts.249.up_proj", + "model.layers.20.mlp.experts.25.down_proj", + "model.layers.20.mlp.experts.25.gate_proj", + "model.layers.20.mlp.experts.25.up_proj", + "model.layers.20.mlp.experts.250.down_proj", + "model.layers.20.mlp.experts.250.gate_proj", + "model.layers.20.mlp.experts.250.up_proj", + "model.layers.20.mlp.experts.251.down_proj", + "model.layers.20.mlp.experts.251.gate_proj", + "model.layers.20.mlp.experts.251.up_proj", + "model.layers.20.mlp.experts.252.down_proj", + "model.layers.20.mlp.experts.252.gate_proj", + "model.layers.20.mlp.experts.252.up_proj", + "model.layers.20.mlp.experts.253.down_proj", + "model.layers.20.mlp.experts.253.gate_proj", + "model.layers.20.mlp.experts.253.up_proj", + "model.layers.20.mlp.experts.254.down_proj", + "model.layers.20.mlp.experts.254.gate_proj", + "model.layers.20.mlp.experts.254.up_proj", + "model.layers.20.mlp.experts.255.down_proj", + "model.layers.20.mlp.experts.255.gate_proj", + "model.layers.20.mlp.experts.255.up_proj", + "model.layers.20.mlp.experts.26.down_proj", + "model.layers.20.mlp.experts.26.gate_proj", + "model.layers.20.mlp.experts.26.up_proj", + "model.layers.20.mlp.experts.27.down_proj", + "model.layers.20.mlp.experts.27.gate_proj", + "model.layers.20.mlp.experts.27.up_proj", + "model.layers.20.mlp.experts.28.down_proj", + "model.layers.20.mlp.experts.28.gate_proj", + "model.layers.20.mlp.experts.28.up_proj", + "model.layers.20.mlp.experts.29.down_proj", + "model.layers.20.mlp.experts.29.gate_proj", + "model.layers.20.mlp.experts.29.up_proj", + "model.layers.20.mlp.experts.3.down_proj", + "model.layers.20.mlp.experts.3.gate_proj", + "model.layers.20.mlp.experts.3.up_proj", + "model.layers.20.mlp.experts.30.down_proj", + "model.layers.20.mlp.experts.30.gate_proj", + "model.layers.20.mlp.experts.30.up_proj", + "model.layers.20.mlp.experts.31.down_proj", + "model.layers.20.mlp.experts.31.gate_proj", + "model.layers.20.mlp.experts.31.up_proj", + "model.layers.20.mlp.experts.32.down_proj", + "model.layers.20.mlp.experts.32.gate_proj", + "model.layers.20.mlp.experts.32.up_proj", + "model.layers.20.mlp.experts.33.down_proj", + "model.layers.20.mlp.experts.33.gate_proj", + "model.layers.20.mlp.experts.33.up_proj", + "model.layers.20.mlp.experts.34.down_proj", + "model.layers.20.mlp.experts.34.gate_proj", + "model.layers.20.mlp.experts.34.up_proj", + "model.layers.20.mlp.experts.35.down_proj", + "model.layers.20.mlp.experts.35.gate_proj", + "model.layers.20.mlp.experts.35.up_proj", + "model.layers.20.mlp.experts.36.down_proj", + "model.layers.20.mlp.experts.36.gate_proj", + "model.layers.20.mlp.experts.36.up_proj", + "model.layers.20.mlp.experts.37.down_proj", + "model.layers.20.mlp.experts.37.gate_proj", + "model.layers.20.mlp.experts.37.up_proj", + "model.layers.20.mlp.experts.38.down_proj", + "model.layers.20.mlp.experts.38.gate_proj", + "model.layers.20.mlp.experts.38.up_proj", + "model.layers.20.mlp.experts.39.down_proj", + "model.layers.20.mlp.experts.39.gate_proj", + "model.layers.20.mlp.experts.39.up_proj", + "model.layers.20.mlp.experts.4.down_proj", + "model.layers.20.mlp.experts.4.gate_proj", + "model.layers.20.mlp.experts.4.up_proj", + "model.layers.20.mlp.experts.40.down_proj", + "model.layers.20.mlp.experts.40.gate_proj", + "model.layers.20.mlp.experts.40.up_proj", + "model.layers.20.mlp.experts.41.down_proj", + "model.layers.20.mlp.experts.41.gate_proj", + "model.layers.20.mlp.experts.41.up_proj", + "model.layers.20.mlp.experts.42.down_proj", + "model.layers.20.mlp.experts.42.gate_proj", + "model.layers.20.mlp.experts.42.up_proj", + "model.layers.20.mlp.experts.43.down_proj", + "model.layers.20.mlp.experts.43.gate_proj", + "model.layers.20.mlp.experts.43.up_proj", + "model.layers.20.mlp.experts.44.down_proj", + "model.layers.20.mlp.experts.44.gate_proj", + "model.layers.20.mlp.experts.44.up_proj", + "model.layers.20.mlp.experts.45.down_proj", + "model.layers.20.mlp.experts.45.gate_proj", + "model.layers.20.mlp.experts.45.up_proj", + "model.layers.20.mlp.experts.46.down_proj", + "model.layers.20.mlp.experts.46.gate_proj", + "model.layers.20.mlp.experts.46.up_proj", + "model.layers.20.mlp.experts.47.down_proj", + "model.layers.20.mlp.experts.47.gate_proj", + "model.layers.20.mlp.experts.47.up_proj", + "model.layers.20.mlp.experts.48.down_proj", + "model.layers.20.mlp.experts.48.gate_proj", + "model.layers.20.mlp.experts.48.up_proj", + "model.layers.20.mlp.experts.49.down_proj", + "model.layers.20.mlp.experts.49.gate_proj", + "model.layers.20.mlp.experts.49.up_proj", + "model.layers.20.mlp.experts.5.down_proj", + "model.layers.20.mlp.experts.5.gate_proj", + "model.layers.20.mlp.experts.5.up_proj", + "model.layers.20.mlp.experts.50.down_proj", + "model.layers.20.mlp.experts.50.gate_proj", + "model.layers.20.mlp.experts.50.up_proj", + "model.layers.20.mlp.experts.51.down_proj", + "model.layers.20.mlp.experts.51.gate_proj", + "model.layers.20.mlp.experts.51.up_proj", + "model.layers.20.mlp.experts.52.down_proj", + "model.layers.20.mlp.experts.52.gate_proj", + "model.layers.20.mlp.experts.52.up_proj", + "model.layers.20.mlp.experts.53.down_proj", + "model.layers.20.mlp.experts.53.gate_proj", + "model.layers.20.mlp.experts.53.up_proj", + "model.layers.20.mlp.experts.54.down_proj", + "model.layers.20.mlp.experts.54.gate_proj", + "model.layers.20.mlp.experts.54.up_proj", + "model.layers.20.mlp.experts.55.down_proj", + "model.layers.20.mlp.experts.55.gate_proj", + "model.layers.20.mlp.experts.55.up_proj", + "model.layers.20.mlp.experts.56.down_proj", + "model.layers.20.mlp.experts.56.gate_proj", + "model.layers.20.mlp.experts.56.up_proj", + "model.layers.20.mlp.experts.57.down_proj", + "model.layers.20.mlp.experts.57.gate_proj", + "model.layers.20.mlp.experts.57.up_proj", + "model.layers.20.mlp.experts.58.down_proj", + "model.layers.20.mlp.experts.58.gate_proj", + "model.layers.20.mlp.experts.58.up_proj", + "model.layers.20.mlp.experts.59.down_proj", + "model.layers.20.mlp.experts.59.gate_proj", + "model.layers.20.mlp.experts.59.up_proj", + "model.layers.20.mlp.experts.6.down_proj", + "model.layers.20.mlp.experts.6.gate_proj", + "model.layers.20.mlp.experts.6.up_proj", + "model.layers.20.mlp.experts.60.down_proj", + "model.layers.20.mlp.experts.60.gate_proj", + "model.layers.20.mlp.experts.60.up_proj", + "model.layers.20.mlp.experts.61.down_proj", + "model.layers.20.mlp.experts.61.gate_proj", + "model.layers.20.mlp.experts.61.up_proj", + "model.layers.20.mlp.experts.62.down_proj", + "model.layers.20.mlp.experts.62.gate_proj", + "model.layers.20.mlp.experts.62.up_proj", + "model.layers.20.mlp.experts.63.down_proj", + "model.layers.20.mlp.experts.63.gate_proj", + "model.layers.20.mlp.experts.63.up_proj", + "model.layers.20.mlp.experts.64.down_proj", + "model.layers.20.mlp.experts.64.gate_proj", + "model.layers.20.mlp.experts.64.up_proj", + "model.layers.20.mlp.experts.65.down_proj", + "model.layers.20.mlp.experts.65.gate_proj", + "model.layers.20.mlp.experts.65.up_proj", + "model.layers.20.mlp.experts.66.down_proj", + "model.layers.20.mlp.experts.66.gate_proj", + "model.layers.20.mlp.experts.66.up_proj", + "model.layers.20.mlp.experts.67.down_proj", + "model.layers.20.mlp.experts.67.gate_proj", + "model.layers.20.mlp.experts.67.up_proj", + "model.layers.20.mlp.experts.68.down_proj", + "model.layers.20.mlp.experts.68.gate_proj", + "model.layers.20.mlp.experts.68.up_proj", + "model.layers.20.mlp.experts.69.down_proj", + "model.layers.20.mlp.experts.69.gate_proj", + "model.layers.20.mlp.experts.69.up_proj", + "model.layers.20.mlp.experts.7.down_proj", + "model.layers.20.mlp.experts.7.gate_proj", + "model.layers.20.mlp.experts.7.up_proj", + "model.layers.20.mlp.experts.70.down_proj", + "model.layers.20.mlp.experts.70.gate_proj", + "model.layers.20.mlp.experts.70.up_proj", + "model.layers.20.mlp.experts.71.down_proj", + "model.layers.20.mlp.experts.71.gate_proj", + "model.layers.20.mlp.experts.71.up_proj", + "model.layers.20.mlp.experts.72.down_proj", + "model.layers.20.mlp.experts.72.gate_proj", + "model.layers.20.mlp.experts.72.up_proj", + "model.layers.20.mlp.experts.73.down_proj", + "model.layers.20.mlp.experts.73.gate_proj", + "model.layers.20.mlp.experts.73.up_proj", + "model.layers.20.mlp.experts.74.down_proj", + "model.layers.20.mlp.experts.74.gate_proj", + "model.layers.20.mlp.experts.74.up_proj", + "model.layers.20.mlp.experts.75.down_proj", + "model.layers.20.mlp.experts.75.gate_proj", + "model.layers.20.mlp.experts.75.up_proj", + "model.layers.20.mlp.experts.76.down_proj", + "model.layers.20.mlp.experts.76.gate_proj", + "model.layers.20.mlp.experts.76.up_proj", + "model.layers.20.mlp.experts.77.down_proj", + "model.layers.20.mlp.experts.77.gate_proj", + "model.layers.20.mlp.experts.77.up_proj", + "model.layers.20.mlp.experts.78.down_proj", + "model.layers.20.mlp.experts.78.gate_proj", + "model.layers.20.mlp.experts.78.up_proj", + "model.layers.20.mlp.experts.79.down_proj", + "model.layers.20.mlp.experts.79.gate_proj", + "model.layers.20.mlp.experts.79.up_proj", + "model.layers.20.mlp.experts.8.down_proj", + "model.layers.20.mlp.experts.8.gate_proj", + "model.layers.20.mlp.experts.8.up_proj", + "model.layers.20.mlp.experts.80.down_proj", + "model.layers.20.mlp.experts.80.gate_proj", + "model.layers.20.mlp.experts.80.up_proj", + "model.layers.20.mlp.experts.81.down_proj", + "model.layers.20.mlp.experts.81.gate_proj", + "model.layers.20.mlp.experts.81.up_proj", + "model.layers.20.mlp.experts.82.down_proj", + "model.layers.20.mlp.experts.82.gate_proj", + "model.layers.20.mlp.experts.82.up_proj", + "model.layers.20.mlp.experts.83.down_proj", + "model.layers.20.mlp.experts.83.gate_proj", + "model.layers.20.mlp.experts.83.up_proj", + "model.layers.20.mlp.experts.84.down_proj", + "model.layers.20.mlp.experts.84.gate_proj", + "model.layers.20.mlp.experts.84.up_proj", + "model.layers.20.mlp.experts.85.down_proj", + "model.layers.20.mlp.experts.85.gate_proj", + "model.layers.20.mlp.experts.85.up_proj", + "model.layers.20.mlp.experts.86.down_proj", + "model.layers.20.mlp.experts.86.gate_proj", + "model.layers.20.mlp.experts.86.up_proj", + "model.layers.20.mlp.experts.87.down_proj", + "model.layers.20.mlp.experts.87.gate_proj", + "model.layers.20.mlp.experts.87.up_proj", + "model.layers.20.mlp.experts.88.down_proj", + "model.layers.20.mlp.experts.88.gate_proj", + "model.layers.20.mlp.experts.88.up_proj", + "model.layers.20.mlp.experts.89.down_proj", + "model.layers.20.mlp.experts.89.gate_proj", + "model.layers.20.mlp.experts.89.up_proj", + "model.layers.20.mlp.experts.9.down_proj", + "model.layers.20.mlp.experts.9.gate_proj", + "model.layers.20.mlp.experts.9.up_proj", + "model.layers.20.mlp.experts.90.down_proj", + "model.layers.20.mlp.experts.90.gate_proj", + "model.layers.20.mlp.experts.90.up_proj", + "model.layers.20.mlp.experts.91.down_proj", + "model.layers.20.mlp.experts.91.gate_proj", + "model.layers.20.mlp.experts.91.up_proj", + "model.layers.20.mlp.experts.92.down_proj", + "model.layers.20.mlp.experts.92.gate_proj", + "model.layers.20.mlp.experts.92.up_proj", + "model.layers.20.mlp.experts.93.down_proj", + "model.layers.20.mlp.experts.93.gate_proj", + "model.layers.20.mlp.experts.93.up_proj", + "model.layers.20.mlp.experts.94.down_proj", + "model.layers.20.mlp.experts.94.gate_proj", + "model.layers.20.mlp.experts.94.up_proj", + "model.layers.20.mlp.experts.95.down_proj", + "model.layers.20.mlp.experts.95.gate_proj", + "model.layers.20.mlp.experts.95.up_proj", + "model.layers.20.mlp.experts.96.down_proj", + "model.layers.20.mlp.experts.96.gate_proj", + "model.layers.20.mlp.experts.96.up_proj", + "model.layers.20.mlp.experts.97.down_proj", + "model.layers.20.mlp.experts.97.gate_proj", + "model.layers.20.mlp.experts.97.up_proj", + "model.layers.20.mlp.experts.98.down_proj", + "model.layers.20.mlp.experts.98.gate_proj", + "model.layers.20.mlp.experts.98.up_proj", + "model.layers.20.mlp.experts.99.down_proj", + "model.layers.20.mlp.experts.99.gate_proj", + "model.layers.20.mlp.experts.99.up_proj", + "model.layers.21.mlp.experts.0.down_proj", + "model.layers.21.mlp.experts.0.gate_proj", + "model.layers.21.mlp.experts.0.up_proj", + "model.layers.21.mlp.experts.1.down_proj", + "model.layers.21.mlp.experts.1.gate_proj", + "model.layers.21.mlp.experts.1.up_proj", + "model.layers.21.mlp.experts.10.down_proj", + "model.layers.21.mlp.experts.10.gate_proj", + "model.layers.21.mlp.experts.10.up_proj", + "model.layers.21.mlp.experts.100.down_proj", + "model.layers.21.mlp.experts.100.gate_proj", + "model.layers.21.mlp.experts.100.up_proj", + "model.layers.21.mlp.experts.101.down_proj", + "model.layers.21.mlp.experts.101.gate_proj", + "model.layers.21.mlp.experts.101.up_proj", + "model.layers.21.mlp.experts.102.down_proj", + "model.layers.21.mlp.experts.102.gate_proj", + "model.layers.21.mlp.experts.102.up_proj", + "model.layers.21.mlp.experts.103.down_proj", + "model.layers.21.mlp.experts.103.gate_proj", + "model.layers.21.mlp.experts.103.up_proj", + "model.layers.21.mlp.experts.104.down_proj", + "model.layers.21.mlp.experts.104.gate_proj", + "model.layers.21.mlp.experts.104.up_proj", + "model.layers.21.mlp.experts.105.down_proj", + "model.layers.21.mlp.experts.105.gate_proj", + "model.layers.21.mlp.experts.105.up_proj", + "model.layers.21.mlp.experts.106.down_proj", + "model.layers.21.mlp.experts.106.gate_proj", + "model.layers.21.mlp.experts.106.up_proj", + "model.layers.21.mlp.experts.107.down_proj", + "model.layers.21.mlp.experts.107.gate_proj", + "model.layers.21.mlp.experts.107.up_proj", + "model.layers.21.mlp.experts.108.down_proj", + "model.layers.21.mlp.experts.108.gate_proj", + "model.layers.21.mlp.experts.108.up_proj", + "model.layers.21.mlp.experts.109.down_proj", + "model.layers.21.mlp.experts.109.gate_proj", + "model.layers.21.mlp.experts.109.up_proj", + "model.layers.21.mlp.experts.11.down_proj", + "model.layers.21.mlp.experts.11.gate_proj", + "model.layers.21.mlp.experts.11.up_proj", + "model.layers.21.mlp.experts.110.down_proj", + "model.layers.21.mlp.experts.110.gate_proj", + "model.layers.21.mlp.experts.110.up_proj", + "model.layers.21.mlp.experts.111.down_proj", + "model.layers.21.mlp.experts.111.gate_proj", + "model.layers.21.mlp.experts.111.up_proj", + "model.layers.21.mlp.experts.112.down_proj", + "model.layers.21.mlp.experts.112.gate_proj", + "model.layers.21.mlp.experts.112.up_proj", + "model.layers.21.mlp.experts.113.down_proj", + "model.layers.21.mlp.experts.113.gate_proj", + "model.layers.21.mlp.experts.113.up_proj", + "model.layers.21.mlp.experts.114.down_proj", + "model.layers.21.mlp.experts.114.gate_proj", + "model.layers.21.mlp.experts.114.up_proj", + "model.layers.21.mlp.experts.115.down_proj", + "model.layers.21.mlp.experts.115.gate_proj", + "model.layers.21.mlp.experts.115.up_proj", + "model.layers.21.mlp.experts.116.down_proj", + "model.layers.21.mlp.experts.116.gate_proj", + "model.layers.21.mlp.experts.116.up_proj", + "model.layers.21.mlp.experts.117.down_proj", + "model.layers.21.mlp.experts.117.gate_proj", + "model.layers.21.mlp.experts.117.up_proj", + "model.layers.21.mlp.experts.118.down_proj", + "model.layers.21.mlp.experts.118.gate_proj", + "model.layers.21.mlp.experts.118.up_proj", + "model.layers.21.mlp.experts.119.down_proj", + "model.layers.21.mlp.experts.119.gate_proj", + "model.layers.21.mlp.experts.119.up_proj", + "model.layers.21.mlp.experts.12.down_proj", + "model.layers.21.mlp.experts.12.gate_proj", + "model.layers.21.mlp.experts.12.up_proj", + "model.layers.21.mlp.experts.120.down_proj", + "model.layers.21.mlp.experts.120.gate_proj", + "model.layers.21.mlp.experts.120.up_proj", + "model.layers.21.mlp.experts.121.down_proj", + "model.layers.21.mlp.experts.121.gate_proj", + "model.layers.21.mlp.experts.121.up_proj", + "model.layers.21.mlp.experts.122.down_proj", + "model.layers.21.mlp.experts.122.gate_proj", + "model.layers.21.mlp.experts.122.up_proj", + "model.layers.21.mlp.experts.123.down_proj", + "model.layers.21.mlp.experts.123.gate_proj", + "model.layers.21.mlp.experts.123.up_proj", + "model.layers.21.mlp.experts.124.down_proj", + "model.layers.21.mlp.experts.124.gate_proj", + "model.layers.21.mlp.experts.124.up_proj", + "model.layers.21.mlp.experts.125.down_proj", + "model.layers.21.mlp.experts.125.gate_proj", + "model.layers.21.mlp.experts.125.up_proj", + "model.layers.21.mlp.experts.126.down_proj", + "model.layers.21.mlp.experts.126.gate_proj", + "model.layers.21.mlp.experts.126.up_proj", + "model.layers.21.mlp.experts.127.down_proj", + "model.layers.21.mlp.experts.127.gate_proj", + "model.layers.21.mlp.experts.127.up_proj", + "model.layers.21.mlp.experts.128.down_proj", + "model.layers.21.mlp.experts.128.gate_proj", + "model.layers.21.mlp.experts.128.up_proj", + "model.layers.21.mlp.experts.129.down_proj", + "model.layers.21.mlp.experts.129.gate_proj", + "model.layers.21.mlp.experts.129.up_proj", + "model.layers.21.mlp.experts.13.down_proj", + "model.layers.21.mlp.experts.13.gate_proj", + "model.layers.21.mlp.experts.13.up_proj", + "model.layers.21.mlp.experts.130.down_proj", + "model.layers.21.mlp.experts.130.gate_proj", + "model.layers.21.mlp.experts.130.up_proj", + "model.layers.21.mlp.experts.131.down_proj", + "model.layers.21.mlp.experts.131.gate_proj", + "model.layers.21.mlp.experts.131.up_proj", + "model.layers.21.mlp.experts.132.down_proj", + "model.layers.21.mlp.experts.132.gate_proj", + "model.layers.21.mlp.experts.132.up_proj", + "model.layers.21.mlp.experts.133.down_proj", + "model.layers.21.mlp.experts.133.gate_proj", + "model.layers.21.mlp.experts.133.up_proj", + "model.layers.21.mlp.experts.134.down_proj", + "model.layers.21.mlp.experts.134.gate_proj", + "model.layers.21.mlp.experts.134.up_proj", + "model.layers.21.mlp.experts.135.down_proj", + "model.layers.21.mlp.experts.135.gate_proj", + "model.layers.21.mlp.experts.135.up_proj", + "model.layers.21.mlp.experts.136.down_proj", + "model.layers.21.mlp.experts.136.gate_proj", + "model.layers.21.mlp.experts.136.up_proj", + "model.layers.21.mlp.experts.137.down_proj", + "model.layers.21.mlp.experts.137.gate_proj", + "model.layers.21.mlp.experts.137.up_proj", + "model.layers.21.mlp.experts.138.down_proj", + "model.layers.21.mlp.experts.138.gate_proj", + "model.layers.21.mlp.experts.138.up_proj", + "model.layers.21.mlp.experts.139.down_proj", + "model.layers.21.mlp.experts.139.gate_proj", + "model.layers.21.mlp.experts.139.up_proj", + "model.layers.21.mlp.experts.14.down_proj", + "model.layers.21.mlp.experts.14.gate_proj", + "model.layers.21.mlp.experts.14.up_proj", + "model.layers.21.mlp.experts.140.down_proj", + "model.layers.21.mlp.experts.140.gate_proj", + "model.layers.21.mlp.experts.140.up_proj", + "model.layers.21.mlp.experts.141.down_proj", + "model.layers.21.mlp.experts.141.gate_proj", + "model.layers.21.mlp.experts.141.up_proj", + "model.layers.21.mlp.experts.142.down_proj", + "model.layers.21.mlp.experts.142.gate_proj", + "model.layers.21.mlp.experts.142.up_proj", + "model.layers.21.mlp.experts.143.down_proj", + "model.layers.21.mlp.experts.143.gate_proj", + "model.layers.21.mlp.experts.143.up_proj", + "model.layers.21.mlp.experts.144.down_proj", + "model.layers.21.mlp.experts.144.gate_proj", + "model.layers.21.mlp.experts.144.up_proj", + "model.layers.21.mlp.experts.145.down_proj", + "model.layers.21.mlp.experts.145.gate_proj", + "model.layers.21.mlp.experts.145.up_proj", + "model.layers.21.mlp.experts.146.down_proj", + "model.layers.21.mlp.experts.146.gate_proj", + "model.layers.21.mlp.experts.146.up_proj", + "model.layers.21.mlp.experts.147.down_proj", + "model.layers.21.mlp.experts.147.gate_proj", + "model.layers.21.mlp.experts.147.up_proj", + "model.layers.21.mlp.experts.148.down_proj", + "model.layers.21.mlp.experts.148.gate_proj", + "model.layers.21.mlp.experts.148.up_proj", + "model.layers.21.mlp.experts.149.down_proj", + "model.layers.21.mlp.experts.149.gate_proj", + "model.layers.21.mlp.experts.149.up_proj", + "model.layers.21.mlp.experts.15.down_proj", + "model.layers.21.mlp.experts.15.gate_proj", + "model.layers.21.mlp.experts.15.up_proj", + "model.layers.21.mlp.experts.150.down_proj", + "model.layers.21.mlp.experts.150.gate_proj", + "model.layers.21.mlp.experts.150.up_proj", + "model.layers.21.mlp.experts.151.down_proj", + "model.layers.21.mlp.experts.151.gate_proj", + "model.layers.21.mlp.experts.151.up_proj", + "model.layers.21.mlp.experts.152.down_proj", + "model.layers.21.mlp.experts.152.gate_proj", + "model.layers.21.mlp.experts.152.up_proj", + "model.layers.21.mlp.experts.153.down_proj", + "model.layers.21.mlp.experts.153.gate_proj", + "model.layers.21.mlp.experts.153.up_proj", + "model.layers.21.mlp.experts.154.down_proj", + "model.layers.21.mlp.experts.154.gate_proj", + "model.layers.21.mlp.experts.154.up_proj", + "model.layers.21.mlp.experts.155.down_proj", + "model.layers.21.mlp.experts.155.gate_proj", + "model.layers.21.mlp.experts.155.up_proj", + "model.layers.21.mlp.experts.156.down_proj", + "model.layers.21.mlp.experts.156.gate_proj", + "model.layers.21.mlp.experts.156.up_proj", + "model.layers.21.mlp.experts.157.down_proj", + "model.layers.21.mlp.experts.157.gate_proj", + "model.layers.21.mlp.experts.157.up_proj", + "model.layers.21.mlp.experts.158.down_proj", + "model.layers.21.mlp.experts.158.gate_proj", + "model.layers.21.mlp.experts.158.up_proj", + "model.layers.21.mlp.experts.159.down_proj", + "model.layers.21.mlp.experts.159.gate_proj", + "model.layers.21.mlp.experts.159.up_proj", + "model.layers.21.mlp.experts.16.down_proj", + "model.layers.21.mlp.experts.16.gate_proj", + "model.layers.21.mlp.experts.16.up_proj", + "model.layers.21.mlp.experts.160.down_proj", + "model.layers.21.mlp.experts.160.gate_proj", + "model.layers.21.mlp.experts.160.up_proj", + "model.layers.21.mlp.experts.161.down_proj", + "model.layers.21.mlp.experts.161.gate_proj", + "model.layers.21.mlp.experts.161.up_proj", + "model.layers.21.mlp.experts.162.down_proj", + "model.layers.21.mlp.experts.162.gate_proj", + "model.layers.21.mlp.experts.162.up_proj", + "model.layers.21.mlp.experts.163.down_proj", + "model.layers.21.mlp.experts.163.gate_proj", + "model.layers.21.mlp.experts.163.up_proj", + "model.layers.21.mlp.experts.164.down_proj", + "model.layers.21.mlp.experts.164.gate_proj", + "model.layers.21.mlp.experts.164.up_proj", + "model.layers.21.mlp.experts.165.down_proj", + "model.layers.21.mlp.experts.165.gate_proj", + "model.layers.21.mlp.experts.165.up_proj", + "model.layers.21.mlp.experts.166.down_proj", + "model.layers.21.mlp.experts.166.gate_proj", + "model.layers.21.mlp.experts.166.up_proj", + "model.layers.21.mlp.experts.167.down_proj", + "model.layers.21.mlp.experts.167.gate_proj", + "model.layers.21.mlp.experts.167.up_proj", + "model.layers.21.mlp.experts.168.down_proj", + "model.layers.21.mlp.experts.168.gate_proj", + "model.layers.21.mlp.experts.168.up_proj", + "model.layers.21.mlp.experts.169.down_proj", + "model.layers.21.mlp.experts.169.gate_proj", + "model.layers.21.mlp.experts.169.up_proj", + "model.layers.21.mlp.experts.17.down_proj", + "model.layers.21.mlp.experts.17.gate_proj", + "model.layers.21.mlp.experts.17.up_proj", + "model.layers.21.mlp.experts.170.down_proj", + "model.layers.21.mlp.experts.170.gate_proj", + "model.layers.21.mlp.experts.170.up_proj", + "model.layers.21.mlp.experts.171.down_proj", + "model.layers.21.mlp.experts.171.gate_proj", + "model.layers.21.mlp.experts.171.up_proj", + "model.layers.21.mlp.experts.172.down_proj", + "model.layers.21.mlp.experts.172.gate_proj", + "model.layers.21.mlp.experts.172.up_proj", + "model.layers.21.mlp.experts.173.down_proj", + "model.layers.21.mlp.experts.173.gate_proj", + "model.layers.21.mlp.experts.173.up_proj", + "model.layers.21.mlp.experts.174.down_proj", + "model.layers.21.mlp.experts.174.gate_proj", + "model.layers.21.mlp.experts.174.up_proj", + "model.layers.21.mlp.experts.175.down_proj", + "model.layers.21.mlp.experts.175.gate_proj", + "model.layers.21.mlp.experts.175.up_proj", + "model.layers.21.mlp.experts.176.down_proj", + "model.layers.21.mlp.experts.176.gate_proj", + "model.layers.21.mlp.experts.176.up_proj", + "model.layers.21.mlp.experts.177.down_proj", + "model.layers.21.mlp.experts.177.gate_proj", + "model.layers.21.mlp.experts.177.up_proj", + "model.layers.21.mlp.experts.178.down_proj", + "model.layers.21.mlp.experts.178.gate_proj", + "model.layers.21.mlp.experts.178.up_proj", + "model.layers.21.mlp.experts.179.down_proj", + "model.layers.21.mlp.experts.179.gate_proj", + "model.layers.21.mlp.experts.179.up_proj", + "model.layers.21.mlp.experts.18.down_proj", + "model.layers.21.mlp.experts.18.gate_proj", + "model.layers.21.mlp.experts.18.up_proj", + "model.layers.21.mlp.experts.180.down_proj", + "model.layers.21.mlp.experts.180.gate_proj", + "model.layers.21.mlp.experts.180.up_proj", + "model.layers.21.mlp.experts.181.down_proj", + "model.layers.21.mlp.experts.181.gate_proj", + "model.layers.21.mlp.experts.181.up_proj", + "model.layers.21.mlp.experts.182.down_proj", + "model.layers.21.mlp.experts.182.gate_proj", + "model.layers.21.mlp.experts.182.up_proj", + "model.layers.21.mlp.experts.183.down_proj", + "model.layers.21.mlp.experts.183.gate_proj", + "model.layers.21.mlp.experts.183.up_proj", + "model.layers.21.mlp.experts.184.down_proj", + "model.layers.21.mlp.experts.184.gate_proj", + "model.layers.21.mlp.experts.184.up_proj", + "model.layers.21.mlp.experts.185.down_proj", + "model.layers.21.mlp.experts.185.gate_proj", + "model.layers.21.mlp.experts.185.up_proj", + "model.layers.21.mlp.experts.186.down_proj", + "model.layers.21.mlp.experts.186.gate_proj", + "model.layers.21.mlp.experts.186.up_proj", + "model.layers.21.mlp.experts.187.down_proj", + "model.layers.21.mlp.experts.187.gate_proj", + "model.layers.21.mlp.experts.187.up_proj", + "model.layers.21.mlp.experts.188.down_proj", + "model.layers.21.mlp.experts.188.gate_proj", + "model.layers.21.mlp.experts.188.up_proj", + "model.layers.21.mlp.experts.189.down_proj", + "model.layers.21.mlp.experts.189.gate_proj", + "model.layers.21.mlp.experts.189.up_proj", + "model.layers.21.mlp.experts.19.down_proj", + "model.layers.21.mlp.experts.19.gate_proj", + "model.layers.21.mlp.experts.19.up_proj", + "model.layers.21.mlp.experts.190.down_proj", + "model.layers.21.mlp.experts.190.gate_proj", + "model.layers.21.mlp.experts.190.up_proj", + "model.layers.21.mlp.experts.191.down_proj", + "model.layers.21.mlp.experts.191.gate_proj", + "model.layers.21.mlp.experts.191.up_proj", + "model.layers.21.mlp.experts.192.down_proj", + "model.layers.21.mlp.experts.192.gate_proj", + "model.layers.21.mlp.experts.192.up_proj", + "model.layers.21.mlp.experts.193.down_proj", + "model.layers.21.mlp.experts.193.gate_proj", + "model.layers.21.mlp.experts.193.up_proj", + "model.layers.21.mlp.experts.194.down_proj", + "model.layers.21.mlp.experts.194.gate_proj", + "model.layers.21.mlp.experts.194.up_proj", + "model.layers.21.mlp.experts.195.down_proj", + "model.layers.21.mlp.experts.195.gate_proj", + "model.layers.21.mlp.experts.195.up_proj", + "model.layers.21.mlp.experts.196.down_proj", + "model.layers.21.mlp.experts.196.gate_proj", + "model.layers.21.mlp.experts.196.up_proj", + "model.layers.21.mlp.experts.197.down_proj", + "model.layers.21.mlp.experts.197.gate_proj", + "model.layers.21.mlp.experts.197.up_proj", + "model.layers.21.mlp.experts.198.down_proj", + "model.layers.21.mlp.experts.198.gate_proj", + "model.layers.21.mlp.experts.198.up_proj", + "model.layers.21.mlp.experts.199.down_proj", + "model.layers.21.mlp.experts.199.gate_proj", + "model.layers.21.mlp.experts.199.up_proj", + "model.layers.21.mlp.experts.2.down_proj", + "model.layers.21.mlp.experts.2.gate_proj", + "model.layers.21.mlp.experts.2.up_proj", + "model.layers.21.mlp.experts.20.down_proj", + "model.layers.21.mlp.experts.20.gate_proj", + "model.layers.21.mlp.experts.20.up_proj", + "model.layers.21.mlp.experts.200.down_proj", + "model.layers.21.mlp.experts.200.gate_proj", + "model.layers.21.mlp.experts.200.up_proj", + "model.layers.21.mlp.experts.201.down_proj", + "model.layers.21.mlp.experts.201.gate_proj", + "model.layers.21.mlp.experts.201.up_proj", + "model.layers.21.mlp.experts.202.down_proj", + "model.layers.21.mlp.experts.202.gate_proj", + "model.layers.21.mlp.experts.202.up_proj", + "model.layers.21.mlp.experts.203.down_proj", + "model.layers.21.mlp.experts.203.gate_proj", + "model.layers.21.mlp.experts.203.up_proj", + "model.layers.21.mlp.experts.204.down_proj", + "model.layers.21.mlp.experts.204.gate_proj", + "model.layers.21.mlp.experts.204.up_proj", + "model.layers.21.mlp.experts.205.down_proj", + "model.layers.21.mlp.experts.205.gate_proj", + "model.layers.21.mlp.experts.205.up_proj", + "model.layers.21.mlp.experts.206.down_proj", + "model.layers.21.mlp.experts.206.gate_proj", + "model.layers.21.mlp.experts.206.up_proj", + "model.layers.21.mlp.experts.207.down_proj", + "model.layers.21.mlp.experts.207.gate_proj", + "model.layers.21.mlp.experts.207.up_proj", + "model.layers.21.mlp.experts.208.down_proj", + "model.layers.21.mlp.experts.208.gate_proj", + "model.layers.21.mlp.experts.208.up_proj", + "model.layers.21.mlp.experts.209.down_proj", + "model.layers.21.mlp.experts.209.gate_proj", + "model.layers.21.mlp.experts.209.up_proj", + "model.layers.21.mlp.experts.21.down_proj", + "model.layers.21.mlp.experts.21.gate_proj", + "model.layers.21.mlp.experts.21.up_proj", + "model.layers.21.mlp.experts.210.down_proj", + "model.layers.21.mlp.experts.210.gate_proj", + "model.layers.21.mlp.experts.210.up_proj", + "model.layers.21.mlp.experts.211.down_proj", + "model.layers.21.mlp.experts.211.gate_proj", + "model.layers.21.mlp.experts.211.up_proj", + "model.layers.21.mlp.experts.212.down_proj", + "model.layers.21.mlp.experts.212.gate_proj", + "model.layers.21.mlp.experts.212.up_proj", + "model.layers.21.mlp.experts.213.down_proj", + "model.layers.21.mlp.experts.213.gate_proj", + "model.layers.21.mlp.experts.213.up_proj", + "model.layers.21.mlp.experts.214.down_proj", + "model.layers.21.mlp.experts.214.gate_proj", + "model.layers.21.mlp.experts.214.up_proj", + "model.layers.21.mlp.experts.215.down_proj", + "model.layers.21.mlp.experts.215.gate_proj", + "model.layers.21.mlp.experts.215.up_proj", + "model.layers.21.mlp.experts.216.down_proj", + "model.layers.21.mlp.experts.216.gate_proj", + "model.layers.21.mlp.experts.216.up_proj", + "model.layers.21.mlp.experts.217.down_proj", + "model.layers.21.mlp.experts.217.gate_proj", + "model.layers.21.mlp.experts.217.up_proj", + "model.layers.21.mlp.experts.218.down_proj", + "model.layers.21.mlp.experts.218.gate_proj", + "model.layers.21.mlp.experts.218.up_proj", + "model.layers.21.mlp.experts.219.down_proj", + "model.layers.21.mlp.experts.219.gate_proj", + "model.layers.21.mlp.experts.219.up_proj", + "model.layers.21.mlp.experts.22.down_proj", + "model.layers.21.mlp.experts.22.gate_proj", + "model.layers.21.mlp.experts.22.up_proj", + "model.layers.21.mlp.experts.220.down_proj", + "model.layers.21.mlp.experts.220.gate_proj", + "model.layers.21.mlp.experts.220.up_proj", + "model.layers.21.mlp.experts.221.down_proj", + "model.layers.21.mlp.experts.221.gate_proj", + "model.layers.21.mlp.experts.221.up_proj", + "model.layers.21.mlp.experts.222.down_proj", + "model.layers.21.mlp.experts.222.gate_proj", + "model.layers.21.mlp.experts.222.up_proj", + "model.layers.21.mlp.experts.223.down_proj", + "model.layers.21.mlp.experts.223.gate_proj", + "model.layers.21.mlp.experts.223.up_proj", + "model.layers.21.mlp.experts.224.down_proj", + "model.layers.21.mlp.experts.224.gate_proj", + "model.layers.21.mlp.experts.224.up_proj", + "model.layers.21.mlp.experts.225.down_proj", + "model.layers.21.mlp.experts.225.gate_proj", + "model.layers.21.mlp.experts.225.up_proj", + "model.layers.21.mlp.experts.226.down_proj", + "model.layers.21.mlp.experts.226.gate_proj", + "model.layers.21.mlp.experts.226.up_proj", + "model.layers.21.mlp.experts.227.down_proj", + "model.layers.21.mlp.experts.227.gate_proj", + "model.layers.21.mlp.experts.227.up_proj", + "model.layers.21.mlp.experts.228.down_proj", + "model.layers.21.mlp.experts.228.gate_proj", + "model.layers.21.mlp.experts.228.up_proj", + "model.layers.21.mlp.experts.229.down_proj", + "model.layers.21.mlp.experts.229.gate_proj", + "model.layers.21.mlp.experts.229.up_proj", + "model.layers.21.mlp.experts.23.down_proj", + "model.layers.21.mlp.experts.23.gate_proj", + "model.layers.21.mlp.experts.23.up_proj", + "model.layers.21.mlp.experts.230.down_proj", + "model.layers.21.mlp.experts.230.gate_proj", + "model.layers.21.mlp.experts.230.up_proj", + "model.layers.21.mlp.experts.231.down_proj", + "model.layers.21.mlp.experts.231.gate_proj", + "model.layers.21.mlp.experts.231.up_proj", + "model.layers.21.mlp.experts.232.down_proj", + "model.layers.21.mlp.experts.232.gate_proj", + "model.layers.21.mlp.experts.232.up_proj", + "model.layers.21.mlp.experts.233.down_proj", + "model.layers.21.mlp.experts.233.gate_proj", + "model.layers.21.mlp.experts.233.up_proj", + "model.layers.21.mlp.experts.234.down_proj", + "model.layers.21.mlp.experts.234.gate_proj", + "model.layers.21.mlp.experts.234.up_proj", + "model.layers.21.mlp.experts.235.down_proj", + "model.layers.21.mlp.experts.235.gate_proj", + "model.layers.21.mlp.experts.235.up_proj", + "model.layers.21.mlp.experts.236.down_proj", + "model.layers.21.mlp.experts.236.gate_proj", + "model.layers.21.mlp.experts.236.up_proj", + "model.layers.21.mlp.experts.237.down_proj", + "model.layers.21.mlp.experts.237.gate_proj", + "model.layers.21.mlp.experts.237.up_proj", + "model.layers.21.mlp.experts.238.down_proj", + "model.layers.21.mlp.experts.238.gate_proj", + "model.layers.21.mlp.experts.238.up_proj", + "model.layers.21.mlp.experts.239.down_proj", + "model.layers.21.mlp.experts.239.gate_proj", + "model.layers.21.mlp.experts.239.up_proj", + "model.layers.21.mlp.experts.24.down_proj", + "model.layers.21.mlp.experts.24.gate_proj", + "model.layers.21.mlp.experts.24.up_proj", + "model.layers.21.mlp.experts.240.down_proj", + "model.layers.21.mlp.experts.240.gate_proj", + "model.layers.21.mlp.experts.240.up_proj", + "model.layers.21.mlp.experts.241.down_proj", + "model.layers.21.mlp.experts.241.gate_proj", + "model.layers.21.mlp.experts.241.up_proj", + "model.layers.21.mlp.experts.242.down_proj", + "model.layers.21.mlp.experts.242.gate_proj", + "model.layers.21.mlp.experts.242.up_proj", + "model.layers.21.mlp.experts.243.down_proj", + "model.layers.21.mlp.experts.243.gate_proj", + "model.layers.21.mlp.experts.243.up_proj", + "model.layers.21.mlp.experts.244.down_proj", + "model.layers.21.mlp.experts.244.gate_proj", + "model.layers.21.mlp.experts.244.up_proj", + "model.layers.21.mlp.experts.245.down_proj", + "model.layers.21.mlp.experts.245.gate_proj", + "model.layers.21.mlp.experts.245.up_proj", + "model.layers.21.mlp.experts.246.down_proj", + "model.layers.21.mlp.experts.246.gate_proj", + "model.layers.21.mlp.experts.246.up_proj", + "model.layers.21.mlp.experts.247.down_proj", + "model.layers.21.mlp.experts.247.gate_proj", + "model.layers.21.mlp.experts.247.up_proj", + "model.layers.21.mlp.experts.248.down_proj", + "model.layers.21.mlp.experts.248.gate_proj", + "model.layers.21.mlp.experts.248.up_proj", + "model.layers.21.mlp.experts.249.down_proj", + "model.layers.21.mlp.experts.249.gate_proj", + "model.layers.21.mlp.experts.249.up_proj", + "model.layers.21.mlp.experts.25.down_proj", + "model.layers.21.mlp.experts.25.gate_proj", + "model.layers.21.mlp.experts.25.up_proj", + "model.layers.21.mlp.experts.250.down_proj", + "model.layers.21.mlp.experts.250.gate_proj", + "model.layers.21.mlp.experts.250.up_proj", + "model.layers.21.mlp.experts.251.down_proj", + "model.layers.21.mlp.experts.251.gate_proj", + "model.layers.21.mlp.experts.251.up_proj", + "model.layers.21.mlp.experts.252.down_proj", + "model.layers.21.mlp.experts.252.gate_proj", + "model.layers.21.mlp.experts.252.up_proj", + "model.layers.21.mlp.experts.253.down_proj", + "model.layers.21.mlp.experts.253.gate_proj", + "model.layers.21.mlp.experts.253.up_proj", + "model.layers.21.mlp.experts.254.down_proj", + "model.layers.21.mlp.experts.254.gate_proj", + "model.layers.21.mlp.experts.254.up_proj", + "model.layers.21.mlp.experts.255.down_proj", + "model.layers.21.mlp.experts.255.gate_proj", + "model.layers.21.mlp.experts.255.up_proj", + "model.layers.21.mlp.experts.26.down_proj", + "model.layers.21.mlp.experts.26.gate_proj", + "model.layers.21.mlp.experts.26.up_proj", + "model.layers.21.mlp.experts.27.down_proj", + "model.layers.21.mlp.experts.27.gate_proj", + "model.layers.21.mlp.experts.27.up_proj", + "model.layers.21.mlp.experts.28.down_proj", + "model.layers.21.mlp.experts.28.gate_proj", + "model.layers.21.mlp.experts.28.up_proj", + "model.layers.21.mlp.experts.29.down_proj", + "model.layers.21.mlp.experts.29.gate_proj", + "model.layers.21.mlp.experts.29.up_proj", + "model.layers.21.mlp.experts.3.down_proj", + "model.layers.21.mlp.experts.3.gate_proj", + "model.layers.21.mlp.experts.3.up_proj", + "model.layers.21.mlp.experts.30.down_proj", + "model.layers.21.mlp.experts.30.gate_proj", + "model.layers.21.mlp.experts.30.up_proj", + "model.layers.21.mlp.experts.31.down_proj", + "model.layers.21.mlp.experts.31.gate_proj", + "model.layers.21.mlp.experts.31.up_proj", + "model.layers.21.mlp.experts.32.down_proj", + "model.layers.21.mlp.experts.32.gate_proj", + "model.layers.21.mlp.experts.32.up_proj", + "model.layers.21.mlp.experts.33.down_proj", + "model.layers.21.mlp.experts.33.gate_proj", + "model.layers.21.mlp.experts.33.up_proj", + "model.layers.21.mlp.experts.34.down_proj", + "model.layers.21.mlp.experts.34.gate_proj", + "model.layers.21.mlp.experts.34.up_proj", + "model.layers.21.mlp.experts.35.down_proj", + "model.layers.21.mlp.experts.35.gate_proj", + "model.layers.21.mlp.experts.35.up_proj", + "model.layers.21.mlp.experts.36.down_proj", + "model.layers.21.mlp.experts.36.gate_proj", + "model.layers.21.mlp.experts.36.up_proj", + "model.layers.21.mlp.experts.37.down_proj", + "model.layers.21.mlp.experts.37.gate_proj", + "model.layers.21.mlp.experts.37.up_proj", + "model.layers.21.mlp.experts.38.down_proj", + "model.layers.21.mlp.experts.38.gate_proj", + "model.layers.21.mlp.experts.38.up_proj", + "model.layers.21.mlp.experts.39.down_proj", + "model.layers.21.mlp.experts.39.gate_proj", + "model.layers.21.mlp.experts.39.up_proj", + "model.layers.21.mlp.experts.4.down_proj", + "model.layers.21.mlp.experts.4.gate_proj", + "model.layers.21.mlp.experts.4.up_proj", + "model.layers.21.mlp.experts.40.down_proj", + "model.layers.21.mlp.experts.40.gate_proj", + "model.layers.21.mlp.experts.40.up_proj", + "model.layers.21.mlp.experts.41.down_proj", + "model.layers.21.mlp.experts.41.gate_proj", + "model.layers.21.mlp.experts.41.up_proj", + "model.layers.21.mlp.experts.42.down_proj", + "model.layers.21.mlp.experts.42.gate_proj", + "model.layers.21.mlp.experts.42.up_proj", + "model.layers.21.mlp.experts.43.down_proj", + "model.layers.21.mlp.experts.43.gate_proj", + "model.layers.21.mlp.experts.43.up_proj", + "model.layers.21.mlp.experts.44.down_proj", + "model.layers.21.mlp.experts.44.gate_proj", + "model.layers.21.mlp.experts.44.up_proj", + "model.layers.21.mlp.experts.45.down_proj", + "model.layers.21.mlp.experts.45.gate_proj", + "model.layers.21.mlp.experts.45.up_proj", + "model.layers.21.mlp.experts.46.down_proj", + "model.layers.21.mlp.experts.46.gate_proj", + "model.layers.21.mlp.experts.46.up_proj", + "model.layers.21.mlp.experts.47.down_proj", + "model.layers.21.mlp.experts.47.gate_proj", + "model.layers.21.mlp.experts.47.up_proj", + "model.layers.21.mlp.experts.48.down_proj", + "model.layers.21.mlp.experts.48.gate_proj", + "model.layers.21.mlp.experts.48.up_proj", + "model.layers.21.mlp.experts.49.down_proj", + "model.layers.21.mlp.experts.49.gate_proj", + "model.layers.21.mlp.experts.49.up_proj", + "model.layers.21.mlp.experts.5.down_proj", + "model.layers.21.mlp.experts.5.gate_proj", + "model.layers.21.mlp.experts.5.up_proj", + "model.layers.21.mlp.experts.50.down_proj", + "model.layers.21.mlp.experts.50.gate_proj", + "model.layers.21.mlp.experts.50.up_proj", + "model.layers.21.mlp.experts.51.down_proj", + "model.layers.21.mlp.experts.51.gate_proj", + "model.layers.21.mlp.experts.51.up_proj", + "model.layers.21.mlp.experts.52.down_proj", + "model.layers.21.mlp.experts.52.gate_proj", + "model.layers.21.mlp.experts.52.up_proj", + "model.layers.21.mlp.experts.53.down_proj", + "model.layers.21.mlp.experts.53.gate_proj", + "model.layers.21.mlp.experts.53.up_proj", + "model.layers.21.mlp.experts.54.down_proj", + "model.layers.21.mlp.experts.54.gate_proj", + "model.layers.21.mlp.experts.54.up_proj", + "model.layers.21.mlp.experts.55.down_proj", + "model.layers.21.mlp.experts.55.gate_proj", + "model.layers.21.mlp.experts.55.up_proj", + "model.layers.21.mlp.experts.56.down_proj", + "model.layers.21.mlp.experts.56.gate_proj", + "model.layers.21.mlp.experts.56.up_proj", + "model.layers.21.mlp.experts.57.down_proj", + "model.layers.21.mlp.experts.57.gate_proj", + "model.layers.21.mlp.experts.57.up_proj", + "model.layers.21.mlp.experts.58.down_proj", + "model.layers.21.mlp.experts.58.gate_proj", + "model.layers.21.mlp.experts.58.up_proj", + "model.layers.21.mlp.experts.59.down_proj", + "model.layers.21.mlp.experts.59.gate_proj", + "model.layers.21.mlp.experts.59.up_proj", + "model.layers.21.mlp.experts.6.down_proj", + "model.layers.21.mlp.experts.6.gate_proj", + "model.layers.21.mlp.experts.6.up_proj", + "model.layers.21.mlp.experts.60.down_proj", + "model.layers.21.mlp.experts.60.gate_proj", + "model.layers.21.mlp.experts.60.up_proj", + "model.layers.21.mlp.experts.61.down_proj", + "model.layers.21.mlp.experts.61.gate_proj", + "model.layers.21.mlp.experts.61.up_proj", + "model.layers.21.mlp.experts.62.down_proj", + "model.layers.21.mlp.experts.62.gate_proj", + "model.layers.21.mlp.experts.62.up_proj", + "model.layers.21.mlp.experts.63.down_proj", + "model.layers.21.mlp.experts.63.gate_proj", + "model.layers.21.mlp.experts.63.up_proj", + "model.layers.21.mlp.experts.64.down_proj", + "model.layers.21.mlp.experts.64.gate_proj", + "model.layers.21.mlp.experts.64.up_proj", + "model.layers.21.mlp.experts.65.down_proj", + "model.layers.21.mlp.experts.65.gate_proj", + "model.layers.21.mlp.experts.65.up_proj", + "model.layers.21.mlp.experts.66.down_proj", + "model.layers.21.mlp.experts.66.gate_proj", + "model.layers.21.mlp.experts.66.up_proj", + "model.layers.21.mlp.experts.67.down_proj", + "model.layers.21.mlp.experts.67.gate_proj", + "model.layers.21.mlp.experts.67.up_proj", + "model.layers.21.mlp.experts.68.down_proj", + "model.layers.21.mlp.experts.68.gate_proj", + "model.layers.21.mlp.experts.68.up_proj", + "model.layers.21.mlp.experts.69.down_proj", + "model.layers.21.mlp.experts.69.gate_proj", + "model.layers.21.mlp.experts.69.up_proj", + "model.layers.21.mlp.experts.7.down_proj", + "model.layers.21.mlp.experts.7.gate_proj", + "model.layers.21.mlp.experts.7.up_proj", + "model.layers.21.mlp.experts.70.down_proj", + "model.layers.21.mlp.experts.70.gate_proj", + "model.layers.21.mlp.experts.70.up_proj", + "model.layers.21.mlp.experts.71.down_proj", + "model.layers.21.mlp.experts.71.gate_proj", + "model.layers.21.mlp.experts.71.up_proj", + "model.layers.21.mlp.experts.72.down_proj", + "model.layers.21.mlp.experts.72.gate_proj", + "model.layers.21.mlp.experts.72.up_proj", + "model.layers.21.mlp.experts.73.down_proj", + "model.layers.21.mlp.experts.73.gate_proj", + "model.layers.21.mlp.experts.73.up_proj", + "model.layers.21.mlp.experts.74.down_proj", + "model.layers.21.mlp.experts.74.gate_proj", + "model.layers.21.mlp.experts.74.up_proj", + "model.layers.21.mlp.experts.75.down_proj", + "model.layers.21.mlp.experts.75.gate_proj", + "model.layers.21.mlp.experts.75.up_proj", + "model.layers.21.mlp.experts.76.down_proj", + "model.layers.21.mlp.experts.76.gate_proj", + "model.layers.21.mlp.experts.76.up_proj", + "model.layers.21.mlp.experts.77.down_proj", + "model.layers.21.mlp.experts.77.gate_proj", + "model.layers.21.mlp.experts.77.up_proj", + "model.layers.21.mlp.experts.78.down_proj", + "model.layers.21.mlp.experts.78.gate_proj", + "model.layers.21.mlp.experts.78.up_proj", + "model.layers.21.mlp.experts.79.down_proj", + "model.layers.21.mlp.experts.79.gate_proj", + "model.layers.21.mlp.experts.79.up_proj", + "model.layers.21.mlp.experts.8.down_proj", + "model.layers.21.mlp.experts.8.gate_proj", + "model.layers.21.mlp.experts.8.up_proj", + "model.layers.21.mlp.experts.80.down_proj", + "model.layers.21.mlp.experts.80.gate_proj", + "model.layers.21.mlp.experts.80.up_proj", + "model.layers.21.mlp.experts.81.down_proj", + "model.layers.21.mlp.experts.81.gate_proj", + "model.layers.21.mlp.experts.81.up_proj", + "model.layers.21.mlp.experts.82.down_proj", + "model.layers.21.mlp.experts.82.gate_proj", + "model.layers.21.mlp.experts.82.up_proj", + "model.layers.21.mlp.experts.83.down_proj", + "model.layers.21.mlp.experts.83.gate_proj", + "model.layers.21.mlp.experts.83.up_proj", + "model.layers.21.mlp.experts.84.down_proj", + "model.layers.21.mlp.experts.84.gate_proj", + "model.layers.21.mlp.experts.84.up_proj", + "model.layers.21.mlp.experts.85.down_proj", + "model.layers.21.mlp.experts.85.gate_proj", + "model.layers.21.mlp.experts.85.up_proj", + "model.layers.21.mlp.experts.86.down_proj", + "model.layers.21.mlp.experts.86.gate_proj", + "model.layers.21.mlp.experts.86.up_proj", + "model.layers.21.mlp.experts.87.down_proj", + "model.layers.21.mlp.experts.87.gate_proj", + "model.layers.21.mlp.experts.87.up_proj", + "model.layers.21.mlp.experts.88.down_proj", + "model.layers.21.mlp.experts.88.gate_proj", + "model.layers.21.mlp.experts.88.up_proj", + "model.layers.21.mlp.experts.89.down_proj", + "model.layers.21.mlp.experts.89.gate_proj", + "model.layers.21.mlp.experts.89.up_proj", + "model.layers.21.mlp.experts.9.down_proj", + "model.layers.21.mlp.experts.9.gate_proj", + "model.layers.21.mlp.experts.9.up_proj", + "model.layers.21.mlp.experts.90.down_proj", + "model.layers.21.mlp.experts.90.gate_proj", + "model.layers.21.mlp.experts.90.up_proj", + "model.layers.21.mlp.experts.91.down_proj", + "model.layers.21.mlp.experts.91.gate_proj", + "model.layers.21.mlp.experts.91.up_proj", + "model.layers.21.mlp.experts.92.down_proj", + "model.layers.21.mlp.experts.92.gate_proj", + "model.layers.21.mlp.experts.92.up_proj", + "model.layers.21.mlp.experts.93.down_proj", + "model.layers.21.mlp.experts.93.gate_proj", + "model.layers.21.mlp.experts.93.up_proj", + "model.layers.21.mlp.experts.94.down_proj", + "model.layers.21.mlp.experts.94.gate_proj", + "model.layers.21.mlp.experts.94.up_proj", + "model.layers.21.mlp.experts.95.down_proj", + "model.layers.21.mlp.experts.95.gate_proj", + "model.layers.21.mlp.experts.95.up_proj", + "model.layers.21.mlp.experts.96.down_proj", + "model.layers.21.mlp.experts.96.gate_proj", + "model.layers.21.mlp.experts.96.up_proj", + "model.layers.21.mlp.experts.97.down_proj", + "model.layers.21.mlp.experts.97.gate_proj", + "model.layers.21.mlp.experts.97.up_proj", + "model.layers.21.mlp.experts.98.down_proj", + "model.layers.21.mlp.experts.98.gate_proj", + "model.layers.21.mlp.experts.98.up_proj", + "model.layers.21.mlp.experts.99.down_proj", + "model.layers.21.mlp.experts.99.gate_proj", + "model.layers.21.mlp.experts.99.up_proj", + "model.layers.22.mlp.experts.0.down_proj", + "model.layers.22.mlp.experts.0.gate_proj", + "model.layers.22.mlp.experts.0.up_proj", + "model.layers.22.mlp.experts.1.down_proj", + "model.layers.22.mlp.experts.1.gate_proj", + "model.layers.22.mlp.experts.1.up_proj", + "model.layers.22.mlp.experts.10.down_proj", + "model.layers.22.mlp.experts.10.gate_proj", + "model.layers.22.mlp.experts.10.up_proj", + "model.layers.22.mlp.experts.100.down_proj", + "model.layers.22.mlp.experts.100.gate_proj", + "model.layers.22.mlp.experts.100.up_proj", + "model.layers.22.mlp.experts.101.down_proj", + "model.layers.22.mlp.experts.101.gate_proj", + "model.layers.22.mlp.experts.101.up_proj", + "model.layers.22.mlp.experts.102.down_proj", + "model.layers.22.mlp.experts.102.gate_proj", + "model.layers.22.mlp.experts.102.up_proj", + "model.layers.22.mlp.experts.103.down_proj", + "model.layers.22.mlp.experts.103.gate_proj", + "model.layers.22.mlp.experts.103.up_proj", + "model.layers.22.mlp.experts.104.down_proj", + "model.layers.22.mlp.experts.104.gate_proj", + "model.layers.22.mlp.experts.104.up_proj", + "model.layers.22.mlp.experts.105.down_proj", + "model.layers.22.mlp.experts.105.gate_proj", + "model.layers.22.mlp.experts.105.up_proj", + "model.layers.22.mlp.experts.106.down_proj", + "model.layers.22.mlp.experts.106.gate_proj", + "model.layers.22.mlp.experts.106.up_proj", + "model.layers.22.mlp.experts.107.down_proj", + "model.layers.22.mlp.experts.107.gate_proj", + "model.layers.22.mlp.experts.107.up_proj", + "model.layers.22.mlp.experts.108.down_proj", + "model.layers.22.mlp.experts.108.gate_proj", + "model.layers.22.mlp.experts.108.up_proj", + "model.layers.22.mlp.experts.109.down_proj", + "model.layers.22.mlp.experts.109.gate_proj", + "model.layers.22.mlp.experts.109.up_proj", + "model.layers.22.mlp.experts.11.down_proj", + "model.layers.22.mlp.experts.11.gate_proj", + "model.layers.22.mlp.experts.11.up_proj", + "model.layers.22.mlp.experts.110.down_proj", + "model.layers.22.mlp.experts.110.gate_proj", + "model.layers.22.mlp.experts.110.up_proj", + "model.layers.22.mlp.experts.111.down_proj", + "model.layers.22.mlp.experts.111.gate_proj", + "model.layers.22.mlp.experts.111.up_proj", + "model.layers.22.mlp.experts.112.down_proj", + "model.layers.22.mlp.experts.112.gate_proj", + "model.layers.22.mlp.experts.112.up_proj", + "model.layers.22.mlp.experts.113.down_proj", + "model.layers.22.mlp.experts.113.gate_proj", + "model.layers.22.mlp.experts.113.up_proj", + "model.layers.22.mlp.experts.114.down_proj", + "model.layers.22.mlp.experts.114.gate_proj", + "model.layers.22.mlp.experts.114.up_proj", + "model.layers.22.mlp.experts.115.down_proj", + "model.layers.22.mlp.experts.115.gate_proj", + "model.layers.22.mlp.experts.115.up_proj", + "model.layers.22.mlp.experts.116.down_proj", + "model.layers.22.mlp.experts.116.gate_proj", + "model.layers.22.mlp.experts.116.up_proj", + "model.layers.22.mlp.experts.117.down_proj", + "model.layers.22.mlp.experts.117.gate_proj", + "model.layers.22.mlp.experts.117.up_proj", + "model.layers.22.mlp.experts.118.down_proj", + "model.layers.22.mlp.experts.118.gate_proj", + "model.layers.22.mlp.experts.118.up_proj", + "model.layers.22.mlp.experts.119.down_proj", + "model.layers.22.mlp.experts.119.gate_proj", + "model.layers.22.mlp.experts.119.up_proj", + "model.layers.22.mlp.experts.12.down_proj", + "model.layers.22.mlp.experts.12.gate_proj", + "model.layers.22.mlp.experts.12.up_proj", + "model.layers.22.mlp.experts.120.down_proj", + "model.layers.22.mlp.experts.120.gate_proj", + "model.layers.22.mlp.experts.120.up_proj", + "model.layers.22.mlp.experts.121.down_proj", + "model.layers.22.mlp.experts.121.gate_proj", + "model.layers.22.mlp.experts.121.up_proj", + "model.layers.22.mlp.experts.122.down_proj", + "model.layers.22.mlp.experts.122.gate_proj", + "model.layers.22.mlp.experts.122.up_proj", + "model.layers.22.mlp.experts.123.down_proj", + "model.layers.22.mlp.experts.123.gate_proj", + "model.layers.22.mlp.experts.123.up_proj", + "model.layers.22.mlp.experts.124.down_proj", + "model.layers.22.mlp.experts.124.gate_proj", + "model.layers.22.mlp.experts.124.up_proj", + "model.layers.22.mlp.experts.125.down_proj", + "model.layers.22.mlp.experts.125.gate_proj", + "model.layers.22.mlp.experts.125.up_proj", + "model.layers.22.mlp.experts.126.down_proj", + "model.layers.22.mlp.experts.126.gate_proj", + "model.layers.22.mlp.experts.126.up_proj", + "model.layers.22.mlp.experts.127.down_proj", + "model.layers.22.mlp.experts.127.gate_proj", + "model.layers.22.mlp.experts.127.up_proj", + "model.layers.22.mlp.experts.128.down_proj", + "model.layers.22.mlp.experts.128.gate_proj", + "model.layers.22.mlp.experts.128.up_proj", + "model.layers.22.mlp.experts.129.down_proj", + "model.layers.22.mlp.experts.129.gate_proj", + "model.layers.22.mlp.experts.129.up_proj", + "model.layers.22.mlp.experts.13.down_proj", + "model.layers.22.mlp.experts.13.gate_proj", + "model.layers.22.mlp.experts.13.up_proj", + "model.layers.22.mlp.experts.130.down_proj", + "model.layers.22.mlp.experts.130.gate_proj", + "model.layers.22.mlp.experts.130.up_proj", + "model.layers.22.mlp.experts.131.down_proj", + "model.layers.22.mlp.experts.131.gate_proj", + "model.layers.22.mlp.experts.131.up_proj", + "model.layers.22.mlp.experts.132.down_proj", + "model.layers.22.mlp.experts.132.gate_proj", + "model.layers.22.mlp.experts.132.up_proj", + "model.layers.22.mlp.experts.133.down_proj", + "model.layers.22.mlp.experts.133.gate_proj", + "model.layers.22.mlp.experts.133.up_proj", + "model.layers.22.mlp.experts.134.down_proj", + "model.layers.22.mlp.experts.134.gate_proj", + "model.layers.22.mlp.experts.134.up_proj", + "model.layers.22.mlp.experts.135.down_proj", + "model.layers.22.mlp.experts.135.gate_proj", + "model.layers.22.mlp.experts.135.up_proj", + "model.layers.22.mlp.experts.136.down_proj", + "model.layers.22.mlp.experts.136.gate_proj", + "model.layers.22.mlp.experts.136.up_proj", + "model.layers.22.mlp.experts.137.down_proj", + "model.layers.22.mlp.experts.137.gate_proj", + "model.layers.22.mlp.experts.137.up_proj", + "model.layers.22.mlp.experts.138.down_proj", + "model.layers.22.mlp.experts.138.gate_proj", + "model.layers.22.mlp.experts.138.up_proj", + "model.layers.22.mlp.experts.139.down_proj", + "model.layers.22.mlp.experts.139.gate_proj", + "model.layers.22.mlp.experts.139.up_proj", + "model.layers.22.mlp.experts.14.down_proj", + "model.layers.22.mlp.experts.14.gate_proj", + "model.layers.22.mlp.experts.14.up_proj", + "model.layers.22.mlp.experts.140.down_proj", + "model.layers.22.mlp.experts.140.gate_proj", + "model.layers.22.mlp.experts.140.up_proj", + "model.layers.22.mlp.experts.141.down_proj", + "model.layers.22.mlp.experts.141.gate_proj", + "model.layers.22.mlp.experts.141.up_proj", + "model.layers.22.mlp.experts.142.down_proj", + "model.layers.22.mlp.experts.142.gate_proj", + "model.layers.22.mlp.experts.142.up_proj", + "model.layers.22.mlp.experts.143.down_proj", + "model.layers.22.mlp.experts.143.gate_proj", + "model.layers.22.mlp.experts.143.up_proj", + "model.layers.22.mlp.experts.144.down_proj", + "model.layers.22.mlp.experts.144.gate_proj", + "model.layers.22.mlp.experts.144.up_proj", + "model.layers.22.mlp.experts.145.down_proj", + "model.layers.22.mlp.experts.145.gate_proj", + "model.layers.22.mlp.experts.145.up_proj", + "model.layers.22.mlp.experts.146.down_proj", + "model.layers.22.mlp.experts.146.gate_proj", + "model.layers.22.mlp.experts.146.up_proj", + "model.layers.22.mlp.experts.147.down_proj", + "model.layers.22.mlp.experts.147.gate_proj", + "model.layers.22.mlp.experts.147.up_proj", + "model.layers.22.mlp.experts.148.down_proj", + "model.layers.22.mlp.experts.148.gate_proj", + "model.layers.22.mlp.experts.148.up_proj", + "model.layers.22.mlp.experts.149.down_proj", + "model.layers.22.mlp.experts.149.gate_proj", + "model.layers.22.mlp.experts.149.up_proj", + "model.layers.22.mlp.experts.15.down_proj", + "model.layers.22.mlp.experts.15.gate_proj", + "model.layers.22.mlp.experts.15.up_proj", + "model.layers.22.mlp.experts.150.down_proj", + "model.layers.22.mlp.experts.150.gate_proj", + "model.layers.22.mlp.experts.150.up_proj", + "model.layers.22.mlp.experts.151.down_proj", + "model.layers.22.mlp.experts.151.gate_proj", + "model.layers.22.mlp.experts.151.up_proj", + "model.layers.22.mlp.experts.152.down_proj", + "model.layers.22.mlp.experts.152.gate_proj", + "model.layers.22.mlp.experts.152.up_proj", + "model.layers.22.mlp.experts.153.down_proj", + "model.layers.22.mlp.experts.153.gate_proj", + "model.layers.22.mlp.experts.153.up_proj", + "model.layers.22.mlp.experts.154.down_proj", + "model.layers.22.mlp.experts.154.gate_proj", + "model.layers.22.mlp.experts.154.up_proj", + "model.layers.22.mlp.experts.155.down_proj", + "model.layers.22.mlp.experts.155.gate_proj", + "model.layers.22.mlp.experts.155.up_proj", + "model.layers.22.mlp.experts.156.down_proj", + "model.layers.22.mlp.experts.156.gate_proj", + "model.layers.22.mlp.experts.156.up_proj", + "model.layers.22.mlp.experts.157.down_proj", + "model.layers.22.mlp.experts.157.gate_proj", + "model.layers.22.mlp.experts.157.up_proj", + "model.layers.22.mlp.experts.158.down_proj", + "model.layers.22.mlp.experts.158.gate_proj", + "model.layers.22.mlp.experts.158.up_proj", + "model.layers.22.mlp.experts.159.down_proj", + "model.layers.22.mlp.experts.159.gate_proj", + "model.layers.22.mlp.experts.159.up_proj", + "model.layers.22.mlp.experts.16.down_proj", + "model.layers.22.mlp.experts.16.gate_proj", + "model.layers.22.mlp.experts.16.up_proj", + "model.layers.22.mlp.experts.160.down_proj", + "model.layers.22.mlp.experts.160.gate_proj", + "model.layers.22.mlp.experts.160.up_proj", + "model.layers.22.mlp.experts.161.down_proj", + "model.layers.22.mlp.experts.161.gate_proj", + "model.layers.22.mlp.experts.161.up_proj", + "model.layers.22.mlp.experts.162.down_proj", + "model.layers.22.mlp.experts.162.gate_proj", + "model.layers.22.mlp.experts.162.up_proj", + "model.layers.22.mlp.experts.163.down_proj", + "model.layers.22.mlp.experts.163.gate_proj", + "model.layers.22.mlp.experts.163.up_proj", + "model.layers.22.mlp.experts.164.down_proj", + "model.layers.22.mlp.experts.164.gate_proj", + "model.layers.22.mlp.experts.164.up_proj", + "model.layers.22.mlp.experts.165.down_proj", + "model.layers.22.mlp.experts.165.gate_proj", + "model.layers.22.mlp.experts.165.up_proj", + "model.layers.22.mlp.experts.166.down_proj", + "model.layers.22.mlp.experts.166.gate_proj", + "model.layers.22.mlp.experts.166.up_proj", + "model.layers.22.mlp.experts.167.down_proj", + "model.layers.22.mlp.experts.167.gate_proj", + "model.layers.22.mlp.experts.167.up_proj", + "model.layers.22.mlp.experts.168.down_proj", + "model.layers.22.mlp.experts.168.gate_proj", + "model.layers.22.mlp.experts.168.up_proj", + "model.layers.22.mlp.experts.169.down_proj", + "model.layers.22.mlp.experts.169.gate_proj", + "model.layers.22.mlp.experts.169.up_proj", + "model.layers.22.mlp.experts.17.down_proj", + "model.layers.22.mlp.experts.17.gate_proj", + "model.layers.22.mlp.experts.17.up_proj", + "model.layers.22.mlp.experts.170.down_proj", + "model.layers.22.mlp.experts.170.gate_proj", + "model.layers.22.mlp.experts.170.up_proj", + "model.layers.22.mlp.experts.171.down_proj", + "model.layers.22.mlp.experts.171.gate_proj", + "model.layers.22.mlp.experts.171.up_proj", + "model.layers.22.mlp.experts.172.down_proj", + "model.layers.22.mlp.experts.172.gate_proj", + "model.layers.22.mlp.experts.172.up_proj", + "model.layers.22.mlp.experts.173.down_proj", + "model.layers.22.mlp.experts.173.gate_proj", + "model.layers.22.mlp.experts.173.up_proj", + "model.layers.22.mlp.experts.174.down_proj", + "model.layers.22.mlp.experts.174.gate_proj", + "model.layers.22.mlp.experts.174.up_proj", + "model.layers.22.mlp.experts.175.down_proj", + "model.layers.22.mlp.experts.175.gate_proj", + "model.layers.22.mlp.experts.175.up_proj", + "model.layers.22.mlp.experts.176.down_proj", + "model.layers.22.mlp.experts.176.gate_proj", + "model.layers.22.mlp.experts.176.up_proj", + "model.layers.22.mlp.experts.177.down_proj", + "model.layers.22.mlp.experts.177.gate_proj", + "model.layers.22.mlp.experts.177.up_proj", + "model.layers.22.mlp.experts.178.down_proj", + "model.layers.22.mlp.experts.178.gate_proj", + "model.layers.22.mlp.experts.178.up_proj", + "model.layers.22.mlp.experts.179.down_proj", + "model.layers.22.mlp.experts.179.gate_proj", + "model.layers.22.mlp.experts.179.up_proj", + "model.layers.22.mlp.experts.18.down_proj", + "model.layers.22.mlp.experts.18.gate_proj", + "model.layers.22.mlp.experts.18.up_proj", + "model.layers.22.mlp.experts.180.down_proj", + "model.layers.22.mlp.experts.180.gate_proj", + "model.layers.22.mlp.experts.180.up_proj", + "model.layers.22.mlp.experts.181.down_proj", + "model.layers.22.mlp.experts.181.gate_proj", + "model.layers.22.mlp.experts.181.up_proj", + "model.layers.22.mlp.experts.182.down_proj", + "model.layers.22.mlp.experts.182.gate_proj", + "model.layers.22.mlp.experts.182.up_proj", + "model.layers.22.mlp.experts.183.down_proj", + "model.layers.22.mlp.experts.183.gate_proj", + "model.layers.22.mlp.experts.183.up_proj", + "model.layers.22.mlp.experts.184.down_proj", + "model.layers.22.mlp.experts.184.gate_proj", + "model.layers.22.mlp.experts.184.up_proj", + "model.layers.22.mlp.experts.185.down_proj", + "model.layers.22.mlp.experts.185.gate_proj", + "model.layers.22.mlp.experts.185.up_proj", + "model.layers.22.mlp.experts.186.down_proj", + "model.layers.22.mlp.experts.186.gate_proj", + "model.layers.22.mlp.experts.186.up_proj", + "model.layers.22.mlp.experts.187.down_proj", + "model.layers.22.mlp.experts.187.gate_proj", + "model.layers.22.mlp.experts.187.up_proj", + "model.layers.22.mlp.experts.188.down_proj", + "model.layers.22.mlp.experts.188.gate_proj", + "model.layers.22.mlp.experts.188.up_proj", + "model.layers.22.mlp.experts.189.down_proj", + "model.layers.22.mlp.experts.189.gate_proj", + "model.layers.22.mlp.experts.189.up_proj", + "model.layers.22.mlp.experts.19.down_proj", + "model.layers.22.mlp.experts.19.gate_proj", + "model.layers.22.mlp.experts.19.up_proj", + "model.layers.22.mlp.experts.190.down_proj", + "model.layers.22.mlp.experts.190.gate_proj", + "model.layers.22.mlp.experts.190.up_proj", + "model.layers.22.mlp.experts.191.down_proj", + "model.layers.22.mlp.experts.191.gate_proj", + "model.layers.22.mlp.experts.191.up_proj", + "model.layers.22.mlp.experts.192.down_proj", + "model.layers.22.mlp.experts.192.gate_proj", + "model.layers.22.mlp.experts.192.up_proj", + "model.layers.22.mlp.experts.193.down_proj", + "model.layers.22.mlp.experts.193.gate_proj", + "model.layers.22.mlp.experts.193.up_proj", + "model.layers.22.mlp.experts.194.down_proj", + "model.layers.22.mlp.experts.194.gate_proj", + "model.layers.22.mlp.experts.194.up_proj", + "model.layers.22.mlp.experts.195.down_proj", + "model.layers.22.mlp.experts.195.gate_proj", + "model.layers.22.mlp.experts.195.up_proj", + "model.layers.22.mlp.experts.196.down_proj", + "model.layers.22.mlp.experts.196.gate_proj", + "model.layers.22.mlp.experts.196.up_proj", + "model.layers.22.mlp.experts.197.down_proj", + "model.layers.22.mlp.experts.197.gate_proj", + "model.layers.22.mlp.experts.197.up_proj", + "model.layers.22.mlp.experts.198.down_proj", + "model.layers.22.mlp.experts.198.gate_proj", + "model.layers.22.mlp.experts.198.up_proj", + "model.layers.22.mlp.experts.199.down_proj", + "model.layers.22.mlp.experts.199.gate_proj", + "model.layers.22.mlp.experts.199.up_proj", + "model.layers.22.mlp.experts.2.down_proj", + "model.layers.22.mlp.experts.2.gate_proj", + "model.layers.22.mlp.experts.2.up_proj", + "model.layers.22.mlp.experts.20.down_proj", + "model.layers.22.mlp.experts.20.gate_proj", + "model.layers.22.mlp.experts.20.up_proj", + "model.layers.22.mlp.experts.200.down_proj", + "model.layers.22.mlp.experts.200.gate_proj", + "model.layers.22.mlp.experts.200.up_proj", + "model.layers.22.mlp.experts.201.down_proj", + "model.layers.22.mlp.experts.201.gate_proj", + "model.layers.22.mlp.experts.201.up_proj", + "model.layers.22.mlp.experts.202.down_proj", + "model.layers.22.mlp.experts.202.gate_proj", + "model.layers.22.mlp.experts.202.up_proj", + "model.layers.22.mlp.experts.203.down_proj", + "model.layers.22.mlp.experts.203.gate_proj", + "model.layers.22.mlp.experts.203.up_proj", + "model.layers.22.mlp.experts.204.down_proj", + "model.layers.22.mlp.experts.204.gate_proj", + "model.layers.22.mlp.experts.204.up_proj", + "model.layers.22.mlp.experts.205.down_proj", + "model.layers.22.mlp.experts.205.gate_proj", + "model.layers.22.mlp.experts.205.up_proj", + "model.layers.22.mlp.experts.206.down_proj", + "model.layers.22.mlp.experts.206.gate_proj", + "model.layers.22.mlp.experts.206.up_proj", + "model.layers.22.mlp.experts.207.down_proj", + "model.layers.22.mlp.experts.207.gate_proj", + "model.layers.22.mlp.experts.207.up_proj", + "model.layers.22.mlp.experts.208.down_proj", + "model.layers.22.mlp.experts.208.gate_proj", + "model.layers.22.mlp.experts.208.up_proj", + "model.layers.22.mlp.experts.209.down_proj", + "model.layers.22.mlp.experts.209.gate_proj", + "model.layers.22.mlp.experts.209.up_proj", + "model.layers.22.mlp.experts.21.down_proj", + "model.layers.22.mlp.experts.21.gate_proj", + "model.layers.22.mlp.experts.21.up_proj", + "model.layers.22.mlp.experts.210.down_proj", + "model.layers.22.mlp.experts.210.gate_proj", + "model.layers.22.mlp.experts.210.up_proj", + "model.layers.22.mlp.experts.211.down_proj", + "model.layers.22.mlp.experts.211.gate_proj", + "model.layers.22.mlp.experts.211.up_proj", + "model.layers.22.mlp.experts.212.down_proj", + "model.layers.22.mlp.experts.212.gate_proj", + "model.layers.22.mlp.experts.212.up_proj", + "model.layers.22.mlp.experts.213.down_proj", + "model.layers.22.mlp.experts.213.gate_proj", + "model.layers.22.mlp.experts.213.up_proj", + "model.layers.22.mlp.experts.214.down_proj", + "model.layers.22.mlp.experts.214.gate_proj", + "model.layers.22.mlp.experts.214.up_proj", + "model.layers.22.mlp.experts.215.down_proj", + "model.layers.22.mlp.experts.215.gate_proj", + "model.layers.22.mlp.experts.215.up_proj", + "model.layers.22.mlp.experts.216.down_proj", + "model.layers.22.mlp.experts.216.gate_proj", + "model.layers.22.mlp.experts.216.up_proj", + "model.layers.22.mlp.experts.217.down_proj", + "model.layers.22.mlp.experts.217.gate_proj", + "model.layers.22.mlp.experts.217.up_proj", + "model.layers.22.mlp.experts.218.down_proj", + "model.layers.22.mlp.experts.218.gate_proj", + "model.layers.22.mlp.experts.218.up_proj", + "model.layers.22.mlp.experts.219.down_proj", + "model.layers.22.mlp.experts.219.gate_proj", + "model.layers.22.mlp.experts.219.up_proj", + "model.layers.22.mlp.experts.22.down_proj", + "model.layers.22.mlp.experts.22.gate_proj", + "model.layers.22.mlp.experts.22.up_proj", + "model.layers.22.mlp.experts.220.down_proj", + "model.layers.22.mlp.experts.220.gate_proj", + "model.layers.22.mlp.experts.220.up_proj", + "model.layers.22.mlp.experts.221.down_proj", + "model.layers.22.mlp.experts.221.gate_proj", + "model.layers.22.mlp.experts.221.up_proj", + "model.layers.22.mlp.experts.222.down_proj", + "model.layers.22.mlp.experts.222.gate_proj", + "model.layers.22.mlp.experts.222.up_proj", + "model.layers.22.mlp.experts.223.down_proj", + "model.layers.22.mlp.experts.223.gate_proj", + "model.layers.22.mlp.experts.223.up_proj", + "model.layers.22.mlp.experts.224.down_proj", + "model.layers.22.mlp.experts.224.gate_proj", + "model.layers.22.mlp.experts.224.up_proj", + "model.layers.22.mlp.experts.225.down_proj", + "model.layers.22.mlp.experts.225.gate_proj", + "model.layers.22.mlp.experts.225.up_proj", + "model.layers.22.mlp.experts.226.down_proj", + "model.layers.22.mlp.experts.226.gate_proj", + "model.layers.22.mlp.experts.226.up_proj", + "model.layers.22.mlp.experts.227.down_proj", + "model.layers.22.mlp.experts.227.gate_proj", + "model.layers.22.mlp.experts.227.up_proj", + "model.layers.22.mlp.experts.228.down_proj", + "model.layers.22.mlp.experts.228.gate_proj", + "model.layers.22.mlp.experts.228.up_proj", + "model.layers.22.mlp.experts.229.down_proj", + "model.layers.22.mlp.experts.229.gate_proj", + "model.layers.22.mlp.experts.229.up_proj", + "model.layers.22.mlp.experts.23.down_proj", + "model.layers.22.mlp.experts.23.gate_proj", + "model.layers.22.mlp.experts.23.up_proj", + "model.layers.22.mlp.experts.230.down_proj", + "model.layers.22.mlp.experts.230.gate_proj", + "model.layers.22.mlp.experts.230.up_proj", + "model.layers.22.mlp.experts.231.down_proj", + "model.layers.22.mlp.experts.231.gate_proj", + "model.layers.22.mlp.experts.231.up_proj", + "model.layers.22.mlp.experts.232.down_proj", + "model.layers.22.mlp.experts.232.gate_proj", + "model.layers.22.mlp.experts.232.up_proj", + "model.layers.22.mlp.experts.233.down_proj", + "model.layers.22.mlp.experts.233.gate_proj", + "model.layers.22.mlp.experts.233.up_proj", + "model.layers.22.mlp.experts.234.down_proj", + "model.layers.22.mlp.experts.234.gate_proj", + "model.layers.22.mlp.experts.234.up_proj", + "model.layers.22.mlp.experts.235.down_proj", + "model.layers.22.mlp.experts.235.gate_proj", + "model.layers.22.mlp.experts.235.up_proj", + "model.layers.22.mlp.experts.236.down_proj", + "model.layers.22.mlp.experts.236.gate_proj", + "model.layers.22.mlp.experts.236.up_proj", + "model.layers.22.mlp.experts.237.down_proj", + "model.layers.22.mlp.experts.237.gate_proj", + "model.layers.22.mlp.experts.237.up_proj", + "model.layers.22.mlp.experts.238.down_proj", + "model.layers.22.mlp.experts.238.gate_proj", + "model.layers.22.mlp.experts.238.up_proj", + "model.layers.22.mlp.experts.239.down_proj", + "model.layers.22.mlp.experts.239.gate_proj", + "model.layers.22.mlp.experts.239.up_proj", + "model.layers.22.mlp.experts.24.down_proj", + "model.layers.22.mlp.experts.24.gate_proj", + "model.layers.22.mlp.experts.24.up_proj", + "model.layers.22.mlp.experts.240.down_proj", + "model.layers.22.mlp.experts.240.gate_proj", + "model.layers.22.mlp.experts.240.up_proj", + "model.layers.22.mlp.experts.241.down_proj", + "model.layers.22.mlp.experts.241.gate_proj", + "model.layers.22.mlp.experts.241.up_proj", + "model.layers.22.mlp.experts.242.down_proj", + "model.layers.22.mlp.experts.242.gate_proj", + "model.layers.22.mlp.experts.242.up_proj", + "model.layers.22.mlp.experts.243.down_proj", + "model.layers.22.mlp.experts.243.gate_proj", + "model.layers.22.mlp.experts.243.up_proj", + "model.layers.22.mlp.experts.244.down_proj", + "model.layers.22.mlp.experts.244.gate_proj", + "model.layers.22.mlp.experts.244.up_proj", + "model.layers.22.mlp.experts.245.down_proj", + "model.layers.22.mlp.experts.245.gate_proj", + "model.layers.22.mlp.experts.245.up_proj", + "model.layers.22.mlp.experts.246.down_proj", + "model.layers.22.mlp.experts.246.gate_proj", + "model.layers.22.mlp.experts.246.up_proj", + "model.layers.22.mlp.experts.247.down_proj", + "model.layers.22.mlp.experts.247.gate_proj", + "model.layers.22.mlp.experts.247.up_proj", + "model.layers.22.mlp.experts.248.down_proj", + "model.layers.22.mlp.experts.248.gate_proj", + "model.layers.22.mlp.experts.248.up_proj", + "model.layers.22.mlp.experts.249.down_proj", + "model.layers.22.mlp.experts.249.gate_proj", + "model.layers.22.mlp.experts.249.up_proj", + "model.layers.22.mlp.experts.25.down_proj", + "model.layers.22.mlp.experts.25.gate_proj", + "model.layers.22.mlp.experts.25.up_proj", + "model.layers.22.mlp.experts.250.down_proj", + "model.layers.22.mlp.experts.250.gate_proj", + "model.layers.22.mlp.experts.250.up_proj", + "model.layers.22.mlp.experts.251.down_proj", + "model.layers.22.mlp.experts.251.gate_proj", + "model.layers.22.mlp.experts.251.up_proj", + "model.layers.22.mlp.experts.252.down_proj", + "model.layers.22.mlp.experts.252.gate_proj", + "model.layers.22.mlp.experts.252.up_proj", + "model.layers.22.mlp.experts.253.down_proj", + "model.layers.22.mlp.experts.253.gate_proj", + "model.layers.22.mlp.experts.253.up_proj", + "model.layers.22.mlp.experts.254.down_proj", + "model.layers.22.mlp.experts.254.gate_proj", + "model.layers.22.mlp.experts.254.up_proj", + "model.layers.22.mlp.experts.255.down_proj", + "model.layers.22.mlp.experts.255.gate_proj", + "model.layers.22.mlp.experts.255.up_proj", + "model.layers.22.mlp.experts.26.down_proj", + "model.layers.22.mlp.experts.26.gate_proj", + "model.layers.22.mlp.experts.26.up_proj", + "model.layers.22.mlp.experts.27.down_proj", + "model.layers.22.mlp.experts.27.gate_proj", + "model.layers.22.mlp.experts.27.up_proj", + "model.layers.22.mlp.experts.28.down_proj", + "model.layers.22.mlp.experts.28.gate_proj", + "model.layers.22.mlp.experts.28.up_proj", + "model.layers.22.mlp.experts.29.down_proj", + "model.layers.22.mlp.experts.29.gate_proj", + "model.layers.22.mlp.experts.29.up_proj", + "model.layers.22.mlp.experts.3.down_proj", + "model.layers.22.mlp.experts.3.gate_proj", + "model.layers.22.mlp.experts.3.up_proj", + "model.layers.22.mlp.experts.30.down_proj", + "model.layers.22.mlp.experts.30.gate_proj", + "model.layers.22.mlp.experts.30.up_proj", + "model.layers.22.mlp.experts.31.down_proj", + "model.layers.22.mlp.experts.31.gate_proj", + "model.layers.22.mlp.experts.31.up_proj", + "model.layers.22.mlp.experts.32.down_proj", + "model.layers.22.mlp.experts.32.gate_proj", + "model.layers.22.mlp.experts.32.up_proj", + "model.layers.22.mlp.experts.33.down_proj", + "model.layers.22.mlp.experts.33.gate_proj", + "model.layers.22.mlp.experts.33.up_proj", + "model.layers.22.mlp.experts.34.down_proj", + "model.layers.22.mlp.experts.34.gate_proj", + "model.layers.22.mlp.experts.34.up_proj", + "model.layers.22.mlp.experts.35.down_proj", + "model.layers.22.mlp.experts.35.gate_proj", + "model.layers.22.mlp.experts.35.up_proj", + "model.layers.22.mlp.experts.36.down_proj", + "model.layers.22.mlp.experts.36.gate_proj", + "model.layers.22.mlp.experts.36.up_proj", + "model.layers.22.mlp.experts.37.down_proj", + "model.layers.22.mlp.experts.37.gate_proj", + "model.layers.22.mlp.experts.37.up_proj", + "model.layers.22.mlp.experts.38.down_proj", + "model.layers.22.mlp.experts.38.gate_proj", + "model.layers.22.mlp.experts.38.up_proj", + "model.layers.22.mlp.experts.39.down_proj", + "model.layers.22.mlp.experts.39.gate_proj", + "model.layers.22.mlp.experts.39.up_proj", + "model.layers.22.mlp.experts.4.down_proj", + "model.layers.22.mlp.experts.4.gate_proj", + "model.layers.22.mlp.experts.4.up_proj", + "model.layers.22.mlp.experts.40.down_proj", + "model.layers.22.mlp.experts.40.gate_proj", + "model.layers.22.mlp.experts.40.up_proj", + "model.layers.22.mlp.experts.41.down_proj", + "model.layers.22.mlp.experts.41.gate_proj", + "model.layers.22.mlp.experts.41.up_proj", + "model.layers.22.mlp.experts.42.down_proj", + "model.layers.22.mlp.experts.42.gate_proj", + "model.layers.22.mlp.experts.42.up_proj", + "model.layers.22.mlp.experts.43.down_proj", + "model.layers.22.mlp.experts.43.gate_proj", + "model.layers.22.mlp.experts.43.up_proj", + "model.layers.22.mlp.experts.44.down_proj", + "model.layers.22.mlp.experts.44.gate_proj", + "model.layers.22.mlp.experts.44.up_proj", + "model.layers.22.mlp.experts.45.down_proj", + "model.layers.22.mlp.experts.45.gate_proj", + "model.layers.22.mlp.experts.45.up_proj", + "model.layers.22.mlp.experts.46.down_proj", + "model.layers.22.mlp.experts.46.gate_proj", + "model.layers.22.mlp.experts.46.up_proj", + "model.layers.22.mlp.experts.47.down_proj", + "model.layers.22.mlp.experts.47.gate_proj", + "model.layers.22.mlp.experts.47.up_proj", + "model.layers.22.mlp.experts.48.down_proj", + "model.layers.22.mlp.experts.48.gate_proj", + "model.layers.22.mlp.experts.48.up_proj", + "model.layers.22.mlp.experts.49.down_proj", + "model.layers.22.mlp.experts.49.gate_proj", + "model.layers.22.mlp.experts.49.up_proj", + "model.layers.22.mlp.experts.5.down_proj", + "model.layers.22.mlp.experts.5.gate_proj", + "model.layers.22.mlp.experts.5.up_proj", + "model.layers.22.mlp.experts.50.down_proj", + "model.layers.22.mlp.experts.50.gate_proj", + "model.layers.22.mlp.experts.50.up_proj", + "model.layers.22.mlp.experts.51.down_proj", + "model.layers.22.mlp.experts.51.gate_proj", + "model.layers.22.mlp.experts.51.up_proj", + "model.layers.22.mlp.experts.52.down_proj", + "model.layers.22.mlp.experts.52.gate_proj", + "model.layers.22.mlp.experts.52.up_proj", + "model.layers.22.mlp.experts.53.down_proj", + "model.layers.22.mlp.experts.53.gate_proj", + "model.layers.22.mlp.experts.53.up_proj", + "model.layers.22.mlp.experts.54.down_proj", + "model.layers.22.mlp.experts.54.gate_proj", + "model.layers.22.mlp.experts.54.up_proj", + "model.layers.22.mlp.experts.55.down_proj", + "model.layers.22.mlp.experts.55.gate_proj", + "model.layers.22.mlp.experts.55.up_proj", + "model.layers.22.mlp.experts.56.down_proj", + "model.layers.22.mlp.experts.56.gate_proj", + "model.layers.22.mlp.experts.56.up_proj", + "model.layers.22.mlp.experts.57.down_proj", + "model.layers.22.mlp.experts.57.gate_proj", + "model.layers.22.mlp.experts.57.up_proj", + "model.layers.22.mlp.experts.58.down_proj", + "model.layers.22.mlp.experts.58.gate_proj", + "model.layers.22.mlp.experts.58.up_proj", + "model.layers.22.mlp.experts.59.down_proj", + "model.layers.22.mlp.experts.59.gate_proj", + "model.layers.22.mlp.experts.59.up_proj", + "model.layers.22.mlp.experts.6.down_proj", + "model.layers.22.mlp.experts.6.gate_proj", + "model.layers.22.mlp.experts.6.up_proj", + "model.layers.22.mlp.experts.60.down_proj", + "model.layers.22.mlp.experts.60.gate_proj", + "model.layers.22.mlp.experts.60.up_proj", + "model.layers.22.mlp.experts.61.down_proj", + "model.layers.22.mlp.experts.61.gate_proj", + "model.layers.22.mlp.experts.61.up_proj", + "model.layers.22.mlp.experts.62.down_proj", + "model.layers.22.mlp.experts.62.gate_proj", + "model.layers.22.mlp.experts.62.up_proj", + "model.layers.22.mlp.experts.63.down_proj", + "model.layers.22.mlp.experts.63.gate_proj", + "model.layers.22.mlp.experts.63.up_proj", + "model.layers.22.mlp.experts.64.down_proj", + "model.layers.22.mlp.experts.64.gate_proj", + "model.layers.22.mlp.experts.64.up_proj", + "model.layers.22.mlp.experts.65.down_proj", + "model.layers.22.mlp.experts.65.gate_proj", + "model.layers.22.mlp.experts.65.up_proj", + "model.layers.22.mlp.experts.66.down_proj", + "model.layers.22.mlp.experts.66.gate_proj", + "model.layers.22.mlp.experts.66.up_proj", + "model.layers.22.mlp.experts.67.down_proj", + "model.layers.22.mlp.experts.67.gate_proj", + "model.layers.22.mlp.experts.67.up_proj", + "model.layers.22.mlp.experts.68.down_proj", + "model.layers.22.mlp.experts.68.gate_proj", + "model.layers.22.mlp.experts.68.up_proj", + "model.layers.22.mlp.experts.69.down_proj", + "model.layers.22.mlp.experts.69.gate_proj", + "model.layers.22.mlp.experts.69.up_proj", + "model.layers.22.mlp.experts.7.down_proj", + "model.layers.22.mlp.experts.7.gate_proj", + "model.layers.22.mlp.experts.7.up_proj", + "model.layers.22.mlp.experts.70.down_proj", + "model.layers.22.mlp.experts.70.gate_proj", + "model.layers.22.mlp.experts.70.up_proj", + "model.layers.22.mlp.experts.71.down_proj", + "model.layers.22.mlp.experts.71.gate_proj", + "model.layers.22.mlp.experts.71.up_proj", + "model.layers.22.mlp.experts.72.down_proj", + "model.layers.22.mlp.experts.72.gate_proj", + "model.layers.22.mlp.experts.72.up_proj", + "model.layers.22.mlp.experts.73.down_proj", + "model.layers.22.mlp.experts.73.gate_proj", + "model.layers.22.mlp.experts.73.up_proj", + "model.layers.22.mlp.experts.74.down_proj", + "model.layers.22.mlp.experts.74.gate_proj", + "model.layers.22.mlp.experts.74.up_proj", + "model.layers.22.mlp.experts.75.down_proj", + "model.layers.22.mlp.experts.75.gate_proj", + "model.layers.22.mlp.experts.75.up_proj", + "model.layers.22.mlp.experts.76.down_proj", + "model.layers.22.mlp.experts.76.gate_proj", + "model.layers.22.mlp.experts.76.up_proj", + "model.layers.22.mlp.experts.77.down_proj", + "model.layers.22.mlp.experts.77.gate_proj", + "model.layers.22.mlp.experts.77.up_proj", + "model.layers.22.mlp.experts.78.down_proj", + "model.layers.22.mlp.experts.78.gate_proj", + "model.layers.22.mlp.experts.78.up_proj", + "model.layers.22.mlp.experts.79.down_proj", + "model.layers.22.mlp.experts.79.gate_proj", + "model.layers.22.mlp.experts.79.up_proj", + "model.layers.22.mlp.experts.8.down_proj", + "model.layers.22.mlp.experts.8.gate_proj", + "model.layers.22.mlp.experts.8.up_proj", + "model.layers.22.mlp.experts.80.down_proj", + "model.layers.22.mlp.experts.80.gate_proj", + "model.layers.22.mlp.experts.80.up_proj", + "model.layers.22.mlp.experts.81.down_proj", + "model.layers.22.mlp.experts.81.gate_proj", + "model.layers.22.mlp.experts.81.up_proj", + "model.layers.22.mlp.experts.82.down_proj", + "model.layers.22.mlp.experts.82.gate_proj", + "model.layers.22.mlp.experts.82.up_proj", + "model.layers.22.mlp.experts.83.down_proj", + "model.layers.22.mlp.experts.83.gate_proj", + "model.layers.22.mlp.experts.83.up_proj", + "model.layers.22.mlp.experts.84.down_proj", + "model.layers.22.mlp.experts.84.gate_proj", + "model.layers.22.mlp.experts.84.up_proj", + "model.layers.22.mlp.experts.85.down_proj", + "model.layers.22.mlp.experts.85.gate_proj", + "model.layers.22.mlp.experts.85.up_proj", + "model.layers.22.mlp.experts.86.down_proj", + "model.layers.22.mlp.experts.86.gate_proj", + "model.layers.22.mlp.experts.86.up_proj", + "model.layers.22.mlp.experts.87.down_proj", + "model.layers.22.mlp.experts.87.gate_proj", + "model.layers.22.mlp.experts.87.up_proj", + "model.layers.22.mlp.experts.88.down_proj", + "model.layers.22.mlp.experts.88.gate_proj", + "model.layers.22.mlp.experts.88.up_proj", + "model.layers.22.mlp.experts.89.down_proj", + "model.layers.22.mlp.experts.89.gate_proj", + "model.layers.22.mlp.experts.89.up_proj", + "model.layers.22.mlp.experts.9.down_proj", + "model.layers.22.mlp.experts.9.gate_proj", + "model.layers.22.mlp.experts.9.up_proj", + "model.layers.22.mlp.experts.90.down_proj", + "model.layers.22.mlp.experts.90.gate_proj", + "model.layers.22.mlp.experts.90.up_proj", + "model.layers.22.mlp.experts.91.down_proj", + "model.layers.22.mlp.experts.91.gate_proj", + "model.layers.22.mlp.experts.91.up_proj", + "model.layers.22.mlp.experts.92.down_proj", + "model.layers.22.mlp.experts.92.gate_proj", + "model.layers.22.mlp.experts.92.up_proj", + "model.layers.22.mlp.experts.93.down_proj", + "model.layers.22.mlp.experts.93.gate_proj", + "model.layers.22.mlp.experts.93.up_proj", + "model.layers.22.mlp.experts.94.down_proj", + "model.layers.22.mlp.experts.94.gate_proj", + "model.layers.22.mlp.experts.94.up_proj", + "model.layers.22.mlp.experts.95.down_proj", + "model.layers.22.mlp.experts.95.gate_proj", + "model.layers.22.mlp.experts.95.up_proj", + "model.layers.22.mlp.experts.96.down_proj", + "model.layers.22.mlp.experts.96.gate_proj", + "model.layers.22.mlp.experts.96.up_proj", + "model.layers.22.mlp.experts.97.down_proj", + "model.layers.22.mlp.experts.97.gate_proj", + "model.layers.22.mlp.experts.97.up_proj", + "model.layers.22.mlp.experts.98.down_proj", + "model.layers.22.mlp.experts.98.gate_proj", + "model.layers.22.mlp.experts.98.up_proj", + "model.layers.22.mlp.experts.99.down_proj", + "model.layers.22.mlp.experts.99.gate_proj", + "model.layers.22.mlp.experts.99.up_proj", + "model.layers.23.mlp.experts.0.down_proj", + "model.layers.23.mlp.experts.0.gate_proj", + "model.layers.23.mlp.experts.0.up_proj", + "model.layers.23.mlp.experts.1.down_proj", + "model.layers.23.mlp.experts.1.gate_proj", + "model.layers.23.mlp.experts.1.up_proj", + "model.layers.23.mlp.experts.10.down_proj", + "model.layers.23.mlp.experts.10.gate_proj", + "model.layers.23.mlp.experts.10.up_proj", + "model.layers.23.mlp.experts.100.down_proj", + "model.layers.23.mlp.experts.100.gate_proj", + "model.layers.23.mlp.experts.100.up_proj", + "model.layers.23.mlp.experts.101.down_proj", + "model.layers.23.mlp.experts.101.gate_proj", + "model.layers.23.mlp.experts.101.up_proj", + "model.layers.23.mlp.experts.102.down_proj", + "model.layers.23.mlp.experts.102.gate_proj", + "model.layers.23.mlp.experts.102.up_proj", + "model.layers.23.mlp.experts.103.down_proj", + "model.layers.23.mlp.experts.103.gate_proj", + "model.layers.23.mlp.experts.103.up_proj", + "model.layers.23.mlp.experts.104.down_proj", + "model.layers.23.mlp.experts.104.gate_proj", + "model.layers.23.mlp.experts.104.up_proj", + "model.layers.23.mlp.experts.105.down_proj", + "model.layers.23.mlp.experts.105.gate_proj", + "model.layers.23.mlp.experts.105.up_proj", + "model.layers.23.mlp.experts.106.down_proj", + "model.layers.23.mlp.experts.106.gate_proj", + "model.layers.23.mlp.experts.106.up_proj", + "model.layers.23.mlp.experts.107.down_proj", + "model.layers.23.mlp.experts.107.gate_proj", + "model.layers.23.mlp.experts.107.up_proj", + "model.layers.23.mlp.experts.108.down_proj", + "model.layers.23.mlp.experts.108.gate_proj", + "model.layers.23.mlp.experts.108.up_proj", + "model.layers.23.mlp.experts.109.down_proj", + "model.layers.23.mlp.experts.109.gate_proj", + "model.layers.23.mlp.experts.109.up_proj", + "model.layers.23.mlp.experts.11.down_proj", + "model.layers.23.mlp.experts.11.gate_proj", + "model.layers.23.mlp.experts.11.up_proj", + "model.layers.23.mlp.experts.110.down_proj", + "model.layers.23.mlp.experts.110.gate_proj", + "model.layers.23.mlp.experts.110.up_proj", + "model.layers.23.mlp.experts.111.down_proj", + "model.layers.23.mlp.experts.111.gate_proj", + "model.layers.23.mlp.experts.111.up_proj", + "model.layers.23.mlp.experts.112.down_proj", + "model.layers.23.mlp.experts.112.gate_proj", + "model.layers.23.mlp.experts.112.up_proj", + "model.layers.23.mlp.experts.113.down_proj", + "model.layers.23.mlp.experts.113.gate_proj", + "model.layers.23.mlp.experts.113.up_proj", + "model.layers.23.mlp.experts.114.down_proj", + "model.layers.23.mlp.experts.114.gate_proj", + "model.layers.23.mlp.experts.114.up_proj", + "model.layers.23.mlp.experts.115.down_proj", + "model.layers.23.mlp.experts.115.gate_proj", + "model.layers.23.mlp.experts.115.up_proj", + "model.layers.23.mlp.experts.116.down_proj", + "model.layers.23.mlp.experts.116.gate_proj", + "model.layers.23.mlp.experts.116.up_proj", + "model.layers.23.mlp.experts.117.down_proj", + "model.layers.23.mlp.experts.117.gate_proj", + "model.layers.23.mlp.experts.117.up_proj", + "model.layers.23.mlp.experts.118.down_proj", + "model.layers.23.mlp.experts.118.gate_proj", + "model.layers.23.mlp.experts.118.up_proj", + "model.layers.23.mlp.experts.119.down_proj", + "model.layers.23.mlp.experts.119.gate_proj", + "model.layers.23.mlp.experts.119.up_proj", + "model.layers.23.mlp.experts.12.down_proj", + "model.layers.23.mlp.experts.12.gate_proj", + "model.layers.23.mlp.experts.12.up_proj", + "model.layers.23.mlp.experts.120.down_proj", + "model.layers.23.mlp.experts.120.gate_proj", + "model.layers.23.mlp.experts.120.up_proj", + "model.layers.23.mlp.experts.121.down_proj", + "model.layers.23.mlp.experts.121.gate_proj", + "model.layers.23.mlp.experts.121.up_proj", + "model.layers.23.mlp.experts.122.down_proj", + "model.layers.23.mlp.experts.122.gate_proj", + "model.layers.23.mlp.experts.122.up_proj", + "model.layers.23.mlp.experts.123.down_proj", + "model.layers.23.mlp.experts.123.gate_proj", + "model.layers.23.mlp.experts.123.up_proj", + "model.layers.23.mlp.experts.124.down_proj", + "model.layers.23.mlp.experts.124.gate_proj", + "model.layers.23.mlp.experts.124.up_proj", + "model.layers.23.mlp.experts.125.down_proj", + "model.layers.23.mlp.experts.125.gate_proj", + "model.layers.23.mlp.experts.125.up_proj", + "model.layers.23.mlp.experts.126.down_proj", + "model.layers.23.mlp.experts.126.gate_proj", + "model.layers.23.mlp.experts.126.up_proj", + "model.layers.23.mlp.experts.127.down_proj", + "model.layers.23.mlp.experts.127.gate_proj", + "model.layers.23.mlp.experts.127.up_proj", + "model.layers.23.mlp.experts.128.down_proj", + "model.layers.23.mlp.experts.128.gate_proj", + "model.layers.23.mlp.experts.128.up_proj", + "model.layers.23.mlp.experts.129.down_proj", + "model.layers.23.mlp.experts.129.gate_proj", + "model.layers.23.mlp.experts.129.up_proj", + "model.layers.23.mlp.experts.13.down_proj", + "model.layers.23.mlp.experts.13.gate_proj", + "model.layers.23.mlp.experts.13.up_proj", + "model.layers.23.mlp.experts.130.down_proj", + "model.layers.23.mlp.experts.130.gate_proj", + "model.layers.23.mlp.experts.130.up_proj", + "model.layers.23.mlp.experts.131.down_proj", + "model.layers.23.mlp.experts.131.gate_proj", + "model.layers.23.mlp.experts.131.up_proj", + "model.layers.23.mlp.experts.132.down_proj", + "model.layers.23.mlp.experts.132.gate_proj", + "model.layers.23.mlp.experts.132.up_proj", + "model.layers.23.mlp.experts.133.down_proj", + "model.layers.23.mlp.experts.133.gate_proj", + "model.layers.23.mlp.experts.133.up_proj", + "model.layers.23.mlp.experts.134.down_proj", + "model.layers.23.mlp.experts.134.gate_proj", + "model.layers.23.mlp.experts.134.up_proj", + "model.layers.23.mlp.experts.135.down_proj", + "model.layers.23.mlp.experts.135.gate_proj", + "model.layers.23.mlp.experts.135.up_proj", + "model.layers.23.mlp.experts.136.down_proj", + "model.layers.23.mlp.experts.136.gate_proj", + "model.layers.23.mlp.experts.136.up_proj", + "model.layers.23.mlp.experts.137.down_proj", + "model.layers.23.mlp.experts.137.gate_proj", + "model.layers.23.mlp.experts.137.up_proj", + "model.layers.23.mlp.experts.138.down_proj", + "model.layers.23.mlp.experts.138.gate_proj", + "model.layers.23.mlp.experts.138.up_proj", + "model.layers.23.mlp.experts.139.down_proj", + "model.layers.23.mlp.experts.139.gate_proj", + "model.layers.23.mlp.experts.139.up_proj", + "model.layers.23.mlp.experts.14.down_proj", + "model.layers.23.mlp.experts.14.gate_proj", + "model.layers.23.mlp.experts.14.up_proj", + "model.layers.23.mlp.experts.140.down_proj", + "model.layers.23.mlp.experts.140.gate_proj", + "model.layers.23.mlp.experts.140.up_proj", + "model.layers.23.mlp.experts.141.down_proj", + "model.layers.23.mlp.experts.141.gate_proj", + "model.layers.23.mlp.experts.141.up_proj", + "model.layers.23.mlp.experts.142.down_proj", + "model.layers.23.mlp.experts.142.gate_proj", + "model.layers.23.mlp.experts.142.up_proj", + "model.layers.23.mlp.experts.143.down_proj", + "model.layers.23.mlp.experts.143.gate_proj", + "model.layers.23.mlp.experts.143.up_proj", + "model.layers.23.mlp.experts.144.down_proj", + "model.layers.23.mlp.experts.144.gate_proj", + "model.layers.23.mlp.experts.144.up_proj", + "model.layers.23.mlp.experts.145.down_proj", + "model.layers.23.mlp.experts.145.gate_proj", + "model.layers.23.mlp.experts.145.up_proj", + "model.layers.23.mlp.experts.146.down_proj", + "model.layers.23.mlp.experts.146.gate_proj", + "model.layers.23.mlp.experts.146.up_proj", + "model.layers.23.mlp.experts.147.down_proj", + "model.layers.23.mlp.experts.147.gate_proj", + "model.layers.23.mlp.experts.147.up_proj", + "model.layers.23.mlp.experts.148.down_proj", + "model.layers.23.mlp.experts.148.gate_proj", + "model.layers.23.mlp.experts.148.up_proj", + "model.layers.23.mlp.experts.149.down_proj", + "model.layers.23.mlp.experts.149.gate_proj", + "model.layers.23.mlp.experts.149.up_proj", + "model.layers.23.mlp.experts.15.down_proj", + "model.layers.23.mlp.experts.15.gate_proj", + "model.layers.23.mlp.experts.15.up_proj", + "model.layers.23.mlp.experts.150.down_proj", + "model.layers.23.mlp.experts.150.gate_proj", + "model.layers.23.mlp.experts.150.up_proj", + "model.layers.23.mlp.experts.151.down_proj", + "model.layers.23.mlp.experts.151.gate_proj", + "model.layers.23.mlp.experts.151.up_proj", + "model.layers.23.mlp.experts.152.down_proj", + "model.layers.23.mlp.experts.152.gate_proj", + "model.layers.23.mlp.experts.152.up_proj", + "model.layers.23.mlp.experts.153.down_proj", + "model.layers.23.mlp.experts.153.gate_proj", + "model.layers.23.mlp.experts.153.up_proj", + "model.layers.23.mlp.experts.154.down_proj", + "model.layers.23.mlp.experts.154.gate_proj", + "model.layers.23.mlp.experts.154.up_proj", + "model.layers.23.mlp.experts.155.down_proj", + "model.layers.23.mlp.experts.155.gate_proj", + "model.layers.23.mlp.experts.155.up_proj", + "model.layers.23.mlp.experts.156.down_proj", + "model.layers.23.mlp.experts.156.gate_proj", + "model.layers.23.mlp.experts.156.up_proj", + "model.layers.23.mlp.experts.157.down_proj", + "model.layers.23.mlp.experts.157.gate_proj", + "model.layers.23.mlp.experts.157.up_proj", + "model.layers.23.mlp.experts.158.down_proj", + "model.layers.23.mlp.experts.158.gate_proj", + "model.layers.23.mlp.experts.158.up_proj", + "model.layers.23.mlp.experts.159.down_proj", + "model.layers.23.mlp.experts.159.gate_proj", + "model.layers.23.mlp.experts.159.up_proj", + "model.layers.23.mlp.experts.16.down_proj", + "model.layers.23.mlp.experts.16.gate_proj", + "model.layers.23.mlp.experts.16.up_proj", + "model.layers.23.mlp.experts.160.down_proj", + "model.layers.23.mlp.experts.160.gate_proj", + "model.layers.23.mlp.experts.160.up_proj", + "model.layers.23.mlp.experts.161.down_proj", + "model.layers.23.mlp.experts.161.gate_proj", + "model.layers.23.mlp.experts.161.up_proj", + "model.layers.23.mlp.experts.162.down_proj", + "model.layers.23.mlp.experts.162.gate_proj", + "model.layers.23.mlp.experts.162.up_proj", + "model.layers.23.mlp.experts.163.down_proj", + "model.layers.23.mlp.experts.163.gate_proj", + "model.layers.23.mlp.experts.163.up_proj", + "model.layers.23.mlp.experts.164.down_proj", + "model.layers.23.mlp.experts.164.gate_proj", + "model.layers.23.mlp.experts.164.up_proj", + "model.layers.23.mlp.experts.165.down_proj", + "model.layers.23.mlp.experts.165.gate_proj", + "model.layers.23.mlp.experts.165.up_proj", + "model.layers.23.mlp.experts.166.down_proj", + "model.layers.23.mlp.experts.166.gate_proj", + "model.layers.23.mlp.experts.166.up_proj", + "model.layers.23.mlp.experts.167.down_proj", + "model.layers.23.mlp.experts.167.gate_proj", + "model.layers.23.mlp.experts.167.up_proj", + "model.layers.23.mlp.experts.168.down_proj", + "model.layers.23.mlp.experts.168.gate_proj", + "model.layers.23.mlp.experts.168.up_proj", + "model.layers.23.mlp.experts.169.down_proj", + "model.layers.23.mlp.experts.169.gate_proj", + "model.layers.23.mlp.experts.169.up_proj", + "model.layers.23.mlp.experts.17.down_proj", + "model.layers.23.mlp.experts.17.gate_proj", + "model.layers.23.mlp.experts.17.up_proj", + "model.layers.23.mlp.experts.170.down_proj", + "model.layers.23.mlp.experts.170.gate_proj", + "model.layers.23.mlp.experts.170.up_proj", + "model.layers.23.mlp.experts.171.down_proj", + "model.layers.23.mlp.experts.171.gate_proj", + "model.layers.23.mlp.experts.171.up_proj", + "model.layers.23.mlp.experts.172.down_proj", + "model.layers.23.mlp.experts.172.gate_proj", + "model.layers.23.mlp.experts.172.up_proj", + "model.layers.23.mlp.experts.173.down_proj", + "model.layers.23.mlp.experts.173.gate_proj", + "model.layers.23.mlp.experts.173.up_proj", + "model.layers.23.mlp.experts.174.down_proj", + "model.layers.23.mlp.experts.174.gate_proj", + "model.layers.23.mlp.experts.174.up_proj", + "model.layers.23.mlp.experts.175.down_proj", + "model.layers.23.mlp.experts.175.gate_proj", + "model.layers.23.mlp.experts.175.up_proj", + "model.layers.23.mlp.experts.176.down_proj", + "model.layers.23.mlp.experts.176.gate_proj", + "model.layers.23.mlp.experts.176.up_proj", + "model.layers.23.mlp.experts.177.down_proj", + "model.layers.23.mlp.experts.177.gate_proj", + "model.layers.23.mlp.experts.177.up_proj", + "model.layers.23.mlp.experts.178.down_proj", + "model.layers.23.mlp.experts.178.gate_proj", + "model.layers.23.mlp.experts.178.up_proj", + "model.layers.23.mlp.experts.179.down_proj", + "model.layers.23.mlp.experts.179.gate_proj", + "model.layers.23.mlp.experts.179.up_proj", + "model.layers.23.mlp.experts.18.down_proj", + "model.layers.23.mlp.experts.18.gate_proj", + "model.layers.23.mlp.experts.18.up_proj", + "model.layers.23.mlp.experts.180.down_proj", + "model.layers.23.mlp.experts.180.gate_proj", + "model.layers.23.mlp.experts.180.up_proj", + "model.layers.23.mlp.experts.181.down_proj", + "model.layers.23.mlp.experts.181.gate_proj", + "model.layers.23.mlp.experts.181.up_proj", + "model.layers.23.mlp.experts.182.down_proj", + "model.layers.23.mlp.experts.182.gate_proj", + "model.layers.23.mlp.experts.182.up_proj", + "model.layers.23.mlp.experts.183.down_proj", + "model.layers.23.mlp.experts.183.gate_proj", + "model.layers.23.mlp.experts.183.up_proj", + "model.layers.23.mlp.experts.184.down_proj", + "model.layers.23.mlp.experts.184.gate_proj", + "model.layers.23.mlp.experts.184.up_proj", + "model.layers.23.mlp.experts.185.down_proj", + "model.layers.23.mlp.experts.185.gate_proj", + "model.layers.23.mlp.experts.185.up_proj", + "model.layers.23.mlp.experts.186.down_proj", + "model.layers.23.mlp.experts.186.gate_proj", + "model.layers.23.mlp.experts.186.up_proj", + "model.layers.23.mlp.experts.187.down_proj", + "model.layers.23.mlp.experts.187.gate_proj", + "model.layers.23.mlp.experts.187.up_proj", + "model.layers.23.mlp.experts.188.down_proj", + "model.layers.23.mlp.experts.188.gate_proj", + "model.layers.23.mlp.experts.188.up_proj", + "model.layers.23.mlp.experts.189.down_proj", + "model.layers.23.mlp.experts.189.gate_proj", + "model.layers.23.mlp.experts.189.up_proj", + "model.layers.23.mlp.experts.19.down_proj", + "model.layers.23.mlp.experts.19.gate_proj", + "model.layers.23.mlp.experts.19.up_proj", + "model.layers.23.mlp.experts.190.down_proj", + "model.layers.23.mlp.experts.190.gate_proj", + "model.layers.23.mlp.experts.190.up_proj", + "model.layers.23.mlp.experts.191.down_proj", + "model.layers.23.mlp.experts.191.gate_proj", + "model.layers.23.mlp.experts.191.up_proj", + "model.layers.23.mlp.experts.192.down_proj", + "model.layers.23.mlp.experts.192.gate_proj", + "model.layers.23.mlp.experts.192.up_proj", + "model.layers.23.mlp.experts.193.down_proj", + "model.layers.23.mlp.experts.193.gate_proj", + "model.layers.23.mlp.experts.193.up_proj", + "model.layers.23.mlp.experts.194.down_proj", + "model.layers.23.mlp.experts.194.gate_proj", + "model.layers.23.mlp.experts.194.up_proj", + "model.layers.23.mlp.experts.195.down_proj", + "model.layers.23.mlp.experts.195.gate_proj", + "model.layers.23.mlp.experts.195.up_proj", + "model.layers.23.mlp.experts.196.down_proj", + "model.layers.23.mlp.experts.196.gate_proj", + "model.layers.23.mlp.experts.196.up_proj", + "model.layers.23.mlp.experts.197.down_proj", + "model.layers.23.mlp.experts.197.gate_proj", + "model.layers.23.mlp.experts.197.up_proj", + "model.layers.23.mlp.experts.198.down_proj", + "model.layers.23.mlp.experts.198.gate_proj", + "model.layers.23.mlp.experts.198.up_proj", + "model.layers.23.mlp.experts.199.down_proj", + "model.layers.23.mlp.experts.199.gate_proj", + "model.layers.23.mlp.experts.199.up_proj", + "model.layers.23.mlp.experts.2.down_proj", + "model.layers.23.mlp.experts.2.gate_proj", + "model.layers.23.mlp.experts.2.up_proj", + "model.layers.23.mlp.experts.20.down_proj", + "model.layers.23.mlp.experts.20.gate_proj", + "model.layers.23.mlp.experts.20.up_proj", + "model.layers.23.mlp.experts.200.down_proj", + "model.layers.23.mlp.experts.200.gate_proj", + "model.layers.23.mlp.experts.200.up_proj", + "model.layers.23.mlp.experts.201.down_proj", + "model.layers.23.mlp.experts.201.gate_proj", + "model.layers.23.mlp.experts.201.up_proj", + "model.layers.23.mlp.experts.202.down_proj", + "model.layers.23.mlp.experts.202.gate_proj", + "model.layers.23.mlp.experts.202.up_proj", + "model.layers.23.mlp.experts.203.down_proj", + "model.layers.23.mlp.experts.203.gate_proj", + "model.layers.23.mlp.experts.203.up_proj", + "model.layers.23.mlp.experts.204.down_proj", + "model.layers.23.mlp.experts.204.gate_proj", + "model.layers.23.mlp.experts.204.up_proj", + "model.layers.23.mlp.experts.205.down_proj", + "model.layers.23.mlp.experts.205.gate_proj", + "model.layers.23.mlp.experts.205.up_proj", + "model.layers.23.mlp.experts.206.down_proj", + "model.layers.23.mlp.experts.206.gate_proj", + "model.layers.23.mlp.experts.206.up_proj", + "model.layers.23.mlp.experts.207.down_proj", + "model.layers.23.mlp.experts.207.gate_proj", + "model.layers.23.mlp.experts.207.up_proj", + "model.layers.23.mlp.experts.208.down_proj", + "model.layers.23.mlp.experts.208.gate_proj", + "model.layers.23.mlp.experts.208.up_proj", + "model.layers.23.mlp.experts.209.down_proj", + "model.layers.23.mlp.experts.209.gate_proj", + "model.layers.23.mlp.experts.209.up_proj", + "model.layers.23.mlp.experts.21.down_proj", + "model.layers.23.mlp.experts.21.gate_proj", + "model.layers.23.mlp.experts.21.up_proj", + "model.layers.23.mlp.experts.210.down_proj", + "model.layers.23.mlp.experts.210.gate_proj", + "model.layers.23.mlp.experts.210.up_proj", + "model.layers.23.mlp.experts.211.down_proj", + "model.layers.23.mlp.experts.211.gate_proj", + "model.layers.23.mlp.experts.211.up_proj", + "model.layers.23.mlp.experts.212.down_proj", + "model.layers.23.mlp.experts.212.gate_proj", + "model.layers.23.mlp.experts.212.up_proj", + "model.layers.23.mlp.experts.213.down_proj", + "model.layers.23.mlp.experts.213.gate_proj", + "model.layers.23.mlp.experts.213.up_proj", + "model.layers.23.mlp.experts.214.down_proj", + "model.layers.23.mlp.experts.214.gate_proj", + "model.layers.23.mlp.experts.214.up_proj", + "model.layers.23.mlp.experts.215.down_proj", + "model.layers.23.mlp.experts.215.gate_proj", + "model.layers.23.mlp.experts.215.up_proj", + "model.layers.23.mlp.experts.216.down_proj", + "model.layers.23.mlp.experts.216.gate_proj", + "model.layers.23.mlp.experts.216.up_proj", + "model.layers.23.mlp.experts.217.down_proj", + "model.layers.23.mlp.experts.217.gate_proj", + "model.layers.23.mlp.experts.217.up_proj", + "model.layers.23.mlp.experts.218.down_proj", + "model.layers.23.mlp.experts.218.gate_proj", + "model.layers.23.mlp.experts.218.up_proj", + "model.layers.23.mlp.experts.219.down_proj", + "model.layers.23.mlp.experts.219.gate_proj", + "model.layers.23.mlp.experts.219.up_proj", + "model.layers.23.mlp.experts.22.down_proj", + "model.layers.23.mlp.experts.22.gate_proj", + "model.layers.23.mlp.experts.22.up_proj", + "model.layers.23.mlp.experts.220.down_proj", + "model.layers.23.mlp.experts.220.gate_proj", + "model.layers.23.mlp.experts.220.up_proj", + "model.layers.23.mlp.experts.221.down_proj", + "model.layers.23.mlp.experts.221.gate_proj", + "model.layers.23.mlp.experts.221.up_proj", + "model.layers.23.mlp.experts.222.down_proj", + "model.layers.23.mlp.experts.222.gate_proj", + "model.layers.23.mlp.experts.222.up_proj", + "model.layers.23.mlp.experts.223.down_proj", + "model.layers.23.mlp.experts.223.gate_proj", + "model.layers.23.mlp.experts.223.up_proj", + "model.layers.23.mlp.experts.224.down_proj", + "model.layers.23.mlp.experts.224.gate_proj", + "model.layers.23.mlp.experts.224.up_proj", + "model.layers.23.mlp.experts.225.down_proj", + "model.layers.23.mlp.experts.225.gate_proj", + "model.layers.23.mlp.experts.225.up_proj", + "model.layers.23.mlp.experts.226.down_proj", + "model.layers.23.mlp.experts.226.gate_proj", + "model.layers.23.mlp.experts.226.up_proj", + "model.layers.23.mlp.experts.227.down_proj", + "model.layers.23.mlp.experts.227.gate_proj", + "model.layers.23.mlp.experts.227.up_proj", + "model.layers.23.mlp.experts.228.down_proj", + "model.layers.23.mlp.experts.228.gate_proj", + "model.layers.23.mlp.experts.228.up_proj", + "model.layers.23.mlp.experts.229.down_proj", + "model.layers.23.mlp.experts.229.gate_proj", + "model.layers.23.mlp.experts.229.up_proj", + "model.layers.23.mlp.experts.23.down_proj", + "model.layers.23.mlp.experts.23.gate_proj", + "model.layers.23.mlp.experts.23.up_proj", + "model.layers.23.mlp.experts.230.down_proj", + "model.layers.23.mlp.experts.230.gate_proj", + "model.layers.23.mlp.experts.230.up_proj", + "model.layers.23.mlp.experts.231.down_proj", + "model.layers.23.mlp.experts.231.gate_proj", + "model.layers.23.mlp.experts.231.up_proj", + "model.layers.23.mlp.experts.232.down_proj", + "model.layers.23.mlp.experts.232.gate_proj", + "model.layers.23.mlp.experts.232.up_proj", + "model.layers.23.mlp.experts.233.down_proj", + "model.layers.23.mlp.experts.233.gate_proj", + "model.layers.23.mlp.experts.233.up_proj", + "model.layers.23.mlp.experts.234.down_proj", + "model.layers.23.mlp.experts.234.gate_proj", + "model.layers.23.mlp.experts.234.up_proj", + "model.layers.23.mlp.experts.235.down_proj", + "model.layers.23.mlp.experts.235.gate_proj", + "model.layers.23.mlp.experts.235.up_proj", + "model.layers.23.mlp.experts.236.down_proj", + "model.layers.23.mlp.experts.236.gate_proj", + "model.layers.23.mlp.experts.236.up_proj", + "model.layers.23.mlp.experts.237.down_proj", + "model.layers.23.mlp.experts.237.gate_proj", + "model.layers.23.mlp.experts.237.up_proj", + "model.layers.23.mlp.experts.238.down_proj", + "model.layers.23.mlp.experts.238.gate_proj", + "model.layers.23.mlp.experts.238.up_proj", + "model.layers.23.mlp.experts.239.down_proj", + "model.layers.23.mlp.experts.239.gate_proj", + "model.layers.23.mlp.experts.239.up_proj", + "model.layers.23.mlp.experts.24.down_proj", + "model.layers.23.mlp.experts.24.gate_proj", + "model.layers.23.mlp.experts.24.up_proj", + "model.layers.23.mlp.experts.240.down_proj", + "model.layers.23.mlp.experts.240.gate_proj", + "model.layers.23.mlp.experts.240.up_proj", + "model.layers.23.mlp.experts.241.down_proj", + "model.layers.23.mlp.experts.241.gate_proj", + "model.layers.23.mlp.experts.241.up_proj", + "model.layers.23.mlp.experts.242.down_proj", + "model.layers.23.mlp.experts.242.gate_proj", + "model.layers.23.mlp.experts.242.up_proj", + "model.layers.23.mlp.experts.243.down_proj", + "model.layers.23.mlp.experts.243.gate_proj", + "model.layers.23.mlp.experts.243.up_proj", + "model.layers.23.mlp.experts.244.down_proj", + "model.layers.23.mlp.experts.244.gate_proj", + "model.layers.23.mlp.experts.244.up_proj", + "model.layers.23.mlp.experts.245.down_proj", + "model.layers.23.mlp.experts.245.gate_proj", + "model.layers.23.mlp.experts.245.up_proj", + "model.layers.23.mlp.experts.246.down_proj", + "model.layers.23.mlp.experts.246.gate_proj", + "model.layers.23.mlp.experts.246.up_proj", + "model.layers.23.mlp.experts.247.down_proj", + "model.layers.23.mlp.experts.247.gate_proj", + "model.layers.23.mlp.experts.247.up_proj", + "model.layers.23.mlp.experts.248.down_proj", + "model.layers.23.mlp.experts.248.gate_proj", + "model.layers.23.mlp.experts.248.up_proj", + "model.layers.23.mlp.experts.249.down_proj", + "model.layers.23.mlp.experts.249.gate_proj", + "model.layers.23.mlp.experts.249.up_proj", + "model.layers.23.mlp.experts.25.down_proj", + "model.layers.23.mlp.experts.25.gate_proj", + "model.layers.23.mlp.experts.25.up_proj", + "model.layers.23.mlp.experts.250.down_proj", + "model.layers.23.mlp.experts.250.gate_proj", + "model.layers.23.mlp.experts.250.up_proj", + "model.layers.23.mlp.experts.251.down_proj", + "model.layers.23.mlp.experts.251.gate_proj", + "model.layers.23.mlp.experts.251.up_proj", + "model.layers.23.mlp.experts.252.down_proj", + "model.layers.23.mlp.experts.252.gate_proj", + "model.layers.23.mlp.experts.252.up_proj", + "model.layers.23.mlp.experts.253.down_proj", + "model.layers.23.mlp.experts.253.gate_proj", + "model.layers.23.mlp.experts.253.up_proj", + "model.layers.23.mlp.experts.254.down_proj", + "model.layers.23.mlp.experts.254.gate_proj", + "model.layers.23.mlp.experts.254.up_proj", + "model.layers.23.mlp.experts.255.down_proj", + "model.layers.23.mlp.experts.255.gate_proj", + "model.layers.23.mlp.experts.255.up_proj", + "model.layers.23.mlp.experts.26.down_proj", + "model.layers.23.mlp.experts.26.gate_proj", + "model.layers.23.mlp.experts.26.up_proj", + "model.layers.23.mlp.experts.27.down_proj", + "model.layers.23.mlp.experts.27.gate_proj", + "model.layers.23.mlp.experts.27.up_proj", + "model.layers.23.mlp.experts.28.down_proj", + "model.layers.23.mlp.experts.28.gate_proj", + "model.layers.23.mlp.experts.28.up_proj", + "model.layers.23.mlp.experts.29.down_proj", + "model.layers.23.mlp.experts.29.gate_proj", + "model.layers.23.mlp.experts.29.up_proj", + "model.layers.23.mlp.experts.3.down_proj", + "model.layers.23.mlp.experts.3.gate_proj", + "model.layers.23.mlp.experts.3.up_proj", + "model.layers.23.mlp.experts.30.down_proj", + "model.layers.23.mlp.experts.30.gate_proj", + "model.layers.23.mlp.experts.30.up_proj", + "model.layers.23.mlp.experts.31.down_proj", + "model.layers.23.mlp.experts.31.gate_proj", + "model.layers.23.mlp.experts.31.up_proj", + "model.layers.23.mlp.experts.32.down_proj", + "model.layers.23.mlp.experts.32.gate_proj", + "model.layers.23.mlp.experts.32.up_proj", + "model.layers.23.mlp.experts.33.down_proj", + "model.layers.23.mlp.experts.33.gate_proj", + "model.layers.23.mlp.experts.33.up_proj", + "model.layers.23.mlp.experts.34.down_proj", + "model.layers.23.mlp.experts.34.gate_proj", + "model.layers.23.mlp.experts.34.up_proj", + "model.layers.23.mlp.experts.35.down_proj", + "model.layers.23.mlp.experts.35.gate_proj", + "model.layers.23.mlp.experts.35.up_proj", + "model.layers.23.mlp.experts.36.down_proj", + "model.layers.23.mlp.experts.36.gate_proj", + "model.layers.23.mlp.experts.36.up_proj", + "model.layers.23.mlp.experts.37.down_proj", + "model.layers.23.mlp.experts.37.gate_proj", + "model.layers.23.mlp.experts.37.up_proj", + "model.layers.23.mlp.experts.38.down_proj", + "model.layers.23.mlp.experts.38.gate_proj", + "model.layers.23.mlp.experts.38.up_proj", + "model.layers.23.mlp.experts.39.down_proj", + "model.layers.23.mlp.experts.39.gate_proj", + "model.layers.23.mlp.experts.39.up_proj", + "model.layers.23.mlp.experts.4.down_proj", + "model.layers.23.mlp.experts.4.gate_proj", + "model.layers.23.mlp.experts.4.up_proj", + "model.layers.23.mlp.experts.40.down_proj", + "model.layers.23.mlp.experts.40.gate_proj", + "model.layers.23.mlp.experts.40.up_proj", + "model.layers.23.mlp.experts.41.down_proj", + "model.layers.23.mlp.experts.41.gate_proj", + "model.layers.23.mlp.experts.41.up_proj", + "model.layers.23.mlp.experts.42.down_proj", + "model.layers.23.mlp.experts.42.gate_proj", + "model.layers.23.mlp.experts.42.up_proj", + "model.layers.23.mlp.experts.43.down_proj", + "model.layers.23.mlp.experts.43.gate_proj", + "model.layers.23.mlp.experts.43.up_proj", + "model.layers.23.mlp.experts.44.down_proj", + "model.layers.23.mlp.experts.44.gate_proj", + "model.layers.23.mlp.experts.44.up_proj", + "model.layers.23.mlp.experts.45.down_proj", + "model.layers.23.mlp.experts.45.gate_proj", + "model.layers.23.mlp.experts.45.up_proj", + "model.layers.23.mlp.experts.46.down_proj", + "model.layers.23.mlp.experts.46.gate_proj", + "model.layers.23.mlp.experts.46.up_proj", + "model.layers.23.mlp.experts.47.down_proj", + "model.layers.23.mlp.experts.47.gate_proj", + "model.layers.23.mlp.experts.47.up_proj", + "model.layers.23.mlp.experts.48.down_proj", + "model.layers.23.mlp.experts.48.gate_proj", + "model.layers.23.mlp.experts.48.up_proj", + "model.layers.23.mlp.experts.49.down_proj", + "model.layers.23.mlp.experts.49.gate_proj", + "model.layers.23.mlp.experts.49.up_proj", + "model.layers.23.mlp.experts.5.down_proj", + "model.layers.23.mlp.experts.5.gate_proj", + "model.layers.23.mlp.experts.5.up_proj", + "model.layers.23.mlp.experts.50.down_proj", + "model.layers.23.mlp.experts.50.gate_proj", + "model.layers.23.mlp.experts.50.up_proj", + "model.layers.23.mlp.experts.51.down_proj", + "model.layers.23.mlp.experts.51.gate_proj", + "model.layers.23.mlp.experts.51.up_proj", + "model.layers.23.mlp.experts.52.down_proj", + "model.layers.23.mlp.experts.52.gate_proj", + "model.layers.23.mlp.experts.52.up_proj", + "model.layers.23.mlp.experts.53.down_proj", + "model.layers.23.mlp.experts.53.gate_proj", + "model.layers.23.mlp.experts.53.up_proj", + "model.layers.23.mlp.experts.54.down_proj", + "model.layers.23.mlp.experts.54.gate_proj", + "model.layers.23.mlp.experts.54.up_proj", + "model.layers.23.mlp.experts.55.down_proj", + "model.layers.23.mlp.experts.55.gate_proj", + "model.layers.23.mlp.experts.55.up_proj", + "model.layers.23.mlp.experts.56.down_proj", + "model.layers.23.mlp.experts.56.gate_proj", + "model.layers.23.mlp.experts.56.up_proj", + "model.layers.23.mlp.experts.57.down_proj", + "model.layers.23.mlp.experts.57.gate_proj", + "model.layers.23.mlp.experts.57.up_proj", + "model.layers.23.mlp.experts.58.down_proj", + "model.layers.23.mlp.experts.58.gate_proj", + "model.layers.23.mlp.experts.58.up_proj", + "model.layers.23.mlp.experts.59.down_proj", + "model.layers.23.mlp.experts.59.gate_proj", + "model.layers.23.mlp.experts.59.up_proj", + "model.layers.23.mlp.experts.6.down_proj", + "model.layers.23.mlp.experts.6.gate_proj", + "model.layers.23.mlp.experts.6.up_proj", + "model.layers.23.mlp.experts.60.down_proj", + "model.layers.23.mlp.experts.60.gate_proj", + "model.layers.23.mlp.experts.60.up_proj", + "model.layers.23.mlp.experts.61.down_proj", + "model.layers.23.mlp.experts.61.gate_proj", + "model.layers.23.mlp.experts.61.up_proj", + "model.layers.23.mlp.experts.62.down_proj", + "model.layers.23.mlp.experts.62.gate_proj", + "model.layers.23.mlp.experts.62.up_proj", + "model.layers.23.mlp.experts.63.down_proj", + "model.layers.23.mlp.experts.63.gate_proj", + "model.layers.23.mlp.experts.63.up_proj", + "model.layers.23.mlp.experts.64.down_proj", + "model.layers.23.mlp.experts.64.gate_proj", + "model.layers.23.mlp.experts.64.up_proj", + "model.layers.23.mlp.experts.65.down_proj", + "model.layers.23.mlp.experts.65.gate_proj", + "model.layers.23.mlp.experts.65.up_proj", + "model.layers.23.mlp.experts.66.down_proj", + "model.layers.23.mlp.experts.66.gate_proj", + "model.layers.23.mlp.experts.66.up_proj", + "model.layers.23.mlp.experts.67.down_proj", + "model.layers.23.mlp.experts.67.gate_proj", + "model.layers.23.mlp.experts.67.up_proj", + "model.layers.23.mlp.experts.68.down_proj", + "model.layers.23.mlp.experts.68.gate_proj", + "model.layers.23.mlp.experts.68.up_proj", + "model.layers.23.mlp.experts.69.down_proj", + "model.layers.23.mlp.experts.69.gate_proj", + "model.layers.23.mlp.experts.69.up_proj", + "model.layers.23.mlp.experts.7.down_proj", + "model.layers.23.mlp.experts.7.gate_proj", + "model.layers.23.mlp.experts.7.up_proj", + "model.layers.23.mlp.experts.70.down_proj", + "model.layers.23.mlp.experts.70.gate_proj", + "model.layers.23.mlp.experts.70.up_proj", + "model.layers.23.mlp.experts.71.down_proj", + "model.layers.23.mlp.experts.71.gate_proj", + "model.layers.23.mlp.experts.71.up_proj", + "model.layers.23.mlp.experts.72.down_proj", + "model.layers.23.mlp.experts.72.gate_proj", + "model.layers.23.mlp.experts.72.up_proj", + "model.layers.23.mlp.experts.73.down_proj", + "model.layers.23.mlp.experts.73.gate_proj", + "model.layers.23.mlp.experts.73.up_proj", + "model.layers.23.mlp.experts.74.down_proj", + "model.layers.23.mlp.experts.74.gate_proj", + "model.layers.23.mlp.experts.74.up_proj", + "model.layers.23.mlp.experts.75.down_proj", + "model.layers.23.mlp.experts.75.gate_proj", + "model.layers.23.mlp.experts.75.up_proj", + "model.layers.23.mlp.experts.76.down_proj", + "model.layers.23.mlp.experts.76.gate_proj", + "model.layers.23.mlp.experts.76.up_proj", + "model.layers.23.mlp.experts.77.down_proj", + "model.layers.23.mlp.experts.77.gate_proj", + "model.layers.23.mlp.experts.77.up_proj", + "model.layers.23.mlp.experts.78.down_proj", + "model.layers.23.mlp.experts.78.gate_proj", + "model.layers.23.mlp.experts.78.up_proj", + "model.layers.23.mlp.experts.79.down_proj", + "model.layers.23.mlp.experts.79.gate_proj", + "model.layers.23.mlp.experts.79.up_proj", + "model.layers.23.mlp.experts.8.down_proj", + "model.layers.23.mlp.experts.8.gate_proj", + "model.layers.23.mlp.experts.8.up_proj", + "model.layers.23.mlp.experts.80.down_proj", + "model.layers.23.mlp.experts.80.gate_proj", + "model.layers.23.mlp.experts.80.up_proj", + "model.layers.23.mlp.experts.81.down_proj", + "model.layers.23.mlp.experts.81.gate_proj", + "model.layers.23.mlp.experts.81.up_proj", + "model.layers.23.mlp.experts.82.down_proj", + "model.layers.23.mlp.experts.82.gate_proj", + "model.layers.23.mlp.experts.82.up_proj", + "model.layers.23.mlp.experts.83.down_proj", + "model.layers.23.mlp.experts.83.gate_proj", + "model.layers.23.mlp.experts.83.up_proj", + "model.layers.23.mlp.experts.84.down_proj", + "model.layers.23.mlp.experts.84.gate_proj", + "model.layers.23.mlp.experts.84.up_proj", + "model.layers.23.mlp.experts.85.down_proj", + "model.layers.23.mlp.experts.85.gate_proj", + "model.layers.23.mlp.experts.85.up_proj", + "model.layers.23.mlp.experts.86.down_proj", + "model.layers.23.mlp.experts.86.gate_proj", + "model.layers.23.mlp.experts.86.up_proj", + "model.layers.23.mlp.experts.87.down_proj", + "model.layers.23.mlp.experts.87.gate_proj", + "model.layers.23.mlp.experts.87.up_proj", + "model.layers.23.mlp.experts.88.down_proj", + "model.layers.23.mlp.experts.88.gate_proj", + "model.layers.23.mlp.experts.88.up_proj", + "model.layers.23.mlp.experts.89.down_proj", + "model.layers.23.mlp.experts.89.gate_proj", + "model.layers.23.mlp.experts.89.up_proj", + "model.layers.23.mlp.experts.9.down_proj", + "model.layers.23.mlp.experts.9.gate_proj", + "model.layers.23.mlp.experts.9.up_proj", + "model.layers.23.mlp.experts.90.down_proj", + "model.layers.23.mlp.experts.90.gate_proj", + "model.layers.23.mlp.experts.90.up_proj", + "model.layers.23.mlp.experts.91.down_proj", + "model.layers.23.mlp.experts.91.gate_proj", + "model.layers.23.mlp.experts.91.up_proj", + "model.layers.23.mlp.experts.92.down_proj", + "model.layers.23.mlp.experts.92.gate_proj", + "model.layers.23.mlp.experts.92.up_proj", + "model.layers.23.mlp.experts.93.down_proj", + "model.layers.23.mlp.experts.93.gate_proj", + "model.layers.23.mlp.experts.93.up_proj", + "model.layers.23.mlp.experts.94.down_proj", + "model.layers.23.mlp.experts.94.gate_proj", + "model.layers.23.mlp.experts.94.up_proj", + "model.layers.23.mlp.experts.95.down_proj", + "model.layers.23.mlp.experts.95.gate_proj", + "model.layers.23.mlp.experts.95.up_proj", + "model.layers.23.mlp.experts.96.down_proj", + "model.layers.23.mlp.experts.96.gate_proj", + "model.layers.23.mlp.experts.96.up_proj", + "model.layers.23.mlp.experts.97.down_proj", + "model.layers.23.mlp.experts.97.gate_proj", + "model.layers.23.mlp.experts.97.up_proj", + "model.layers.23.mlp.experts.98.down_proj", + "model.layers.23.mlp.experts.98.gate_proj", + "model.layers.23.mlp.experts.98.up_proj", + "model.layers.23.mlp.experts.99.down_proj", + "model.layers.23.mlp.experts.99.gate_proj", + "model.layers.23.mlp.experts.99.up_proj", + "model.layers.24.mlp.experts.0.down_proj", + "model.layers.24.mlp.experts.0.gate_proj", + "model.layers.24.mlp.experts.0.up_proj", + "model.layers.24.mlp.experts.1.down_proj", + "model.layers.24.mlp.experts.1.gate_proj", + "model.layers.24.mlp.experts.1.up_proj", + "model.layers.24.mlp.experts.10.down_proj", + "model.layers.24.mlp.experts.10.gate_proj", + "model.layers.24.mlp.experts.10.up_proj", + "model.layers.24.mlp.experts.100.down_proj", + "model.layers.24.mlp.experts.100.gate_proj", + "model.layers.24.mlp.experts.100.up_proj", + "model.layers.24.mlp.experts.101.down_proj", + "model.layers.24.mlp.experts.101.gate_proj", + "model.layers.24.mlp.experts.101.up_proj", + "model.layers.24.mlp.experts.102.down_proj", + "model.layers.24.mlp.experts.102.gate_proj", + "model.layers.24.mlp.experts.102.up_proj", + "model.layers.24.mlp.experts.103.down_proj", + "model.layers.24.mlp.experts.103.gate_proj", + "model.layers.24.mlp.experts.103.up_proj", + "model.layers.24.mlp.experts.104.down_proj", + "model.layers.24.mlp.experts.104.gate_proj", + "model.layers.24.mlp.experts.104.up_proj", + "model.layers.24.mlp.experts.105.down_proj", + "model.layers.24.mlp.experts.105.gate_proj", + "model.layers.24.mlp.experts.105.up_proj", + "model.layers.24.mlp.experts.106.down_proj", + "model.layers.24.mlp.experts.106.gate_proj", + "model.layers.24.mlp.experts.106.up_proj", + "model.layers.24.mlp.experts.107.down_proj", + "model.layers.24.mlp.experts.107.gate_proj", + "model.layers.24.mlp.experts.107.up_proj", + "model.layers.24.mlp.experts.108.down_proj", + "model.layers.24.mlp.experts.108.gate_proj", + "model.layers.24.mlp.experts.108.up_proj", + "model.layers.24.mlp.experts.109.down_proj", + "model.layers.24.mlp.experts.109.gate_proj", + "model.layers.24.mlp.experts.109.up_proj", + "model.layers.24.mlp.experts.11.down_proj", + "model.layers.24.mlp.experts.11.gate_proj", + "model.layers.24.mlp.experts.11.up_proj", + "model.layers.24.mlp.experts.110.down_proj", + "model.layers.24.mlp.experts.110.gate_proj", + "model.layers.24.mlp.experts.110.up_proj", + "model.layers.24.mlp.experts.111.down_proj", + "model.layers.24.mlp.experts.111.gate_proj", + "model.layers.24.mlp.experts.111.up_proj", + "model.layers.24.mlp.experts.112.down_proj", + "model.layers.24.mlp.experts.112.gate_proj", + "model.layers.24.mlp.experts.112.up_proj", + "model.layers.24.mlp.experts.113.down_proj", + "model.layers.24.mlp.experts.113.gate_proj", + "model.layers.24.mlp.experts.113.up_proj", + "model.layers.24.mlp.experts.114.down_proj", + "model.layers.24.mlp.experts.114.gate_proj", + "model.layers.24.mlp.experts.114.up_proj", + "model.layers.24.mlp.experts.115.down_proj", + "model.layers.24.mlp.experts.115.gate_proj", + "model.layers.24.mlp.experts.115.up_proj", + "model.layers.24.mlp.experts.116.down_proj", + "model.layers.24.mlp.experts.116.gate_proj", + "model.layers.24.mlp.experts.116.up_proj", + "model.layers.24.mlp.experts.117.down_proj", + "model.layers.24.mlp.experts.117.gate_proj", + "model.layers.24.mlp.experts.117.up_proj", + "model.layers.24.mlp.experts.118.down_proj", + "model.layers.24.mlp.experts.118.gate_proj", + "model.layers.24.mlp.experts.118.up_proj", + "model.layers.24.mlp.experts.119.down_proj", + "model.layers.24.mlp.experts.119.gate_proj", + "model.layers.24.mlp.experts.119.up_proj", + "model.layers.24.mlp.experts.12.down_proj", + "model.layers.24.mlp.experts.12.gate_proj", + "model.layers.24.mlp.experts.12.up_proj", + "model.layers.24.mlp.experts.120.down_proj", + "model.layers.24.mlp.experts.120.gate_proj", + "model.layers.24.mlp.experts.120.up_proj", + "model.layers.24.mlp.experts.121.down_proj", + "model.layers.24.mlp.experts.121.gate_proj", + "model.layers.24.mlp.experts.121.up_proj", + "model.layers.24.mlp.experts.122.down_proj", + "model.layers.24.mlp.experts.122.gate_proj", + "model.layers.24.mlp.experts.122.up_proj", + "model.layers.24.mlp.experts.123.down_proj", + "model.layers.24.mlp.experts.123.gate_proj", + "model.layers.24.mlp.experts.123.up_proj", + "model.layers.24.mlp.experts.124.down_proj", + "model.layers.24.mlp.experts.124.gate_proj", + "model.layers.24.mlp.experts.124.up_proj", + "model.layers.24.mlp.experts.125.down_proj", + "model.layers.24.mlp.experts.125.gate_proj", + "model.layers.24.mlp.experts.125.up_proj", + "model.layers.24.mlp.experts.126.down_proj", + "model.layers.24.mlp.experts.126.gate_proj", + "model.layers.24.mlp.experts.126.up_proj", + "model.layers.24.mlp.experts.127.down_proj", + "model.layers.24.mlp.experts.127.gate_proj", + "model.layers.24.mlp.experts.127.up_proj", + "model.layers.24.mlp.experts.128.down_proj", + "model.layers.24.mlp.experts.128.gate_proj", + "model.layers.24.mlp.experts.128.up_proj", + "model.layers.24.mlp.experts.129.down_proj", + "model.layers.24.mlp.experts.129.gate_proj", + "model.layers.24.mlp.experts.129.up_proj", + "model.layers.24.mlp.experts.13.down_proj", + "model.layers.24.mlp.experts.13.gate_proj", + "model.layers.24.mlp.experts.13.up_proj", + "model.layers.24.mlp.experts.130.down_proj", + "model.layers.24.mlp.experts.130.gate_proj", + "model.layers.24.mlp.experts.130.up_proj", + "model.layers.24.mlp.experts.131.down_proj", + "model.layers.24.mlp.experts.131.gate_proj", + "model.layers.24.mlp.experts.131.up_proj", + "model.layers.24.mlp.experts.132.down_proj", + "model.layers.24.mlp.experts.132.gate_proj", + "model.layers.24.mlp.experts.132.up_proj", + "model.layers.24.mlp.experts.133.down_proj", + "model.layers.24.mlp.experts.133.gate_proj", + "model.layers.24.mlp.experts.133.up_proj", + "model.layers.24.mlp.experts.134.down_proj", + "model.layers.24.mlp.experts.134.gate_proj", + "model.layers.24.mlp.experts.134.up_proj", + "model.layers.24.mlp.experts.135.down_proj", + "model.layers.24.mlp.experts.135.gate_proj", + "model.layers.24.mlp.experts.135.up_proj", + "model.layers.24.mlp.experts.136.down_proj", + "model.layers.24.mlp.experts.136.gate_proj", + "model.layers.24.mlp.experts.136.up_proj", + "model.layers.24.mlp.experts.137.down_proj", + "model.layers.24.mlp.experts.137.gate_proj", + "model.layers.24.mlp.experts.137.up_proj", + "model.layers.24.mlp.experts.138.down_proj", + "model.layers.24.mlp.experts.138.gate_proj", + "model.layers.24.mlp.experts.138.up_proj", + "model.layers.24.mlp.experts.139.down_proj", + "model.layers.24.mlp.experts.139.gate_proj", + "model.layers.24.mlp.experts.139.up_proj", + "model.layers.24.mlp.experts.14.down_proj", + "model.layers.24.mlp.experts.14.gate_proj", + "model.layers.24.mlp.experts.14.up_proj", + "model.layers.24.mlp.experts.140.down_proj", + "model.layers.24.mlp.experts.140.gate_proj", + "model.layers.24.mlp.experts.140.up_proj", + "model.layers.24.mlp.experts.141.down_proj", + "model.layers.24.mlp.experts.141.gate_proj", + "model.layers.24.mlp.experts.141.up_proj", + "model.layers.24.mlp.experts.142.down_proj", + "model.layers.24.mlp.experts.142.gate_proj", + "model.layers.24.mlp.experts.142.up_proj", + "model.layers.24.mlp.experts.143.down_proj", + "model.layers.24.mlp.experts.143.gate_proj", + "model.layers.24.mlp.experts.143.up_proj", + "model.layers.24.mlp.experts.144.down_proj", + "model.layers.24.mlp.experts.144.gate_proj", + "model.layers.24.mlp.experts.144.up_proj", + "model.layers.24.mlp.experts.145.down_proj", + "model.layers.24.mlp.experts.145.gate_proj", + "model.layers.24.mlp.experts.145.up_proj", + "model.layers.24.mlp.experts.146.down_proj", + "model.layers.24.mlp.experts.146.gate_proj", + "model.layers.24.mlp.experts.146.up_proj", + "model.layers.24.mlp.experts.147.down_proj", + "model.layers.24.mlp.experts.147.gate_proj", + "model.layers.24.mlp.experts.147.up_proj", + "model.layers.24.mlp.experts.148.down_proj", + "model.layers.24.mlp.experts.148.gate_proj", + "model.layers.24.mlp.experts.148.up_proj", + "model.layers.24.mlp.experts.149.down_proj", + "model.layers.24.mlp.experts.149.gate_proj", + "model.layers.24.mlp.experts.149.up_proj", + "model.layers.24.mlp.experts.15.down_proj", + "model.layers.24.mlp.experts.15.gate_proj", + "model.layers.24.mlp.experts.15.up_proj", + "model.layers.24.mlp.experts.150.down_proj", + "model.layers.24.mlp.experts.150.gate_proj", + "model.layers.24.mlp.experts.150.up_proj", + "model.layers.24.mlp.experts.151.down_proj", + "model.layers.24.mlp.experts.151.gate_proj", + "model.layers.24.mlp.experts.151.up_proj", + "model.layers.24.mlp.experts.152.down_proj", + "model.layers.24.mlp.experts.152.gate_proj", + "model.layers.24.mlp.experts.152.up_proj", + "model.layers.24.mlp.experts.153.down_proj", + "model.layers.24.mlp.experts.153.gate_proj", + "model.layers.24.mlp.experts.153.up_proj", + "model.layers.24.mlp.experts.154.down_proj", + "model.layers.24.mlp.experts.154.gate_proj", + "model.layers.24.mlp.experts.154.up_proj", + "model.layers.24.mlp.experts.155.down_proj", + "model.layers.24.mlp.experts.155.gate_proj", + "model.layers.24.mlp.experts.155.up_proj", + "model.layers.24.mlp.experts.156.down_proj", + "model.layers.24.mlp.experts.156.gate_proj", + "model.layers.24.mlp.experts.156.up_proj", + "model.layers.24.mlp.experts.157.down_proj", + "model.layers.24.mlp.experts.157.gate_proj", + "model.layers.24.mlp.experts.157.up_proj", + "model.layers.24.mlp.experts.158.down_proj", + "model.layers.24.mlp.experts.158.gate_proj", + "model.layers.24.mlp.experts.158.up_proj", + "model.layers.24.mlp.experts.159.down_proj", + "model.layers.24.mlp.experts.159.gate_proj", + "model.layers.24.mlp.experts.159.up_proj", + "model.layers.24.mlp.experts.16.down_proj", + "model.layers.24.mlp.experts.16.gate_proj", + "model.layers.24.mlp.experts.16.up_proj", + "model.layers.24.mlp.experts.160.down_proj", + "model.layers.24.mlp.experts.160.gate_proj", + "model.layers.24.mlp.experts.160.up_proj", + "model.layers.24.mlp.experts.161.down_proj", + "model.layers.24.mlp.experts.161.gate_proj", + "model.layers.24.mlp.experts.161.up_proj", + "model.layers.24.mlp.experts.162.down_proj", + "model.layers.24.mlp.experts.162.gate_proj", + "model.layers.24.mlp.experts.162.up_proj", + "model.layers.24.mlp.experts.163.down_proj", + "model.layers.24.mlp.experts.163.gate_proj", + "model.layers.24.mlp.experts.163.up_proj", + "model.layers.24.mlp.experts.164.down_proj", + "model.layers.24.mlp.experts.164.gate_proj", + "model.layers.24.mlp.experts.164.up_proj", + "model.layers.24.mlp.experts.165.down_proj", + "model.layers.24.mlp.experts.165.gate_proj", + "model.layers.24.mlp.experts.165.up_proj", + "model.layers.24.mlp.experts.166.down_proj", + "model.layers.24.mlp.experts.166.gate_proj", + "model.layers.24.mlp.experts.166.up_proj", + "model.layers.24.mlp.experts.167.down_proj", + "model.layers.24.mlp.experts.167.gate_proj", + "model.layers.24.mlp.experts.167.up_proj", + "model.layers.24.mlp.experts.168.down_proj", + "model.layers.24.mlp.experts.168.gate_proj", + "model.layers.24.mlp.experts.168.up_proj", + "model.layers.24.mlp.experts.169.down_proj", + "model.layers.24.mlp.experts.169.gate_proj", + "model.layers.24.mlp.experts.169.up_proj", + "model.layers.24.mlp.experts.17.down_proj", + "model.layers.24.mlp.experts.17.gate_proj", + "model.layers.24.mlp.experts.17.up_proj", + "model.layers.24.mlp.experts.170.down_proj", + "model.layers.24.mlp.experts.170.gate_proj", + "model.layers.24.mlp.experts.170.up_proj", + "model.layers.24.mlp.experts.171.down_proj", + "model.layers.24.mlp.experts.171.gate_proj", + "model.layers.24.mlp.experts.171.up_proj", + "model.layers.24.mlp.experts.172.down_proj", + "model.layers.24.mlp.experts.172.gate_proj", + "model.layers.24.mlp.experts.172.up_proj", + "model.layers.24.mlp.experts.173.down_proj", + "model.layers.24.mlp.experts.173.gate_proj", + "model.layers.24.mlp.experts.173.up_proj", + "model.layers.24.mlp.experts.174.down_proj", + "model.layers.24.mlp.experts.174.gate_proj", + "model.layers.24.mlp.experts.174.up_proj", + "model.layers.24.mlp.experts.175.down_proj", + "model.layers.24.mlp.experts.175.gate_proj", + "model.layers.24.mlp.experts.175.up_proj", + "model.layers.24.mlp.experts.176.down_proj", + "model.layers.24.mlp.experts.176.gate_proj", + "model.layers.24.mlp.experts.176.up_proj", + "model.layers.24.mlp.experts.177.down_proj", + "model.layers.24.mlp.experts.177.gate_proj", + "model.layers.24.mlp.experts.177.up_proj", + "model.layers.24.mlp.experts.178.down_proj", + "model.layers.24.mlp.experts.178.gate_proj", + "model.layers.24.mlp.experts.178.up_proj", + "model.layers.24.mlp.experts.179.down_proj", + "model.layers.24.mlp.experts.179.gate_proj", + "model.layers.24.mlp.experts.179.up_proj", + "model.layers.24.mlp.experts.18.down_proj", + "model.layers.24.mlp.experts.18.gate_proj", + "model.layers.24.mlp.experts.18.up_proj", + "model.layers.24.mlp.experts.180.down_proj", + "model.layers.24.mlp.experts.180.gate_proj", + "model.layers.24.mlp.experts.180.up_proj", + "model.layers.24.mlp.experts.181.down_proj", + "model.layers.24.mlp.experts.181.gate_proj", + "model.layers.24.mlp.experts.181.up_proj", + "model.layers.24.mlp.experts.182.down_proj", + "model.layers.24.mlp.experts.182.gate_proj", + "model.layers.24.mlp.experts.182.up_proj", + "model.layers.24.mlp.experts.183.down_proj", + "model.layers.24.mlp.experts.183.gate_proj", + "model.layers.24.mlp.experts.183.up_proj", + "model.layers.24.mlp.experts.184.down_proj", + "model.layers.24.mlp.experts.184.gate_proj", + "model.layers.24.mlp.experts.184.up_proj", + "model.layers.24.mlp.experts.185.down_proj", + "model.layers.24.mlp.experts.185.gate_proj", + "model.layers.24.mlp.experts.185.up_proj", + "model.layers.24.mlp.experts.186.down_proj", + "model.layers.24.mlp.experts.186.gate_proj", + "model.layers.24.mlp.experts.186.up_proj", + "model.layers.24.mlp.experts.187.down_proj", + "model.layers.24.mlp.experts.187.gate_proj", + "model.layers.24.mlp.experts.187.up_proj", + "model.layers.24.mlp.experts.188.down_proj", + "model.layers.24.mlp.experts.188.gate_proj", + "model.layers.24.mlp.experts.188.up_proj", + "model.layers.24.mlp.experts.189.down_proj", + "model.layers.24.mlp.experts.189.gate_proj", + "model.layers.24.mlp.experts.189.up_proj", + "model.layers.24.mlp.experts.19.down_proj", + "model.layers.24.mlp.experts.19.gate_proj", + "model.layers.24.mlp.experts.19.up_proj", + "model.layers.24.mlp.experts.190.down_proj", + "model.layers.24.mlp.experts.190.gate_proj", + "model.layers.24.mlp.experts.190.up_proj", + "model.layers.24.mlp.experts.191.down_proj", + "model.layers.24.mlp.experts.191.gate_proj", + "model.layers.24.mlp.experts.191.up_proj", + "model.layers.24.mlp.experts.192.down_proj", + "model.layers.24.mlp.experts.192.gate_proj", + "model.layers.24.mlp.experts.192.up_proj", + "model.layers.24.mlp.experts.193.down_proj", + "model.layers.24.mlp.experts.193.gate_proj", + "model.layers.24.mlp.experts.193.up_proj", + "model.layers.24.mlp.experts.194.down_proj", + "model.layers.24.mlp.experts.194.gate_proj", + "model.layers.24.mlp.experts.194.up_proj", + "model.layers.24.mlp.experts.195.down_proj", + "model.layers.24.mlp.experts.195.gate_proj", + "model.layers.24.mlp.experts.195.up_proj", + "model.layers.24.mlp.experts.196.down_proj", + "model.layers.24.mlp.experts.196.gate_proj", + "model.layers.24.mlp.experts.196.up_proj", + "model.layers.24.mlp.experts.197.down_proj", + "model.layers.24.mlp.experts.197.gate_proj", + "model.layers.24.mlp.experts.197.up_proj", + "model.layers.24.mlp.experts.198.down_proj", + "model.layers.24.mlp.experts.198.gate_proj", + "model.layers.24.mlp.experts.198.up_proj", + "model.layers.24.mlp.experts.199.down_proj", + "model.layers.24.mlp.experts.199.gate_proj", + "model.layers.24.mlp.experts.199.up_proj", + "model.layers.24.mlp.experts.2.down_proj", + "model.layers.24.mlp.experts.2.gate_proj", + "model.layers.24.mlp.experts.2.up_proj", + "model.layers.24.mlp.experts.20.down_proj", + "model.layers.24.mlp.experts.20.gate_proj", + "model.layers.24.mlp.experts.20.up_proj", + "model.layers.24.mlp.experts.200.down_proj", + "model.layers.24.mlp.experts.200.gate_proj", + "model.layers.24.mlp.experts.200.up_proj", + "model.layers.24.mlp.experts.201.down_proj", + "model.layers.24.mlp.experts.201.gate_proj", + "model.layers.24.mlp.experts.201.up_proj", + "model.layers.24.mlp.experts.202.down_proj", + "model.layers.24.mlp.experts.202.gate_proj", + "model.layers.24.mlp.experts.202.up_proj", + "model.layers.24.mlp.experts.203.down_proj", + "model.layers.24.mlp.experts.203.gate_proj", + "model.layers.24.mlp.experts.203.up_proj", + "model.layers.24.mlp.experts.204.down_proj", + "model.layers.24.mlp.experts.204.gate_proj", + "model.layers.24.mlp.experts.204.up_proj", + "model.layers.24.mlp.experts.205.down_proj", + "model.layers.24.mlp.experts.205.gate_proj", + "model.layers.24.mlp.experts.205.up_proj", + "model.layers.24.mlp.experts.206.down_proj", + "model.layers.24.mlp.experts.206.gate_proj", + "model.layers.24.mlp.experts.206.up_proj", + "model.layers.24.mlp.experts.207.down_proj", + "model.layers.24.mlp.experts.207.gate_proj", + "model.layers.24.mlp.experts.207.up_proj", + "model.layers.24.mlp.experts.208.down_proj", + "model.layers.24.mlp.experts.208.gate_proj", + "model.layers.24.mlp.experts.208.up_proj", + "model.layers.24.mlp.experts.209.down_proj", + "model.layers.24.mlp.experts.209.gate_proj", + "model.layers.24.mlp.experts.209.up_proj", + "model.layers.24.mlp.experts.21.down_proj", + "model.layers.24.mlp.experts.21.gate_proj", + "model.layers.24.mlp.experts.21.up_proj", + "model.layers.24.mlp.experts.210.down_proj", + "model.layers.24.mlp.experts.210.gate_proj", + "model.layers.24.mlp.experts.210.up_proj", + "model.layers.24.mlp.experts.211.down_proj", + "model.layers.24.mlp.experts.211.gate_proj", + "model.layers.24.mlp.experts.211.up_proj", + "model.layers.24.mlp.experts.212.down_proj", + "model.layers.24.mlp.experts.212.gate_proj", + "model.layers.24.mlp.experts.212.up_proj", + "model.layers.24.mlp.experts.213.down_proj", + "model.layers.24.mlp.experts.213.gate_proj", + "model.layers.24.mlp.experts.213.up_proj", + "model.layers.24.mlp.experts.214.down_proj", + "model.layers.24.mlp.experts.214.gate_proj", + "model.layers.24.mlp.experts.214.up_proj", + "model.layers.24.mlp.experts.215.down_proj", + "model.layers.24.mlp.experts.215.gate_proj", + "model.layers.24.mlp.experts.215.up_proj", + "model.layers.24.mlp.experts.216.down_proj", + "model.layers.24.mlp.experts.216.gate_proj", + "model.layers.24.mlp.experts.216.up_proj", + "model.layers.24.mlp.experts.217.down_proj", + "model.layers.24.mlp.experts.217.gate_proj", + "model.layers.24.mlp.experts.217.up_proj", + "model.layers.24.mlp.experts.218.down_proj", + "model.layers.24.mlp.experts.218.gate_proj", + "model.layers.24.mlp.experts.218.up_proj", + "model.layers.24.mlp.experts.219.down_proj", + "model.layers.24.mlp.experts.219.gate_proj", + "model.layers.24.mlp.experts.219.up_proj", + "model.layers.24.mlp.experts.22.down_proj", + "model.layers.24.mlp.experts.22.gate_proj", + "model.layers.24.mlp.experts.22.up_proj", + "model.layers.24.mlp.experts.220.down_proj", + "model.layers.24.mlp.experts.220.gate_proj", + "model.layers.24.mlp.experts.220.up_proj", + "model.layers.24.mlp.experts.221.down_proj", + "model.layers.24.mlp.experts.221.gate_proj", + "model.layers.24.mlp.experts.221.up_proj", + "model.layers.24.mlp.experts.222.down_proj", + "model.layers.24.mlp.experts.222.gate_proj", + "model.layers.24.mlp.experts.222.up_proj", + "model.layers.24.mlp.experts.223.down_proj", + "model.layers.24.mlp.experts.223.gate_proj", + "model.layers.24.mlp.experts.223.up_proj", + "model.layers.24.mlp.experts.224.down_proj", + "model.layers.24.mlp.experts.224.gate_proj", + "model.layers.24.mlp.experts.224.up_proj", + "model.layers.24.mlp.experts.225.down_proj", + "model.layers.24.mlp.experts.225.gate_proj", + "model.layers.24.mlp.experts.225.up_proj", + "model.layers.24.mlp.experts.226.down_proj", + "model.layers.24.mlp.experts.226.gate_proj", + "model.layers.24.mlp.experts.226.up_proj", + "model.layers.24.mlp.experts.227.down_proj", + "model.layers.24.mlp.experts.227.gate_proj", + "model.layers.24.mlp.experts.227.up_proj", + "model.layers.24.mlp.experts.228.down_proj", + "model.layers.24.mlp.experts.228.gate_proj", + "model.layers.24.mlp.experts.228.up_proj", + "model.layers.24.mlp.experts.229.down_proj", + "model.layers.24.mlp.experts.229.gate_proj", + "model.layers.24.mlp.experts.229.up_proj", + "model.layers.24.mlp.experts.23.down_proj", + "model.layers.24.mlp.experts.23.gate_proj", + "model.layers.24.mlp.experts.23.up_proj", + "model.layers.24.mlp.experts.230.down_proj", + "model.layers.24.mlp.experts.230.gate_proj", + "model.layers.24.mlp.experts.230.up_proj", + "model.layers.24.mlp.experts.231.down_proj", + "model.layers.24.mlp.experts.231.gate_proj", + "model.layers.24.mlp.experts.231.up_proj", + "model.layers.24.mlp.experts.232.down_proj", + "model.layers.24.mlp.experts.232.gate_proj", + "model.layers.24.mlp.experts.232.up_proj", + "model.layers.24.mlp.experts.233.down_proj", + "model.layers.24.mlp.experts.233.gate_proj", + "model.layers.24.mlp.experts.233.up_proj", + "model.layers.24.mlp.experts.234.down_proj", + "model.layers.24.mlp.experts.234.gate_proj", + "model.layers.24.mlp.experts.234.up_proj", + "model.layers.24.mlp.experts.235.down_proj", + "model.layers.24.mlp.experts.235.gate_proj", + "model.layers.24.mlp.experts.235.up_proj", + "model.layers.24.mlp.experts.236.down_proj", + "model.layers.24.mlp.experts.236.gate_proj", + "model.layers.24.mlp.experts.236.up_proj", + "model.layers.24.mlp.experts.237.down_proj", + "model.layers.24.mlp.experts.237.gate_proj", + "model.layers.24.mlp.experts.237.up_proj", + "model.layers.24.mlp.experts.238.down_proj", + "model.layers.24.mlp.experts.238.gate_proj", + "model.layers.24.mlp.experts.238.up_proj", + "model.layers.24.mlp.experts.239.down_proj", + "model.layers.24.mlp.experts.239.gate_proj", + "model.layers.24.mlp.experts.239.up_proj", + "model.layers.24.mlp.experts.24.down_proj", + "model.layers.24.mlp.experts.24.gate_proj", + "model.layers.24.mlp.experts.24.up_proj", + "model.layers.24.mlp.experts.240.down_proj", + "model.layers.24.mlp.experts.240.gate_proj", + "model.layers.24.mlp.experts.240.up_proj", + "model.layers.24.mlp.experts.241.down_proj", + "model.layers.24.mlp.experts.241.gate_proj", + "model.layers.24.mlp.experts.241.up_proj", + "model.layers.24.mlp.experts.242.down_proj", + "model.layers.24.mlp.experts.242.gate_proj", + "model.layers.24.mlp.experts.242.up_proj", + "model.layers.24.mlp.experts.243.down_proj", + "model.layers.24.mlp.experts.243.gate_proj", + "model.layers.24.mlp.experts.243.up_proj", + "model.layers.24.mlp.experts.244.down_proj", + "model.layers.24.mlp.experts.244.gate_proj", + "model.layers.24.mlp.experts.244.up_proj", + "model.layers.24.mlp.experts.245.down_proj", + "model.layers.24.mlp.experts.245.gate_proj", + "model.layers.24.mlp.experts.245.up_proj", + "model.layers.24.mlp.experts.246.down_proj", + "model.layers.24.mlp.experts.246.gate_proj", + "model.layers.24.mlp.experts.246.up_proj", + "model.layers.24.mlp.experts.247.down_proj", + "model.layers.24.mlp.experts.247.gate_proj", + "model.layers.24.mlp.experts.247.up_proj", + "model.layers.24.mlp.experts.248.down_proj", + "model.layers.24.mlp.experts.248.gate_proj", + "model.layers.24.mlp.experts.248.up_proj", + "model.layers.24.mlp.experts.249.down_proj", + "model.layers.24.mlp.experts.249.gate_proj", + "model.layers.24.mlp.experts.249.up_proj", + "model.layers.24.mlp.experts.25.down_proj", + "model.layers.24.mlp.experts.25.gate_proj", + "model.layers.24.mlp.experts.25.up_proj", + "model.layers.24.mlp.experts.250.down_proj", + "model.layers.24.mlp.experts.250.gate_proj", + "model.layers.24.mlp.experts.250.up_proj", + "model.layers.24.mlp.experts.251.down_proj", + "model.layers.24.mlp.experts.251.gate_proj", + "model.layers.24.mlp.experts.251.up_proj", + "model.layers.24.mlp.experts.252.down_proj", + "model.layers.24.mlp.experts.252.gate_proj", + "model.layers.24.mlp.experts.252.up_proj", + "model.layers.24.mlp.experts.253.down_proj", + "model.layers.24.mlp.experts.253.gate_proj", + "model.layers.24.mlp.experts.253.up_proj", + "model.layers.24.mlp.experts.254.down_proj", + "model.layers.24.mlp.experts.254.gate_proj", + "model.layers.24.mlp.experts.254.up_proj", + "model.layers.24.mlp.experts.255.down_proj", + "model.layers.24.mlp.experts.255.gate_proj", + "model.layers.24.mlp.experts.255.up_proj", + "model.layers.24.mlp.experts.26.down_proj", + "model.layers.24.mlp.experts.26.gate_proj", + "model.layers.24.mlp.experts.26.up_proj", + "model.layers.24.mlp.experts.27.down_proj", + "model.layers.24.mlp.experts.27.gate_proj", + "model.layers.24.mlp.experts.27.up_proj", + "model.layers.24.mlp.experts.28.down_proj", + "model.layers.24.mlp.experts.28.gate_proj", + "model.layers.24.mlp.experts.28.up_proj", + "model.layers.24.mlp.experts.29.down_proj", + "model.layers.24.mlp.experts.29.gate_proj", + "model.layers.24.mlp.experts.29.up_proj", + "model.layers.24.mlp.experts.3.down_proj", + "model.layers.24.mlp.experts.3.gate_proj", + "model.layers.24.mlp.experts.3.up_proj", + "model.layers.24.mlp.experts.30.down_proj", + "model.layers.24.mlp.experts.30.gate_proj", + "model.layers.24.mlp.experts.30.up_proj", + "model.layers.24.mlp.experts.31.down_proj", + "model.layers.24.mlp.experts.31.gate_proj", + "model.layers.24.mlp.experts.31.up_proj", + "model.layers.24.mlp.experts.32.down_proj", + "model.layers.24.mlp.experts.32.gate_proj", + "model.layers.24.mlp.experts.32.up_proj", + "model.layers.24.mlp.experts.33.down_proj", + "model.layers.24.mlp.experts.33.gate_proj", + "model.layers.24.mlp.experts.33.up_proj", + "model.layers.24.mlp.experts.34.down_proj", + "model.layers.24.mlp.experts.34.gate_proj", + "model.layers.24.mlp.experts.34.up_proj", + "model.layers.24.mlp.experts.35.down_proj", + "model.layers.24.mlp.experts.35.gate_proj", + "model.layers.24.mlp.experts.35.up_proj", + "model.layers.24.mlp.experts.36.down_proj", + "model.layers.24.mlp.experts.36.gate_proj", + "model.layers.24.mlp.experts.36.up_proj", + "model.layers.24.mlp.experts.37.down_proj", + "model.layers.24.mlp.experts.37.gate_proj", + "model.layers.24.mlp.experts.37.up_proj", + "model.layers.24.mlp.experts.38.down_proj", + "model.layers.24.mlp.experts.38.gate_proj", + "model.layers.24.mlp.experts.38.up_proj", + "model.layers.24.mlp.experts.39.down_proj", + "model.layers.24.mlp.experts.39.gate_proj", + "model.layers.24.mlp.experts.39.up_proj", + "model.layers.24.mlp.experts.4.down_proj", + "model.layers.24.mlp.experts.4.gate_proj", + "model.layers.24.mlp.experts.4.up_proj", + "model.layers.24.mlp.experts.40.down_proj", + "model.layers.24.mlp.experts.40.gate_proj", + "model.layers.24.mlp.experts.40.up_proj", + "model.layers.24.mlp.experts.41.down_proj", + "model.layers.24.mlp.experts.41.gate_proj", + "model.layers.24.mlp.experts.41.up_proj", + "model.layers.24.mlp.experts.42.down_proj", + "model.layers.24.mlp.experts.42.gate_proj", + "model.layers.24.mlp.experts.42.up_proj", + "model.layers.24.mlp.experts.43.down_proj", + "model.layers.24.mlp.experts.43.gate_proj", + "model.layers.24.mlp.experts.43.up_proj", + "model.layers.24.mlp.experts.44.down_proj", + "model.layers.24.mlp.experts.44.gate_proj", + "model.layers.24.mlp.experts.44.up_proj", + "model.layers.24.mlp.experts.45.down_proj", + "model.layers.24.mlp.experts.45.gate_proj", + "model.layers.24.mlp.experts.45.up_proj", + "model.layers.24.mlp.experts.46.down_proj", + "model.layers.24.mlp.experts.46.gate_proj", + "model.layers.24.mlp.experts.46.up_proj", + "model.layers.24.mlp.experts.47.down_proj", + "model.layers.24.mlp.experts.47.gate_proj", + "model.layers.24.mlp.experts.47.up_proj", + "model.layers.24.mlp.experts.48.down_proj", + "model.layers.24.mlp.experts.48.gate_proj", + "model.layers.24.mlp.experts.48.up_proj", + "model.layers.24.mlp.experts.49.down_proj", + "model.layers.24.mlp.experts.49.gate_proj", + "model.layers.24.mlp.experts.49.up_proj", + "model.layers.24.mlp.experts.5.down_proj", + "model.layers.24.mlp.experts.5.gate_proj", + "model.layers.24.mlp.experts.5.up_proj", + "model.layers.24.mlp.experts.50.down_proj", + "model.layers.24.mlp.experts.50.gate_proj", + "model.layers.24.mlp.experts.50.up_proj", + "model.layers.24.mlp.experts.51.down_proj", + "model.layers.24.mlp.experts.51.gate_proj", + "model.layers.24.mlp.experts.51.up_proj", + "model.layers.24.mlp.experts.52.down_proj", + "model.layers.24.mlp.experts.52.gate_proj", + "model.layers.24.mlp.experts.52.up_proj", + "model.layers.24.mlp.experts.53.down_proj", + "model.layers.24.mlp.experts.53.gate_proj", + "model.layers.24.mlp.experts.53.up_proj", + "model.layers.24.mlp.experts.54.down_proj", + "model.layers.24.mlp.experts.54.gate_proj", + "model.layers.24.mlp.experts.54.up_proj", + "model.layers.24.mlp.experts.55.down_proj", + "model.layers.24.mlp.experts.55.gate_proj", + "model.layers.24.mlp.experts.55.up_proj", + "model.layers.24.mlp.experts.56.down_proj", + "model.layers.24.mlp.experts.56.gate_proj", + "model.layers.24.mlp.experts.56.up_proj", + "model.layers.24.mlp.experts.57.down_proj", + "model.layers.24.mlp.experts.57.gate_proj", + "model.layers.24.mlp.experts.57.up_proj", + "model.layers.24.mlp.experts.58.down_proj", + "model.layers.24.mlp.experts.58.gate_proj", + "model.layers.24.mlp.experts.58.up_proj", + "model.layers.24.mlp.experts.59.down_proj", + "model.layers.24.mlp.experts.59.gate_proj", + "model.layers.24.mlp.experts.59.up_proj", + "model.layers.24.mlp.experts.6.down_proj", + "model.layers.24.mlp.experts.6.gate_proj", + "model.layers.24.mlp.experts.6.up_proj", + "model.layers.24.mlp.experts.60.down_proj", + "model.layers.24.mlp.experts.60.gate_proj", + "model.layers.24.mlp.experts.60.up_proj", + "model.layers.24.mlp.experts.61.down_proj", + "model.layers.24.mlp.experts.61.gate_proj", + "model.layers.24.mlp.experts.61.up_proj", + "model.layers.24.mlp.experts.62.down_proj", + "model.layers.24.mlp.experts.62.gate_proj", + "model.layers.24.mlp.experts.62.up_proj", + "model.layers.24.mlp.experts.63.down_proj", + "model.layers.24.mlp.experts.63.gate_proj", + "model.layers.24.mlp.experts.63.up_proj", + "model.layers.24.mlp.experts.64.down_proj", + "model.layers.24.mlp.experts.64.gate_proj", + "model.layers.24.mlp.experts.64.up_proj", + "model.layers.24.mlp.experts.65.down_proj", + "model.layers.24.mlp.experts.65.gate_proj", + "model.layers.24.mlp.experts.65.up_proj", + "model.layers.24.mlp.experts.66.down_proj", + "model.layers.24.mlp.experts.66.gate_proj", + "model.layers.24.mlp.experts.66.up_proj", + "model.layers.24.mlp.experts.67.down_proj", + "model.layers.24.mlp.experts.67.gate_proj", + "model.layers.24.mlp.experts.67.up_proj", + "model.layers.24.mlp.experts.68.down_proj", + "model.layers.24.mlp.experts.68.gate_proj", + "model.layers.24.mlp.experts.68.up_proj", + "model.layers.24.mlp.experts.69.down_proj", + "model.layers.24.mlp.experts.69.gate_proj", + "model.layers.24.mlp.experts.69.up_proj", + "model.layers.24.mlp.experts.7.down_proj", + "model.layers.24.mlp.experts.7.gate_proj", + "model.layers.24.mlp.experts.7.up_proj", + "model.layers.24.mlp.experts.70.down_proj", + "model.layers.24.mlp.experts.70.gate_proj", + "model.layers.24.mlp.experts.70.up_proj", + "model.layers.24.mlp.experts.71.down_proj", + "model.layers.24.mlp.experts.71.gate_proj", + "model.layers.24.mlp.experts.71.up_proj", + "model.layers.24.mlp.experts.72.down_proj", + "model.layers.24.mlp.experts.72.gate_proj", + "model.layers.24.mlp.experts.72.up_proj", + "model.layers.24.mlp.experts.73.down_proj", + "model.layers.24.mlp.experts.73.gate_proj", + "model.layers.24.mlp.experts.73.up_proj", + "model.layers.24.mlp.experts.74.down_proj", + "model.layers.24.mlp.experts.74.gate_proj", + "model.layers.24.mlp.experts.74.up_proj", + "model.layers.24.mlp.experts.75.down_proj", + "model.layers.24.mlp.experts.75.gate_proj", + "model.layers.24.mlp.experts.75.up_proj", + "model.layers.24.mlp.experts.76.down_proj", + "model.layers.24.mlp.experts.76.gate_proj", + "model.layers.24.mlp.experts.76.up_proj", + "model.layers.24.mlp.experts.77.down_proj", + "model.layers.24.mlp.experts.77.gate_proj", + "model.layers.24.mlp.experts.77.up_proj", + "model.layers.24.mlp.experts.78.down_proj", + "model.layers.24.mlp.experts.78.gate_proj", + "model.layers.24.mlp.experts.78.up_proj", + "model.layers.24.mlp.experts.79.down_proj", + "model.layers.24.mlp.experts.79.gate_proj", + "model.layers.24.mlp.experts.79.up_proj", + "model.layers.24.mlp.experts.8.down_proj", + "model.layers.24.mlp.experts.8.gate_proj", + "model.layers.24.mlp.experts.8.up_proj", + "model.layers.24.mlp.experts.80.down_proj", + "model.layers.24.mlp.experts.80.gate_proj", + "model.layers.24.mlp.experts.80.up_proj", + "model.layers.24.mlp.experts.81.down_proj", + "model.layers.24.mlp.experts.81.gate_proj", + "model.layers.24.mlp.experts.81.up_proj", + "model.layers.24.mlp.experts.82.down_proj", + "model.layers.24.mlp.experts.82.gate_proj", + "model.layers.24.mlp.experts.82.up_proj", + "model.layers.24.mlp.experts.83.down_proj", + "model.layers.24.mlp.experts.83.gate_proj", + "model.layers.24.mlp.experts.83.up_proj", + "model.layers.24.mlp.experts.84.down_proj", + "model.layers.24.mlp.experts.84.gate_proj", + "model.layers.24.mlp.experts.84.up_proj", + "model.layers.24.mlp.experts.85.down_proj", + "model.layers.24.mlp.experts.85.gate_proj", + "model.layers.24.mlp.experts.85.up_proj", + "model.layers.24.mlp.experts.86.down_proj", + "model.layers.24.mlp.experts.86.gate_proj", + "model.layers.24.mlp.experts.86.up_proj", + "model.layers.24.mlp.experts.87.down_proj", + "model.layers.24.mlp.experts.87.gate_proj", + "model.layers.24.mlp.experts.87.up_proj", + "model.layers.24.mlp.experts.88.down_proj", + "model.layers.24.mlp.experts.88.gate_proj", + "model.layers.24.mlp.experts.88.up_proj", + "model.layers.24.mlp.experts.89.down_proj", + "model.layers.24.mlp.experts.89.gate_proj", + "model.layers.24.mlp.experts.89.up_proj", + "model.layers.24.mlp.experts.9.down_proj", + "model.layers.24.mlp.experts.9.gate_proj", + "model.layers.24.mlp.experts.9.up_proj", + "model.layers.24.mlp.experts.90.down_proj", + "model.layers.24.mlp.experts.90.gate_proj", + "model.layers.24.mlp.experts.90.up_proj", + "model.layers.24.mlp.experts.91.down_proj", + "model.layers.24.mlp.experts.91.gate_proj", + "model.layers.24.mlp.experts.91.up_proj", + "model.layers.24.mlp.experts.92.down_proj", + "model.layers.24.mlp.experts.92.gate_proj", + "model.layers.24.mlp.experts.92.up_proj", + "model.layers.24.mlp.experts.93.down_proj", + "model.layers.24.mlp.experts.93.gate_proj", + "model.layers.24.mlp.experts.93.up_proj", + "model.layers.24.mlp.experts.94.down_proj", + "model.layers.24.mlp.experts.94.gate_proj", + "model.layers.24.mlp.experts.94.up_proj", + "model.layers.24.mlp.experts.95.down_proj", + "model.layers.24.mlp.experts.95.gate_proj", + "model.layers.24.mlp.experts.95.up_proj", + "model.layers.24.mlp.experts.96.down_proj", + "model.layers.24.mlp.experts.96.gate_proj", + "model.layers.24.mlp.experts.96.up_proj", + "model.layers.24.mlp.experts.97.down_proj", + "model.layers.24.mlp.experts.97.gate_proj", + "model.layers.24.mlp.experts.97.up_proj", + "model.layers.24.mlp.experts.98.down_proj", + "model.layers.24.mlp.experts.98.gate_proj", + "model.layers.24.mlp.experts.98.up_proj", + "model.layers.24.mlp.experts.99.down_proj", + "model.layers.24.mlp.experts.99.gate_proj", + "model.layers.24.mlp.experts.99.up_proj", + "model.layers.25.mlp.experts.0.down_proj", + "model.layers.25.mlp.experts.0.gate_proj", + "model.layers.25.mlp.experts.0.up_proj", + "model.layers.25.mlp.experts.1.down_proj", + "model.layers.25.mlp.experts.1.gate_proj", + "model.layers.25.mlp.experts.1.up_proj", + "model.layers.25.mlp.experts.10.down_proj", + "model.layers.25.mlp.experts.10.gate_proj", + "model.layers.25.mlp.experts.10.up_proj", + "model.layers.25.mlp.experts.100.down_proj", + "model.layers.25.mlp.experts.100.gate_proj", + "model.layers.25.mlp.experts.100.up_proj", + "model.layers.25.mlp.experts.101.down_proj", + "model.layers.25.mlp.experts.101.gate_proj", + "model.layers.25.mlp.experts.101.up_proj", + "model.layers.25.mlp.experts.102.down_proj", + "model.layers.25.mlp.experts.102.gate_proj", + "model.layers.25.mlp.experts.102.up_proj", + "model.layers.25.mlp.experts.103.down_proj", + "model.layers.25.mlp.experts.103.gate_proj", + "model.layers.25.mlp.experts.103.up_proj", + "model.layers.25.mlp.experts.104.down_proj", + "model.layers.25.mlp.experts.104.gate_proj", + "model.layers.25.mlp.experts.104.up_proj", + "model.layers.25.mlp.experts.105.down_proj", + "model.layers.25.mlp.experts.105.gate_proj", + "model.layers.25.mlp.experts.105.up_proj", + "model.layers.25.mlp.experts.106.down_proj", + "model.layers.25.mlp.experts.106.gate_proj", + "model.layers.25.mlp.experts.106.up_proj", + "model.layers.25.mlp.experts.107.down_proj", + "model.layers.25.mlp.experts.107.gate_proj", + "model.layers.25.mlp.experts.107.up_proj", + "model.layers.25.mlp.experts.108.down_proj", + "model.layers.25.mlp.experts.108.gate_proj", + "model.layers.25.mlp.experts.108.up_proj", + "model.layers.25.mlp.experts.109.down_proj", + "model.layers.25.mlp.experts.109.gate_proj", + "model.layers.25.mlp.experts.109.up_proj", + "model.layers.25.mlp.experts.11.down_proj", + "model.layers.25.mlp.experts.11.gate_proj", + "model.layers.25.mlp.experts.11.up_proj", + "model.layers.25.mlp.experts.110.down_proj", + "model.layers.25.mlp.experts.110.gate_proj", + "model.layers.25.mlp.experts.110.up_proj", + "model.layers.25.mlp.experts.111.down_proj", + "model.layers.25.mlp.experts.111.gate_proj", + "model.layers.25.mlp.experts.111.up_proj", + "model.layers.25.mlp.experts.112.down_proj", + "model.layers.25.mlp.experts.112.gate_proj", + "model.layers.25.mlp.experts.112.up_proj", + "model.layers.25.mlp.experts.113.down_proj", + "model.layers.25.mlp.experts.113.gate_proj", + "model.layers.25.mlp.experts.113.up_proj", + "model.layers.25.mlp.experts.114.down_proj", + "model.layers.25.mlp.experts.114.gate_proj", + "model.layers.25.mlp.experts.114.up_proj", + "model.layers.25.mlp.experts.115.down_proj", + "model.layers.25.mlp.experts.115.gate_proj", + "model.layers.25.mlp.experts.115.up_proj", + "model.layers.25.mlp.experts.116.down_proj", + "model.layers.25.mlp.experts.116.gate_proj", + "model.layers.25.mlp.experts.116.up_proj", + "model.layers.25.mlp.experts.117.down_proj", + "model.layers.25.mlp.experts.117.gate_proj", + "model.layers.25.mlp.experts.117.up_proj", + "model.layers.25.mlp.experts.118.down_proj", + "model.layers.25.mlp.experts.118.gate_proj", + "model.layers.25.mlp.experts.118.up_proj", + "model.layers.25.mlp.experts.119.down_proj", + "model.layers.25.mlp.experts.119.gate_proj", + "model.layers.25.mlp.experts.119.up_proj", + "model.layers.25.mlp.experts.12.down_proj", + "model.layers.25.mlp.experts.12.gate_proj", + "model.layers.25.mlp.experts.12.up_proj", + "model.layers.25.mlp.experts.120.down_proj", + "model.layers.25.mlp.experts.120.gate_proj", + "model.layers.25.mlp.experts.120.up_proj", + "model.layers.25.mlp.experts.121.down_proj", + "model.layers.25.mlp.experts.121.gate_proj", + "model.layers.25.mlp.experts.121.up_proj", + "model.layers.25.mlp.experts.122.down_proj", + "model.layers.25.mlp.experts.122.gate_proj", + "model.layers.25.mlp.experts.122.up_proj", + "model.layers.25.mlp.experts.123.down_proj", + "model.layers.25.mlp.experts.123.gate_proj", + "model.layers.25.mlp.experts.123.up_proj", + "model.layers.25.mlp.experts.124.down_proj", + "model.layers.25.mlp.experts.124.gate_proj", + "model.layers.25.mlp.experts.124.up_proj", + "model.layers.25.mlp.experts.125.down_proj", + "model.layers.25.mlp.experts.125.gate_proj", + "model.layers.25.mlp.experts.125.up_proj", + "model.layers.25.mlp.experts.126.down_proj", + "model.layers.25.mlp.experts.126.gate_proj", + "model.layers.25.mlp.experts.126.up_proj", + "model.layers.25.mlp.experts.127.down_proj", + "model.layers.25.mlp.experts.127.gate_proj", + "model.layers.25.mlp.experts.127.up_proj", + "model.layers.25.mlp.experts.128.down_proj", + "model.layers.25.mlp.experts.128.gate_proj", + "model.layers.25.mlp.experts.128.up_proj", + "model.layers.25.mlp.experts.129.down_proj", + "model.layers.25.mlp.experts.129.gate_proj", + "model.layers.25.mlp.experts.129.up_proj", + "model.layers.25.mlp.experts.13.down_proj", + "model.layers.25.mlp.experts.13.gate_proj", + "model.layers.25.mlp.experts.13.up_proj", + "model.layers.25.mlp.experts.130.down_proj", + "model.layers.25.mlp.experts.130.gate_proj", + "model.layers.25.mlp.experts.130.up_proj", + "model.layers.25.mlp.experts.131.down_proj", + "model.layers.25.mlp.experts.131.gate_proj", + "model.layers.25.mlp.experts.131.up_proj", + "model.layers.25.mlp.experts.132.down_proj", + "model.layers.25.mlp.experts.132.gate_proj", + "model.layers.25.mlp.experts.132.up_proj", + "model.layers.25.mlp.experts.133.down_proj", + "model.layers.25.mlp.experts.133.gate_proj", + "model.layers.25.mlp.experts.133.up_proj", + "model.layers.25.mlp.experts.134.down_proj", + "model.layers.25.mlp.experts.134.gate_proj", + "model.layers.25.mlp.experts.134.up_proj", + "model.layers.25.mlp.experts.135.down_proj", + "model.layers.25.mlp.experts.135.gate_proj", + "model.layers.25.mlp.experts.135.up_proj", + "model.layers.25.mlp.experts.136.down_proj", + "model.layers.25.mlp.experts.136.gate_proj", + "model.layers.25.mlp.experts.136.up_proj", + "model.layers.25.mlp.experts.137.down_proj", + "model.layers.25.mlp.experts.137.gate_proj", + "model.layers.25.mlp.experts.137.up_proj", + "model.layers.25.mlp.experts.138.down_proj", + "model.layers.25.mlp.experts.138.gate_proj", + "model.layers.25.mlp.experts.138.up_proj", + "model.layers.25.mlp.experts.139.down_proj", + "model.layers.25.mlp.experts.139.gate_proj", + "model.layers.25.mlp.experts.139.up_proj", + "model.layers.25.mlp.experts.14.down_proj", + "model.layers.25.mlp.experts.14.gate_proj", + "model.layers.25.mlp.experts.14.up_proj", + "model.layers.25.mlp.experts.140.down_proj", + "model.layers.25.mlp.experts.140.gate_proj", + "model.layers.25.mlp.experts.140.up_proj", + "model.layers.25.mlp.experts.141.down_proj", + "model.layers.25.mlp.experts.141.gate_proj", + "model.layers.25.mlp.experts.141.up_proj", + "model.layers.25.mlp.experts.142.down_proj", + "model.layers.25.mlp.experts.142.gate_proj", + "model.layers.25.mlp.experts.142.up_proj", + "model.layers.25.mlp.experts.143.down_proj", + "model.layers.25.mlp.experts.143.gate_proj", + "model.layers.25.mlp.experts.143.up_proj", + "model.layers.25.mlp.experts.144.down_proj", + "model.layers.25.mlp.experts.144.gate_proj", + "model.layers.25.mlp.experts.144.up_proj", + "model.layers.25.mlp.experts.145.down_proj", + "model.layers.25.mlp.experts.145.gate_proj", + "model.layers.25.mlp.experts.145.up_proj", + "model.layers.25.mlp.experts.146.down_proj", + "model.layers.25.mlp.experts.146.gate_proj", + "model.layers.25.mlp.experts.146.up_proj", + "model.layers.25.mlp.experts.147.down_proj", + "model.layers.25.mlp.experts.147.gate_proj", + "model.layers.25.mlp.experts.147.up_proj", + "model.layers.25.mlp.experts.148.down_proj", + "model.layers.25.mlp.experts.148.gate_proj", + "model.layers.25.mlp.experts.148.up_proj", + "model.layers.25.mlp.experts.149.down_proj", + "model.layers.25.mlp.experts.149.gate_proj", + "model.layers.25.mlp.experts.149.up_proj", + "model.layers.25.mlp.experts.15.down_proj", + "model.layers.25.mlp.experts.15.gate_proj", + "model.layers.25.mlp.experts.15.up_proj", + "model.layers.25.mlp.experts.150.down_proj", + "model.layers.25.mlp.experts.150.gate_proj", + "model.layers.25.mlp.experts.150.up_proj", + "model.layers.25.mlp.experts.151.down_proj", + "model.layers.25.mlp.experts.151.gate_proj", + "model.layers.25.mlp.experts.151.up_proj", + "model.layers.25.mlp.experts.152.down_proj", + "model.layers.25.mlp.experts.152.gate_proj", + "model.layers.25.mlp.experts.152.up_proj", + "model.layers.25.mlp.experts.153.down_proj", + "model.layers.25.mlp.experts.153.gate_proj", + "model.layers.25.mlp.experts.153.up_proj", + "model.layers.25.mlp.experts.154.down_proj", + "model.layers.25.mlp.experts.154.gate_proj", + "model.layers.25.mlp.experts.154.up_proj", + "model.layers.25.mlp.experts.155.down_proj", + "model.layers.25.mlp.experts.155.gate_proj", + "model.layers.25.mlp.experts.155.up_proj", + "model.layers.25.mlp.experts.156.down_proj", + "model.layers.25.mlp.experts.156.gate_proj", + "model.layers.25.mlp.experts.156.up_proj", + "model.layers.25.mlp.experts.157.down_proj", + "model.layers.25.mlp.experts.157.gate_proj", + "model.layers.25.mlp.experts.157.up_proj", + "model.layers.25.mlp.experts.158.down_proj", + "model.layers.25.mlp.experts.158.gate_proj", + "model.layers.25.mlp.experts.158.up_proj", + "model.layers.25.mlp.experts.159.down_proj", + "model.layers.25.mlp.experts.159.gate_proj", + "model.layers.25.mlp.experts.159.up_proj", + "model.layers.25.mlp.experts.16.down_proj", + "model.layers.25.mlp.experts.16.gate_proj", + "model.layers.25.mlp.experts.16.up_proj", + "model.layers.25.mlp.experts.160.down_proj", + "model.layers.25.mlp.experts.160.gate_proj", + "model.layers.25.mlp.experts.160.up_proj", + "model.layers.25.mlp.experts.161.down_proj", + "model.layers.25.mlp.experts.161.gate_proj", + "model.layers.25.mlp.experts.161.up_proj", + "model.layers.25.mlp.experts.162.down_proj", + "model.layers.25.mlp.experts.162.gate_proj", + "model.layers.25.mlp.experts.162.up_proj", + "model.layers.25.mlp.experts.163.down_proj", + "model.layers.25.mlp.experts.163.gate_proj", + "model.layers.25.mlp.experts.163.up_proj", + "model.layers.25.mlp.experts.164.down_proj", + "model.layers.25.mlp.experts.164.gate_proj", + "model.layers.25.mlp.experts.164.up_proj", + "model.layers.25.mlp.experts.165.down_proj", + "model.layers.25.mlp.experts.165.gate_proj", + "model.layers.25.mlp.experts.165.up_proj", + "model.layers.25.mlp.experts.166.down_proj", + "model.layers.25.mlp.experts.166.gate_proj", + "model.layers.25.mlp.experts.166.up_proj", + "model.layers.25.mlp.experts.167.down_proj", + "model.layers.25.mlp.experts.167.gate_proj", + "model.layers.25.mlp.experts.167.up_proj", + "model.layers.25.mlp.experts.168.down_proj", + "model.layers.25.mlp.experts.168.gate_proj", + "model.layers.25.mlp.experts.168.up_proj", + "model.layers.25.mlp.experts.169.down_proj", + "model.layers.25.mlp.experts.169.gate_proj", + "model.layers.25.mlp.experts.169.up_proj", + "model.layers.25.mlp.experts.17.down_proj", + "model.layers.25.mlp.experts.17.gate_proj", + "model.layers.25.mlp.experts.17.up_proj", + "model.layers.25.mlp.experts.170.down_proj", + "model.layers.25.mlp.experts.170.gate_proj", + "model.layers.25.mlp.experts.170.up_proj", + "model.layers.25.mlp.experts.171.down_proj", + "model.layers.25.mlp.experts.171.gate_proj", + "model.layers.25.mlp.experts.171.up_proj", + "model.layers.25.mlp.experts.172.down_proj", + "model.layers.25.mlp.experts.172.gate_proj", + "model.layers.25.mlp.experts.172.up_proj", + "model.layers.25.mlp.experts.173.down_proj", + "model.layers.25.mlp.experts.173.gate_proj", + "model.layers.25.mlp.experts.173.up_proj", + "model.layers.25.mlp.experts.174.down_proj", + "model.layers.25.mlp.experts.174.gate_proj", + "model.layers.25.mlp.experts.174.up_proj", + "model.layers.25.mlp.experts.175.down_proj", + "model.layers.25.mlp.experts.175.gate_proj", + "model.layers.25.mlp.experts.175.up_proj", + "model.layers.25.mlp.experts.176.down_proj", + "model.layers.25.mlp.experts.176.gate_proj", + "model.layers.25.mlp.experts.176.up_proj", + "model.layers.25.mlp.experts.177.down_proj", + "model.layers.25.mlp.experts.177.gate_proj", + "model.layers.25.mlp.experts.177.up_proj", + "model.layers.25.mlp.experts.178.down_proj", + "model.layers.25.mlp.experts.178.gate_proj", + "model.layers.25.mlp.experts.178.up_proj", + "model.layers.25.mlp.experts.179.down_proj", + "model.layers.25.mlp.experts.179.gate_proj", + "model.layers.25.mlp.experts.179.up_proj", + "model.layers.25.mlp.experts.18.down_proj", + "model.layers.25.mlp.experts.18.gate_proj", + "model.layers.25.mlp.experts.18.up_proj", + "model.layers.25.mlp.experts.180.down_proj", + "model.layers.25.mlp.experts.180.gate_proj", + "model.layers.25.mlp.experts.180.up_proj", + "model.layers.25.mlp.experts.181.down_proj", + "model.layers.25.mlp.experts.181.gate_proj", + "model.layers.25.mlp.experts.181.up_proj", + "model.layers.25.mlp.experts.182.down_proj", + "model.layers.25.mlp.experts.182.gate_proj", + "model.layers.25.mlp.experts.182.up_proj", + "model.layers.25.mlp.experts.183.down_proj", + "model.layers.25.mlp.experts.183.gate_proj", + "model.layers.25.mlp.experts.183.up_proj", + "model.layers.25.mlp.experts.184.down_proj", + "model.layers.25.mlp.experts.184.gate_proj", + "model.layers.25.mlp.experts.184.up_proj", + "model.layers.25.mlp.experts.185.down_proj", + "model.layers.25.mlp.experts.185.gate_proj", + "model.layers.25.mlp.experts.185.up_proj", + "model.layers.25.mlp.experts.186.down_proj", + "model.layers.25.mlp.experts.186.gate_proj", + "model.layers.25.mlp.experts.186.up_proj", + "model.layers.25.mlp.experts.187.down_proj", + "model.layers.25.mlp.experts.187.gate_proj", + "model.layers.25.mlp.experts.187.up_proj", + "model.layers.25.mlp.experts.188.down_proj", + "model.layers.25.mlp.experts.188.gate_proj", + "model.layers.25.mlp.experts.188.up_proj", + "model.layers.25.mlp.experts.189.down_proj", + "model.layers.25.mlp.experts.189.gate_proj", + "model.layers.25.mlp.experts.189.up_proj", + "model.layers.25.mlp.experts.19.down_proj", + "model.layers.25.mlp.experts.19.gate_proj", + "model.layers.25.mlp.experts.19.up_proj", + "model.layers.25.mlp.experts.190.down_proj", + "model.layers.25.mlp.experts.190.gate_proj", + "model.layers.25.mlp.experts.190.up_proj", + "model.layers.25.mlp.experts.191.down_proj", + "model.layers.25.mlp.experts.191.gate_proj", + "model.layers.25.mlp.experts.191.up_proj", + "model.layers.25.mlp.experts.192.down_proj", + "model.layers.25.mlp.experts.192.gate_proj", + "model.layers.25.mlp.experts.192.up_proj", + "model.layers.25.mlp.experts.193.down_proj", + "model.layers.25.mlp.experts.193.gate_proj", + "model.layers.25.mlp.experts.193.up_proj", + "model.layers.25.mlp.experts.194.down_proj", + "model.layers.25.mlp.experts.194.gate_proj", + "model.layers.25.mlp.experts.194.up_proj", + "model.layers.25.mlp.experts.195.down_proj", + "model.layers.25.mlp.experts.195.gate_proj", + "model.layers.25.mlp.experts.195.up_proj", + "model.layers.25.mlp.experts.196.down_proj", + "model.layers.25.mlp.experts.196.gate_proj", + "model.layers.25.mlp.experts.196.up_proj", + "model.layers.25.mlp.experts.197.down_proj", + "model.layers.25.mlp.experts.197.gate_proj", + "model.layers.25.mlp.experts.197.up_proj", + "model.layers.25.mlp.experts.198.down_proj", + "model.layers.25.mlp.experts.198.gate_proj", + "model.layers.25.mlp.experts.198.up_proj", + "model.layers.25.mlp.experts.199.down_proj", + "model.layers.25.mlp.experts.199.gate_proj", + "model.layers.25.mlp.experts.199.up_proj", + "model.layers.25.mlp.experts.2.down_proj", + "model.layers.25.mlp.experts.2.gate_proj", + "model.layers.25.mlp.experts.2.up_proj", + "model.layers.25.mlp.experts.20.down_proj", + "model.layers.25.mlp.experts.20.gate_proj", + "model.layers.25.mlp.experts.20.up_proj", + "model.layers.25.mlp.experts.200.down_proj", + "model.layers.25.mlp.experts.200.gate_proj", + "model.layers.25.mlp.experts.200.up_proj", + "model.layers.25.mlp.experts.201.down_proj", + "model.layers.25.mlp.experts.201.gate_proj", + "model.layers.25.mlp.experts.201.up_proj", + "model.layers.25.mlp.experts.202.down_proj", + "model.layers.25.mlp.experts.202.gate_proj", + "model.layers.25.mlp.experts.202.up_proj", + "model.layers.25.mlp.experts.203.down_proj", + "model.layers.25.mlp.experts.203.gate_proj", + "model.layers.25.mlp.experts.203.up_proj", + "model.layers.25.mlp.experts.204.down_proj", + "model.layers.25.mlp.experts.204.gate_proj", + "model.layers.25.mlp.experts.204.up_proj", + "model.layers.25.mlp.experts.205.down_proj", + "model.layers.25.mlp.experts.205.gate_proj", + "model.layers.25.mlp.experts.205.up_proj", + "model.layers.25.mlp.experts.206.down_proj", + "model.layers.25.mlp.experts.206.gate_proj", + "model.layers.25.mlp.experts.206.up_proj", + "model.layers.25.mlp.experts.207.down_proj", + "model.layers.25.mlp.experts.207.gate_proj", + "model.layers.25.mlp.experts.207.up_proj", + "model.layers.25.mlp.experts.208.down_proj", + "model.layers.25.mlp.experts.208.gate_proj", + "model.layers.25.mlp.experts.208.up_proj", + "model.layers.25.mlp.experts.209.down_proj", + "model.layers.25.mlp.experts.209.gate_proj", + "model.layers.25.mlp.experts.209.up_proj", + "model.layers.25.mlp.experts.21.down_proj", + "model.layers.25.mlp.experts.21.gate_proj", + "model.layers.25.mlp.experts.21.up_proj", + "model.layers.25.mlp.experts.210.down_proj", + "model.layers.25.mlp.experts.210.gate_proj", + "model.layers.25.mlp.experts.210.up_proj", + "model.layers.25.mlp.experts.211.down_proj", + "model.layers.25.mlp.experts.211.gate_proj", + "model.layers.25.mlp.experts.211.up_proj", + "model.layers.25.mlp.experts.212.down_proj", + "model.layers.25.mlp.experts.212.gate_proj", + "model.layers.25.mlp.experts.212.up_proj", + "model.layers.25.mlp.experts.213.down_proj", + "model.layers.25.mlp.experts.213.gate_proj", + "model.layers.25.mlp.experts.213.up_proj", + "model.layers.25.mlp.experts.214.down_proj", + "model.layers.25.mlp.experts.214.gate_proj", + "model.layers.25.mlp.experts.214.up_proj", + "model.layers.25.mlp.experts.215.down_proj", + "model.layers.25.mlp.experts.215.gate_proj", + "model.layers.25.mlp.experts.215.up_proj", + "model.layers.25.mlp.experts.216.down_proj", + "model.layers.25.mlp.experts.216.gate_proj", + "model.layers.25.mlp.experts.216.up_proj", + "model.layers.25.mlp.experts.217.down_proj", + "model.layers.25.mlp.experts.217.gate_proj", + "model.layers.25.mlp.experts.217.up_proj", + "model.layers.25.mlp.experts.218.down_proj", + "model.layers.25.mlp.experts.218.gate_proj", + "model.layers.25.mlp.experts.218.up_proj", + "model.layers.25.mlp.experts.219.down_proj", + "model.layers.25.mlp.experts.219.gate_proj", + "model.layers.25.mlp.experts.219.up_proj", + "model.layers.25.mlp.experts.22.down_proj", + "model.layers.25.mlp.experts.22.gate_proj", + "model.layers.25.mlp.experts.22.up_proj", + "model.layers.25.mlp.experts.220.down_proj", + "model.layers.25.mlp.experts.220.gate_proj", + "model.layers.25.mlp.experts.220.up_proj", + "model.layers.25.mlp.experts.221.down_proj", + "model.layers.25.mlp.experts.221.gate_proj", + "model.layers.25.mlp.experts.221.up_proj", + "model.layers.25.mlp.experts.222.down_proj", + "model.layers.25.mlp.experts.222.gate_proj", + "model.layers.25.mlp.experts.222.up_proj", + "model.layers.25.mlp.experts.223.down_proj", + "model.layers.25.mlp.experts.223.gate_proj", + "model.layers.25.mlp.experts.223.up_proj", + "model.layers.25.mlp.experts.224.down_proj", + "model.layers.25.mlp.experts.224.gate_proj", + "model.layers.25.mlp.experts.224.up_proj", + "model.layers.25.mlp.experts.225.down_proj", + "model.layers.25.mlp.experts.225.gate_proj", + "model.layers.25.mlp.experts.225.up_proj", + "model.layers.25.mlp.experts.226.down_proj", + "model.layers.25.mlp.experts.226.gate_proj", + "model.layers.25.mlp.experts.226.up_proj", + "model.layers.25.mlp.experts.227.down_proj", + "model.layers.25.mlp.experts.227.gate_proj", + "model.layers.25.mlp.experts.227.up_proj", + "model.layers.25.mlp.experts.228.down_proj", + "model.layers.25.mlp.experts.228.gate_proj", + "model.layers.25.mlp.experts.228.up_proj", + "model.layers.25.mlp.experts.229.down_proj", + "model.layers.25.mlp.experts.229.gate_proj", + "model.layers.25.mlp.experts.229.up_proj", + "model.layers.25.mlp.experts.23.down_proj", + "model.layers.25.mlp.experts.23.gate_proj", + "model.layers.25.mlp.experts.23.up_proj", + "model.layers.25.mlp.experts.230.down_proj", + "model.layers.25.mlp.experts.230.gate_proj", + "model.layers.25.mlp.experts.230.up_proj", + "model.layers.25.mlp.experts.231.down_proj", + "model.layers.25.mlp.experts.231.gate_proj", + "model.layers.25.mlp.experts.231.up_proj", + "model.layers.25.mlp.experts.232.down_proj", + "model.layers.25.mlp.experts.232.gate_proj", + "model.layers.25.mlp.experts.232.up_proj", + "model.layers.25.mlp.experts.233.down_proj", + "model.layers.25.mlp.experts.233.gate_proj", + "model.layers.25.mlp.experts.233.up_proj", + "model.layers.25.mlp.experts.234.down_proj", + "model.layers.25.mlp.experts.234.gate_proj", + "model.layers.25.mlp.experts.234.up_proj", + "model.layers.25.mlp.experts.235.down_proj", + "model.layers.25.mlp.experts.235.gate_proj", + "model.layers.25.mlp.experts.235.up_proj", + "model.layers.25.mlp.experts.236.down_proj", + "model.layers.25.mlp.experts.236.gate_proj", + "model.layers.25.mlp.experts.236.up_proj", + "model.layers.25.mlp.experts.237.down_proj", + "model.layers.25.mlp.experts.237.gate_proj", + "model.layers.25.mlp.experts.237.up_proj", + "model.layers.25.mlp.experts.238.down_proj", + "model.layers.25.mlp.experts.238.gate_proj", + "model.layers.25.mlp.experts.238.up_proj", + "model.layers.25.mlp.experts.239.down_proj", + "model.layers.25.mlp.experts.239.gate_proj", + "model.layers.25.mlp.experts.239.up_proj", + "model.layers.25.mlp.experts.24.down_proj", + "model.layers.25.mlp.experts.24.gate_proj", + "model.layers.25.mlp.experts.24.up_proj", + "model.layers.25.mlp.experts.240.down_proj", + "model.layers.25.mlp.experts.240.gate_proj", + "model.layers.25.mlp.experts.240.up_proj", + "model.layers.25.mlp.experts.241.down_proj", + "model.layers.25.mlp.experts.241.gate_proj", + "model.layers.25.mlp.experts.241.up_proj", + "model.layers.25.mlp.experts.242.down_proj", + "model.layers.25.mlp.experts.242.gate_proj", + "model.layers.25.mlp.experts.242.up_proj", + "model.layers.25.mlp.experts.243.down_proj", + "model.layers.25.mlp.experts.243.gate_proj", + "model.layers.25.mlp.experts.243.up_proj", + "model.layers.25.mlp.experts.244.down_proj", + "model.layers.25.mlp.experts.244.gate_proj", + "model.layers.25.mlp.experts.244.up_proj", + "model.layers.25.mlp.experts.245.down_proj", + "model.layers.25.mlp.experts.245.gate_proj", + "model.layers.25.mlp.experts.245.up_proj", + "model.layers.25.mlp.experts.246.down_proj", + "model.layers.25.mlp.experts.246.gate_proj", + "model.layers.25.mlp.experts.246.up_proj", + "model.layers.25.mlp.experts.247.down_proj", + "model.layers.25.mlp.experts.247.gate_proj", + "model.layers.25.mlp.experts.247.up_proj", + "model.layers.25.mlp.experts.248.down_proj", + "model.layers.25.mlp.experts.248.gate_proj", + "model.layers.25.mlp.experts.248.up_proj", + "model.layers.25.mlp.experts.249.down_proj", + "model.layers.25.mlp.experts.249.gate_proj", + "model.layers.25.mlp.experts.249.up_proj", + "model.layers.25.mlp.experts.25.down_proj", + "model.layers.25.mlp.experts.25.gate_proj", + "model.layers.25.mlp.experts.25.up_proj", + "model.layers.25.mlp.experts.250.down_proj", + "model.layers.25.mlp.experts.250.gate_proj", + "model.layers.25.mlp.experts.250.up_proj", + "model.layers.25.mlp.experts.251.down_proj", + "model.layers.25.mlp.experts.251.gate_proj", + "model.layers.25.mlp.experts.251.up_proj", + "model.layers.25.mlp.experts.252.down_proj", + "model.layers.25.mlp.experts.252.gate_proj", + "model.layers.25.mlp.experts.252.up_proj", + "model.layers.25.mlp.experts.253.down_proj", + "model.layers.25.mlp.experts.253.gate_proj", + "model.layers.25.mlp.experts.253.up_proj", + "model.layers.25.mlp.experts.254.down_proj", + "model.layers.25.mlp.experts.254.gate_proj", + "model.layers.25.mlp.experts.254.up_proj", + "model.layers.25.mlp.experts.255.down_proj", + "model.layers.25.mlp.experts.255.gate_proj", + "model.layers.25.mlp.experts.255.up_proj", + "model.layers.25.mlp.experts.26.down_proj", + "model.layers.25.mlp.experts.26.gate_proj", + "model.layers.25.mlp.experts.26.up_proj", + "model.layers.25.mlp.experts.27.down_proj", + "model.layers.25.mlp.experts.27.gate_proj", + "model.layers.25.mlp.experts.27.up_proj", + "model.layers.25.mlp.experts.28.down_proj", + "model.layers.25.mlp.experts.28.gate_proj", + "model.layers.25.mlp.experts.28.up_proj", + "model.layers.25.mlp.experts.29.down_proj", + "model.layers.25.mlp.experts.29.gate_proj", + "model.layers.25.mlp.experts.29.up_proj", + "model.layers.25.mlp.experts.3.down_proj", + "model.layers.25.mlp.experts.3.gate_proj", + "model.layers.25.mlp.experts.3.up_proj", + "model.layers.25.mlp.experts.30.down_proj", + "model.layers.25.mlp.experts.30.gate_proj", + "model.layers.25.mlp.experts.30.up_proj", + "model.layers.25.mlp.experts.31.down_proj", + "model.layers.25.mlp.experts.31.gate_proj", + "model.layers.25.mlp.experts.31.up_proj", + "model.layers.25.mlp.experts.32.down_proj", + "model.layers.25.mlp.experts.32.gate_proj", + "model.layers.25.mlp.experts.32.up_proj", + "model.layers.25.mlp.experts.33.down_proj", + "model.layers.25.mlp.experts.33.gate_proj", + "model.layers.25.mlp.experts.33.up_proj", + "model.layers.25.mlp.experts.34.down_proj", + "model.layers.25.mlp.experts.34.gate_proj", + "model.layers.25.mlp.experts.34.up_proj", + "model.layers.25.mlp.experts.35.down_proj", + "model.layers.25.mlp.experts.35.gate_proj", + "model.layers.25.mlp.experts.35.up_proj", + "model.layers.25.mlp.experts.36.down_proj", + "model.layers.25.mlp.experts.36.gate_proj", + "model.layers.25.mlp.experts.36.up_proj", + "model.layers.25.mlp.experts.37.down_proj", + "model.layers.25.mlp.experts.37.gate_proj", + "model.layers.25.mlp.experts.37.up_proj", + "model.layers.25.mlp.experts.38.down_proj", + "model.layers.25.mlp.experts.38.gate_proj", + "model.layers.25.mlp.experts.38.up_proj", + "model.layers.25.mlp.experts.39.down_proj", + "model.layers.25.mlp.experts.39.gate_proj", + "model.layers.25.mlp.experts.39.up_proj", + "model.layers.25.mlp.experts.4.down_proj", + "model.layers.25.mlp.experts.4.gate_proj", + "model.layers.25.mlp.experts.4.up_proj", + "model.layers.25.mlp.experts.40.down_proj", + "model.layers.25.mlp.experts.40.gate_proj", + "model.layers.25.mlp.experts.40.up_proj", + "model.layers.25.mlp.experts.41.down_proj", + "model.layers.25.mlp.experts.41.gate_proj", + "model.layers.25.mlp.experts.41.up_proj", + "model.layers.25.mlp.experts.42.down_proj", + "model.layers.25.mlp.experts.42.gate_proj", + "model.layers.25.mlp.experts.42.up_proj", + "model.layers.25.mlp.experts.43.down_proj", + "model.layers.25.mlp.experts.43.gate_proj", + "model.layers.25.mlp.experts.43.up_proj", + "model.layers.25.mlp.experts.44.down_proj", + "model.layers.25.mlp.experts.44.gate_proj", + "model.layers.25.mlp.experts.44.up_proj", + "model.layers.25.mlp.experts.45.down_proj", + "model.layers.25.mlp.experts.45.gate_proj", + "model.layers.25.mlp.experts.45.up_proj", + "model.layers.25.mlp.experts.46.down_proj", + "model.layers.25.mlp.experts.46.gate_proj", + "model.layers.25.mlp.experts.46.up_proj", + "model.layers.25.mlp.experts.47.down_proj", + "model.layers.25.mlp.experts.47.gate_proj", + "model.layers.25.mlp.experts.47.up_proj", + "model.layers.25.mlp.experts.48.down_proj", + "model.layers.25.mlp.experts.48.gate_proj", + "model.layers.25.mlp.experts.48.up_proj", + "model.layers.25.mlp.experts.49.down_proj", + "model.layers.25.mlp.experts.49.gate_proj", + "model.layers.25.mlp.experts.49.up_proj", + "model.layers.25.mlp.experts.5.down_proj", + "model.layers.25.mlp.experts.5.gate_proj", + "model.layers.25.mlp.experts.5.up_proj", + "model.layers.25.mlp.experts.50.down_proj", + "model.layers.25.mlp.experts.50.gate_proj", + "model.layers.25.mlp.experts.50.up_proj", + "model.layers.25.mlp.experts.51.down_proj", + "model.layers.25.mlp.experts.51.gate_proj", + "model.layers.25.mlp.experts.51.up_proj", + "model.layers.25.mlp.experts.52.down_proj", + "model.layers.25.mlp.experts.52.gate_proj", + "model.layers.25.mlp.experts.52.up_proj", + "model.layers.25.mlp.experts.53.down_proj", + "model.layers.25.mlp.experts.53.gate_proj", + "model.layers.25.mlp.experts.53.up_proj", + "model.layers.25.mlp.experts.54.down_proj", + "model.layers.25.mlp.experts.54.gate_proj", + "model.layers.25.mlp.experts.54.up_proj", + "model.layers.25.mlp.experts.55.down_proj", + "model.layers.25.mlp.experts.55.gate_proj", + "model.layers.25.mlp.experts.55.up_proj", + "model.layers.25.mlp.experts.56.down_proj", + "model.layers.25.mlp.experts.56.gate_proj", + "model.layers.25.mlp.experts.56.up_proj", + "model.layers.25.mlp.experts.57.down_proj", + "model.layers.25.mlp.experts.57.gate_proj", + "model.layers.25.mlp.experts.57.up_proj", + "model.layers.25.mlp.experts.58.down_proj", + "model.layers.25.mlp.experts.58.gate_proj", + "model.layers.25.mlp.experts.58.up_proj", + "model.layers.25.mlp.experts.59.down_proj", + "model.layers.25.mlp.experts.59.gate_proj", + "model.layers.25.mlp.experts.59.up_proj", + "model.layers.25.mlp.experts.6.down_proj", + "model.layers.25.mlp.experts.6.gate_proj", + "model.layers.25.mlp.experts.6.up_proj", + "model.layers.25.mlp.experts.60.down_proj", + "model.layers.25.mlp.experts.60.gate_proj", + "model.layers.25.mlp.experts.60.up_proj", + "model.layers.25.mlp.experts.61.down_proj", + "model.layers.25.mlp.experts.61.gate_proj", + "model.layers.25.mlp.experts.61.up_proj", + "model.layers.25.mlp.experts.62.down_proj", + "model.layers.25.mlp.experts.62.gate_proj", + "model.layers.25.mlp.experts.62.up_proj", + "model.layers.25.mlp.experts.63.down_proj", + "model.layers.25.mlp.experts.63.gate_proj", + "model.layers.25.mlp.experts.63.up_proj", + "model.layers.25.mlp.experts.64.down_proj", + "model.layers.25.mlp.experts.64.gate_proj", + "model.layers.25.mlp.experts.64.up_proj", + "model.layers.25.mlp.experts.65.down_proj", + "model.layers.25.mlp.experts.65.gate_proj", + "model.layers.25.mlp.experts.65.up_proj", + "model.layers.25.mlp.experts.66.down_proj", + "model.layers.25.mlp.experts.66.gate_proj", + "model.layers.25.mlp.experts.66.up_proj", + "model.layers.25.mlp.experts.67.down_proj", + "model.layers.25.mlp.experts.67.gate_proj", + "model.layers.25.mlp.experts.67.up_proj", + "model.layers.25.mlp.experts.68.down_proj", + "model.layers.25.mlp.experts.68.gate_proj", + "model.layers.25.mlp.experts.68.up_proj", + "model.layers.25.mlp.experts.69.down_proj", + "model.layers.25.mlp.experts.69.gate_proj", + "model.layers.25.mlp.experts.69.up_proj", + "model.layers.25.mlp.experts.7.down_proj", + "model.layers.25.mlp.experts.7.gate_proj", + "model.layers.25.mlp.experts.7.up_proj", + "model.layers.25.mlp.experts.70.down_proj", + "model.layers.25.mlp.experts.70.gate_proj", + "model.layers.25.mlp.experts.70.up_proj", + "model.layers.25.mlp.experts.71.down_proj", + "model.layers.25.mlp.experts.71.gate_proj", + "model.layers.25.mlp.experts.71.up_proj", + "model.layers.25.mlp.experts.72.down_proj", + "model.layers.25.mlp.experts.72.gate_proj", + "model.layers.25.mlp.experts.72.up_proj", + "model.layers.25.mlp.experts.73.down_proj", + "model.layers.25.mlp.experts.73.gate_proj", + "model.layers.25.mlp.experts.73.up_proj", + "model.layers.25.mlp.experts.74.down_proj", + "model.layers.25.mlp.experts.74.gate_proj", + "model.layers.25.mlp.experts.74.up_proj", + "model.layers.25.mlp.experts.75.down_proj", + "model.layers.25.mlp.experts.75.gate_proj", + "model.layers.25.mlp.experts.75.up_proj", + "model.layers.25.mlp.experts.76.down_proj", + "model.layers.25.mlp.experts.76.gate_proj", + "model.layers.25.mlp.experts.76.up_proj", + "model.layers.25.mlp.experts.77.down_proj", + "model.layers.25.mlp.experts.77.gate_proj", + "model.layers.25.mlp.experts.77.up_proj", + "model.layers.25.mlp.experts.78.down_proj", + "model.layers.25.mlp.experts.78.gate_proj", + "model.layers.25.mlp.experts.78.up_proj", + "model.layers.25.mlp.experts.79.down_proj", + "model.layers.25.mlp.experts.79.gate_proj", + "model.layers.25.mlp.experts.79.up_proj", + "model.layers.25.mlp.experts.8.down_proj", + "model.layers.25.mlp.experts.8.gate_proj", + "model.layers.25.mlp.experts.8.up_proj", + "model.layers.25.mlp.experts.80.down_proj", + "model.layers.25.mlp.experts.80.gate_proj", + "model.layers.25.mlp.experts.80.up_proj", + "model.layers.25.mlp.experts.81.down_proj", + "model.layers.25.mlp.experts.81.gate_proj", + "model.layers.25.mlp.experts.81.up_proj", + "model.layers.25.mlp.experts.82.down_proj", + "model.layers.25.mlp.experts.82.gate_proj", + "model.layers.25.mlp.experts.82.up_proj", + "model.layers.25.mlp.experts.83.down_proj", + "model.layers.25.mlp.experts.83.gate_proj", + "model.layers.25.mlp.experts.83.up_proj", + "model.layers.25.mlp.experts.84.down_proj", + "model.layers.25.mlp.experts.84.gate_proj", + "model.layers.25.mlp.experts.84.up_proj", + "model.layers.25.mlp.experts.85.down_proj", + "model.layers.25.mlp.experts.85.gate_proj", + "model.layers.25.mlp.experts.85.up_proj", + "model.layers.25.mlp.experts.86.down_proj", + "model.layers.25.mlp.experts.86.gate_proj", + "model.layers.25.mlp.experts.86.up_proj", + "model.layers.25.mlp.experts.87.down_proj", + "model.layers.25.mlp.experts.87.gate_proj", + "model.layers.25.mlp.experts.87.up_proj", + "model.layers.25.mlp.experts.88.down_proj", + "model.layers.25.mlp.experts.88.gate_proj", + "model.layers.25.mlp.experts.88.up_proj", + "model.layers.25.mlp.experts.89.down_proj", + "model.layers.25.mlp.experts.89.gate_proj", + "model.layers.25.mlp.experts.89.up_proj", + "model.layers.25.mlp.experts.9.down_proj", + "model.layers.25.mlp.experts.9.gate_proj", + "model.layers.25.mlp.experts.9.up_proj", + "model.layers.25.mlp.experts.90.down_proj", + "model.layers.25.mlp.experts.90.gate_proj", + "model.layers.25.mlp.experts.90.up_proj", + "model.layers.25.mlp.experts.91.down_proj", + "model.layers.25.mlp.experts.91.gate_proj", + "model.layers.25.mlp.experts.91.up_proj", + "model.layers.25.mlp.experts.92.down_proj", + "model.layers.25.mlp.experts.92.gate_proj", + "model.layers.25.mlp.experts.92.up_proj", + "model.layers.25.mlp.experts.93.down_proj", + "model.layers.25.mlp.experts.93.gate_proj", + "model.layers.25.mlp.experts.93.up_proj", + "model.layers.25.mlp.experts.94.down_proj", + "model.layers.25.mlp.experts.94.gate_proj", + "model.layers.25.mlp.experts.94.up_proj", + "model.layers.25.mlp.experts.95.down_proj", + "model.layers.25.mlp.experts.95.gate_proj", + "model.layers.25.mlp.experts.95.up_proj", + "model.layers.25.mlp.experts.96.down_proj", + "model.layers.25.mlp.experts.96.gate_proj", + "model.layers.25.mlp.experts.96.up_proj", + "model.layers.25.mlp.experts.97.down_proj", + "model.layers.25.mlp.experts.97.gate_proj", + "model.layers.25.mlp.experts.97.up_proj", + "model.layers.25.mlp.experts.98.down_proj", + "model.layers.25.mlp.experts.98.gate_proj", + "model.layers.25.mlp.experts.98.up_proj", + "model.layers.25.mlp.experts.99.down_proj", + "model.layers.25.mlp.experts.99.gate_proj", + "model.layers.25.mlp.experts.99.up_proj", + "model.layers.26.mlp.experts.0.down_proj", + "model.layers.26.mlp.experts.0.gate_proj", + "model.layers.26.mlp.experts.0.up_proj", + "model.layers.26.mlp.experts.1.down_proj", + "model.layers.26.mlp.experts.1.gate_proj", + "model.layers.26.mlp.experts.1.up_proj", + "model.layers.26.mlp.experts.10.down_proj", + "model.layers.26.mlp.experts.10.gate_proj", + "model.layers.26.mlp.experts.10.up_proj", + "model.layers.26.mlp.experts.100.down_proj", + "model.layers.26.mlp.experts.100.gate_proj", + "model.layers.26.mlp.experts.100.up_proj", + "model.layers.26.mlp.experts.101.down_proj", + "model.layers.26.mlp.experts.101.gate_proj", + "model.layers.26.mlp.experts.101.up_proj", + "model.layers.26.mlp.experts.102.down_proj", + "model.layers.26.mlp.experts.102.gate_proj", + "model.layers.26.mlp.experts.102.up_proj", + "model.layers.26.mlp.experts.103.down_proj", + "model.layers.26.mlp.experts.103.gate_proj", + "model.layers.26.mlp.experts.103.up_proj", + "model.layers.26.mlp.experts.104.down_proj", + "model.layers.26.mlp.experts.104.gate_proj", + "model.layers.26.mlp.experts.104.up_proj", + "model.layers.26.mlp.experts.105.down_proj", + "model.layers.26.mlp.experts.105.gate_proj", + "model.layers.26.mlp.experts.105.up_proj", + "model.layers.26.mlp.experts.106.down_proj", + "model.layers.26.mlp.experts.106.gate_proj", + "model.layers.26.mlp.experts.106.up_proj", + "model.layers.26.mlp.experts.107.down_proj", + "model.layers.26.mlp.experts.107.gate_proj", + "model.layers.26.mlp.experts.107.up_proj", + "model.layers.26.mlp.experts.108.down_proj", + "model.layers.26.mlp.experts.108.gate_proj", + "model.layers.26.mlp.experts.108.up_proj", + "model.layers.26.mlp.experts.109.down_proj", + "model.layers.26.mlp.experts.109.gate_proj", + "model.layers.26.mlp.experts.109.up_proj", + "model.layers.26.mlp.experts.11.down_proj", + "model.layers.26.mlp.experts.11.gate_proj", + "model.layers.26.mlp.experts.11.up_proj", + "model.layers.26.mlp.experts.110.down_proj", + "model.layers.26.mlp.experts.110.gate_proj", + "model.layers.26.mlp.experts.110.up_proj", + "model.layers.26.mlp.experts.111.down_proj", + "model.layers.26.mlp.experts.111.gate_proj", + "model.layers.26.mlp.experts.111.up_proj", + "model.layers.26.mlp.experts.112.down_proj", + "model.layers.26.mlp.experts.112.gate_proj", + "model.layers.26.mlp.experts.112.up_proj", + "model.layers.26.mlp.experts.113.down_proj", + "model.layers.26.mlp.experts.113.gate_proj", + "model.layers.26.mlp.experts.113.up_proj", + "model.layers.26.mlp.experts.114.down_proj", + "model.layers.26.mlp.experts.114.gate_proj", + "model.layers.26.mlp.experts.114.up_proj", + "model.layers.26.mlp.experts.115.down_proj", + "model.layers.26.mlp.experts.115.gate_proj", + "model.layers.26.mlp.experts.115.up_proj", + "model.layers.26.mlp.experts.116.down_proj", + "model.layers.26.mlp.experts.116.gate_proj", + "model.layers.26.mlp.experts.116.up_proj", + "model.layers.26.mlp.experts.117.down_proj", + "model.layers.26.mlp.experts.117.gate_proj", + "model.layers.26.mlp.experts.117.up_proj", + "model.layers.26.mlp.experts.118.down_proj", + "model.layers.26.mlp.experts.118.gate_proj", + "model.layers.26.mlp.experts.118.up_proj", + "model.layers.26.mlp.experts.119.down_proj", + "model.layers.26.mlp.experts.119.gate_proj", + "model.layers.26.mlp.experts.119.up_proj", + "model.layers.26.mlp.experts.12.down_proj", + "model.layers.26.mlp.experts.12.gate_proj", + "model.layers.26.mlp.experts.12.up_proj", + "model.layers.26.mlp.experts.120.down_proj", + "model.layers.26.mlp.experts.120.gate_proj", + "model.layers.26.mlp.experts.120.up_proj", + "model.layers.26.mlp.experts.121.down_proj", + "model.layers.26.mlp.experts.121.gate_proj", + "model.layers.26.mlp.experts.121.up_proj", + "model.layers.26.mlp.experts.122.down_proj", + "model.layers.26.mlp.experts.122.gate_proj", + "model.layers.26.mlp.experts.122.up_proj", + "model.layers.26.mlp.experts.123.down_proj", + "model.layers.26.mlp.experts.123.gate_proj", + "model.layers.26.mlp.experts.123.up_proj", + "model.layers.26.mlp.experts.124.down_proj", + "model.layers.26.mlp.experts.124.gate_proj", + "model.layers.26.mlp.experts.124.up_proj", + "model.layers.26.mlp.experts.125.down_proj", + "model.layers.26.mlp.experts.125.gate_proj", + "model.layers.26.mlp.experts.125.up_proj", + "model.layers.26.mlp.experts.126.down_proj", + "model.layers.26.mlp.experts.126.gate_proj", + "model.layers.26.mlp.experts.126.up_proj", + "model.layers.26.mlp.experts.127.down_proj", + "model.layers.26.mlp.experts.127.gate_proj", + "model.layers.26.mlp.experts.127.up_proj", + "model.layers.26.mlp.experts.128.down_proj", + "model.layers.26.mlp.experts.128.gate_proj", + "model.layers.26.mlp.experts.128.up_proj", + "model.layers.26.mlp.experts.129.down_proj", + "model.layers.26.mlp.experts.129.gate_proj", + "model.layers.26.mlp.experts.129.up_proj", + "model.layers.26.mlp.experts.13.down_proj", + "model.layers.26.mlp.experts.13.gate_proj", + "model.layers.26.mlp.experts.13.up_proj", + "model.layers.26.mlp.experts.130.down_proj", + "model.layers.26.mlp.experts.130.gate_proj", + "model.layers.26.mlp.experts.130.up_proj", + "model.layers.26.mlp.experts.131.down_proj", + "model.layers.26.mlp.experts.131.gate_proj", + "model.layers.26.mlp.experts.131.up_proj", + "model.layers.26.mlp.experts.132.down_proj", + "model.layers.26.mlp.experts.132.gate_proj", + "model.layers.26.mlp.experts.132.up_proj", + "model.layers.26.mlp.experts.133.down_proj", + "model.layers.26.mlp.experts.133.gate_proj", + "model.layers.26.mlp.experts.133.up_proj", + "model.layers.26.mlp.experts.134.down_proj", + "model.layers.26.mlp.experts.134.gate_proj", + "model.layers.26.mlp.experts.134.up_proj", + "model.layers.26.mlp.experts.135.down_proj", + "model.layers.26.mlp.experts.135.gate_proj", + "model.layers.26.mlp.experts.135.up_proj", + "model.layers.26.mlp.experts.136.down_proj", + "model.layers.26.mlp.experts.136.gate_proj", + "model.layers.26.mlp.experts.136.up_proj", + "model.layers.26.mlp.experts.137.down_proj", + "model.layers.26.mlp.experts.137.gate_proj", + "model.layers.26.mlp.experts.137.up_proj", + "model.layers.26.mlp.experts.138.down_proj", + "model.layers.26.mlp.experts.138.gate_proj", + "model.layers.26.mlp.experts.138.up_proj", + "model.layers.26.mlp.experts.139.down_proj", + "model.layers.26.mlp.experts.139.gate_proj", + "model.layers.26.mlp.experts.139.up_proj", + "model.layers.26.mlp.experts.14.down_proj", + "model.layers.26.mlp.experts.14.gate_proj", + "model.layers.26.mlp.experts.14.up_proj", + "model.layers.26.mlp.experts.140.down_proj", + "model.layers.26.mlp.experts.140.gate_proj", + "model.layers.26.mlp.experts.140.up_proj", + "model.layers.26.mlp.experts.141.down_proj", + "model.layers.26.mlp.experts.141.gate_proj", + "model.layers.26.mlp.experts.141.up_proj", + "model.layers.26.mlp.experts.142.down_proj", + "model.layers.26.mlp.experts.142.gate_proj", + "model.layers.26.mlp.experts.142.up_proj", + "model.layers.26.mlp.experts.143.down_proj", + "model.layers.26.mlp.experts.143.gate_proj", + "model.layers.26.mlp.experts.143.up_proj", + "model.layers.26.mlp.experts.144.down_proj", + "model.layers.26.mlp.experts.144.gate_proj", + "model.layers.26.mlp.experts.144.up_proj", + "model.layers.26.mlp.experts.145.down_proj", + "model.layers.26.mlp.experts.145.gate_proj", + "model.layers.26.mlp.experts.145.up_proj", + "model.layers.26.mlp.experts.146.down_proj", + "model.layers.26.mlp.experts.146.gate_proj", + "model.layers.26.mlp.experts.146.up_proj", + "model.layers.26.mlp.experts.147.down_proj", + "model.layers.26.mlp.experts.147.gate_proj", + "model.layers.26.mlp.experts.147.up_proj", + "model.layers.26.mlp.experts.148.down_proj", + "model.layers.26.mlp.experts.148.gate_proj", + "model.layers.26.mlp.experts.148.up_proj", + "model.layers.26.mlp.experts.149.down_proj", + "model.layers.26.mlp.experts.149.gate_proj", + "model.layers.26.mlp.experts.149.up_proj", + "model.layers.26.mlp.experts.15.down_proj", + "model.layers.26.mlp.experts.15.gate_proj", + "model.layers.26.mlp.experts.15.up_proj", + "model.layers.26.mlp.experts.150.down_proj", + "model.layers.26.mlp.experts.150.gate_proj", + "model.layers.26.mlp.experts.150.up_proj", + "model.layers.26.mlp.experts.151.down_proj", + "model.layers.26.mlp.experts.151.gate_proj", + "model.layers.26.mlp.experts.151.up_proj", + "model.layers.26.mlp.experts.152.down_proj", + "model.layers.26.mlp.experts.152.gate_proj", + "model.layers.26.mlp.experts.152.up_proj", + "model.layers.26.mlp.experts.153.down_proj", + "model.layers.26.mlp.experts.153.gate_proj", + "model.layers.26.mlp.experts.153.up_proj", + "model.layers.26.mlp.experts.154.down_proj", + "model.layers.26.mlp.experts.154.gate_proj", + "model.layers.26.mlp.experts.154.up_proj", + "model.layers.26.mlp.experts.155.down_proj", + "model.layers.26.mlp.experts.155.gate_proj", + "model.layers.26.mlp.experts.155.up_proj", + "model.layers.26.mlp.experts.156.down_proj", + "model.layers.26.mlp.experts.156.gate_proj", + "model.layers.26.mlp.experts.156.up_proj", + "model.layers.26.mlp.experts.157.down_proj", + "model.layers.26.mlp.experts.157.gate_proj", + "model.layers.26.mlp.experts.157.up_proj", + "model.layers.26.mlp.experts.158.down_proj", + "model.layers.26.mlp.experts.158.gate_proj", + "model.layers.26.mlp.experts.158.up_proj", + "model.layers.26.mlp.experts.159.down_proj", + "model.layers.26.mlp.experts.159.gate_proj", + "model.layers.26.mlp.experts.159.up_proj", + "model.layers.26.mlp.experts.16.down_proj", + "model.layers.26.mlp.experts.16.gate_proj", + "model.layers.26.mlp.experts.16.up_proj", + "model.layers.26.mlp.experts.160.down_proj", + "model.layers.26.mlp.experts.160.gate_proj", + "model.layers.26.mlp.experts.160.up_proj", + "model.layers.26.mlp.experts.161.down_proj", + "model.layers.26.mlp.experts.161.gate_proj", + "model.layers.26.mlp.experts.161.up_proj", + "model.layers.26.mlp.experts.162.down_proj", + "model.layers.26.mlp.experts.162.gate_proj", + "model.layers.26.mlp.experts.162.up_proj", + "model.layers.26.mlp.experts.163.down_proj", + "model.layers.26.mlp.experts.163.gate_proj", + "model.layers.26.mlp.experts.163.up_proj", + "model.layers.26.mlp.experts.164.down_proj", + "model.layers.26.mlp.experts.164.gate_proj", + "model.layers.26.mlp.experts.164.up_proj", + "model.layers.26.mlp.experts.165.down_proj", + "model.layers.26.mlp.experts.165.gate_proj", + "model.layers.26.mlp.experts.165.up_proj", + "model.layers.26.mlp.experts.166.down_proj", + "model.layers.26.mlp.experts.166.gate_proj", + "model.layers.26.mlp.experts.166.up_proj", + "model.layers.26.mlp.experts.167.down_proj", + "model.layers.26.mlp.experts.167.gate_proj", + "model.layers.26.mlp.experts.167.up_proj", + "model.layers.26.mlp.experts.168.down_proj", + "model.layers.26.mlp.experts.168.gate_proj", + "model.layers.26.mlp.experts.168.up_proj", + "model.layers.26.mlp.experts.169.down_proj", + "model.layers.26.mlp.experts.169.gate_proj", + "model.layers.26.mlp.experts.169.up_proj", + "model.layers.26.mlp.experts.17.down_proj", + "model.layers.26.mlp.experts.17.gate_proj", + "model.layers.26.mlp.experts.17.up_proj", + "model.layers.26.mlp.experts.170.down_proj", + "model.layers.26.mlp.experts.170.gate_proj", + "model.layers.26.mlp.experts.170.up_proj", + "model.layers.26.mlp.experts.171.down_proj", + "model.layers.26.mlp.experts.171.gate_proj", + "model.layers.26.mlp.experts.171.up_proj", + "model.layers.26.mlp.experts.172.down_proj", + "model.layers.26.mlp.experts.172.gate_proj", + "model.layers.26.mlp.experts.172.up_proj", + "model.layers.26.mlp.experts.173.down_proj", + "model.layers.26.mlp.experts.173.gate_proj", + "model.layers.26.mlp.experts.173.up_proj", + "model.layers.26.mlp.experts.174.down_proj", + "model.layers.26.mlp.experts.174.gate_proj", + "model.layers.26.mlp.experts.174.up_proj", + "model.layers.26.mlp.experts.175.down_proj", + "model.layers.26.mlp.experts.175.gate_proj", + "model.layers.26.mlp.experts.175.up_proj", + "model.layers.26.mlp.experts.176.down_proj", + "model.layers.26.mlp.experts.176.gate_proj", + "model.layers.26.mlp.experts.176.up_proj", + "model.layers.26.mlp.experts.177.down_proj", + "model.layers.26.mlp.experts.177.gate_proj", + "model.layers.26.mlp.experts.177.up_proj", + "model.layers.26.mlp.experts.178.down_proj", + "model.layers.26.mlp.experts.178.gate_proj", + "model.layers.26.mlp.experts.178.up_proj", + "model.layers.26.mlp.experts.179.down_proj", + "model.layers.26.mlp.experts.179.gate_proj", + "model.layers.26.mlp.experts.179.up_proj", + "model.layers.26.mlp.experts.18.down_proj", + "model.layers.26.mlp.experts.18.gate_proj", + "model.layers.26.mlp.experts.18.up_proj", + "model.layers.26.mlp.experts.180.down_proj", + "model.layers.26.mlp.experts.180.gate_proj", + "model.layers.26.mlp.experts.180.up_proj", + "model.layers.26.mlp.experts.181.down_proj", + "model.layers.26.mlp.experts.181.gate_proj", + "model.layers.26.mlp.experts.181.up_proj", + "model.layers.26.mlp.experts.182.down_proj", + "model.layers.26.mlp.experts.182.gate_proj", + "model.layers.26.mlp.experts.182.up_proj", + "model.layers.26.mlp.experts.183.down_proj", + "model.layers.26.mlp.experts.183.gate_proj", + "model.layers.26.mlp.experts.183.up_proj", + "model.layers.26.mlp.experts.184.down_proj", + "model.layers.26.mlp.experts.184.gate_proj", + "model.layers.26.mlp.experts.184.up_proj", + "model.layers.26.mlp.experts.185.down_proj", + "model.layers.26.mlp.experts.185.gate_proj", + "model.layers.26.mlp.experts.185.up_proj", + "model.layers.26.mlp.experts.186.down_proj", + "model.layers.26.mlp.experts.186.gate_proj", + "model.layers.26.mlp.experts.186.up_proj", + "model.layers.26.mlp.experts.187.down_proj", + "model.layers.26.mlp.experts.187.gate_proj", + "model.layers.26.mlp.experts.187.up_proj", + "model.layers.26.mlp.experts.188.down_proj", + "model.layers.26.mlp.experts.188.gate_proj", + "model.layers.26.mlp.experts.188.up_proj", + "model.layers.26.mlp.experts.189.down_proj", + "model.layers.26.mlp.experts.189.gate_proj", + "model.layers.26.mlp.experts.189.up_proj", + "model.layers.26.mlp.experts.19.down_proj", + "model.layers.26.mlp.experts.19.gate_proj", + "model.layers.26.mlp.experts.19.up_proj", + "model.layers.26.mlp.experts.190.down_proj", + "model.layers.26.mlp.experts.190.gate_proj", + "model.layers.26.mlp.experts.190.up_proj", + "model.layers.26.mlp.experts.191.down_proj", + "model.layers.26.mlp.experts.191.gate_proj", + "model.layers.26.mlp.experts.191.up_proj", + "model.layers.26.mlp.experts.192.down_proj", + "model.layers.26.mlp.experts.192.gate_proj", + "model.layers.26.mlp.experts.192.up_proj", + "model.layers.26.mlp.experts.193.down_proj", + "model.layers.26.mlp.experts.193.gate_proj", + "model.layers.26.mlp.experts.193.up_proj", + "model.layers.26.mlp.experts.194.down_proj", + "model.layers.26.mlp.experts.194.gate_proj", + "model.layers.26.mlp.experts.194.up_proj", + "model.layers.26.mlp.experts.195.down_proj", + "model.layers.26.mlp.experts.195.gate_proj", + "model.layers.26.mlp.experts.195.up_proj", + "model.layers.26.mlp.experts.196.down_proj", + "model.layers.26.mlp.experts.196.gate_proj", + "model.layers.26.mlp.experts.196.up_proj", + "model.layers.26.mlp.experts.197.down_proj", + "model.layers.26.mlp.experts.197.gate_proj", + "model.layers.26.mlp.experts.197.up_proj", + "model.layers.26.mlp.experts.198.down_proj", + "model.layers.26.mlp.experts.198.gate_proj", + "model.layers.26.mlp.experts.198.up_proj", + "model.layers.26.mlp.experts.199.down_proj", + "model.layers.26.mlp.experts.199.gate_proj", + "model.layers.26.mlp.experts.199.up_proj", + "model.layers.26.mlp.experts.2.down_proj", + "model.layers.26.mlp.experts.2.gate_proj", + "model.layers.26.mlp.experts.2.up_proj", + "model.layers.26.mlp.experts.20.down_proj", + "model.layers.26.mlp.experts.20.gate_proj", + "model.layers.26.mlp.experts.20.up_proj", + "model.layers.26.mlp.experts.200.down_proj", + "model.layers.26.mlp.experts.200.gate_proj", + "model.layers.26.mlp.experts.200.up_proj", + "model.layers.26.mlp.experts.201.down_proj", + "model.layers.26.mlp.experts.201.gate_proj", + "model.layers.26.mlp.experts.201.up_proj", + "model.layers.26.mlp.experts.202.down_proj", + "model.layers.26.mlp.experts.202.gate_proj", + "model.layers.26.mlp.experts.202.up_proj", + "model.layers.26.mlp.experts.203.down_proj", + "model.layers.26.mlp.experts.203.gate_proj", + "model.layers.26.mlp.experts.203.up_proj", + "model.layers.26.mlp.experts.204.down_proj", + "model.layers.26.mlp.experts.204.gate_proj", + "model.layers.26.mlp.experts.204.up_proj", + "model.layers.26.mlp.experts.205.down_proj", + "model.layers.26.mlp.experts.205.gate_proj", + "model.layers.26.mlp.experts.205.up_proj", + "model.layers.26.mlp.experts.206.down_proj", + "model.layers.26.mlp.experts.206.gate_proj", + "model.layers.26.mlp.experts.206.up_proj", + "model.layers.26.mlp.experts.207.down_proj", + "model.layers.26.mlp.experts.207.gate_proj", + "model.layers.26.mlp.experts.207.up_proj", + "model.layers.26.mlp.experts.208.down_proj", + "model.layers.26.mlp.experts.208.gate_proj", + "model.layers.26.mlp.experts.208.up_proj", + "model.layers.26.mlp.experts.209.down_proj", + "model.layers.26.mlp.experts.209.gate_proj", + "model.layers.26.mlp.experts.209.up_proj", + "model.layers.26.mlp.experts.21.down_proj", + "model.layers.26.mlp.experts.21.gate_proj", + "model.layers.26.mlp.experts.21.up_proj", + "model.layers.26.mlp.experts.210.down_proj", + "model.layers.26.mlp.experts.210.gate_proj", + "model.layers.26.mlp.experts.210.up_proj", + "model.layers.26.mlp.experts.211.down_proj", + "model.layers.26.mlp.experts.211.gate_proj", + "model.layers.26.mlp.experts.211.up_proj", + "model.layers.26.mlp.experts.212.down_proj", + "model.layers.26.mlp.experts.212.gate_proj", + "model.layers.26.mlp.experts.212.up_proj", + "model.layers.26.mlp.experts.213.down_proj", + "model.layers.26.mlp.experts.213.gate_proj", + "model.layers.26.mlp.experts.213.up_proj", + "model.layers.26.mlp.experts.214.down_proj", + "model.layers.26.mlp.experts.214.gate_proj", + "model.layers.26.mlp.experts.214.up_proj", + "model.layers.26.mlp.experts.215.down_proj", + "model.layers.26.mlp.experts.215.gate_proj", + "model.layers.26.mlp.experts.215.up_proj", + "model.layers.26.mlp.experts.216.down_proj", + "model.layers.26.mlp.experts.216.gate_proj", + "model.layers.26.mlp.experts.216.up_proj", + "model.layers.26.mlp.experts.217.down_proj", + "model.layers.26.mlp.experts.217.gate_proj", + "model.layers.26.mlp.experts.217.up_proj", + "model.layers.26.mlp.experts.218.down_proj", + "model.layers.26.mlp.experts.218.gate_proj", + "model.layers.26.mlp.experts.218.up_proj", + "model.layers.26.mlp.experts.219.down_proj", + "model.layers.26.mlp.experts.219.gate_proj", + "model.layers.26.mlp.experts.219.up_proj", + "model.layers.26.mlp.experts.22.down_proj", + "model.layers.26.mlp.experts.22.gate_proj", + "model.layers.26.mlp.experts.22.up_proj", + "model.layers.26.mlp.experts.220.down_proj", + "model.layers.26.mlp.experts.220.gate_proj", + "model.layers.26.mlp.experts.220.up_proj", + "model.layers.26.mlp.experts.221.down_proj", + "model.layers.26.mlp.experts.221.gate_proj", + "model.layers.26.mlp.experts.221.up_proj", + "model.layers.26.mlp.experts.222.down_proj", + "model.layers.26.mlp.experts.222.gate_proj", + "model.layers.26.mlp.experts.222.up_proj", + "model.layers.26.mlp.experts.223.down_proj", + "model.layers.26.mlp.experts.223.gate_proj", + "model.layers.26.mlp.experts.223.up_proj", + "model.layers.26.mlp.experts.224.down_proj", + "model.layers.26.mlp.experts.224.gate_proj", + "model.layers.26.mlp.experts.224.up_proj", + "model.layers.26.mlp.experts.225.down_proj", + "model.layers.26.mlp.experts.225.gate_proj", + "model.layers.26.mlp.experts.225.up_proj", + "model.layers.26.mlp.experts.226.down_proj", + "model.layers.26.mlp.experts.226.gate_proj", + "model.layers.26.mlp.experts.226.up_proj", + "model.layers.26.mlp.experts.227.down_proj", + "model.layers.26.mlp.experts.227.gate_proj", + "model.layers.26.mlp.experts.227.up_proj", + "model.layers.26.mlp.experts.228.down_proj", + "model.layers.26.mlp.experts.228.gate_proj", + "model.layers.26.mlp.experts.228.up_proj", + "model.layers.26.mlp.experts.229.down_proj", + "model.layers.26.mlp.experts.229.gate_proj", + "model.layers.26.mlp.experts.229.up_proj", + "model.layers.26.mlp.experts.23.down_proj", + "model.layers.26.mlp.experts.23.gate_proj", + "model.layers.26.mlp.experts.23.up_proj", + "model.layers.26.mlp.experts.230.down_proj", + "model.layers.26.mlp.experts.230.gate_proj", + "model.layers.26.mlp.experts.230.up_proj", + "model.layers.26.mlp.experts.231.down_proj", + "model.layers.26.mlp.experts.231.gate_proj", + "model.layers.26.mlp.experts.231.up_proj", + "model.layers.26.mlp.experts.232.down_proj", + "model.layers.26.mlp.experts.232.gate_proj", + "model.layers.26.mlp.experts.232.up_proj", + "model.layers.26.mlp.experts.233.down_proj", + "model.layers.26.mlp.experts.233.gate_proj", + "model.layers.26.mlp.experts.233.up_proj", + "model.layers.26.mlp.experts.234.down_proj", + "model.layers.26.mlp.experts.234.gate_proj", + "model.layers.26.mlp.experts.234.up_proj", + "model.layers.26.mlp.experts.235.down_proj", + "model.layers.26.mlp.experts.235.gate_proj", + "model.layers.26.mlp.experts.235.up_proj", + "model.layers.26.mlp.experts.236.down_proj", + "model.layers.26.mlp.experts.236.gate_proj", + "model.layers.26.mlp.experts.236.up_proj", + "model.layers.26.mlp.experts.237.down_proj", + "model.layers.26.mlp.experts.237.gate_proj", + "model.layers.26.mlp.experts.237.up_proj", + "model.layers.26.mlp.experts.238.down_proj", + "model.layers.26.mlp.experts.238.gate_proj", + "model.layers.26.mlp.experts.238.up_proj", + "model.layers.26.mlp.experts.239.down_proj", + "model.layers.26.mlp.experts.239.gate_proj", + "model.layers.26.mlp.experts.239.up_proj", + "model.layers.26.mlp.experts.24.down_proj", + "model.layers.26.mlp.experts.24.gate_proj", + "model.layers.26.mlp.experts.24.up_proj", + "model.layers.26.mlp.experts.240.down_proj", + "model.layers.26.mlp.experts.240.gate_proj", + "model.layers.26.mlp.experts.240.up_proj", + "model.layers.26.mlp.experts.241.down_proj", + "model.layers.26.mlp.experts.241.gate_proj", + "model.layers.26.mlp.experts.241.up_proj", + "model.layers.26.mlp.experts.242.down_proj", + "model.layers.26.mlp.experts.242.gate_proj", + "model.layers.26.mlp.experts.242.up_proj", + "model.layers.26.mlp.experts.243.down_proj", + "model.layers.26.mlp.experts.243.gate_proj", + "model.layers.26.mlp.experts.243.up_proj", + "model.layers.26.mlp.experts.244.down_proj", + "model.layers.26.mlp.experts.244.gate_proj", + "model.layers.26.mlp.experts.244.up_proj", + "model.layers.26.mlp.experts.245.down_proj", + "model.layers.26.mlp.experts.245.gate_proj", + "model.layers.26.mlp.experts.245.up_proj", + "model.layers.26.mlp.experts.246.down_proj", + "model.layers.26.mlp.experts.246.gate_proj", + "model.layers.26.mlp.experts.246.up_proj", + "model.layers.26.mlp.experts.247.down_proj", + "model.layers.26.mlp.experts.247.gate_proj", + "model.layers.26.mlp.experts.247.up_proj", + "model.layers.26.mlp.experts.248.down_proj", + "model.layers.26.mlp.experts.248.gate_proj", + "model.layers.26.mlp.experts.248.up_proj", + "model.layers.26.mlp.experts.249.down_proj", + "model.layers.26.mlp.experts.249.gate_proj", + "model.layers.26.mlp.experts.249.up_proj", + "model.layers.26.mlp.experts.25.down_proj", + "model.layers.26.mlp.experts.25.gate_proj", + "model.layers.26.mlp.experts.25.up_proj", + "model.layers.26.mlp.experts.250.down_proj", + "model.layers.26.mlp.experts.250.gate_proj", + "model.layers.26.mlp.experts.250.up_proj", + "model.layers.26.mlp.experts.251.down_proj", + "model.layers.26.mlp.experts.251.gate_proj", + "model.layers.26.mlp.experts.251.up_proj", + "model.layers.26.mlp.experts.252.down_proj", + "model.layers.26.mlp.experts.252.gate_proj", + "model.layers.26.mlp.experts.252.up_proj", + "model.layers.26.mlp.experts.253.down_proj", + "model.layers.26.mlp.experts.253.gate_proj", + "model.layers.26.mlp.experts.253.up_proj", + "model.layers.26.mlp.experts.254.down_proj", + "model.layers.26.mlp.experts.254.gate_proj", + "model.layers.26.mlp.experts.254.up_proj", + "model.layers.26.mlp.experts.255.down_proj", + "model.layers.26.mlp.experts.255.gate_proj", + "model.layers.26.mlp.experts.255.up_proj", + "model.layers.26.mlp.experts.26.down_proj", + "model.layers.26.mlp.experts.26.gate_proj", + "model.layers.26.mlp.experts.26.up_proj", + "model.layers.26.mlp.experts.27.down_proj", + "model.layers.26.mlp.experts.27.gate_proj", + "model.layers.26.mlp.experts.27.up_proj", + "model.layers.26.mlp.experts.28.down_proj", + "model.layers.26.mlp.experts.28.gate_proj", + "model.layers.26.mlp.experts.28.up_proj", + "model.layers.26.mlp.experts.29.down_proj", + "model.layers.26.mlp.experts.29.gate_proj", + "model.layers.26.mlp.experts.29.up_proj", + "model.layers.26.mlp.experts.3.down_proj", + "model.layers.26.mlp.experts.3.gate_proj", + "model.layers.26.mlp.experts.3.up_proj", + "model.layers.26.mlp.experts.30.down_proj", + "model.layers.26.mlp.experts.30.gate_proj", + "model.layers.26.mlp.experts.30.up_proj", + "model.layers.26.mlp.experts.31.down_proj", + "model.layers.26.mlp.experts.31.gate_proj", + "model.layers.26.mlp.experts.31.up_proj", + "model.layers.26.mlp.experts.32.down_proj", + "model.layers.26.mlp.experts.32.gate_proj", + "model.layers.26.mlp.experts.32.up_proj", + "model.layers.26.mlp.experts.33.down_proj", + "model.layers.26.mlp.experts.33.gate_proj", + "model.layers.26.mlp.experts.33.up_proj", + "model.layers.26.mlp.experts.34.down_proj", + "model.layers.26.mlp.experts.34.gate_proj", + "model.layers.26.mlp.experts.34.up_proj", + "model.layers.26.mlp.experts.35.down_proj", + "model.layers.26.mlp.experts.35.gate_proj", + "model.layers.26.mlp.experts.35.up_proj", + "model.layers.26.mlp.experts.36.down_proj", + "model.layers.26.mlp.experts.36.gate_proj", + "model.layers.26.mlp.experts.36.up_proj", + "model.layers.26.mlp.experts.37.down_proj", + "model.layers.26.mlp.experts.37.gate_proj", + "model.layers.26.mlp.experts.37.up_proj", + "model.layers.26.mlp.experts.38.down_proj", + "model.layers.26.mlp.experts.38.gate_proj", + "model.layers.26.mlp.experts.38.up_proj", + "model.layers.26.mlp.experts.39.down_proj", + "model.layers.26.mlp.experts.39.gate_proj", + "model.layers.26.mlp.experts.39.up_proj", + "model.layers.26.mlp.experts.4.down_proj", + "model.layers.26.mlp.experts.4.gate_proj", + "model.layers.26.mlp.experts.4.up_proj", + "model.layers.26.mlp.experts.40.down_proj", + "model.layers.26.mlp.experts.40.gate_proj", + "model.layers.26.mlp.experts.40.up_proj", + "model.layers.26.mlp.experts.41.down_proj", + "model.layers.26.mlp.experts.41.gate_proj", + "model.layers.26.mlp.experts.41.up_proj", + "model.layers.26.mlp.experts.42.down_proj", + "model.layers.26.mlp.experts.42.gate_proj", + "model.layers.26.mlp.experts.42.up_proj", + "model.layers.26.mlp.experts.43.down_proj", + "model.layers.26.mlp.experts.43.gate_proj", + "model.layers.26.mlp.experts.43.up_proj", + "model.layers.26.mlp.experts.44.down_proj", + "model.layers.26.mlp.experts.44.gate_proj", + "model.layers.26.mlp.experts.44.up_proj", + "model.layers.26.mlp.experts.45.down_proj", + "model.layers.26.mlp.experts.45.gate_proj", + "model.layers.26.mlp.experts.45.up_proj", + "model.layers.26.mlp.experts.46.down_proj", + "model.layers.26.mlp.experts.46.gate_proj", + "model.layers.26.mlp.experts.46.up_proj", + "model.layers.26.mlp.experts.47.down_proj", + "model.layers.26.mlp.experts.47.gate_proj", + "model.layers.26.mlp.experts.47.up_proj", + "model.layers.26.mlp.experts.48.down_proj", + "model.layers.26.mlp.experts.48.gate_proj", + "model.layers.26.mlp.experts.48.up_proj", + "model.layers.26.mlp.experts.49.down_proj", + "model.layers.26.mlp.experts.49.gate_proj", + "model.layers.26.mlp.experts.49.up_proj", + "model.layers.26.mlp.experts.5.down_proj", + "model.layers.26.mlp.experts.5.gate_proj", + "model.layers.26.mlp.experts.5.up_proj", + "model.layers.26.mlp.experts.50.down_proj", + "model.layers.26.mlp.experts.50.gate_proj", + "model.layers.26.mlp.experts.50.up_proj", + "model.layers.26.mlp.experts.51.down_proj", + "model.layers.26.mlp.experts.51.gate_proj", + "model.layers.26.mlp.experts.51.up_proj", + "model.layers.26.mlp.experts.52.down_proj", + "model.layers.26.mlp.experts.52.gate_proj", + "model.layers.26.mlp.experts.52.up_proj", + "model.layers.26.mlp.experts.53.down_proj", + "model.layers.26.mlp.experts.53.gate_proj", + "model.layers.26.mlp.experts.53.up_proj", + "model.layers.26.mlp.experts.54.down_proj", + "model.layers.26.mlp.experts.54.gate_proj", + "model.layers.26.mlp.experts.54.up_proj", + "model.layers.26.mlp.experts.55.down_proj", + "model.layers.26.mlp.experts.55.gate_proj", + "model.layers.26.mlp.experts.55.up_proj", + "model.layers.26.mlp.experts.56.down_proj", + "model.layers.26.mlp.experts.56.gate_proj", + "model.layers.26.mlp.experts.56.up_proj", + "model.layers.26.mlp.experts.57.down_proj", + "model.layers.26.mlp.experts.57.gate_proj", + "model.layers.26.mlp.experts.57.up_proj", + "model.layers.26.mlp.experts.58.down_proj", + "model.layers.26.mlp.experts.58.gate_proj", + "model.layers.26.mlp.experts.58.up_proj", + "model.layers.26.mlp.experts.59.down_proj", + "model.layers.26.mlp.experts.59.gate_proj", + "model.layers.26.mlp.experts.59.up_proj", + "model.layers.26.mlp.experts.6.down_proj", + "model.layers.26.mlp.experts.6.gate_proj", + "model.layers.26.mlp.experts.6.up_proj", + "model.layers.26.mlp.experts.60.down_proj", + "model.layers.26.mlp.experts.60.gate_proj", + "model.layers.26.mlp.experts.60.up_proj", + "model.layers.26.mlp.experts.61.down_proj", + "model.layers.26.mlp.experts.61.gate_proj", + "model.layers.26.mlp.experts.61.up_proj", + "model.layers.26.mlp.experts.62.down_proj", + "model.layers.26.mlp.experts.62.gate_proj", + "model.layers.26.mlp.experts.62.up_proj", + "model.layers.26.mlp.experts.63.down_proj", + "model.layers.26.mlp.experts.63.gate_proj", + "model.layers.26.mlp.experts.63.up_proj", + "model.layers.26.mlp.experts.64.down_proj", + "model.layers.26.mlp.experts.64.gate_proj", + "model.layers.26.mlp.experts.64.up_proj", + "model.layers.26.mlp.experts.65.down_proj", + "model.layers.26.mlp.experts.65.gate_proj", + "model.layers.26.mlp.experts.65.up_proj", + "model.layers.26.mlp.experts.66.down_proj", + "model.layers.26.mlp.experts.66.gate_proj", + "model.layers.26.mlp.experts.66.up_proj", + "model.layers.26.mlp.experts.67.down_proj", + "model.layers.26.mlp.experts.67.gate_proj", + "model.layers.26.mlp.experts.67.up_proj", + "model.layers.26.mlp.experts.68.down_proj", + "model.layers.26.mlp.experts.68.gate_proj", + "model.layers.26.mlp.experts.68.up_proj", + "model.layers.26.mlp.experts.69.down_proj", + "model.layers.26.mlp.experts.69.gate_proj", + "model.layers.26.mlp.experts.69.up_proj", + "model.layers.26.mlp.experts.7.down_proj", + "model.layers.26.mlp.experts.7.gate_proj", + "model.layers.26.mlp.experts.7.up_proj", + "model.layers.26.mlp.experts.70.down_proj", + "model.layers.26.mlp.experts.70.gate_proj", + "model.layers.26.mlp.experts.70.up_proj", + "model.layers.26.mlp.experts.71.down_proj", + "model.layers.26.mlp.experts.71.gate_proj", + "model.layers.26.mlp.experts.71.up_proj", + "model.layers.26.mlp.experts.72.down_proj", + "model.layers.26.mlp.experts.72.gate_proj", + "model.layers.26.mlp.experts.72.up_proj", + "model.layers.26.mlp.experts.73.down_proj", + "model.layers.26.mlp.experts.73.gate_proj", + "model.layers.26.mlp.experts.73.up_proj", + "model.layers.26.mlp.experts.74.down_proj", + "model.layers.26.mlp.experts.74.gate_proj", + "model.layers.26.mlp.experts.74.up_proj", + "model.layers.26.mlp.experts.75.down_proj", + "model.layers.26.mlp.experts.75.gate_proj", + "model.layers.26.mlp.experts.75.up_proj", + "model.layers.26.mlp.experts.76.down_proj", + "model.layers.26.mlp.experts.76.gate_proj", + "model.layers.26.mlp.experts.76.up_proj", + "model.layers.26.mlp.experts.77.down_proj", + "model.layers.26.mlp.experts.77.gate_proj", + "model.layers.26.mlp.experts.77.up_proj", + "model.layers.26.mlp.experts.78.down_proj", + "model.layers.26.mlp.experts.78.gate_proj", + "model.layers.26.mlp.experts.78.up_proj", + "model.layers.26.mlp.experts.79.down_proj", + "model.layers.26.mlp.experts.79.gate_proj", + "model.layers.26.mlp.experts.79.up_proj", + "model.layers.26.mlp.experts.8.down_proj", + "model.layers.26.mlp.experts.8.gate_proj", + "model.layers.26.mlp.experts.8.up_proj", + "model.layers.26.mlp.experts.80.down_proj", + "model.layers.26.mlp.experts.80.gate_proj", + "model.layers.26.mlp.experts.80.up_proj", + "model.layers.26.mlp.experts.81.down_proj", + "model.layers.26.mlp.experts.81.gate_proj", + "model.layers.26.mlp.experts.81.up_proj", + "model.layers.26.mlp.experts.82.down_proj", + "model.layers.26.mlp.experts.82.gate_proj", + "model.layers.26.mlp.experts.82.up_proj", + "model.layers.26.mlp.experts.83.down_proj", + "model.layers.26.mlp.experts.83.gate_proj", + "model.layers.26.mlp.experts.83.up_proj", + "model.layers.26.mlp.experts.84.down_proj", + "model.layers.26.mlp.experts.84.gate_proj", + "model.layers.26.mlp.experts.84.up_proj", + "model.layers.26.mlp.experts.85.down_proj", + "model.layers.26.mlp.experts.85.gate_proj", + "model.layers.26.mlp.experts.85.up_proj", + "model.layers.26.mlp.experts.86.down_proj", + "model.layers.26.mlp.experts.86.gate_proj", + "model.layers.26.mlp.experts.86.up_proj", + "model.layers.26.mlp.experts.87.down_proj", + "model.layers.26.mlp.experts.87.gate_proj", + "model.layers.26.mlp.experts.87.up_proj", + "model.layers.26.mlp.experts.88.down_proj", + "model.layers.26.mlp.experts.88.gate_proj", + "model.layers.26.mlp.experts.88.up_proj", + "model.layers.26.mlp.experts.89.down_proj", + "model.layers.26.mlp.experts.89.gate_proj", + "model.layers.26.mlp.experts.89.up_proj", + "model.layers.26.mlp.experts.9.down_proj", + "model.layers.26.mlp.experts.9.gate_proj", + "model.layers.26.mlp.experts.9.up_proj", + "model.layers.26.mlp.experts.90.down_proj", + "model.layers.26.mlp.experts.90.gate_proj", + "model.layers.26.mlp.experts.90.up_proj", + "model.layers.26.mlp.experts.91.down_proj", + "model.layers.26.mlp.experts.91.gate_proj", + "model.layers.26.mlp.experts.91.up_proj", + "model.layers.26.mlp.experts.92.down_proj", + "model.layers.26.mlp.experts.92.gate_proj", + "model.layers.26.mlp.experts.92.up_proj", + "model.layers.26.mlp.experts.93.down_proj", + "model.layers.26.mlp.experts.93.gate_proj", + "model.layers.26.mlp.experts.93.up_proj", + "model.layers.26.mlp.experts.94.down_proj", + "model.layers.26.mlp.experts.94.gate_proj", + "model.layers.26.mlp.experts.94.up_proj", + "model.layers.26.mlp.experts.95.down_proj", + "model.layers.26.mlp.experts.95.gate_proj", + "model.layers.26.mlp.experts.95.up_proj", + "model.layers.26.mlp.experts.96.down_proj", + "model.layers.26.mlp.experts.96.gate_proj", + "model.layers.26.mlp.experts.96.up_proj", + "model.layers.26.mlp.experts.97.down_proj", + "model.layers.26.mlp.experts.97.gate_proj", + "model.layers.26.mlp.experts.97.up_proj", + "model.layers.26.mlp.experts.98.down_proj", + "model.layers.26.mlp.experts.98.gate_proj", + "model.layers.26.mlp.experts.98.up_proj", + "model.layers.26.mlp.experts.99.down_proj", + "model.layers.26.mlp.experts.99.gate_proj", + "model.layers.26.mlp.experts.99.up_proj", + "model.layers.27.mlp.experts.0.down_proj", + "model.layers.27.mlp.experts.0.gate_proj", + "model.layers.27.mlp.experts.0.up_proj", + "model.layers.27.mlp.experts.1.down_proj", + "model.layers.27.mlp.experts.1.gate_proj", + "model.layers.27.mlp.experts.1.up_proj", + "model.layers.27.mlp.experts.10.down_proj", + "model.layers.27.mlp.experts.10.gate_proj", + "model.layers.27.mlp.experts.10.up_proj", + "model.layers.27.mlp.experts.100.down_proj", + "model.layers.27.mlp.experts.100.gate_proj", + "model.layers.27.mlp.experts.100.up_proj", + "model.layers.27.mlp.experts.101.down_proj", + "model.layers.27.mlp.experts.101.gate_proj", + "model.layers.27.mlp.experts.101.up_proj", + "model.layers.27.mlp.experts.102.down_proj", + "model.layers.27.mlp.experts.102.gate_proj", + "model.layers.27.mlp.experts.102.up_proj", + "model.layers.27.mlp.experts.103.down_proj", + "model.layers.27.mlp.experts.103.gate_proj", + "model.layers.27.mlp.experts.103.up_proj", + "model.layers.27.mlp.experts.104.down_proj", + "model.layers.27.mlp.experts.104.gate_proj", + "model.layers.27.mlp.experts.104.up_proj", + "model.layers.27.mlp.experts.105.down_proj", + "model.layers.27.mlp.experts.105.gate_proj", + "model.layers.27.mlp.experts.105.up_proj", + "model.layers.27.mlp.experts.106.down_proj", + "model.layers.27.mlp.experts.106.gate_proj", + "model.layers.27.mlp.experts.106.up_proj", + "model.layers.27.mlp.experts.107.down_proj", + "model.layers.27.mlp.experts.107.gate_proj", + "model.layers.27.mlp.experts.107.up_proj", + "model.layers.27.mlp.experts.108.down_proj", + "model.layers.27.mlp.experts.108.gate_proj", + "model.layers.27.mlp.experts.108.up_proj", + "model.layers.27.mlp.experts.109.down_proj", + "model.layers.27.mlp.experts.109.gate_proj", + "model.layers.27.mlp.experts.109.up_proj", + "model.layers.27.mlp.experts.11.down_proj", + "model.layers.27.mlp.experts.11.gate_proj", + "model.layers.27.mlp.experts.11.up_proj", + "model.layers.27.mlp.experts.110.down_proj", + "model.layers.27.mlp.experts.110.gate_proj", + "model.layers.27.mlp.experts.110.up_proj", + "model.layers.27.mlp.experts.111.down_proj", + "model.layers.27.mlp.experts.111.gate_proj", + "model.layers.27.mlp.experts.111.up_proj", + "model.layers.27.mlp.experts.112.down_proj", + "model.layers.27.mlp.experts.112.gate_proj", + "model.layers.27.mlp.experts.112.up_proj", + "model.layers.27.mlp.experts.113.down_proj", + "model.layers.27.mlp.experts.113.gate_proj", + "model.layers.27.mlp.experts.113.up_proj", + "model.layers.27.mlp.experts.114.down_proj", + "model.layers.27.mlp.experts.114.gate_proj", + "model.layers.27.mlp.experts.114.up_proj", + "model.layers.27.mlp.experts.115.down_proj", + "model.layers.27.mlp.experts.115.gate_proj", + "model.layers.27.mlp.experts.115.up_proj", + "model.layers.27.mlp.experts.116.down_proj", + "model.layers.27.mlp.experts.116.gate_proj", + "model.layers.27.mlp.experts.116.up_proj", + "model.layers.27.mlp.experts.117.down_proj", + "model.layers.27.mlp.experts.117.gate_proj", + "model.layers.27.mlp.experts.117.up_proj", + "model.layers.27.mlp.experts.118.down_proj", + "model.layers.27.mlp.experts.118.gate_proj", + "model.layers.27.mlp.experts.118.up_proj", + "model.layers.27.mlp.experts.119.down_proj", + "model.layers.27.mlp.experts.119.gate_proj", + "model.layers.27.mlp.experts.119.up_proj", + "model.layers.27.mlp.experts.12.down_proj", + "model.layers.27.mlp.experts.12.gate_proj", + "model.layers.27.mlp.experts.12.up_proj", + "model.layers.27.mlp.experts.120.down_proj", + "model.layers.27.mlp.experts.120.gate_proj", + "model.layers.27.mlp.experts.120.up_proj", + "model.layers.27.mlp.experts.121.down_proj", + "model.layers.27.mlp.experts.121.gate_proj", + "model.layers.27.mlp.experts.121.up_proj", + "model.layers.27.mlp.experts.122.down_proj", + "model.layers.27.mlp.experts.122.gate_proj", + "model.layers.27.mlp.experts.122.up_proj", + "model.layers.27.mlp.experts.123.down_proj", + "model.layers.27.mlp.experts.123.gate_proj", + "model.layers.27.mlp.experts.123.up_proj", + "model.layers.27.mlp.experts.124.down_proj", + "model.layers.27.mlp.experts.124.gate_proj", + "model.layers.27.mlp.experts.124.up_proj", + "model.layers.27.mlp.experts.125.down_proj", + "model.layers.27.mlp.experts.125.gate_proj", + "model.layers.27.mlp.experts.125.up_proj", + "model.layers.27.mlp.experts.126.down_proj", + "model.layers.27.mlp.experts.126.gate_proj", + "model.layers.27.mlp.experts.126.up_proj", + "model.layers.27.mlp.experts.127.down_proj", + "model.layers.27.mlp.experts.127.gate_proj", + "model.layers.27.mlp.experts.127.up_proj", + "model.layers.27.mlp.experts.128.down_proj", + "model.layers.27.mlp.experts.128.gate_proj", + "model.layers.27.mlp.experts.128.up_proj", + "model.layers.27.mlp.experts.129.down_proj", + "model.layers.27.mlp.experts.129.gate_proj", + "model.layers.27.mlp.experts.129.up_proj", + "model.layers.27.mlp.experts.13.down_proj", + "model.layers.27.mlp.experts.13.gate_proj", + "model.layers.27.mlp.experts.13.up_proj", + "model.layers.27.mlp.experts.130.down_proj", + "model.layers.27.mlp.experts.130.gate_proj", + "model.layers.27.mlp.experts.130.up_proj", + "model.layers.27.mlp.experts.131.down_proj", + "model.layers.27.mlp.experts.131.gate_proj", + "model.layers.27.mlp.experts.131.up_proj", + "model.layers.27.mlp.experts.132.down_proj", + "model.layers.27.mlp.experts.132.gate_proj", + "model.layers.27.mlp.experts.132.up_proj", + "model.layers.27.mlp.experts.133.down_proj", + "model.layers.27.mlp.experts.133.gate_proj", + "model.layers.27.mlp.experts.133.up_proj", + "model.layers.27.mlp.experts.134.down_proj", + "model.layers.27.mlp.experts.134.gate_proj", + "model.layers.27.mlp.experts.134.up_proj", + "model.layers.27.mlp.experts.135.down_proj", + "model.layers.27.mlp.experts.135.gate_proj", + "model.layers.27.mlp.experts.135.up_proj", + "model.layers.27.mlp.experts.136.down_proj", + "model.layers.27.mlp.experts.136.gate_proj", + "model.layers.27.mlp.experts.136.up_proj", + "model.layers.27.mlp.experts.137.down_proj", + "model.layers.27.mlp.experts.137.gate_proj", + "model.layers.27.mlp.experts.137.up_proj", + "model.layers.27.mlp.experts.138.down_proj", + "model.layers.27.mlp.experts.138.gate_proj", + "model.layers.27.mlp.experts.138.up_proj", + "model.layers.27.mlp.experts.139.down_proj", + "model.layers.27.mlp.experts.139.gate_proj", + "model.layers.27.mlp.experts.139.up_proj", + "model.layers.27.mlp.experts.14.down_proj", + "model.layers.27.mlp.experts.14.gate_proj", + "model.layers.27.mlp.experts.14.up_proj", + "model.layers.27.mlp.experts.140.down_proj", + "model.layers.27.mlp.experts.140.gate_proj", + "model.layers.27.mlp.experts.140.up_proj", + "model.layers.27.mlp.experts.141.down_proj", + "model.layers.27.mlp.experts.141.gate_proj", + "model.layers.27.mlp.experts.141.up_proj", + "model.layers.27.mlp.experts.142.down_proj", + "model.layers.27.mlp.experts.142.gate_proj", + "model.layers.27.mlp.experts.142.up_proj", + "model.layers.27.mlp.experts.143.down_proj", + "model.layers.27.mlp.experts.143.gate_proj", + "model.layers.27.mlp.experts.143.up_proj", + "model.layers.27.mlp.experts.144.down_proj", + "model.layers.27.mlp.experts.144.gate_proj", + "model.layers.27.mlp.experts.144.up_proj", + "model.layers.27.mlp.experts.145.down_proj", + "model.layers.27.mlp.experts.145.gate_proj", + "model.layers.27.mlp.experts.145.up_proj", + "model.layers.27.mlp.experts.146.down_proj", + "model.layers.27.mlp.experts.146.gate_proj", + "model.layers.27.mlp.experts.146.up_proj", + "model.layers.27.mlp.experts.147.down_proj", + "model.layers.27.mlp.experts.147.gate_proj", + "model.layers.27.mlp.experts.147.up_proj", + "model.layers.27.mlp.experts.148.down_proj", + "model.layers.27.mlp.experts.148.gate_proj", + "model.layers.27.mlp.experts.148.up_proj", + "model.layers.27.mlp.experts.149.down_proj", + "model.layers.27.mlp.experts.149.gate_proj", + "model.layers.27.mlp.experts.149.up_proj", + "model.layers.27.mlp.experts.15.down_proj", + "model.layers.27.mlp.experts.15.gate_proj", + "model.layers.27.mlp.experts.15.up_proj", + "model.layers.27.mlp.experts.150.down_proj", + "model.layers.27.mlp.experts.150.gate_proj", + "model.layers.27.mlp.experts.150.up_proj", + "model.layers.27.mlp.experts.151.down_proj", + "model.layers.27.mlp.experts.151.gate_proj", + "model.layers.27.mlp.experts.151.up_proj", + "model.layers.27.mlp.experts.152.down_proj", + "model.layers.27.mlp.experts.152.gate_proj", + "model.layers.27.mlp.experts.152.up_proj", + "model.layers.27.mlp.experts.153.down_proj", + "model.layers.27.mlp.experts.153.gate_proj", + "model.layers.27.mlp.experts.153.up_proj", + "model.layers.27.mlp.experts.154.down_proj", + "model.layers.27.mlp.experts.154.gate_proj", + "model.layers.27.mlp.experts.154.up_proj", + "model.layers.27.mlp.experts.155.down_proj", + "model.layers.27.mlp.experts.155.gate_proj", + "model.layers.27.mlp.experts.155.up_proj", + "model.layers.27.mlp.experts.156.down_proj", + "model.layers.27.mlp.experts.156.gate_proj", + "model.layers.27.mlp.experts.156.up_proj", + "model.layers.27.mlp.experts.157.down_proj", + "model.layers.27.mlp.experts.157.gate_proj", + "model.layers.27.mlp.experts.157.up_proj", + "model.layers.27.mlp.experts.158.down_proj", + "model.layers.27.mlp.experts.158.gate_proj", + "model.layers.27.mlp.experts.158.up_proj", + "model.layers.27.mlp.experts.159.down_proj", + "model.layers.27.mlp.experts.159.gate_proj", + "model.layers.27.mlp.experts.159.up_proj", + "model.layers.27.mlp.experts.16.down_proj", + "model.layers.27.mlp.experts.16.gate_proj", + "model.layers.27.mlp.experts.16.up_proj", + "model.layers.27.mlp.experts.160.down_proj", + "model.layers.27.mlp.experts.160.gate_proj", + "model.layers.27.mlp.experts.160.up_proj", + "model.layers.27.mlp.experts.161.down_proj", + "model.layers.27.mlp.experts.161.gate_proj", + "model.layers.27.mlp.experts.161.up_proj", + "model.layers.27.mlp.experts.162.down_proj", + "model.layers.27.mlp.experts.162.gate_proj", + "model.layers.27.mlp.experts.162.up_proj", + "model.layers.27.mlp.experts.163.down_proj", + "model.layers.27.mlp.experts.163.gate_proj", + "model.layers.27.mlp.experts.163.up_proj", + "model.layers.27.mlp.experts.164.down_proj", + "model.layers.27.mlp.experts.164.gate_proj", + "model.layers.27.mlp.experts.164.up_proj", + "model.layers.27.mlp.experts.165.down_proj", + "model.layers.27.mlp.experts.165.gate_proj", + "model.layers.27.mlp.experts.165.up_proj", + "model.layers.27.mlp.experts.166.down_proj", + "model.layers.27.mlp.experts.166.gate_proj", + "model.layers.27.mlp.experts.166.up_proj", + "model.layers.27.mlp.experts.167.down_proj", + "model.layers.27.mlp.experts.167.gate_proj", + "model.layers.27.mlp.experts.167.up_proj", + "model.layers.27.mlp.experts.168.down_proj", + "model.layers.27.mlp.experts.168.gate_proj", + "model.layers.27.mlp.experts.168.up_proj", + "model.layers.27.mlp.experts.169.down_proj", + "model.layers.27.mlp.experts.169.gate_proj", + "model.layers.27.mlp.experts.169.up_proj", + "model.layers.27.mlp.experts.17.down_proj", + "model.layers.27.mlp.experts.17.gate_proj", + "model.layers.27.mlp.experts.17.up_proj", + "model.layers.27.mlp.experts.170.down_proj", + "model.layers.27.mlp.experts.170.gate_proj", + "model.layers.27.mlp.experts.170.up_proj", + "model.layers.27.mlp.experts.171.down_proj", + "model.layers.27.mlp.experts.171.gate_proj", + "model.layers.27.mlp.experts.171.up_proj", + "model.layers.27.mlp.experts.172.down_proj", + "model.layers.27.mlp.experts.172.gate_proj", + "model.layers.27.mlp.experts.172.up_proj", + "model.layers.27.mlp.experts.173.down_proj", + "model.layers.27.mlp.experts.173.gate_proj", + "model.layers.27.mlp.experts.173.up_proj", + "model.layers.27.mlp.experts.174.down_proj", + "model.layers.27.mlp.experts.174.gate_proj", + "model.layers.27.mlp.experts.174.up_proj", + "model.layers.27.mlp.experts.175.down_proj", + "model.layers.27.mlp.experts.175.gate_proj", + "model.layers.27.mlp.experts.175.up_proj", + "model.layers.27.mlp.experts.176.down_proj", + "model.layers.27.mlp.experts.176.gate_proj", + "model.layers.27.mlp.experts.176.up_proj", + "model.layers.27.mlp.experts.177.down_proj", + "model.layers.27.mlp.experts.177.gate_proj", + "model.layers.27.mlp.experts.177.up_proj", + "model.layers.27.mlp.experts.178.down_proj", + "model.layers.27.mlp.experts.178.gate_proj", + "model.layers.27.mlp.experts.178.up_proj", + "model.layers.27.mlp.experts.179.down_proj", + "model.layers.27.mlp.experts.179.gate_proj", + "model.layers.27.mlp.experts.179.up_proj", + "model.layers.27.mlp.experts.18.down_proj", + "model.layers.27.mlp.experts.18.gate_proj", + "model.layers.27.mlp.experts.18.up_proj", + "model.layers.27.mlp.experts.180.down_proj", + "model.layers.27.mlp.experts.180.gate_proj", + "model.layers.27.mlp.experts.180.up_proj", + "model.layers.27.mlp.experts.181.down_proj", + "model.layers.27.mlp.experts.181.gate_proj", + "model.layers.27.mlp.experts.181.up_proj", + "model.layers.27.mlp.experts.182.down_proj", + "model.layers.27.mlp.experts.182.gate_proj", + "model.layers.27.mlp.experts.182.up_proj", + "model.layers.27.mlp.experts.183.down_proj", + "model.layers.27.mlp.experts.183.gate_proj", + "model.layers.27.mlp.experts.183.up_proj", + "model.layers.27.mlp.experts.184.down_proj", + "model.layers.27.mlp.experts.184.gate_proj", + "model.layers.27.mlp.experts.184.up_proj", + "model.layers.27.mlp.experts.185.down_proj", + "model.layers.27.mlp.experts.185.gate_proj", + "model.layers.27.mlp.experts.185.up_proj", + "model.layers.27.mlp.experts.186.down_proj", + "model.layers.27.mlp.experts.186.gate_proj", + "model.layers.27.mlp.experts.186.up_proj", + "model.layers.27.mlp.experts.187.down_proj", + "model.layers.27.mlp.experts.187.gate_proj", + "model.layers.27.mlp.experts.187.up_proj", + "model.layers.27.mlp.experts.188.down_proj", + "model.layers.27.mlp.experts.188.gate_proj", + "model.layers.27.mlp.experts.188.up_proj", + "model.layers.27.mlp.experts.189.down_proj", + "model.layers.27.mlp.experts.189.gate_proj", + "model.layers.27.mlp.experts.189.up_proj", + "model.layers.27.mlp.experts.19.down_proj", + "model.layers.27.mlp.experts.19.gate_proj", + "model.layers.27.mlp.experts.19.up_proj", + "model.layers.27.mlp.experts.190.down_proj", + "model.layers.27.mlp.experts.190.gate_proj", + "model.layers.27.mlp.experts.190.up_proj", + "model.layers.27.mlp.experts.191.down_proj", + "model.layers.27.mlp.experts.191.gate_proj", + "model.layers.27.mlp.experts.191.up_proj", + "model.layers.27.mlp.experts.192.down_proj", + "model.layers.27.mlp.experts.192.gate_proj", + "model.layers.27.mlp.experts.192.up_proj", + "model.layers.27.mlp.experts.193.down_proj", + "model.layers.27.mlp.experts.193.gate_proj", + "model.layers.27.mlp.experts.193.up_proj", + "model.layers.27.mlp.experts.194.down_proj", + "model.layers.27.mlp.experts.194.gate_proj", + "model.layers.27.mlp.experts.194.up_proj", + "model.layers.27.mlp.experts.195.down_proj", + "model.layers.27.mlp.experts.195.gate_proj", + "model.layers.27.mlp.experts.195.up_proj", + "model.layers.27.mlp.experts.196.down_proj", + "model.layers.27.mlp.experts.196.gate_proj", + "model.layers.27.mlp.experts.196.up_proj", + "model.layers.27.mlp.experts.197.down_proj", + "model.layers.27.mlp.experts.197.gate_proj", + "model.layers.27.mlp.experts.197.up_proj", + "model.layers.27.mlp.experts.198.down_proj", + "model.layers.27.mlp.experts.198.gate_proj", + "model.layers.27.mlp.experts.198.up_proj", + "model.layers.27.mlp.experts.199.down_proj", + "model.layers.27.mlp.experts.199.gate_proj", + "model.layers.27.mlp.experts.199.up_proj", + "model.layers.27.mlp.experts.2.down_proj", + "model.layers.27.mlp.experts.2.gate_proj", + "model.layers.27.mlp.experts.2.up_proj", + "model.layers.27.mlp.experts.20.down_proj", + "model.layers.27.mlp.experts.20.gate_proj", + "model.layers.27.mlp.experts.20.up_proj", + "model.layers.27.mlp.experts.200.down_proj", + "model.layers.27.mlp.experts.200.gate_proj", + "model.layers.27.mlp.experts.200.up_proj", + "model.layers.27.mlp.experts.201.down_proj", + "model.layers.27.mlp.experts.201.gate_proj", + "model.layers.27.mlp.experts.201.up_proj", + "model.layers.27.mlp.experts.202.down_proj", + "model.layers.27.mlp.experts.202.gate_proj", + "model.layers.27.mlp.experts.202.up_proj", + "model.layers.27.mlp.experts.203.down_proj", + "model.layers.27.mlp.experts.203.gate_proj", + "model.layers.27.mlp.experts.203.up_proj", + "model.layers.27.mlp.experts.204.down_proj", + "model.layers.27.mlp.experts.204.gate_proj", + "model.layers.27.mlp.experts.204.up_proj", + "model.layers.27.mlp.experts.205.down_proj", + "model.layers.27.mlp.experts.205.gate_proj", + "model.layers.27.mlp.experts.205.up_proj", + "model.layers.27.mlp.experts.206.down_proj", + "model.layers.27.mlp.experts.206.gate_proj", + "model.layers.27.mlp.experts.206.up_proj", + "model.layers.27.mlp.experts.207.down_proj", + "model.layers.27.mlp.experts.207.gate_proj", + "model.layers.27.mlp.experts.207.up_proj", + "model.layers.27.mlp.experts.208.down_proj", + "model.layers.27.mlp.experts.208.gate_proj", + "model.layers.27.mlp.experts.208.up_proj", + "model.layers.27.mlp.experts.209.down_proj", + "model.layers.27.mlp.experts.209.gate_proj", + "model.layers.27.mlp.experts.209.up_proj", + "model.layers.27.mlp.experts.21.down_proj", + "model.layers.27.mlp.experts.21.gate_proj", + "model.layers.27.mlp.experts.21.up_proj", + "model.layers.27.mlp.experts.210.down_proj", + "model.layers.27.mlp.experts.210.gate_proj", + "model.layers.27.mlp.experts.210.up_proj", + "model.layers.27.mlp.experts.211.down_proj", + "model.layers.27.mlp.experts.211.gate_proj", + "model.layers.27.mlp.experts.211.up_proj", + "model.layers.27.mlp.experts.212.down_proj", + "model.layers.27.mlp.experts.212.gate_proj", + "model.layers.27.mlp.experts.212.up_proj", + "model.layers.27.mlp.experts.213.down_proj", + "model.layers.27.mlp.experts.213.gate_proj", + "model.layers.27.mlp.experts.213.up_proj", + "model.layers.27.mlp.experts.214.down_proj", + "model.layers.27.mlp.experts.214.gate_proj", + "model.layers.27.mlp.experts.214.up_proj", + "model.layers.27.mlp.experts.215.down_proj", + "model.layers.27.mlp.experts.215.gate_proj", + "model.layers.27.mlp.experts.215.up_proj", + "model.layers.27.mlp.experts.216.down_proj", + "model.layers.27.mlp.experts.216.gate_proj", + "model.layers.27.mlp.experts.216.up_proj", + "model.layers.27.mlp.experts.217.down_proj", + "model.layers.27.mlp.experts.217.gate_proj", + "model.layers.27.mlp.experts.217.up_proj", + "model.layers.27.mlp.experts.218.down_proj", + "model.layers.27.mlp.experts.218.gate_proj", + "model.layers.27.mlp.experts.218.up_proj", + "model.layers.27.mlp.experts.219.down_proj", + "model.layers.27.mlp.experts.219.gate_proj", + "model.layers.27.mlp.experts.219.up_proj", + "model.layers.27.mlp.experts.22.down_proj", + "model.layers.27.mlp.experts.22.gate_proj", + "model.layers.27.mlp.experts.22.up_proj", + "model.layers.27.mlp.experts.220.down_proj", + "model.layers.27.mlp.experts.220.gate_proj", + "model.layers.27.mlp.experts.220.up_proj", + "model.layers.27.mlp.experts.221.down_proj", + "model.layers.27.mlp.experts.221.gate_proj", + "model.layers.27.mlp.experts.221.up_proj", + "model.layers.27.mlp.experts.222.down_proj", + "model.layers.27.mlp.experts.222.gate_proj", + "model.layers.27.mlp.experts.222.up_proj", + "model.layers.27.mlp.experts.223.down_proj", + "model.layers.27.mlp.experts.223.gate_proj", + "model.layers.27.mlp.experts.223.up_proj", + "model.layers.27.mlp.experts.224.down_proj", + "model.layers.27.mlp.experts.224.gate_proj", + "model.layers.27.mlp.experts.224.up_proj", + "model.layers.27.mlp.experts.225.down_proj", + "model.layers.27.mlp.experts.225.gate_proj", + "model.layers.27.mlp.experts.225.up_proj", + "model.layers.27.mlp.experts.226.down_proj", + "model.layers.27.mlp.experts.226.gate_proj", + "model.layers.27.mlp.experts.226.up_proj", + "model.layers.27.mlp.experts.227.down_proj", + "model.layers.27.mlp.experts.227.gate_proj", + "model.layers.27.mlp.experts.227.up_proj", + "model.layers.27.mlp.experts.228.down_proj", + "model.layers.27.mlp.experts.228.gate_proj", + "model.layers.27.mlp.experts.228.up_proj", + "model.layers.27.mlp.experts.229.down_proj", + "model.layers.27.mlp.experts.229.gate_proj", + "model.layers.27.mlp.experts.229.up_proj", + "model.layers.27.mlp.experts.23.down_proj", + "model.layers.27.mlp.experts.23.gate_proj", + "model.layers.27.mlp.experts.23.up_proj", + "model.layers.27.mlp.experts.230.down_proj", + "model.layers.27.mlp.experts.230.gate_proj", + "model.layers.27.mlp.experts.230.up_proj", + "model.layers.27.mlp.experts.231.down_proj", + "model.layers.27.mlp.experts.231.gate_proj", + "model.layers.27.mlp.experts.231.up_proj", + "model.layers.27.mlp.experts.232.down_proj", + "model.layers.27.mlp.experts.232.gate_proj", + "model.layers.27.mlp.experts.232.up_proj", + "model.layers.27.mlp.experts.233.down_proj", + "model.layers.27.mlp.experts.233.gate_proj", + "model.layers.27.mlp.experts.233.up_proj", + "model.layers.27.mlp.experts.234.down_proj", + "model.layers.27.mlp.experts.234.gate_proj", + "model.layers.27.mlp.experts.234.up_proj", + "model.layers.27.mlp.experts.235.down_proj", + "model.layers.27.mlp.experts.235.gate_proj", + "model.layers.27.mlp.experts.235.up_proj", + "model.layers.27.mlp.experts.236.down_proj", + "model.layers.27.mlp.experts.236.gate_proj", + "model.layers.27.mlp.experts.236.up_proj", + "model.layers.27.mlp.experts.237.down_proj", + "model.layers.27.mlp.experts.237.gate_proj", + "model.layers.27.mlp.experts.237.up_proj", + "model.layers.27.mlp.experts.238.down_proj", + "model.layers.27.mlp.experts.238.gate_proj", + "model.layers.27.mlp.experts.238.up_proj", + "model.layers.27.mlp.experts.239.down_proj", + "model.layers.27.mlp.experts.239.gate_proj", + "model.layers.27.mlp.experts.239.up_proj", + "model.layers.27.mlp.experts.24.down_proj", + "model.layers.27.mlp.experts.24.gate_proj", + "model.layers.27.mlp.experts.24.up_proj", + "model.layers.27.mlp.experts.240.down_proj", + "model.layers.27.mlp.experts.240.gate_proj", + "model.layers.27.mlp.experts.240.up_proj", + "model.layers.27.mlp.experts.241.down_proj", + "model.layers.27.mlp.experts.241.gate_proj", + "model.layers.27.mlp.experts.241.up_proj", + "model.layers.27.mlp.experts.242.down_proj", + "model.layers.27.mlp.experts.242.gate_proj", + "model.layers.27.mlp.experts.242.up_proj", + "model.layers.27.mlp.experts.243.down_proj", + "model.layers.27.mlp.experts.243.gate_proj", + "model.layers.27.mlp.experts.243.up_proj", + "model.layers.27.mlp.experts.244.down_proj", + "model.layers.27.mlp.experts.244.gate_proj", + "model.layers.27.mlp.experts.244.up_proj", + "model.layers.27.mlp.experts.245.down_proj", + "model.layers.27.mlp.experts.245.gate_proj", + "model.layers.27.mlp.experts.245.up_proj", + "model.layers.27.mlp.experts.246.down_proj", + "model.layers.27.mlp.experts.246.gate_proj", + "model.layers.27.mlp.experts.246.up_proj", + "model.layers.27.mlp.experts.247.down_proj", + "model.layers.27.mlp.experts.247.gate_proj", + "model.layers.27.mlp.experts.247.up_proj", + "model.layers.27.mlp.experts.248.down_proj", + "model.layers.27.mlp.experts.248.gate_proj", + "model.layers.27.mlp.experts.248.up_proj", + "model.layers.27.mlp.experts.249.down_proj", + "model.layers.27.mlp.experts.249.gate_proj", + "model.layers.27.mlp.experts.249.up_proj", + "model.layers.27.mlp.experts.25.down_proj", + "model.layers.27.mlp.experts.25.gate_proj", + "model.layers.27.mlp.experts.25.up_proj", + "model.layers.27.mlp.experts.250.down_proj", + "model.layers.27.mlp.experts.250.gate_proj", + "model.layers.27.mlp.experts.250.up_proj", + "model.layers.27.mlp.experts.251.down_proj", + "model.layers.27.mlp.experts.251.gate_proj", + "model.layers.27.mlp.experts.251.up_proj", + "model.layers.27.mlp.experts.252.down_proj", + "model.layers.27.mlp.experts.252.gate_proj", + "model.layers.27.mlp.experts.252.up_proj", + "model.layers.27.mlp.experts.253.down_proj", + "model.layers.27.mlp.experts.253.gate_proj", + "model.layers.27.mlp.experts.253.up_proj", + "model.layers.27.mlp.experts.254.down_proj", + "model.layers.27.mlp.experts.254.gate_proj", + "model.layers.27.mlp.experts.254.up_proj", + "model.layers.27.mlp.experts.255.down_proj", + "model.layers.27.mlp.experts.255.gate_proj", + "model.layers.27.mlp.experts.255.up_proj", + "model.layers.27.mlp.experts.26.down_proj", + "model.layers.27.mlp.experts.26.gate_proj", + "model.layers.27.mlp.experts.26.up_proj", + "model.layers.27.mlp.experts.27.down_proj", + "model.layers.27.mlp.experts.27.gate_proj", + "model.layers.27.mlp.experts.27.up_proj", + "model.layers.27.mlp.experts.28.down_proj", + "model.layers.27.mlp.experts.28.gate_proj", + "model.layers.27.mlp.experts.28.up_proj", + "model.layers.27.mlp.experts.29.down_proj", + "model.layers.27.mlp.experts.29.gate_proj", + "model.layers.27.mlp.experts.29.up_proj", + "model.layers.27.mlp.experts.3.down_proj", + "model.layers.27.mlp.experts.3.gate_proj", + "model.layers.27.mlp.experts.3.up_proj", + "model.layers.27.mlp.experts.30.down_proj", + "model.layers.27.mlp.experts.30.gate_proj", + "model.layers.27.mlp.experts.30.up_proj", + "model.layers.27.mlp.experts.31.down_proj", + "model.layers.27.mlp.experts.31.gate_proj", + "model.layers.27.mlp.experts.31.up_proj", + "model.layers.27.mlp.experts.32.down_proj", + "model.layers.27.mlp.experts.32.gate_proj", + "model.layers.27.mlp.experts.32.up_proj", + "model.layers.27.mlp.experts.33.down_proj", + "model.layers.27.mlp.experts.33.gate_proj", + "model.layers.27.mlp.experts.33.up_proj", + "model.layers.27.mlp.experts.34.down_proj", + "model.layers.27.mlp.experts.34.gate_proj", + "model.layers.27.mlp.experts.34.up_proj", + "model.layers.27.mlp.experts.35.down_proj", + "model.layers.27.mlp.experts.35.gate_proj", + "model.layers.27.mlp.experts.35.up_proj", + "model.layers.27.mlp.experts.36.down_proj", + "model.layers.27.mlp.experts.36.gate_proj", + "model.layers.27.mlp.experts.36.up_proj", + "model.layers.27.mlp.experts.37.down_proj", + "model.layers.27.mlp.experts.37.gate_proj", + "model.layers.27.mlp.experts.37.up_proj", + "model.layers.27.mlp.experts.38.down_proj", + "model.layers.27.mlp.experts.38.gate_proj", + "model.layers.27.mlp.experts.38.up_proj", + "model.layers.27.mlp.experts.39.down_proj", + "model.layers.27.mlp.experts.39.gate_proj", + "model.layers.27.mlp.experts.39.up_proj", + "model.layers.27.mlp.experts.4.down_proj", + "model.layers.27.mlp.experts.4.gate_proj", + "model.layers.27.mlp.experts.4.up_proj", + "model.layers.27.mlp.experts.40.down_proj", + "model.layers.27.mlp.experts.40.gate_proj", + "model.layers.27.mlp.experts.40.up_proj", + "model.layers.27.mlp.experts.41.down_proj", + "model.layers.27.mlp.experts.41.gate_proj", + "model.layers.27.mlp.experts.41.up_proj", + "model.layers.27.mlp.experts.42.down_proj", + "model.layers.27.mlp.experts.42.gate_proj", + "model.layers.27.mlp.experts.42.up_proj", + "model.layers.27.mlp.experts.43.down_proj", + "model.layers.27.mlp.experts.43.gate_proj", + "model.layers.27.mlp.experts.43.up_proj", + "model.layers.27.mlp.experts.44.down_proj", + "model.layers.27.mlp.experts.44.gate_proj", + "model.layers.27.mlp.experts.44.up_proj", + "model.layers.27.mlp.experts.45.down_proj", + "model.layers.27.mlp.experts.45.gate_proj", + "model.layers.27.mlp.experts.45.up_proj", + "model.layers.27.mlp.experts.46.down_proj", + "model.layers.27.mlp.experts.46.gate_proj", + "model.layers.27.mlp.experts.46.up_proj", + "model.layers.27.mlp.experts.47.down_proj", + "model.layers.27.mlp.experts.47.gate_proj", + "model.layers.27.mlp.experts.47.up_proj", + "model.layers.27.mlp.experts.48.down_proj", + "model.layers.27.mlp.experts.48.gate_proj", + "model.layers.27.mlp.experts.48.up_proj", + "model.layers.27.mlp.experts.49.down_proj", + "model.layers.27.mlp.experts.49.gate_proj", + "model.layers.27.mlp.experts.49.up_proj", + "model.layers.27.mlp.experts.5.down_proj", + "model.layers.27.mlp.experts.5.gate_proj", + "model.layers.27.mlp.experts.5.up_proj", + "model.layers.27.mlp.experts.50.down_proj", + "model.layers.27.mlp.experts.50.gate_proj", + "model.layers.27.mlp.experts.50.up_proj", + "model.layers.27.mlp.experts.51.down_proj", + "model.layers.27.mlp.experts.51.gate_proj", + "model.layers.27.mlp.experts.51.up_proj", + "model.layers.27.mlp.experts.52.down_proj", + "model.layers.27.mlp.experts.52.gate_proj", + "model.layers.27.mlp.experts.52.up_proj", + "model.layers.27.mlp.experts.53.down_proj", + "model.layers.27.mlp.experts.53.gate_proj", + "model.layers.27.mlp.experts.53.up_proj", + "model.layers.27.mlp.experts.54.down_proj", + "model.layers.27.mlp.experts.54.gate_proj", + "model.layers.27.mlp.experts.54.up_proj", + "model.layers.27.mlp.experts.55.down_proj", + "model.layers.27.mlp.experts.55.gate_proj", + "model.layers.27.mlp.experts.55.up_proj", + "model.layers.27.mlp.experts.56.down_proj", + "model.layers.27.mlp.experts.56.gate_proj", + "model.layers.27.mlp.experts.56.up_proj", + "model.layers.27.mlp.experts.57.down_proj", + "model.layers.27.mlp.experts.57.gate_proj", + "model.layers.27.mlp.experts.57.up_proj", + "model.layers.27.mlp.experts.58.down_proj", + "model.layers.27.mlp.experts.58.gate_proj", + "model.layers.27.mlp.experts.58.up_proj", + "model.layers.27.mlp.experts.59.down_proj", + "model.layers.27.mlp.experts.59.gate_proj", + "model.layers.27.mlp.experts.59.up_proj", + "model.layers.27.mlp.experts.6.down_proj", + "model.layers.27.mlp.experts.6.gate_proj", + "model.layers.27.mlp.experts.6.up_proj", + "model.layers.27.mlp.experts.60.down_proj", + "model.layers.27.mlp.experts.60.gate_proj", + "model.layers.27.mlp.experts.60.up_proj", + "model.layers.27.mlp.experts.61.down_proj", + "model.layers.27.mlp.experts.61.gate_proj", + "model.layers.27.mlp.experts.61.up_proj", + "model.layers.27.mlp.experts.62.down_proj", + "model.layers.27.mlp.experts.62.gate_proj", + "model.layers.27.mlp.experts.62.up_proj", + "model.layers.27.mlp.experts.63.down_proj", + "model.layers.27.mlp.experts.63.gate_proj", + "model.layers.27.mlp.experts.63.up_proj", + "model.layers.27.mlp.experts.64.down_proj", + "model.layers.27.mlp.experts.64.gate_proj", + "model.layers.27.mlp.experts.64.up_proj", + "model.layers.27.mlp.experts.65.down_proj", + "model.layers.27.mlp.experts.65.gate_proj", + "model.layers.27.mlp.experts.65.up_proj", + "model.layers.27.mlp.experts.66.down_proj", + "model.layers.27.mlp.experts.66.gate_proj", + "model.layers.27.mlp.experts.66.up_proj", + "model.layers.27.mlp.experts.67.down_proj", + "model.layers.27.mlp.experts.67.gate_proj", + "model.layers.27.mlp.experts.67.up_proj", + "model.layers.27.mlp.experts.68.down_proj", + "model.layers.27.mlp.experts.68.gate_proj", + "model.layers.27.mlp.experts.68.up_proj", + "model.layers.27.mlp.experts.69.down_proj", + "model.layers.27.mlp.experts.69.gate_proj", + "model.layers.27.mlp.experts.69.up_proj", + "model.layers.27.mlp.experts.7.down_proj", + "model.layers.27.mlp.experts.7.gate_proj", + "model.layers.27.mlp.experts.7.up_proj", + "model.layers.27.mlp.experts.70.down_proj", + "model.layers.27.mlp.experts.70.gate_proj", + "model.layers.27.mlp.experts.70.up_proj", + "model.layers.27.mlp.experts.71.down_proj", + "model.layers.27.mlp.experts.71.gate_proj", + "model.layers.27.mlp.experts.71.up_proj", + "model.layers.27.mlp.experts.72.down_proj", + "model.layers.27.mlp.experts.72.gate_proj", + "model.layers.27.mlp.experts.72.up_proj", + "model.layers.27.mlp.experts.73.down_proj", + "model.layers.27.mlp.experts.73.gate_proj", + "model.layers.27.mlp.experts.73.up_proj", + "model.layers.27.mlp.experts.74.down_proj", + "model.layers.27.mlp.experts.74.gate_proj", + "model.layers.27.mlp.experts.74.up_proj", + "model.layers.27.mlp.experts.75.down_proj", + "model.layers.27.mlp.experts.75.gate_proj", + "model.layers.27.mlp.experts.75.up_proj", + "model.layers.27.mlp.experts.76.down_proj", + "model.layers.27.mlp.experts.76.gate_proj", + "model.layers.27.mlp.experts.76.up_proj", + "model.layers.27.mlp.experts.77.down_proj", + "model.layers.27.mlp.experts.77.gate_proj", + "model.layers.27.mlp.experts.77.up_proj", + "model.layers.27.mlp.experts.78.down_proj", + "model.layers.27.mlp.experts.78.gate_proj", + "model.layers.27.mlp.experts.78.up_proj", + "model.layers.27.mlp.experts.79.down_proj", + "model.layers.27.mlp.experts.79.gate_proj", + "model.layers.27.mlp.experts.79.up_proj", + "model.layers.27.mlp.experts.8.down_proj", + "model.layers.27.mlp.experts.8.gate_proj", + "model.layers.27.mlp.experts.8.up_proj", + "model.layers.27.mlp.experts.80.down_proj", + "model.layers.27.mlp.experts.80.gate_proj", + "model.layers.27.mlp.experts.80.up_proj", + "model.layers.27.mlp.experts.81.down_proj", + "model.layers.27.mlp.experts.81.gate_proj", + "model.layers.27.mlp.experts.81.up_proj", + "model.layers.27.mlp.experts.82.down_proj", + "model.layers.27.mlp.experts.82.gate_proj", + "model.layers.27.mlp.experts.82.up_proj", + "model.layers.27.mlp.experts.83.down_proj", + "model.layers.27.mlp.experts.83.gate_proj", + "model.layers.27.mlp.experts.83.up_proj", + "model.layers.27.mlp.experts.84.down_proj", + "model.layers.27.mlp.experts.84.gate_proj", + "model.layers.27.mlp.experts.84.up_proj", + "model.layers.27.mlp.experts.85.down_proj", + "model.layers.27.mlp.experts.85.gate_proj", + "model.layers.27.mlp.experts.85.up_proj", + "model.layers.27.mlp.experts.86.down_proj", + "model.layers.27.mlp.experts.86.gate_proj", + "model.layers.27.mlp.experts.86.up_proj", + "model.layers.27.mlp.experts.87.down_proj", + "model.layers.27.mlp.experts.87.gate_proj", + "model.layers.27.mlp.experts.87.up_proj", + "model.layers.27.mlp.experts.88.down_proj", + "model.layers.27.mlp.experts.88.gate_proj", + "model.layers.27.mlp.experts.88.up_proj", + "model.layers.27.mlp.experts.89.down_proj", + "model.layers.27.mlp.experts.89.gate_proj", + "model.layers.27.mlp.experts.89.up_proj", + "model.layers.27.mlp.experts.9.down_proj", + "model.layers.27.mlp.experts.9.gate_proj", + "model.layers.27.mlp.experts.9.up_proj", + "model.layers.27.mlp.experts.90.down_proj", + "model.layers.27.mlp.experts.90.gate_proj", + "model.layers.27.mlp.experts.90.up_proj", + "model.layers.27.mlp.experts.91.down_proj", + "model.layers.27.mlp.experts.91.gate_proj", + "model.layers.27.mlp.experts.91.up_proj", + "model.layers.27.mlp.experts.92.down_proj", + "model.layers.27.mlp.experts.92.gate_proj", + "model.layers.27.mlp.experts.92.up_proj", + "model.layers.27.mlp.experts.93.down_proj", + "model.layers.27.mlp.experts.93.gate_proj", + "model.layers.27.mlp.experts.93.up_proj", + "model.layers.27.mlp.experts.94.down_proj", + "model.layers.27.mlp.experts.94.gate_proj", + "model.layers.27.mlp.experts.94.up_proj", + "model.layers.27.mlp.experts.95.down_proj", + "model.layers.27.mlp.experts.95.gate_proj", + "model.layers.27.mlp.experts.95.up_proj", + "model.layers.27.mlp.experts.96.down_proj", + "model.layers.27.mlp.experts.96.gate_proj", + "model.layers.27.mlp.experts.96.up_proj", + "model.layers.27.mlp.experts.97.down_proj", + "model.layers.27.mlp.experts.97.gate_proj", + "model.layers.27.mlp.experts.97.up_proj", + "model.layers.27.mlp.experts.98.down_proj", + "model.layers.27.mlp.experts.98.gate_proj", + "model.layers.27.mlp.experts.98.up_proj", + "model.layers.27.mlp.experts.99.down_proj", + "model.layers.27.mlp.experts.99.gate_proj", + "model.layers.27.mlp.experts.99.up_proj", + "model.layers.28.mlp.experts.0.down_proj", + "model.layers.28.mlp.experts.0.gate_proj", + "model.layers.28.mlp.experts.0.up_proj", + "model.layers.28.mlp.experts.1.down_proj", + "model.layers.28.mlp.experts.1.gate_proj", + "model.layers.28.mlp.experts.1.up_proj", + "model.layers.28.mlp.experts.10.down_proj", + "model.layers.28.mlp.experts.10.gate_proj", + "model.layers.28.mlp.experts.10.up_proj", + "model.layers.28.mlp.experts.100.down_proj", + "model.layers.28.mlp.experts.100.gate_proj", + "model.layers.28.mlp.experts.100.up_proj", + "model.layers.28.mlp.experts.101.down_proj", + "model.layers.28.mlp.experts.101.gate_proj", + "model.layers.28.mlp.experts.101.up_proj", + "model.layers.28.mlp.experts.102.down_proj", + "model.layers.28.mlp.experts.102.gate_proj", + "model.layers.28.mlp.experts.102.up_proj", + "model.layers.28.mlp.experts.103.down_proj", + "model.layers.28.mlp.experts.103.gate_proj", + "model.layers.28.mlp.experts.103.up_proj", + "model.layers.28.mlp.experts.104.down_proj", + "model.layers.28.mlp.experts.104.gate_proj", + "model.layers.28.mlp.experts.104.up_proj", + "model.layers.28.mlp.experts.105.down_proj", + "model.layers.28.mlp.experts.105.gate_proj", + "model.layers.28.mlp.experts.105.up_proj", + "model.layers.28.mlp.experts.106.down_proj", + "model.layers.28.mlp.experts.106.gate_proj", + "model.layers.28.mlp.experts.106.up_proj", + "model.layers.28.mlp.experts.107.down_proj", + "model.layers.28.mlp.experts.107.gate_proj", + "model.layers.28.mlp.experts.107.up_proj", + "model.layers.28.mlp.experts.108.down_proj", + "model.layers.28.mlp.experts.108.gate_proj", + "model.layers.28.mlp.experts.108.up_proj", + "model.layers.28.mlp.experts.109.down_proj", + "model.layers.28.mlp.experts.109.gate_proj", + "model.layers.28.mlp.experts.109.up_proj", + "model.layers.28.mlp.experts.11.down_proj", + "model.layers.28.mlp.experts.11.gate_proj", + "model.layers.28.mlp.experts.11.up_proj", + "model.layers.28.mlp.experts.110.down_proj", + "model.layers.28.mlp.experts.110.gate_proj", + "model.layers.28.mlp.experts.110.up_proj", + "model.layers.28.mlp.experts.111.down_proj", + "model.layers.28.mlp.experts.111.gate_proj", + "model.layers.28.mlp.experts.111.up_proj", + "model.layers.28.mlp.experts.112.down_proj", + "model.layers.28.mlp.experts.112.gate_proj", + "model.layers.28.mlp.experts.112.up_proj", + "model.layers.28.mlp.experts.113.down_proj", + "model.layers.28.mlp.experts.113.gate_proj", + "model.layers.28.mlp.experts.113.up_proj", + "model.layers.28.mlp.experts.114.down_proj", + "model.layers.28.mlp.experts.114.gate_proj", + "model.layers.28.mlp.experts.114.up_proj", + "model.layers.28.mlp.experts.115.down_proj", + "model.layers.28.mlp.experts.115.gate_proj", + "model.layers.28.mlp.experts.115.up_proj", + "model.layers.28.mlp.experts.116.down_proj", + "model.layers.28.mlp.experts.116.gate_proj", + "model.layers.28.mlp.experts.116.up_proj", + "model.layers.28.mlp.experts.117.down_proj", + "model.layers.28.mlp.experts.117.gate_proj", + "model.layers.28.mlp.experts.117.up_proj", + "model.layers.28.mlp.experts.118.down_proj", + "model.layers.28.mlp.experts.118.gate_proj", + "model.layers.28.mlp.experts.118.up_proj", + "model.layers.28.mlp.experts.119.down_proj", + "model.layers.28.mlp.experts.119.gate_proj", + "model.layers.28.mlp.experts.119.up_proj", + "model.layers.28.mlp.experts.12.down_proj", + "model.layers.28.mlp.experts.12.gate_proj", + "model.layers.28.mlp.experts.12.up_proj", + "model.layers.28.mlp.experts.120.down_proj", + "model.layers.28.mlp.experts.120.gate_proj", + "model.layers.28.mlp.experts.120.up_proj", + "model.layers.28.mlp.experts.121.down_proj", + "model.layers.28.mlp.experts.121.gate_proj", + "model.layers.28.mlp.experts.121.up_proj", + "model.layers.28.mlp.experts.122.down_proj", + "model.layers.28.mlp.experts.122.gate_proj", + "model.layers.28.mlp.experts.122.up_proj", + "model.layers.28.mlp.experts.123.down_proj", + "model.layers.28.mlp.experts.123.gate_proj", + "model.layers.28.mlp.experts.123.up_proj", + "model.layers.28.mlp.experts.124.down_proj", + "model.layers.28.mlp.experts.124.gate_proj", + "model.layers.28.mlp.experts.124.up_proj", + "model.layers.28.mlp.experts.125.down_proj", + "model.layers.28.mlp.experts.125.gate_proj", + "model.layers.28.mlp.experts.125.up_proj", + "model.layers.28.mlp.experts.126.down_proj", + "model.layers.28.mlp.experts.126.gate_proj", + "model.layers.28.mlp.experts.126.up_proj", + "model.layers.28.mlp.experts.127.down_proj", + "model.layers.28.mlp.experts.127.gate_proj", + "model.layers.28.mlp.experts.127.up_proj", + "model.layers.28.mlp.experts.128.down_proj", + "model.layers.28.mlp.experts.128.gate_proj", + "model.layers.28.mlp.experts.128.up_proj", + "model.layers.28.mlp.experts.129.down_proj", + "model.layers.28.mlp.experts.129.gate_proj", + "model.layers.28.mlp.experts.129.up_proj", + "model.layers.28.mlp.experts.13.down_proj", + "model.layers.28.mlp.experts.13.gate_proj", + "model.layers.28.mlp.experts.13.up_proj", + "model.layers.28.mlp.experts.130.down_proj", + "model.layers.28.mlp.experts.130.gate_proj", + "model.layers.28.mlp.experts.130.up_proj", + "model.layers.28.mlp.experts.131.down_proj", + "model.layers.28.mlp.experts.131.gate_proj", + "model.layers.28.mlp.experts.131.up_proj", + "model.layers.28.mlp.experts.132.down_proj", + "model.layers.28.mlp.experts.132.gate_proj", + "model.layers.28.mlp.experts.132.up_proj", + "model.layers.28.mlp.experts.133.down_proj", + "model.layers.28.mlp.experts.133.gate_proj", + "model.layers.28.mlp.experts.133.up_proj", + "model.layers.28.mlp.experts.134.down_proj", + "model.layers.28.mlp.experts.134.gate_proj", + "model.layers.28.mlp.experts.134.up_proj", + "model.layers.28.mlp.experts.135.down_proj", + "model.layers.28.mlp.experts.135.gate_proj", + "model.layers.28.mlp.experts.135.up_proj", + "model.layers.28.mlp.experts.136.down_proj", + "model.layers.28.mlp.experts.136.gate_proj", + "model.layers.28.mlp.experts.136.up_proj", + "model.layers.28.mlp.experts.137.down_proj", + "model.layers.28.mlp.experts.137.gate_proj", + "model.layers.28.mlp.experts.137.up_proj", + "model.layers.28.mlp.experts.138.down_proj", + "model.layers.28.mlp.experts.138.gate_proj", + "model.layers.28.mlp.experts.138.up_proj", + "model.layers.28.mlp.experts.139.down_proj", + "model.layers.28.mlp.experts.139.gate_proj", + "model.layers.28.mlp.experts.139.up_proj", + "model.layers.28.mlp.experts.14.down_proj", + "model.layers.28.mlp.experts.14.gate_proj", + "model.layers.28.mlp.experts.14.up_proj", + "model.layers.28.mlp.experts.140.down_proj", + "model.layers.28.mlp.experts.140.gate_proj", + "model.layers.28.mlp.experts.140.up_proj", + "model.layers.28.mlp.experts.141.down_proj", + "model.layers.28.mlp.experts.141.gate_proj", + "model.layers.28.mlp.experts.141.up_proj", + "model.layers.28.mlp.experts.142.down_proj", + "model.layers.28.mlp.experts.142.gate_proj", + "model.layers.28.mlp.experts.142.up_proj", + "model.layers.28.mlp.experts.143.down_proj", + "model.layers.28.mlp.experts.143.gate_proj", + "model.layers.28.mlp.experts.143.up_proj", + "model.layers.28.mlp.experts.144.down_proj", + "model.layers.28.mlp.experts.144.gate_proj", + "model.layers.28.mlp.experts.144.up_proj", + "model.layers.28.mlp.experts.145.down_proj", + "model.layers.28.mlp.experts.145.gate_proj", + "model.layers.28.mlp.experts.145.up_proj", + "model.layers.28.mlp.experts.146.down_proj", + "model.layers.28.mlp.experts.146.gate_proj", + "model.layers.28.mlp.experts.146.up_proj", + "model.layers.28.mlp.experts.147.down_proj", + "model.layers.28.mlp.experts.147.gate_proj", + "model.layers.28.mlp.experts.147.up_proj", + "model.layers.28.mlp.experts.148.down_proj", + "model.layers.28.mlp.experts.148.gate_proj", + "model.layers.28.mlp.experts.148.up_proj", + "model.layers.28.mlp.experts.149.down_proj", + "model.layers.28.mlp.experts.149.gate_proj", + "model.layers.28.mlp.experts.149.up_proj", + "model.layers.28.mlp.experts.15.down_proj", + "model.layers.28.mlp.experts.15.gate_proj", + "model.layers.28.mlp.experts.15.up_proj", + "model.layers.28.mlp.experts.150.down_proj", + "model.layers.28.mlp.experts.150.gate_proj", + "model.layers.28.mlp.experts.150.up_proj", + "model.layers.28.mlp.experts.151.down_proj", + "model.layers.28.mlp.experts.151.gate_proj", + "model.layers.28.mlp.experts.151.up_proj", + "model.layers.28.mlp.experts.152.down_proj", + "model.layers.28.mlp.experts.152.gate_proj", + "model.layers.28.mlp.experts.152.up_proj", + "model.layers.28.mlp.experts.153.down_proj", + "model.layers.28.mlp.experts.153.gate_proj", + "model.layers.28.mlp.experts.153.up_proj", + "model.layers.28.mlp.experts.154.down_proj", + "model.layers.28.mlp.experts.154.gate_proj", + "model.layers.28.mlp.experts.154.up_proj", + "model.layers.28.mlp.experts.155.down_proj", + "model.layers.28.mlp.experts.155.gate_proj", + "model.layers.28.mlp.experts.155.up_proj", + "model.layers.28.mlp.experts.156.down_proj", + "model.layers.28.mlp.experts.156.gate_proj", + "model.layers.28.mlp.experts.156.up_proj", + "model.layers.28.mlp.experts.157.down_proj", + "model.layers.28.mlp.experts.157.gate_proj", + "model.layers.28.mlp.experts.157.up_proj", + "model.layers.28.mlp.experts.158.down_proj", + "model.layers.28.mlp.experts.158.gate_proj", + "model.layers.28.mlp.experts.158.up_proj", + "model.layers.28.mlp.experts.159.down_proj", + "model.layers.28.mlp.experts.159.gate_proj", + "model.layers.28.mlp.experts.159.up_proj", + "model.layers.28.mlp.experts.16.down_proj", + "model.layers.28.mlp.experts.16.gate_proj", + "model.layers.28.mlp.experts.16.up_proj", + "model.layers.28.mlp.experts.160.down_proj", + "model.layers.28.mlp.experts.160.gate_proj", + "model.layers.28.mlp.experts.160.up_proj", + "model.layers.28.mlp.experts.161.down_proj", + "model.layers.28.mlp.experts.161.gate_proj", + "model.layers.28.mlp.experts.161.up_proj", + "model.layers.28.mlp.experts.162.down_proj", + "model.layers.28.mlp.experts.162.gate_proj", + "model.layers.28.mlp.experts.162.up_proj", + "model.layers.28.mlp.experts.163.down_proj", + "model.layers.28.mlp.experts.163.gate_proj", + "model.layers.28.mlp.experts.163.up_proj", + "model.layers.28.mlp.experts.164.down_proj", + "model.layers.28.mlp.experts.164.gate_proj", + "model.layers.28.mlp.experts.164.up_proj", + "model.layers.28.mlp.experts.165.down_proj", + "model.layers.28.mlp.experts.165.gate_proj", + "model.layers.28.mlp.experts.165.up_proj", + "model.layers.28.mlp.experts.166.down_proj", + "model.layers.28.mlp.experts.166.gate_proj", + "model.layers.28.mlp.experts.166.up_proj", + "model.layers.28.mlp.experts.167.down_proj", + "model.layers.28.mlp.experts.167.gate_proj", + "model.layers.28.mlp.experts.167.up_proj", + "model.layers.28.mlp.experts.168.down_proj", + "model.layers.28.mlp.experts.168.gate_proj", + "model.layers.28.mlp.experts.168.up_proj", + "model.layers.28.mlp.experts.169.down_proj", + "model.layers.28.mlp.experts.169.gate_proj", + "model.layers.28.mlp.experts.169.up_proj", + "model.layers.28.mlp.experts.17.down_proj", + "model.layers.28.mlp.experts.17.gate_proj", + "model.layers.28.mlp.experts.17.up_proj", + "model.layers.28.mlp.experts.170.down_proj", + "model.layers.28.mlp.experts.170.gate_proj", + "model.layers.28.mlp.experts.170.up_proj", + "model.layers.28.mlp.experts.171.down_proj", + "model.layers.28.mlp.experts.171.gate_proj", + "model.layers.28.mlp.experts.171.up_proj", + "model.layers.28.mlp.experts.172.down_proj", + "model.layers.28.mlp.experts.172.gate_proj", + "model.layers.28.mlp.experts.172.up_proj", + "model.layers.28.mlp.experts.173.down_proj", + "model.layers.28.mlp.experts.173.gate_proj", + "model.layers.28.mlp.experts.173.up_proj", + "model.layers.28.mlp.experts.174.down_proj", + "model.layers.28.mlp.experts.174.gate_proj", + "model.layers.28.mlp.experts.174.up_proj", + "model.layers.28.mlp.experts.175.down_proj", + "model.layers.28.mlp.experts.175.gate_proj", + "model.layers.28.mlp.experts.175.up_proj", + "model.layers.28.mlp.experts.176.down_proj", + "model.layers.28.mlp.experts.176.gate_proj", + "model.layers.28.mlp.experts.176.up_proj", + "model.layers.28.mlp.experts.177.down_proj", + "model.layers.28.mlp.experts.177.gate_proj", + "model.layers.28.mlp.experts.177.up_proj", + "model.layers.28.mlp.experts.178.down_proj", + "model.layers.28.mlp.experts.178.gate_proj", + "model.layers.28.mlp.experts.178.up_proj", + "model.layers.28.mlp.experts.179.down_proj", + "model.layers.28.mlp.experts.179.gate_proj", + "model.layers.28.mlp.experts.179.up_proj", + "model.layers.28.mlp.experts.18.down_proj", + "model.layers.28.mlp.experts.18.gate_proj", + "model.layers.28.mlp.experts.18.up_proj", + "model.layers.28.mlp.experts.180.down_proj", + "model.layers.28.mlp.experts.180.gate_proj", + "model.layers.28.mlp.experts.180.up_proj", + "model.layers.28.mlp.experts.181.down_proj", + "model.layers.28.mlp.experts.181.gate_proj", + "model.layers.28.mlp.experts.181.up_proj", + "model.layers.28.mlp.experts.182.down_proj", + "model.layers.28.mlp.experts.182.gate_proj", + "model.layers.28.mlp.experts.182.up_proj", + "model.layers.28.mlp.experts.183.down_proj", + "model.layers.28.mlp.experts.183.gate_proj", + "model.layers.28.mlp.experts.183.up_proj", + "model.layers.28.mlp.experts.184.down_proj", + "model.layers.28.mlp.experts.184.gate_proj", + "model.layers.28.mlp.experts.184.up_proj", + "model.layers.28.mlp.experts.185.down_proj", + "model.layers.28.mlp.experts.185.gate_proj", + "model.layers.28.mlp.experts.185.up_proj", + "model.layers.28.mlp.experts.186.down_proj", + "model.layers.28.mlp.experts.186.gate_proj", + "model.layers.28.mlp.experts.186.up_proj", + "model.layers.28.mlp.experts.187.down_proj", + "model.layers.28.mlp.experts.187.gate_proj", + "model.layers.28.mlp.experts.187.up_proj", + "model.layers.28.mlp.experts.188.down_proj", + "model.layers.28.mlp.experts.188.gate_proj", + "model.layers.28.mlp.experts.188.up_proj", + "model.layers.28.mlp.experts.189.down_proj", + "model.layers.28.mlp.experts.189.gate_proj", + "model.layers.28.mlp.experts.189.up_proj", + "model.layers.28.mlp.experts.19.down_proj", + "model.layers.28.mlp.experts.19.gate_proj", + "model.layers.28.mlp.experts.19.up_proj", + "model.layers.28.mlp.experts.190.down_proj", + "model.layers.28.mlp.experts.190.gate_proj", + "model.layers.28.mlp.experts.190.up_proj", + "model.layers.28.mlp.experts.191.down_proj", + "model.layers.28.mlp.experts.191.gate_proj", + "model.layers.28.mlp.experts.191.up_proj", + "model.layers.28.mlp.experts.192.down_proj", + "model.layers.28.mlp.experts.192.gate_proj", + "model.layers.28.mlp.experts.192.up_proj", + "model.layers.28.mlp.experts.193.down_proj", + "model.layers.28.mlp.experts.193.gate_proj", + "model.layers.28.mlp.experts.193.up_proj", + "model.layers.28.mlp.experts.194.down_proj", + "model.layers.28.mlp.experts.194.gate_proj", + "model.layers.28.mlp.experts.194.up_proj", + "model.layers.28.mlp.experts.195.down_proj", + "model.layers.28.mlp.experts.195.gate_proj", + "model.layers.28.mlp.experts.195.up_proj", + "model.layers.28.mlp.experts.196.down_proj", + "model.layers.28.mlp.experts.196.gate_proj", + "model.layers.28.mlp.experts.196.up_proj", + "model.layers.28.mlp.experts.197.down_proj", + "model.layers.28.mlp.experts.197.gate_proj", + "model.layers.28.mlp.experts.197.up_proj", + "model.layers.28.mlp.experts.198.down_proj", + "model.layers.28.mlp.experts.198.gate_proj", + "model.layers.28.mlp.experts.198.up_proj", + "model.layers.28.mlp.experts.199.down_proj", + "model.layers.28.mlp.experts.199.gate_proj", + "model.layers.28.mlp.experts.199.up_proj", + "model.layers.28.mlp.experts.2.down_proj", + "model.layers.28.mlp.experts.2.gate_proj", + "model.layers.28.mlp.experts.2.up_proj", + "model.layers.28.mlp.experts.20.down_proj", + "model.layers.28.mlp.experts.20.gate_proj", + "model.layers.28.mlp.experts.20.up_proj", + "model.layers.28.mlp.experts.200.down_proj", + "model.layers.28.mlp.experts.200.gate_proj", + "model.layers.28.mlp.experts.200.up_proj", + "model.layers.28.mlp.experts.201.down_proj", + "model.layers.28.mlp.experts.201.gate_proj", + "model.layers.28.mlp.experts.201.up_proj", + "model.layers.28.mlp.experts.202.down_proj", + "model.layers.28.mlp.experts.202.gate_proj", + "model.layers.28.mlp.experts.202.up_proj", + "model.layers.28.mlp.experts.203.down_proj", + "model.layers.28.mlp.experts.203.gate_proj", + "model.layers.28.mlp.experts.203.up_proj", + "model.layers.28.mlp.experts.204.down_proj", + "model.layers.28.mlp.experts.204.gate_proj", + "model.layers.28.mlp.experts.204.up_proj", + "model.layers.28.mlp.experts.205.down_proj", + "model.layers.28.mlp.experts.205.gate_proj", + "model.layers.28.mlp.experts.205.up_proj", + "model.layers.28.mlp.experts.206.down_proj", + "model.layers.28.mlp.experts.206.gate_proj", + "model.layers.28.mlp.experts.206.up_proj", + "model.layers.28.mlp.experts.207.down_proj", + "model.layers.28.mlp.experts.207.gate_proj", + "model.layers.28.mlp.experts.207.up_proj", + "model.layers.28.mlp.experts.208.down_proj", + "model.layers.28.mlp.experts.208.gate_proj", + "model.layers.28.mlp.experts.208.up_proj", + "model.layers.28.mlp.experts.209.down_proj", + "model.layers.28.mlp.experts.209.gate_proj", + "model.layers.28.mlp.experts.209.up_proj", + "model.layers.28.mlp.experts.21.down_proj", + "model.layers.28.mlp.experts.21.gate_proj", + "model.layers.28.mlp.experts.21.up_proj", + "model.layers.28.mlp.experts.210.down_proj", + "model.layers.28.mlp.experts.210.gate_proj", + "model.layers.28.mlp.experts.210.up_proj", + "model.layers.28.mlp.experts.211.down_proj", + "model.layers.28.mlp.experts.211.gate_proj", + "model.layers.28.mlp.experts.211.up_proj", + "model.layers.28.mlp.experts.212.down_proj", + "model.layers.28.mlp.experts.212.gate_proj", + "model.layers.28.mlp.experts.212.up_proj", + "model.layers.28.mlp.experts.213.down_proj", + "model.layers.28.mlp.experts.213.gate_proj", + "model.layers.28.mlp.experts.213.up_proj", + "model.layers.28.mlp.experts.214.down_proj", + "model.layers.28.mlp.experts.214.gate_proj", + "model.layers.28.mlp.experts.214.up_proj", + "model.layers.28.mlp.experts.215.down_proj", + "model.layers.28.mlp.experts.215.gate_proj", + "model.layers.28.mlp.experts.215.up_proj", + "model.layers.28.mlp.experts.216.down_proj", + "model.layers.28.mlp.experts.216.gate_proj", + "model.layers.28.mlp.experts.216.up_proj", + "model.layers.28.mlp.experts.217.down_proj", + "model.layers.28.mlp.experts.217.gate_proj", + "model.layers.28.mlp.experts.217.up_proj", + "model.layers.28.mlp.experts.218.down_proj", + "model.layers.28.mlp.experts.218.gate_proj", + "model.layers.28.mlp.experts.218.up_proj", + "model.layers.28.mlp.experts.219.down_proj", + "model.layers.28.mlp.experts.219.gate_proj", + "model.layers.28.mlp.experts.219.up_proj", + "model.layers.28.mlp.experts.22.down_proj", + "model.layers.28.mlp.experts.22.gate_proj", + "model.layers.28.mlp.experts.22.up_proj", + "model.layers.28.mlp.experts.220.down_proj", + "model.layers.28.mlp.experts.220.gate_proj", + "model.layers.28.mlp.experts.220.up_proj", + "model.layers.28.mlp.experts.221.down_proj", + "model.layers.28.mlp.experts.221.gate_proj", + "model.layers.28.mlp.experts.221.up_proj", + "model.layers.28.mlp.experts.222.down_proj", + "model.layers.28.mlp.experts.222.gate_proj", + "model.layers.28.mlp.experts.222.up_proj", + "model.layers.28.mlp.experts.223.down_proj", + "model.layers.28.mlp.experts.223.gate_proj", + "model.layers.28.mlp.experts.223.up_proj", + "model.layers.28.mlp.experts.224.down_proj", + "model.layers.28.mlp.experts.224.gate_proj", + "model.layers.28.mlp.experts.224.up_proj", + "model.layers.28.mlp.experts.225.down_proj", + "model.layers.28.mlp.experts.225.gate_proj", + "model.layers.28.mlp.experts.225.up_proj", + "model.layers.28.mlp.experts.226.down_proj", + "model.layers.28.mlp.experts.226.gate_proj", + "model.layers.28.mlp.experts.226.up_proj", + "model.layers.28.mlp.experts.227.down_proj", + "model.layers.28.mlp.experts.227.gate_proj", + "model.layers.28.mlp.experts.227.up_proj", + "model.layers.28.mlp.experts.228.down_proj", + "model.layers.28.mlp.experts.228.gate_proj", + "model.layers.28.mlp.experts.228.up_proj", + "model.layers.28.mlp.experts.229.down_proj", + "model.layers.28.mlp.experts.229.gate_proj", + "model.layers.28.mlp.experts.229.up_proj", + "model.layers.28.mlp.experts.23.down_proj", + "model.layers.28.mlp.experts.23.gate_proj", + "model.layers.28.mlp.experts.23.up_proj", + "model.layers.28.mlp.experts.230.down_proj", + "model.layers.28.mlp.experts.230.gate_proj", + "model.layers.28.mlp.experts.230.up_proj", + "model.layers.28.mlp.experts.231.down_proj", + "model.layers.28.mlp.experts.231.gate_proj", + "model.layers.28.mlp.experts.231.up_proj", + "model.layers.28.mlp.experts.232.down_proj", + "model.layers.28.mlp.experts.232.gate_proj", + "model.layers.28.mlp.experts.232.up_proj", + "model.layers.28.mlp.experts.233.down_proj", + "model.layers.28.mlp.experts.233.gate_proj", + "model.layers.28.mlp.experts.233.up_proj", + "model.layers.28.mlp.experts.234.down_proj", + "model.layers.28.mlp.experts.234.gate_proj", + "model.layers.28.mlp.experts.234.up_proj", + "model.layers.28.mlp.experts.235.down_proj", + "model.layers.28.mlp.experts.235.gate_proj", + "model.layers.28.mlp.experts.235.up_proj", + "model.layers.28.mlp.experts.236.down_proj", + "model.layers.28.mlp.experts.236.gate_proj", + "model.layers.28.mlp.experts.236.up_proj", + "model.layers.28.mlp.experts.237.down_proj", + "model.layers.28.mlp.experts.237.gate_proj", + "model.layers.28.mlp.experts.237.up_proj", + "model.layers.28.mlp.experts.238.down_proj", + "model.layers.28.mlp.experts.238.gate_proj", + "model.layers.28.mlp.experts.238.up_proj", + "model.layers.28.mlp.experts.239.down_proj", + "model.layers.28.mlp.experts.239.gate_proj", + "model.layers.28.mlp.experts.239.up_proj", + "model.layers.28.mlp.experts.24.down_proj", + "model.layers.28.mlp.experts.24.gate_proj", + "model.layers.28.mlp.experts.24.up_proj", + "model.layers.28.mlp.experts.240.down_proj", + "model.layers.28.mlp.experts.240.gate_proj", + "model.layers.28.mlp.experts.240.up_proj", + "model.layers.28.mlp.experts.241.down_proj", + "model.layers.28.mlp.experts.241.gate_proj", + "model.layers.28.mlp.experts.241.up_proj", + "model.layers.28.mlp.experts.242.down_proj", + "model.layers.28.mlp.experts.242.gate_proj", + "model.layers.28.mlp.experts.242.up_proj", + "model.layers.28.mlp.experts.243.down_proj", + "model.layers.28.mlp.experts.243.gate_proj", + "model.layers.28.mlp.experts.243.up_proj", + "model.layers.28.mlp.experts.244.down_proj", + "model.layers.28.mlp.experts.244.gate_proj", + "model.layers.28.mlp.experts.244.up_proj", + "model.layers.28.mlp.experts.245.down_proj", + "model.layers.28.mlp.experts.245.gate_proj", + "model.layers.28.mlp.experts.245.up_proj", + "model.layers.28.mlp.experts.246.down_proj", + "model.layers.28.mlp.experts.246.gate_proj", + "model.layers.28.mlp.experts.246.up_proj", + "model.layers.28.mlp.experts.247.down_proj", + "model.layers.28.mlp.experts.247.gate_proj", + "model.layers.28.mlp.experts.247.up_proj", + "model.layers.28.mlp.experts.248.down_proj", + "model.layers.28.mlp.experts.248.gate_proj", + "model.layers.28.mlp.experts.248.up_proj", + "model.layers.28.mlp.experts.249.down_proj", + "model.layers.28.mlp.experts.249.gate_proj", + "model.layers.28.mlp.experts.249.up_proj", + "model.layers.28.mlp.experts.25.down_proj", + "model.layers.28.mlp.experts.25.gate_proj", + "model.layers.28.mlp.experts.25.up_proj", + "model.layers.28.mlp.experts.250.down_proj", + "model.layers.28.mlp.experts.250.gate_proj", + "model.layers.28.mlp.experts.250.up_proj", + "model.layers.28.mlp.experts.251.down_proj", + "model.layers.28.mlp.experts.251.gate_proj", + "model.layers.28.mlp.experts.251.up_proj", + "model.layers.28.mlp.experts.252.down_proj", + "model.layers.28.mlp.experts.252.gate_proj", + "model.layers.28.mlp.experts.252.up_proj", + "model.layers.28.mlp.experts.253.down_proj", + "model.layers.28.mlp.experts.253.gate_proj", + "model.layers.28.mlp.experts.253.up_proj", + "model.layers.28.mlp.experts.254.down_proj", + "model.layers.28.mlp.experts.254.gate_proj", + "model.layers.28.mlp.experts.254.up_proj", + "model.layers.28.mlp.experts.255.down_proj", + "model.layers.28.mlp.experts.255.gate_proj", + "model.layers.28.mlp.experts.255.up_proj", + "model.layers.28.mlp.experts.26.down_proj", + "model.layers.28.mlp.experts.26.gate_proj", + "model.layers.28.mlp.experts.26.up_proj", + "model.layers.28.mlp.experts.27.down_proj", + "model.layers.28.mlp.experts.27.gate_proj", + "model.layers.28.mlp.experts.27.up_proj", + "model.layers.28.mlp.experts.28.down_proj", + "model.layers.28.mlp.experts.28.gate_proj", + "model.layers.28.mlp.experts.28.up_proj", + "model.layers.28.mlp.experts.29.down_proj", + "model.layers.28.mlp.experts.29.gate_proj", + "model.layers.28.mlp.experts.29.up_proj", + "model.layers.28.mlp.experts.3.down_proj", + "model.layers.28.mlp.experts.3.gate_proj", + "model.layers.28.mlp.experts.3.up_proj", + "model.layers.28.mlp.experts.30.down_proj", + "model.layers.28.mlp.experts.30.gate_proj", + "model.layers.28.mlp.experts.30.up_proj", + "model.layers.28.mlp.experts.31.down_proj", + "model.layers.28.mlp.experts.31.gate_proj", + "model.layers.28.mlp.experts.31.up_proj", + "model.layers.28.mlp.experts.32.down_proj", + "model.layers.28.mlp.experts.32.gate_proj", + "model.layers.28.mlp.experts.32.up_proj", + "model.layers.28.mlp.experts.33.down_proj", + "model.layers.28.mlp.experts.33.gate_proj", + "model.layers.28.mlp.experts.33.up_proj", + "model.layers.28.mlp.experts.34.down_proj", + "model.layers.28.mlp.experts.34.gate_proj", + "model.layers.28.mlp.experts.34.up_proj", + "model.layers.28.mlp.experts.35.down_proj", + "model.layers.28.mlp.experts.35.gate_proj", + "model.layers.28.mlp.experts.35.up_proj", + "model.layers.28.mlp.experts.36.down_proj", + "model.layers.28.mlp.experts.36.gate_proj", + "model.layers.28.mlp.experts.36.up_proj", + "model.layers.28.mlp.experts.37.down_proj", + "model.layers.28.mlp.experts.37.gate_proj", + "model.layers.28.mlp.experts.37.up_proj", + "model.layers.28.mlp.experts.38.down_proj", + "model.layers.28.mlp.experts.38.gate_proj", + "model.layers.28.mlp.experts.38.up_proj", + "model.layers.28.mlp.experts.39.down_proj", + "model.layers.28.mlp.experts.39.gate_proj", + "model.layers.28.mlp.experts.39.up_proj", + "model.layers.28.mlp.experts.4.down_proj", + "model.layers.28.mlp.experts.4.gate_proj", + "model.layers.28.mlp.experts.4.up_proj", + "model.layers.28.mlp.experts.40.down_proj", + "model.layers.28.mlp.experts.40.gate_proj", + "model.layers.28.mlp.experts.40.up_proj", + "model.layers.28.mlp.experts.41.down_proj", + "model.layers.28.mlp.experts.41.gate_proj", + "model.layers.28.mlp.experts.41.up_proj", + "model.layers.28.mlp.experts.42.down_proj", + "model.layers.28.mlp.experts.42.gate_proj", + "model.layers.28.mlp.experts.42.up_proj", + "model.layers.28.mlp.experts.43.down_proj", + "model.layers.28.mlp.experts.43.gate_proj", + "model.layers.28.mlp.experts.43.up_proj", + "model.layers.28.mlp.experts.44.down_proj", + "model.layers.28.mlp.experts.44.gate_proj", + "model.layers.28.mlp.experts.44.up_proj", + "model.layers.28.mlp.experts.45.down_proj", + "model.layers.28.mlp.experts.45.gate_proj", + "model.layers.28.mlp.experts.45.up_proj", + "model.layers.28.mlp.experts.46.down_proj", + "model.layers.28.mlp.experts.46.gate_proj", + "model.layers.28.mlp.experts.46.up_proj", + "model.layers.28.mlp.experts.47.down_proj", + "model.layers.28.mlp.experts.47.gate_proj", + "model.layers.28.mlp.experts.47.up_proj", + "model.layers.28.mlp.experts.48.down_proj", + "model.layers.28.mlp.experts.48.gate_proj", + "model.layers.28.mlp.experts.48.up_proj", + "model.layers.28.mlp.experts.49.down_proj", + "model.layers.28.mlp.experts.49.gate_proj", + "model.layers.28.mlp.experts.49.up_proj", + "model.layers.28.mlp.experts.5.down_proj", + "model.layers.28.mlp.experts.5.gate_proj", + "model.layers.28.mlp.experts.5.up_proj", + "model.layers.28.mlp.experts.50.down_proj", + "model.layers.28.mlp.experts.50.gate_proj", + "model.layers.28.mlp.experts.50.up_proj", + "model.layers.28.mlp.experts.51.down_proj", + "model.layers.28.mlp.experts.51.gate_proj", + "model.layers.28.mlp.experts.51.up_proj", + "model.layers.28.mlp.experts.52.down_proj", + "model.layers.28.mlp.experts.52.gate_proj", + "model.layers.28.mlp.experts.52.up_proj", + "model.layers.28.mlp.experts.53.down_proj", + "model.layers.28.mlp.experts.53.gate_proj", + "model.layers.28.mlp.experts.53.up_proj", + "model.layers.28.mlp.experts.54.down_proj", + "model.layers.28.mlp.experts.54.gate_proj", + "model.layers.28.mlp.experts.54.up_proj", + "model.layers.28.mlp.experts.55.down_proj", + "model.layers.28.mlp.experts.55.gate_proj", + "model.layers.28.mlp.experts.55.up_proj", + "model.layers.28.mlp.experts.56.down_proj", + "model.layers.28.mlp.experts.56.gate_proj", + "model.layers.28.mlp.experts.56.up_proj", + "model.layers.28.mlp.experts.57.down_proj", + "model.layers.28.mlp.experts.57.gate_proj", + "model.layers.28.mlp.experts.57.up_proj", + "model.layers.28.mlp.experts.58.down_proj", + "model.layers.28.mlp.experts.58.gate_proj", + "model.layers.28.mlp.experts.58.up_proj", + "model.layers.28.mlp.experts.59.down_proj", + "model.layers.28.mlp.experts.59.gate_proj", + "model.layers.28.mlp.experts.59.up_proj", + "model.layers.28.mlp.experts.6.down_proj", + "model.layers.28.mlp.experts.6.gate_proj", + "model.layers.28.mlp.experts.6.up_proj", + "model.layers.28.mlp.experts.60.down_proj", + "model.layers.28.mlp.experts.60.gate_proj", + "model.layers.28.mlp.experts.60.up_proj", + "model.layers.28.mlp.experts.61.down_proj", + "model.layers.28.mlp.experts.61.gate_proj", + "model.layers.28.mlp.experts.61.up_proj", + "model.layers.28.mlp.experts.62.down_proj", + "model.layers.28.mlp.experts.62.gate_proj", + "model.layers.28.mlp.experts.62.up_proj", + "model.layers.28.mlp.experts.63.down_proj", + "model.layers.28.mlp.experts.63.gate_proj", + "model.layers.28.mlp.experts.63.up_proj", + "model.layers.28.mlp.experts.64.down_proj", + "model.layers.28.mlp.experts.64.gate_proj", + "model.layers.28.mlp.experts.64.up_proj", + "model.layers.28.mlp.experts.65.down_proj", + "model.layers.28.mlp.experts.65.gate_proj", + "model.layers.28.mlp.experts.65.up_proj", + "model.layers.28.mlp.experts.66.down_proj", + "model.layers.28.mlp.experts.66.gate_proj", + "model.layers.28.mlp.experts.66.up_proj", + "model.layers.28.mlp.experts.67.down_proj", + "model.layers.28.mlp.experts.67.gate_proj", + "model.layers.28.mlp.experts.67.up_proj", + "model.layers.28.mlp.experts.68.down_proj", + "model.layers.28.mlp.experts.68.gate_proj", + "model.layers.28.mlp.experts.68.up_proj", + "model.layers.28.mlp.experts.69.down_proj", + "model.layers.28.mlp.experts.69.gate_proj", + "model.layers.28.mlp.experts.69.up_proj", + "model.layers.28.mlp.experts.7.down_proj", + "model.layers.28.mlp.experts.7.gate_proj", + "model.layers.28.mlp.experts.7.up_proj", + "model.layers.28.mlp.experts.70.down_proj", + "model.layers.28.mlp.experts.70.gate_proj", + "model.layers.28.mlp.experts.70.up_proj", + "model.layers.28.mlp.experts.71.down_proj", + "model.layers.28.mlp.experts.71.gate_proj", + "model.layers.28.mlp.experts.71.up_proj", + "model.layers.28.mlp.experts.72.down_proj", + "model.layers.28.mlp.experts.72.gate_proj", + "model.layers.28.mlp.experts.72.up_proj", + "model.layers.28.mlp.experts.73.down_proj", + "model.layers.28.mlp.experts.73.gate_proj", + "model.layers.28.mlp.experts.73.up_proj", + "model.layers.28.mlp.experts.74.down_proj", + "model.layers.28.mlp.experts.74.gate_proj", + "model.layers.28.mlp.experts.74.up_proj", + "model.layers.28.mlp.experts.75.down_proj", + "model.layers.28.mlp.experts.75.gate_proj", + "model.layers.28.mlp.experts.75.up_proj", + "model.layers.28.mlp.experts.76.down_proj", + "model.layers.28.mlp.experts.76.gate_proj", + "model.layers.28.mlp.experts.76.up_proj", + "model.layers.28.mlp.experts.77.down_proj", + "model.layers.28.mlp.experts.77.gate_proj", + "model.layers.28.mlp.experts.77.up_proj", + "model.layers.28.mlp.experts.78.down_proj", + "model.layers.28.mlp.experts.78.gate_proj", + "model.layers.28.mlp.experts.78.up_proj", + "model.layers.28.mlp.experts.79.down_proj", + "model.layers.28.mlp.experts.79.gate_proj", + "model.layers.28.mlp.experts.79.up_proj", + "model.layers.28.mlp.experts.8.down_proj", + "model.layers.28.mlp.experts.8.gate_proj", + "model.layers.28.mlp.experts.8.up_proj", + "model.layers.28.mlp.experts.80.down_proj", + "model.layers.28.mlp.experts.80.gate_proj", + "model.layers.28.mlp.experts.80.up_proj", + "model.layers.28.mlp.experts.81.down_proj", + "model.layers.28.mlp.experts.81.gate_proj", + "model.layers.28.mlp.experts.81.up_proj", + "model.layers.28.mlp.experts.82.down_proj", + "model.layers.28.mlp.experts.82.gate_proj", + "model.layers.28.mlp.experts.82.up_proj", + "model.layers.28.mlp.experts.83.down_proj", + "model.layers.28.mlp.experts.83.gate_proj", + "model.layers.28.mlp.experts.83.up_proj", + "model.layers.28.mlp.experts.84.down_proj", + "model.layers.28.mlp.experts.84.gate_proj", + "model.layers.28.mlp.experts.84.up_proj", + "model.layers.28.mlp.experts.85.down_proj", + "model.layers.28.mlp.experts.85.gate_proj", + "model.layers.28.mlp.experts.85.up_proj", + "model.layers.28.mlp.experts.86.down_proj", + "model.layers.28.mlp.experts.86.gate_proj", + "model.layers.28.mlp.experts.86.up_proj", + "model.layers.28.mlp.experts.87.down_proj", + "model.layers.28.mlp.experts.87.gate_proj", + "model.layers.28.mlp.experts.87.up_proj", + "model.layers.28.mlp.experts.88.down_proj", + "model.layers.28.mlp.experts.88.gate_proj", + "model.layers.28.mlp.experts.88.up_proj", + "model.layers.28.mlp.experts.89.down_proj", + "model.layers.28.mlp.experts.89.gate_proj", + "model.layers.28.mlp.experts.89.up_proj", + "model.layers.28.mlp.experts.9.down_proj", + "model.layers.28.mlp.experts.9.gate_proj", + "model.layers.28.mlp.experts.9.up_proj", + "model.layers.28.mlp.experts.90.down_proj", + "model.layers.28.mlp.experts.90.gate_proj", + "model.layers.28.mlp.experts.90.up_proj", + "model.layers.28.mlp.experts.91.down_proj", + "model.layers.28.mlp.experts.91.gate_proj", + "model.layers.28.mlp.experts.91.up_proj", + "model.layers.28.mlp.experts.92.down_proj", + "model.layers.28.mlp.experts.92.gate_proj", + "model.layers.28.mlp.experts.92.up_proj", + "model.layers.28.mlp.experts.93.down_proj", + "model.layers.28.mlp.experts.93.gate_proj", + "model.layers.28.mlp.experts.93.up_proj", + "model.layers.28.mlp.experts.94.down_proj", + "model.layers.28.mlp.experts.94.gate_proj", + "model.layers.28.mlp.experts.94.up_proj", + "model.layers.28.mlp.experts.95.down_proj", + "model.layers.28.mlp.experts.95.gate_proj", + "model.layers.28.mlp.experts.95.up_proj", + "model.layers.28.mlp.experts.96.down_proj", + "model.layers.28.mlp.experts.96.gate_proj", + "model.layers.28.mlp.experts.96.up_proj", + "model.layers.28.mlp.experts.97.down_proj", + "model.layers.28.mlp.experts.97.gate_proj", + "model.layers.28.mlp.experts.97.up_proj", + "model.layers.28.mlp.experts.98.down_proj", + "model.layers.28.mlp.experts.98.gate_proj", + "model.layers.28.mlp.experts.98.up_proj", + "model.layers.28.mlp.experts.99.down_proj", + "model.layers.28.mlp.experts.99.gate_proj", + "model.layers.28.mlp.experts.99.up_proj", + "model.layers.29.mlp.experts.0.down_proj", + "model.layers.29.mlp.experts.0.gate_proj", + "model.layers.29.mlp.experts.0.up_proj", + "model.layers.29.mlp.experts.1.down_proj", + "model.layers.29.mlp.experts.1.gate_proj", + "model.layers.29.mlp.experts.1.up_proj", + "model.layers.29.mlp.experts.10.down_proj", + "model.layers.29.mlp.experts.10.gate_proj", + "model.layers.29.mlp.experts.10.up_proj", + "model.layers.29.mlp.experts.100.down_proj", + "model.layers.29.mlp.experts.100.gate_proj", + "model.layers.29.mlp.experts.100.up_proj", + "model.layers.29.mlp.experts.101.down_proj", + "model.layers.29.mlp.experts.101.gate_proj", + "model.layers.29.mlp.experts.101.up_proj", + "model.layers.29.mlp.experts.102.down_proj", + "model.layers.29.mlp.experts.102.gate_proj", + "model.layers.29.mlp.experts.102.up_proj", + "model.layers.29.mlp.experts.103.down_proj", + "model.layers.29.mlp.experts.103.gate_proj", + "model.layers.29.mlp.experts.103.up_proj", + "model.layers.29.mlp.experts.104.down_proj", + "model.layers.29.mlp.experts.104.gate_proj", + "model.layers.29.mlp.experts.104.up_proj", + "model.layers.29.mlp.experts.105.down_proj", + "model.layers.29.mlp.experts.105.gate_proj", + "model.layers.29.mlp.experts.105.up_proj", + "model.layers.29.mlp.experts.106.down_proj", + "model.layers.29.mlp.experts.106.gate_proj", + "model.layers.29.mlp.experts.106.up_proj", + "model.layers.29.mlp.experts.107.down_proj", + "model.layers.29.mlp.experts.107.gate_proj", + "model.layers.29.mlp.experts.107.up_proj", + "model.layers.29.mlp.experts.108.down_proj", + "model.layers.29.mlp.experts.108.gate_proj", + "model.layers.29.mlp.experts.108.up_proj", + "model.layers.29.mlp.experts.109.down_proj", + "model.layers.29.mlp.experts.109.gate_proj", + "model.layers.29.mlp.experts.109.up_proj", + "model.layers.29.mlp.experts.11.down_proj", + "model.layers.29.mlp.experts.11.gate_proj", + "model.layers.29.mlp.experts.11.up_proj", + "model.layers.29.mlp.experts.110.down_proj", + "model.layers.29.mlp.experts.110.gate_proj", + "model.layers.29.mlp.experts.110.up_proj", + "model.layers.29.mlp.experts.111.down_proj", + "model.layers.29.mlp.experts.111.gate_proj", + "model.layers.29.mlp.experts.111.up_proj", + "model.layers.29.mlp.experts.112.down_proj", + "model.layers.29.mlp.experts.112.gate_proj", + "model.layers.29.mlp.experts.112.up_proj", + "model.layers.29.mlp.experts.113.down_proj", + "model.layers.29.mlp.experts.113.gate_proj", + "model.layers.29.mlp.experts.113.up_proj", + "model.layers.29.mlp.experts.114.down_proj", + "model.layers.29.mlp.experts.114.gate_proj", + "model.layers.29.mlp.experts.114.up_proj", + "model.layers.29.mlp.experts.115.down_proj", + "model.layers.29.mlp.experts.115.gate_proj", + "model.layers.29.mlp.experts.115.up_proj", + "model.layers.29.mlp.experts.116.down_proj", + "model.layers.29.mlp.experts.116.gate_proj", + "model.layers.29.mlp.experts.116.up_proj", + "model.layers.29.mlp.experts.117.down_proj", + "model.layers.29.mlp.experts.117.gate_proj", + "model.layers.29.mlp.experts.117.up_proj", + "model.layers.29.mlp.experts.118.down_proj", + "model.layers.29.mlp.experts.118.gate_proj", + "model.layers.29.mlp.experts.118.up_proj", + "model.layers.29.mlp.experts.119.down_proj", + "model.layers.29.mlp.experts.119.gate_proj", + "model.layers.29.mlp.experts.119.up_proj", + "model.layers.29.mlp.experts.12.down_proj", + "model.layers.29.mlp.experts.12.gate_proj", + "model.layers.29.mlp.experts.12.up_proj", + "model.layers.29.mlp.experts.120.down_proj", + "model.layers.29.mlp.experts.120.gate_proj", + "model.layers.29.mlp.experts.120.up_proj", + "model.layers.29.mlp.experts.121.down_proj", + "model.layers.29.mlp.experts.121.gate_proj", + "model.layers.29.mlp.experts.121.up_proj", + "model.layers.29.mlp.experts.122.down_proj", + "model.layers.29.mlp.experts.122.gate_proj", + "model.layers.29.mlp.experts.122.up_proj", + "model.layers.29.mlp.experts.123.down_proj", + "model.layers.29.mlp.experts.123.gate_proj", + "model.layers.29.mlp.experts.123.up_proj", + "model.layers.29.mlp.experts.124.down_proj", + "model.layers.29.mlp.experts.124.gate_proj", + "model.layers.29.mlp.experts.124.up_proj", + "model.layers.29.mlp.experts.125.down_proj", + "model.layers.29.mlp.experts.125.gate_proj", + "model.layers.29.mlp.experts.125.up_proj", + "model.layers.29.mlp.experts.126.down_proj", + "model.layers.29.mlp.experts.126.gate_proj", + "model.layers.29.mlp.experts.126.up_proj", + "model.layers.29.mlp.experts.127.down_proj", + "model.layers.29.mlp.experts.127.gate_proj", + "model.layers.29.mlp.experts.127.up_proj", + "model.layers.29.mlp.experts.128.down_proj", + "model.layers.29.mlp.experts.128.gate_proj", + "model.layers.29.mlp.experts.128.up_proj", + "model.layers.29.mlp.experts.129.down_proj", + "model.layers.29.mlp.experts.129.gate_proj", + "model.layers.29.mlp.experts.129.up_proj", + "model.layers.29.mlp.experts.13.down_proj", + "model.layers.29.mlp.experts.13.gate_proj", + "model.layers.29.mlp.experts.13.up_proj", + "model.layers.29.mlp.experts.130.down_proj", + "model.layers.29.mlp.experts.130.gate_proj", + "model.layers.29.mlp.experts.130.up_proj", + "model.layers.29.mlp.experts.131.down_proj", + "model.layers.29.mlp.experts.131.gate_proj", + "model.layers.29.mlp.experts.131.up_proj", + "model.layers.29.mlp.experts.132.down_proj", + "model.layers.29.mlp.experts.132.gate_proj", + "model.layers.29.mlp.experts.132.up_proj", + "model.layers.29.mlp.experts.133.down_proj", + "model.layers.29.mlp.experts.133.gate_proj", + "model.layers.29.mlp.experts.133.up_proj", + "model.layers.29.mlp.experts.134.down_proj", + "model.layers.29.mlp.experts.134.gate_proj", + "model.layers.29.mlp.experts.134.up_proj", + "model.layers.29.mlp.experts.135.down_proj", + "model.layers.29.mlp.experts.135.gate_proj", + "model.layers.29.mlp.experts.135.up_proj", + "model.layers.29.mlp.experts.136.down_proj", + "model.layers.29.mlp.experts.136.gate_proj", + "model.layers.29.mlp.experts.136.up_proj", + "model.layers.29.mlp.experts.137.down_proj", + "model.layers.29.mlp.experts.137.gate_proj", + "model.layers.29.mlp.experts.137.up_proj", + "model.layers.29.mlp.experts.138.down_proj", + "model.layers.29.mlp.experts.138.gate_proj", + "model.layers.29.mlp.experts.138.up_proj", + "model.layers.29.mlp.experts.139.down_proj", + "model.layers.29.mlp.experts.139.gate_proj", + "model.layers.29.mlp.experts.139.up_proj", + "model.layers.29.mlp.experts.14.down_proj", + "model.layers.29.mlp.experts.14.gate_proj", + "model.layers.29.mlp.experts.14.up_proj", + "model.layers.29.mlp.experts.140.down_proj", + "model.layers.29.mlp.experts.140.gate_proj", + "model.layers.29.mlp.experts.140.up_proj", + "model.layers.29.mlp.experts.141.down_proj", + "model.layers.29.mlp.experts.141.gate_proj", + "model.layers.29.mlp.experts.141.up_proj", + "model.layers.29.mlp.experts.142.down_proj", + "model.layers.29.mlp.experts.142.gate_proj", + "model.layers.29.mlp.experts.142.up_proj", + "model.layers.29.mlp.experts.143.down_proj", + "model.layers.29.mlp.experts.143.gate_proj", + "model.layers.29.mlp.experts.143.up_proj", + "model.layers.29.mlp.experts.144.down_proj", + "model.layers.29.mlp.experts.144.gate_proj", + "model.layers.29.mlp.experts.144.up_proj", + "model.layers.29.mlp.experts.145.down_proj", + "model.layers.29.mlp.experts.145.gate_proj", + "model.layers.29.mlp.experts.145.up_proj", + "model.layers.29.mlp.experts.146.down_proj", + "model.layers.29.mlp.experts.146.gate_proj", + "model.layers.29.mlp.experts.146.up_proj", + "model.layers.29.mlp.experts.147.down_proj", + "model.layers.29.mlp.experts.147.gate_proj", + "model.layers.29.mlp.experts.147.up_proj", + "model.layers.29.mlp.experts.148.down_proj", + "model.layers.29.mlp.experts.148.gate_proj", + "model.layers.29.mlp.experts.148.up_proj", + "model.layers.29.mlp.experts.149.down_proj", + "model.layers.29.mlp.experts.149.gate_proj", + "model.layers.29.mlp.experts.149.up_proj", + "model.layers.29.mlp.experts.15.down_proj", + "model.layers.29.mlp.experts.15.gate_proj", + "model.layers.29.mlp.experts.15.up_proj", + "model.layers.29.mlp.experts.150.down_proj", + "model.layers.29.mlp.experts.150.gate_proj", + "model.layers.29.mlp.experts.150.up_proj", + "model.layers.29.mlp.experts.151.down_proj", + "model.layers.29.mlp.experts.151.gate_proj", + "model.layers.29.mlp.experts.151.up_proj", + "model.layers.29.mlp.experts.152.down_proj", + "model.layers.29.mlp.experts.152.gate_proj", + "model.layers.29.mlp.experts.152.up_proj", + "model.layers.29.mlp.experts.153.down_proj", + "model.layers.29.mlp.experts.153.gate_proj", + "model.layers.29.mlp.experts.153.up_proj", + "model.layers.29.mlp.experts.154.down_proj", + "model.layers.29.mlp.experts.154.gate_proj", + "model.layers.29.mlp.experts.154.up_proj", + "model.layers.29.mlp.experts.155.down_proj", + "model.layers.29.mlp.experts.155.gate_proj", + "model.layers.29.mlp.experts.155.up_proj", + "model.layers.29.mlp.experts.156.down_proj", + "model.layers.29.mlp.experts.156.gate_proj", + "model.layers.29.mlp.experts.156.up_proj", + "model.layers.29.mlp.experts.157.down_proj", + "model.layers.29.mlp.experts.157.gate_proj", + "model.layers.29.mlp.experts.157.up_proj", + "model.layers.29.mlp.experts.158.down_proj", + "model.layers.29.mlp.experts.158.gate_proj", + "model.layers.29.mlp.experts.158.up_proj", + "model.layers.29.mlp.experts.159.down_proj", + "model.layers.29.mlp.experts.159.gate_proj", + "model.layers.29.mlp.experts.159.up_proj", + "model.layers.29.mlp.experts.16.down_proj", + "model.layers.29.mlp.experts.16.gate_proj", + "model.layers.29.mlp.experts.16.up_proj", + "model.layers.29.mlp.experts.160.down_proj", + "model.layers.29.mlp.experts.160.gate_proj", + "model.layers.29.mlp.experts.160.up_proj", + "model.layers.29.mlp.experts.161.down_proj", + "model.layers.29.mlp.experts.161.gate_proj", + "model.layers.29.mlp.experts.161.up_proj", + "model.layers.29.mlp.experts.162.down_proj", + "model.layers.29.mlp.experts.162.gate_proj", + "model.layers.29.mlp.experts.162.up_proj", + "model.layers.29.mlp.experts.163.down_proj", + "model.layers.29.mlp.experts.163.gate_proj", + "model.layers.29.mlp.experts.163.up_proj", + "model.layers.29.mlp.experts.164.down_proj", + "model.layers.29.mlp.experts.164.gate_proj", + "model.layers.29.mlp.experts.164.up_proj", + "model.layers.29.mlp.experts.165.down_proj", + "model.layers.29.mlp.experts.165.gate_proj", + "model.layers.29.mlp.experts.165.up_proj", + "model.layers.29.mlp.experts.166.down_proj", + "model.layers.29.mlp.experts.166.gate_proj", + "model.layers.29.mlp.experts.166.up_proj", + "model.layers.29.mlp.experts.167.down_proj", + "model.layers.29.mlp.experts.167.gate_proj", + "model.layers.29.mlp.experts.167.up_proj", + "model.layers.29.mlp.experts.168.down_proj", + "model.layers.29.mlp.experts.168.gate_proj", + "model.layers.29.mlp.experts.168.up_proj", + "model.layers.29.mlp.experts.169.down_proj", + "model.layers.29.mlp.experts.169.gate_proj", + "model.layers.29.mlp.experts.169.up_proj", + "model.layers.29.mlp.experts.17.down_proj", + "model.layers.29.mlp.experts.17.gate_proj", + "model.layers.29.mlp.experts.17.up_proj", + "model.layers.29.mlp.experts.170.down_proj", + "model.layers.29.mlp.experts.170.gate_proj", + "model.layers.29.mlp.experts.170.up_proj", + "model.layers.29.mlp.experts.171.down_proj", + "model.layers.29.mlp.experts.171.gate_proj", + "model.layers.29.mlp.experts.171.up_proj", + "model.layers.29.mlp.experts.172.down_proj", + "model.layers.29.mlp.experts.172.gate_proj", + "model.layers.29.mlp.experts.172.up_proj", + "model.layers.29.mlp.experts.173.down_proj", + "model.layers.29.mlp.experts.173.gate_proj", + "model.layers.29.mlp.experts.173.up_proj", + "model.layers.29.mlp.experts.174.down_proj", + "model.layers.29.mlp.experts.174.gate_proj", + "model.layers.29.mlp.experts.174.up_proj", + "model.layers.29.mlp.experts.175.down_proj", + "model.layers.29.mlp.experts.175.gate_proj", + "model.layers.29.mlp.experts.175.up_proj", + "model.layers.29.mlp.experts.176.down_proj", + "model.layers.29.mlp.experts.176.gate_proj", + "model.layers.29.mlp.experts.176.up_proj", + "model.layers.29.mlp.experts.177.down_proj", + "model.layers.29.mlp.experts.177.gate_proj", + "model.layers.29.mlp.experts.177.up_proj", + "model.layers.29.mlp.experts.178.down_proj", + "model.layers.29.mlp.experts.178.gate_proj", + "model.layers.29.mlp.experts.178.up_proj", + "model.layers.29.mlp.experts.179.down_proj", + "model.layers.29.mlp.experts.179.gate_proj", + "model.layers.29.mlp.experts.179.up_proj", + "model.layers.29.mlp.experts.18.down_proj", + "model.layers.29.mlp.experts.18.gate_proj", + "model.layers.29.mlp.experts.18.up_proj", + "model.layers.29.mlp.experts.180.down_proj", + "model.layers.29.mlp.experts.180.gate_proj", + "model.layers.29.mlp.experts.180.up_proj", + "model.layers.29.mlp.experts.181.down_proj", + "model.layers.29.mlp.experts.181.gate_proj", + "model.layers.29.mlp.experts.181.up_proj", + "model.layers.29.mlp.experts.182.down_proj", + "model.layers.29.mlp.experts.182.gate_proj", + "model.layers.29.mlp.experts.182.up_proj", + "model.layers.29.mlp.experts.183.down_proj", + "model.layers.29.mlp.experts.183.gate_proj", + "model.layers.29.mlp.experts.183.up_proj", + "model.layers.29.mlp.experts.184.down_proj", + "model.layers.29.mlp.experts.184.gate_proj", + "model.layers.29.mlp.experts.184.up_proj", + "model.layers.29.mlp.experts.185.down_proj", + "model.layers.29.mlp.experts.185.gate_proj", + "model.layers.29.mlp.experts.185.up_proj", + "model.layers.29.mlp.experts.186.down_proj", + "model.layers.29.mlp.experts.186.gate_proj", + "model.layers.29.mlp.experts.186.up_proj", + "model.layers.29.mlp.experts.187.down_proj", + "model.layers.29.mlp.experts.187.gate_proj", + "model.layers.29.mlp.experts.187.up_proj", + "model.layers.29.mlp.experts.188.down_proj", + "model.layers.29.mlp.experts.188.gate_proj", + "model.layers.29.mlp.experts.188.up_proj", + "model.layers.29.mlp.experts.189.down_proj", + "model.layers.29.mlp.experts.189.gate_proj", + "model.layers.29.mlp.experts.189.up_proj", + "model.layers.29.mlp.experts.19.down_proj", + "model.layers.29.mlp.experts.19.gate_proj", + "model.layers.29.mlp.experts.19.up_proj", + "model.layers.29.mlp.experts.190.down_proj", + "model.layers.29.mlp.experts.190.gate_proj", + "model.layers.29.mlp.experts.190.up_proj", + "model.layers.29.mlp.experts.191.down_proj", + "model.layers.29.mlp.experts.191.gate_proj", + "model.layers.29.mlp.experts.191.up_proj", + "model.layers.29.mlp.experts.192.down_proj", + "model.layers.29.mlp.experts.192.gate_proj", + "model.layers.29.mlp.experts.192.up_proj", + "model.layers.29.mlp.experts.193.down_proj", + "model.layers.29.mlp.experts.193.gate_proj", + "model.layers.29.mlp.experts.193.up_proj", + "model.layers.29.mlp.experts.194.down_proj", + "model.layers.29.mlp.experts.194.gate_proj", + "model.layers.29.mlp.experts.194.up_proj", + "model.layers.29.mlp.experts.195.down_proj", + "model.layers.29.mlp.experts.195.gate_proj", + "model.layers.29.mlp.experts.195.up_proj", + "model.layers.29.mlp.experts.196.down_proj", + "model.layers.29.mlp.experts.196.gate_proj", + "model.layers.29.mlp.experts.196.up_proj", + "model.layers.29.mlp.experts.197.down_proj", + "model.layers.29.mlp.experts.197.gate_proj", + "model.layers.29.mlp.experts.197.up_proj", + "model.layers.29.mlp.experts.198.down_proj", + "model.layers.29.mlp.experts.198.gate_proj", + "model.layers.29.mlp.experts.198.up_proj", + "model.layers.29.mlp.experts.199.down_proj", + "model.layers.29.mlp.experts.199.gate_proj", + "model.layers.29.mlp.experts.199.up_proj", + "model.layers.29.mlp.experts.2.down_proj", + "model.layers.29.mlp.experts.2.gate_proj", + "model.layers.29.mlp.experts.2.up_proj", + "model.layers.29.mlp.experts.20.down_proj", + "model.layers.29.mlp.experts.20.gate_proj", + "model.layers.29.mlp.experts.20.up_proj", + "model.layers.29.mlp.experts.200.down_proj", + "model.layers.29.mlp.experts.200.gate_proj", + "model.layers.29.mlp.experts.200.up_proj", + "model.layers.29.mlp.experts.201.down_proj", + "model.layers.29.mlp.experts.201.gate_proj", + "model.layers.29.mlp.experts.201.up_proj", + "model.layers.29.mlp.experts.202.down_proj", + "model.layers.29.mlp.experts.202.gate_proj", + "model.layers.29.mlp.experts.202.up_proj", + "model.layers.29.mlp.experts.203.down_proj", + "model.layers.29.mlp.experts.203.gate_proj", + "model.layers.29.mlp.experts.203.up_proj", + "model.layers.29.mlp.experts.204.down_proj", + "model.layers.29.mlp.experts.204.gate_proj", + "model.layers.29.mlp.experts.204.up_proj", + "model.layers.29.mlp.experts.205.down_proj", + "model.layers.29.mlp.experts.205.gate_proj", + "model.layers.29.mlp.experts.205.up_proj", + "model.layers.29.mlp.experts.206.down_proj", + "model.layers.29.mlp.experts.206.gate_proj", + "model.layers.29.mlp.experts.206.up_proj", + "model.layers.29.mlp.experts.207.down_proj", + "model.layers.29.mlp.experts.207.gate_proj", + "model.layers.29.mlp.experts.207.up_proj", + "model.layers.29.mlp.experts.208.down_proj", + "model.layers.29.mlp.experts.208.gate_proj", + "model.layers.29.mlp.experts.208.up_proj", + "model.layers.29.mlp.experts.209.down_proj", + "model.layers.29.mlp.experts.209.gate_proj", + "model.layers.29.mlp.experts.209.up_proj", + "model.layers.29.mlp.experts.21.down_proj", + "model.layers.29.mlp.experts.21.gate_proj", + "model.layers.29.mlp.experts.21.up_proj", + "model.layers.29.mlp.experts.210.down_proj", + "model.layers.29.mlp.experts.210.gate_proj", + "model.layers.29.mlp.experts.210.up_proj", + "model.layers.29.mlp.experts.211.down_proj", + "model.layers.29.mlp.experts.211.gate_proj", + "model.layers.29.mlp.experts.211.up_proj", + "model.layers.29.mlp.experts.212.down_proj", + "model.layers.29.mlp.experts.212.gate_proj", + "model.layers.29.mlp.experts.212.up_proj", + "model.layers.29.mlp.experts.213.down_proj", + "model.layers.29.mlp.experts.213.gate_proj", + "model.layers.29.mlp.experts.213.up_proj", + "model.layers.29.mlp.experts.214.down_proj", + "model.layers.29.mlp.experts.214.gate_proj", + "model.layers.29.mlp.experts.214.up_proj", + "model.layers.29.mlp.experts.215.down_proj", + "model.layers.29.mlp.experts.215.gate_proj", + "model.layers.29.mlp.experts.215.up_proj", + "model.layers.29.mlp.experts.216.down_proj", + "model.layers.29.mlp.experts.216.gate_proj", + "model.layers.29.mlp.experts.216.up_proj", + "model.layers.29.mlp.experts.217.down_proj", + "model.layers.29.mlp.experts.217.gate_proj", + "model.layers.29.mlp.experts.217.up_proj", + "model.layers.29.mlp.experts.218.down_proj", + "model.layers.29.mlp.experts.218.gate_proj", + "model.layers.29.mlp.experts.218.up_proj", + "model.layers.29.mlp.experts.219.down_proj", + "model.layers.29.mlp.experts.219.gate_proj", + "model.layers.29.mlp.experts.219.up_proj", + "model.layers.29.mlp.experts.22.down_proj", + "model.layers.29.mlp.experts.22.gate_proj", + "model.layers.29.mlp.experts.22.up_proj", + "model.layers.29.mlp.experts.220.down_proj", + "model.layers.29.mlp.experts.220.gate_proj", + "model.layers.29.mlp.experts.220.up_proj", + "model.layers.29.mlp.experts.221.down_proj", + "model.layers.29.mlp.experts.221.gate_proj", + "model.layers.29.mlp.experts.221.up_proj", + "model.layers.29.mlp.experts.222.down_proj", + "model.layers.29.mlp.experts.222.gate_proj", + "model.layers.29.mlp.experts.222.up_proj", + "model.layers.29.mlp.experts.223.down_proj", + "model.layers.29.mlp.experts.223.gate_proj", + "model.layers.29.mlp.experts.223.up_proj", + "model.layers.29.mlp.experts.224.down_proj", + "model.layers.29.mlp.experts.224.gate_proj", + "model.layers.29.mlp.experts.224.up_proj", + "model.layers.29.mlp.experts.225.down_proj", + "model.layers.29.mlp.experts.225.gate_proj", + "model.layers.29.mlp.experts.225.up_proj", + "model.layers.29.mlp.experts.226.down_proj", + "model.layers.29.mlp.experts.226.gate_proj", + "model.layers.29.mlp.experts.226.up_proj", + "model.layers.29.mlp.experts.227.down_proj", + "model.layers.29.mlp.experts.227.gate_proj", + "model.layers.29.mlp.experts.227.up_proj", + "model.layers.29.mlp.experts.228.down_proj", + "model.layers.29.mlp.experts.228.gate_proj", + "model.layers.29.mlp.experts.228.up_proj", + "model.layers.29.mlp.experts.229.down_proj", + "model.layers.29.mlp.experts.229.gate_proj", + "model.layers.29.mlp.experts.229.up_proj", + "model.layers.29.mlp.experts.23.down_proj", + "model.layers.29.mlp.experts.23.gate_proj", + "model.layers.29.mlp.experts.23.up_proj", + "model.layers.29.mlp.experts.230.down_proj", + "model.layers.29.mlp.experts.230.gate_proj", + "model.layers.29.mlp.experts.230.up_proj", + "model.layers.29.mlp.experts.231.down_proj", + "model.layers.29.mlp.experts.231.gate_proj", + "model.layers.29.mlp.experts.231.up_proj", + "model.layers.29.mlp.experts.232.down_proj", + "model.layers.29.mlp.experts.232.gate_proj", + "model.layers.29.mlp.experts.232.up_proj", + "model.layers.29.mlp.experts.233.down_proj", + "model.layers.29.mlp.experts.233.gate_proj", + "model.layers.29.mlp.experts.233.up_proj", + "model.layers.29.mlp.experts.234.down_proj", + "model.layers.29.mlp.experts.234.gate_proj", + "model.layers.29.mlp.experts.234.up_proj", + "model.layers.29.mlp.experts.235.down_proj", + "model.layers.29.mlp.experts.235.gate_proj", + "model.layers.29.mlp.experts.235.up_proj", + "model.layers.29.mlp.experts.236.down_proj", + "model.layers.29.mlp.experts.236.gate_proj", + "model.layers.29.mlp.experts.236.up_proj", + "model.layers.29.mlp.experts.237.down_proj", + "model.layers.29.mlp.experts.237.gate_proj", + "model.layers.29.mlp.experts.237.up_proj", + "model.layers.29.mlp.experts.238.down_proj", + "model.layers.29.mlp.experts.238.gate_proj", + "model.layers.29.mlp.experts.238.up_proj", + "model.layers.29.mlp.experts.239.down_proj", + "model.layers.29.mlp.experts.239.gate_proj", + "model.layers.29.mlp.experts.239.up_proj", + "model.layers.29.mlp.experts.24.down_proj", + "model.layers.29.mlp.experts.24.gate_proj", + "model.layers.29.mlp.experts.24.up_proj", + "model.layers.29.mlp.experts.240.down_proj", + "model.layers.29.mlp.experts.240.gate_proj", + "model.layers.29.mlp.experts.240.up_proj", + "model.layers.29.mlp.experts.241.down_proj", + "model.layers.29.mlp.experts.241.gate_proj", + "model.layers.29.mlp.experts.241.up_proj", + "model.layers.29.mlp.experts.242.down_proj", + "model.layers.29.mlp.experts.242.gate_proj", + "model.layers.29.mlp.experts.242.up_proj", + "model.layers.29.mlp.experts.243.down_proj", + "model.layers.29.mlp.experts.243.gate_proj", + "model.layers.29.mlp.experts.243.up_proj", + "model.layers.29.mlp.experts.244.down_proj", + "model.layers.29.mlp.experts.244.gate_proj", + "model.layers.29.mlp.experts.244.up_proj", + "model.layers.29.mlp.experts.245.down_proj", + "model.layers.29.mlp.experts.245.gate_proj", + "model.layers.29.mlp.experts.245.up_proj", + "model.layers.29.mlp.experts.246.down_proj", + "model.layers.29.mlp.experts.246.gate_proj", + "model.layers.29.mlp.experts.246.up_proj", + "model.layers.29.mlp.experts.247.down_proj", + "model.layers.29.mlp.experts.247.gate_proj", + "model.layers.29.mlp.experts.247.up_proj", + "model.layers.29.mlp.experts.248.down_proj", + "model.layers.29.mlp.experts.248.gate_proj", + "model.layers.29.mlp.experts.248.up_proj", + "model.layers.29.mlp.experts.249.down_proj", + "model.layers.29.mlp.experts.249.gate_proj", + "model.layers.29.mlp.experts.249.up_proj", + "model.layers.29.mlp.experts.25.down_proj", + "model.layers.29.mlp.experts.25.gate_proj", + "model.layers.29.mlp.experts.25.up_proj", + "model.layers.29.mlp.experts.250.down_proj", + "model.layers.29.mlp.experts.250.gate_proj", + "model.layers.29.mlp.experts.250.up_proj", + "model.layers.29.mlp.experts.251.down_proj", + "model.layers.29.mlp.experts.251.gate_proj", + "model.layers.29.mlp.experts.251.up_proj", + "model.layers.29.mlp.experts.252.down_proj", + "model.layers.29.mlp.experts.252.gate_proj", + "model.layers.29.mlp.experts.252.up_proj", + "model.layers.29.mlp.experts.253.down_proj", + "model.layers.29.mlp.experts.253.gate_proj", + "model.layers.29.mlp.experts.253.up_proj", + "model.layers.29.mlp.experts.254.down_proj", + "model.layers.29.mlp.experts.254.gate_proj", + "model.layers.29.mlp.experts.254.up_proj", + "model.layers.29.mlp.experts.255.down_proj", + "model.layers.29.mlp.experts.255.gate_proj", + "model.layers.29.mlp.experts.255.up_proj", + "model.layers.29.mlp.experts.26.down_proj", + "model.layers.29.mlp.experts.26.gate_proj", + "model.layers.29.mlp.experts.26.up_proj", + "model.layers.29.mlp.experts.27.down_proj", + "model.layers.29.mlp.experts.27.gate_proj", + "model.layers.29.mlp.experts.27.up_proj", + "model.layers.29.mlp.experts.28.down_proj", + "model.layers.29.mlp.experts.28.gate_proj", + "model.layers.29.mlp.experts.28.up_proj", + "model.layers.29.mlp.experts.29.down_proj", + "model.layers.29.mlp.experts.29.gate_proj", + "model.layers.29.mlp.experts.29.up_proj", + "model.layers.29.mlp.experts.3.down_proj", + "model.layers.29.mlp.experts.3.gate_proj", + "model.layers.29.mlp.experts.3.up_proj", + "model.layers.29.mlp.experts.30.down_proj", + "model.layers.29.mlp.experts.30.gate_proj", + "model.layers.29.mlp.experts.30.up_proj", + "model.layers.29.mlp.experts.31.down_proj", + "model.layers.29.mlp.experts.31.gate_proj", + "model.layers.29.mlp.experts.31.up_proj", + "model.layers.29.mlp.experts.32.down_proj", + "model.layers.29.mlp.experts.32.gate_proj", + "model.layers.29.mlp.experts.32.up_proj", + "model.layers.29.mlp.experts.33.down_proj", + "model.layers.29.mlp.experts.33.gate_proj", + "model.layers.29.mlp.experts.33.up_proj", + "model.layers.29.mlp.experts.34.down_proj", + "model.layers.29.mlp.experts.34.gate_proj", + "model.layers.29.mlp.experts.34.up_proj", + "model.layers.29.mlp.experts.35.down_proj", + "model.layers.29.mlp.experts.35.gate_proj", + "model.layers.29.mlp.experts.35.up_proj", + "model.layers.29.mlp.experts.36.down_proj", + "model.layers.29.mlp.experts.36.gate_proj", + "model.layers.29.mlp.experts.36.up_proj", + "model.layers.29.mlp.experts.37.down_proj", + "model.layers.29.mlp.experts.37.gate_proj", + "model.layers.29.mlp.experts.37.up_proj", + "model.layers.29.mlp.experts.38.down_proj", + "model.layers.29.mlp.experts.38.gate_proj", + "model.layers.29.mlp.experts.38.up_proj", + "model.layers.29.mlp.experts.39.down_proj", + "model.layers.29.mlp.experts.39.gate_proj", + "model.layers.29.mlp.experts.39.up_proj", + "model.layers.29.mlp.experts.4.down_proj", + "model.layers.29.mlp.experts.4.gate_proj", + "model.layers.29.mlp.experts.4.up_proj", + "model.layers.29.mlp.experts.40.down_proj", + "model.layers.29.mlp.experts.40.gate_proj", + "model.layers.29.mlp.experts.40.up_proj", + "model.layers.29.mlp.experts.41.down_proj", + "model.layers.29.mlp.experts.41.gate_proj", + "model.layers.29.mlp.experts.41.up_proj", + "model.layers.29.mlp.experts.42.down_proj", + "model.layers.29.mlp.experts.42.gate_proj", + "model.layers.29.mlp.experts.42.up_proj", + "model.layers.29.mlp.experts.43.down_proj", + "model.layers.29.mlp.experts.43.gate_proj", + "model.layers.29.mlp.experts.43.up_proj", + "model.layers.29.mlp.experts.44.down_proj", + "model.layers.29.mlp.experts.44.gate_proj", + "model.layers.29.mlp.experts.44.up_proj", + "model.layers.29.mlp.experts.45.down_proj", + "model.layers.29.mlp.experts.45.gate_proj", + "model.layers.29.mlp.experts.45.up_proj", + "model.layers.29.mlp.experts.46.down_proj", + "model.layers.29.mlp.experts.46.gate_proj", + "model.layers.29.mlp.experts.46.up_proj", + "model.layers.29.mlp.experts.47.down_proj", + "model.layers.29.mlp.experts.47.gate_proj", + "model.layers.29.mlp.experts.47.up_proj", + "model.layers.29.mlp.experts.48.down_proj", + "model.layers.29.mlp.experts.48.gate_proj", + "model.layers.29.mlp.experts.48.up_proj", + "model.layers.29.mlp.experts.49.down_proj", + "model.layers.29.mlp.experts.49.gate_proj", + "model.layers.29.mlp.experts.49.up_proj", + "model.layers.29.mlp.experts.5.down_proj", + "model.layers.29.mlp.experts.5.gate_proj", + "model.layers.29.mlp.experts.5.up_proj", + "model.layers.29.mlp.experts.50.down_proj", + "model.layers.29.mlp.experts.50.gate_proj", + "model.layers.29.mlp.experts.50.up_proj", + "model.layers.29.mlp.experts.51.down_proj", + "model.layers.29.mlp.experts.51.gate_proj", + "model.layers.29.mlp.experts.51.up_proj", + "model.layers.29.mlp.experts.52.down_proj", + "model.layers.29.mlp.experts.52.gate_proj", + "model.layers.29.mlp.experts.52.up_proj", + "model.layers.29.mlp.experts.53.down_proj", + "model.layers.29.mlp.experts.53.gate_proj", + "model.layers.29.mlp.experts.53.up_proj", + "model.layers.29.mlp.experts.54.down_proj", + "model.layers.29.mlp.experts.54.gate_proj", + "model.layers.29.mlp.experts.54.up_proj", + "model.layers.29.mlp.experts.55.down_proj", + "model.layers.29.mlp.experts.55.gate_proj", + "model.layers.29.mlp.experts.55.up_proj", + "model.layers.29.mlp.experts.56.down_proj", + "model.layers.29.mlp.experts.56.gate_proj", + "model.layers.29.mlp.experts.56.up_proj", + "model.layers.29.mlp.experts.57.down_proj", + "model.layers.29.mlp.experts.57.gate_proj", + "model.layers.29.mlp.experts.57.up_proj", + "model.layers.29.mlp.experts.58.down_proj", + "model.layers.29.mlp.experts.58.gate_proj", + "model.layers.29.mlp.experts.58.up_proj", + "model.layers.29.mlp.experts.59.down_proj", + "model.layers.29.mlp.experts.59.gate_proj", + "model.layers.29.mlp.experts.59.up_proj", + "model.layers.29.mlp.experts.6.down_proj", + "model.layers.29.mlp.experts.6.gate_proj", + "model.layers.29.mlp.experts.6.up_proj", + "model.layers.29.mlp.experts.60.down_proj", + "model.layers.29.mlp.experts.60.gate_proj", + "model.layers.29.mlp.experts.60.up_proj", + "model.layers.29.mlp.experts.61.down_proj", + "model.layers.29.mlp.experts.61.gate_proj", + "model.layers.29.mlp.experts.61.up_proj", + "model.layers.29.mlp.experts.62.down_proj", + "model.layers.29.mlp.experts.62.gate_proj", + "model.layers.29.mlp.experts.62.up_proj", + "model.layers.29.mlp.experts.63.down_proj", + "model.layers.29.mlp.experts.63.gate_proj", + "model.layers.29.mlp.experts.63.up_proj", + "model.layers.29.mlp.experts.64.down_proj", + "model.layers.29.mlp.experts.64.gate_proj", + "model.layers.29.mlp.experts.64.up_proj", + "model.layers.29.mlp.experts.65.down_proj", + "model.layers.29.mlp.experts.65.gate_proj", + "model.layers.29.mlp.experts.65.up_proj", + "model.layers.29.mlp.experts.66.down_proj", + "model.layers.29.mlp.experts.66.gate_proj", + "model.layers.29.mlp.experts.66.up_proj", + "model.layers.29.mlp.experts.67.down_proj", + "model.layers.29.mlp.experts.67.gate_proj", + "model.layers.29.mlp.experts.67.up_proj", + "model.layers.29.mlp.experts.68.down_proj", + "model.layers.29.mlp.experts.68.gate_proj", + "model.layers.29.mlp.experts.68.up_proj", + "model.layers.29.mlp.experts.69.down_proj", + "model.layers.29.mlp.experts.69.gate_proj", + "model.layers.29.mlp.experts.69.up_proj", + "model.layers.29.mlp.experts.7.down_proj", + "model.layers.29.mlp.experts.7.gate_proj", + "model.layers.29.mlp.experts.7.up_proj", + "model.layers.29.mlp.experts.70.down_proj", + "model.layers.29.mlp.experts.70.gate_proj", + "model.layers.29.mlp.experts.70.up_proj", + "model.layers.29.mlp.experts.71.down_proj", + "model.layers.29.mlp.experts.71.gate_proj", + "model.layers.29.mlp.experts.71.up_proj", + "model.layers.29.mlp.experts.72.down_proj", + "model.layers.29.mlp.experts.72.gate_proj", + "model.layers.29.mlp.experts.72.up_proj", + "model.layers.29.mlp.experts.73.down_proj", + "model.layers.29.mlp.experts.73.gate_proj", + "model.layers.29.mlp.experts.73.up_proj", + "model.layers.29.mlp.experts.74.down_proj", + "model.layers.29.mlp.experts.74.gate_proj", + "model.layers.29.mlp.experts.74.up_proj", + "model.layers.29.mlp.experts.75.down_proj", + "model.layers.29.mlp.experts.75.gate_proj", + "model.layers.29.mlp.experts.75.up_proj", + "model.layers.29.mlp.experts.76.down_proj", + "model.layers.29.mlp.experts.76.gate_proj", + "model.layers.29.mlp.experts.76.up_proj", + "model.layers.29.mlp.experts.77.down_proj", + "model.layers.29.mlp.experts.77.gate_proj", + "model.layers.29.mlp.experts.77.up_proj", + "model.layers.29.mlp.experts.78.down_proj", + "model.layers.29.mlp.experts.78.gate_proj", + "model.layers.29.mlp.experts.78.up_proj", + "model.layers.29.mlp.experts.79.down_proj", + "model.layers.29.mlp.experts.79.gate_proj", + "model.layers.29.mlp.experts.79.up_proj", + "model.layers.29.mlp.experts.8.down_proj", + "model.layers.29.mlp.experts.8.gate_proj", + "model.layers.29.mlp.experts.8.up_proj", + "model.layers.29.mlp.experts.80.down_proj", + "model.layers.29.mlp.experts.80.gate_proj", + "model.layers.29.mlp.experts.80.up_proj", + "model.layers.29.mlp.experts.81.down_proj", + "model.layers.29.mlp.experts.81.gate_proj", + "model.layers.29.mlp.experts.81.up_proj", + "model.layers.29.mlp.experts.82.down_proj", + "model.layers.29.mlp.experts.82.gate_proj", + "model.layers.29.mlp.experts.82.up_proj", + "model.layers.29.mlp.experts.83.down_proj", + "model.layers.29.mlp.experts.83.gate_proj", + "model.layers.29.mlp.experts.83.up_proj", + "model.layers.29.mlp.experts.84.down_proj", + "model.layers.29.mlp.experts.84.gate_proj", + "model.layers.29.mlp.experts.84.up_proj", + "model.layers.29.mlp.experts.85.down_proj", + "model.layers.29.mlp.experts.85.gate_proj", + "model.layers.29.mlp.experts.85.up_proj", + "model.layers.29.mlp.experts.86.down_proj", + "model.layers.29.mlp.experts.86.gate_proj", + "model.layers.29.mlp.experts.86.up_proj", + "model.layers.29.mlp.experts.87.down_proj", + "model.layers.29.mlp.experts.87.gate_proj", + "model.layers.29.mlp.experts.87.up_proj", + "model.layers.29.mlp.experts.88.down_proj", + "model.layers.29.mlp.experts.88.gate_proj", + "model.layers.29.mlp.experts.88.up_proj", + "model.layers.29.mlp.experts.89.down_proj", + "model.layers.29.mlp.experts.89.gate_proj", + "model.layers.29.mlp.experts.89.up_proj", + "model.layers.29.mlp.experts.9.down_proj", + "model.layers.29.mlp.experts.9.gate_proj", + "model.layers.29.mlp.experts.9.up_proj", + "model.layers.29.mlp.experts.90.down_proj", + "model.layers.29.mlp.experts.90.gate_proj", + "model.layers.29.mlp.experts.90.up_proj", + "model.layers.29.mlp.experts.91.down_proj", + "model.layers.29.mlp.experts.91.gate_proj", + "model.layers.29.mlp.experts.91.up_proj", + "model.layers.29.mlp.experts.92.down_proj", + "model.layers.29.mlp.experts.92.gate_proj", + "model.layers.29.mlp.experts.92.up_proj", + "model.layers.29.mlp.experts.93.down_proj", + "model.layers.29.mlp.experts.93.gate_proj", + "model.layers.29.mlp.experts.93.up_proj", + "model.layers.29.mlp.experts.94.down_proj", + "model.layers.29.mlp.experts.94.gate_proj", + "model.layers.29.mlp.experts.94.up_proj", + "model.layers.29.mlp.experts.95.down_proj", + "model.layers.29.mlp.experts.95.gate_proj", + "model.layers.29.mlp.experts.95.up_proj", + "model.layers.29.mlp.experts.96.down_proj", + "model.layers.29.mlp.experts.96.gate_proj", + "model.layers.29.mlp.experts.96.up_proj", + "model.layers.29.mlp.experts.97.down_proj", + "model.layers.29.mlp.experts.97.gate_proj", + "model.layers.29.mlp.experts.97.up_proj", + "model.layers.29.mlp.experts.98.down_proj", + "model.layers.29.mlp.experts.98.gate_proj", + "model.layers.29.mlp.experts.98.up_proj", + "model.layers.29.mlp.experts.99.down_proj", + "model.layers.29.mlp.experts.99.gate_proj", + "model.layers.29.mlp.experts.99.up_proj", + "model.layers.3.mlp.experts.0.down_proj", + "model.layers.3.mlp.experts.0.gate_proj", + "model.layers.3.mlp.experts.0.up_proj", + "model.layers.3.mlp.experts.1.down_proj", + "model.layers.3.mlp.experts.1.gate_proj", + "model.layers.3.mlp.experts.1.up_proj", + "model.layers.3.mlp.experts.10.down_proj", + "model.layers.3.mlp.experts.10.gate_proj", + "model.layers.3.mlp.experts.10.up_proj", + "model.layers.3.mlp.experts.100.down_proj", + "model.layers.3.mlp.experts.100.gate_proj", + "model.layers.3.mlp.experts.100.up_proj", + "model.layers.3.mlp.experts.101.down_proj", + "model.layers.3.mlp.experts.101.gate_proj", + "model.layers.3.mlp.experts.101.up_proj", + "model.layers.3.mlp.experts.102.down_proj", + "model.layers.3.mlp.experts.102.gate_proj", + "model.layers.3.mlp.experts.102.up_proj", + "model.layers.3.mlp.experts.103.down_proj", + "model.layers.3.mlp.experts.103.gate_proj", + "model.layers.3.mlp.experts.103.up_proj", + "model.layers.3.mlp.experts.104.down_proj", + "model.layers.3.mlp.experts.104.gate_proj", + "model.layers.3.mlp.experts.104.up_proj", + "model.layers.3.mlp.experts.105.down_proj", + "model.layers.3.mlp.experts.105.gate_proj", + "model.layers.3.mlp.experts.105.up_proj", + "model.layers.3.mlp.experts.106.down_proj", + "model.layers.3.mlp.experts.106.gate_proj", + "model.layers.3.mlp.experts.106.up_proj", + "model.layers.3.mlp.experts.107.down_proj", + "model.layers.3.mlp.experts.107.gate_proj", + "model.layers.3.mlp.experts.107.up_proj", + "model.layers.3.mlp.experts.108.down_proj", + "model.layers.3.mlp.experts.108.gate_proj", + "model.layers.3.mlp.experts.108.up_proj", + "model.layers.3.mlp.experts.109.down_proj", + "model.layers.3.mlp.experts.109.gate_proj", + "model.layers.3.mlp.experts.109.up_proj", + "model.layers.3.mlp.experts.11.down_proj", + "model.layers.3.mlp.experts.11.gate_proj", + "model.layers.3.mlp.experts.11.up_proj", + "model.layers.3.mlp.experts.110.down_proj", + "model.layers.3.mlp.experts.110.gate_proj", + "model.layers.3.mlp.experts.110.up_proj", + "model.layers.3.mlp.experts.111.down_proj", + "model.layers.3.mlp.experts.111.gate_proj", + "model.layers.3.mlp.experts.111.up_proj", + "model.layers.3.mlp.experts.112.down_proj", + "model.layers.3.mlp.experts.112.gate_proj", + "model.layers.3.mlp.experts.112.up_proj", + "model.layers.3.mlp.experts.113.down_proj", + "model.layers.3.mlp.experts.113.gate_proj", + "model.layers.3.mlp.experts.113.up_proj", + "model.layers.3.mlp.experts.114.down_proj", + "model.layers.3.mlp.experts.114.gate_proj", + "model.layers.3.mlp.experts.114.up_proj", + "model.layers.3.mlp.experts.115.down_proj", + "model.layers.3.mlp.experts.115.gate_proj", + "model.layers.3.mlp.experts.115.up_proj", + "model.layers.3.mlp.experts.116.down_proj", + "model.layers.3.mlp.experts.116.gate_proj", + "model.layers.3.mlp.experts.116.up_proj", + "model.layers.3.mlp.experts.117.down_proj", + "model.layers.3.mlp.experts.117.gate_proj", + "model.layers.3.mlp.experts.117.up_proj", + "model.layers.3.mlp.experts.118.down_proj", + "model.layers.3.mlp.experts.118.gate_proj", + "model.layers.3.mlp.experts.118.up_proj", + "model.layers.3.mlp.experts.119.down_proj", + "model.layers.3.mlp.experts.119.gate_proj", + "model.layers.3.mlp.experts.119.up_proj", + "model.layers.3.mlp.experts.12.down_proj", + "model.layers.3.mlp.experts.12.gate_proj", + "model.layers.3.mlp.experts.12.up_proj", + "model.layers.3.mlp.experts.120.down_proj", + "model.layers.3.mlp.experts.120.gate_proj", + "model.layers.3.mlp.experts.120.up_proj", + "model.layers.3.mlp.experts.121.down_proj", + "model.layers.3.mlp.experts.121.gate_proj", + "model.layers.3.mlp.experts.121.up_proj", + "model.layers.3.mlp.experts.122.down_proj", + "model.layers.3.mlp.experts.122.gate_proj", + "model.layers.3.mlp.experts.122.up_proj", + "model.layers.3.mlp.experts.123.down_proj", + "model.layers.3.mlp.experts.123.gate_proj", + "model.layers.3.mlp.experts.123.up_proj", + "model.layers.3.mlp.experts.124.down_proj", + "model.layers.3.mlp.experts.124.gate_proj", + "model.layers.3.mlp.experts.124.up_proj", + "model.layers.3.mlp.experts.125.down_proj", + "model.layers.3.mlp.experts.125.gate_proj", + "model.layers.3.mlp.experts.125.up_proj", + "model.layers.3.mlp.experts.126.down_proj", + "model.layers.3.mlp.experts.126.gate_proj", + "model.layers.3.mlp.experts.126.up_proj", + "model.layers.3.mlp.experts.127.down_proj", + "model.layers.3.mlp.experts.127.gate_proj", + "model.layers.3.mlp.experts.127.up_proj", + "model.layers.3.mlp.experts.128.down_proj", + "model.layers.3.mlp.experts.128.gate_proj", + "model.layers.3.mlp.experts.128.up_proj", + "model.layers.3.mlp.experts.129.down_proj", + "model.layers.3.mlp.experts.129.gate_proj", + "model.layers.3.mlp.experts.129.up_proj", + "model.layers.3.mlp.experts.13.down_proj", + "model.layers.3.mlp.experts.13.gate_proj", + "model.layers.3.mlp.experts.13.up_proj", + "model.layers.3.mlp.experts.130.down_proj", + "model.layers.3.mlp.experts.130.gate_proj", + "model.layers.3.mlp.experts.130.up_proj", + "model.layers.3.mlp.experts.131.down_proj", + "model.layers.3.mlp.experts.131.gate_proj", + "model.layers.3.mlp.experts.131.up_proj", + "model.layers.3.mlp.experts.132.down_proj", + "model.layers.3.mlp.experts.132.gate_proj", + "model.layers.3.mlp.experts.132.up_proj", + "model.layers.3.mlp.experts.133.down_proj", + "model.layers.3.mlp.experts.133.gate_proj", + "model.layers.3.mlp.experts.133.up_proj", + "model.layers.3.mlp.experts.134.down_proj", + "model.layers.3.mlp.experts.134.gate_proj", + "model.layers.3.mlp.experts.134.up_proj", + "model.layers.3.mlp.experts.135.down_proj", + "model.layers.3.mlp.experts.135.gate_proj", + "model.layers.3.mlp.experts.135.up_proj", + "model.layers.3.mlp.experts.136.down_proj", + "model.layers.3.mlp.experts.136.gate_proj", + "model.layers.3.mlp.experts.136.up_proj", + "model.layers.3.mlp.experts.137.down_proj", + "model.layers.3.mlp.experts.137.gate_proj", + "model.layers.3.mlp.experts.137.up_proj", + "model.layers.3.mlp.experts.138.down_proj", + "model.layers.3.mlp.experts.138.gate_proj", + "model.layers.3.mlp.experts.138.up_proj", + "model.layers.3.mlp.experts.139.down_proj", + "model.layers.3.mlp.experts.139.gate_proj", + "model.layers.3.mlp.experts.139.up_proj", + "model.layers.3.mlp.experts.14.down_proj", + "model.layers.3.mlp.experts.14.gate_proj", + "model.layers.3.mlp.experts.14.up_proj", + "model.layers.3.mlp.experts.140.down_proj", + "model.layers.3.mlp.experts.140.gate_proj", + "model.layers.3.mlp.experts.140.up_proj", + "model.layers.3.mlp.experts.141.down_proj", + "model.layers.3.mlp.experts.141.gate_proj", + "model.layers.3.mlp.experts.141.up_proj", + "model.layers.3.mlp.experts.142.down_proj", + "model.layers.3.mlp.experts.142.gate_proj", + "model.layers.3.mlp.experts.142.up_proj", + "model.layers.3.mlp.experts.143.down_proj", + "model.layers.3.mlp.experts.143.gate_proj", + "model.layers.3.mlp.experts.143.up_proj", + "model.layers.3.mlp.experts.144.down_proj", + "model.layers.3.mlp.experts.144.gate_proj", + "model.layers.3.mlp.experts.144.up_proj", + "model.layers.3.mlp.experts.145.down_proj", + "model.layers.3.mlp.experts.145.gate_proj", + "model.layers.3.mlp.experts.145.up_proj", + "model.layers.3.mlp.experts.146.down_proj", + "model.layers.3.mlp.experts.146.gate_proj", + "model.layers.3.mlp.experts.146.up_proj", + "model.layers.3.mlp.experts.147.down_proj", + "model.layers.3.mlp.experts.147.gate_proj", + "model.layers.3.mlp.experts.147.up_proj", + "model.layers.3.mlp.experts.148.down_proj", + "model.layers.3.mlp.experts.148.gate_proj", + "model.layers.3.mlp.experts.148.up_proj", + "model.layers.3.mlp.experts.149.down_proj", + "model.layers.3.mlp.experts.149.gate_proj", + "model.layers.3.mlp.experts.149.up_proj", + "model.layers.3.mlp.experts.15.down_proj", + "model.layers.3.mlp.experts.15.gate_proj", + "model.layers.3.mlp.experts.15.up_proj", + "model.layers.3.mlp.experts.150.down_proj", + "model.layers.3.mlp.experts.150.gate_proj", + "model.layers.3.mlp.experts.150.up_proj", + "model.layers.3.mlp.experts.151.down_proj", + "model.layers.3.mlp.experts.151.gate_proj", + "model.layers.3.mlp.experts.151.up_proj", + "model.layers.3.mlp.experts.152.down_proj", + "model.layers.3.mlp.experts.152.gate_proj", + "model.layers.3.mlp.experts.152.up_proj", + "model.layers.3.mlp.experts.153.down_proj", + "model.layers.3.mlp.experts.153.gate_proj", + "model.layers.3.mlp.experts.153.up_proj", + "model.layers.3.mlp.experts.154.down_proj", + "model.layers.3.mlp.experts.154.gate_proj", + "model.layers.3.mlp.experts.154.up_proj", + "model.layers.3.mlp.experts.155.down_proj", + "model.layers.3.mlp.experts.155.gate_proj", + "model.layers.3.mlp.experts.155.up_proj", + "model.layers.3.mlp.experts.156.down_proj", + "model.layers.3.mlp.experts.156.gate_proj", + "model.layers.3.mlp.experts.156.up_proj", + "model.layers.3.mlp.experts.157.down_proj", + "model.layers.3.mlp.experts.157.gate_proj", + "model.layers.3.mlp.experts.157.up_proj", + "model.layers.3.mlp.experts.158.down_proj", + "model.layers.3.mlp.experts.158.gate_proj", + "model.layers.3.mlp.experts.158.up_proj", + "model.layers.3.mlp.experts.159.down_proj", + "model.layers.3.mlp.experts.159.gate_proj", + "model.layers.3.mlp.experts.159.up_proj", + "model.layers.3.mlp.experts.16.down_proj", + "model.layers.3.mlp.experts.16.gate_proj", + "model.layers.3.mlp.experts.16.up_proj", + "model.layers.3.mlp.experts.160.down_proj", + "model.layers.3.mlp.experts.160.gate_proj", + "model.layers.3.mlp.experts.160.up_proj", + "model.layers.3.mlp.experts.161.down_proj", + "model.layers.3.mlp.experts.161.gate_proj", + "model.layers.3.mlp.experts.161.up_proj", + "model.layers.3.mlp.experts.162.down_proj", + "model.layers.3.mlp.experts.162.gate_proj", + "model.layers.3.mlp.experts.162.up_proj", + "model.layers.3.mlp.experts.163.down_proj", + "model.layers.3.mlp.experts.163.gate_proj", + "model.layers.3.mlp.experts.163.up_proj", + "model.layers.3.mlp.experts.164.down_proj", + "model.layers.3.mlp.experts.164.gate_proj", + "model.layers.3.mlp.experts.164.up_proj", + "model.layers.3.mlp.experts.165.down_proj", + "model.layers.3.mlp.experts.165.gate_proj", + "model.layers.3.mlp.experts.165.up_proj", + "model.layers.3.mlp.experts.166.down_proj", + "model.layers.3.mlp.experts.166.gate_proj", + "model.layers.3.mlp.experts.166.up_proj", + "model.layers.3.mlp.experts.167.down_proj", + "model.layers.3.mlp.experts.167.gate_proj", + "model.layers.3.mlp.experts.167.up_proj", + "model.layers.3.mlp.experts.168.down_proj", + "model.layers.3.mlp.experts.168.gate_proj", + "model.layers.3.mlp.experts.168.up_proj", + "model.layers.3.mlp.experts.169.down_proj", + "model.layers.3.mlp.experts.169.gate_proj", + "model.layers.3.mlp.experts.169.up_proj", + "model.layers.3.mlp.experts.17.down_proj", + "model.layers.3.mlp.experts.17.gate_proj", + "model.layers.3.mlp.experts.17.up_proj", + "model.layers.3.mlp.experts.170.down_proj", + "model.layers.3.mlp.experts.170.gate_proj", + "model.layers.3.mlp.experts.170.up_proj", + "model.layers.3.mlp.experts.171.down_proj", + "model.layers.3.mlp.experts.171.gate_proj", + "model.layers.3.mlp.experts.171.up_proj", + "model.layers.3.mlp.experts.172.down_proj", + "model.layers.3.mlp.experts.172.gate_proj", + "model.layers.3.mlp.experts.172.up_proj", + "model.layers.3.mlp.experts.173.down_proj", + "model.layers.3.mlp.experts.173.gate_proj", + "model.layers.3.mlp.experts.173.up_proj", + "model.layers.3.mlp.experts.174.down_proj", + "model.layers.3.mlp.experts.174.gate_proj", + "model.layers.3.mlp.experts.174.up_proj", + "model.layers.3.mlp.experts.175.down_proj", + "model.layers.3.mlp.experts.175.gate_proj", + "model.layers.3.mlp.experts.175.up_proj", + "model.layers.3.mlp.experts.176.down_proj", + "model.layers.3.mlp.experts.176.gate_proj", + "model.layers.3.mlp.experts.176.up_proj", + "model.layers.3.mlp.experts.177.down_proj", + "model.layers.3.mlp.experts.177.gate_proj", + "model.layers.3.mlp.experts.177.up_proj", + "model.layers.3.mlp.experts.178.down_proj", + "model.layers.3.mlp.experts.178.gate_proj", + "model.layers.3.mlp.experts.178.up_proj", + "model.layers.3.mlp.experts.179.down_proj", + "model.layers.3.mlp.experts.179.gate_proj", + "model.layers.3.mlp.experts.179.up_proj", + "model.layers.3.mlp.experts.18.down_proj", + "model.layers.3.mlp.experts.18.gate_proj", + "model.layers.3.mlp.experts.18.up_proj", + "model.layers.3.mlp.experts.180.down_proj", + "model.layers.3.mlp.experts.180.gate_proj", + "model.layers.3.mlp.experts.180.up_proj", + "model.layers.3.mlp.experts.181.down_proj", + "model.layers.3.mlp.experts.181.gate_proj", + "model.layers.3.mlp.experts.181.up_proj", + "model.layers.3.mlp.experts.182.down_proj", + "model.layers.3.mlp.experts.182.gate_proj", + "model.layers.3.mlp.experts.182.up_proj", + "model.layers.3.mlp.experts.183.down_proj", + "model.layers.3.mlp.experts.183.gate_proj", + "model.layers.3.mlp.experts.183.up_proj", + "model.layers.3.mlp.experts.184.down_proj", + "model.layers.3.mlp.experts.184.gate_proj", + "model.layers.3.mlp.experts.184.up_proj", + "model.layers.3.mlp.experts.185.down_proj", + "model.layers.3.mlp.experts.185.gate_proj", + "model.layers.3.mlp.experts.185.up_proj", + "model.layers.3.mlp.experts.186.down_proj", + "model.layers.3.mlp.experts.186.gate_proj", + "model.layers.3.mlp.experts.186.up_proj", + "model.layers.3.mlp.experts.187.down_proj", + "model.layers.3.mlp.experts.187.gate_proj", + "model.layers.3.mlp.experts.187.up_proj", + "model.layers.3.mlp.experts.188.down_proj", + "model.layers.3.mlp.experts.188.gate_proj", + "model.layers.3.mlp.experts.188.up_proj", + "model.layers.3.mlp.experts.189.down_proj", + "model.layers.3.mlp.experts.189.gate_proj", + "model.layers.3.mlp.experts.189.up_proj", + "model.layers.3.mlp.experts.19.down_proj", + "model.layers.3.mlp.experts.19.gate_proj", + "model.layers.3.mlp.experts.19.up_proj", + "model.layers.3.mlp.experts.190.down_proj", + "model.layers.3.mlp.experts.190.gate_proj", + "model.layers.3.mlp.experts.190.up_proj", + "model.layers.3.mlp.experts.191.down_proj", + "model.layers.3.mlp.experts.191.gate_proj", + "model.layers.3.mlp.experts.191.up_proj", + "model.layers.3.mlp.experts.192.down_proj", + "model.layers.3.mlp.experts.192.gate_proj", + "model.layers.3.mlp.experts.192.up_proj", + "model.layers.3.mlp.experts.193.down_proj", + "model.layers.3.mlp.experts.193.gate_proj", + "model.layers.3.mlp.experts.193.up_proj", + "model.layers.3.mlp.experts.194.down_proj", + "model.layers.3.mlp.experts.194.gate_proj", + "model.layers.3.mlp.experts.194.up_proj", + "model.layers.3.mlp.experts.195.down_proj", + "model.layers.3.mlp.experts.195.gate_proj", + "model.layers.3.mlp.experts.195.up_proj", + "model.layers.3.mlp.experts.196.down_proj", + "model.layers.3.mlp.experts.196.gate_proj", + "model.layers.3.mlp.experts.196.up_proj", + "model.layers.3.mlp.experts.197.down_proj", + "model.layers.3.mlp.experts.197.gate_proj", + "model.layers.3.mlp.experts.197.up_proj", + "model.layers.3.mlp.experts.198.down_proj", + "model.layers.3.mlp.experts.198.gate_proj", + "model.layers.3.mlp.experts.198.up_proj", + "model.layers.3.mlp.experts.199.down_proj", + "model.layers.3.mlp.experts.199.gate_proj", + "model.layers.3.mlp.experts.199.up_proj", + "model.layers.3.mlp.experts.2.down_proj", + "model.layers.3.mlp.experts.2.gate_proj", + "model.layers.3.mlp.experts.2.up_proj", + "model.layers.3.mlp.experts.20.down_proj", + "model.layers.3.mlp.experts.20.gate_proj", + "model.layers.3.mlp.experts.20.up_proj", + "model.layers.3.mlp.experts.200.down_proj", + "model.layers.3.mlp.experts.200.gate_proj", + "model.layers.3.mlp.experts.200.up_proj", + "model.layers.3.mlp.experts.201.down_proj", + "model.layers.3.mlp.experts.201.gate_proj", + "model.layers.3.mlp.experts.201.up_proj", + "model.layers.3.mlp.experts.202.down_proj", + "model.layers.3.mlp.experts.202.gate_proj", + "model.layers.3.mlp.experts.202.up_proj", + "model.layers.3.mlp.experts.203.down_proj", + "model.layers.3.mlp.experts.203.gate_proj", + "model.layers.3.mlp.experts.203.up_proj", + "model.layers.3.mlp.experts.204.down_proj", + "model.layers.3.mlp.experts.204.gate_proj", + "model.layers.3.mlp.experts.204.up_proj", + "model.layers.3.mlp.experts.205.down_proj", + "model.layers.3.mlp.experts.205.gate_proj", + "model.layers.3.mlp.experts.205.up_proj", + "model.layers.3.mlp.experts.206.down_proj", + "model.layers.3.mlp.experts.206.gate_proj", + "model.layers.3.mlp.experts.206.up_proj", + "model.layers.3.mlp.experts.207.down_proj", + "model.layers.3.mlp.experts.207.gate_proj", + "model.layers.3.mlp.experts.207.up_proj", + "model.layers.3.mlp.experts.208.down_proj", + "model.layers.3.mlp.experts.208.gate_proj", + "model.layers.3.mlp.experts.208.up_proj", + "model.layers.3.mlp.experts.209.down_proj", + "model.layers.3.mlp.experts.209.gate_proj", + "model.layers.3.mlp.experts.209.up_proj", + "model.layers.3.mlp.experts.21.down_proj", + "model.layers.3.mlp.experts.21.gate_proj", + "model.layers.3.mlp.experts.21.up_proj", + "model.layers.3.mlp.experts.210.down_proj", + "model.layers.3.mlp.experts.210.gate_proj", + "model.layers.3.mlp.experts.210.up_proj", + "model.layers.3.mlp.experts.211.down_proj", + "model.layers.3.mlp.experts.211.gate_proj", + "model.layers.3.mlp.experts.211.up_proj", + "model.layers.3.mlp.experts.212.down_proj", + "model.layers.3.mlp.experts.212.gate_proj", + "model.layers.3.mlp.experts.212.up_proj", + "model.layers.3.mlp.experts.213.down_proj", + "model.layers.3.mlp.experts.213.gate_proj", + "model.layers.3.mlp.experts.213.up_proj", + "model.layers.3.mlp.experts.214.down_proj", + "model.layers.3.mlp.experts.214.gate_proj", + "model.layers.3.mlp.experts.214.up_proj", + "model.layers.3.mlp.experts.215.down_proj", + "model.layers.3.mlp.experts.215.gate_proj", + "model.layers.3.mlp.experts.215.up_proj", + "model.layers.3.mlp.experts.216.down_proj", + "model.layers.3.mlp.experts.216.gate_proj", + "model.layers.3.mlp.experts.216.up_proj", + "model.layers.3.mlp.experts.217.down_proj", + "model.layers.3.mlp.experts.217.gate_proj", + "model.layers.3.mlp.experts.217.up_proj", + "model.layers.3.mlp.experts.218.down_proj", + "model.layers.3.mlp.experts.218.gate_proj", + "model.layers.3.mlp.experts.218.up_proj", + "model.layers.3.mlp.experts.219.down_proj", + "model.layers.3.mlp.experts.219.gate_proj", + "model.layers.3.mlp.experts.219.up_proj", + "model.layers.3.mlp.experts.22.down_proj", + "model.layers.3.mlp.experts.22.gate_proj", + "model.layers.3.mlp.experts.22.up_proj", + "model.layers.3.mlp.experts.220.down_proj", + "model.layers.3.mlp.experts.220.gate_proj", + "model.layers.3.mlp.experts.220.up_proj", + "model.layers.3.mlp.experts.221.down_proj", + "model.layers.3.mlp.experts.221.gate_proj", + "model.layers.3.mlp.experts.221.up_proj", + "model.layers.3.mlp.experts.222.down_proj", + "model.layers.3.mlp.experts.222.gate_proj", + "model.layers.3.mlp.experts.222.up_proj", + "model.layers.3.mlp.experts.223.down_proj", + "model.layers.3.mlp.experts.223.gate_proj", + "model.layers.3.mlp.experts.223.up_proj", + "model.layers.3.mlp.experts.224.down_proj", + "model.layers.3.mlp.experts.224.gate_proj", + "model.layers.3.mlp.experts.224.up_proj", + "model.layers.3.mlp.experts.225.down_proj", + "model.layers.3.mlp.experts.225.gate_proj", + "model.layers.3.mlp.experts.225.up_proj", + "model.layers.3.mlp.experts.226.down_proj", + "model.layers.3.mlp.experts.226.gate_proj", + "model.layers.3.mlp.experts.226.up_proj", + "model.layers.3.mlp.experts.227.down_proj", + "model.layers.3.mlp.experts.227.gate_proj", + "model.layers.3.mlp.experts.227.up_proj", + "model.layers.3.mlp.experts.228.down_proj", + "model.layers.3.mlp.experts.228.gate_proj", + "model.layers.3.mlp.experts.228.up_proj", + "model.layers.3.mlp.experts.229.down_proj", + "model.layers.3.mlp.experts.229.gate_proj", + "model.layers.3.mlp.experts.229.up_proj", + "model.layers.3.mlp.experts.23.down_proj", + "model.layers.3.mlp.experts.23.gate_proj", + "model.layers.3.mlp.experts.23.up_proj", + "model.layers.3.mlp.experts.230.down_proj", + "model.layers.3.mlp.experts.230.gate_proj", + "model.layers.3.mlp.experts.230.up_proj", + "model.layers.3.mlp.experts.231.down_proj", + "model.layers.3.mlp.experts.231.gate_proj", + "model.layers.3.mlp.experts.231.up_proj", + "model.layers.3.mlp.experts.232.down_proj", + "model.layers.3.mlp.experts.232.gate_proj", + "model.layers.3.mlp.experts.232.up_proj", + "model.layers.3.mlp.experts.233.down_proj", + "model.layers.3.mlp.experts.233.gate_proj", + "model.layers.3.mlp.experts.233.up_proj", + "model.layers.3.mlp.experts.234.down_proj", + "model.layers.3.mlp.experts.234.gate_proj", + "model.layers.3.mlp.experts.234.up_proj", + "model.layers.3.mlp.experts.235.down_proj", + "model.layers.3.mlp.experts.235.gate_proj", + "model.layers.3.mlp.experts.235.up_proj", + "model.layers.3.mlp.experts.236.down_proj", + "model.layers.3.mlp.experts.236.gate_proj", + "model.layers.3.mlp.experts.236.up_proj", + "model.layers.3.mlp.experts.237.down_proj", + "model.layers.3.mlp.experts.237.gate_proj", + "model.layers.3.mlp.experts.237.up_proj", + "model.layers.3.mlp.experts.238.down_proj", + "model.layers.3.mlp.experts.238.gate_proj", + "model.layers.3.mlp.experts.238.up_proj", + "model.layers.3.mlp.experts.239.down_proj", + "model.layers.3.mlp.experts.239.gate_proj", + "model.layers.3.mlp.experts.239.up_proj", + "model.layers.3.mlp.experts.24.down_proj", + "model.layers.3.mlp.experts.24.gate_proj", + "model.layers.3.mlp.experts.24.up_proj", + "model.layers.3.mlp.experts.240.down_proj", + "model.layers.3.mlp.experts.240.gate_proj", + "model.layers.3.mlp.experts.240.up_proj", + "model.layers.3.mlp.experts.241.down_proj", + "model.layers.3.mlp.experts.241.gate_proj", + "model.layers.3.mlp.experts.241.up_proj", + "model.layers.3.mlp.experts.242.down_proj", + "model.layers.3.mlp.experts.242.gate_proj", + "model.layers.3.mlp.experts.242.up_proj", + "model.layers.3.mlp.experts.243.down_proj", + "model.layers.3.mlp.experts.243.gate_proj", + "model.layers.3.mlp.experts.243.up_proj", + "model.layers.3.mlp.experts.244.down_proj", + "model.layers.3.mlp.experts.244.gate_proj", + "model.layers.3.mlp.experts.244.up_proj", + "model.layers.3.mlp.experts.245.down_proj", + "model.layers.3.mlp.experts.245.gate_proj", + "model.layers.3.mlp.experts.245.up_proj", + "model.layers.3.mlp.experts.246.down_proj", + "model.layers.3.mlp.experts.246.gate_proj", + "model.layers.3.mlp.experts.246.up_proj", + "model.layers.3.mlp.experts.247.down_proj", + "model.layers.3.mlp.experts.247.gate_proj", + "model.layers.3.mlp.experts.247.up_proj", + "model.layers.3.mlp.experts.248.down_proj", + "model.layers.3.mlp.experts.248.gate_proj", + "model.layers.3.mlp.experts.248.up_proj", + "model.layers.3.mlp.experts.249.down_proj", + "model.layers.3.mlp.experts.249.gate_proj", + "model.layers.3.mlp.experts.249.up_proj", + "model.layers.3.mlp.experts.25.down_proj", + "model.layers.3.mlp.experts.25.gate_proj", + "model.layers.3.mlp.experts.25.up_proj", + "model.layers.3.mlp.experts.250.down_proj", + "model.layers.3.mlp.experts.250.gate_proj", + "model.layers.3.mlp.experts.250.up_proj", + "model.layers.3.mlp.experts.251.down_proj", + "model.layers.3.mlp.experts.251.gate_proj", + "model.layers.3.mlp.experts.251.up_proj", + "model.layers.3.mlp.experts.252.down_proj", + "model.layers.3.mlp.experts.252.gate_proj", + "model.layers.3.mlp.experts.252.up_proj", + "model.layers.3.mlp.experts.253.down_proj", + "model.layers.3.mlp.experts.253.gate_proj", + "model.layers.3.mlp.experts.253.up_proj", + "model.layers.3.mlp.experts.254.down_proj", + "model.layers.3.mlp.experts.254.gate_proj", + "model.layers.3.mlp.experts.254.up_proj", + "model.layers.3.mlp.experts.255.down_proj", + "model.layers.3.mlp.experts.255.gate_proj", + "model.layers.3.mlp.experts.255.up_proj", + "model.layers.3.mlp.experts.26.down_proj", + "model.layers.3.mlp.experts.26.gate_proj", + "model.layers.3.mlp.experts.26.up_proj", + "model.layers.3.mlp.experts.27.down_proj", + "model.layers.3.mlp.experts.27.gate_proj", + "model.layers.3.mlp.experts.27.up_proj", + "model.layers.3.mlp.experts.28.down_proj", + "model.layers.3.mlp.experts.28.gate_proj", + "model.layers.3.mlp.experts.28.up_proj", + "model.layers.3.mlp.experts.29.down_proj", + "model.layers.3.mlp.experts.29.gate_proj", + "model.layers.3.mlp.experts.29.up_proj", + "model.layers.3.mlp.experts.3.down_proj", + "model.layers.3.mlp.experts.3.gate_proj", + "model.layers.3.mlp.experts.3.up_proj", + "model.layers.3.mlp.experts.30.down_proj", + "model.layers.3.mlp.experts.30.gate_proj", + "model.layers.3.mlp.experts.30.up_proj", + "model.layers.3.mlp.experts.31.down_proj", + "model.layers.3.mlp.experts.31.gate_proj", + "model.layers.3.mlp.experts.31.up_proj", + "model.layers.3.mlp.experts.32.down_proj", + "model.layers.3.mlp.experts.32.gate_proj", + "model.layers.3.mlp.experts.32.up_proj", + "model.layers.3.mlp.experts.33.down_proj", + "model.layers.3.mlp.experts.33.gate_proj", + "model.layers.3.mlp.experts.33.up_proj", + "model.layers.3.mlp.experts.34.down_proj", + "model.layers.3.mlp.experts.34.gate_proj", + "model.layers.3.mlp.experts.34.up_proj", + "model.layers.3.mlp.experts.35.down_proj", + "model.layers.3.mlp.experts.35.gate_proj", + "model.layers.3.mlp.experts.35.up_proj", + "model.layers.3.mlp.experts.36.down_proj", + "model.layers.3.mlp.experts.36.gate_proj", + "model.layers.3.mlp.experts.36.up_proj", + "model.layers.3.mlp.experts.37.down_proj", + "model.layers.3.mlp.experts.37.gate_proj", + "model.layers.3.mlp.experts.37.up_proj", + "model.layers.3.mlp.experts.38.down_proj", + "model.layers.3.mlp.experts.38.gate_proj", + "model.layers.3.mlp.experts.38.up_proj", + "model.layers.3.mlp.experts.39.down_proj", + "model.layers.3.mlp.experts.39.gate_proj", + "model.layers.3.mlp.experts.39.up_proj", + "model.layers.3.mlp.experts.4.down_proj", + "model.layers.3.mlp.experts.4.gate_proj", + "model.layers.3.mlp.experts.4.up_proj", + "model.layers.3.mlp.experts.40.down_proj", + "model.layers.3.mlp.experts.40.gate_proj", + "model.layers.3.mlp.experts.40.up_proj", + "model.layers.3.mlp.experts.41.down_proj", + "model.layers.3.mlp.experts.41.gate_proj", + "model.layers.3.mlp.experts.41.up_proj", + "model.layers.3.mlp.experts.42.down_proj", + "model.layers.3.mlp.experts.42.gate_proj", + "model.layers.3.mlp.experts.42.up_proj", + "model.layers.3.mlp.experts.43.down_proj", + "model.layers.3.mlp.experts.43.gate_proj", + "model.layers.3.mlp.experts.43.up_proj", + "model.layers.3.mlp.experts.44.down_proj", + "model.layers.3.mlp.experts.44.gate_proj", + "model.layers.3.mlp.experts.44.up_proj", + "model.layers.3.mlp.experts.45.down_proj", + "model.layers.3.mlp.experts.45.gate_proj", + "model.layers.3.mlp.experts.45.up_proj", + "model.layers.3.mlp.experts.46.down_proj", + "model.layers.3.mlp.experts.46.gate_proj", + "model.layers.3.mlp.experts.46.up_proj", + "model.layers.3.mlp.experts.47.down_proj", + "model.layers.3.mlp.experts.47.gate_proj", + "model.layers.3.mlp.experts.47.up_proj", + "model.layers.3.mlp.experts.48.down_proj", + "model.layers.3.mlp.experts.48.gate_proj", + "model.layers.3.mlp.experts.48.up_proj", + "model.layers.3.mlp.experts.49.down_proj", + "model.layers.3.mlp.experts.49.gate_proj", + "model.layers.3.mlp.experts.49.up_proj", + "model.layers.3.mlp.experts.5.down_proj", + "model.layers.3.mlp.experts.5.gate_proj", + "model.layers.3.mlp.experts.5.up_proj", + "model.layers.3.mlp.experts.50.down_proj", + "model.layers.3.mlp.experts.50.gate_proj", + "model.layers.3.mlp.experts.50.up_proj", + "model.layers.3.mlp.experts.51.down_proj", + "model.layers.3.mlp.experts.51.gate_proj", + "model.layers.3.mlp.experts.51.up_proj", + "model.layers.3.mlp.experts.52.down_proj", + "model.layers.3.mlp.experts.52.gate_proj", + "model.layers.3.mlp.experts.52.up_proj", + "model.layers.3.mlp.experts.53.down_proj", + "model.layers.3.mlp.experts.53.gate_proj", + "model.layers.3.mlp.experts.53.up_proj", + "model.layers.3.mlp.experts.54.down_proj", + "model.layers.3.mlp.experts.54.gate_proj", + "model.layers.3.mlp.experts.54.up_proj", + "model.layers.3.mlp.experts.55.down_proj", + "model.layers.3.mlp.experts.55.gate_proj", + "model.layers.3.mlp.experts.55.up_proj", + "model.layers.3.mlp.experts.56.down_proj", + "model.layers.3.mlp.experts.56.gate_proj", + "model.layers.3.mlp.experts.56.up_proj", + "model.layers.3.mlp.experts.57.down_proj", + "model.layers.3.mlp.experts.57.gate_proj", + "model.layers.3.mlp.experts.57.up_proj", + "model.layers.3.mlp.experts.58.down_proj", + "model.layers.3.mlp.experts.58.gate_proj", + "model.layers.3.mlp.experts.58.up_proj", + "model.layers.3.mlp.experts.59.down_proj", + "model.layers.3.mlp.experts.59.gate_proj", + "model.layers.3.mlp.experts.59.up_proj", + "model.layers.3.mlp.experts.6.down_proj", + "model.layers.3.mlp.experts.6.gate_proj", + "model.layers.3.mlp.experts.6.up_proj", + "model.layers.3.mlp.experts.60.down_proj", + "model.layers.3.mlp.experts.60.gate_proj", + "model.layers.3.mlp.experts.60.up_proj", + "model.layers.3.mlp.experts.61.down_proj", + "model.layers.3.mlp.experts.61.gate_proj", + "model.layers.3.mlp.experts.61.up_proj", + "model.layers.3.mlp.experts.62.down_proj", + "model.layers.3.mlp.experts.62.gate_proj", + "model.layers.3.mlp.experts.62.up_proj", + "model.layers.3.mlp.experts.63.down_proj", + "model.layers.3.mlp.experts.63.gate_proj", + "model.layers.3.mlp.experts.63.up_proj", + "model.layers.3.mlp.experts.64.down_proj", + "model.layers.3.mlp.experts.64.gate_proj", + "model.layers.3.mlp.experts.64.up_proj", + "model.layers.3.mlp.experts.65.down_proj", + "model.layers.3.mlp.experts.65.gate_proj", + "model.layers.3.mlp.experts.65.up_proj", + "model.layers.3.mlp.experts.66.down_proj", + "model.layers.3.mlp.experts.66.gate_proj", + "model.layers.3.mlp.experts.66.up_proj", + "model.layers.3.mlp.experts.67.down_proj", + "model.layers.3.mlp.experts.67.gate_proj", + "model.layers.3.mlp.experts.67.up_proj", + "model.layers.3.mlp.experts.68.down_proj", + "model.layers.3.mlp.experts.68.gate_proj", + "model.layers.3.mlp.experts.68.up_proj", + "model.layers.3.mlp.experts.69.down_proj", + "model.layers.3.mlp.experts.69.gate_proj", + "model.layers.3.mlp.experts.69.up_proj", + "model.layers.3.mlp.experts.7.down_proj", + "model.layers.3.mlp.experts.7.gate_proj", + "model.layers.3.mlp.experts.7.up_proj", + "model.layers.3.mlp.experts.70.down_proj", + "model.layers.3.mlp.experts.70.gate_proj", + "model.layers.3.mlp.experts.70.up_proj", + "model.layers.3.mlp.experts.71.down_proj", + "model.layers.3.mlp.experts.71.gate_proj", + "model.layers.3.mlp.experts.71.up_proj", + "model.layers.3.mlp.experts.72.down_proj", + "model.layers.3.mlp.experts.72.gate_proj", + "model.layers.3.mlp.experts.72.up_proj", + "model.layers.3.mlp.experts.73.down_proj", + "model.layers.3.mlp.experts.73.gate_proj", + "model.layers.3.mlp.experts.73.up_proj", + "model.layers.3.mlp.experts.74.down_proj", + "model.layers.3.mlp.experts.74.gate_proj", + "model.layers.3.mlp.experts.74.up_proj", + "model.layers.3.mlp.experts.75.down_proj", + "model.layers.3.mlp.experts.75.gate_proj", + "model.layers.3.mlp.experts.75.up_proj", + "model.layers.3.mlp.experts.76.down_proj", + "model.layers.3.mlp.experts.76.gate_proj", + "model.layers.3.mlp.experts.76.up_proj", + "model.layers.3.mlp.experts.77.down_proj", + "model.layers.3.mlp.experts.77.gate_proj", + "model.layers.3.mlp.experts.77.up_proj", + "model.layers.3.mlp.experts.78.down_proj", + "model.layers.3.mlp.experts.78.gate_proj", + "model.layers.3.mlp.experts.78.up_proj", + "model.layers.3.mlp.experts.79.down_proj", + "model.layers.3.mlp.experts.79.gate_proj", + "model.layers.3.mlp.experts.79.up_proj", + "model.layers.3.mlp.experts.8.down_proj", + "model.layers.3.mlp.experts.8.gate_proj", + "model.layers.3.mlp.experts.8.up_proj", + "model.layers.3.mlp.experts.80.down_proj", + "model.layers.3.mlp.experts.80.gate_proj", + "model.layers.3.mlp.experts.80.up_proj", + "model.layers.3.mlp.experts.81.down_proj", + "model.layers.3.mlp.experts.81.gate_proj", + "model.layers.3.mlp.experts.81.up_proj", + "model.layers.3.mlp.experts.82.down_proj", + "model.layers.3.mlp.experts.82.gate_proj", + "model.layers.3.mlp.experts.82.up_proj", + "model.layers.3.mlp.experts.83.down_proj", + "model.layers.3.mlp.experts.83.gate_proj", + "model.layers.3.mlp.experts.83.up_proj", + "model.layers.3.mlp.experts.84.down_proj", + "model.layers.3.mlp.experts.84.gate_proj", + "model.layers.3.mlp.experts.84.up_proj", + "model.layers.3.mlp.experts.85.down_proj", + "model.layers.3.mlp.experts.85.gate_proj", + "model.layers.3.mlp.experts.85.up_proj", + "model.layers.3.mlp.experts.86.down_proj", + "model.layers.3.mlp.experts.86.gate_proj", + "model.layers.3.mlp.experts.86.up_proj", + "model.layers.3.mlp.experts.87.down_proj", + "model.layers.3.mlp.experts.87.gate_proj", + "model.layers.3.mlp.experts.87.up_proj", + "model.layers.3.mlp.experts.88.down_proj", + "model.layers.3.mlp.experts.88.gate_proj", + "model.layers.3.mlp.experts.88.up_proj", + "model.layers.3.mlp.experts.89.down_proj", + "model.layers.3.mlp.experts.89.gate_proj", + "model.layers.3.mlp.experts.89.up_proj", + "model.layers.3.mlp.experts.9.down_proj", + "model.layers.3.mlp.experts.9.gate_proj", + "model.layers.3.mlp.experts.9.up_proj", + "model.layers.3.mlp.experts.90.down_proj", + "model.layers.3.mlp.experts.90.gate_proj", + "model.layers.3.mlp.experts.90.up_proj", + "model.layers.3.mlp.experts.91.down_proj", + "model.layers.3.mlp.experts.91.gate_proj", + "model.layers.3.mlp.experts.91.up_proj", + "model.layers.3.mlp.experts.92.down_proj", + "model.layers.3.mlp.experts.92.gate_proj", + "model.layers.3.mlp.experts.92.up_proj", + "model.layers.3.mlp.experts.93.down_proj", + "model.layers.3.mlp.experts.93.gate_proj", + "model.layers.3.mlp.experts.93.up_proj", + "model.layers.3.mlp.experts.94.down_proj", + "model.layers.3.mlp.experts.94.gate_proj", + "model.layers.3.mlp.experts.94.up_proj", + "model.layers.3.mlp.experts.95.down_proj", + "model.layers.3.mlp.experts.95.gate_proj", + "model.layers.3.mlp.experts.95.up_proj", + "model.layers.3.mlp.experts.96.down_proj", + "model.layers.3.mlp.experts.96.gate_proj", + "model.layers.3.mlp.experts.96.up_proj", + "model.layers.3.mlp.experts.97.down_proj", + "model.layers.3.mlp.experts.97.gate_proj", + "model.layers.3.mlp.experts.97.up_proj", + "model.layers.3.mlp.experts.98.down_proj", + "model.layers.3.mlp.experts.98.gate_proj", + "model.layers.3.mlp.experts.98.up_proj", + "model.layers.3.mlp.experts.99.down_proj", + "model.layers.3.mlp.experts.99.gate_proj", + "model.layers.3.mlp.experts.99.up_proj", + "model.layers.30.mlp.experts.0.down_proj", + "model.layers.30.mlp.experts.0.gate_proj", + "model.layers.30.mlp.experts.0.up_proj", + "model.layers.30.mlp.experts.1.down_proj", + "model.layers.30.mlp.experts.1.gate_proj", + "model.layers.30.mlp.experts.1.up_proj", + "model.layers.30.mlp.experts.10.down_proj", + "model.layers.30.mlp.experts.10.gate_proj", + "model.layers.30.mlp.experts.10.up_proj", + "model.layers.30.mlp.experts.100.down_proj", + "model.layers.30.mlp.experts.100.gate_proj", + "model.layers.30.mlp.experts.100.up_proj", + "model.layers.30.mlp.experts.101.down_proj", + "model.layers.30.mlp.experts.101.gate_proj", + "model.layers.30.mlp.experts.101.up_proj", + "model.layers.30.mlp.experts.102.down_proj", + "model.layers.30.mlp.experts.102.gate_proj", + "model.layers.30.mlp.experts.102.up_proj", + "model.layers.30.mlp.experts.103.down_proj", + "model.layers.30.mlp.experts.103.gate_proj", + "model.layers.30.mlp.experts.103.up_proj", + "model.layers.30.mlp.experts.104.down_proj", + "model.layers.30.mlp.experts.104.gate_proj", + "model.layers.30.mlp.experts.104.up_proj", + "model.layers.30.mlp.experts.105.down_proj", + "model.layers.30.mlp.experts.105.gate_proj", + "model.layers.30.mlp.experts.105.up_proj", + "model.layers.30.mlp.experts.106.down_proj", + "model.layers.30.mlp.experts.106.gate_proj", + "model.layers.30.mlp.experts.106.up_proj", + "model.layers.30.mlp.experts.107.down_proj", + "model.layers.30.mlp.experts.107.gate_proj", + "model.layers.30.mlp.experts.107.up_proj", + "model.layers.30.mlp.experts.108.down_proj", + "model.layers.30.mlp.experts.108.gate_proj", + "model.layers.30.mlp.experts.108.up_proj", + "model.layers.30.mlp.experts.109.down_proj", + "model.layers.30.mlp.experts.109.gate_proj", + "model.layers.30.mlp.experts.109.up_proj", + "model.layers.30.mlp.experts.11.down_proj", + "model.layers.30.mlp.experts.11.gate_proj", + "model.layers.30.mlp.experts.11.up_proj", + "model.layers.30.mlp.experts.110.down_proj", + "model.layers.30.mlp.experts.110.gate_proj", + "model.layers.30.mlp.experts.110.up_proj", + "model.layers.30.mlp.experts.111.down_proj", + "model.layers.30.mlp.experts.111.gate_proj", + "model.layers.30.mlp.experts.111.up_proj", + "model.layers.30.mlp.experts.112.down_proj", + "model.layers.30.mlp.experts.112.gate_proj", + "model.layers.30.mlp.experts.112.up_proj", + "model.layers.30.mlp.experts.113.down_proj", + "model.layers.30.mlp.experts.113.gate_proj", + "model.layers.30.mlp.experts.113.up_proj", + "model.layers.30.mlp.experts.114.down_proj", + "model.layers.30.mlp.experts.114.gate_proj", + "model.layers.30.mlp.experts.114.up_proj", + "model.layers.30.mlp.experts.115.down_proj", + "model.layers.30.mlp.experts.115.gate_proj", + "model.layers.30.mlp.experts.115.up_proj", + "model.layers.30.mlp.experts.116.down_proj", + "model.layers.30.mlp.experts.116.gate_proj", + "model.layers.30.mlp.experts.116.up_proj", + "model.layers.30.mlp.experts.117.down_proj", + "model.layers.30.mlp.experts.117.gate_proj", + "model.layers.30.mlp.experts.117.up_proj", + "model.layers.30.mlp.experts.118.down_proj", + "model.layers.30.mlp.experts.118.gate_proj", + "model.layers.30.mlp.experts.118.up_proj", + "model.layers.30.mlp.experts.119.down_proj", + "model.layers.30.mlp.experts.119.gate_proj", + "model.layers.30.mlp.experts.119.up_proj", + "model.layers.30.mlp.experts.12.down_proj", + "model.layers.30.mlp.experts.12.gate_proj", + "model.layers.30.mlp.experts.12.up_proj", + "model.layers.30.mlp.experts.120.down_proj", + "model.layers.30.mlp.experts.120.gate_proj", + "model.layers.30.mlp.experts.120.up_proj", + "model.layers.30.mlp.experts.121.down_proj", + "model.layers.30.mlp.experts.121.gate_proj", + "model.layers.30.mlp.experts.121.up_proj", + "model.layers.30.mlp.experts.122.down_proj", + "model.layers.30.mlp.experts.122.gate_proj", + "model.layers.30.mlp.experts.122.up_proj", + "model.layers.30.mlp.experts.123.down_proj", + "model.layers.30.mlp.experts.123.gate_proj", + "model.layers.30.mlp.experts.123.up_proj", + "model.layers.30.mlp.experts.124.down_proj", + "model.layers.30.mlp.experts.124.gate_proj", + "model.layers.30.mlp.experts.124.up_proj", + "model.layers.30.mlp.experts.125.down_proj", + "model.layers.30.mlp.experts.125.gate_proj", + "model.layers.30.mlp.experts.125.up_proj", + "model.layers.30.mlp.experts.126.down_proj", + "model.layers.30.mlp.experts.126.gate_proj", + "model.layers.30.mlp.experts.126.up_proj", + "model.layers.30.mlp.experts.127.down_proj", + "model.layers.30.mlp.experts.127.gate_proj", + "model.layers.30.mlp.experts.127.up_proj", + "model.layers.30.mlp.experts.128.down_proj", + "model.layers.30.mlp.experts.128.gate_proj", + "model.layers.30.mlp.experts.128.up_proj", + "model.layers.30.mlp.experts.129.down_proj", + "model.layers.30.mlp.experts.129.gate_proj", + "model.layers.30.mlp.experts.129.up_proj", + "model.layers.30.mlp.experts.13.down_proj", + "model.layers.30.mlp.experts.13.gate_proj", + "model.layers.30.mlp.experts.13.up_proj", + "model.layers.30.mlp.experts.130.down_proj", + "model.layers.30.mlp.experts.130.gate_proj", + "model.layers.30.mlp.experts.130.up_proj", + "model.layers.30.mlp.experts.131.down_proj", + "model.layers.30.mlp.experts.131.gate_proj", + "model.layers.30.mlp.experts.131.up_proj", + "model.layers.30.mlp.experts.132.down_proj", + "model.layers.30.mlp.experts.132.gate_proj", + "model.layers.30.mlp.experts.132.up_proj", + "model.layers.30.mlp.experts.133.down_proj", + "model.layers.30.mlp.experts.133.gate_proj", + "model.layers.30.mlp.experts.133.up_proj", + "model.layers.30.mlp.experts.134.down_proj", + "model.layers.30.mlp.experts.134.gate_proj", + "model.layers.30.mlp.experts.134.up_proj", + "model.layers.30.mlp.experts.135.down_proj", + "model.layers.30.mlp.experts.135.gate_proj", + "model.layers.30.mlp.experts.135.up_proj", + "model.layers.30.mlp.experts.136.down_proj", + "model.layers.30.mlp.experts.136.gate_proj", + "model.layers.30.mlp.experts.136.up_proj", + "model.layers.30.mlp.experts.137.down_proj", + "model.layers.30.mlp.experts.137.gate_proj", + "model.layers.30.mlp.experts.137.up_proj", + "model.layers.30.mlp.experts.138.down_proj", + "model.layers.30.mlp.experts.138.gate_proj", + "model.layers.30.mlp.experts.138.up_proj", + "model.layers.30.mlp.experts.139.down_proj", + "model.layers.30.mlp.experts.139.gate_proj", + "model.layers.30.mlp.experts.139.up_proj", + "model.layers.30.mlp.experts.14.down_proj", + "model.layers.30.mlp.experts.14.gate_proj", + "model.layers.30.mlp.experts.14.up_proj", + "model.layers.30.mlp.experts.140.down_proj", + "model.layers.30.mlp.experts.140.gate_proj", + "model.layers.30.mlp.experts.140.up_proj", + "model.layers.30.mlp.experts.141.down_proj", + "model.layers.30.mlp.experts.141.gate_proj", + "model.layers.30.mlp.experts.141.up_proj", + "model.layers.30.mlp.experts.142.down_proj", + "model.layers.30.mlp.experts.142.gate_proj", + "model.layers.30.mlp.experts.142.up_proj", + "model.layers.30.mlp.experts.143.down_proj", + "model.layers.30.mlp.experts.143.gate_proj", + "model.layers.30.mlp.experts.143.up_proj", + "model.layers.30.mlp.experts.144.down_proj", + "model.layers.30.mlp.experts.144.gate_proj", + "model.layers.30.mlp.experts.144.up_proj", + "model.layers.30.mlp.experts.145.down_proj", + "model.layers.30.mlp.experts.145.gate_proj", + "model.layers.30.mlp.experts.145.up_proj", + "model.layers.30.mlp.experts.146.down_proj", + "model.layers.30.mlp.experts.146.gate_proj", + "model.layers.30.mlp.experts.146.up_proj", + "model.layers.30.mlp.experts.147.down_proj", + "model.layers.30.mlp.experts.147.gate_proj", + "model.layers.30.mlp.experts.147.up_proj", + "model.layers.30.mlp.experts.148.down_proj", + "model.layers.30.mlp.experts.148.gate_proj", + "model.layers.30.mlp.experts.148.up_proj", + "model.layers.30.mlp.experts.149.down_proj", + "model.layers.30.mlp.experts.149.gate_proj", + "model.layers.30.mlp.experts.149.up_proj", + "model.layers.30.mlp.experts.15.down_proj", + "model.layers.30.mlp.experts.15.gate_proj", + "model.layers.30.mlp.experts.15.up_proj", + "model.layers.30.mlp.experts.150.down_proj", + "model.layers.30.mlp.experts.150.gate_proj", + "model.layers.30.mlp.experts.150.up_proj", + "model.layers.30.mlp.experts.151.down_proj", + "model.layers.30.mlp.experts.151.gate_proj", + "model.layers.30.mlp.experts.151.up_proj", + "model.layers.30.mlp.experts.152.down_proj", + "model.layers.30.mlp.experts.152.gate_proj", + "model.layers.30.mlp.experts.152.up_proj", + "model.layers.30.mlp.experts.153.down_proj", + "model.layers.30.mlp.experts.153.gate_proj", + "model.layers.30.mlp.experts.153.up_proj", + "model.layers.30.mlp.experts.154.down_proj", + "model.layers.30.mlp.experts.154.gate_proj", + "model.layers.30.mlp.experts.154.up_proj", + "model.layers.30.mlp.experts.155.down_proj", + "model.layers.30.mlp.experts.155.gate_proj", + "model.layers.30.mlp.experts.155.up_proj", + "model.layers.30.mlp.experts.156.down_proj", + "model.layers.30.mlp.experts.156.gate_proj", + "model.layers.30.mlp.experts.156.up_proj", + "model.layers.30.mlp.experts.157.down_proj", + "model.layers.30.mlp.experts.157.gate_proj", + "model.layers.30.mlp.experts.157.up_proj", + "model.layers.30.mlp.experts.158.down_proj", + "model.layers.30.mlp.experts.158.gate_proj", + "model.layers.30.mlp.experts.158.up_proj", + "model.layers.30.mlp.experts.159.down_proj", + "model.layers.30.mlp.experts.159.gate_proj", + "model.layers.30.mlp.experts.159.up_proj", + "model.layers.30.mlp.experts.16.down_proj", + "model.layers.30.mlp.experts.16.gate_proj", + "model.layers.30.mlp.experts.16.up_proj", + "model.layers.30.mlp.experts.160.down_proj", + "model.layers.30.mlp.experts.160.gate_proj", + "model.layers.30.mlp.experts.160.up_proj", + "model.layers.30.mlp.experts.161.down_proj", + "model.layers.30.mlp.experts.161.gate_proj", + "model.layers.30.mlp.experts.161.up_proj", + "model.layers.30.mlp.experts.162.down_proj", + "model.layers.30.mlp.experts.162.gate_proj", + "model.layers.30.mlp.experts.162.up_proj", + "model.layers.30.mlp.experts.163.down_proj", + "model.layers.30.mlp.experts.163.gate_proj", + "model.layers.30.mlp.experts.163.up_proj", + "model.layers.30.mlp.experts.164.down_proj", + "model.layers.30.mlp.experts.164.gate_proj", + "model.layers.30.mlp.experts.164.up_proj", + "model.layers.30.mlp.experts.165.down_proj", + "model.layers.30.mlp.experts.165.gate_proj", + "model.layers.30.mlp.experts.165.up_proj", + "model.layers.30.mlp.experts.166.down_proj", + "model.layers.30.mlp.experts.166.gate_proj", + "model.layers.30.mlp.experts.166.up_proj", + "model.layers.30.mlp.experts.167.down_proj", + "model.layers.30.mlp.experts.167.gate_proj", + "model.layers.30.mlp.experts.167.up_proj", + "model.layers.30.mlp.experts.168.down_proj", + "model.layers.30.mlp.experts.168.gate_proj", + "model.layers.30.mlp.experts.168.up_proj", + "model.layers.30.mlp.experts.169.down_proj", + "model.layers.30.mlp.experts.169.gate_proj", + "model.layers.30.mlp.experts.169.up_proj", + "model.layers.30.mlp.experts.17.down_proj", + "model.layers.30.mlp.experts.17.gate_proj", + "model.layers.30.mlp.experts.17.up_proj", + "model.layers.30.mlp.experts.170.down_proj", + "model.layers.30.mlp.experts.170.gate_proj", + "model.layers.30.mlp.experts.170.up_proj", + "model.layers.30.mlp.experts.171.down_proj", + "model.layers.30.mlp.experts.171.gate_proj", + "model.layers.30.mlp.experts.171.up_proj", + "model.layers.30.mlp.experts.172.down_proj", + "model.layers.30.mlp.experts.172.gate_proj", + "model.layers.30.mlp.experts.172.up_proj", + "model.layers.30.mlp.experts.173.down_proj", + "model.layers.30.mlp.experts.173.gate_proj", + "model.layers.30.mlp.experts.173.up_proj", + "model.layers.30.mlp.experts.174.down_proj", + "model.layers.30.mlp.experts.174.gate_proj", + "model.layers.30.mlp.experts.174.up_proj", + "model.layers.30.mlp.experts.175.down_proj", + "model.layers.30.mlp.experts.175.gate_proj", + "model.layers.30.mlp.experts.175.up_proj", + "model.layers.30.mlp.experts.176.down_proj", + "model.layers.30.mlp.experts.176.gate_proj", + "model.layers.30.mlp.experts.176.up_proj", + "model.layers.30.mlp.experts.177.down_proj", + "model.layers.30.mlp.experts.177.gate_proj", + "model.layers.30.mlp.experts.177.up_proj", + "model.layers.30.mlp.experts.178.down_proj", + "model.layers.30.mlp.experts.178.gate_proj", + "model.layers.30.mlp.experts.178.up_proj", + "model.layers.30.mlp.experts.179.down_proj", + "model.layers.30.mlp.experts.179.gate_proj", + "model.layers.30.mlp.experts.179.up_proj", + "model.layers.30.mlp.experts.18.down_proj", + "model.layers.30.mlp.experts.18.gate_proj", + "model.layers.30.mlp.experts.18.up_proj", + "model.layers.30.mlp.experts.180.down_proj", + "model.layers.30.mlp.experts.180.gate_proj", + "model.layers.30.mlp.experts.180.up_proj", + "model.layers.30.mlp.experts.181.down_proj", + "model.layers.30.mlp.experts.181.gate_proj", + "model.layers.30.mlp.experts.181.up_proj", + "model.layers.30.mlp.experts.182.down_proj", + "model.layers.30.mlp.experts.182.gate_proj", + "model.layers.30.mlp.experts.182.up_proj", + "model.layers.30.mlp.experts.183.down_proj", + "model.layers.30.mlp.experts.183.gate_proj", + "model.layers.30.mlp.experts.183.up_proj", + "model.layers.30.mlp.experts.184.down_proj", + "model.layers.30.mlp.experts.184.gate_proj", + "model.layers.30.mlp.experts.184.up_proj", + "model.layers.30.mlp.experts.185.down_proj", + "model.layers.30.mlp.experts.185.gate_proj", + "model.layers.30.mlp.experts.185.up_proj", + "model.layers.30.mlp.experts.186.down_proj", + "model.layers.30.mlp.experts.186.gate_proj", + "model.layers.30.mlp.experts.186.up_proj", + "model.layers.30.mlp.experts.187.down_proj", + "model.layers.30.mlp.experts.187.gate_proj", + "model.layers.30.mlp.experts.187.up_proj", + "model.layers.30.mlp.experts.188.down_proj", + "model.layers.30.mlp.experts.188.gate_proj", + "model.layers.30.mlp.experts.188.up_proj", + "model.layers.30.mlp.experts.189.down_proj", + "model.layers.30.mlp.experts.189.gate_proj", + "model.layers.30.mlp.experts.189.up_proj", + "model.layers.30.mlp.experts.19.down_proj", + "model.layers.30.mlp.experts.19.gate_proj", + "model.layers.30.mlp.experts.19.up_proj", + "model.layers.30.mlp.experts.190.down_proj", + "model.layers.30.mlp.experts.190.gate_proj", + "model.layers.30.mlp.experts.190.up_proj", + "model.layers.30.mlp.experts.191.down_proj", + "model.layers.30.mlp.experts.191.gate_proj", + "model.layers.30.mlp.experts.191.up_proj", + "model.layers.30.mlp.experts.192.down_proj", + "model.layers.30.mlp.experts.192.gate_proj", + "model.layers.30.mlp.experts.192.up_proj", + "model.layers.30.mlp.experts.193.down_proj", + "model.layers.30.mlp.experts.193.gate_proj", + "model.layers.30.mlp.experts.193.up_proj", + "model.layers.30.mlp.experts.194.down_proj", + "model.layers.30.mlp.experts.194.gate_proj", + "model.layers.30.mlp.experts.194.up_proj", + "model.layers.30.mlp.experts.195.down_proj", + "model.layers.30.mlp.experts.195.gate_proj", + "model.layers.30.mlp.experts.195.up_proj", + "model.layers.30.mlp.experts.196.down_proj", + "model.layers.30.mlp.experts.196.gate_proj", + "model.layers.30.mlp.experts.196.up_proj", + "model.layers.30.mlp.experts.197.down_proj", + "model.layers.30.mlp.experts.197.gate_proj", + "model.layers.30.mlp.experts.197.up_proj", + "model.layers.30.mlp.experts.198.down_proj", + "model.layers.30.mlp.experts.198.gate_proj", + "model.layers.30.mlp.experts.198.up_proj", + "model.layers.30.mlp.experts.199.down_proj", + "model.layers.30.mlp.experts.199.gate_proj", + "model.layers.30.mlp.experts.199.up_proj", + "model.layers.30.mlp.experts.2.down_proj", + "model.layers.30.mlp.experts.2.gate_proj", + "model.layers.30.mlp.experts.2.up_proj", + "model.layers.30.mlp.experts.20.down_proj", + "model.layers.30.mlp.experts.20.gate_proj", + "model.layers.30.mlp.experts.20.up_proj", + "model.layers.30.mlp.experts.200.down_proj", + "model.layers.30.mlp.experts.200.gate_proj", + "model.layers.30.mlp.experts.200.up_proj", + "model.layers.30.mlp.experts.201.down_proj", + "model.layers.30.mlp.experts.201.gate_proj", + "model.layers.30.mlp.experts.201.up_proj", + "model.layers.30.mlp.experts.202.down_proj", + "model.layers.30.mlp.experts.202.gate_proj", + "model.layers.30.mlp.experts.202.up_proj", + "model.layers.30.mlp.experts.203.down_proj", + "model.layers.30.mlp.experts.203.gate_proj", + "model.layers.30.mlp.experts.203.up_proj", + "model.layers.30.mlp.experts.204.down_proj", + "model.layers.30.mlp.experts.204.gate_proj", + "model.layers.30.mlp.experts.204.up_proj", + "model.layers.30.mlp.experts.205.down_proj", + "model.layers.30.mlp.experts.205.gate_proj", + "model.layers.30.mlp.experts.205.up_proj", + "model.layers.30.mlp.experts.206.down_proj", + "model.layers.30.mlp.experts.206.gate_proj", + "model.layers.30.mlp.experts.206.up_proj", + "model.layers.30.mlp.experts.207.down_proj", + "model.layers.30.mlp.experts.207.gate_proj", + "model.layers.30.mlp.experts.207.up_proj", + "model.layers.30.mlp.experts.208.down_proj", + "model.layers.30.mlp.experts.208.gate_proj", + "model.layers.30.mlp.experts.208.up_proj", + "model.layers.30.mlp.experts.209.down_proj", + "model.layers.30.mlp.experts.209.gate_proj", + "model.layers.30.mlp.experts.209.up_proj", + "model.layers.30.mlp.experts.21.down_proj", + "model.layers.30.mlp.experts.21.gate_proj", + "model.layers.30.mlp.experts.21.up_proj", + "model.layers.30.mlp.experts.210.down_proj", + "model.layers.30.mlp.experts.210.gate_proj", + "model.layers.30.mlp.experts.210.up_proj", + "model.layers.30.mlp.experts.211.down_proj", + "model.layers.30.mlp.experts.211.gate_proj", + "model.layers.30.mlp.experts.211.up_proj", + "model.layers.30.mlp.experts.212.down_proj", + "model.layers.30.mlp.experts.212.gate_proj", + "model.layers.30.mlp.experts.212.up_proj", + "model.layers.30.mlp.experts.213.down_proj", + "model.layers.30.mlp.experts.213.gate_proj", + "model.layers.30.mlp.experts.213.up_proj", + "model.layers.30.mlp.experts.214.down_proj", + "model.layers.30.mlp.experts.214.gate_proj", + "model.layers.30.mlp.experts.214.up_proj", + "model.layers.30.mlp.experts.215.down_proj", + "model.layers.30.mlp.experts.215.gate_proj", + "model.layers.30.mlp.experts.215.up_proj", + "model.layers.30.mlp.experts.216.down_proj", + "model.layers.30.mlp.experts.216.gate_proj", + "model.layers.30.mlp.experts.216.up_proj", + "model.layers.30.mlp.experts.217.down_proj", + "model.layers.30.mlp.experts.217.gate_proj", + "model.layers.30.mlp.experts.217.up_proj", + "model.layers.30.mlp.experts.218.down_proj", + "model.layers.30.mlp.experts.218.gate_proj", + "model.layers.30.mlp.experts.218.up_proj", + "model.layers.30.mlp.experts.219.down_proj", + "model.layers.30.mlp.experts.219.gate_proj", + "model.layers.30.mlp.experts.219.up_proj", + "model.layers.30.mlp.experts.22.down_proj", + "model.layers.30.mlp.experts.22.gate_proj", + "model.layers.30.mlp.experts.22.up_proj", + "model.layers.30.mlp.experts.220.down_proj", + "model.layers.30.mlp.experts.220.gate_proj", + "model.layers.30.mlp.experts.220.up_proj", + "model.layers.30.mlp.experts.221.down_proj", + "model.layers.30.mlp.experts.221.gate_proj", + "model.layers.30.mlp.experts.221.up_proj", + "model.layers.30.mlp.experts.222.down_proj", + "model.layers.30.mlp.experts.222.gate_proj", + "model.layers.30.mlp.experts.222.up_proj", + "model.layers.30.mlp.experts.223.down_proj", + "model.layers.30.mlp.experts.223.gate_proj", + "model.layers.30.mlp.experts.223.up_proj", + "model.layers.30.mlp.experts.224.down_proj", + "model.layers.30.mlp.experts.224.gate_proj", + "model.layers.30.mlp.experts.224.up_proj", + "model.layers.30.mlp.experts.225.down_proj", + "model.layers.30.mlp.experts.225.gate_proj", + "model.layers.30.mlp.experts.225.up_proj", + "model.layers.30.mlp.experts.226.down_proj", + "model.layers.30.mlp.experts.226.gate_proj", + "model.layers.30.mlp.experts.226.up_proj", + "model.layers.30.mlp.experts.227.down_proj", + "model.layers.30.mlp.experts.227.gate_proj", + "model.layers.30.mlp.experts.227.up_proj", + "model.layers.30.mlp.experts.228.down_proj", + "model.layers.30.mlp.experts.228.gate_proj", + "model.layers.30.mlp.experts.228.up_proj", + "model.layers.30.mlp.experts.229.down_proj", + "model.layers.30.mlp.experts.229.gate_proj", + "model.layers.30.mlp.experts.229.up_proj", + "model.layers.30.mlp.experts.23.down_proj", + "model.layers.30.mlp.experts.23.gate_proj", + "model.layers.30.mlp.experts.23.up_proj", + "model.layers.30.mlp.experts.230.down_proj", + "model.layers.30.mlp.experts.230.gate_proj", + "model.layers.30.mlp.experts.230.up_proj", + "model.layers.30.mlp.experts.231.down_proj", + "model.layers.30.mlp.experts.231.gate_proj", + "model.layers.30.mlp.experts.231.up_proj", + "model.layers.30.mlp.experts.232.down_proj", + "model.layers.30.mlp.experts.232.gate_proj", + "model.layers.30.mlp.experts.232.up_proj", + "model.layers.30.mlp.experts.233.down_proj", + "model.layers.30.mlp.experts.233.gate_proj", + "model.layers.30.mlp.experts.233.up_proj", + "model.layers.30.mlp.experts.234.down_proj", + "model.layers.30.mlp.experts.234.gate_proj", + "model.layers.30.mlp.experts.234.up_proj", + "model.layers.30.mlp.experts.235.down_proj", + "model.layers.30.mlp.experts.235.gate_proj", + "model.layers.30.mlp.experts.235.up_proj", + "model.layers.30.mlp.experts.236.down_proj", + "model.layers.30.mlp.experts.236.gate_proj", + "model.layers.30.mlp.experts.236.up_proj", + "model.layers.30.mlp.experts.237.down_proj", + "model.layers.30.mlp.experts.237.gate_proj", + "model.layers.30.mlp.experts.237.up_proj", + "model.layers.30.mlp.experts.238.down_proj", + "model.layers.30.mlp.experts.238.gate_proj", + "model.layers.30.mlp.experts.238.up_proj", + "model.layers.30.mlp.experts.239.down_proj", + "model.layers.30.mlp.experts.239.gate_proj", + "model.layers.30.mlp.experts.239.up_proj", + "model.layers.30.mlp.experts.24.down_proj", + "model.layers.30.mlp.experts.24.gate_proj", + "model.layers.30.mlp.experts.24.up_proj", + "model.layers.30.mlp.experts.240.down_proj", + "model.layers.30.mlp.experts.240.gate_proj", + "model.layers.30.mlp.experts.240.up_proj", + "model.layers.30.mlp.experts.241.down_proj", + "model.layers.30.mlp.experts.241.gate_proj", + "model.layers.30.mlp.experts.241.up_proj", + "model.layers.30.mlp.experts.242.down_proj", + "model.layers.30.mlp.experts.242.gate_proj", + "model.layers.30.mlp.experts.242.up_proj", + "model.layers.30.mlp.experts.243.down_proj", + "model.layers.30.mlp.experts.243.gate_proj", + "model.layers.30.mlp.experts.243.up_proj", + "model.layers.30.mlp.experts.244.down_proj", + "model.layers.30.mlp.experts.244.gate_proj", + "model.layers.30.mlp.experts.244.up_proj", + "model.layers.30.mlp.experts.245.down_proj", + "model.layers.30.mlp.experts.245.gate_proj", + "model.layers.30.mlp.experts.245.up_proj", + "model.layers.30.mlp.experts.246.down_proj", + "model.layers.30.mlp.experts.246.gate_proj", + "model.layers.30.mlp.experts.246.up_proj", + "model.layers.30.mlp.experts.247.down_proj", + "model.layers.30.mlp.experts.247.gate_proj", + "model.layers.30.mlp.experts.247.up_proj", + "model.layers.30.mlp.experts.248.down_proj", + "model.layers.30.mlp.experts.248.gate_proj", + "model.layers.30.mlp.experts.248.up_proj", + "model.layers.30.mlp.experts.249.down_proj", + "model.layers.30.mlp.experts.249.gate_proj", + "model.layers.30.mlp.experts.249.up_proj", + "model.layers.30.mlp.experts.25.down_proj", + "model.layers.30.mlp.experts.25.gate_proj", + "model.layers.30.mlp.experts.25.up_proj", + "model.layers.30.mlp.experts.250.down_proj", + "model.layers.30.mlp.experts.250.gate_proj", + "model.layers.30.mlp.experts.250.up_proj", + "model.layers.30.mlp.experts.251.down_proj", + "model.layers.30.mlp.experts.251.gate_proj", + "model.layers.30.mlp.experts.251.up_proj", + "model.layers.30.mlp.experts.252.down_proj", + "model.layers.30.mlp.experts.252.gate_proj", + "model.layers.30.mlp.experts.252.up_proj", + "model.layers.30.mlp.experts.253.down_proj", + "model.layers.30.mlp.experts.253.gate_proj", + "model.layers.30.mlp.experts.253.up_proj", + "model.layers.30.mlp.experts.254.down_proj", + "model.layers.30.mlp.experts.254.gate_proj", + "model.layers.30.mlp.experts.254.up_proj", + "model.layers.30.mlp.experts.255.down_proj", + "model.layers.30.mlp.experts.255.gate_proj", + "model.layers.30.mlp.experts.255.up_proj", + "model.layers.30.mlp.experts.26.down_proj", + "model.layers.30.mlp.experts.26.gate_proj", + "model.layers.30.mlp.experts.26.up_proj", + "model.layers.30.mlp.experts.27.down_proj", + "model.layers.30.mlp.experts.27.gate_proj", + "model.layers.30.mlp.experts.27.up_proj", + "model.layers.30.mlp.experts.28.down_proj", + "model.layers.30.mlp.experts.28.gate_proj", + "model.layers.30.mlp.experts.28.up_proj", + "model.layers.30.mlp.experts.29.down_proj", + "model.layers.30.mlp.experts.29.gate_proj", + "model.layers.30.mlp.experts.29.up_proj", + "model.layers.30.mlp.experts.3.down_proj", + "model.layers.30.mlp.experts.3.gate_proj", + "model.layers.30.mlp.experts.3.up_proj", + "model.layers.30.mlp.experts.30.down_proj", + "model.layers.30.mlp.experts.30.gate_proj", + "model.layers.30.mlp.experts.30.up_proj", + "model.layers.30.mlp.experts.31.down_proj", + "model.layers.30.mlp.experts.31.gate_proj", + "model.layers.30.mlp.experts.31.up_proj", + "model.layers.30.mlp.experts.32.down_proj", + "model.layers.30.mlp.experts.32.gate_proj", + "model.layers.30.mlp.experts.32.up_proj", + "model.layers.30.mlp.experts.33.down_proj", + "model.layers.30.mlp.experts.33.gate_proj", + "model.layers.30.mlp.experts.33.up_proj", + "model.layers.30.mlp.experts.34.down_proj", + "model.layers.30.mlp.experts.34.gate_proj", + "model.layers.30.mlp.experts.34.up_proj", + "model.layers.30.mlp.experts.35.down_proj", + "model.layers.30.mlp.experts.35.gate_proj", + "model.layers.30.mlp.experts.35.up_proj", + "model.layers.30.mlp.experts.36.down_proj", + "model.layers.30.mlp.experts.36.gate_proj", + "model.layers.30.mlp.experts.36.up_proj", + "model.layers.30.mlp.experts.37.down_proj", + "model.layers.30.mlp.experts.37.gate_proj", + "model.layers.30.mlp.experts.37.up_proj", + "model.layers.30.mlp.experts.38.down_proj", + "model.layers.30.mlp.experts.38.gate_proj", + "model.layers.30.mlp.experts.38.up_proj", + "model.layers.30.mlp.experts.39.down_proj", + "model.layers.30.mlp.experts.39.gate_proj", + "model.layers.30.mlp.experts.39.up_proj", + "model.layers.30.mlp.experts.4.down_proj", + "model.layers.30.mlp.experts.4.gate_proj", + "model.layers.30.mlp.experts.4.up_proj", + "model.layers.30.mlp.experts.40.down_proj", + "model.layers.30.mlp.experts.40.gate_proj", + "model.layers.30.mlp.experts.40.up_proj", + "model.layers.30.mlp.experts.41.down_proj", + "model.layers.30.mlp.experts.41.gate_proj", + "model.layers.30.mlp.experts.41.up_proj", + "model.layers.30.mlp.experts.42.down_proj", + "model.layers.30.mlp.experts.42.gate_proj", + "model.layers.30.mlp.experts.42.up_proj", + "model.layers.30.mlp.experts.43.down_proj", + "model.layers.30.mlp.experts.43.gate_proj", + "model.layers.30.mlp.experts.43.up_proj", + "model.layers.30.mlp.experts.44.down_proj", + "model.layers.30.mlp.experts.44.gate_proj", + "model.layers.30.mlp.experts.44.up_proj", + "model.layers.30.mlp.experts.45.down_proj", + "model.layers.30.mlp.experts.45.gate_proj", + "model.layers.30.mlp.experts.45.up_proj", + "model.layers.30.mlp.experts.46.down_proj", + "model.layers.30.mlp.experts.46.gate_proj", + "model.layers.30.mlp.experts.46.up_proj", + "model.layers.30.mlp.experts.47.down_proj", + "model.layers.30.mlp.experts.47.gate_proj", + "model.layers.30.mlp.experts.47.up_proj", + "model.layers.30.mlp.experts.48.down_proj", + "model.layers.30.mlp.experts.48.gate_proj", + "model.layers.30.mlp.experts.48.up_proj", + "model.layers.30.mlp.experts.49.down_proj", + "model.layers.30.mlp.experts.49.gate_proj", + "model.layers.30.mlp.experts.49.up_proj", + "model.layers.30.mlp.experts.5.down_proj", + "model.layers.30.mlp.experts.5.gate_proj", + "model.layers.30.mlp.experts.5.up_proj", + "model.layers.30.mlp.experts.50.down_proj", + "model.layers.30.mlp.experts.50.gate_proj", + "model.layers.30.mlp.experts.50.up_proj", + "model.layers.30.mlp.experts.51.down_proj", + "model.layers.30.mlp.experts.51.gate_proj", + "model.layers.30.mlp.experts.51.up_proj", + "model.layers.30.mlp.experts.52.down_proj", + "model.layers.30.mlp.experts.52.gate_proj", + "model.layers.30.mlp.experts.52.up_proj", + "model.layers.30.mlp.experts.53.down_proj", + "model.layers.30.mlp.experts.53.gate_proj", + "model.layers.30.mlp.experts.53.up_proj", + "model.layers.30.mlp.experts.54.down_proj", + "model.layers.30.mlp.experts.54.gate_proj", + "model.layers.30.mlp.experts.54.up_proj", + "model.layers.30.mlp.experts.55.down_proj", + "model.layers.30.mlp.experts.55.gate_proj", + "model.layers.30.mlp.experts.55.up_proj", + "model.layers.30.mlp.experts.56.down_proj", + "model.layers.30.mlp.experts.56.gate_proj", + "model.layers.30.mlp.experts.56.up_proj", + "model.layers.30.mlp.experts.57.down_proj", + "model.layers.30.mlp.experts.57.gate_proj", + "model.layers.30.mlp.experts.57.up_proj", + "model.layers.30.mlp.experts.58.down_proj", + "model.layers.30.mlp.experts.58.gate_proj", + "model.layers.30.mlp.experts.58.up_proj", + "model.layers.30.mlp.experts.59.down_proj", + "model.layers.30.mlp.experts.59.gate_proj", + "model.layers.30.mlp.experts.59.up_proj", + "model.layers.30.mlp.experts.6.down_proj", + "model.layers.30.mlp.experts.6.gate_proj", + "model.layers.30.mlp.experts.6.up_proj", + "model.layers.30.mlp.experts.60.down_proj", + "model.layers.30.mlp.experts.60.gate_proj", + "model.layers.30.mlp.experts.60.up_proj", + "model.layers.30.mlp.experts.61.down_proj", + "model.layers.30.mlp.experts.61.gate_proj", + "model.layers.30.mlp.experts.61.up_proj", + "model.layers.30.mlp.experts.62.down_proj", + "model.layers.30.mlp.experts.62.gate_proj", + "model.layers.30.mlp.experts.62.up_proj", + "model.layers.30.mlp.experts.63.down_proj", + "model.layers.30.mlp.experts.63.gate_proj", + "model.layers.30.mlp.experts.63.up_proj", + "model.layers.30.mlp.experts.64.down_proj", + "model.layers.30.mlp.experts.64.gate_proj", + "model.layers.30.mlp.experts.64.up_proj", + "model.layers.30.mlp.experts.65.down_proj", + "model.layers.30.mlp.experts.65.gate_proj", + "model.layers.30.mlp.experts.65.up_proj", + "model.layers.30.mlp.experts.66.down_proj", + "model.layers.30.mlp.experts.66.gate_proj", + "model.layers.30.mlp.experts.66.up_proj", + "model.layers.30.mlp.experts.67.down_proj", + "model.layers.30.mlp.experts.67.gate_proj", + "model.layers.30.mlp.experts.67.up_proj", + "model.layers.30.mlp.experts.68.down_proj", + "model.layers.30.mlp.experts.68.gate_proj", + "model.layers.30.mlp.experts.68.up_proj", + "model.layers.30.mlp.experts.69.down_proj", + "model.layers.30.mlp.experts.69.gate_proj", + "model.layers.30.mlp.experts.69.up_proj", + "model.layers.30.mlp.experts.7.down_proj", + "model.layers.30.mlp.experts.7.gate_proj", + "model.layers.30.mlp.experts.7.up_proj", + "model.layers.30.mlp.experts.70.down_proj", + "model.layers.30.mlp.experts.70.gate_proj", + "model.layers.30.mlp.experts.70.up_proj", + "model.layers.30.mlp.experts.71.down_proj", + "model.layers.30.mlp.experts.71.gate_proj", + "model.layers.30.mlp.experts.71.up_proj", + "model.layers.30.mlp.experts.72.down_proj", + "model.layers.30.mlp.experts.72.gate_proj", + "model.layers.30.mlp.experts.72.up_proj", + "model.layers.30.mlp.experts.73.down_proj", + "model.layers.30.mlp.experts.73.gate_proj", + "model.layers.30.mlp.experts.73.up_proj", + "model.layers.30.mlp.experts.74.down_proj", + "model.layers.30.mlp.experts.74.gate_proj", + "model.layers.30.mlp.experts.74.up_proj", + "model.layers.30.mlp.experts.75.down_proj", + "model.layers.30.mlp.experts.75.gate_proj", + "model.layers.30.mlp.experts.75.up_proj", + "model.layers.30.mlp.experts.76.down_proj", + "model.layers.30.mlp.experts.76.gate_proj", + "model.layers.30.mlp.experts.76.up_proj", + "model.layers.30.mlp.experts.77.down_proj", + "model.layers.30.mlp.experts.77.gate_proj", + "model.layers.30.mlp.experts.77.up_proj", + "model.layers.30.mlp.experts.78.down_proj", + "model.layers.30.mlp.experts.78.gate_proj", + "model.layers.30.mlp.experts.78.up_proj", + "model.layers.30.mlp.experts.79.down_proj", + "model.layers.30.mlp.experts.79.gate_proj", + "model.layers.30.mlp.experts.79.up_proj", + "model.layers.30.mlp.experts.8.down_proj", + "model.layers.30.mlp.experts.8.gate_proj", + "model.layers.30.mlp.experts.8.up_proj", + "model.layers.30.mlp.experts.80.down_proj", + "model.layers.30.mlp.experts.80.gate_proj", + "model.layers.30.mlp.experts.80.up_proj", + "model.layers.30.mlp.experts.81.down_proj", + "model.layers.30.mlp.experts.81.gate_proj", + "model.layers.30.mlp.experts.81.up_proj", + "model.layers.30.mlp.experts.82.down_proj", + "model.layers.30.mlp.experts.82.gate_proj", + "model.layers.30.mlp.experts.82.up_proj", + "model.layers.30.mlp.experts.83.down_proj", + "model.layers.30.mlp.experts.83.gate_proj", + "model.layers.30.mlp.experts.83.up_proj", + "model.layers.30.mlp.experts.84.down_proj", + "model.layers.30.mlp.experts.84.gate_proj", + "model.layers.30.mlp.experts.84.up_proj", + "model.layers.30.mlp.experts.85.down_proj", + "model.layers.30.mlp.experts.85.gate_proj", + "model.layers.30.mlp.experts.85.up_proj", + "model.layers.30.mlp.experts.86.down_proj", + "model.layers.30.mlp.experts.86.gate_proj", + "model.layers.30.mlp.experts.86.up_proj", + "model.layers.30.mlp.experts.87.down_proj", + "model.layers.30.mlp.experts.87.gate_proj", + "model.layers.30.mlp.experts.87.up_proj", + "model.layers.30.mlp.experts.88.down_proj", + "model.layers.30.mlp.experts.88.gate_proj", + "model.layers.30.mlp.experts.88.up_proj", + "model.layers.30.mlp.experts.89.down_proj", + "model.layers.30.mlp.experts.89.gate_proj", + "model.layers.30.mlp.experts.89.up_proj", + "model.layers.30.mlp.experts.9.down_proj", + "model.layers.30.mlp.experts.9.gate_proj", + "model.layers.30.mlp.experts.9.up_proj", + "model.layers.30.mlp.experts.90.down_proj", + "model.layers.30.mlp.experts.90.gate_proj", + "model.layers.30.mlp.experts.90.up_proj", + "model.layers.30.mlp.experts.91.down_proj", + "model.layers.30.mlp.experts.91.gate_proj", + "model.layers.30.mlp.experts.91.up_proj", + "model.layers.30.mlp.experts.92.down_proj", + "model.layers.30.mlp.experts.92.gate_proj", + "model.layers.30.mlp.experts.92.up_proj", + "model.layers.30.mlp.experts.93.down_proj", + "model.layers.30.mlp.experts.93.gate_proj", + "model.layers.30.mlp.experts.93.up_proj", + "model.layers.30.mlp.experts.94.down_proj", + "model.layers.30.mlp.experts.94.gate_proj", + "model.layers.30.mlp.experts.94.up_proj", + "model.layers.30.mlp.experts.95.down_proj", + "model.layers.30.mlp.experts.95.gate_proj", + "model.layers.30.mlp.experts.95.up_proj", + "model.layers.30.mlp.experts.96.down_proj", + "model.layers.30.mlp.experts.96.gate_proj", + "model.layers.30.mlp.experts.96.up_proj", + "model.layers.30.mlp.experts.97.down_proj", + "model.layers.30.mlp.experts.97.gate_proj", + "model.layers.30.mlp.experts.97.up_proj", + "model.layers.30.mlp.experts.98.down_proj", + "model.layers.30.mlp.experts.98.gate_proj", + "model.layers.30.mlp.experts.98.up_proj", + "model.layers.30.mlp.experts.99.down_proj", + "model.layers.30.mlp.experts.99.gate_proj", + "model.layers.30.mlp.experts.99.up_proj", + "model.layers.31.mlp.experts.0.down_proj", + "model.layers.31.mlp.experts.0.gate_proj", + "model.layers.31.mlp.experts.0.up_proj", + "model.layers.31.mlp.experts.1.down_proj", + "model.layers.31.mlp.experts.1.gate_proj", + "model.layers.31.mlp.experts.1.up_proj", + "model.layers.31.mlp.experts.10.down_proj", + "model.layers.31.mlp.experts.10.gate_proj", + "model.layers.31.mlp.experts.10.up_proj", + "model.layers.31.mlp.experts.100.down_proj", + "model.layers.31.mlp.experts.100.gate_proj", + "model.layers.31.mlp.experts.100.up_proj", + "model.layers.31.mlp.experts.101.down_proj", + "model.layers.31.mlp.experts.101.gate_proj", + "model.layers.31.mlp.experts.101.up_proj", + "model.layers.31.mlp.experts.102.down_proj", + "model.layers.31.mlp.experts.102.gate_proj", + "model.layers.31.mlp.experts.102.up_proj", + "model.layers.31.mlp.experts.103.down_proj", + "model.layers.31.mlp.experts.103.gate_proj", + "model.layers.31.mlp.experts.103.up_proj", + "model.layers.31.mlp.experts.104.down_proj", + "model.layers.31.mlp.experts.104.gate_proj", + "model.layers.31.mlp.experts.104.up_proj", + "model.layers.31.mlp.experts.105.down_proj", + "model.layers.31.mlp.experts.105.gate_proj", + "model.layers.31.mlp.experts.105.up_proj", + "model.layers.31.mlp.experts.106.down_proj", + "model.layers.31.mlp.experts.106.gate_proj", + "model.layers.31.mlp.experts.106.up_proj", + "model.layers.31.mlp.experts.107.down_proj", + "model.layers.31.mlp.experts.107.gate_proj", + "model.layers.31.mlp.experts.107.up_proj", + "model.layers.31.mlp.experts.108.down_proj", + "model.layers.31.mlp.experts.108.gate_proj", + "model.layers.31.mlp.experts.108.up_proj", + "model.layers.31.mlp.experts.109.down_proj", + "model.layers.31.mlp.experts.109.gate_proj", + "model.layers.31.mlp.experts.109.up_proj", + "model.layers.31.mlp.experts.11.down_proj", + "model.layers.31.mlp.experts.11.gate_proj", + "model.layers.31.mlp.experts.11.up_proj", + "model.layers.31.mlp.experts.110.down_proj", + "model.layers.31.mlp.experts.110.gate_proj", + "model.layers.31.mlp.experts.110.up_proj", + "model.layers.31.mlp.experts.111.down_proj", + "model.layers.31.mlp.experts.111.gate_proj", + "model.layers.31.mlp.experts.111.up_proj", + "model.layers.31.mlp.experts.112.down_proj", + "model.layers.31.mlp.experts.112.gate_proj", + "model.layers.31.mlp.experts.112.up_proj", + "model.layers.31.mlp.experts.113.down_proj", + "model.layers.31.mlp.experts.113.gate_proj", + "model.layers.31.mlp.experts.113.up_proj", + "model.layers.31.mlp.experts.114.down_proj", + "model.layers.31.mlp.experts.114.gate_proj", + "model.layers.31.mlp.experts.114.up_proj", + "model.layers.31.mlp.experts.115.down_proj", + "model.layers.31.mlp.experts.115.gate_proj", + "model.layers.31.mlp.experts.115.up_proj", + "model.layers.31.mlp.experts.116.down_proj", + "model.layers.31.mlp.experts.116.gate_proj", + "model.layers.31.mlp.experts.116.up_proj", + "model.layers.31.mlp.experts.117.down_proj", + "model.layers.31.mlp.experts.117.gate_proj", + "model.layers.31.mlp.experts.117.up_proj", + "model.layers.31.mlp.experts.118.down_proj", + "model.layers.31.mlp.experts.118.gate_proj", + "model.layers.31.mlp.experts.118.up_proj", + "model.layers.31.mlp.experts.119.down_proj", + "model.layers.31.mlp.experts.119.gate_proj", + "model.layers.31.mlp.experts.119.up_proj", + "model.layers.31.mlp.experts.12.down_proj", + "model.layers.31.mlp.experts.12.gate_proj", + "model.layers.31.mlp.experts.12.up_proj", + "model.layers.31.mlp.experts.120.down_proj", + "model.layers.31.mlp.experts.120.gate_proj", + "model.layers.31.mlp.experts.120.up_proj", + "model.layers.31.mlp.experts.121.down_proj", + "model.layers.31.mlp.experts.121.gate_proj", + "model.layers.31.mlp.experts.121.up_proj", + "model.layers.31.mlp.experts.122.down_proj", + "model.layers.31.mlp.experts.122.gate_proj", + "model.layers.31.mlp.experts.122.up_proj", + "model.layers.31.mlp.experts.123.down_proj", + "model.layers.31.mlp.experts.123.gate_proj", + "model.layers.31.mlp.experts.123.up_proj", + "model.layers.31.mlp.experts.124.down_proj", + "model.layers.31.mlp.experts.124.gate_proj", + "model.layers.31.mlp.experts.124.up_proj", + "model.layers.31.mlp.experts.125.down_proj", + "model.layers.31.mlp.experts.125.gate_proj", + "model.layers.31.mlp.experts.125.up_proj", + "model.layers.31.mlp.experts.126.down_proj", + "model.layers.31.mlp.experts.126.gate_proj", + "model.layers.31.mlp.experts.126.up_proj", + "model.layers.31.mlp.experts.127.down_proj", + "model.layers.31.mlp.experts.127.gate_proj", + "model.layers.31.mlp.experts.127.up_proj", + "model.layers.31.mlp.experts.128.down_proj", + "model.layers.31.mlp.experts.128.gate_proj", + "model.layers.31.mlp.experts.128.up_proj", + "model.layers.31.mlp.experts.129.down_proj", + "model.layers.31.mlp.experts.129.gate_proj", + "model.layers.31.mlp.experts.129.up_proj", + "model.layers.31.mlp.experts.13.down_proj", + "model.layers.31.mlp.experts.13.gate_proj", + "model.layers.31.mlp.experts.13.up_proj", + "model.layers.31.mlp.experts.130.down_proj", + "model.layers.31.mlp.experts.130.gate_proj", + "model.layers.31.mlp.experts.130.up_proj", + "model.layers.31.mlp.experts.131.down_proj", + "model.layers.31.mlp.experts.131.gate_proj", + "model.layers.31.mlp.experts.131.up_proj", + "model.layers.31.mlp.experts.132.down_proj", + "model.layers.31.mlp.experts.132.gate_proj", + "model.layers.31.mlp.experts.132.up_proj", + "model.layers.31.mlp.experts.133.down_proj", + "model.layers.31.mlp.experts.133.gate_proj", + "model.layers.31.mlp.experts.133.up_proj", + "model.layers.31.mlp.experts.134.down_proj", + "model.layers.31.mlp.experts.134.gate_proj", + "model.layers.31.mlp.experts.134.up_proj", + "model.layers.31.mlp.experts.135.down_proj", + "model.layers.31.mlp.experts.135.gate_proj", + "model.layers.31.mlp.experts.135.up_proj", + "model.layers.31.mlp.experts.136.down_proj", + "model.layers.31.mlp.experts.136.gate_proj", + "model.layers.31.mlp.experts.136.up_proj", + "model.layers.31.mlp.experts.137.down_proj", + "model.layers.31.mlp.experts.137.gate_proj", + "model.layers.31.mlp.experts.137.up_proj", + "model.layers.31.mlp.experts.138.down_proj", + "model.layers.31.mlp.experts.138.gate_proj", + "model.layers.31.mlp.experts.138.up_proj", + "model.layers.31.mlp.experts.139.down_proj", + "model.layers.31.mlp.experts.139.gate_proj", + "model.layers.31.mlp.experts.139.up_proj", + "model.layers.31.mlp.experts.14.down_proj", + "model.layers.31.mlp.experts.14.gate_proj", + "model.layers.31.mlp.experts.14.up_proj", + "model.layers.31.mlp.experts.140.down_proj", + "model.layers.31.mlp.experts.140.gate_proj", + "model.layers.31.mlp.experts.140.up_proj", + "model.layers.31.mlp.experts.141.down_proj", + "model.layers.31.mlp.experts.141.gate_proj", + "model.layers.31.mlp.experts.141.up_proj", + "model.layers.31.mlp.experts.142.down_proj", + "model.layers.31.mlp.experts.142.gate_proj", + "model.layers.31.mlp.experts.142.up_proj", + "model.layers.31.mlp.experts.143.down_proj", + "model.layers.31.mlp.experts.143.gate_proj", + "model.layers.31.mlp.experts.143.up_proj", + "model.layers.31.mlp.experts.144.down_proj", + "model.layers.31.mlp.experts.144.gate_proj", + "model.layers.31.mlp.experts.144.up_proj", + "model.layers.31.mlp.experts.145.down_proj", + "model.layers.31.mlp.experts.145.gate_proj", + "model.layers.31.mlp.experts.145.up_proj", + "model.layers.31.mlp.experts.146.down_proj", + "model.layers.31.mlp.experts.146.gate_proj", + "model.layers.31.mlp.experts.146.up_proj", + "model.layers.31.mlp.experts.147.down_proj", + "model.layers.31.mlp.experts.147.gate_proj", + "model.layers.31.mlp.experts.147.up_proj", + "model.layers.31.mlp.experts.148.down_proj", + "model.layers.31.mlp.experts.148.gate_proj", + "model.layers.31.mlp.experts.148.up_proj", + "model.layers.31.mlp.experts.149.down_proj", + "model.layers.31.mlp.experts.149.gate_proj", + "model.layers.31.mlp.experts.149.up_proj", + "model.layers.31.mlp.experts.15.down_proj", + "model.layers.31.mlp.experts.15.gate_proj", + "model.layers.31.mlp.experts.15.up_proj", + "model.layers.31.mlp.experts.150.down_proj", + "model.layers.31.mlp.experts.150.gate_proj", + "model.layers.31.mlp.experts.150.up_proj", + "model.layers.31.mlp.experts.151.down_proj", + "model.layers.31.mlp.experts.151.gate_proj", + "model.layers.31.mlp.experts.151.up_proj", + "model.layers.31.mlp.experts.152.down_proj", + "model.layers.31.mlp.experts.152.gate_proj", + "model.layers.31.mlp.experts.152.up_proj", + "model.layers.31.mlp.experts.153.down_proj", + "model.layers.31.mlp.experts.153.gate_proj", + "model.layers.31.mlp.experts.153.up_proj", + "model.layers.31.mlp.experts.154.down_proj", + "model.layers.31.mlp.experts.154.gate_proj", + "model.layers.31.mlp.experts.154.up_proj", + "model.layers.31.mlp.experts.155.down_proj", + "model.layers.31.mlp.experts.155.gate_proj", + "model.layers.31.mlp.experts.155.up_proj", + "model.layers.31.mlp.experts.156.down_proj", + "model.layers.31.mlp.experts.156.gate_proj", + "model.layers.31.mlp.experts.156.up_proj", + "model.layers.31.mlp.experts.157.down_proj", + "model.layers.31.mlp.experts.157.gate_proj", + "model.layers.31.mlp.experts.157.up_proj", + "model.layers.31.mlp.experts.158.down_proj", + "model.layers.31.mlp.experts.158.gate_proj", + "model.layers.31.mlp.experts.158.up_proj", + "model.layers.31.mlp.experts.159.down_proj", + "model.layers.31.mlp.experts.159.gate_proj", + "model.layers.31.mlp.experts.159.up_proj", + "model.layers.31.mlp.experts.16.down_proj", + "model.layers.31.mlp.experts.16.gate_proj", + "model.layers.31.mlp.experts.16.up_proj", + "model.layers.31.mlp.experts.160.down_proj", + "model.layers.31.mlp.experts.160.gate_proj", + "model.layers.31.mlp.experts.160.up_proj", + "model.layers.31.mlp.experts.161.down_proj", + "model.layers.31.mlp.experts.161.gate_proj", + "model.layers.31.mlp.experts.161.up_proj", + "model.layers.31.mlp.experts.162.down_proj", + "model.layers.31.mlp.experts.162.gate_proj", + "model.layers.31.mlp.experts.162.up_proj", + "model.layers.31.mlp.experts.163.down_proj", + "model.layers.31.mlp.experts.163.gate_proj", + "model.layers.31.mlp.experts.163.up_proj", + "model.layers.31.mlp.experts.164.down_proj", + "model.layers.31.mlp.experts.164.gate_proj", + "model.layers.31.mlp.experts.164.up_proj", + "model.layers.31.mlp.experts.165.down_proj", + "model.layers.31.mlp.experts.165.gate_proj", + "model.layers.31.mlp.experts.165.up_proj", + "model.layers.31.mlp.experts.166.down_proj", + "model.layers.31.mlp.experts.166.gate_proj", + "model.layers.31.mlp.experts.166.up_proj", + "model.layers.31.mlp.experts.167.down_proj", + "model.layers.31.mlp.experts.167.gate_proj", + "model.layers.31.mlp.experts.167.up_proj", + "model.layers.31.mlp.experts.168.down_proj", + "model.layers.31.mlp.experts.168.gate_proj", + "model.layers.31.mlp.experts.168.up_proj", + "model.layers.31.mlp.experts.169.down_proj", + "model.layers.31.mlp.experts.169.gate_proj", + "model.layers.31.mlp.experts.169.up_proj", + "model.layers.31.mlp.experts.17.down_proj", + "model.layers.31.mlp.experts.17.gate_proj", + "model.layers.31.mlp.experts.17.up_proj", + "model.layers.31.mlp.experts.170.down_proj", + "model.layers.31.mlp.experts.170.gate_proj", + "model.layers.31.mlp.experts.170.up_proj", + "model.layers.31.mlp.experts.171.down_proj", + "model.layers.31.mlp.experts.171.gate_proj", + "model.layers.31.mlp.experts.171.up_proj", + "model.layers.31.mlp.experts.172.down_proj", + "model.layers.31.mlp.experts.172.gate_proj", + "model.layers.31.mlp.experts.172.up_proj", + "model.layers.31.mlp.experts.173.down_proj", + "model.layers.31.mlp.experts.173.gate_proj", + "model.layers.31.mlp.experts.173.up_proj", + "model.layers.31.mlp.experts.174.down_proj", + "model.layers.31.mlp.experts.174.gate_proj", + "model.layers.31.mlp.experts.174.up_proj", + "model.layers.31.mlp.experts.175.down_proj", + "model.layers.31.mlp.experts.175.gate_proj", + "model.layers.31.mlp.experts.175.up_proj", + "model.layers.31.mlp.experts.176.down_proj", + "model.layers.31.mlp.experts.176.gate_proj", + "model.layers.31.mlp.experts.176.up_proj", + "model.layers.31.mlp.experts.177.down_proj", + "model.layers.31.mlp.experts.177.gate_proj", + "model.layers.31.mlp.experts.177.up_proj", + "model.layers.31.mlp.experts.178.down_proj", + "model.layers.31.mlp.experts.178.gate_proj", + "model.layers.31.mlp.experts.178.up_proj", + "model.layers.31.mlp.experts.179.down_proj", + "model.layers.31.mlp.experts.179.gate_proj", + "model.layers.31.mlp.experts.179.up_proj", + "model.layers.31.mlp.experts.18.down_proj", + "model.layers.31.mlp.experts.18.gate_proj", + "model.layers.31.mlp.experts.18.up_proj", + "model.layers.31.mlp.experts.180.down_proj", + "model.layers.31.mlp.experts.180.gate_proj", + "model.layers.31.mlp.experts.180.up_proj", + "model.layers.31.mlp.experts.181.down_proj", + "model.layers.31.mlp.experts.181.gate_proj", + "model.layers.31.mlp.experts.181.up_proj", + "model.layers.31.mlp.experts.182.down_proj", + "model.layers.31.mlp.experts.182.gate_proj", + "model.layers.31.mlp.experts.182.up_proj", + "model.layers.31.mlp.experts.183.down_proj", + "model.layers.31.mlp.experts.183.gate_proj", + "model.layers.31.mlp.experts.183.up_proj", + "model.layers.31.mlp.experts.184.down_proj", + "model.layers.31.mlp.experts.184.gate_proj", + "model.layers.31.mlp.experts.184.up_proj", + "model.layers.31.mlp.experts.185.down_proj", + "model.layers.31.mlp.experts.185.gate_proj", + "model.layers.31.mlp.experts.185.up_proj", + "model.layers.31.mlp.experts.186.down_proj", + "model.layers.31.mlp.experts.186.gate_proj", + "model.layers.31.mlp.experts.186.up_proj", + "model.layers.31.mlp.experts.187.down_proj", + "model.layers.31.mlp.experts.187.gate_proj", + "model.layers.31.mlp.experts.187.up_proj", + "model.layers.31.mlp.experts.188.down_proj", + "model.layers.31.mlp.experts.188.gate_proj", + "model.layers.31.mlp.experts.188.up_proj", + "model.layers.31.mlp.experts.189.down_proj", + "model.layers.31.mlp.experts.189.gate_proj", + "model.layers.31.mlp.experts.189.up_proj", + "model.layers.31.mlp.experts.19.down_proj", + "model.layers.31.mlp.experts.19.gate_proj", + "model.layers.31.mlp.experts.19.up_proj", + "model.layers.31.mlp.experts.190.down_proj", + "model.layers.31.mlp.experts.190.gate_proj", + "model.layers.31.mlp.experts.190.up_proj", + "model.layers.31.mlp.experts.191.down_proj", + "model.layers.31.mlp.experts.191.gate_proj", + "model.layers.31.mlp.experts.191.up_proj", + "model.layers.31.mlp.experts.192.down_proj", + "model.layers.31.mlp.experts.192.gate_proj", + "model.layers.31.mlp.experts.192.up_proj", + "model.layers.31.mlp.experts.193.down_proj", + "model.layers.31.mlp.experts.193.gate_proj", + "model.layers.31.mlp.experts.193.up_proj", + "model.layers.31.mlp.experts.194.down_proj", + "model.layers.31.mlp.experts.194.gate_proj", + "model.layers.31.mlp.experts.194.up_proj", + "model.layers.31.mlp.experts.195.down_proj", + "model.layers.31.mlp.experts.195.gate_proj", + "model.layers.31.mlp.experts.195.up_proj", + "model.layers.31.mlp.experts.196.down_proj", + "model.layers.31.mlp.experts.196.gate_proj", + "model.layers.31.mlp.experts.196.up_proj", + "model.layers.31.mlp.experts.197.down_proj", + "model.layers.31.mlp.experts.197.gate_proj", + "model.layers.31.mlp.experts.197.up_proj", + "model.layers.31.mlp.experts.198.down_proj", + "model.layers.31.mlp.experts.198.gate_proj", + "model.layers.31.mlp.experts.198.up_proj", + "model.layers.31.mlp.experts.199.down_proj", + "model.layers.31.mlp.experts.199.gate_proj", + "model.layers.31.mlp.experts.199.up_proj", + "model.layers.31.mlp.experts.2.down_proj", + "model.layers.31.mlp.experts.2.gate_proj", + "model.layers.31.mlp.experts.2.up_proj", + "model.layers.31.mlp.experts.20.down_proj", + "model.layers.31.mlp.experts.20.gate_proj", + "model.layers.31.mlp.experts.20.up_proj", + "model.layers.31.mlp.experts.200.down_proj", + "model.layers.31.mlp.experts.200.gate_proj", + "model.layers.31.mlp.experts.200.up_proj", + "model.layers.31.mlp.experts.201.down_proj", + "model.layers.31.mlp.experts.201.gate_proj", + "model.layers.31.mlp.experts.201.up_proj", + "model.layers.31.mlp.experts.202.down_proj", + "model.layers.31.mlp.experts.202.gate_proj", + "model.layers.31.mlp.experts.202.up_proj", + "model.layers.31.mlp.experts.203.down_proj", + "model.layers.31.mlp.experts.203.gate_proj", + "model.layers.31.mlp.experts.203.up_proj", + "model.layers.31.mlp.experts.204.down_proj", + "model.layers.31.mlp.experts.204.gate_proj", + "model.layers.31.mlp.experts.204.up_proj", + "model.layers.31.mlp.experts.205.down_proj", + "model.layers.31.mlp.experts.205.gate_proj", + "model.layers.31.mlp.experts.205.up_proj", + "model.layers.31.mlp.experts.206.down_proj", + "model.layers.31.mlp.experts.206.gate_proj", + "model.layers.31.mlp.experts.206.up_proj", + "model.layers.31.mlp.experts.207.down_proj", + "model.layers.31.mlp.experts.207.gate_proj", + "model.layers.31.mlp.experts.207.up_proj", + "model.layers.31.mlp.experts.208.down_proj", + "model.layers.31.mlp.experts.208.gate_proj", + "model.layers.31.mlp.experts.208.up_proj", + "model.layers.31.mlp.experts.209.down_proj", + "model.layers.31.mlp.experts.209.gate_proj", + "model.layers.31.mlp.experts.209.up_proj", + "model.layers.31.mlp.experts.21.down_proj", + "model.layers.31.mlp.experts.21.gate_proj", + "model.layers.31.mlp.experts.21.up_proj", + "model.layers.31.mlp.experts.210.down_proj", + "model.layers.31.mlp.experts.210.gate_proj", + "model.layers.31.mlp.experts.210.up_proj", + "model.layers.31.mlp.experts.211.down_proj", + "model.layers.31.mlp.experts.211.gate_proj", + "model.layers.31.mlp.experts.211.up_proj", + "model.layers.31.mlp.experts.212.down_proj", + "model.layers.31.mlp.experts.212.gate_proj", + "model.layers.31.mlp.experts.212.up_proj", + "model.layers.31.mlp.experts.213.down_proj", + "model.layers.31.mlp.experts.213.gate_proj", + "model.layers.31.mlp.experts.213.up_proj", + "model.layers.31.mlp.experts.214.down_proj", + "model.layers.31.mlp.experts.214.gate_proj", + "model.layers.31.mlp.experts.214.up_proj", + "model.layers.31.mlp.experts.215.down_proj", + "model.layers.31.mlp.experts.215.gate_proj", + "model.layers.31.mlp.experts.215.up_proj", + "model.layers.31.mlp.experts.216.down_proj", + "model.layers.31.mlp.experts.216.gate_proj", + "model.layers.31.mlp.experts.216.up_proj", + "model.layers.31.mlp.experts.217.down_proj", + "model.layers.31.mlp.experts.217.gate_proj", + "model.layers.31.mlp.experts.217.up_proj", + "model.layers.31.mlp.experts.218.down_proj", + "model.layers.31.mlp.experts.218.gate_proj", + "model.layers.31.mlp.experts.218.up_proj", + "model.layers.31.mlp.experts.219.down_proj", + "model.layers.31.mlp.experts.219.gate_proj", + "model.layers.31.mlp.experts.219.up_proj", + "model.layers.31.mlp.experts.22.down_proj", + "model.layers.31.mlp.experts.22.gate_proj", + "model.layers.31.mlp.experts.22.up_proj", + "model.layers.31.mlp.experts.220.down_proj", + "model.layers.31.mlp.experts.220.gate_proj", + "model.layers.31.mlp.experts.220.up_proj", + "model.layers.31.mlp.experts.221.down_proj", + "model.layers.31.mlp.experts.221.gate_proj", + "model.layers.31.mlp.experts.221.up_proj", + "model.layers.31.mlp.experts.222.down_proj", + "model.layers.31.mlp.experts.222.gate_proj", + "model.layers.31.mlp.experts.222.up_proj", + "model.layers.31.mlp.experts.223.down_proj", + "model.layers.31.mlp.experts.223.gate_proj", + "model.layers.31.mlp.experts.223.up_proj", + "model.layers.31.mlp.experts.224.down_proj", + "model.layers.31.mlp.experts.224.gate_proj", + "model.layers.31.mlp.experts.224.up_proj", + "model.layers.31.mlp.experts.225.down_proj", + "model.layers.31.mlp.experts.225.gate_proj", + "model.layers.31.mlp.experts.225.up_proj", + "model.layers.31.mlp.experts.226.down_proj", + "model.layers.31.mlp.experts.226.gate_proj", + "model.layers.31.mlp.experts.226.up_proj", + "model.layers.31.mlp.experts.227.down_proj", + "model.layers.31.mlp.experts.227.gate_proj", + "model.layers.31.mlp.experts.227.up_proj", + "model.layers.31.mlp.experts.228.down_proj", + "model.layers.31.mlp.experts.228.gate_proj", + "model.layers.31.mlp.experts.228.up_proj", + "model.layers.31.mlp.experts.229.down_proj", + "model.layers.31.mlp.experts.229.gate_proj", + "model.layers.31.mlp.experts.229.up_proj", + "model.layers.31.mlp.experts.23.down_proj", + "model.layers.31.mlp.experts.23.gate_proj", + "model.layers.31.mlp.experts.23.up_proj", + "model.layers.31.mlp.experts.230.down_proj", + "model.layers.31.mlp.experts.230.gate_proj", + "model.layers.31.mlp.experts.230.up_proj", + "model.layers.31.mlp.experts.231.down_proj", + "model.layers.31.mlp.experts.231.gate_proj", + "model.layers.31.mlp.experts.231.up_proj", + "model.layers.31.mlp.experts.232.down_proj", + "model.layers.31.mlp.experts.232.gate_proj", + "model.layers.31.mlp.experts.232.up_proj", + "model.layers.31.mlp.experts.233.down_proj", + "model.layers.31.mlp.experts.233.gate_proj", + "model.layers.31.mlp.experts.233.up_proj", + "model.layers.31.mlp.experts.234.down_proj", + "model.layers.31.mlp.experts.234.gate_proj", + "model.layers.31.mlp.experts.234.up_proj", + "model.layers.31.mlp.experts.235.down_proj", + "model.layers.31.mlp.experts.235.gate_proj", + "model.layers.31.mlp.experts.235.up_proj", + "model.layers.31.mlp.experts.236.down_proj", + "model.layers.31.mlp.experts.236.gate_proj", + "model.layers.31.mlp.experts.236.up_proj", + "model.layers.31.mlp.experts.237.down_proj", + "model.layers.31.mlp.experts.237.gate_proj", + "model.layers.31.mlp.experts.237.up_proj", + "model.layers.31.mlp.experts.238.down_proj", + "model.layers.31.mlp.experts.238.gate_proj", + "model.layers.31.mlp.experts.238.up_proj", + "model.layers.31.mlp.experts.239.down_proj", + "model.layers.31.mlp.experts.239.gate_proj", + "model.layers.31.mlp.experts.239.up_proj", + "model.layers.31.mlp.experts.24.down_proj", + "model.layers.31.mlp.experts.24.gate_proj", + "model.layers.31.mlp.experts.24.up_proj", + "model.layers.31.mlp.experts.240.down_proj", + "model.layers.31.mlp.experts.240.gate_proj", + "model.layers.31.mlp.experts.240.up_proj", + "model.layers.31.mlp.experts.241.down_proj", + "model.layers.31.mlp.experts.241.gate_proj", + "model.layers.31.mlp.experts.241.up_proj", + "model.layers.31.mlp.experts.242.down_proj", + "model.layers.31.mlp.experts.242.gate_proj", + "model.layers.31.mlp.experts.242.up_proj", + "model.layers.31.mlp.experts.243.down_proj", + "model.layers.31.mlp.experts.243.gate_proj", + "model.layers.31.mlp.experts.243.up_proj", + "model.layers.31.mlp.experts.244.down_proj", + "model.layers.31.mlp.experts.244.gate_proj", + "model.layers.31.mlp.experts.244.up_proj", + "model.layers.31.mlp.experts.245.down_proj", + "model.layers.31.mlp.experts.245.gate_proj", + "model.layers.31.mlp.experts.245.up_proj", + "model.layers.31.mlp.experts.246.down_proj", + "model.layers.31.mlp.experts.246.gate_proj", + "model.layers.31.mlp.experts.246.up_proj", + "model.layers.31.mlp.experts.247.down_proj", + "model.layers.31.mlp.experts.247.gate_proj", + "model.layers.31.mlp.experts.247.up_proj", + "model.layers.31.mlp.experts.248.down_proj", + "model.layers.31.mlp.experts.248.gate_proj", + "model.layers.31.mlp.experts.248.up_proj", + "model.layers.31.mlp.experts.249.down_proj", + "model.layers.31.mlp.experts.249.gate_proj", + "model.layers.31.mlp.experts.249.up_proj", + "model.layers.31.mlp.experts.25.down_proj", + "model.layers.31.mlp.experts.25.gate_proj", + "model.layers.31.mlp.experts.25.up_proj", + "model.layers.31.mlp.experts.250.down_proj", + "model.layers.31.mlp.experts.250.gate_proj", + "model.layers.31.mlp.experts.250.up_proj", + "model.layers.31.mlp.experts.251.down_proj", + "model.layers.31.mlp.experts.251.gate_proj", + "model.layers.31.mlp.experts.251.up_proj", + "model.layers.31.mlp.experts.252.down_proj", + "model.layers.31.mlp.experts.252.gate_proj", + "model.layers.31.mlp.experts.252.up_proj", + "model.layers.31.mlp.experts.253.down_proj", + "model.layers.31.mlp.experts.253.gate_proj", + "model.layers.31.mlp.experts.253.up_proj", + "model.layers.31.mlp.experts.254.down_proj", + "model.layers.31.mlp.experts.254.gate_proj", + "model.layers.31.mlp.experts.254.up_proj", + "model.layers.31.mlp.experts.255.down_proj", + "model.layers.31.mlp.experts.255.gate_proj", + "model.layers.31.mlp.experts.255.up_proj", + "model.layers.31.mlp.experts.26.down_proj", + "model.layers.31.mlp.experts.26.gate_proj", + "model.layers.31.mlp.experts.26.up_proj", + "model.layers.31.mlp.experts.27.down_proj", + "model.layers.31.mlp.experts.27.gate_proj", + "model.layers.31.mlp.experts.27.up_proj", + "model.layers.31.mlp.experts.28.down_proj", + "model.layers.31.mlp.experts.28.gate_proj", + "model.layers.31.mlp.experts.28.up_proj", + "model.layers.31.mlp.experts.29.down_proj", + "model.layers.31.mlp.experts.29.gate_proj", + "model.layers.31.mlp.experts.29.up_proj", + "model.layers.31.mlp.experts.3.down_proj", + "model.layers.31.mlp.experts.3.gate_proj", + "model.layers.31.mlp.experts.3.up_proj", + "model.layers.31.mlp.experts.30.down_proj", + "model.layers.31.mlp.experts.30.gate_proj", + "model.layers.31.mlp.experts.30.up_proj", + "model.layers.31.mlp.experts.31.down_proj", + "model.layers.31.mlp.experts.31.gate_proj", + "model.layers.31.mlp.experts.31.up_proj", + "model.layers.31.mlp.experts.32.down_proj", + "model.layers.31.mlp.experts.32.gate_proj", + "model.layers.31.mlp.experts.32.up_proj", + "model.layers.31.mlp.experts.33.down_proj", + "model.layers.31.mlp.experts.33.gate_proj", + "model.layers.31.mlp.experts.33.up_proj", + "model.layers.31.mlp.experts.34.down_proj", + "model.layers.31.mlp.experts.34.gate_proj", + "model.layers.31.mlp.experts.34.up_proj", + "model.layers.31.mlp.experts.35.down_proj", + "model.layers.31.mlp.experts.35.gate_proj", + "model.layers.31.mlp.experts.35.up_proj", + "model.layers.31.mlp.experts.36.down_proj", + "model.layers.31.mlp.experts.36.gate_proj", + "model.layers.31.mlp.experts.36.up_proj", + "model.layers.31.mlp.experts.37.down_proj", + "model.layers.31.mlp.experts.37.gate_proj", + "model.layers.31.mlp.experts.37.up_proj", + "model.layers.31.mlp.experts.38.down_proj", + "model.layers.31.mlp.experts.38.gate_proj", + "model.layers.31.mlp.experts.38.up_proj", + "model.layers.31.mlp.experts.39.down_proj", + "model.layers.31.mlp.experts.39.gate_proj", + "model.layers.31.mlp.experts.39.up_proj", + "model.layers.31.mlp.experts.4.down_proj", + "model.layers.31.mlp.experts.4.gate_proj", + "model.layers.31.mlp.experts.4.up_proj", + "model.layers.31.mlp.experts.40.down_proj", + "model.layers.31.mlp.experts.40.gate_proj", + "model.layers.31.mlp.experts.40.up_proj", + "model.layers.31.mlp.experts.41.down_proj", + "model.layers.31.mlp.experts.41.gate_proj", + "model.layers.31.mlp.experts.41.up_proj", + "model.layers.31.mlp.experts.42.down_proj", + "model.layers.31.mlp.experts.42.gate_proj", + "model.layers.31.mlp.experts.42.up_proj", + "model.layers.31.mlp.experts.43.down_proj", + "model.layers.31.mlp.experts.43.gate_proj", + "model.layers.31.mlp.experts.43.up_proj", + "model.layers.31.mlp.experts.44.down_proj", + "model.layers.31.mlp.experts.44.gate_proj", + "model.layers.31.mlp.experts.44.up_proj", + "model.layers.31.mlp.experts.45.down_proj", + "model.layers.31.mlp.experts.45.gate_proj", + "model.layers.31.mlp.experts.45.up_proj", + "model.layers.31.mlp.experts.46.down_proj", + "model.layers.31.mlp.experts.46.gate_proj", + "model.layers.31.mlp.experts.46.up_proj", + "model.layers.31.mlp.experts.47.down_proj", + "model.layers.31.mlp.experts.47.gate_proj", + "model.layers.31.mlp.experts.47.up_proj", + "model.layers.31.mlp.experts.48.down_proj", + "model.layers.31.mlp.experts.48.gate_proj", + "model.layers.31.mlp.experts.48.up_proj", + "model.layers.31.mlp.experts.49.down_proj", + "model.layers.31.mlp.experts.49.gate_proj", + "model.layers.31.mlp.experts.49.up_proj", + "model.layers.31.mlp.experts.5.down_proj", + "model.layers.31.mlp.experts.5.gate_proj", + "model.layers.31.mlp.experts.5.up_proj", + "model.layers.31.mlp.experts.50.down_proj", + "model.layers.31.mlp.experts.50.gate_proj", + "model.layers.31.mlp.experts.50.up_proj", + "model.layers.31.mlp.experts.51.down_proj", + "model.layers.31.mlp.experts.51.gate_proj", + "model.layers.31.mlp.experts.51.up_proj", + "model.layers.31.mlp.experts.52.down_proj", + "model.layers.31.mlp.experts.52.gate_proj", + "model.layers.31.mlp.experts.52.up_proj", + "model.layers.31.mlp.experts.53.down_proj", + "model.layers.31.mlp.experts.53.gate_proj", + "model.layers.31.mlp.experts.53.up_proj", + "model.layers.31.mlp.experts.54.down_proj", + "model.layers.31.mlp.experts.54.gate_proj", + "model.layers.31.mlp.experts.54.up_proj", + "model.layers.31.mlp.experts.55.down_proj", + "model.layers.31.mlp.experts.55.gate_proj", + "model.layers.31.mlp.experts.55.up_proj", + "model.layers.31.mlp.experts.56.down_proj", + "model.layers.31.mlp.experts.56.gate_proj", + "model.layers.31.mlp.experts.56.up_proj", + "model.layers.31.mlp.experts.57.down_proj", + "model.layers.31.mlp.experts.57.gate_proj", + "model.layers.31.mlp.experts.57.up_proj", + "model.layers.31.mlp.experts.58.down_proj", + "model.layers.31.mlp.experts.58.gate_proj", + "model.layers.31.mlp.experts.58.up_proj", + "model.layers.31.mlp.experts.59.down_proj", + "model.layers.31.mlp.experts.59.gate_proj", + "model.layers.31.mlp.experts.59.up_proj", + "model.layers.31.mlp.experts.6.down_proj", + "model.layers.31.mlp.experts.6.gate_proj", + "model.layers.31.mlp.experts.6.up_proj", + "model.layers.31.mlp.experts.60.down_proj", + "model.layers.31.mlp.experts.60.gate_proj", + "model.layers.31.mlp.experts.60.up_proj", + "model.layers.31.mlp.experts.61.down_proj", + "model.layers.31.mlp.experts.61.gate_proj", + "model.layers.31.mlp.experts.61.up_proj", + "model.layers.31.mlp.experts.62.down_proj", + "model.layers.31.mlp.experts.62.gate_proj", + "model.layers.31.mlp.experts.62.up_proj", + "model.layers.31.mlp.experts.63.down_proj", + "model.layers.31.mlp.experts.63.gate_proj", + "model.layers.31.mlp.experts.63.up_proj", + "model.layers.31.mlp.experts.64.down_proj", + "model.layers.31.mlp.experts.64.gate_proj", + "model.layers.31.mlp.experts.64.up_proj", + "model.layers.31.mlp.experts.65.down_proj", + "model.layers.31.mlp.experts.65.gate_proj", + "model.layers.31.mlp.experts.65.up_proj", + "model.layers.31.mlp.experts.66.down_proj", + "model.layers.31.mlp.experts.66.gate_proj", + "model.layers.31.mlp.experts.66.up_proj", + "model.layers.31.mlp.experts.67.down_proj", + "model.layers.31.mlp.experts.67.gate_proj", + "model.layers.31.mlp.experts.67.up_proj", + "model.layers.31.mlp.experts.68.down_proj", + "model.layers.31.mlp.experts.68.gate_proj", + "model.layers.31.mlp.experts.68.up_proj", + "model.layers.31.mlp.experts.69.down_proj", + "model.layers.31.mlp.experts.69.gate_proj", + "model.layers.31.mlp.experts.69.up_proj", + "model.layers.31.mlp.experts.7.down_proj", + "model.layers.31.mlp.experts.7.gate_proj", + "model.layers.31.mlp.experts.7.up_proj", + "model.layers.31.mlp.experts.70.down_proj", + "model.layers.31.mlp.experts.70.gate_proj", + "model.layers.31.mlp.experts.70.up_proj", + "model.layers.31.mlp.experts.71.down_proj", + "model.layers.31.mlp.experts.71.gate_proj", + "model.layers.31.mlp.experts.71.up_proj", + "model.layers.31.mlp.experts.72.down_proj", + "model.layers.31.mlp.experts.72.gate_proj", + "model.layers.31.mlp.experts.72.up_proj", + "model.layers.31.mlp.experts.73.down_proj", + "model.layers.31.mlp.experts.73.gate_proj", + "model.layers.31.mlp.experts.73.up_proj", + "model.layers.31.mlp.experts.74.down_proj", + "model.layers.31.mlp.experts.74.gate_proj", + "model.layers.31.mlp.experts.74.up_proj", + "model.layers.31.mlp.experts.75.down_proj", + "model.layers.31.mlp.experts.75.gate_proj", + "model.layers.31.mlp.experts.75.up_proj", + "model.layers.31.mlp.experts.76.down_proj", + "model.layers.31.mlp.experts.76.gate_proj", + "model.layers.31.mlp.experts.76.up_proj", + "model.layers.31.mlp.experts.77.down_proj", + "model.layers.31.mlp.experts.77.gate_proj", + "model.layers.31.mlp.experts.77.up_proj", + "model.layers.31.mlp.experts.78.down_proj", + "model.layers.31.mlp.experts.78.gate_proj", + "model.layers.31.mlp.experts.78.up_proj", + "model.layers.31.mlp.experts.79.down_proj", + "model.layers.31.mlp.experts.79.gate_proj", + "model.layers.31.mlp.experts.79.up_proj", + "model.layers.31.mlp.experts.8.down_proj", + "model.layers.31.mlp.experts.8.gate_proj", + "model.layers.31.mlp.experts.8.up_proj", + "model.layers.31.mlp.experts.80.down_proj", + "model.layers.31.mlp.experts.80.gate_proj", + "model.layers.31.mlp.experts.80.up_proj", + "model.layers.31.mlp.experts.81.down_proj", + "model.layers.31.mlp.experts.81.gate_proj", + "model.layers.31.mlp.experts.81.up_proj", + "model.layers.31.mlp.experts.82.down_proj", + "model.layers.31.mlp.experts.82.gate_proj", + "model.layers.31.mlp.experts.82.up_proj", + "model.layers.31.mlp.experts.83.down_proj", + "model.layers.31.mlp.experts.83.gate_proj", + "model.layers.31.mlp.experts.83.up_proj", + "model.layers.31.mlp.experts.84.down_proj", + "model.layers.31.mlp.experts.84.gate_proj", + "model.layers.31.mlp.experts.84.up_proj", + "model.layers.31.mlp.experts.85.down_proj", + "model.layers.31.mlp.experts.85.gate_proj", + "model.layers.31.mlp.experts.85.up_proj", + "model.layers.31.mlp.experts.86.down_proj", + "model.layers.31.mlp.experts.86.gate_proj", + "model.layers.31.mlp.experts.86.up_proj", + "model.layers.31.mlp.experts.87.down_proj", + "model.layers.31.mlp.experts.87.gate_proj", + "model.layers.31.mlp.experts.87.up_proj", + "model.layers.31.mlp.experts.88.down_proj", + "model.layers.31.mlp.experts.88.gate_proj", + "model.layers.31.mlp.experts.88.up_proj", + "model.layers.31.mlp.experts.89.down_proj", + "model.layers.31.mlp.experts.89.gate_proj", + "model.layers.31.mlp.experts.89.up_proj", + "model.layers.31.mlp.experts.9.down_proj", + "model.layers.31.mlp.experts.9.gate_proj", + "model.layers.31.mlp.experts.9.up_proj", + "model.layers.31.mlp.experts.90.down_proj", + "model.layers.31.mlp.experts.90.gate_proj", + "model.layers.31.mlp.experts.90.up_proj", + "model.layers.31.mlp.experts.91.down_proj", + "model.layers.31.mlp.experts.91.gate_proj", + "model.layers.31.mlp.experts.91.up_proj", + "model.layers.31.mlp.experts.92.down_proj", + "model.layers.31.mlp.experts.92.gate_proj", + "model.layers.31.mlp.experts.92.up_proj", + "model.layers.31.mlp.experts.93.down_proj", + "model.layers.31.mlp.experts.93.gate_proj", + "model.layers.31.mlp.experts.93.up_proj", + "model.layers.31.mlp.experts.94.down_proj", + "model.layers.31.mlp.experts.94.gate_proj", + "model.layers.31.mlp.experts.94.up_proj", + "model.layers.31.mlp.experts.95.down_proj", + "model.layers.31.mlp.experts.95.gate_proj", + "model.layers.31.mlp.experts.95.up_proj", + "model.layers.31.mlp.experts.96.down_proj", + "model.layers.31.mlp.experts.96.gate_proj", + "model.layers.31.mlp.experts.96.up_proj", + "model.layers.31.mlp.experts.97.down_proj", + "model.layers.31.mlp.experts.97.gate_proj", + "model.layers.31.mlp.experts.97.up_proj", + "model.layers.31.mlp.experts.98.down_proj", + "model.layers.31.mlp.experts.98.gate_proj", + "model.layers.31.mlp.experts.98.up_proj", + "model.layers.31.mlp.experts.99.down_proj", + "model.layers.31.mlp.experts.99.gate_proj", + "model.layers.31.mlp.experts.99.up_proj", + "model.layers.32.mlp.experts.0.down_proj", + "model.layers.32.mlp.experts.0.gate_proj", + "model.layers.32.mlp.experts.0.up_proj", + "model.layers.32.mlp.experts.1.down_proj", + "model.layers.32.mlp.experts.1.gate_proj", + "model.layers.32.mlp.experts.1.up_proj", + "model.layers.32.mlp.experts.10.down_proj", + "model.layers.32.mlp.experts.10.gate_proj", + "model.layers.32.mlp.experts.10.up_proj", + "model.layers.32.mlp.experts.100.down_proj", + "model.layers.32.mlp.experts.100.gate_proj", + "model.layers.32.mlp.experts.100.up_proj", + "model.layers.32.mlp.experts.101.down_proj", + "model.layers.32.mlp.experts.101.gate_proj", + "model.layers.32.mlp.experts.101.up_proj", + "model.layers.32.mlp.experts.102.down_proj", + "model.layers.32.mlp.experts.102.gate_proj", + "model.layers.32.mlp.experts.102.up_proj", + "model.layers.32.mlp.experts.103.down_proj", + "model.layers.32.mlp.experts.103.gate_proj", + "model.layers.32.mlp.experts.103.up_proj", + "model.layers.32.mlp.experts.104.down_proj", + "model.layers.32.mlp.experts.104.gate_proj", + "model.layers.32.mlp.experts.104.up_proj", + "model.layers.32.mlp.experts.105.down_proj", + "model.layers.32.mlp.experts.105.gate_proj", + "model.layers.32.mlp.experts.105.up_proj", + "model.layers.32.mlp.experts.106.down_proj", + "model.layers.32.mlp.experts.106.gate_proj", + "model.layers.32.mlp.experts.106.up_proj", + "model.layers.32.mlp.experts.107.down_proj", + "model.layers.32.mlp.experts.107.gate_proj", + "model.layers.32.mlp.experts.107.up_proj", + "model.layers.32.mlp.experts.108.down_proj", + "model.layers.32.mlp.experts.108.gate_proj", + "model.layers.32.mlp.experts.108.up_proj", + "model.layers.32.mlp.experts.109.down_proj", + "model.layers.32.mlp.experts.109.gate_proj", + "model.layers.32.mlp.experts.109.up_proj", + "model.layers.32.mlp.experts.11.down_proj", + "model.layers.32.mlp.experts.11.gate_proj", + "model.layers.32.mlp.experts.11.up_proj", + "model.layers.32.mlp.experts.110.down_proj", + "model.layers.32.mlp.experts.110.gate_proj", + "model.layers.32.mlp.experts.110.up_proj", + "model.layers.32.mlp.experts.111.down_proj", + "model.layers.32.mlp.experts.111.gate_proj", + "model.layers.32.mlp.experts.111.up_proj", + "model.layers.32.mlp.experts.112.down_proj", + "model.layers.32.mlp.experts.112.gate_proj", + "model.layers.32.mlp.experts.112.up_proj", + "model.layers.32.mlp.experts.113.down_proj", + "model.layers.32.mlp.experts.113.gate_proj", + "model.layers.32.mlp.experts.113.up_proj", + "model.layers.32.mlp.experts.114.down_proj", + "model.layers.32.mlp.experts.114.gate_proj", + "model.layers.32.mlp.experts.114.up_proj", + "model.layers.32.mlp.experts.115.down_proj", + "model.layers.32.mlp.experts.115.gate_proj", + "model.layers.32.mlp.experts.115.up_proj", + "model.layers.32.mlp.experts.116.down_proj", + "model.layers.32.mlp.experts.116.gate_proj", + "model.layers.32.mlp.experts.116.up_proj", + "model.layers.32.mlp.experts.117.down_proj", + "model.layers.32.mlp.experts.117.gate_proj", + "model.layers.32.mlp.experts.117.up_proj", + "model.layers.32.mlp.experts.118.down_proj", + "model.layers.32.mlp.experts.118.gate_proj", + "model.layers.32.mlp.experts.118.up_proj", + "model.layers.32.mlp.experts.119.down_proj", + "model.layers.32.mlp.experts.119.gate_proj", + "model.layers.32.mlp.experts.119.up_proj", + "model.layers.32.mlp.experts.12.down_proj", + "model.layers.32.mlp.experts.12.gate_proj", + "model.layers.32.mlp.experts.12.up_proj", + "model.layers.32.mlp.experts.120.down_proj", + "model.layers.32.mlp.experts.120.gate_proj", + "model.layers.32.mlp.experts.120.up_proj", + "model.layers.32.mlp.experts.121.down_proj", + "model.layers.32.mlp.experts.121.gate_proj", + "model.layers.32.mlp.experts.121.up_proj", + "model.layers.32.mlp.experts.122.down_proj", + "model.layers.32.mlp.experts.122.gate_proj", + "model.layers.32.mlp.experts.122.up_proj", + "model.layers.32.mlp.experts.123.down_proj", + "model.layers.32.mlp.experts.123.gate_proj", + "model.layers.32.mlp.experts.123.up_proj", + "model.layers.32.mlp.experts.124.down_proj", + "model.layers.32.mlp.experts.124.gate_proj", + "model.layers.32.mlp.experts.124.up_proj", + "model.layers.32.mlp.experts.125.down_proj", + "model.layers.32.mlp.experts.125.gate_proj", + "model.layers.32.mlp.experts.125.up_proj", + "model.layers.32.mlp.experts.126.down_proj", + "model.layers.32.mlp.experts.126.gate_proj", + "model.layers.32.mlp.experts.126.up_proj", + "model.layers.32.mlp.experts.127.down_proj", + "model.layers.32.mlp.experts.127.gate_proj", + "model.layers.32.mlp.experts.127.up_proj", + "model.layers.32.mlp.experts.128.down_proj", + "model.layers.32.mlp.experts.128.gate_proj", + "model.layers.32.mlp.experts.128.up_proj", + "model.layers.32.mlp.experts.129.down_proj", + "model.layers.32.mlp.experts.129.gate_proj", + "model.layers.32.mlp.experts.129.up_proj", + "model.layers.32.mlp.experts.13.down_proj", + "model.layers.32.mlp.experts.13.gate_proj", + "model.layers.32.mlp.experts.13.up_proj", + "model.layers.32.mlp.experts.130.down_proj", + "model.layers.32.mlp.experts.130.gate_proj", + "model.layers.32.mlp.experts.130.up_proj", + "model.layers.32.mlp.experts.131.down_proj", + "model.layers.32.mlp.experts.131.gate_proj", + "model.layers.32.mlp.experts.131.up_proj", + "model.layers.32.mlp.experts.132.down_proj", + "model.layers.32.mlp.experts.132.gate_proj", + "model.layers.32.mlp.experts.132.up_proj", + "model.layers.32.mlp.experts.133.down_proj", + "model.layers.32.mlp.experts.133.gate_proj", + "model.layers.32.mlp.experts.133.up_proj", + "model.layers.32.mlp.experts.134.down_proj", + "model.layers.32.mlp.experts.134.gate_proj", + "model.layers.32.mlp.experts.134.up_proj", + "model.layers.32.mlp.experts.135.down_proj", + "model.layers.32.mlp.experts.135.gate_proj", + "model.layers.32.mlp.experts.135.up_proj", + "model.layers.32.mlp.experts.136.down_proj", + "model.layers.32.mlp.experts.136.gate_proj", + "model.layers.32.mlp.experts.136.up_proj", + "model.layers.32.mlp.experts.137.down_proj", + "model.layers.32.mlp.experts.137.gate_proj", + "model.layers.32.mlp.experts.137.up_proj", + "model.layers.32.mlp.experts.138.down_proj", + "model.layers.32.mlp.experts.138.gate_proj", + "model.layers.32.mlp.experts.138.up_proj", + "model.layers.32.mlp.experts.139.down_proj", + "model.layers.32.mlp.experts.139.gate_proj", + "model.layers.32.mlp.experts.139.up_proj", + "model.layers.32.mlp.experts.14.down_proj", + "model.layers.32.mlp.experts.14.gate_proj", + "model.layers.32.mlp.experts.14.up_proj", + "model.layers.32.mlp.experts.140.down_proj", + "model.layers.32.mlp.experts.140.gate_proj", + "model.layers.32.mlp.experts.140.up_proj", + "model.layers.32.mlp.experts.141.down_proj", + "model.layers.32.mlp.experts.141.gate_proj", + "model.layers.32.mlp.experts.141.up_proj", + "model.layers.32.mlp.experts.142.down_proj", + "model.layers.32.mlp.experts.142.gate_proj", + "model.layers.32.mlp.experts.142.up_proj", + "model.layers.32.mlp.experts.143.down_proj", + "model.layers.32.mlp.experts.143.gate_proj", + "model.layers.32.mlp.experts.143.up_proj", + "model.layers.32.mlp.experts.144.down_proj", + "model.layers.32.mlp.experts.144.gate_proj", + "model.layers.32.mlp.experts.144.up_proj", + "model.layers.32.mlp.experts.145.down_proj", + "model.layers.32.mlp.experts.145.gate_proj", + "model.layers.32.mlp.experts.145.up_proj", + "model.layers.32.mlp.experts.146.down_proj", + "model.layers.32.mlp.experts.146.gate_proj", + "model.layers.32.mlp.experts.146.up_proj", + "model.layers.32.mlp.experts.147.down_proj", + "model.layers.32.mlp.experts.147.gate_proj", + "model.layers.32.mlp.experts.147.up_proj", + "model.layers.32.mlp.experts.148.down_proj", + "model.layers.32.mlp.experts.148.gate_proj", + "model.layers.32.mlp.experts.148.up_proj", + "model.layers.32.mlp.experts.149.down_proj", + "model.layers.32.mlp.experts.149.gate_proj", + "model.layers.32.mlp.experts.149.up_proj", + "model.layers.32.mlp.experts.15.down_proj", + "model.layers.32.mlp.experts.15.gate_proj", + "model.layers.32.mlp.experts.15.up_proj", + "model.layers.32.mlp.experts.150.down_proj", + "model.layers.32.mlp.experts.150.gate_proj", + "model.layers.32.mlp.experts.150.up_proj", + "model.layers.32.mlp.experts.151.down_proj", + "model.layers.32.mlp.experts.151.gate_proj", + "model.layers.32.mlp.experts.151.up_proj", + "model.layers.32.mlp.experts.152.down_proj", + "model.layers.32.mlp.experts.152.gate_proj", + "model.layers.32.mlp.experts.152.up_proj", + "model.layers.32.mlp.experts.153.down_proj", + "model.layers.32.mlp.experts.153.gate_proj", + "model.layers.32.mlp.experts.153.up_proj", + "model.layers.32.mlp.experts.154.down_proj", + "model.layers.32.mlp.experts.154.gate_proj", + "model.layers.32.mlp.experts.154.up_proj", + "model.layers.32.mlp.experts.155.down_proj", + "model.layers.32.mlp.experts.155.gate_proj", + "model.layers.32.mlp.experts.155.up_proj", + "model.layers.32.mlp.experts.156.down_proj", + "model.layers.32.mlp.experts.156.gate_proj", + "model.layers.32.mlp.experts.156.up_proj", + "model.layers.32.mlp.experts.157.down_proj", + "model.layers.32.mlp.experts.157.gate_proj", + "model.layers.32.mlp.experts.157.up_proj", + "model.layers.32.mlp.experts.158.down_proj", + "model.layers.32.mlp.experts.158.gate_proj", + "model.layers.32.mlp.experts.158.up_proj", + "model.layers.32.mlp.experts.159.down_proj", + "model.layers.32.mlp.experts.159.gate_proj", + "model.layers.32.mlp.experts.159.up_proj", + "model.layers.32.mlp.experts.16.down_proj", + "model.layers.32.mlp.experts.16.gate_proj", + "model.layers.32.mlp.experts.16.up_proj", + "model.layers.32.mlp.experts.160.down_proj", + "model.layers.32.mlp.experts.160.gate_proj", + "model.layers.32.mlp.experts.160.up_proj", + "model.layers.32.mlp.experts.161.down_proj", + "model.layers.32.mlp.experts.161.gate_proj", + "model.layers.32.mlp.experts.161.up_proj", + "model.layers.32.mlp.experts.162.down_proj", + "model.layers.32.mlp.experts.162.gate_proj", + "model.layers.32.mlp.experts.162.up_proj", + "model.layers.32.mlp.experts.163.down_proj", + "model.layers.32.mlp.experts.163.gate_proj", + "model.layers.32.mlp.experts.163.up_proj", + "model.layers.32.mlp.experts.164.down_proj", + "model.layers.32.mlp.experts.164.gate_proj", + "model.layers.32.mlp.experts.164.up_proj", + "model.layers.32.mlp.experts.165.down_proj", + "model.layers.32.mlp.experts.165.gate_proj", + "model.layers.32.mlp.experts.165.up_proj", + "model.layers.32.mlp.experts.166.down_proj", + "model.layers.32.mlp.experts.166.gate_proj", + "model.layers.32.mlp.experts.166.up_proj", + "model.layers.32.mlp.experts.167.down_proj", + "model.layers.32.mlp.experts.167.gate_proj", + "model.layers.32.mlp.experts.167.up_proj", + "model.layers.32.mlp.experts.168.down_proj", + "model.layers.32.mlp.experts.168.gate_proj", + "model.layers.32.mlp.experts.168.up_proj", + "model.layers.32.mlp.experts.169.down_proj", + "model.layers.32.mlp.experts.169.gate_proj", + "model.layers.32.mlp.experts.169.up_proj", + "model.layers.32.mlp.experts.17.down_proj", + "model.layers.32.mlp.experts.17.gate_proj", + "model.layers.32.mlp.experts.17.up_proj", + "model.layers.32.mlp.experts.170.down_proj", + "model.layers.32.mlp.experts.170.gate_proj", + "model.layers.32.mlp.experts.170.up_proj", + "model.layers.32.mlp.experts.171.down_proj", + "model.layers.32.mlp.experts.171.gate_proj", + "model.layers.32.mlp.experts.171.up_proj", + "model.layers.32.mlp.experts.172.down_proj", + "model.layers.32.mlp.experts.172.gate_proj", + "model.layers.32.mlp.experts.172.up_proj", + "model.layers.32.mlp.experts.173.down_proj", + "model.layers.32.mlp.experts.173.gate_proj", + "model.layers.32.mlp.experts.173.up_proj", + "model.layers.32.mlp.experts.174.down_proj", + "model.layers.32.mlp.experts.174.gate_proj", + "model.layers.32.mlp.experts.174.up_proj", + "model.layers.32.mlp.experts.175.down_proj", + "model.layers.32.mlp.experts.175.gate_proj", + "model.layers.32.mlp.experts.175.up_proj", + "model.layers.32.mlp.experts.176.down_proj", + "model.layers.32.mlp.experts.176.gate_proj", + "model.layers.32.mlp.experts.176.up_proj", + "model.layers.32.mlp.experts.177.down_proj", + "model.layers.32.mlp.experts.177.gate_proj", + "model.layers.32.mlp.experts.177.up_proj", + "model.layers.32.mlp.experts.178.down_proj", + "model.layers.32.mlp.experts.178.gate_proj", + "model.layers.32.mlp.experts.178.up_proj", + "model.layers.32.mlp.experts.179.down_proj", + "model.layers.32.mlp.experts.179.gate_proj", + "model.layers.32.mlp.experts.179.up_proj", + "model.layers.32.mlp.experts.18.down_proj", + "model.layers.32.mlp.experts.18.gate_proj", + "model.layers.32.mlp.experts.18.up_proj", + "model.layers.32.mlp.experts.180.down_proj", + "model.layers.32.mlp.experts.180.gate_proj", + "model.layers.32.mlp.experts.180.up_proj", + "model.layers.32.mlp.experts.181.down_proj", + "model.layers.32.mlp.experts.181.gate_proj", + "model.layers.32.mlp.experts.181.up_proj", + "model.layers.32.mlp.experts.182.down_proj", + "model.layers.32.mlp.experts.182.gate_proj", + "model.layers.32.mlp.experts.182.up_proj", + "model.layers.32.mlp.experts.183.down_proj", + "model.layers.32.mlp.experts.183.gate_proj", + "model.layers.32.mlp.experts.183.up_proj", + "model.layers.32.mlp.experts.184.down_proj", + "model.layers.32.mlp.experts.184.gate_proj", + "model.layers.32.mlp.experts.184.up_proj", + "model.layers.32.mlp.experts.185.down_proj", + "model.layers.32.mlp.experts.185.gate_proj", + "model.layers.32.mlp.experts.185.up_proj", + "model.layers.32.mlp.experts.186.down_proj", + "model.layers.32.mlp.experts.186.gate_proj", + "model.layers.32.mlp.experts.186.up_proj", + "model.layers.32.mlp.experts.187.down_proj", + "model.layers.32.mlp.experts.187.gate_proj", + "model.layers.32.mlp.experts.187.up_proj", + "model.layers.32.mlp.experts.188.down_proj", + "model.layers.32.mlp.experts.188.gate_proj", + "model.layers.32.mlp.experts.188.up_proj", + "model.layers.32.mlp.experts.189.down_proj", + "model.layers.32.mlp.experts.189.gate_proj", + "model.layers.32.mlp.experts.189.up_proj", + "model.layers.32.mlp.experts.19.down_proj", + "model.layers.32.mlp.experts.19.gate_proj", + "model.layers.32.mlp.experts.19.up_proj", + "model.layers.32.mlp.experts.190.down_proj", + "model.layers.32.mlp.experts.190.gate_proj", + "model.layers.32.mlp.experts.190.up_proj", + "model.layers.32.mlp.experts.191.down_proj", + "model.layers.32.mlp.experts.191.gate_proj", + "model.layers.32.mlp.experts.191.up_proj", + "model.layers.32.mlp.experts.192.down_proj", + "model.layers.32.mlp.experts.192.gate_proj", + "model.layers.32.mlp.experts.192.up_proj", + "model.layers.32.mlp.experts.193.down_proj", + "model.layers.32.mlp.experts.193.gate_proj", + "model.layers.32.mlp.experts.193.up_proj", + "model.layers.32.mlp.experts.194.down_proj", + "model.layers.32.mlp.experts.194.gate_proj", + "model.layers.32.mlp.experts.194.up_proj", + "model.layers.32.mlp.experts.195.down_proj", + "model.layers.32.mlp.experts.195.gate_proj", + "model.layers.32.mlp.experts.195.up_proj", + "model.layers.32.mlp.experts.196.down_proj", + "model.layers.32.mlp.experts.196.gate_proj", + "model.layers.32.mlp.experts.196.up_proj", + "model.layers.32.mlp.experts.197.down_proj", + "model.layers.32.mlp.experts.197.gate_proj", + "model.layers.32.mlp.experts.197.up_proj", + "model.layers.32.mlp.experts.198.down_proj", + "model.layers.32.mlp.experts.198.gate_proj", + "model.layers.32.mlp.experts.198.up_proj", + "model.layers.32.mlp.experts.199.down_proj", + "model.layers.32.mlp.experts.199.gate_proj", + "model.layers.32.mlp.experts.199.up_proj", + "model.layers.32.mlp.experts.2.down_proj", + "model.layers.32.mlp.experts.2.gate_proj", + "model.layers.32.mlp.experts.2.up_proj", + "model.layers.32.mlp.experts.20.down_proj", + "model.layers.32.mlp.experts.20.gate_proj", + "model.layers.32.mlp.experts.20.up_proj", + "model.layers.32.mlp.experts.200.down_proj", + "model.layers.32.mlp.experts.200.gate_proj", + "model.layers.32.mlp.experts.200.up_proj", + "model.layers.32.mlp.experts.201.down_proj", + "model.layers.32.mlp.experts.201.gate_proj", + "model.layers.32.mlp.experts.201.up_proj", + "model.layers.32.mlp.experts.202.down_proj", + "model.layers.32.mlp.experts.202.gate_proj", + "model.layers.32.mlp.experts.202.up_proj", + "model.layers.32.mlp.experts.203.down_proj", + "model.layers.32.mlp.experts.203.gate_proj", + "model.layers.32.mlp.experts.203.up_proj", + "model.layers.32.mlp.experts.204.down_proj", + "model.layers.32.mlp.experts.204.gate_proj", + "model.layers.32.mlp.experts.204.up_proj", + "model.layers.32.mlp.experts.205.down_proj", + "model.layers.32.mlp.experts.205.gate_proj", + "model.layers.32.mlp.experts.205.up_proj", + "model.layers.32.mlp.experts.206.down_proj", + "model.layers.32.mlp.experts.206.gate_proj", + "model.layers.32.mlp.experts.206.up_proj", + "model.layers.32.mlp.experts.207.down_proj", + "model.layers.32.mlp.experts.207.gate_proj", + "model.layers.32.mlp.experts.207.up_proj", + "model.layers.32.mlp.experts.208.down_proj", + "model.layers.32.mlp.experts.208.gate_proj", + "model.layers.32.mlp.experts.208.up_proj", + "model.layers.32.mlp.experts.209.down_proj", + "model.layers.32.mlp.experts.209.gate_proj", + "model.layers.32.mlp.experts.209.up_proj", + "model.layers.32.mlp.experts.21.down_proj", + "model.layers.32.mlp.experts.21.gate_proj", + "model.layers.32.mlp.experts.21.up_proj", + "model.layers.32.mlp.experts.210.down_proj", + "model.layers.32.mlp.experts.210.gate_proj", + "model.layers.32.mlp.experts.210.up_proj", + "model.layers.32.mlp.experts.211.down_proj", + "model.layers.32.mlp.experts.211.gate_proj", + "model.layers.32.mlp.experts.211.up_proj", + "model.layers.32.mlp.experts.212.down_proj", + "model.layers.32.mlp.experts.212.gate_proj", + "model.layers.32.mlp.experts.212.up_proj", + "model.layers.32.mlp.experts.213.down_proj", + "model.layers.32.mlp.experts.213.gate_proj", + "model.layers.32.mlp.experts.213.up_proj", + "model.layers.32.mlp.experts.214.down_proj", + "model.layers.32.mlp.experts.214.gate_proj", + "model.layers.32.mlp.experts.214.up_proj", + "model.layers.32.mlp.experts.215.down_proj", + "model.layers.32.mlp.experts.215.gate_proj", + "model.layers.32.mlp.experts.215.up_proj", + "model.layers.32.mlp.experts.216.down_proj", + "model.layers.32.mlp.experts.216.gate_proj", + "model.layers.32.mlp.experts.216.up_proj", + "model.layers.32.mlp.experts.217.down_proj", + "model.layers.32.mlp.experts.217.gate_proj", + "model.layers.32.mlp.experts.217.up_proj", + "model.layers.32.mlp.experts.218.down_proj", + "model.layers.32.mlp.experts.218.gate_proj", + "model.layers.32.mlp.experts.218.up_proj", + "model.layers.32.mlp.experts.219.down_proj", + "model.layers.32.mlp.experts.219.gate_proj", + "model.layers.32.mlp.experts.219.up_proj", + "model.layers.32.mlp.experts.22.down_proj", + "model.layers.32.mlp.experts.22.gate_proj", + "model.layers.32.mlp.experts.22.up_proj", + "model.layers.32.mlp.experts.220.down_proj", + "model.layers.32.mlp.experts.220.gate_proj", + "model.layers.32.mlp.experts.220.up_proj", + "model.layers.32.mlp.experts.221.down_proj", + "model.layers.32.mlp.experts.221.gate_proj", + "model.layers.32.mlp.experts.221.up_proj", + "model.layers.32.mlp.experts.222.down_proj", + "model.layers.32.mlp.experts.222.gate_proj", + "model.layers.32.mlp.experts.222.up_proj", + "model.layers.32.mlp.experts.223.down_proj", + "model.layers.32.mlp.experts.223.gate_proj", + "model.layers.32.mlp.experts.223.up_proj", + "model.layers.32.mlp.experts.224.down_proj", + "model.layers.32.mlp.experts.224.gate_proj", + "model.layers.32.mlp.experts.224.up_proj", + "model.layers.32.mlp.experts.225.down_proj", + "model.layers.32.mlp.experts.225.gate_proj", + "model.layers.32.mlp.experts.225.up_proj", + "model.layers.32.mlp.experts.226.down_proj", + "model.layers.32.mlp.experts.226.gate_proj", + "model.layers.32.mlp.experts.226.up_proj", + "model.layers.32.mlp.experts.227.down_proj", + "model.layers.32.mlp.experts.227.gate_proj", + "model.layers.32.mlp.experts.227.up_proj", + "model.layers.32.mlp.experts.228.down_proj", + "model.layers.32.mlp.experts.228.gate_proj", + "model.layers.32.mlp.experts.228.up_proj", + "model.layers.32.mlp.experts.229.down_proj", + "model.layers.32.mlp.experts.229.gate_proj", + "model.layers.32.mlp.experts.229.up_proj", + "model.layers.32.mlp.experts.23.down_proj", + "model.layers.32.mlp.experts.23.gate_proj", + "model.layers.32.mlp.experts.23.up_proj", + "model.layers.32.mlp.experts.230.down_proj", + "model.layers.32.mlp.experts.230.gate_proj", + "model.layers.32.mlp.experts.230.up_proj", + "model.layers.32.mlp.experts.231.down_proj", + "model.layers.32.mlp.experts.231.gate_proj", + "model.layers.32.mlp.experts.231.up_proj", + "model.layers.32.mlp.experts.232.down_proj", + "model.layers.32.mlp.experts.232.gate_proj", + "model.layers.32.mlp.experts.232.up_proj", + "model.layers.32.mlp.experts.233.down_proj", + "model.layers.32.mlp.experts.233.gate_proj", + "model.layers.32.mlp.experts.233.up_proj", + "model.layers.32.mlp.experts.234.down_proj", + "model.layers.32.mlp.experts.234.gate_proj", + "model.layers.32.mlp.experts.234.up_proj", + "model.layers.32.mlp.experts.235.down_proj", + "model.layers.32.mlp.experts.235.gate_proj", + "model.layers.32.mlp.experts.235.up_proj", + "model.layers.32.mlp.experts.236.down_proj", + "model.layers.32.mlp.experts.236.gate_proj", + "model.layers.32.mlp.experts.236.up_proj", + "model.layers.32.mlp.experts.237.down_proj", + "model.layers.32.mlp.experts.237.gate_proj", + "model.layers.32.mlp.experts.237.up_proj", + "model.layers.32.mlp.experts.238.down_proj", + "model.layers.32.mlp.experts.238.gate_proj", + "model.layers.32.mlp.experts.238.up_proj", + "model.layers.32.mlp.experts.239.down_proj", + "model.layers.32.mlp.experts.239.gate_proj", + "model.layers.32.mlp.experts.239.up_proj", + "model.layers.32.mlp.experts.24.down_proj", + "model.layers.32.mlp.experts.24.gate_proj", + "model.layers.32.mlp.experts.24.up_proj", + "model.layers.32.mlp.experts.240.down_proj", + "model.layers.32.mlp.experts.240.gate_proj", + "model.layers.32.mlp.experts.240.up_proj", + "model.layers.32.mlp.experts.241.down_proj", + "model.layers.32.mlp.experts.241.gate_proj", + "model.layers.32.mlp.experts.241.up_proj", + "model.layers.32.mlp.experts.242.down_proj", + "model.layers.32.mlp.experts.242.gate_proj", + "model.layers.32.mlp.experts.242.up_proj", + "model.layers.32.mlp.experts.243.down_proj", + "model.layers.32.mlp.experts.243.gate_proj", + "model.layers.32.mlp.experts.243.up_proj", + "model.layers.32.mlp.experts.244.down_proj", + "model.layers.32.mlp.experts.244.gate_proj", + "model.layers.32.mlp.experts.244.up_proj", + "model.layers.32.mlp.experts.245.down_proj", + "model.layers.32.mlp.experts.245.gate_proj", + "model.layers.32.mlp.experts.245.up_proj", + "model.layers.32.mlp.experts.246.down_proj", + "model.layers.32.mlp.experts.246.gate_proj", + "model.layers.32.mlp.experts.246.up_proj", + "model.layers.32.mlp.experts.247.down_proj", + "model.layers.32.mlp.experts.247.gate_proj", + "model.layers.32.mlp.experts.247.up_proj", + "model.layers.32.mlp.experts.248.down_proj", + "model.layers.32.mlp.experts.248.gate_proj", + "model.layers.32.mlp.experts.248.up_proj", + "model.layers.32.mlp.experts.249.down_proj", + "model.layers.32.mlp.experts.249.gate_proj", + "model.layers.32.mlp.experts.249.up_proj", + "model.layers.32.mlp.experts.25.down_proj", + "model.layers.32.mlp.experts.25.gate_proj", + "model.layers.32.mlp.experts.25.up_proj", + "model.layers.32.mlp.experts.250.down_proj", + "model.layers.32.mlp.experts.250.gate_proj", + "model.layers.32.mlp.experts.250.up_proj", + "model.layers.32.mlp.experts.251.down_proj", + "model.layers.32.mlp.experts.251.gate_proj", + "model.layers.32.mlp.experts.251.up_proj", + "model.layers.32.mlp.experts.252.down_proj", + "model.layers.32.mlp.experts.252.gate_proj", + "model.layers.32.mlp.experts.252.up_proj", + "model.layers.32.mlp.experts.253.down_proj", + "model.layers.32.mlp.experts.253.gate_proj", + "model.layers.32.mlp.experts.253.up_proj", + "model.layers.32.mlp.experts.254.down_proj", + "model.layers.32.mlp.experts.254.gate_proj", + "model.layers.32.mlp.experts.254.up_proj", + "model.layers.32.mlp.experts.255.down_proj", + "model.layers.32.mlp.experts.255.gate_proj", + "model.layers.32.mlp.experts.255.up_proj", + "model.layers.32.mlp.experts.26.down_proj", + "model.layers.32.mlp.experts.26.gate_proj", + "model.layers.32.mlp.experts.26.up_proj", + "model.layers.32.mlp.experts.27.down_proj", + "model.layers.32.mlp.experts.27.gate_proj", + "model.layers.32.mlp.experts.27.up_proj", + "model.layers.32.mlp.experts.28.down_proj", + "model.layers.32.mlp.experts.28.gate_proj", + "model.layers.32.mlp.experts.28.up_proj", + "model.layers.32.mlp.experts.29.down_proj", + "model.layers.32.mlp.experts.29.gate_proj", + "model.layers.32.mlp.experts.29.up_proj", + "model.layers.32.mlp.experts.3.down_proj", + "model.layers.32.mlp.experts.3.gate_proj", + "model.layers.32.mlp.experts.3.up_proj", + "model.layers.32.mlp.experts.30.down_proj", + "model.layers.32.mlp.experts.30.gate_proj", + "model.layers.32.mlp.experts.30.up_proj", + "model.layers.32.mlp.experts.31.down_proj", + "model.layers.32.mlp.experts.31.gate_proj", + "model.layers.32.mlp.experts.31.up_proj", + "model.layers.32.mlp.experts.32.down_proj", + "model.layers.32.mlp.experts.32.gate_proj", + "model.layers.32.mlp.experts.32.up_proj", + "model.layers.32.mlp.experts.33.down_proj", + "model.layers.32.mlp.experts.33.gate_proj", + "model.layers.32.mlp.experts.33.up_proj", + "model.layers.32.mlp.experts.34.down_proj", + "model.layers.32.mlp.experts.34.gate_proj", + "model.layers.32.mlp.experts.34.up_proj", + "model.layers.32.mlp.experts.35.down_proj", + "model.layers.32.mlp.experts.35.gate_proj", + "model.layers.32.mlp.experts.35.up_proj", + "model.layers.32.mlp.experts.36.down_proj", + "model.layers.32.mlp.experts.36.gate_proj", + "model.layers.32.mlp.experts.36.up_proj", + "model.layers.32.mlp.experts.37.down_proj", + "model.layers.32.mlp.experts.37.gate_proj", + "model.layers.32.mlp.experts.37.up_proj", + "model.layers.32.mlp.experts.38.down_proj", + "model.layers.32.mlp.experts.38.gate_proj", + "model.layers.32.mlp.experts.38.up_proj", + "model.layers.32.mlp.experts.39.down_proj", + "model.layers.32.mlp.experts.39.gate_proj", + "model.layers.32.mlp.experts.39.up_proj", + "model.layers.32.mlp.experts.4.down_proj", + "model.layers.32.mlp.experts.4.gate_proj", + "model.layers.32.mlp.experts.4.up_proj", + "model.layers.32.mlp.experts.40.down_proj", + "model.layers.32.mlp.experts.40.gate_proj", + "model.layers.32.mlp.experts.40.up_proj", + "model.layers.32.mlp.experts.41.down_proj", + "model.layers.32.mlp.experts.41.gate_proj", + "model.layers.32.mlp.experts.41.up_proj", + "model.layers.32.mlp.experts.42.down_proj", + "model.layers.32.mlp.experts.42.gate_proj", + "model.layers.32.mlp.experts.42.up_proj", + "model.layers.32.mlp.experts.43.down_proj", + "model.layers.32.mlp.experts.43.gate_proj", + "model.layers.32.mlp.experts.43.up_proj", + "model.layers.32.mlp.experts.44.down_proj", + "model.layers.32.mlp.experts.44.gate_proj", + "model.layers.32.mlp.experts.44.up_proj", + "model.layers.32.mlp.experts.45.down_proj", + "model.layers.32.mlp.experts.45.gate_proj", + "model.layers.32.mlp.experts.45.up_proj", + "model.layers.32.mlp.experts.46.down_proj", + "model.layers.32.mlp.experts.46.gate_proj", + "model.layers.32.mlp.experts.46.up_proj", + "model.layers.32.mlp.experts.47.down_proj", + "model.layers.32.mlp.experts.47.gate_proj", + "model.layers.32.mlp.experts.47.up_proj", + "model.layers.32.mlp.experts.48.down_proj", + "model.layers.32.mlp.experts.48.gate_proj", + "model.layers.32.mlp.experts.48.up_proj", + "model.layers.32.mlp.experts.49.down_proj", + "model.layers.32.mlp.experts.49.gate_proj", + "model.layers.32.mlp.experts.49.up_proj", + "model.layers.32.mlp.experts.5.down_proj", + "model.layers.32.mlp.experts.5.gate_proj", + "model.layers.32.mlp.experts.5.up_proj", + "model.layers.32.mlp.experts.50.down_proj", + "model.layers.32.mlp.experts.50.gate_proj", + "model.layers.32.mlp.experts.50.up_proj", + "model.layers.32.mlp.experts.51.down_proj", + "model.layers.32.mlp.experts.51.gate_proj", + "model.layers.32.mlp.experts.51.up_proj", + "model.layers.32.mlp.experts.52.down_proj", + "model.layers.32.mlp.experts.52.gate_proj", + "model.layers.32.mlp.experts.52.up_proj", + "model.layers.32.mlp.experts.53.down_proj", + "model.layers.32.mlp.experts.53.gate_proj", + "model.layers.32.mlp.experts.53.up_proj", + "model.layers.32.mlp.experts.54.down_proj", + "model.layers.32.mlp.experts.54.gate_proj", + "model.layers.32.mlp.experts.54.up_proj", + "model.layers.32.mlp.experts.55.down_proj", + "model.layers.32.mlp.experts.55.gate_proj", + "model.layers.32.mlp.experts.55.up_proj", + "model.layers.32.mlp.experts.56.down_proj", + "model.layers.32.mlp.experts.56.gate_proj", + "model.layers.32.mlp.experts.56.up_proj", + "model.layers.32.mlp.experts.57.down_proj", + "model.layers.32.mlp.experts.57.gate_proj", + "model.layers.32.mlp.experts.57.up_proj", + "model.layers.32.mlp.experts.58.down_proj", + "model.layers.32.mlp.experts.58.gate_proj", + "model.layers.32.mlp.experts.58.up_proj", + "model.layers.32.mlp.experts.59.down_proj", + "model.layers.32.mlp.experts.59.gate_proj", + "model.layers.32.mlp.experts.59.up_proj", + "model.layers.32.mlp.experts.6.down_proj", + "model.layers.32.mlp.experts.6.gate_proj", + "model.layers.32.mlp.experts.6.up_proj", + "model.layers.32.mlp.experts.60.down_proj", + "model.layers.32.mlp.experts.60.gate_proj", + "model.layers.32.mlp.experts.60.up_proj", + "model.layers.32.mlp.experts.61.down_proj", + "model.layers.32.mlp.experts.61.gate_proj", + "model.layers.32.mlp.experts.61.up_proj", + "model.layers.32.mlp.experts.62.down_proj", + "model.layers.32.mlp.experts.62.gate_proj", + "model.layers.32.mlp.experts.62.up_proj", + "model.layers.32.mlp.experts.63.down_proj", + "model.layers.32.mlp.experts.63.gate_proj", + "model.layers.32.mlp.experts.63.up_proj", + "model.layers.32.mlp.experts.64.down_proj", + "model.layers.32.mlp.experts.64.gate_proj", + "model.layers.32.mlp.experts.64.up_proj", + "model.layers.32.mlp.experts.65.down_proj", + "model.layers.32.mlp.experts.65.gate_proj", + "model.layers.32.mlp.experts.65.up_proj", + "model.layers.32.mlp.experts.66.down_proj", + "model.layers.32.mlp.experts.66.gate_proj", + "model.layers.32.mlp.experts.66.up_proj", + "model.layers.32.mlp.experts.67.down_proj", + "model.layers.32.mlp.experts.67.gate_proj", + "model.layers.32.mlp.experts.67.up_proj", + "model.layers.32.mlp.experts.68.down_proj", + "model.layers.32.mlp.experts.68.gate_proj", + "model.layers.32.mlp.experts.68.up_proj", + "model.layers.32.mlp.experts.69.down_proj", + "model.layers.32.mlp.experts.69.gate_proj", + "model.layers.32.mlp.experts.69.up_proj", + "model.layers.32.mlp.experts.7.down_proj", + "model.layers.32.mlp.experts.7.gate_proj", + "model.layers.32.mlp.experts.7.up_proj", + "model.layers.32.mlp.experts.70.down_proj", + "model.layers.32.mlp.experts.70.gate_proj", + "model.layers.32.mlp.experts.70.up_proj", + "model.layers.32.mlp.experts.71.down_proj", + "model.layers.32.mlp.experts.71.gate_proj", + "model.layers.32.mlp.experts.71.up_proj", + "model.layers.32.mlp.experts.72.down_proj", + "model.layers.32.mlp.experts.72.gate_proj", + "model.layers.32.mlp.experts.72.up_proj", + "model.layers.32.mlp.experts.73.down_proj", + "model.layers.32.mlp.experts.73.gate_proj", + "model.layers.32.mlp.experts.73.up_proj", + "model.layers.32.mlp.experts.74.down_proj", + "model.layers.32.mlp.experts.74.gate_proj", + "model.layers.32.mlp.experts.74.up_proj", + "model.layers.32.mlp.experts.75.down_proj", + "model.layers.32.mlp.experts.75.gate_proj", + "model.layers.32.mlp.experts.75.up_proj", + "model.layers.32.mlp.experts.76.down_proj", + "model.layers.32.mlp.experts.76.gate_proj", + "model.layers.32.mlp.experts.76.up_proj", + "model.layers.32.mlp.experts.77.down_proj", + "model.layers.32.mlp.experts.77.gate_proj", + "model.layers.32.mlp.experts.77.up_proj", + "model.layers.32.mlp.experts.78.down_proj", + "model.layers.32.mlp.experts.78.gate_proj", + "model.layers.32.mlp.experts.78.up_proj", + "model.layers.32.mlp.experts.79.down_proj", + "model.layers.32.mlp.experts.79.gate_proj", + "model.layers.32.mlp.experts.79.up_proj", + "model.layers.32.mlp.experts.8.down_proj", + "model.layers.32.mlp.experts.8.gate_proj", + "model.layers.32.mlp.experts.8.up_proj", + "model.layers.32.mlp.experts.80.down_proj", + "model.layers.32.mlp.experts.80.gate_proj", + "model.layers.32.mlp.experts.80.up_proj", + "model.layers.32.mlp.experts.81.down_proj", + "model.layers.32.mlp.experts.81.gate_proj", + "model.layers.32.mlp.experts.81.up_proj", + "model.layers.32.mlp.experts.82.down_proj", + "model.layers.32.mlp.experts.82.gate_proj", + "model.layers.32.mlp.experts.82.up_proj", + "model.layers.32.mlp.experts.83.down_proj", + "model.layers.32.mlp.experts.83.gate_proj", + "model.layers.32.mlp.experts.83.up_proj", + "model.layers.32.mlp.experts.84.down_proj", + "model.layers.32.mlp.experts.84.gate_proj", + "model.layers.32.mlp.experts.84.up_proj", + "model.layers.32.mlp.experts.85.down_proj", + "model.layers.32.mlp.experts.85.gate_proj", + "model.layers.32.mlp.experts.85.up_proj", + "model.layers.32.mlp.experts.86.down_proj", + "model.layers.32.mlp.experts.86.gate_proj", + "model.layers.32.mlp.experts.86.up_proj", + "model.layers.32.mlp.experts.87.down_proj", + "model.layers.32.mlp.experts.87.gate_proj", + "model.layers.32.mlp.experts.87.up_proj", + "model.layers.32.mlp.experts.88.down_proj", + "model.layers.32.mlp.experts.88.gate_proj", + "model.layers.32.mlp.experts.88.up_proj", + "model.layers.32.mlp.experts.89.down_proj", + "model.layers.32.mlp.experts.89.gate_proj", + "model.layers.32.mlp.experts.89.up_proj", + "model.layers.32.mlp.experts.9.down_proj", + "model.layers.32.mlp.experts.9.gate_proj", + "model.layers.32.mlp.experts.9.up_proj", + "model.layers.32.mlp.experts.90.down_proj", + "model.layers.32.mlp.experts.90.gate_proj", + "model.layers.32.mlp.experts.90.up_proj", + "model.layers.32.mlp.experts.91.down_proj", + "model.layers.32.mlp.experts.91.gate_proj", + "model.layers.32.mlp.experts.91.up_proj", + "model.layers.32.mlp.experts.92.down_proj", + "model.layers.32.mlp.experts.92.gate_proj", + "model.layers.32.mlp.experts.92.up_proj", + "model.layers.32.mlp.experts.93.down_proj", + "model.layers.32.mlp.experts.93.gate_proj", + "model.layers.32.mlp.experts.93.up_proj", + "model.layers.32.mlp.experts.94.down_proj", + "model.layers.32.mlp.experts.94.gate_proj", + "model.layers.32.mlp.experts.94.up_proj", + "model.layers.32.mlp.experts.95.down_proj", + "model.layers.32.mlp.experts.95.gate_proj", + "model.layers.32.mlp.experts.95.up_proj", + "model.layers.32.mlp.experts.96.down_proj", + "model.layers.32.mlp.experts.96.gate_proj", + "model.layers.32.mlp.experts.96.up_proj", + "model.layers.32.mlp.experts.97.down_proj", + "model.layers.32.mlp.experts.97.gate_proj", + "model.layers.32.mlp.experts.97.up_proj", + "model.layers.32.mlp.experts.98.down_proj", + "model.layers.32.mlp.experts.98.gate_proj", + "model.layers.32.mlp.experts.98.up_proj", + "model.layers.32.mlp.experts.99.down_proj", + "model.layers.32.mlp.experts.99.gate_proj", + "model.layers.32.mlp.experts.99.up_proj", + "model.layers.33.mlp.experts.0.down_proj", + "model.layers.33.mlp.experts.0.gate_proj", + "model.layers.33.mlp.experts.0.up_proj", + "model.layers.33.mlp.experts.1.down_proj", + "model.layers.33.mlp.experts.1.gate_proj", + "model.layers.33.mlp.experts.1.up_proj", + "model.layers.33.mlp.experts.10.down_proj", + "model.layers.33.mlp.experts.10.gate_proj", + "model.layers.33.mlp.experts.10.up_proj", + "model.layers.33.mlp.experts.100.down_proj", + "model.layers.33.mlp.experts.100.gate_proj", + "model.layers.33.mlp.experts.100.up_proj", + "model.layers.33.mlp.experts.101.down_proj", + "model.layers.33.mlp.experts.101.gate_proj", + "model.layers.33.mlp.experts.101.up_proj", + "model.layers.33.mlp.experts.102.down_proj", + "model.layers.33.mlp.experts.102.gate_proj", + "model.layers.33.mlp.experts.102.up_proj", + "model.layers.33.mlp.experts.103.down_proj", + "model.layers.33.mlp.experts.103.gate_proj", + "model.layers.33.mlp.experts.103.up_proj", + "model.layers.33.mlp.experts.104.down_proj", + "model.layers.33.mlp.experts.104.gate_proj", + "model.layers.33.mlp.experts.104.up_proj", + "model.layers.33.mlp.experts.105.down_proj", + "model.layers.33.mlp.experts.105.gate_proj", + "model.layers.33.mlp.experts.105.up_proj", + "model.layers.33.mlp.experts.106.down_proj", + "model.layers.33.mlp.experts.106.gate_proj", + "model.layers.33.mlp.experts.106.up_proj", + "model.layers.33.mlp.experts.107.down_proj", + "model.layers.33.mlp.experts.107.gate_proj", + "model.layers.33.mlp.experts.107.up_proj", + "model.layers.33.mlp.experts.108.down_proj", + "model.layers.33.mlp.experts.108.gate_proj", + "model.layers.33.mlp.experts.108.up_proj", + "model.layers.33.mlp.experts.109.down_proj", + "model.layers.33.mlp.experts.109.gate_proj", + "model.layers.33.mlp.experts.109.up_proj", + "model.layers.33.mlp.experts.11.down_proj", + "model.layers.33.mlp.experts.11.gate_proj", + "model.layers.33.mlp.experts.11.up_proj", + "model.layers.33.mlp.experts.110.down_proj", + "model.layers.33.mlp.experts.110.gate_proj", + "model.layers.33.mlp.experts.110.up_proj", + "model.layers.33.mlp.experts.111.down_proj", + "model.layers.33.mlp.experts.111.gate_proj", + "model.layers.33.mlp.experts.111.up_proj", + "model.layers.33.mlp.experts.112.down_proj", + "model.layers.33.mlp.experts.112.gate_proj", + "model.layers.33.mlp.experts.112.up_proj", + "model.layers.33.mlp.experts.113.down_proj", + "model.layers.33.mlp.experts.113.gate_proj", + "model.layers.33.mlp.experts.113.up_proj", + "model.layers.33.mlp.experts.114.down_proj", + "model.layers.33.mlp.experts.114.gate_proj", + "model.layers.33.mlp.experts.114.up_proj", + "model.layers.33.mlp.experts.115.down_proj", + "model.layers.33.mlp.experts.115.gate_proj", + "model.layers.33.mlp.experts.115.up_proj", + "model.layers.33.mlp.experts.116.down_proj", + "model.layers.33.mlp.experts.116.gate_proj", + "model.layers.33.mlp.experts.116.up_proj", + "model.layers.33.mlp.experts.117.down_proj", + "model.layers.33.mlp.experts.117.gate_proj", + "model.layers.33.mlp.experts.117.up_proj", + "model.layers.33.mlp.experts.118.down_proj", + "model.layers.33.mlp.experts.118.gate_proj", + "model.layers.33.mlp.experts.118.up_proj", + "model.layers.33.mlp.experts.119.down_proj", + "model.layers.33.mlp.experts.119.gate_proj", + "model.layers.33.mlp.experts.119.up_proj", + "model.layers.33.mlp.experts.12.down_proj", + "model.layers.33.mlp.experts.12.gate_proj", + "model.layers.33.mlp.experts.12.up_proj", + "model.layers.33.mlp.experts.120.down_proj", + "model.layers.33.mlp.experts.120.gate_proj", + "model.layers.33.mlp.experts.120.up_proj", + "model.layers.33.mlp.experts.121.down_proj", + "model.layers.33.mlp.experts.121.gate_proj", + "model.layers.33.mlp.experts.121.up_proj", + "model.layers.33.mlp.experts.122.down_proj", + "model.layers.33.mlp.experts.122.gate_proj", + "model.layers.33.mlp.experts.122.up_proj", + "model.layers.33.mlp.experts.123.down_proj", + "model.layers.33.mlp.experts.123.gate_proj", + "model.layers.33.mlp.experts.123.up_proj", + "model.layers.33.mlp.experts.124.down_proj", + "model.layers.33.mlp.experts.124.gate_proj", + "model.layers.33.mlp.experts.124.up_proj", + "model.layers.33.mlp.experts.125.down_proj", + "model.layers.33.mlp.experts.125.gate_proj", + "model.layers.33.mlp.experts.125.up_proj", + "model.layers.33.mlp.experts.126.down_proj", + "model.layers.33.mlp.experts.126.gate_proj", + "model.layers.33.mlp.experts.126.up_proj", + "model.layers.33.mlp.experts.127.down_proj", + "model.layers.33.mlp.experts.127.gate_proj", + "model.layers.33.mlp.experts.127.up_proj", + "model.layers.33.mlp.experts.128.down_proj", + "model.layers.33.mlp.experts.128.gate_proj", + "model.layers.33.mlp.experts.128.up_proj", + "model.layers.33.mlp.experts.129.down_proj", + "model.layers.33.mlp.experts.129.gate_proj", + "model.layers.33.mlp.experts.129.up_proj", + "model.layers.33.mlp.experts.13.down_proj", + "model.layers.33.mlp.experts.13.gate_proj", + "model.layers.33.mlp.experts.13.up_proj", + "model.layers.33.mlp.experts.130.down_proj", + "model.layers.33.mlp.experts.130.gate_proj", + "model.layers.33.mlp.experts.130.up_proj", + "model.layers.33.mlp.experts.131.down_proj", + "model.layers.33.mlp.experts.131.gate_proj", + "model.layers.33.mlp.experts.131.up_proj", + "model.layers.33.mlp.experts.132.down_proj", + "model.layers.33.mlp.experts.132.gate_proj", + "model.layers.33.mlp.experts.132.up_proj", + "model.layers.33.mlp.experts.133.down_proj", + "model.layers.33.mlp.experts.133.gate_proj", + "model.layers.33.mlp.experts.133.up_proj", + "model.layers.33.mlp.experts.134.down_proj", + "model.layers.33.mlp.experts.134.gate_proj", + "model.layers.33.mlp.experts.134.up_proj", + "model.layers.33.mlp.experts.135.down_proj", + "model.layers.33.mlp.experts.135.gate_proj", + "model.layers.33.mlp.experts.135.up_proj", + "model.layers.33.mlp.experts.136.down_proj", + "model.layers.33.mlp.experts.136.gate_proj", + "model.layers.33.mlp.experts.136.up_proj", + "model.layers.33.mlp.experts.137.down_proj", + "model.layers.33.mlp.experts.137.gate_proj", + "model.layers.33.mlp.experts.137.up_proj", + "model.layers.33.mlp.experts.138.down_proj", + "model.layers.33.mlp.experts.138.gate_proj", + "model.layers.33.mlp.experts.138.up_proj", + "model.layers.33.mlp.experts.139.down_proj", + "model.layers.33.mlp.experts.139.gate_proj", + "model.layers.33.mlp.experts.139.up_proj", + "model.layers.33.mlp.experts.14.down_proj", + "model.layers.33.mlp.experts.14.gate_proj", + "model.layers.33.mlp.experts.14.up_proj", + "model.layers.33.mlp.experts.140.down_proj", + "model.layers.33.mlp.experts.140.gate_proj", + "model.layers.33.mlp.experts.140.up_proj", + "model.layers.33.mlp.experts.141.down_proj", + "model.layers.33.mlp.experts.141.gate_proj", + "model.layers.33.mlp.experts.141.up_proj", + "model.layers.33.mlp.experts.142.down_proj", + "model.layers.33.mlp.experts.142.gate_proj", + "model.layers.33.mlp.experts.142.up_proj", + "model.layers.33.mlp.experts.143.down_proj", + "model.layers.33.mlp.experts.143.gate_proj", + "model.layers.33.mlp.experts.143.up_proj", + "model.layers.33.mlp.experts.144.down_proj", + "model.layers.33.mlp.experts.144.gate_proj", + "model.layers.33.mlp.experts.144.up_proj", + "model.layers.33.mlp.experts.145.down_proj", + "model.layers.33.mlp.experts.145.gate_proj", + "model.layers.33.mlp.experts.145.up_proj", + "model.layers.33.mlp.experts.146.down_proj", + "model.layers.33.mlp.experts.146.gate_proj", + "model.layers.33.mlp.experts.146.up_proj", + "model.layers.33.mlp.experts.147.down_proj", + "model.layers.33.mlp.experts.147.gate_proj", + "model.layers.33.mlp.experts.147.up_proj", + "model.layers.33.mlp.experts.148.down_proj", + "model.layers.33.mlp.experts.148.gate_proj", + "model.layers.33.mlp.experts.148.up_proj", + "model.layers.33.mlp.experts.149.down_proj", + "model.layers.33.mlp.experts.149.gate_proj", + "model.layers.33.mlp.experts.149.up_proj", + "model.layers.33.mlp.experts.15.down_proj", + "model.layers.33.mlp.experts.15.gate_proj", + "model.layers.33.mlp.experts.15.up_proj", + "model.layers.33.mlp.experts.150.down_proj", + "model.layers.33.mlp.experts.150.gate_proj", + "model.layers.33.mlp.experts.150.up_proj", + "model.layers.33.mlp.experts.151.down_proj", + "model.layers.33.mlp.experts.151.gate_proj", + "model.layers.33.mlp.experts.151.up_proj", + "model.layers.33.mlp.experts.152.down_proj", + "model.layers.33.mlp.experts.152.gate_proj", + "model.layers.33.mlp.experts.152.up_proj", + "model.layers.33.mlp.experts.153.down_proj", + "model.layers.33.mlp.experts.153.gate_proj", + "model.layers.33.mlp.experts.153.up_proj", + "model.layers.33.mlp.experts.154.down_proj", + "model.layers.33.mlp.experts.154.gate_proj", + "model.layers.33.mlp.experts.154.up_proj", + "model.layers.33.mlp.experts.155.down_proj", + "model.layers.33.mlp.experts.155.gate_proj", + "model.layers.33.mlp.experts.155.up_proj", + "model.layers.33.mlp.experts.156.down_proj", + "model.layers.33.mlp.experts.156.gate_proj", + "model.layers.33.mlp.experts.156.up_proj", + "model.layers.33.mlp.experts.157.down_proj", + "model.layers.33.mlp.experts.157.gate_proj", + "model.layers.33.mlp.experts.157.up_proj", + "model.layers.33.mlp.experts.158.down_proj", + "model.layers.33.mlp.experts.158.gate_proj", + "model.layers.33.mlp.experts.158.up_proj", + "model.layers.33.mlp.experts.159.down_proj", + "model.layers.33.mlp.experts.159.gate_proj", + "model.layers.33.mlp.experts.159.up_proj", + "model.layers.33.mlp.experts.16.down_proj", + "model.layers.33.mlp.experts.16.gate_proj", + "model.layers.33.mlp.experts.16.up_proj", + "model.layers.33.mlp.experts.160.down_proj", + "model.layers.33.mlp.experts.160.gate_proj", + "model.layers.33.mlp.experts.160.up_proj", + "model.layers.33.mlp.experts.161.down_proj", + "model.layers.33.mlp.experts.161.gate_proj", + "model.layers.33.mlp.experts.161.up_proj", + "model.layers.33.mlp.experts.162.down_proj", + "model.layers.33.mlp.experts.162.gate_proj", + "model.layers.33.mlp.experts.162.up_proj", + "model.layers.33.mlp.experts.163.down_proj", + "model.layers.33.mlp.experts.163.gate_proj", + "model.layers.33.mlp.experts.163.up_proj", + "model.layers.33.mlp.experts.164.down_proj", + "model.layers.33.mlp.experts.164.gate_proj", + "model.layers.33.mlp.experts.164.up_proj", + "model.layers.33.mlp.experts.165.down_proj", + "model.layers.33.mlp.experts.165.gate_proj", + "model.layers.33.mlp.experts.165.up_proj", + "model.layers.33.mlp.experts.166.down_proj", + "model.layers.33.mlp.experts.166.gate_proj", + "model.layers.33.mlp.experts.166.up_proj", + "model.layers.33.mlp.experts.167.down_proj", + "model.layers.33.mlp.experts.167.gate_proj", + "model.layers.33.mlp.experts.167.up_proj", + "model.layers.33.mlp.experts.168.down_proj", + "model.layers.33.mlp.experts.168.gate_proj", + "model.layers.33.mlp.experts.168.up_proj", + "model.layers.33.mlp.experts.169.down_proj", + "model.layers.33.mlp.experts.169.gate_proj", + "model.layers.33.mlp.experts.169.up_proj", + "model.layers.33.mlp.experts.17.down_proj", + "model.layers.33.mlp.experts.17.gate_proj", + "model.layers.33.mlp.experts.17.up_proj", + "model.layers.33.mlp.experts.170.down_proj", + "model.layers.33.mlp.experts.170.gate_proj", + "model.layers.33.mlp.experts.170.up_proj", + "model.layers.33.mlp.experts.171.down_proj", + "model.layers.33.mlp.experts.171.gate_proj", + "model.layers.33.mlp.experts.171.up_proj", + "model.layers.33.mlp.experts.172.down_proj", + "model.layers.33.mlp.experts.172.gate_proj", + "model.layers.33.mlp.experts.172.up_proj", + "model.layers.33.mlp.experts.173.down_proj", + "model.layers.33.mlp.experts.173.gate_proj", + "model.layers.33.mlp.experts.173.up_proj", + "model.layers.33.mlp.experts.174.down_proj", + "model.layers.33.mlp.experts.174.gate_proj", + "model.layers.33.mlp.experts.174.up_proj", + "model.layers.33.mlp.experts.175.down_proj", + "model.layers.33.mlp.experts.175.gate_proj", + "model.layers.33.mlp.experts.175.up_proj", + "model.layers.33.mlp.experts.176.down_proj", + "model.layers.33.mlp.experts.176.gate_proj", + "model.layers.33.mlp.experts.176.up_proj", + "model.layers.33.mlp.experts.177.down_proj", + "model.layers.33.mlp.experts.177.gate_proj", + "model.layers.33.mlp.experts.177.up_proj", + "model.layers.33.mlp.experts.178.down_proj", + "model.layers.33.mlp.experts.178.gate_proj", + "model.layers.33.mlp.experts.178.up_proj", + "model.layers.33.mlp.experts.179.down_proj", + "model.layers.33.mlp.experts.179.gate_proj", + "model.layers.33.mlp.experts.179.up_proj", + "model.layers.33.mlp.experts.18.down_proj", + "model.layers.33.mlp.experts.18.gate_proj", + "model.layers.33.mlp.experts.18.up_proj", + "model.layers.33.mlp.experts.180.down_proj", + "model.layers.33.mlp.experts.180.gate_proj", + "model.layers.33.mlp.experts.180.up_proj", + "model.layers.33.mlp.experts.181.down_proj", + "model.layers.33.mlp.experts.181.gate_proj", + "model.layers.33.mlp.experts.181.up_proj", + "model.layers.33.mlp.experts.182.down_proj", + "model.layers.33.mlp.experts.182.gate_proj", + "model.layers.33.mlp.experts.182.up_proj", + "model.layers.33.mlp.experts.183.down_proj", + "model.layers.33.mlp.experts.183.gate_proj", + "model.layers.33.mlp.experts.183.up_proj", + "model.layers.33.mlp.experts.184.down_proj", + "model.layers.33.mlp.experts.184.gate_proj", + "model.layers.33.mlp.experts.184.up_proj", + "model.layers.33.mlp.experts.185.down_proj", + "model.layers.33.mlp.experts.185.gate_proj", + "model.layers.33.mlp.experts.185.up_proj", + "model.layers.33.mlp.experts.186.down_proj", + "model.layers.33.mlp.experts.186.gate_proj", + "model.layers.33.mlp.experts.186.up_proj", + "model.layers.33.mlp.experts.187.down_proj", + "model.layers.33.mlp.experts.187.gate_proj", + "model.layers.33.mlp.experts.187.up_proj", + "model.layers.33.mlp.experts.188.down_proj", + "model.layers.33.mlp.experts.188.gate_proj", + "model.layers.33.mlp.experts.188.up_proj", + "model.layers.33.mlp.experts.189.down_proj", + "model.layers.33.mlp.experts.189.gate_proj", + "model.layers.33.mlp.experts.189.up_proj", + "model.layers.33.mlp.experts.19.down_proj", + "model.layers.33.mlp.experts.19.gate_proj", + "model.layers.33.mlp.experts.19.up_proj", + "model.layers.33.mlp.experts.190.down_proj", + "model.layers.33.mlp.experts.190.gate_proj", + "model.layers.33.mlp.experts.190.up_proj", + "model.layers.33.mlp.experts.191.down_proj", + "model.layers.33.mlp.experts.191.gate_proj", + "model.layers.33.mlp.experts.191.up_proj", + "model.layers.33.mlp.experts.192.down_proj", + "model.layers.33.mlp.experts.192.gate_proj", + "model.layers.33.mlp.experts.192.up_proj", + "model.layers.33.mlp.experts.193.down_proj", + "model.layers.33.mlp.experts.193.gate_proj", + "model.layers.33.mlp.experts.193.up_proj", + "model.layers.33.mlp.experts.194.down_proj", + "model.layers.33.mlp.experts.194.gate_proj", + "model.layers.33.mlp.experts.194.up_proj", + "model.layers.33.mlp.experts.195.down_proj", + "model.layers.33.mlp.experts.195.gate_proj", + "model.layers.33.mlp.experts.195.up_proj", + "model.layers.33.mlp.experts.196.down_proj", + "model.layers.33.mlp.experts.196.gate_proj", + "model.layers.33.mlp.experts.196.up_proj", + "model.layers.33.mlp.experts.197.down_proj", + "model.layers.33.mlp.experts.197.gate_proj", + "model.layers.33.mlp.experts.197.up_proj", + "model.layers.33.mlp.experts.198.down_proj", + "model.layers.33.mlp.experts.198.gate_proj", + "model.layers.33.mlp.experts.198.up_proj", + "model.layers.33.mlp.experts.199.down_proj", + "model.layers.33.mlp.experts.199.gate_proj", + "model.layers.33.mlp.experts.199.up_proj", + "model.layers.33.mlp.experts.2.down_proj", + "model.layers.33.mlp.experts.2.gate_proj", + "model.layers.33.mlp.experts.2.up_proj", + "model.layers.33.mlp.experts.20.down_proj", + "model.layers.33.mlp.experts.20.gate_proj", + "model.layers.33.mlp.experts.20.up_proj", + "model.layers.33.mlp.experts.200.down_proj", + "model.layers.33.mlp.experts.200.gate_proj", + "model.layers.33.mlp.experts.200.up_proj", + "model.layers.33.mlp.experts.201.down_proj", + "model.layers.33.mlp.experts.201.gate_proj", + "model.layers.33.mlp.experts.201.up_proj", + "model.layers.33.mlp.experts.202.down_proj", + "model.layers.33.mlp.experts.202.gate_proj", + "model.layers.33.mlp.experts.202.up_proj", + "model.layers.33.mlp.experts.203.down_proj", + "model.layers.33.mlp.experts.203.gate_proj", + "model.layers.33.mlp.experts.203.up_proj", + "model.layers.33.mlp.experts.204.down_proj", + "model.layers.33.mlp.experts.204.gate_proj", + "model.layers.33.mlp.experts.204.up_proj", + "model.layers.33.mlp.experts.205.down_proj", + "model.layers.33.mlp.experts.205.gate_proj", + "model.layers.33.mlp.experts.205.up_proj", + "model.layers.33.mlp.experts.206.down_proj", + "model.layers.33.mlp.experts.206.gate_proj", + "model.layers.33.mlp.experts.206.up_proj", + "model.layers.33.mlp.experts.207.down_proj", + "model.layers.33.mlp.experts.207.gate_proj", + "model.layers.33.mlp.experts.207.up_proj", + "model.layers.33.mlp.experts.208.down_proj", + "model.layers.33.mlp.experts.208.gate_proj", + "model.layers.33.mlp.experts.208.up_proj", + "model.layers.33.mlp.experts.209.down_proj", + "model.layers.33.mlp.experts.209.gate_proj", + "model.layers.33.mlp.experts.209.up_proj", + "model.layers.33.mlp.experts.21.down_proj", + "model.layers.33.mlp.experts.21.gate_proj", + "model.layers.33.mlp.experts.21.up_proj", + "model.layers.33.mlp.experts.210.down_proj", + "model.layers.33.mlp.experts.210.gate_proj", + "model.layers.33.mlp.experts.210.up_proj", + "model.layers.33.mlp.experts.211.down_proj", + "model.layers.33.mlp.experts.211.gate_proj", + "model.layers.33.mlp.experts.211.up_proj", + "model.layers.33.mlp.experts.212.down_proj", + "model.layers.33.mlp.experts.212.gate_proj", + "model.layers.33.mlp.experts.212.up_proj", + "model.layers.33.mlp.experts.213.down_proj", + "model.layers.33.mlp.experts.213.gate_proj", + "model.layers.33.mlp.experts.213.up_proj", + "model.layers.33.mlp.experts.214.down_proj", + "model.layers.33.mlp.experts.214.gate_proj", + "model.layers.33.mlp.experts.214.up_proj", + "model.layers.33.mlp.experts.215.down_proj", + "model.layers.33.mlp.experts.215.gate_proj", + "model.layers.33.mlp.experts.215.up_proj", + "model.layers.33.mlp.experts.216.down_proj", + "model.layers.33.mlp.experts.216.gate_proj", + "model.layers.33.mlp.experts.216.up_proj", + "model.layers.33.mlp.experts.217.down_proj", + "model.layers.33.mlp.experts.217.gate_proj", + "model.layers.33.mlp.experts.217.up_proj", + "model.layers.33.mlp.experts.218.down_proj", + "model.layers.33.mlp.experts.218.gate_proj", + "model.layers.33.mlp.experts.218.up_proj", + "model.layers.33.mlp.experts.219.down_proj", + "model.layers.33.mlp.experts.219.gate_proj", + "model.layers.33.mlp.experts.219.up_proj", + "model.layers.33.mlp.experts.22.down_proj", + "model.layers.33.mlp.experts.22.gate_proj", + "model.layers.33.mlp.experts.22.up_proj", + "model.layers.33.mlp.experts.220.down_proj", + "model.layers.33.mlp.experts.220.gate_proj", + "model.layers.33.mlp.experts.220.up_proj", + "model.layers.33.mlp.experts.221.down_proj", + "model.layers.33.mlp.experts.221.gate_proj", + "model.layers.33.mlp.experts.221.up_proj", + "model.layers.33.mlp.experts.222.down_proj", + "model.layers.33.mlp.experts.222.gate_proj", + "model.layers.33.mlp.experts.222.up_proj", + "model.layers.33.mlp.experts.223.down_proj", + "model.layers.33.mlp.experts.223.gate_proj", + "model.layers.33.mlp.experts.223.up_proj", + "model.layers.33.mlp.experts.224.down_proj", + "model.layers.33.mlp.experts.224.gate_proj", + "model.layers.33.mlp.experts.224.up_proj", + "model.layers.33.mlp.experts.225.down_proj", + "model.layers.33.mlp.experts.225.gate_proj", + "model.layers.33.mlp.experts.225.up_proj", + "model.layers.33.mlp.experts.226.down_proj", + "model.layers.33.mlp.experts.226.gate_proj", + "model.layers.33.mlp.experts.226.up_proj", + "model.layers.33.mlp.experts.227.down_proj", + "model.layers.33.mlp.experts.227.gate_proj", + "model.layers.33.mlp.experts.227.up_proj", + "model.layers.33.mlp.experts.228.down_proj", + "model.layers.33.mlp.experts.228.gate_proj", + "model.layers.33.mlp.experts.228.up_proj", + "model.layers.33.mlp.experts.229.down_proj", + "model.layers.33.mlp.experts.229.gate_proj", + "model.layers.33.mlp.experts.229.up_proj", + "model.layers.33.mlp.experts.23.down_proj", + "model.layers.33.mlp.experts.23.gate_proj", + "model.layers.33.mlp.experts.23.up_proj", + "model.layers.33.mlp.experts.230.down_proj", + "model.layers.33.mlp.experts.230.gate_proj", + "model.layers.33.mlp.experts.230.up_proj", + "model.layers.33.mlp.experts.231.down_proj", + "model.layers.33.mlp.experts.231.gate_proj", + "model.layers.33.mlp.experts.231.up_proj", + "model.layers.33.mlp.experts.232.down_proj", + "model.layers.33.mlp.experts.232.gate_proj", + "model.layers.33.mlp.experts.232.up_proj", + "model.layers.33.mlp.experts.233.down_proj", + "model.layers.33.mlp.experts.233.gate_proj", + "model.layers.33.mlp.experts.233.up_proj", + "model.layers.33.mlp.experts.234.down_proj", + "model.layers.33.mlp.experts.234.gate_proj", + "model.layers.33.mlp.experts.234.up_proj", + "model.layers.33.mlp.experts.235.down_proj", + "model.layers.33.mlp.experts.235.gate_proj", + "model.layers.33.mlp.experts.235.up_proj", + "model.layers.33.mlp.experts.236.down_proj", + "model.layers.33.mlp.experts.236.gate_proj", + "model.layers.33.mlp.experts.236.up_proj", + "model.layers.33.mlp.experts.237.down_proj", + "model.layers.33.mlp.experts.237.gate_proj", + "model.layers.33.mlp.experts.237.up_proj", + "model.layers.33.mlp.experts.238.down_proj", + "model.layers.33.mlp.experts.238.gate_proj", + "model.layers.33.mlp.experts.238.up_proj", + "model.layers.33.mlp.experts.239.down_proj", + "model.layers.33.mlp.experts.239.gate_proj", + "model.layers.33.mlp.experts.239.up_proj", + "model.layers.33.mlp.experts.24.down_proj", + "model.layers.33.mlp.experts.24.gate_proj", + "model.layers.33.mlp.experts.24.up_proj", + "model.layers.33.mlp.experts.240.down_proj", + "model.layers.33.mlp.experts.240.gate_proj", + "model.layers.33.mlp.experts.240.up_proj", + "model.layers.33.mlp.experts.241.down_proj", + "model.layers.33.mlp.experts.241.gate_proj", + "model.layers.33.mlp.experts.241.up_proj", + "model.layers.33.mlp.experts.242.down_proj", + "model.layers.33.mlp.experts.242.gate_proj", + "model.layers.33.mlp.experts.242.up_proj", + "model.layers.33.mlp.experts.243.down_proj", + "model.layers.33.mlp.experts.243.gate_proj", + "model.layers.33.mlp.experts.243.up_proj", + "model.layers.33.mlp.experts.244.down_proj", + "model.layers.33.mlp.experts.244.gate_proj", + "model.layers.33.mlp.experts.244.up_proj", + "model.layers.33.mlp.experts.245.down_proj", + "model.layers.33.mlp.experts.245.gate_proj", + "model.layers.33.mlp.experts.245.up_proj", + "model.layers.33.mlp.experts.246.down_proj", + "model.layers.33.mlp.experts.246.gate_proj", + "model.layers.33.mlp.experts.246.up_proj", + "model.layers.33.mlp.experts.247.down_proj", + "model.layers.33.mlp.experts.247.gate_proj", + "model.layers.33.mlp.experts.247.up_proj", + "model.layers.33.mlp.experts.248.down_proj", + "model.layers.33.mlp.experts.248.gate_proj", + "model.layers.33.mlp.experts.248.up_proj", + "model.layers.33.mlp.experts.249.down_proj", + "model.layers.33.mlp.experts.249.gate_proj", + "model.layers.33.mlp.experts.249.up_proj", + "model.layers.33.mlp.experts.25.down_proj", + "model.layers.33.mlp.experts.25.gate_proj", + "model.layers.33.mlp.experts.25.up_proj", + "model.layers.33.mlp.experts.250.down_proj", + "model.layers.33.mlp.experts.250.gate_proj", + "model.layers.33.mlp.experts.250.up_proj", + "model.layers.33.mlp.experts.251.down_proj", + "model.layers.33.mlp.experts.251.gate_proj", + "model.layers.33.mlp.experts.251.up_proj", + "model.layers.33.mlp.experts.252.down_proj", + "model.layers.33.mlp.experts.252.gate_proj", + "model.layers.33.mlp.experts.252.up_proj", + "model.layers.33.mlp.experts.253.down_proj", + "model.layers.33.mlp.experts.253.gate_proj", + "model.layers.33.mlp.experts.253.up_proj", + "model.layers.33.mlp.experts.254.down_proj", + "model.layers.33.mlp.experts.254.gate_proj", + "model.layers.33.mlp.experts.254.up_proj", + "model.layers.33.mlp.experts.255.down_proj", + "model.layers.33.mlp.experts.255.gate_proj", + "model.layers.33.mlp.experts.255.up_proj", + "model.layers.33.mlp.experts.26.down_proj", + "model.layers.33.mlp.experts.26.gate_proj", + "model.layers.33.mlp.experts.26.up_proj", + "model.layers.33.mlp.experts.27.down_proj", + "model.layers.33.mlp.experts.27.gate_proj", + "model.layers.33.mlp.experts.27.up_proj", + "model.layers.33.mlp.experts.28.down_proj", + "model.layers.33.mlp.experts.28.gate_proj", + "model.layers.33.mlp.experts.28.up_proj", + "model.layers.33.mlp.experts.29.down_proj", + "model.layers.33.mlp.experts.29.gate_proj", + "model.layers.33.mlp.experts.29.up_proj", + "model.layers.33.mlp.experts.3.down_proj", + "model.layers.33.mlp.experts.3.gate_proj", + "model.layers.33.mlp.experts.3.up_proj", + "model.layers.33.mlp.experts.30.down_proj", + "model.layers.33.mlp.experts.30.gate_proj", + "model.layers.33.mlp.experts.30.up_proj", + "model.layers.33.mlp.experts.31.down_proj", + "model.layers.33.mlp.experts.31.gate_proj", + "model.layers.33.mlp.experts.31.up_proj", + "model.layers.33.mlp.experts.32.down_proj", + "model.layers.33.mlp.experts.32.gate_proj", + "model.layers.33.mlp.experts.32.up_proj", + "model.layers.33.mlp.experts.33.down_proj", + "model.layers.33.mlp.experts.33.gate_proj", + "model.layers.33.mlp.experts.33.up_proj", + "model.layers.33.mlp.experts.34.down_proj", + "model.layers.33.mlp.experts.34.gate_proj", + "model.layers.33.mlp.experts.34.up_proj", + "model.layers.33.mlp.experts.35.down_proj", + "model.layers.33.mlp.experts.35.gate_proj", + "model.layers.33.mlp.experts.35.up_proj", + "model.layers.33.mlp.experts.36.down_proj", + "model.layers.33.mlp.experts.36.gate_proj", + "model.layers.33.mlp.experts.36.up_proj", + "model.layers.33.mlp.experts.37.down_proj", + "model.layers.33.mlp.experts.37.gate_proj", + "model.layers.33.mlp.experts.37.up_proj", + "model.layers.33.mlp.experts.38.down_proj", + "model.layers.33.mlp.experts.38.gate_proj", + "model.layers.33.mlp.experts.38.up_proj", + "model.layers.33.mlp.experts.39.down_proj", + "model.layers.33.mlp.experts.39.gate_proj", + "model.layers.33.mlp.experts.39.up_proj", + "model.layers.33.mlp.experts.4.down_proj", + "model.layers.33.mlp.experts.4.gate_proj", + "model.layers.33.mlp.experts.4.up_proj", + "model.layers.33.mlp.experts.40.down_proj", + "model.layers.33.mlp.experts.40.gate_proj", + "model.layers.33.mlp.experts.40.up_proj", + "model.layers.33.mlp.experts.41.down_proj", + "model.layers.33.mlp.experts.41.gate_proj", + "model.layers.33.mlp.experts.41.up_proj", + "model.layers.33.mlp.experts.42.down_proj", + "model.layers.33.mlp.experts.42.gate_proj", + "model.layers.33.mlp.experts.42.up_proj", + "model.layers.33.mlp.experts.43.down_proj", + "model.layers.33.mlp.experts.43.gate_proj", + "model.layers.33.mlp.experts.43.up_proj", + "model.layers.33.mlp.experts.44.down_proj", + "model.layers.33.mlp.experts.44.gate_proj", + "model.layers.33.mlp.experts.44.up_proj", + "model.layers.33.mlp.experts.45.down_proj", + "model.layers.33.mlp.experts.45.gate_proj", + "model.layers.33.mlp.experts.45.up_proj", + "model.layers.33.mlp.experts.46.down_proj", + "model.layers.33.mlp.experts.46.gate_proj", + "model.layers.33.mlp.experts.46.up_proj", + "model.layers.33.mlp.experts.47.down_proj", + "model.layers.33.mlp.experts.47.gate_proj", + "model.layers.33.mlp.experts.47.up_proj", + "model.layers.33.mlp.experts.48.down_proj", + "model.layers.33.mlp.experts.48.gate_proj", + "model.layers.33.mlp.experts.48.up_proj", + "model.layers.33.mlp.experts.49.down_proj", + "model.layers.33.mlp.experts.49.gate_proj", + "model.layers.33.mlp.experts.49.up_proj", + "model.layers.33.mlp.experts.5.down_proj", + "model.layers.33.mlp.experts.5.gate_proj", + "model.layers.33.mlp.experts.5.up_proj", + "model.layers.33.mlp.experts.50.down_proj", + "model.layers.33.mlp.experts.50.gate_proj", + "model.layers.33.mlp.experts.50.up_proj", + "model.layers.33.mlp.experts.51.down_proj", + "model.layers.33.mlp.experts.51.gate_proj", + "model.layers.33.mlp.experts.51.up_proj", + "model.layers.33.mlp.experts.52.down_proj", + "model.layers.33.mlp.experts.52.gate_proj", + "model.layers.33.mlp.experts.52.up_proj", + "model.layers.33.mlp.experts.53.down_proj", + "model.layers.33.mlp.experts.53.gate_proj", + "model.layers.33.mlp.experts.53.up_proj", + "model.layers.33.mlp.experts.54.down_proj", + "model.layers.33.mlp.experts.54.gate_proj", + "model.layers.33.mlp.experts.54.up_proj", + "model.layers.33.mlp.experts.55.down_proj", + "model.layers.33.mlp.experts.55.gate_proj", + "model.layers.33.mlp.experts.55.up_proj", + "model.layers.33.mlp.experts.56.down_proj", + "model.layers.33.mlp.experts.56.gate_proj", + "model.layers.33.mlp.experts.56.up_proj", + "model.layers.33.mlp.experts.57.down_proj", + "model.layers.33.mlp.experts.57.gate_proj", + "model.layers.33.mlp.experts.57.up_proj", + "model.layers.33.mlp.experts.58.down_proj", + "model.layers.33.mlp.experts.58.gate_proj", + "model.layers.33.mlp.experts.58.up_proj", + "model.layers.33.mlp.experts.59.down_proj", + "model.layers.33.mlp.experts.59.gate_proj", + "model.layers.33.mlp.experts.59.up_proj", + "model.layers.33.mlp.experts.6.down_proj", + "model.layers.33.mlp.experts.6.gate_proj", + "model.layers.33.mlp.experts.6.up_proj", + "model.layers.33.mlp.experts.60.down_proj", + "model.layers.33.mlp.experts.60.gate_proj", + "model.layers.33.mlp.experts.60.up_proj", + "model.layers.33.mlp.experts.61.down_proj", + "model.layers.33.mlp.experts.61.gate_proj", + "model.layers.33.mlp.experts.61.up_proj", + "model.layers.33.mlp.experts.62.down_proj", + "model.layers.33.mlp.experts.62.gate_proj", + "model.layers.33.mlp.experts.62.up_proj", + "model.layers.33.mlp.experts.63.down_proj", + "model.layers.33.mlp.experts.63.gate_proj", + "model.layers.33.mlp.experts.63.up_proj", + "model.layers.33.mlp.experts.64.down_proj", + "model.layers.33.mlp.experts.64.gate_proj", + "model.layers.33.mlp.experts.64.up_proj", + "model.layers.33.mlp.experts.65.down_proj", + "model.layers.33.mlp.experts.65.gate_proj", + "model.layers.33.mlp.experts.65.up_proj", + "model.layers.33.mlp.experts.66.down_proj", + "model.layers.33.mlp.experts.66.gate_proj", + "model.layers.33.mlp.experts.66.up_proj", + "model.layers.33.mlp.experts.67.down_proj", + "model.layers.33.mlp.experts.67.gate_proj", + "model.layers.33.mlp.experts.67.up_proj", + "model.layers.33.mlp.experts.68.down_proj", + "model.layers.33.mlp.experts.68.gate_proj", + "model.layers.33.mlp.experts.68.up_proj", + "model.layers.33.mlp.experts.69.down_proj", + "model.layers.33.mlp.experts.69.gate_proj", + "model.layers.33.mlp.experts.69.up_proj", + "model.layers.33.mlp.experts.7.down_proj", + "model.layers.33.mlp.experts.7.gate_proj", + "model.layers.33.mlp.experts.7.up_proj", + "model.layers.33.mlp.experts.70.down_proj", + "model.layers.33.mlp.experts.70.gate_proj", + "model.layers.33.mlp.experts.70.up_proj", + "model.layers.33.mlp.experts.71.down_proj", + "model.layers.33.mlp.experts.71.gate_proj", + "model.layers.33.mlp.experts.71.up_proj", + "model.layers.33.mlp.experts.72.down_proj", + "model.layers.33.mlp.experts.72.gate_proj", + "model.layers.33.mlp.experts.72.up_proj", + "model.layers.33.mlp.experts.73.down_proj", + "model.layers.33.mlp.experts.73.gate_proj", + "model.layers.33.mlp.experts.73.up_proj", + "model.layers.33.mlp.experts.74.down_proj", + "model.layers.33.mlp.experts.74.gate_proj", + "model.layers.33.mlp.experts.74.up_proj", + "model.layers.33.mlp.experts.75.down_proj", + "model.layers.33.mlp.experts.75.gate_proj", + "model.layers.33.mlp.experts.75.up_proj", + "model.layers.33.mlp.experts.76.down_proj", + "model.layers.33.mlp.experts.76.gate_proj", + "model.layers.33.mlp.experts.76.up_proj", + "model.layers.33.mlp.experts.77.down_proj", + "model.layers.33.mlp.experts.77.gate_proj", + "model.layers.33.mlp.experts.77.up_proj", + "model.layers.33.mlp.experts.78.down_proj", + "model.layers.33.mlp.experts.78.gate_proj", + "model.layers.33.mlp.experts.78.up_proj", + "model.layers.33.mlp.experts.79.down_proj", + "model.layers.33.mlp.experts.79.gate_proj", + "model.layers.33.mlp.experts.79.up_proj", + "model.layers.33.mlp.experts.8.down_proj", + "model.layers.33.mlp.experts.8.gate_proj", + "model.layers.33.mlp.experts.8.up_proj", + "model.layers.33.mlp.experts.80.down_proj", + "model.layers.33.mlp.experts.80.gate_proj", + "model.layers.33.mlp.experts.80.up_proj", + "model.layers.33.mlp.experts.81.down_proj", + "model.layers.33.mlp.experts.81.gate_proj", + "model.layers.33.mlp.experts.81.up_proj", + "model.layers.33.mlp.experts.82.down_proj", + "model.layers.33.mlp.experts.82.gate_proj", + "model.layers.33.mlp.experts.82.up_proj", + "model.layers.33.mlp.experts.83.down_proj", + "model.layers.33.mlp.experts.83.gate_proj", + "model.layers.33.mlp.experts.83.up_proj", + "model.layers.33.mlp.experts.84.down_proj", + "model.layers.33.mlp.experts.84.gate_proj", + "model.layers.33.mlp.experts.84.up_proj", + "model.layers.33.mlp.experts.85.down_proj", + "model.layers.33.mlp.experts.85.gate_proj", + "model.layers.33.mlp.experts.85.up_proj", + "model.layers.33.mlp.experts.86.down_proj", + "model.layers.33.mlp.experts.86.gate_proj", + "model.layers.33.mlp.experts.86.up_proj", + "model.layers.33.mlp.experts.87.down_proj", + "model.layers.33.mlp.experts.87.gate_proj", + "model.layers.33.mlp.experts.87.up_proj", + "model.layers.33.mlp.experts.88.down_proj", + "model.layers.33.mlp.experts.88.gate_proj", + "model.layers.33.mlp.experts.88.up_proj", + "model.layers.33.mlp.experts.89.down_proj", + "model.layers.33.mlp.experts.89.gate_proj", + "model.layers.33.mlp.experts.89.up_proj", + "model.layers.33.mlp.experts.9.down_proj", + "model.layers.33.mlp.experts.9.gate_proj", + "model.layers.33.mlp.experts.9.up_proj", + "model.layers.33.mlp.experts.90.down_proj", + "model.layers.33.mlp.experts.90.gate_proj", + "model.layers.33.mlp.experts.90.up_proj", + "model.layers.33.mlp.experts.91.down_proj", + "model.layers.33.mlp.experts.91.gate_proj", + "model.layers.33.mlp.experts.91.up_proj", + "model.layers.33.mlp.experts.92.down_proj", + "model.layers.33.mlp.experts.92.gate_proj", + "model.layers.33.mlp.experts.92.up_proj", + "model.layers.33.mlp.experts.93.down_proj", + "model.layers.33.mlp.experts.93.gate_proj", + "model.layers.33.mlp.experts.93.up_proj", + "model.layers.33.mlp.experts.94.down_proj", + "model.layers.33.mlp.experts.94.gate_proj", + "model.layers.33.mlp.experts.94.up_proj", + "model.layers.33.mlp.experts.95.down_proj", + "model.layers.33.mlp.experts.95.gate_proj", + "model.layers.33.mlp.experts.95.up_proj", + "model.layers.33.mlp.experts.96.down_proj", + "model.layers.33.mlp.experts.96.gate_proj", + "model.layers.33.mlp.experts.96.up_proj", + "model.layers.33.mlp.experts.97.down_proj", + "model.layers.33.mlp.experts.97.gate_proj", + "model.layers.33.mlp.experts.97.up_proj", + "model.layers.33.mlp.experts.98.down_proj", + "model.layers.33.mlp.experts.98.gate_proj", + "model.layers.33.mlp.experts.98.up_proj", + "model.layers.33.mlp.experts.99.down_proj", + "model.layers.33.mlp.experts.99.gate_proj", + "model.layers.33.mlp.experts.99.up_proj", + "model.layers.34.mlp.experts.0.down_proj", + "model.layers.34.mlp.experts.0.gate_proj", + "model.layers.34.mlp.experts.0.up_proj", + "model.layers.34.mlp.experts.1.down_proj", + "model.layers.34.mlp.experts.1.gate_proj", + "model.layers.34.mlp.experts.1.up_proj", + "model.layers.34.mlp.experts.10.down_proj", + "model.layers.34.mlp.experts.10.gate_proj", + "model.layers.34.mlp.experts.10.up_proj", + "model.layers.34.mlp.experts.100.down_proj", + "model.layers.34.mlp.experts.100.gate_proj", + "model.layers.34.mlp.experts.100.up_proj", + "model.layers.34.mlp.experts.101.down_proj", + "model.layers.34.mlp.experts.101.gate_proj", + "model.layers.34.mlp.experts.101.up_proj", + "model.layers.34.mlp.experts.102.down_proj", + "model.layers.34.mlp.experts.102.gate_proj", + "model.layers.34.mlp.experts.102.up_proj", + "model.layers.34.mlp.experts.103.down_proj", + "model.layers.34.mlp.experts.103.gate_proj", + "model.layers.34.mlp.experts.103.up_proj", + "model.layers.34.mlp.experts.104.down_proj", + "model.layers.34.mlp.experts.104.gate_proj", + "model.layers.34.mlp.experts.104.up_proj", + "model.layers.34.mlp.experts.105.down_proj", + "model.layers.34.mlp.experts.105.gate_proj", + "model.layers.34.mlp.experts.105.up_proj", + "model.layers.34.mlp.experts.106.down_proj", + "model.layers.34.mlp.experts.106.gate_proj", + "model.layers.34.mlp.experts.106.up_proj", + "model.layers.34.mlp.experts.107.down_proj", + "model.layers.34.mlp.experts.107.gate_proj", + "model.layers.34.mlp.experts.107.up_proj", + "model.layers.34.mlp.experts.108.down_proj", + "model.layers.34.mlp.experts.108.gate_proj", + "model.layers.34.mlp.experts.108.up_proj", + "model.layers.34.mlp.experts.109.down_proj", + "model.layers.34.mlp.experts.109.gate_proj", + "model.layers.34.mlp.experts.109.up_proj", + "model.layers.34.mlp.experts.11.down_proj", + "model.layers.34.mlp.experts.11.gate_proj", + "model.layers.34.mlp.experts.11.up_proj", + "model.layers.34.mlp.experts.110.down_proj", + "model.layers.34.mlp.experts.110.gate_proj", + "model.layers.34.mlp.experts.110.up_proj", + "model.layers.34.mlp.experts.111.down_proj", + "model.layers.34.mlp.experts.111.gate_proj", + "model.layers.34.mlp.experts.111.up_proj", + "model.layers.34.mlp.experts.112.down_proj", + "model.layers.34.mlp.experts.112.gate_proj", + "model.layers.34.mlp.experts.112.up_proj", + "model.layers.34.mlp.experts.113.down_proj", + "model.layers.34.mlp.experts.113.gate_proj", + "model.layers.34.mlp.experts.113.up_proj", + "model.layers.34.mlp.experts.114.down_proj", + "model.layers.34.mlp.experts.114.gate_proj", + "model.layers.34.mlp.experts.114.up_proj", + "model.layers.34.mlp.experts.115.down_proj", + "model.layers.34.mlp.experts.115.gate_proj", + "model.layers.34.mlp.experts.115.up_proj", + "model.layers.34.mlp.experts.116.down_proj", + "model.layers.34.mlp.experts.116.gate_proj", + "model.layers.34.mlp.experts.116.up_proj", + "model.layers.34.mlp.experts.117.down_proj", + "model.layers.34.mlp.experts.117.gate_proj", + "model.layers.34.mlp.experts.117.up_proj", + "model.layers.34.mlp.experts.118.down_proj", + "model.layers.34.mlp.experts.118.gate_proj", + "model.layers.34.mlp.experts.118.up_proj", + "model.layers.34.mlp.experts.119.down_proj", + "model.layers.34.mlp.experts.119.gate_proj", + "model.layers.34.mlp.experts.119.up_proj", + "model.layers.34.mlp.experts.12.down_proj", + "model.layers.34.mlp.experts.12.gate_proj", + "model.layers.34.mlp.experts.12.up_proj", + "model.layers.34.mlp.experts.120.down_proj", + "model.layers.34.mlp.experts.120.gate_proj", + "model.layers.34.mlp.experts.120.up_proj", + "model.layers.34.mlp.experts.121.down_proj", + "model.layers.34.mlp.experts.121.gate_proj", + "model.layers.34.mlp.experts.121.up_proj", + "model.layers.34.mlp.experts.122.down_proj", + "model.layers.34.mlp.experts.122.gate_proj", + "model.layers.34.mlp.experts.122.up_proj", + "model.layers.34.mlp.experts.123.down_proj", + "model.layers.34.mlp.experts.123.gate_proj", + "model.layers.34.mlp.experts.123.up_proj", + "model.layers.34.mlp.experts.124.down_proj", + "model.layers.34.mlp.experts.124.gate_proj", + "model.layers.34.mlp.experts.124.up_proj", + "model.layers.34.mlp.experts.125.down_proj", + "model.layers.34.mlp.experts.125.gate_proj", + "model.layers.34.mlp.experts.125.up_proj", + "model.layers.34.mlp.experts.126.down_proj", + "model.layers.34.mlp.experts.126.gate_proj", + "model.layers.34.mlp.experts.126.up_proj", + "model.layers.34.mlp.experts.127.down_proj", + "model.layers.34.mlp.experts.127.gate_proj", + "model.layers.34.mlp.experts.127.up_proj", + "model.layers.34.mlp.experts.128.down_proj", + "model.layers.34.mlp.experts.128.gate_proj", + "model.layers.34.mlp.experts.128.up_proj", + "model.layers.34.mlp.experts.129.down_proj", + "model.layers.34.mlp.experts.129.gate_proj", + "model.layers.34.mlp.experts.129.up_proj", + "model.layers.34.mlp.experts.13.down_proj", + "model.layers.34.mlp.experts.13.gate_proj", + "model.layers.34.mlp.experts.13.up_proj", + "model.layers.34.mlp.experts.130.down_proj", + "model.layers.34.mlp.experts.130.gate_proj", + "model.layers.34.mlp.experts.130.up_proj", + "model.layers.34.mlp.experts.131.down_proj", + "model.layers.34.mlp.experts.131.gate_proj", + "model.layers.34.mlp.experts.131.up_proj", + "model.layers.34.mlp.experts.132.down_proj", + "model.layers.34.mlp.experts.132.gate_proj", + "model.layers.34.mlp.experts.132.up_proj", + "model.layers.34.mlp.experts.133.down_proj", + "model.layers.34.mlp.experts.133.gate_proj", + "model.layers.34.mlp.experts.133.up_proj", + "model.layers.34.mlp.experts.134.down_proj", + "model.layers.34.mlp.experts.134.gate_proj", + "model.layers.34.mlp.experts.134.up_proj", + "model.layers.34.mlp.experts.135.down_proj", + "model.layers.34.mlp.experts.135.gate_proj", + "model.layers.34.mlp.experts.135.up_proj", + "model.layers.34.mlp.experts.136.down_proj", + "model.layers.34.mlp.experts.136.gate_proj", + "model.layers.34.mlp.experts.136.up_proj", + "model.layers.34.mlp.experts.137.down_proj", + "model.layers.34.mlp.experts.137.gate_proj", + "model.layers.34.mlp.experts.137.up_proj", + "model.layers.34.mlp.experts.138.down_proj", + "model.layers.34.mlp.experts.138.gate_proj", + "model.layers.34.mlp.experts.138.up_proj", + "model.layers.34.mlp.experts.139.down_proj", + "model.layers.34.mlp.experts.139.gate_proj", + "model.layers.34.mlp.experts.139.up_proj", + "model.layers.34.mlp.experts.14.down_proj", + "model.layers.34.mlp.experts.14.gate_proj", + "model.layers.34.mlp.experts.14.up_proj", + "model.layers.34.mlp.experts.140.down_proj", + "model.layers.34.mlp.experts.140.gate_proj", + "model.layers.34.mlp.experts.140.up_proj", + "model.layers.34.mlp.experts.141.down_proj", + "model.layers.34.mlp.experts.141.gate_proj", + "model.layers.34.mlp.experts.141.up_proj", + "model.layers.34.mlp.experts.142.down_proj", + "model.layers.34.mlp.experts.142.gate_proj", + "model.layers.34.mlp.experts.142.up_proj", + "model.layers.34.mlp.experts.143.down_proj", + "model.layers.34.mlp.experts.143.gate_proj", + "model.layers.34.mlp.experts.143.up_proj", + "model.layers.34.mlp.experts.144.down_proj", + "model.layers.34.mlp.experts.144.gate_proj", + "model.layers.34.mlp.experts.144.up_proj", + "model.layers.34.mlp.experts.145.down_proj", + "model.layers.34.mlp.experts.145.gate_proj", + "model.layers.34.mlp.experts.145.up_proj", + "model.layers.34.mlp.experts.146.down_proj", + "model.layers.34.mlp.experts.146.gate_proj", + "model.layers.34.mlp.experts.146.up_proj", + "model.layers.34.mlp.experts.147.down_proj", + "model.layers.34.mlp.experts.147.gate_proj", + "model.layers.34.mlp.experts.147.up_proj", + "model.layers.34.mlp.experts.148.down_proj", + "model.layers.34.mlp.experts.148.gate_proj", + "model.layers.34.mlp.experts.148.up_proj", + "model.layers.34.mlp.experts.149.down_proj", + "model.layers.34.mlp.experts.149.gate_proj", + "model.layers.34.mlp.experts.149.up_proj", + "model.layers.34.mlp.experts.15.down_proj", + "model.layers.34.mlp.experts.15.gate_proj", + "model.layers.34.mlp.experts.15.up_proj", + "model.layers.34.mlp.experts.150.down_proj", + "model.layers.34.mlp.experts.150.gate_proj", + "model.layers.34.mlp.experts.150.up_proj", + "model.layers.34.mlp.experts.151.down_proj", + "model.layers.34.mlp.experts.151.gate_proj", + "model.layers.34.mlp.experts.151.up_proj", + "model.layers.34.mlp.experts.152.down_proj", + "model.layers.34.mlp.experts.152.gate_proj", + "model.layers.34.mlp.experts.152.up_proj", + "model.layers.34.mlp.experts.153.down_proj", + "model.layers.34.mlp.experts.153.gate_proj", + "model.layers.34.mlp.experts.153.up_proj", + "model.layers.34.mlp.experts.154.down_proj", + "model.layers.34.mlp.experts.154.gate_proj", + "model.layers.34.mlp.experts.154.up_proj", + "model.layers.34.mlp.experts.155.down_proj", + "model.layers.34.mlp.experts.155.gate_proj", + "model.layers.34.mlp.experts.155.up_proj", + "model.layers.34.mlp.experts.156.down_proj", + "model.layers.34.mlp.experts.156.gate_proj", + "model.layers.34.mlp.experts.156.up_proj", + "model.layers.34.mlp.experts.157.down_proj", + "model.layers.34.mlp.experts.157.gate_proj", + "model.layers.34.mlp.experts.157.up_proj", + "model.layers.34.mlp.experts.158.down_proj", + "model.layers.34.mlp.experts.158.gate_proj", + "model.layers.34.mlp.experts.158.up_proj", + "model.layers.34.mlp.experts.159.down_proj", + "model.layers.34.mlp.experts.159.gate_proj", + "model.layers.34.mlp.experts.159.up_proj", + "model.layers.34.mlp.experts.16.down_proj", + "model.layers.34.mlp.experts.16.gate_proj", + "model.layers.34.mlp.experts.16.up_proj", + "model.layers.34.mlp.experts.160.down_proj", + "model.layers.34.mlp.experts.160.gate_proj", + "model.layers.34.mlp.experts.160.up_proj", + "model.layers.34.mlp.experts.161.down_proj", + "model.layers.34.mlp.experts.161.gate_proj", + "model.layers.34.mlp.experts.161.up_proj", + "model.layers.34.mlp.experts.162.down_proj", + "model.layers.34.mlp.experts.162.gate_proj", + "model.layers.34.mlp.experts.162.up_proj", + "model.layers.34.mlp.experts.163.down_proj", + "model.layers.34.mlp.experts.163.gate_proj", + "model.layers.34.mlp.experts.163.up_proj", + "model.layers.34.mlp.experts.164.down_proj", + "model.layers.34.mlp.experts.164.gate_proj", + "model.layers.34.mlp.experts.164.up_proj", + "model.layers.34.mlp.experts.165.down_proj", + "model.layers.34.mlp.experts.165.gate_proj", + "model.layers.34.mlp.experts.165.up_proj", + "model.layers.34.mlp.experts.166.down_proj", + "model.layers.34.mlp.experts.166.gate_proj", + "model.layers.34.mlp.experts.166.up_proj", + "model.layers.34.mlp.experts.167.down_proj", + "model.layers.34.mlp.experts.167.gate_proj", + "model.layers.34.mlp.experts.167.up_proj", + "model.layers.34.mlp.experts.168.down_proj", + "model.layers.34.mlp.experts.168.gate_proj", + "model.layers.34.mlp.experts.168.up_proj", + "model.layers.34.mlp.experts.169.down_proj", + "model.layers.34.mlp.experts.169.gate_proj", + "model.layers.34.mlp.experts.169.up_proj", + "model.layers.34.mlp.experts.17.down_proj", + "model.layers.34.mlp.experts.17.gate_proj", + "model.layers.34.mlp.experts.17.up_proj", + "model.layers.34.mlp.experts.170.down_proj", + "model.layers.34.mlp.experts.170.gate_proj", + "model.layers.34.mlp.experts.170.up_proj", + "model.layers.34.mlp.experts.171.down_proj", + "model.layers.34.mlp.experts.171.gate_proj", + "model.layers.34.mlp.experts.171.up_proj", + "model.layers.34.mlp.experts.172.down_proj", + "model.layers.34.mlp.experts.172.gate_proj", + "model.layers.34.mlp.experts.172.up_proj", + "model.layers.34.mlp.experts.173.down_proj", + "model.layers.34.mlp.experts.173.gate_proj", + "model.layers.34.mlp.experts.173.up_proj", + "model.layers.34.mlp.experts.174.down_proj", + "model.layers.34.mlp.experts.174.gate_proj", + "model.layers.34.mlp.experts.174.up_proj", + "model.layers.34.mlp.experts.175.down_proj", + "model.layers.34.mlp.experts.175.gate_proj", + "model.layers.34.mlp.experts.175.up_proj", + "model.layers.34.mlp.experts.176.down_proj", + "model.layers.34.mlp.experts.176.gate_proj", + "model.layers.34.mlp.experts.176.up_proj", + "model.layers.34.mlp.experts.177.down_proj", + "model.layers.34.mlp.experts.177.gate_proj", + "model.layers.34.mlp.experts.177.up_proj", + "model.layers.34.mlp.experts.178.down_proj", + "model.layers.34.mlp.experts.178.gate_proj", + "model.layers.34.mlp.experts.178.up_proj", + "model.layers.34.mlp.experts.179.down_proj", + "model.layers.34.mlp.experts.179.gate_proj", + "model.layers.34.mlp.experts.179.up_proj", + "model.layers.34.mlp.experts.18.down_proj", + "model.layers.34.mlp.experts.18.gate_proj", + "model.layers.34.mlp.experts.18.up_proj", + "model.layers.34.mlp.experts.180.down_proj", + "model.layers.34.mlp.experts.180.gate_proj", + "model.layers.34.mlp.experts.180.up_proj", + "model.layers.34.mlp.experts.181.down_proj", + "model.layers.34.mlp.experts.181.gate_proj", + "model.layers.34.mlp.experts.181.up_proj", + "model.layers.34.mlp.experts.182.down_proj", + "model.layers.34.mlp.experts.182.gate_proj", + "model.layers.34.mlp.experts.182.up_proj", + "model.layers.34.mlp.experts.183.down_proj", + "model.layers.34.mlp.experts.183.gate_proj", + "model.layers.34.mlp.experts.183.up_proj", + "model.layers.34.mlp.experts.184.down_proj", + "model.layers.34.mlp.experts.184.gate_proj", + "model.layers.34.mlp.experts.184.up_proj", + "model.layers.34.mlp.experts.185.down_proj", + "model.layers.34.mlp.experts.185.gate_proj", + "model.layers.34.mlp.experts.185.up_proj", + "model.layers.34.mlp.experts.186.down_proj", + "model.layers.34.mlp.experts.186.gate_proj", + "model.layers.34.mlp.experts.186.up_proj", + "model.layers.34.mlp.experts.187.down_proj", + "model.layers.34.mlp.experts.187.gate_proj", + "model.layers.34.mlp.experts.187.up_proj", + "model.layers.34.mlp.experts.188.down_proj", + "model.layers.34.mlp.experts.188.gate_proj", + "model.layers.34.mlp.experts.188.up_proj", + "model.layers.34.mlp.experts.189.down_proj", + "model.layers.34.mlp.experts.189.gate_proj", + "model.layers.34.mlp.experts.189.up_proj", + "model.layers.34.mlp.experts.19.down_proj", + "model.layers.34.mlp.experts.19.gate_proj", + "model.layers.34.mlp.experts.19.up_proj", + "model.layers.34.mlp.experts.190.down_proj", + "model.layers.34.mlp.experts.190.gate_proj", + "model.layers.34.mlp.experts.190.up_proj", + "model.layers.34.mlp.experts.191.down_proj", + "model.layers.34.mlp.experts.191.gate_proj", + "model.layers.34.mlp.experts.191.up_proj", + "model.layers.34.mlp.experts.192.down_proj", + "model.layers.34.mlp.experts.192.gate_proj", + "model.layers.34.mlp.experts.192.up_proj", + "model.layers.34.mlp.experts.193.down_proj", + "model.layers.34.mlp.experts.193.gate_proj", + "model.layers.34.mlp.experts.193.up_proj", + "model.layers.34.mlp.experts.194.down_proj", + "model.layers.34.mlp.experts.194.gate_proj", + "model.layers.34.mlp.experts.194.up_proj", + "model.layers.34.mlp.experts.195.down_proj", + "model.layers.34.mlp.experts.195.gate_proj", + "model.layers.34.mlp.experts.195.up_proj", + "model.layers.34.mlp.experts.196.down_proj", + "model.layers.34.mlp.experts.196.gate_proj", + "model.layers.34.mlp.experts.196.up_proj", + "model.layers.34.mlp.experts.197.down_proj", + "model.layers.34.mlp.experts.197.gate_proj", + "model.layers.34.mlp.experts.197.up_proj", + "model.layers.34.mlp.experts.198.down_proj", + "model.layers.34.mlp.experts.198.gate_proj", + "model.layers.34.mlp.experts.198.up_proj", + "model.layers.34.mlp.experts.199.down_proj", + "model.layers.34.mlp.experts.199.gate_proj", + "model.layers.34.mlp.experts.199.up_proj", + "model.layers.34.mlp.experts.2.down_proj", + "model.layers.34.mlp.experts.2.gate_proj", + "model.layers.34.mlp.experts.2.up_proj", + "model.layers.34.mlp.experts.20.down_proj", + "model.layers.34.mlp.experts.20.gate_proj", + "model.layers.34.mlp.experts.20.up_proj", + "model.layers.34.mlp.experts.200.down_proj", + "model.layers.34.mlp.experts.200.gate_proj", + "model.layers.34.mlp.experts.200.up_proj", + "model.layers.34.mlp.experts.201.down_proj", + "model.layers.34.mlp.experts.201.gate_proj", + "model.layers.34.mlp.experts.201.up_proj", + "model.layers.34.mlp.experts.202.down_proj", + "model.layers.34.mlp.experts.202.gate_proj", + "model.layers.34.mlp.experts.202.up_proj", + "model.layers.34.mlp.experts.203.down_proj", + "model.layers.34.mlp.experts.203.gate_proj", + "model.layers.34.mlp.experts.203.up_proj", + "model.layers.34.mlp.experts.204.down_proj", + "model.layers.34.mlp.experts.204.gate_proj", + "model.layers.34.mlp.experts.204.up_proj", + "model.layers.34.mlp.experts.205.down_proj", + "model.layers.34.mlp.experts.205.gate_proj", + "model.layers.34.mlp.experts.205.up_proj", + "model.layers.34.mlp.experts.206.down_proj", + "model.layers.34.mlp.experts.206.gate_proj", + "model.layers.34.mlp.experts.206.up_proj", + "model.layers.34.mlp.experts.207.down_proj", + "model.layers.34.mlp.experts.207.gate_proj", + "model.layers.34.mlp.experts.207.up_proj", + "model.layers.34.mlp.experts.208.down_proj", + "model.layers.34.mlp.experts.208.gate_proj", + "model.layers.34.mlp.experts.208.up_proj", + "model.layers.34.mlp.experts.209.down_proj", + "model.layers.34.mlp.experts.209.gate_proj", + "model.layers.34.mlp.experts.209.up_proj", + "model.layers.34.mlp.experts.21.down_proj", + "model.layers.34.mlp.experts.21.gate_proj", + "model.layers.34.mlp.experts.21.up_proj", + "model.layers.34.mlp.experts.210.down_proj", + "model.layers.34.mlp.experts.210.gate_proj", + "model.layers.34.mlp.experts.210.up_proj", + "model.layers.34.mlp.experts.211.down_proj", + "model.layers.34.mlp.experts.211.gate_proj", + "model.layers.34.mlp.experts.211.up_proj", + "model.layers.34.mlp.experts.212.down_proj", + "model.layers.34.mlp.experts.212.gate_proj", + "model.layers.34.mlp.experts.212.up_proj", + "model.layers.34.mlp.experts.213.down_proj", + "model.layers.34.mlp.experts.213.gate_proj", + "model.layers.34.mlp.experts.213.up_proj", + "model.layers.34.mlp.experts.214.down_proj", + "model.layers.34.mlp.experts.214.gate_proj", + "model.layers.34.mlp.experts.214.up_proj", + "model.layers.34.mlp.experts.215.down_proj", + "model.layers.34.mlp.experts.215.gate_proj", + "model.layers.34.mlp.experts.215.up_proj", + "model.layers.34.mlp.experts.216.down_proj", + "model.layers.34.mlp.experts.216.gate_proj", + "model.layers.34.mlp.experts.216.up_proj", + "model.layers.34.mlp.experts.217.down_proj", + "model.layers.34.mlp.experts.217.gate_proj", + "model.layers.34.mlp.experts.217.up_proj", + "model.layers.34.mlp.experts.218.down_proj", + "model.layers.34.mlp.experts.218.gate_proj", + "model.layers.34.mlp.experts.218.up_proj", + "model.layers.34.mlp.experts.219.down_proj", + "model.layers.34.mlp.experts.219.gate_proj", + "model.layers.34.mlp.experts.219.up_proj", + "model.layers.34.mlp.experts.22.down_proj", + "model.layers.34.mlp.experts.22.gate_proj", + "model.layers.34.mlp.experts.22.up_proj", + "model.layers.34.mlp.experts.220.down_proj", + "model.layers.34.mlp.experts.220.gate_proj", + "model.layers.34.mlp.experts.220.up_proj", + "model.layers.34.mlp.experts.221.down_proj", + "model.layers.34.mlp.experts.221.gate_proj", + "model.layers.34.mlp.experts.221.up_proj", + "model.layers.34.mlp.experts.222.down_proj", + "model.layers.34.mlp.experts.222.gate_proj", + "model.layers.34.mlp.experts.222.up_proj", + "model.layers.34.mlp.experts.223.down_proj", + "model.layers.34.mlp.experts.223.gate_proj", + "model.layers.34.mlp.experts.223.up_proj", + "model.layers.34.mlp.experts.224.down_proj", + "model.layers.34.mlp.experts.224.gate_proj", + "model.layers.34.mlp.experts.224.up_proj", + "model.layers.34.mlp.experts.225.down_proj", + "model.layers.34.mlp.experts.225.gate_proj", + "model.layers.34.mlp.experts.225.up_proj", + "model.layers.34.mlp.experts.226.down_proj", + "model.layers.34.mlp.experts.226.gate_proj", + "model.layers.34.mlp.experts.226.up_proj", + "model.layers.34.mlp.experts.227.down_proj", + "model.layers.34.mlp.experts.227.gate_proj", + "model.layers.34.mlp.experts.227.up_proj", + "model.layers.34.mlp.experts.228.down_proj", + "model.layers.34.mlp.experts.228.gate_proj", + "model.layers.34.mlp.experts.228.up_proj", + "model.layers.34.mlp.experts.229.down_proj", + "model.layers.34.mlp.experts.229.gate_proj", + "model.layers.34.mlp.experts.229.up_proj", + "model.layers.34.mlp.experts.23.down_proj", + "model.layers.34.mlp.experts.23.gate_proj", + "model.layers.34.mlp.experts.23.up_proj", + "model.layers.34.mlp.experts.230.down_proj", + "model.layers.34.mlp.experts.230.gate_proj", + "model.layers.34.mlp.experts.230.up_proj", + "model.layers.34.mlp.experts.231.down_proj", + "model.layers.34.mlp.experts.231.gate_proj", + "model.layers.34.mlp.experts.231.up_proj", + "model.layers.34.mlp.experts.232.down_proj", + "model.layers.34.mlp.experts.232.gate_proj", + "model.layers.34.mlp.experts.232.up_proj", + "model.layers.34.mlp.experts.233.down_proj", + "model.layers.34.mlp.experts.233.gate_proj", + "model.layers.34.mlp.experts.233.up_proj", + "model.layers.34.mlp.experts.234.down_proj", + "model.layers.34.mlp.experts.234.gate_proj", + "model.layers.34.mlp.experts.234.up_proj", + "model.layers.34.mlp.experts.235.down_proj", + "model.layers.34.mlp.experts.235.gate_proj", + "model.layers.34.mlp.experts.235.up_proj", + "model.layers.34.mlp.experts.236.down_proj", + "model.layers.34.mlp.experts.236.gate_proj", + "model.layers.34.mlp.experts.236.up_proj", + "model.layers.34.mlp.experts.237.down_proj", + "model.layers.34.mlp.experts.237.gate_proj", + "model.layers.34.mlp.experts.237.up_proj", + "model.layers.34.mlp.experts.238.down_proj", + "model.layers.34.mlp.experts.238.gate_proj", + "model.layers.34.mlp.experts.238.up_proj", + "model.layers.34.mlp.experts.239.down_proj", + "model.layers.34.mlp.experts.239.gate_proj", + "model.layers.34.mlp.experts.239.up_proj", + "model.layers.34.mlp.experts.24.down_proj", + "model.layers.34.mlp.experts.24.gate_proj", + "model.layers.34.mlp.experts.24.up_proj", + "model.layers.34.mlp.experts.240.down_proj", + "model.layers.34.mlp.experts.240.gate_proj", + "model.layers.34.mlp.experts.240.up_proj", + "model.layers.34.mlp.experts.241.down_proj", + "model.layers.34.mlp.experts.241.gate_proj", + "model.layers.34.mlp.experts.241.up_proj", + "model.layers.34.mlp.experts.242.down_proj", + "model.layers.34.mlp.experts.242.gate_proj", + "model.layers.34.mlp.experts.242.up_proj", + "model.layers.34.mlp.experts.243.down_proj", + "model.layers.34.mlp.experts.243.gate_proj", + "model.layers.34.mlp.experts.243.up_proj", + "model.layers.34.mlp.experts.244.down_proj", + "model.layers.34.mlp.experts.244.gate_proj", + "model.layers.34.mlp.experts.244.up_proj", + "model.layers.34.mlp.experts.245.down_proj", + "model.layers.34.mlp.experts.245.gate_proj", + "model.layers.34.mlp.experts.245.up_proj", + "model.layers.34.mlp.experts.246.down_proj", + "model.layers.34.mlp.experts.246.gate_proj", + "model.layers.34.mlp.experts.246.up_proj", + "model.layers.34.mlp.experts.247.down_proj", + "model.layers.34.mlp.experts.247.gate_proj", + "model.layers.34.mlp.experts.247.up_proj", + "model.layers.34.mlp.experts.248.down_proj", + "model.layers.34.mlp.experts.248.gate_proj", + "model.layers.34.mlp.experts.248.up_proj", + "model.layers.34.mlp.experts.249.down_proj", + "model.layers.34.mlp.experts.249.gate_proj", + "model.layers.34.mlp.experts.249.up_proj", + "model.layers.34.mlp.experts.25.down_proj", + "model.layers.34.mlp.experts.25.gate_proj", + "model.layers.34.mlp.experts.25.up_proj", + "model.layers.34.mlp.experts.250.down_proj", + "model.layers.34.mlp.experts.250.gate_proj", + "model.layers.34.mlp.experts.250.up_proj", + "model.layers.34.mlp.experts.251.down_proj", + "model.layers.34.mlp.experts.251.gate_proj", + "model.layers.34.mlp.experts.251.up_proj", + "model.layers.34.mlp.experts.252.down_proj", + "model.layers.34.mlp.experts.252.gate_proj", + "model.layers.34.mlp.experts.252.up_proj", + "model.layers.34.mlp.experts.253.down_proj", + "model.layers.34.mlp.experts.253.gate_proj", + "model.layers.34.mlp.experts.253.up_proj", + "model.layers.34.mlp.experts.254.down_proj", + "model.layers.34.mlp.experts.254.gate_proj", + "model.layers.34.mlp.experts.254.up_proj", + "model.layers.34.mlp.experts.255.down_proj", + "model.layers.34.mlp.experts.255.gate_proj", + "model.layers.34.mlp.experts.255.up_proj", + "model.layers.34.mlp.experts.26.down_proj", + "model.layers.34.mlp.experts.26.gate_proj", + "model.layers.34.mlp.experts.26.up_proj", + "model.layers.34.mlp.experts.27.down_proj", + "model.layers.34.mlp.experts.27.gate_proj", + "model.layers.34.mlp.experts.27.up_proj", + "model.layers.34.mlp.experts.28.down_proj", + "model.layers.34.mlp.experts.28.gate_proj", + "model.layers.34.mlp.experts.28.up_proj", + "model.layers.34.mlp.experts.29.down_proj", + "model.layers.34.mlp.experts.29.gate_proj", + "model.layers.34.mlp.experts.29.up_proj", + "model.layers.34.mlp.experts.3.down_proj", + "model.layers.34.mlp.experts.3.gate_proj", + "model.layers.34.mlp.experts.3.up_proj", + "model.layers.34.mlp.experts.30.down_proj", + "model.layers.34.mlp.experts.30.gate_proj", + "model.layers.34.mlp.experts.30.up_proj", + "model.layers.34.mlp.experts.31.down_proj", + "model.layers.34.mlp.experts.31.gate_proj", + "model.layers.34.mlp.experts.31.up_proj", + "model.layers.34.mlp.experts.32.down_proj", + "model.layers.34.mlp.experts.32.gate_proj", + "model.layers.34.mlp.experts.32.up_proj", + "model.layers.34.mlp.experts.33.down_proj", + "model.layers.34.mlp.experts.33.gate_proj", + "model.layers.34.mlp.experts.33.up_proj", + "model.layers.34.mlp.experts.34.down_proj", + "model.layers.34.mlp.experts.34.gate_proj", + "model.layers.34.mlp.experts.34.up_proj", + "model.layers.34.mlp.experts.35.down_proj", + "model.layers.34.mlp.experts.35.gate_proj", + "model.layers.34.mlp.experts.35.up_proj", + "model.layers.34.mlp.experts.36.down_proj", + "model.layers.34.mlp.experts.36.gate_proj", + "model.layers.34.mlp.experts.36.up_proj", + "model.layers.34.mlp.experts.37.down_proj", + "model.layers.34.mlp.experts.37.gate_proj", + "model.layers.34.mlp.experts.37.up_proj", + "model.layers.34.mlp.experts.38.down_proj", + "model.layers.34.mlp.experts.38.gate_proj", + "model.layers.34.mlp.experts.38.up_proj", + "model.layers.34.mlp.experts.39.down_proj", + "model.layers.34.mlp.experts.39.gate_proj", + "model.layers.34.mlp.experts.39.up_proj", + "model.layers.34.mlp.experts.4.down_proj", + "model.layers.34.mlp.experts.4.gate_proj", + "model.layers.34.mlp.experts.4.up_proj", + "model.layers.34.mlp.experts.40.down_proj", + "model.layers.34.mlp.experts.40.gate_proj", + "model.layers.34.mlp.experts.40.up_proj", + "model.layers.34.mlp.experts.41.down_proj", + "model.layers.34.mlp.experts.41.gate_proj", + "model.layers.34.mlp.experts.41.up_proj", + "model.layers.34.mlp.experts.42.down_proj", + "model.layers.34.mlp.experts.42.gate_proj", + "model.layers.34.mlp.experts.42.up_proj", + "model.layers.34.mlp.experts.43.down_proj", + "model.layers.34.mlp.experts.43.gate_proj", + "model.layers.34.mlp.experts.43.up_proj", + "model.layers.34.mlp.experts.44.down_proj", + "model.layers.34.mlp.experts.44.gate_proj", + "model.layers.34.mlp.experts.44.up_proj", + "model.layers.34.mlp.experts.45.down_proj", + "model.layers.34.mlp.experts.45.gate_proj", + "model.layers.34.mlp.experts.45.up_proj", + "model.layers.34.mlp.experts.46.down_proj", + "model.layers.34.mlp.experts.46.gate_proj", + "model.layers.34.mlp.experts.46.up_proj", + "model.layers.34.mlp.experts.47.down_proj", + "model.layers.34.mlp.experts.47.gate_proj", + "model.layers.34.mlp.experts.47.up_proj", + "model.layers.34.mlp.experts.48.down_proj", + "model.layers.34.mlp.experts.48.gate_proj", + "model.layers.34.mlp.experts.48.up_proj", + "model.layers.34.mlp.experts.49.down_proj", + "model.layers.34.mlp.experts.49.gate_proj", + "model.layers.34.mlp.experts.49.up_proj", + "model.layers.34.mlp.experts.5.down_proj", + "model.layers.34.mlp.experts.5.gate_proj", + "model.layers.34.mlp.experts.5.up_proj", + "model.layers.34.mlp.experts.50.down_proj", + "model.layers.34.mlp.experts.50.gate_proj", + "model.layers.34.mlp.experts.50.up_proj", + "model.layers.34.mlp.experts.51.down_proj", + "model.layers.34.mlp.experts.51.gate_proj", + "model.layers.34.mlp.experts.51.up_proj", + "model.layers.34.mlp.experts.52.down_proj", + "model.layers.34.mlp.experts.52.gate_proj", + "model.layers.34.mlp.experts.52.up_proj", + "model.layers.34.mlp.experts.53.down_proj", + "model.layers.34.mlp.experts.53.gate_proj", + "model.layers.34.mlp.experts.53.up_proj", + "model.layers.34.mlp.experts.54.down_proj", + "model.layers.34.mlp.experts.54.gate_proj", + "model.layers.34.mlp.experts.54.up_proj", + "model.layers.34.mlp.experts.55.down_proj", + "model.layers.34.mlp.experts.55.gate_proj", + "model.layers.34.mlp.experts.55.up_proj", + "model.layers.34.mlp.experts.56.down_proj", + "model.layers.34.mlp.experts.56.gate_proj", + "model.layers.34.mlp.experts.56.up_proj", + "model.layers.34.mlp.experts.57.down_proj", + "model.layers.34.mlp.experts.57.gate_proj", + "model.layers.34.mlp.experts.57.up_proj", + "model.layers.34.mlp.experts.58.down_proj", + "model.layers.34.mlp.experts.58.gate_proj", + "model.layers.34.mlp.experts.58.up_proj", + "model.layers.34.mlp.experts.59.down_proj", + "model.layers.34.mlp.experts.59.gate_proj", + "model.layers.34.mlp.experts.59.up_proj", + "model.layers.34.mlp.experts.6.down_proj", + "model.layers.34.mlp.experts.6.gate_proj", + "model.layers.34.mlp.experts.6.up_proj", + "model.layers.34.mlp.experts.60.down_proj", + "model.layers.34.mlp.experts.60.gate_proj", + "model.layers.34.mlp.experts.60.up_proj", + "model.layers.34.mlp.experts.61.down_proj", + "model.layers.34.mlp.experts.61.gate_proj", + "model.layers.34.mlp.experts.61.up_proj", + "model.layers.34.mlp.experts.62.down_proj", + "model.layers.34.mlp.experts.62.gate_proj", + "model.layers.34.mlp.experts.62.up_proj", + "model.layers.34.mlp.experts.63.down_proj", + "model.layers.34.mlp.experts.63.gate_proj", + "model.layers.34.mlp.experts.63.up_proj", + "model.layers.34.mlp.experts.64.down_proj", + "model.layers.34.mlp.experts.64.gate_proj", + "model.layers.34.mlp.experts.64.up_proj", + "model.layers.34.mlp.experts.65.down_proj", + "model.layers.34.mlp.experts.65.gate_proj", + "model.layers.34.mlp.experts.65.up_proj", + "model.layers.34.mlp.experts.66.down_proj", + "model.layers.34.mlp.experts.66.gate_proj", + "model.layers.34.mlp.experts.66.up_proj", + "model.layers.34.mlp.experts.67.down_proj", + "model.layers.34.mlp.experts.67.gate_proj", + "model.layers.34.mlp.experts.67.up_proj", + "model.layers.34.mlp.experts.68.down_proj", + "model.layers.34.mlp.experts.68.gate_proj", + "model.layers.34.mlp.experts.68.up_proj", + "model.layers.34.mlp.experts.69.down_proj", + "model.layers.34.mlp.experts.69.gate_proj", + "model.layers.34.mlp.experts.69.up_proj", + "model.layers.34.mlp.experts.7.down_proj", + "model.layers.34.mlp.experts.7.gate_proj", + "model.layers.34.mlp.experts.7.up_proj", + "model.layers.34.mlp.experts.70.down_proj", + "model.layers.34.mlp.experts.70.gate_proj", + "model.layers.34.mlp.experts.70.up_proj", + "model.layers.34.mlp.experts.71.down_proj", + "model.layers.34.mlp.experts.71.gate_proj", + "model.layers.34.mlp.experts.71.up_proj", + "model.layers.34.mlp.experts.72.down_proj", + "model.layers.34.mlp.experts.72.gate_proj", + "model.layers.34.mlp.experts.72.up_proj", + "model.layers.34.mlp.experts.73.down_proj", + "model.layers.34.mlp.experts.73.gate_proj", + "model.layers.34.mlp.experts.73.up_proj", + "model.layers.34.mlp.experts.74.down_proj", + "model.layers.34.mlp.experts.74.gate_proj", + "model.layers.34.mlp.experts.74.up_proj", + "model.layers.34.mlp.experts.75.down_proj", + "model.layers.34.mlp.experts.75.gate_proj", + "model.layers.34.mlp.experts.75.up_proj", + "model.layers.34.mlp.experts.76.down_proj", + "model.layers.34.mlp.experts.76.gate_proj", + "model.layers.34.mlp.experts.76.up_proj", + "model.layers.34.mlp.experts.77.down_proj", + "model.layers.34.mlp.experts.77.gate_proj", + "model.layers.34.mlp.experts.77.up_proj", + "model.layers.34.mlp.experts.78.down_proj", + "model.layers.34.mlp.experts.78.gate_proj", + "model.layers.34.mlp.experts.78.up_proj", + "model.layers.34.mlp.experts.79.down_proj", + "model.layers.34.mlp.experts.79.gate_proj", + "model.layers.34.mlp.experts.79.up_proj", + "model.layers.34.mlp.experts.8.down_proj", + "model.layers.34.mlp.experts.8.gate_proj", + "model.layers.34.mlp.experts.8.up_proj", + "model.layers.34.mlp.experts.80.down_proj", + "model.layers.34.mlp.experts.80.gate_proj", + "model.layers.34.mlp.experts.80.up_proj", + "model.layers.34.mlp.experts.81.down_proj", + "model.layers.34.mlp.experts.81.gate_proj", + "model.layers.34.mlp.experts.81.up_proj", + "model.layers.34.mlp.experts.82.down_proj", + "model.layers.34.mlp.experts.82.gate_proj", + "model.layers.34.mlp.experts.82.up_proj", + "model.layers.34.mlp.experts.83.down_proj", + "model.layers.34.mlp.experts.83.gate_proj", + "model.layers.34.mlp.experts.83.up_proj", + "model.layers.34.mlp.experts.84.down_proj", + "model.layers.34.mlp.experts.84.gate_proj", + "model.layers.34.mlp.experts.84.up_proj", + "model.layers.34.mlp.experts.85.down_proj", + "model.layers.34.mlp.experts.85.gate_proj", + "model.layers.34.mlp.experts.85.up_proj", + "model.layers.34.mlp.experts.86.down_proj", + "model.layers.34.mlp.experts.86.gate_proj", + "model.layers.34.mlp.experts.86.up_proj", + "model.layers.34.mlp.experts.87.down_proj", + "model.layers.34.mlp.experts.87.gate_proj", + "model.layers.34.mlp.experts.87.up_proj", + "model.layers.34.mlp.experts.88.down_proj", + "model.layers.34.mlp.experts.88.gate_proj", + "model.layers.34.mlp.experts.88.up_proj", + "model.layers.34.mlp.experts.89.down_proj", + "model.layers.34.mlp.experts.89.gate_proj", + "model.layers.34.mlp.experts.89.up_proj", + "model.layers.34.mlp.experts.9.down_proj", + "model.layers.34.mlp.experts.9.gate_proj", + "model.layers.34.mlp.experts.9.up_proj", + "model.layers.34.mlp.experts.90.down_proj", + "model.layers.34.mlp.experts.90.gate_proj", + "model.layers.34.mlp.experts.90.up_proj", + "model.layers.34.mlp.experts.91.down_proj", + "model.layers.34.mlp.experts.91.gate_proj", + "model.layers.34.mlp.experts.91.up_proj", + "model.layers.34.mlp.experts.92.down_proj", + "model.layers.34.mlp.experts.92.gate_proj", + "model.layers.34.mlp.experts.92.up_proj", + "model.layers.34.mlp.experts.93.down_proj", + "model.layers.34.mlp.experts.93.gate_proj", + "model.layers.34.mlp.experts.93.up_proj", + "model.layers.34.mlp.experts.94.down_proj", + "model.layers.34.mlp.experts.94.gate_proj", + "model.layers.34.mlp.experts.94.up_proj", + "model.layers.34.mlp.experts.95.down_proj", + "model.layers.34.mlp.experts.95.gate_proj", + "model.layers.34.mlp.experts.95.up_proj", + "model.layers.34.mlp.experts.96.down_proj", + "model.layers.34.mlp.experts.96.gate_proj", + "model.layers.34.mlp.experts.96.up_proj", + "model.layers.34.mlp.experts.97.down_proj", + "model.layers.34.mlp.experts.97.gate_proj", + "model.layers.34.mlp.experts.97.up_proj", + "model.layers.34.mlp.experts.98.down_proj", + "model.layers.34.mlp.experts.98.gate_proj", + "model.layers.34.mlp.experts.98.up_proj", + "model.layers.34.mlp.experts.99.down_proj", + "model.layers.34.mlp.experts.99.gate_proj", + "model.layers.34.mlp.experts.99.up_proj", + "model.layers.35.mlp.experts.0.down_proj", + "model.layers.35.mlp.experts.0.gate_proj", + "model.layers.35.mlp.experts.0.up_proj", + "model.layers.35.mlp.experts.1.down_proj", + "model.layers.35.mlp.experts.1.gate_proj", + "model.layers.35.mlp.experts.1.up_proj", + "model.layers.35.mlp.experts.10.down_proj", + "model.layers.35.mlp.experts.10.gate_proj", + "model.layers.35.mlp.experts.10.up_proj", + "model.layers.35.mlp.experts.100.down_proj", + "model.layers.35.mlp.experts.100.gate_proj", + "model.layers.35.mlp.experts.100.up_proj", + "model.layers.35.mlp.experts.101.down_proj", + "model.layers.35.mlp.experts.101.gate_proj", + "model.layers.35.mlp.experts.101.up_proj", + "model.layers.35.mlp.experts.102.down_proj", + "model.layers.35.mlp.experts.102.gate_proj", + "model.layers.35.mlp.experts.102.up_proj", + "model.layers.35.mlp.experts.103.down_proj", + "model.layers.35.mlp.experts.103.gate_proj", + "model.layers.35.mlp.experts.103.up_proj", + "model.layers.35.mlp.experts.104.down_proj", + "model.layers.35.mlp.experts.104.gate_proj", + "model.layers.35.mlp.experts.104.up_proj", + "model.layers.35.mlp.experts.105.down_proj", + "model.layers.35.mlp.experts.105.gate_proj", + "model.layers.35.mlp.experts.105.up_proj", + "model.layers.35.mlp.experts.106.down_proj", + "model.layers.35.mlp.experts.106.gate_proj", + "model.layers.35.mlp.experts.106.up_proj", + "model.layers.35.mlp.experts.107.down_proj", + "model.layers.35.mlp.experts.107.gate_proj", + "model.layers.35.mlp.experts.107.up_proj", + "model.layers.35.mlp.experts.108.down_proj", + "model.layers.35.mlp.experts.108.gate_proj", + "model.layers.35.mlp.experts.108.up_proj", + "model.layers.35.mlp.experts.109.down_proj", + "model.layers.35.mlp.experts.109.gate_proj", + "model.layers.35.mlp.experts.109.up_proj", + "model.layers.35.mlp.experts.11.down_proj", + "model.layers.35.mlp.experts.11.gate_proj", + "model.layers.35.mlp.experts.11.up_proj", + "model.layers.35.mlp.experts.110.down_proj", + "model.layers.35.mlp.experts.110.gate_proj", + "model.layers.35.mlp.experts.110.up_proj", + "model.layers.35.mlp.experts.111.down_proj", + "model.layers.35.mlp.experts.111.gate_proj", + "model.layers.35.mlp.experts.111.up_proj", + "model.layers.35.mlp.experts.112.down_proj", + "model.layers.35.mlp.experts.112.gate_proj", + "model.layers.35.mlp.experts.112.up_proj", + "model.layers.35.mlp.experts.113.down_proj", + "model.layers.35.mlp.experts.113.gate_proj", + "model.layers.35.mlp.experts.113.up_proj", + "model.layers.35.mlp.experts.114.down_proj", + "model.layers.35.mlp.experts.114.gate_proj", + "model.layers.35.mlp.experts.114.up_proj", + "model.layers.35.mlp.experts.115.down_proj", + "model.layers.35.mlp.experts.115.gate_proj", + "model.layers.35.mlp.experts.115.up_proj", + "model.layers.35.mlp.experts.116.down_proj", + "model.layers.35.mlp.experts.116.gate_proj", + "model.layers.35.mlp.experts.116.up_proj", + "model.layers.35.mlp.experts.117.down_proj", + "model.layers.35.mlp.experts.117.gate_proj", + "model.layers.35.mlp.experts.117.up_proj", + "model.layers.35.mlp.experts.118.down_proj", + "model.layers.35.mlp.experts.118.gate_proj", + "model.layers.35.mlp.experts.118.up_proj", + "model.layers.35.mlp.experts.119.down_proj", + "model.layers.35.mlp.experts.119.gate_proj", + "model.layers.35.mlp.experts.119.up_proj", + "model.layers.35.mlp.experts.12.down_proj", + "model.layers.35.mlp.experts.12.gate_proj", + "model.layers.35.mlp.experts.12.up_proj", + "model.layers.35.mlp.experts.120.down_proj", + "model.layers.35.mlp.experts.120.gate_proj", + "model.layers.35.mlp.experts.120.up_proj", + "model.layers.35.mlp.experts.121.down_proj", + "model.layers.35.mlp.experts.121.gate_proj", + "model.layers.35.mlp.experts.121.up_proj", + "model.layers.35.mlp.experts.122.down_proj", + "model.layers.35.mlp.experts.122.gate_proj", + "model.layers.35.mlp.experts.122.up_proj", + "model.layers.35.mlp.experts.123.down_proj", + "model.layers.35.mlp.experts.123.gate_proj", + "model.layers.35.mlp.experts.123.up_proj", + "model.layers.35.mlp.experts.124.down_proj", + "model.layers.35.mlp.experts.124.gate_proj", + "model.layers.35.mlp.experts.124.up_proj", + "model.layers.35.mlp.experts.125.down_proj", + "model.layers.35.mlp.experts.125.gate_proj", + "model.layers.35.mlp.experts.125.up_proj", + "model.layers.35.mlp.experts.126.down_proj", + "model.layers.35.mlp.experts.126.gate_proj", + "model.layers.35.mlp.experts.126.up_proj", + "model.layers.35.mlp.experts.127.down_proj", + "model.layers.35.mlp.experts.127.gate_proj", + "model.layers.35.mlp.experts.127.up_proj", + "model.layers.35.mlp.experts.128.down_proj", + "model.layers.35.mlp.experts.128.gate_proj", + "model.layers.35.mlp.experts.128.up_proj", + "model.layers.35.mlp.experts.129.down_proj", + "model.layers.35.mlp.experts.129.gate_proj", + "model.layers.35.mlp.experts.129.up_proj", + "model.layers.35.mlp.experts.13.down_proj", + "model.layers.35.mlp.experts.13.gate_proj", + "model.layers.35.mlp.experts.13.up_proj", + "model.layers.35.mlp.experts.130.down_proj", + "model.layers.35.mlp.experts.130.gate_proj", + "model.layers.35.mlp.experts.130.up_proj", + "model.layers.35.mlp.experts.131.down_proj", + "model.layers.35.mlp.experts.131.gate_proj", + "model.layers.35.mlp.experts.131.up_proj", + "model.layers.35.mlp.experts.132.down_proj", + "model.layers.35.mlp.experts.132.gate_proj", + "model.layers.35.mlp.experts.132.up_proj", + "model.layers.35.mlp.experts.133.down_proj", + "model.layers.35.mlp.experts.133.gate_proj", + "model.layers.35.mlp.experts.133.up_proj", + "model.layers.35.mlp.experts.134.down_proj", + "model.layers.35.mlp.experts.134.gate_proj", + "model.layers.35.mlp.experts.134.up_proj", + "model.layers.35.mlp.experts.135.down_proj", + "model.layers.35.mlp.experts.135.gate_proj", + "model.layers.35.mlp.experts.135.up_proj", + "model.layers.35.mlp.experts.136.down_proj", + "model.layers.35.mlp.experts.136.gate_proj", + "model.layers.35.mlp.experts.136.up_proj", + "model.layers.35.mlp.experts.137.down_proj", + "model.layers.35.mlp.experts.137.gate_proj", + "model.layers.35.mlp.experts.137.up_proj", + "model.layers.35.mlp.experts.138.down_proj", + "model.layers.35.mlp.experts.138.gate_proj", + "model.layers.35.mlp.experts.138.up_proj", + "model.layers.35.mlp.experts.139.down_proj", + "model.layers.35.mlp.experts.139.gate_proj", + "model.layers.35.mlp.experts.139.up_proj", + "model.layers.35.mlp.experts.14.down_proj", + "model.layers.35.mlp.experts.14.gate_proj", + "model.layers.35.mlp.experts.14.up_proj", + "model.layers.35.mlp.experts.140.down_proj", + "model.layers.35.mlp.experts.140.gate_proj", + "model.layers.35.mlp.experts.140.up_proj", + "model.layers.35.mlp.experts.141.down_proj", + "model.layers.35.mlp.experts.141.gate_proj", + "model.layers.35.mlp.experts.141.up_proj", + "model.layers.35.mlp.experts.142.down_proj", + "model.layers.35.mlp.experts.142.gate_proj", + "model.layers.35.mlp.experts.142.up_proj", + "model.layers.35.mlp.experts.143.down_proj", + "model.layers.35.mlp.experts.143.gate_proj", + "model.layers.35.mlp.experts.143.up_proj", + "model.layers.35.mlp.experts.144.down_proj", + "model.layers.35.mlp.experts.144.gate_proj", + "model.layers.35.mlp.experts.144.up_proj", + "model.layers.35.mlp.experts.145.down_proj", + "model.layers.35.mlp.experts.145.gate_proj", + "model.layers.35.mlp.experts.145.up_proj", + "model.layers.35.mlp.experts.146.down_proj", + "model.layers.35.mlp.experts.146.gate_proj", + "model.layers.35.mlp.experts.146.up_proj", + "model.layers.35.mlp.experts.147.down_proj", + "model.layers.35.mlp.experts.147.gate_proj", + "model.layers.35.mlp.experts.147.up_proj", + "model.layers.35.mlp.experts.148.down_proj", + "model.layers.35.mlp.experts.148.gate_proj", + "model.layers.35.mlp.experts.148.up_proj", + "model.layers.35.mlp.experts.149.down_proj", + "model.layers.35.mlp.experts.149.gate_proj", + "model.layers.35.mlp.experts.149.up_proj", + "model.layers.35.mlp.experts.15.down_proj", + "model.layers.35.mlp.experts.15.gate_proj", + "model.layers.35.mlp.experts.15.up_proj", + "model.layers.35.mlp.experts.150.down_proj", + "model.layers.35.mlp.experts.150.gate_proj", + "model.layers.35.mlp.experts.150.up_proj", + "model.layers.35.mlp.experts.151.down_proj", + "model.layers.35.mlp.experts.151.gate_proj", + "model.layers.35.mlp.experts.151.up_proj", + "model.layers.35.mlp.experts.152.down_proj", + "model.layers.35.mlp.experts.152.gate_proj", + "model.layers.35.mlp.experts.152.up_proj", + "model.layers.35.mlp.experts.153.down_proj", + "model.layers.35.mlp.experts.153.gate_proj", + "model.layers.35.mlp.experts.153.up_proj", + "model.layers.35.mlp.experts.154.down_proj", + "model.layers.35.mlp.experts.154.gate_proj", + "model.layers.35.mlp.experts.154.up_proj", + "model.layers.35.mlp.experts.155.down_proj", + "model.layers.35.mlp.experts.155.gate_proj", + "model.layers.35.mlp.experts.155.up_proj", + "model.layers.35.mlp.experts.156.down_proj", + "model.layers.35.mlp.experts.156.gate_proj", + "model.layers.35.mlp.experts.156.up_proj", + "model.layers.35.mlp.experts.157.down_proj", + "model.layers.35.mlp.experts.157.gate_proj", + "model.layers.35.mlp.experts.157.up_proj", + "model.layers.35.mlp.experts.158.down_proj", + "model.layers.35.mlp.experts.158.gate_proj", + "model.layers.35.mlp.experts.158.up_proj", + "model.layers.35.mlp.experts.159.down_proj", + "model.layers.35.mlp.experts.159.gate_proj", + "model.layers.35.mlp.experts.159.up_proj", + "model.layers.35.mlp.experts.16.down_proj", + "model.layers.35.mlp.experts.16.gate_proj", + "model.layers.35.mlp.experts.16.up_proj", + "model.layers.35.mlp.experts.160.down_proj", + "model.layers.35.mlp.experts.160.gate_proj", + "model.layers.35.mlp.experts.160.up_proj", + "model.layers.35.mlp.experts.161.down_proj", + "model.layers.35.mlp.experts.161.gate_proj", + "model.layers.35.mlp.experts.161.up_proj", + "model.layers.35.mlp.experts.162.down_proj", + "model.layers.35.mlp.experts.162.gate_proj", + "model.layers.35.mlp.experts.162.up_proj", + "model.layers.35.mlp.experts.163.down_proj", + "model.layers.35.mlp.experts.163.gate_proj", + "model.layers.35.mlp.experts.163.up_proj", + "model.layers.35.mlp.experts.164.down_proj", + "model.layers.35.mlp.experts.164.gate_proj", + "model.layers.35.mlp.experts.164.up_proj", + "model.layers.35.mlp.experts.165.down_proj", + "model.layers.35.mlp.experts.165.gate_proj", + "model.layers.35.mlp.experts.165.up_proj", + "model.layers.35.mlp.experts.166.down_proj", + "model.layers.35.mlp.experts.166.gate_proj", + "model.layers.35.mlp.experts.166.up_proj", + "model.layers.35.mlp.experts.167.down_proj", + "model.layers.35.mlp.experts.167.gate_proj", + "model.layers.35.mlp.experts.167.up_proj", + "model.layers.35.mlp.experts.168.down_proj", + "model.layers.35.mlp.experts.168.gate_proj", + "model.layers.35.mlp.experts.168.up_proj", + "model.layers.35.mlp.experts.169.down_proj", + "model.layers.35.mlp.experts.169.gate_proj", + "model.layers.35.mlp.experts.169.up_proj", + "model.layers.35.mlp.experts.17.down_proj", + "model.layers.35.mlp.experts.17.gate_proj", + "model.layers.35.mlp.experts.17.up_proj", + "model.layers.35.mlp.experts.170.down_proj", + "model.layers.35.mlp.experts.170.gate_proj", + "model.layers.35.mlp.experts.170.up_proj", + "model.layers.35.mlp.experts.171.down_proj", + "model.layers.35.mlp.experts.171.gate_proj", + "model.layers.35.mlp.experts.171.up_proj", + "model.layers.35.mlp.experts.172.down_proj", + "model.layers.35.mlp.experts.172.gate_proj", + "model.layers.35.mlp.experts.172.up_proj", + "model.layers.35.mlp.experts.173.down_proj", + "model.layers.35.mlp.experts.173.gate_proj", + "model.layers.35.mlp.experts.173.up_proj", + "model.layers.35.mlp.experts.174.down_proj", + "model.layers.35.mlp.experts.174.gate_proj", + "model.layers.35.mlp.experts.174.up_proj", + "model.layers.35.mlp.experts.175.down_proj", + "model.layers.35.mlp.experts.175.gate_proj", + "model.layers.35.mlp.experts.175.up_proj", + "model.layers.35.mlp.experts.176.down_proj", + "model.layers.35.mlp.experts.176.gate_proj", + "model.layers.35.mlp.experts.176.up_proj", + "model.layers.35.mlp.experts.177.down_proj", + "model.layers.35.mlp.experts.177.gate_proj", + "model.layers.35.mlp.experts.177.up_proj", + "model.layers.35.mlp.experts.178.down_proj", + "model.layers.35.mlp.experts.178.gate_proj", + "model.layers.35.mlp.experts.178.up_proj", + "model.layers.35.mlp.experts.179.down_proj", + "model.layers.35.mlp.experts.179.gate_proj", + "model.layers.35.mlp.experts.179.up_proj", + "model.layers.35.mlp.experts.18.down_proj", + "model.layers.35.mlp.experts.18.gate_proj", + "model.layers.35.mlp.experts.18.up_proj", + "model.layers.35.mlp.experts.180.down_proj", + "model.layers.35.mlp.experts.180.gate_proj", + "model.layers.35.mlp.experts.180.up_proj", + "model.layers.35.mlp.experts.181.down_proj", + "model.layers.35.mlp.experts.181.gate_proj", + "model.layers.35.mlp.experts.181.up_proj", + "model.layers.35.mlp.experts.182.down_proj", + "model.layers.35.mlp.experts.182.gate_proj", + "model.layers.35.mlp.experts.182.up_proj", + "model.layers.35.mlp.experts.183.down_proj", + "model.layers.35.mlp.experts.183.gate_proj", + "model.layers.35.mlp.experts.183.up_proj", + "model.layers.35.mlp.experts.184.down_proj", + "model.layers.35.mlp.experts.184.gate_proj", + "model.layers.35.mlp.experts.184.up_proj", + "model.layers.35.mlp.experts.185.down_proj", + "model.layers.35.mlp.experts.185.gate_proj", + "model.layers.35.mlp.experts.185.up_proj", + "model.layers.35.mlp.experts.186.down_proj", + "model.layers.35.mlp.experts.186.gate_proj", + "model.layers.35.mlp.experts.186.up_proj", + "model.layers.35.mlp.experts.187.down_proj", + "model.layers.35.mlp.experts.187.gate_proj", + "model.layers.35.mlp.experts.187.up_proj", + "model.layers.35.mlp.experts.188.down_proj", + "model.layers.35.mlp.experts.188.gate_proj", + "model.layers.35.mlp.experts.188.up_proj", + "model.layers.35.mlp.experts.189.down_proj", + "model.layers.35.mlp.experts.189.gate_proj", + "model.layers.35.mlp.experts.189.up_proj", + "model.layers.35.mlp.experts.19.down_proj", + "model.layers.35.mlp.experts.19.gate_proj", + "model.layers.35.mlp.experts.19.up_proj", + "model.layers.35.mlp.experts.190.down_proj", + "model.layers.35.mlp.experts.190.gate_proj", + "model.layers.35.mlp.experts.190.up_proj", + "model.layers.35.mlp.experts.191.down_proj", + "model.layers.35.mlp.experts.191.gate_proj", + "model.layers.35.mlp.experts.191.up_proj", + "model.layers.35.mlp.experts.192.down_proj", + "model.layers.35.mlp.experts.192.gate_proj", + "model.layers.35.mlp.experts.192.up_proj", + "model.layers.35.mlp.experts.193.down_proj", + "model.layers.35.mlp.experts.193.gate_proj", + "model.layers.35.mlp.experts.193.up_proj", + "model.layers.35.mlp.experts.194.down_proj", + "model.layers.35.mlp.experts.194.gate_proj", + "model.layers.35.mlp.experts.194.up_proj", + "model.layers.35.mlp.experts.195.down_proj", + "model.layers.35.mlp.experts.195.gate_proj", + "model.layers.35.mlp.experts.195.up_proj", + "model.layers.35.mlp.experts.196.down_proj", + "model.layers.35.mlp.experts.196.gate_proj", + "model.layers.35.mlp.experts.196.up_proj", + "model.layers.35.mlp.experts.197.down_proj", + "model.layers.35.mlp.experts.197.gate_proj", + "model.layers.35.mlp.experts.197.up_proj", + "model.layers.35.mlp.experts.198.down_proj", + "model.layers.35.mlp.experts.198.gate_proj", + "model.layers.35.mlp.experts.198.up_proj", + "model.layers.35.mlp.experts.199.down_proj", + "model.layers.35.mlp.experts.199.gate_proj", + "model.layers.35.mlp.experts.199.up_proj", + "model.layers.35.mlp.experts.2.down_proj", + "model.layers.35.mlp.experts.2.gate_proj", + "model.layers.35.mlp.experts.2.up_proj", + "model.layers.35.mlp.experts.20.down_proj", + "model.layers.35.mlp.experts.20.gate_proj", + "model.layers.35.mlp.experts.20.up_proj", + "model.layers.35.mlp.experts.200.down_proj", + "model.layers.35.mlp.experts.200.gate_proj", + "model.layers.35.mlp.experts.200.up_proj", + "model.layers.35.mlp.experts.201.down_proj", + "model.layers.35.mlp.experts.201.gate_proj", + "model.layers.35.mlp.experts.201.up_proj", + "model.layers.35.mlp.experts.202.down_proj", + "model.layers.35.mlp.experts.202.gate_proj", + "model.layers.35.mlp.experts.202.up_proj", + "model.layers.35.mlp.experts.203.down_proj", + "model.layers.35.mlp.experts.203.gate_proj", + "model.layers.35.mlp.experts.203.up_proj", + "model.layers.35.mlp.experts.204.down_proj", + "model.layers.35.mlp.experts.204.gate_proj", + "model.layers.35.mlp.experts.204.up_proj", + "model.layers.35.mlp.experts.205.down_proj", + "model.layers.35.mlp.experts.205.gate_proj", + "model.layers.35.mlp.experts.205.up_proj", + "model.layers.35.mlp.experts.206.down_proj", + "model.layers.35.mlp.experts.206.gate_proj", + "model.layers.35.mlp.experts.206.up_proj", + "model.layers.35.mlp.experts.207.down_proj", + "model.layers.35.mlp.experts.207.gate_proj", + "model.layers.35.mlp.experts.207.up_proj", + "model.layers.35.mlp.experts.208.down_proj", + "model.layers.35.mlp.experts.208.gate_proj", + "model.layers.35.mlp.experts.208.up_proj", + "model.layers.35.mlp.experts.209.down_proj", + "model.layers.35.mlp.experts.209.gate_proj", + "model.layers.35.mlp.experts.209.up_proj", + "model.layers.35.mlp.experts.21.down_proj", + "model.layers.35.mlp.experts.21.gate_proj", + "model.layers.35.mlp.experts.21.up_proj", + "model.layers.35.mlp.experts.210.down_proj", + "model.layers.35.mlp.experts.210.gate_proj", + "model.layers.35.mlp.experts.210.up_proj", + "model.layers.35.mlp.experts.211.down_proj", + "model.layers.35.mlp.experts.211.gate_proj", + "model.layers.35.mlp.experts.211.up_proj", + "model.layers.35.mlp.experts.212.down_proj", + "model.layers.35.mlp.experts.212.gate_proj", + "model.layers.35.mlp.experts.212.up_proj", + "model.layers.35.mlp.experts.213.down_proj", + "model.layers.35.mlp.experts.213.gate_proj", + "model.layers.35.mlp.experts.213.up_proj", + "model.layers.35.mlp.experts.214.down_proj", + "model.layers.35.mlp.experts.214.gate_proj", + "model.layers.35.mlp.experts.214.up_proj", + "model.layers.35.mlp.experts.215.down_proj", + "model.layers.35.mlp.experts.215.gate_proj", + "model.layers.35.mlp.experts.215.up_proj", + "model.layers.35.mlp.experts.216.down_proj", + "model.layers.35.mlp.experts.216.gate_proj", + "model.layers.35.mlp.experts.216.up_proj", + "model.layers.35.mlp.experts.217.down_proj", + "model.layers.35.mlp.experts.217.gate_proj", + "model.layers.35.mlp.experts.217.up_proj", + "model.layers.35.mlp.experts.218.down_proj", + "model.layers.35.mlp.experts.218.gate_proj", + "model.layers.35.mlp.experts.218.up_proj", + "model.layers.35.mlp.experts.219.down_proj", + "model.layers.35.mlp.experts.219.gate_proj", + "model.layers.35.mlp.experts.219.up_proj", + "model.layers.35.mlp.experts.22.down_proj", + "model.layers.35.mlp.experts.22.gate_proj", + "model.layers.35.mlp.experts.22.up_proj", + "model.layers.35.mlp.experts.220.down_proj", + "model.layers.35.mlp.experts.220.gate_proj", + "model.layers.35.mlp.experts.220.up_proj", + "model.layers.35.mlp.experts.221.down_proj", + "model.layers.35.mlp.experts.221.gate_proj", + "model.layers.35.mlp.experts.221.up_proj", + "model.layers.35.mlp.experts.222.down_proj", + "model.layers.35.mlp.experts.222.gate_proj", + "model.layers.35.mlp.experts.222.up_proj", + "model.layers.35.mlp.experts.223.down_proj", + "model.layers.35.mlp.experts.223.gate_proj", + "model.layers.35.mlp.experts.223.up_proj", + "model.layers.35.mlp.experts.224.down_proj", + "model.layers.35.mlp.experts.224.gate_proj", + "model.layers.35.mlp.experts.224.up_proj", + "model.layers.35.mlp.experts.225.down_proj", + "model.layers.35.mlp.experts.225.gate_proj", + "model.layers.35.mlp.experts.225.up_proj", + "model.layers.35.mlp.experts.226.down_proj", + "model.layers.35.mlp.experts.226.gate_proj", + "model.layers.35.mlp.experts.226.up_proj", + "model.layers.35.mlp.experts.227.down_proj", + "model.layers.35.mlp.experts.227.gate_proj", + "model.layers.35.mlp.experts.227.up_proj", + "model.layers.35.mlp.experts.228.down_proj", + "model.layers.35.mlp.experts.228.gate_proj", + "model.layers.35.mlp.experts.228.up_proj", + "model.layers.35.mlp.experts.229.down_proj", + "model.layers.35.mlp.experts.229.gate_proj", + "model.layers.35.mlp.experts.229.up_proj", + "model.layers.35.mlp.experts.23.down_proj", + "model.layers.35.mlp.experts.23.gate_proj", + "model.layers.35.mlp.experts.23.up_proj", + "model.layers.35.mlp.experts.230.down_proj", + "model.layers.35.mlp.experts.230.gate_proj", + "model.layers.35.mlp.experts.230.up_proj", + "model.layers.35.mlp.experts.231.down_proj", + "model.layers.35.mlp.experts.231.gate_proj", + "model.layers.35.mlp.experts.231.up_proj", + "model.layers.35.mlp.experts.232.down_proj", + "model.layers.35.mlp.experts.232.gate_proj", + "model.layers.35.mlp.experts.232.up_proj", + "model.layers.35.mlp.experts.233.down_proj", + "model.layers.35.mlp.experts.233.gate_proj", + "model.layers.35.mlp.experts.233.up_proj", + "model.layers.35.mlp.experts.234.down_proj", + "model.layers.35.mlp.experts.234.gate_proj", + "model.layers.35.mlp.experts.234.up_proj", + "model.layers.35.mlp.experts.235.down_proj", + "model.layers.35.mlp.experts.235.gate_proj", + "model.layers.35.mlp.experts.235.up_proj", + "model.layers.35.mlp.experts.236.down_proj", + "model.layers.35.mlp.experts.236.gate_proj", + "model.layers.35.mlp.experts.236.up_proj", + "model.layers.35.mlp.experts.237.down_proj", + "model.layers.35.mlp.experts.237.gate_proj", + "model.layers.35.mlp.experts.237.up_proj", + "model.layers.35.mlp.experts.238.down_proj", + "model.layers.35.mlp.experts.238.gate_proj", + "model.layers.35.mlp.experts.238.up_proj", + "model.layers.35.mlp.experts.239.down_proj", + "model.layers.35.mlp.experts.239.gate_proj", + "model.layers.35.mlp.experts.239.up_proj", + "model.layers.35.mlp.experts.24.down_proj", + "model.layers.35.mlp.experts.24.gate_proj", + "model.layers.35.mlp.experts.24.up_proj", + "model.layers.35.mlp.experts.240.down_proj", + "model.layers.35.mlp.experts.240.gate_proj", + "model.layers.35.mlp.experts.240.up_proj", + "model.layers.35.mlp.experts.241.down_proj", + "model.layers.35.mlp.experts.241.gate_proj", + "model.layers.35.mlp.experts.241.up_proj", + "model.layers.35.mlp.experts.242.down_proj", + "model.layers.35.mlp.experts.242.gate_proj", + "model.layers.35.mlp.experts.242.up_proj", + "model.layers.35.mlp.experts.243.down_proj", + "model.layers.35.mlp.experts.243.gate_proj", + "model.layers.35.mlp.experts.243.up_proj", + "model.layers.35.mlp.experts.244.down_proj", + "model.layers.35.mlp.experts.244.gate_proj", + "model.layers.35.mlp.experts.244.up_proj", + "model.layers.35.mlp.experts.245.down_proj", + "model.layers.35.mlp.experts.245.gate_proj", + "model.layers.35.mlp.experts.245.up_proj", + "model.layers.35.mlp.experts.246.down_proj", + "model.layers.35.mlp.experts.246.gate_proj", + "model.layers.35.mlp.experts.246.up_proj", + "model.layers.35.mlp.experts.247.down_proj", + "model.layers.35.mlp.experts.247.gate_proj", + "model.layers.35.mlp.experts.247.up_proj", + "model.layers.35.mlp.experts.248.down_proj", + "model.layers.35.mlp.experts.248.gate_proj", + "model.layers.35.mlp.experts.248.up_proj", + "model.layers.35.mlp.experts.249.down_proj", + "model.layers.35.mlp.experts.249.gate_proj", + "model.layers.35.mlp.experts.249.up_proj", + "model.layers.35.mlp.experts.25.down_proj", + "model.layers.35.mlp.experts.25.gate_proj", + "model.layers.35.mlp.experts.25.up_proj", + "model.layers.35.mlp.experts.250.down_proj", + "model.layers.35.mlp.experts.250.gate_proj", + "model.layers.35.mlp.experts.250.up_proj", + "model.layers.35.mlp.experts.251.down_proj", + "model.layers.35.mlp.experts.251.gate_proj", + "model.layers.35.mlp.experts.251.up_proj", + "model.layers.35.mlp.experts.252.down_proj", + "model.layers.35.mlp.experts.252.gate_proj", + "model.layers.35.mlp.experts.252.up_proj", + "model.layers.35.mlp.experts.253.down_proj", + "model.layers.35.mlp.experts.253.gate_proj", + "model.layers.35.mlp.experts.253.up_proj", + "model.layers.35.mlp.experts.254.down_proj", + "model.layers.35.mlp.experts.254.gate_proj", + "model.layers.35.mlp.experts.254.up_proj", + "model.layers.35.mlp.experts.255.down_proj", + "model.layers.35.mlp.experts.255.gate_proj", + "model.layers.35.mlp.experts.255.up_proj", + "model.layers.35.mlp.experts.26.down_proj", + "model.layers.35.mlp.experts.26.gate_proj", + "model.layers.35.mlp.experts.26.up_proj", + "model.layers.35.mlp.experts.27.down_proj", + "model.layers.35.mlp.experts.27.gate_proj", + "model.layers.35.mlp.experts.27.up_proj", + "model.layers.35.mlp.experts.28.down_proj", + "model.layers.35.mlp.experts.28.gate_proj", + "model.layers.35.mlp.experts.28.up_proj", + "model.layers.35.mlp.experts.29.down_proj", + "model.layers.35.mlp.experts.29.gate_proj", + "model.layers.35.mlp.experts.29.up_proj", + "model.layers.35.mlp.experts.3.down_proj", + "model.layers.35.mlp.experts.3.gate_proj", + "model.layers.35.mlp.experts.3.up_proj", + "model.layers.35.mlp.experts.30.down_proj", + "model.layers.35.mlp.experts.30.gate_proj", + "model.layers.35.mlp.experts.30.up_proj", + "model.layers.35.mlp.experts.31.down_proj", + "model.layers.35.mlp.experts.31.gate_proj", + "model.layers.35.mlp.experts.31.up_proj", + "model.layers.35.mlp.experts.32.down_proj", + "model.layers.35.mlp.experts.32.gate_proj", + "model.layers.35.mlp.experts.32.up_proj", + "model.layers.35.mlp.experts.33.down_proj", + "model.layers.35.mlp.experts.33.gate_proj", + "model.layers.35.mlp.experts.33.up_proj", + "model.layers.35.mlp.experts.34.down_proj", + "model.layers.35.mlp.experts.34.gate_proj", + "model.layers.35.mlp.experts.34.up_proj", + "model.layers.35.mlp.experts.35.down_proj", + "model.layers.35.mlp.experts.35.gate_proj", + "model.layers.35.mlp.experts.35.up_proj", + "model.layers.35.mlp.experts.36.down_proj", + "model.layers.35.mlp.experts.36.gate_proj", + "model.layers.35.mlp.experts.36.up_proj", + "model.layers.35.mlp.experts.37.down_proj", + "model.layers.35.mlp.experts.37.gate_proj", + "model.layers.35.mlp.experts.37.up_proj", + "model.layers.35.mlp.experts.38.down_proj", + "model.layers.35.mlp.experts.38.gate_proj", + "model.layers.35.mlp.experts.38.up_proj", + "model.layers.35.mlp.experts.39.down_proj", + "model.layers.35.mlp.experts.39.gate_proj", + "model.layers.35.mlp.experts.39.up_proj", + "model.layers.35.mlp.experts.4.down_proj", + "model.layers.35.mlp.experts.4.gate_proj", + "model.layers.35.mlp.experts.4.up_proj", + "model.layers.35.mlp.experts.40.down_proj", + "model.layers.35.mlp.experts.40.gate_proj", + "model.layers.35.mlp.experts.40.up_proj", + "model.layers.35.mlp.experts.41.down_proj", + "model.layers.35.mlp.experts.41.gate_proj", + "model.layers.35.mlp.experts.41.up_proj", + "model.layers.35.mlp.experts.42.down_proj", + "model.layers.35.mlp.experts.42.gate_proj", + "model.layers.35.mlp.experts.42.up_proj", + "model.layers.35.mlp.experts.43.down_proj", + "model.layers.35.mlp.experts.43.gate_proj", + "model.layers.35.mlp.experts.43.up_proj", + "model.layers.35.mlp.experts.44.down_proj", + "model.layers.35.mlp.experts.44.gate_proj", + "model.layers.35.mlp.experts.44.up_proj", + "model.layers.35.mlp.experts.45.down_proj", + "model.layers.35.mlp.experts.45.gate_proj", + "model.layers.35.mlp.experts.45.up_proj", + "model.layers.35.mlp.experts.46.down_proj", + "model.layers.35.mlp.experts.46.gate_proj", + "model.layers.35.mlp.experts.46.up_proj", + "model.layers.35.mlp.experts.47.down_proj", + "model.layers.35.mlp.experts.47.gate_proj", + "model.layers.35.mlp.experts.47.up_proj", + "model.layers.35.mlp.experts.48.down_proj", + "model.layers.35.mlp.experts.48.gate_proj", + "model.layers.35.mlp.experts.48.up_proj", + "model.layers.35.mlp.experts.49.down_proj", + "model.layers.35.mlp.experts.49.gate_proj", + "model.layers.35.mlp.experts.49.up_proj", + "model.layers.35.mlp.experts.5.down_proj", + "model.layers.35.mlp.experts.5.gate_proj", + "model.layers.35.mlp.experts.5.up_proj", + "model.layers.35.mlp.experts.50.down_proj", + "model.layers.35.mlp.experts.50.gate_proj", + "model.layers.35.mlp.experts.50.up_proj", + "model.layers.35.mlp.experts.51.down_proj", + "model.layers.35.mlp.experts.51.gate_proj", + "model.layers.35.mlp.experts.51.up_proj", + "model.layers.35.mlp.experts.52.down_proj", + "model.layers.35.mlp.experts.52.gate_proj", + "model.layers.35.mlp.experts.52.up_proj", + "model.layers.35.mlp.experts.53.down_proj", + "model.layers.35.mlp.experts.53.gate_proj", + "model.layers.35.mlp.experts.53.up_proj", + "model.layers.35.mlp.experts.54.down_proj", + "model.layers.35.mlp.experts.54.gate_proj", + "model.layers.35.mlp.experts.54.up_proj", + "model.layers.35.mlp.experts.55.down_proj", + "model.layers.35.mlp.experts.55.gate_proj", + "model.layers.35.mlp.experts.55.up_proj", + "model.layers.35.mlp.experts.56.down_proj", + "model.layers.35.mlp.experts.56.gate_proj", + "model.layers.35.mlp.experts.56.up_proj", + "model.layers.35.mlp.experts.57.down_proj", + "model.layers.35.mlp.experts.57.gate_proj", + "model.layers.35.mlp.experts.57.up_proj", + "model.layers.35.mlp.experts.58.down_proj", + "model.layers.35.mlp.experts.58.gate_proj", + "model.layers.35.mlp.experts.58.up_proj", + "model.layers.35.mlp.experts.59.down_proj", + "model.layers.35.mlp.experts.59.gate_proj", + "model.layers.35.mlp.experts.59.up_proj", + "model.layers.35.mlp.experts.6.down_proj", + "model.layers.35.mlp.experts.6.gate_proj", + "model.layers.35.mlp.experts.6.up_proj", + "model.layers.35.mlp.experts.60.down_proj", + "model.layers.35.mlp.experts.60.gate_proj", + "model.layers.35.mlp.experts.60.up_proj", + "model.layers.35.mlp.experts.61.down_proj", + "model.layers.35.mlp.experts.61.gate_proj", + "model.layers.35.mlp.experts.61.up_proj", + "model.layers.35.mlp.experts.62.down_proj", + "model.layers.35.mlp.experts.62.gate_proj", + "model.layers.35.mlp.experts.62.up_proj", + "model.layers.35.mlp.experts.63.down_proj", + "model.layers.35.mlp.experts.63.gate_proj", + "model.layers.35.mlp.experts.63.up_proj", + "model.layers.35.mlp.experts.64.down_proj", + "model.layers.35.mlp.experts.64.gate_proj", + "model.layers.35.mlp.experts.64.up_proj", + "model.layers.35.mlp.experts.65.down_proj", + "model.layers.35.mlp.experts.65.gate_proj", + "model.layers.35.mlp.experts.65.up_proj", + "model.layers.35.mlp.experts.66.down_proj", + "model.layers.35.mlp.experts.66.gate_proj", + "model.layers.35.mlp.experts.66.up_proj", + "model.layers.35.mlp.experts.67.down_proj", + "model.layers.35.mlp.experts.67.gate_proj", + "model.layers.35.mlp.experts.67.up_proj", + "model.layers.35.mlp.experts.68.down_proj", + "model.layers.35.mlp.experts.68.gate_proj", + "model.layers.35.mlp.experts.68.up_proj", + "model.layers.35.mlp.experts.69.down_proj", + "model.layers.35.mlp.experts.69.gate_proj", + "model.layers.35.mlp.experts.69.up_proj", + "model.layers.35.mlp.experts.7.down_proj", + "model.layers.35.mlp.experts.7.gate_proj", + "model.layers.35.mlp.experts.7.up_proj", + "model.layers.35.mlp.experts.70.down_proj", + "model.layers.35.mlp.experts.70.gate_proj", + "model.layers.35.mlp.experts.70.up_proj", + "model.layers.35.mlp.experts.71.down_proj", + "model.layers.35.mlp.experts.71.gate_proj", + "model.layers.35.mlp.experts.71.up_proj", + "model.layers.35.mlp.experts.72.down_proj", + "model.layers.35.mlp.experts.72.gate_proj", + "model.layers.35.mlp.experts.72.up_proj", + "model.layers.35.mlp.experts.73.down_proj", + "model.layers.35.mlp.experts.73.gate_proj", + "model.layers.35.mlp.experts.73.up_proj", + "model.layers.35.mlp.experts.74.down_proj", + "model.layers.35.mlp.experts.74.gate_proj", + "model.layers.35.mlp.experts.74.up_proj", + "model.layers.35.mlp.experts.75.down_proj", + "model.layers.35.mlp.experts.75.gate_proj", + "model.layers.35.mlp.experts.75.up_proj", + "model.layers.35.mlp.experts.76.down_proj", + "model.layers.35.mlp.experts.76.gate_proj", + "model.layers.35.mlp.experts.76.up_proj", + "model.layers.35.mlp.experts.77.down_proj", + "model.layers.35.mlp.experts.77.gate_proj", + "model.layers.35.mlp.experts.77.up_proj", + "model.layers.35.mlp.experts.78.down_proj", + "model.layers.35.mlp.experts.78.gate_proj", + "model.layers.35.mlp.experts.78.up_proj", + "model.layers.35.mlp.experts.79.down_proj", + "model.layers.35.mlp.experts.79.gate_proj", + "model.layers.35.mlp.experts.79.up_proj", + "model.layers.35.mlp.experts.8.down_proj", + "model.layers.35.mlp.experts.8.gate_proj", + "model.layers.35.mlp.experts.8.up_proj", + "model.layers.35.mlp.experts.80.down_proj", + "model.layers.35.mlp.experts.80.gate_proj", + "model.layers.35.mlp.experts.80.up_proj", + "model.layers.35.mlp.experts.81.down_proj", + "model.layers.35.mlp.experts.81.gate_proj", + "model.layers.35.mlp.experts.81.up_proj", + "model.layers.35.mlp.experts.82.down_proj", + "model.layers.35.mlp.experts.82.gate_proj", + "model.layers.35.mlp.experts.82.up_proj", + "model.layers.35.mlp.experts.83.down_proj", + "model.layers.35.mlp.experts.83.gate_proj", + "model.layers.35.mlp.experts.83.up_proj", + "model.layers.35.mlp.experts.84.down_proj", + "model.layers.35.mlp.experts.84.gate_proj", + "model.layers.35.mlp.experts.84.up_proj", + "model.layers.35.mlp.experts.85.down_proj", + "model.layers.35.mlp.experts.85.gate_proj", + "model.layers.35.mlp.experts.85.up_proj", + "model.layers.35.mlp.experts.86.down_proj", + "model.layers.35.mlp.experts.86.gate_proj", + "model.layers.35.mlp.experts.86.up_proj", + "model.layers.35.mlp.experts.87.down_proj", + "model.layers.35.mlp.experts.87.gate_proj", + "model.layers.35.mlp.experts.87.up_proj", + "model.layers.35.mlp.experts.88.down_proj", + "model.layers.35.mlp.experts.88.gate_proj", + "model.layers.35.mlp.experts.88.up_proj", + "model.layers.35.mlp.experts.89.down_proj", + "model.layers.35.mlp.experts.89.gate_proj", + "model.layers.35.mlp.experts.89.up_proj", + "model.layers.35.mlp.experts.9.down_proj", + "model.layers.35.mlp.experts.9.gate_proj", + "model.layers.35.mlp.experts.9.up_proj", + "model.layers.35.mlp.experts.90.down_proj", + "model.layers.35.mlp.experts.90.gate_proj", + "model.layers.35.mlp.experts.90.up_proj", + "model.layers.35.mlp.experts.91.down_proj", + "model.layers.35.mlp.experts.91.gate_proj", + "model.layers.35.mlp.experts.91.up_proj", + "model.layers.35.mlp.experts.92.down_proj", + "model.layers.35.mlp.experts.92.gate_proj", + "model.layers.35.mlp.experts.92.up_proj", + "model.layers.35.mlp.experts.93.down_proj", + "model.layers.35.mlp.experts.93.gate_proj", + "model.layers.35.mlp.experts.93.up_proj", + "model.layers.35.mlp.experts.94.down_proj", + "model.layers.35.mlp.experts.94.gate_proj", + "model.layers.35.mlp.experts.94.up_proj", + "model.layers.35.mlp.experts.95.down_proj", + "model.layers.35.mlp.experts.95.gate_proj", + "model.layers.35.mlp.experts.95.up_proj", + "model.layers.35.mlp.experts.96.down_proj", + "model.layers.35.mlp.experts.96.gate_proj", + "model.layers.35.mlp.experts.96.up_proj", + "model.layers.35.mlp.experts.97.down_proj", + "model.layers.35.mlp.experts.97.gate_proj", + "model.layers.35.mlp.experts.97.up_proj", + "model.layers.35.mlp.experts.98.down_proj", + "model.layers.35.mlp.experts.98.gate_proj", + "model.layers.35.mlp.experts.98.up_proj", + "model.layers.35.mlp.experts.99.down_proj", + "model.layers.35.mlp.experts.99.gate_proj", + "model.layers.35.mlp.experts.99.up_proj", + "model.layers.36.mlp.experts.0.down_proj", + "model.layers.36.mlp.experts.0.gate_proj", + "model.layers.36.mlp.experts.0.up_proj", + "model.layers.36.mlp.experts.1.down_proj", + "model.layers.36.mlp.experts.1.gate_proj", + "model.layers.36.mlp.experts.1.up_proj", + "model.layers.36.mlp.experts.10.down_proj", + "model.layers.36.mlp.experts.10.gate_proj", + "model.layers.36.mlp.experts.10.up_proj", + "model.layers.36.mlp.experts.100.down_proj", + "model.layers.36.mlp.experts.100.gate_proj", + "model.layers.36.mlp.experts.100.up_proj", + "model.layers.36.mlp.experts.101.down_proj", + "model.layers.36.mlp.experts.101.gate_proj", + "model.layers.36.mlp.experts.101.up_proj", + "model.layers.36.mlp.experts.102.down_proj", + "model.layers.36.mlp.experts.102.gate_proj", + "model.layers.36.mlp.experts.102.up_proj", + "model.layers.36.mlp.experts.103.down_proj", + "model.layers.36.mlp.experts.103.gate_proj", + "model.layers.36.mlp.experts.103.up_proj", + "model.layers.36.mlp.experts.104.down_proj", + "model.layers.36.mlp.experts.104.gate_proj", + "model.layers.36.mlp.experts.104.up_proj", + "model.layers.36.mlp.experts.105.down_proj", + "model.layers.36.mlp.experts.105.gate_proj", + "model.layers.36.mlp.experts.105.up_proj", + "model.layers.36.mlp.experts.106.down_proj", + "model.layers.36.mlp.experts.106.gate_proj", + "model.layers.36.mlp.experts.106.up_proj", + "model.layers.36.mlp.experts.107.down_proj", + "model.layers.36.mlp.experts.107.gate_proj", + "model.layers.36.mlp.experts.107.up_proj", + "model.layers.36.mlp.experts.108.down_proj", + "model.layers.36.mlp.experts.108.gate_proj", + "model.layers.36.mlp.experts.108.up_proj", + "model.layers.36.mlp.experts.109.down_proj", + "model.layers.36.mlp.experts.109.gate_proj", + "model.layers.36.mlp.experts.109.up_proj", + "model.layers.36.mlp.experts.11.down_proj", + "model.layers.36.mlp.experts.11.gate_proj", + "model.layers.36.mlp.experts.11.up_proj", + "model.layers.36.mlp.experts.110.down_proj", + "model.layers.36.mlp.experts.110.gate_proj", + "model.layers.36.mlp.experts.110.up_proj", + "model.layers.36.mlp.experts.111.down_proj", + "model.layers.36.mlp.experts.111.gate_proj", + "model.layers.36.mlp.experts.111.up_proj", + "model.layers.36.mlp.experts.112.down_proj", + "model.layers.36.mlp.experts.112.gate_proj", + "model.layers.36.mlp.experts.112.up_proj", + "model.layers.36.mlp.experts.113.down_proj", + "model.layers.36.mlp.experts.113.gate_proj", + "model.layers.36.mlp.experts.113.up_proj", + "model.layers.36.mlp.experts.114.down_proj", + "model.layers.36.mlp.experts.114.gate_proj", + "model.layers.36.mlp.experts.114.up_proj", + "model.layers.36.mlp.experts.115.down_proj", + "model.layers.36.mlp.experts.115.gate_proj", + "model.layers.36.mlp.experts.115.up_proj", + "model.layers.36.mlp.experts.116.down_proj", + "model.layers.36.mlp.experts.116.gate_proj", + "model.layers.36.mlp.experts.116.up_proj", + "model.layers.36.mlp.experts.117.down_proj", + "model.layers.36.mlp.experts.117.gate_proj", + "model.layers.36.mlp.experts.117.up_proj", + "model.layers.36.mlp.experts.118.down_proj", + "model.layers.36.mlp.experts.118.gate_proj", + "model.layers.36.mlp.experts.118.up_proj", + "model.layers.36.mlp.experts.119.down_proj", + "model.layers.36.mlp.experts.119.gate_proj", + "model.layers.36.mlp.experts.119.up_proj", + "model.layers.36.mlp.experts.12.down_proj", + "model.layers.36.mlp.experts.12.gate_proj", + "model.layers.36.mlp.experts.12.up_proj", + "model.layers.36.mlp.experts.120.down_proj", + "model.layers.36.mlp.experts.120.gate_proj", + "model.layers.36.mlp.experts.120.up_proj", + "model.layers.36.mlp.experts.121.down_proj", + "model.layers.36.mlp.experts.121.gate_proj", + "model.layers.36.mlp.experts.121.up_proj", + "model.layers.36.mlp.experts.122.down_proj", + "model.layers.36.mlp.experts.122.gate_proj", + "model.layers.36.mlp.experts.122.up_proj", + "model.layers.36.mlp.experts.123.down_proj", + "model.layers.36.mlp.experts.123.gate_proj", + "model.layers.36.mlp.experts.123.up_proj", + "model.layers.36.mlp.experts.124.down_proj", + "model.layers.36.mlp.experts.124.gate_proj", + "model.layers.36.mlp.experts.124.up_proj", + "model.layers.36.mlp.experts.125.down_proj", + "model.layers.36.mlp.experts.125.gate_proj", + "model.layers.36.mlp.experts.125.up_proj", + "model.layers.36.mlp.experts.126.down_proj", + "model.layers.36.mlp.experts.126.gate_proj", + "model.layers.36.mlp.experts.126.up_proj", + "model.layers.36.mlp.experts.127.down_proj", + "model.layers.36.mlp.experts.127.gate_proj", + "model.layers.36.mlp.experts.127.up_proj", + "model.layers.36.mlp.experts.128.down_proj", + "model.layers.36.mlp.experts.128.gate_proj", + "model.layers.36.mlp.experts.128.up_proj", + "model.layers.36.mlp.experts.129.down_proj", + "model.layers.36.mlp.experts.129.gate_proj", + "model.layers.36.mlp.experts.129.up_proj", + "model.layers.36.mlp.experts.13.down_proj", + "model.layers.36.mlp.experts.13.gate_proj", + "model.layers.36.mlp.experts.13.up_proj", + "model.layers.36.mlp.experts.130.down_proj", + "model.layers.36.mlp.experts.130.gate_proj", + "model.layers.36.mlp.experts.130.up_proj", + "model.layers.36.mlp.experts.131.down_proj", + "model.layers.36.mlp.experts.131.gate_proj", + "model.layers.36.mlp.experts.131.up_proj", + "model.layers.36.mlp.experts.132.down_proj", + "model.layers.36.mlp.experts.132.gate_proj", + "model.layers.36.mlp.experts.132.up_proj", + "model.layers.36.mlp.experts.133.down_proj", + "model.layers.36.mlp.experts.133.gate_proj", + "model.layers.36.mlp.experts.133.up_proj", + "model.layers.36.mlp.experts.134.down_proj", + "model.layers.36.mlp.experts.134.gate_proj", + "model.layers.36.mlp.experts.134.up_proj", + "model.layers.36.mlp.experts.135.down_proj", + "model.layers.36.mlp.experts.135.gate_proj", + "model.layers.36.mlp.experts.135.up_proj", + "model.layers.36.mlp.experts.136.down_proj", + "model.layers.36.mlp.experts.136.gate_proj", + "model.layers.36.mlp.experts.136.up_proj", + "model.layers.36.mlp.experts.137.down_proj", + "model.layers.36.mlp.experts.137.gate_proj", + "model.layers.36.mlp.experts.137.up_proj", + "model.layers.36.mlp.experts.138.down_proj", + "model.layers.36.mlp.experts.138.gate_proj", + "model.layers.36.mlp.experts.138.up_proj", + "model.layers.36.mlp.experts.139.down_proj", + "model.layers.36.mlp.experts.139.gate_proj", + "model.layers.36.mlp.experts.139.up_proj", + "model.layers.36.mlp.experts.14.down_proj", + "model.layers.36.mlp.experts.14.gate_proj", + "model.layers.36.mlp.experts.14.up_proj", + "model.layers.36.mlp.experts.140.down_proj", + "model.layers.36.mlp.experts.140.gate_proj", + "model.layers.36.mlp.experts.140.up_proj", + "model.layers.36.mlp.experts.141.down_proj", + "model.layers.36.mlp.experts.141.gate_proj", + "model.layers.36.mlp.experts.141.up_proj", + "model.layers.36.mlp.experts.142.down_proj", + "model.layers.36.mlp.experts.142.gate_proj", + "model.layers.36.mlp.experts.142.up_proj", + "model.layers.36.mlp.experts.143.down_proj", + "model.layers.36.mlp.experts.143.gate_proj", + "model.layers.36.mlp.experts.143.up_proj", + "model.layers.36.mlp.experts.144.down_proj", + "model.layers.36.mlp.experts.144.gate_proj", + "model.layers.36.mlp.experts.144.up_proj", + "model.layers.36.mlp.experts.145.down_proj", + "model.layers.36.mlp.experts.145.gate_proj", + "model.layers.36.mlp.experts.145.up_proj", + "model.layers.36.mlp.experts.146.down_proj", + "model.layers.36.mlp.experts.146.gate_proj", + "model.layers.36.mlp.experts.146.up_proj", + "model.layers.36.mlp.experts.147.down_proj", + "model.layers.36.mlp.experts.147.gate_proj", + "model.layers.36.mlp.experts.147.up_proj", + "model.layers.36.mlp.experts.148.down_proj", + "model.layers.36.mlp.experts.148.gate_proj", + "model.layers.36.mlp.experts.148.up_proj", + "model.layers.36.mlp.experts.149.down_proj", + "model.layers.36.mlp.experts.149.gate_proj", + "model.layers.36.mlp.experts.149.up_proj", + "model.layers.36.mlp.experts.15.down_proj", + "model.layers.36.mlp.experts.15.gate_proj", + "model.layers.36.mlp.experts.15.up_proj", + "model.layers.36.mlp.experts.150.down_proj", + "model.layers.36.mlp.experts.150.gate_proj", + "model.layers.36.mlp.experts.150.up_proj", + "model.layers.36.mlp.experts.151.down_proj", + "model.layers.36.mlp.experts.151.gate_proj", + "model.layers.36.mlp.experts.151.up_proj", + "model.layers.36.mlp.experts.152.down_proj", + "model.layers.36.mlp.experts.152.gate_proj", + "model.layers.36.mlp.experts.152.up_proj", + "model.layers.36.mlp.experts.153.down_proj", + "model.layers.36.mlp.experts.153.gate_proj", + "model.layers.36.mlp.experts.153.up_proj", + "model.layers.36.mlp.experts.154.down_proj", + "model.layers.36.mlp.experts.154.gate_proj", + "model.layers.36.mlp.experts.154.up_proj", + "model.layers.36.mlp.experts.155.down_proj", + "model.layers.36.mlp.experts.155.gate_proj", + "model.layers.36.mlp.experts.155.up_proj", + "model.layers.36.mlp.experts.156.down_proj", + "model.layers.36.mlp.experts.156.gate_proj", + "model.layers.36.mlp.experts.156.up_proj", + "model.layers.36.mlp.experts.157.down_proj", + "model.layers.36.mlp.experts.157.gate_proj", + "model.layers.36.mlp.experts.157.up_proj", + "model.layers.36.mlp.experts.158.down_proj", + "model.layers.36.mlp.experts.158.gate_proj", + "model.layers.36.mlp.experts.158.up_proj", + "model.layers.36.mlp.experts.159.down_proj", + "model.layers.36.mlp.experts.159.gate_proj", + "model.layers.36.mlp.experts.159.up_proj", + "model.layers.36.mlp.experts.16.down_proj", + "model.layers.36.mlp.experts.16.gate_proj", + "model.layers.36.mlp.experts.16.up_proj", + "model.layers.36.mlp.experts.160.down_proj", + "model.layers.36.mlp.experts.160.gate_proj", + "model.layers.36.mlp.experts.160.up_proj", + "model.layers.36.mlp.experts.161.down_proj", + "model.layers.36.mlp.experts.161.gate_proj", + "model.layers.36.mlp.experts.161.up_proj", + "model.layers.36.mlp.experts.162.down_proj", + "model.layers.36.mlp.experts.162.gate_proj", + "model.layers.36.mlp.experts.162.up_proj", + "model.layers.36.mlp.experts.163.down_proj", + "model.layers.36.mlp.experts.163.gate_proj", + "model.layers.36.mlp.experts.163.up_proj", + "model.layers.36.mlp.experts.164.down_proj", + "model.layers.36.mlp.experts.164.gate_proj", + "model.layers.36.mlp.experts.164.up_proj", + "model.layers.36.mlp.experts.165.down_proj", + "model.layers.36.mlp.experts.165.gate_proj", + "model.layers.36.mlp.experts.165.up_proj", + "model.layers.36.mlp.experts.166.down_proj", + "model.layers.36.mlp.experts.166.gate_proj", + "model.layers.36.mlp.experts.166.up_proj", + "model.layers.36.mlp.experts.167.down_proj", + "model.layers.36.mlp.experts.167.gate_proj", + "model.layers.36.mlp.experts.167.up_proj", + "model.layers.36.mlp.experts.168.down_proj", + "model.layers.36.mlp.experts.168.gate_proj", + "model.layers.36.mlp.experts.168.up_proj", + "model.layers.36.mlp.experts.169.down_proj", + "model.layers.36.mlp.experts.169.gate_proj", + "model.layers.36.mlp.experts.169.up_proj", + "model.layers.36.mlp.experts.17.down_proj", + "model.layers.36.mlp.experts.17.gate_proj", + "model.layers.36.mlp.experts.17.up_proj", + "model.layers.36.mlp.experts.170.down_proj", + "model.layers.36.mlp.experts.170.gate_proj", + "model.layers.36.mlp.experts.170.up_proj", + "model.layers.36.mlp.experts.171.down_proj", + "model.layers.36.mlp.experts.171.gate_proj", + "model.layers.36.mlp.experts.171.up_proj", + "model.layers.36.mlp.experts.172.down_proj", + "model.layers.36.mlp.experts.172.gate_proj", + "model.layers.36.mlp.experts.172.up_proj", + "model.layers.36.mlp.experts.173.down_proj", + "model.layers.36.mlp.experts.173.gate_proj", + "model.layers.36.mlp.experts.173.up_proj", + "model.layers.36.mlp.experts.174.down_proj", + "model.layers.36.mlp.experts.174.gate_proj", + "model.layers.36.mlp.experts.174.up_proj", + "model.layers.36.mlp.experts.175.down_proj", + "model.layers.36.mlp.experts.175.gate_proj", + "model.layers.36.mlp.experts.175.up_proj", + "model.layers.36.mlp.experts.176.down_proj", + "model.layers.36.mlp.experts.176.gate_proj", + "model.layers.36.mlp.experts.176.up_proj", + "model.layers.36.mlp.experts.177.down_proj", + "model.layers.36.mlp.experts.177.gate_proj", + "model.layers.36.mlp.experts.177.up_proj", + "model.layers.36.mlp.experts.178.down_proj", + "model.layers.36.mlp.experts.178.gate_proj", + "model.layers.36.mlp.experts.178.up_proj", + "model.layers.36.mlp.experts.179.down_proj", + "model.layers.36.mlp.experts.179.gate_proj", + "model.layers.36.mlp.experts.179.up_proj", + "model.layers.36.mlp.experts.18.down_proj", + "model.layers.36.mlp.experts.18.gate_proj", + "model.layers.36.mlp.experts.18.up_proj", + "model.layers.36.mlp.experts.180.down_proj", + "model.layers.36.mlp.experts.180.gate_proj", + "model.layers.36.mlp.experts.180.up_proj", + "model.layers.36.mlp.experts.181.down_proj", + "model.layers.36.mlp.experts.181.gate_proj", + "model.layers.36.mlp.experts.181.up_proj", + "model.layers.36.mlp.experts.182.down_proj", + "model.layers.36.mlp.experts.182.gate_proj", + "model.layers.36.mlp.experts.182.up_proj", + "model.layers.36.mlp.experts.183.down_proj", + "model.layers.36.mlp.experts.183.gate_proj", + "model.layers.36.mlp.experts.183.up_proj", + "model.layers.36.mlp.experts.184.down_proj", + "model.layers.36.mlp.experts.184.gate_proj", + "model.layers.36.mlp.experts.184.up_proj", + "model.layers.36.mlp.experts.185.down_proj", + "model.layers.36.mlp.experts.185.gate_proj", + "model.layers.36.mlp.experts.185.up_proj", + "model.layers.36.mlp.experts.186.down_proj", + "model.layers.36.mlp.experts.186.gate_proj", + "model.layers.36.mlp.experts.186.up_proj", + "model.layers.36.mlp.experts.187.down_proj", + "model.layers.36.mlp.experts.187.gate_proj", + "model.layers.36.mlp.experts.187.up_proj", + "model.layers.36.mlp.experts.188.down_proj", + "model.layers.36.mlp.experts.188.gate_proj", + "model.layers.36.mlp.experts.188.up_proj", + "model.layers.36.mlp.experts.189.down_proj", + "model.layers.36.mlp.experts.189.gate_proj", + "model.layers.36.mlp.experts.189.up_proj", + "model.layers.36.mlp.experts.19.down_proj", + "model.layers.36.mlp.experts.19.gate_proj", + "model.layers.36.mlp.experts.19.up_proj", + "model.layers.36.mlp.experts.190.down_proj", + "model.layers.36.mlp.experts.190.gate_proj", + "model.layers.36.mlp.experts.190.up_proj", + "model.layers.36.mlp.experts.191.down_proj", + "model.layers.36.mlp.experts.191.gate_proj", + "model.layers.36.mlp.experts.191.up_proj", + "model.layers.36.mlp.experts.192.down_proj", + "model.layers.36.mlp.experts.192.gate_proj", + "model.layers.36.mlp.experts.192.up_proj", + "model.layers.36.mlp.experts.193.down_proj", + "model.layers.36.mlp.experts.193.gate_proj", + "model.layers.36.mlp.experts.193.up_proj", + "model.layers.36.mlp.experts.194.down_proj", + "model.layers.36.mlp.experts.194.gate_proj", + "model.layers.36.mlp.experts.194.up_proj", + "model.layers.36.mlp.experts.195.down_proj", + "model.layers.36.mlp.experts.195.gate_proj", + "model.layers.36.mlp.experts.195.up_proj", + "model.layers.36.mlp.experts.196.down_proj", + "model.layers.36.mlp.experts.196.gate_proj", + "model.layers.36.mlp.experts.196.up_proj", + "model.layers.36.mlp.experts.197.down_proj", + "model.layers.36.mlp.experts.197.gate_proj", + "model.layers.36.mlp.experts.197.up_proj", + "model.layers.36.mlp.experts.198.down_proj", + "model.layers.36.mlp.experts.198.gate_proj", + "model.layers.36.mlp.experts.198.up_proj", + "model.layers.36.mlp.experts.199.down_proj", + "model.layers.36.mlp.experts.199.gate_proj", + "model.layers.36.mlp.experts.199.up_proj", + "model.layers.36.mlp.experts.2.down_proj", + "model.layers.36.mlp.experts.2.gate_proj", + "model.layers.36.mlp.experts.2.up_proj", + "model.layers.36.mlp.experts.20.down_proj", + "model.layers.36.mlp.experts.20.gate_proj", + "model.layers.36.mlp.experts.20.up_proj", + "model.layers.36.mlp.experts.200.down_proj", + "model.layers.36.mlp.experts.200.gate_proj", + "model.layers.36.mlp.experts.200.up_proj", + "model.layers.36.mlp.experts.201.down_proj", + "model.layers.36.mlp.experts.201.gate_proj", + "model.layers.36.mlp.experts.201.up_proj", + "model.layers.36.mlp.experts.202.down_proj", + "model.layers.36.mlp.experts.202.gate_proj", + "model.layers.36.mlp.experts.202.up_proj", + "model.layers.36.mlp.experts.203.down_proj", + "model.layers.36.mlp.experts.203.gate_proj", + "model.layers.36.mlp.experts.203.up_proj", + "model.layers.36.mlp.experts.204.down_proj", + "model.layers.36.mlp.experts.204.gate_proj", + "model.layers.36.mlp.experts.204.up_proj", + "model.layers.36.mlp.experts.205.down_proj", + "model.layers.36.mlp.experts.205.gate_proj", + "model.layers.36.mlp.experts.205.up_proj", + "model.layers.36.mlp.experts.206.down_proj", + "model.layers.36.mlp.experts.206.gate_proj", + "model.layers.36.mlp.experts.206.up_proj", + "model.layers.36.mlp.experts.207.down_proj", + "model.layers.36.mlp.experts.207.gate_proj", + "model.layers.36.mlp.experts.207.up_proj", + "model.layers.36.mlp.experts.208.down_proj", + "model.layers.36.mlp.experts.208.gate_proj", + "model.layers.36.mlp.experts.208.up_proj", + "model.layers.36.mlp.experts.209.down_proj", + "model.layers.36.mlp.experts.209.gate_proj", + "model.layers.36.mlp.experts.209.up_proj", + "model.layers.36.mlp.experts.21.down_proj", + "model.layers.36.mlp.experts.21.gate_proj", + "model.layers.36.mlp.experts.21.up_proj", + "model.layers.36.mlp.experts.210.down_proj", + "model.layers.36.mlp.experts.210.gate_proj", + "model.layers.36.mlp.experts.210.up_proj", + "model.layers.36.mlp.experts.211.down_proj", + "model.layers.36.mlp.experts.211.gate_proj", + "model.layers.36.mlp.experts.211.up_proj", + "model.layers.36.mlp.experts.212.down_proj", + "model.layers.36.mlp.experts.212.gate_proj", + "model.layers.36.mlp.experts.212.up_proj", + "model.layers.36.mlp.experts.213.down_proj", + "model.layers.36.mlp.experts.213.gate_proj", + "model.layers.36.mlp.experts.213.up_proj", + "model.layers.36.mlp.experts.214.down_proj", + "model.layers.36.mlp.experts.214.gate_proj", + "model.layers.36.mlp.experts.214.up_proj", + "model.layers.36.mlp.experts.215.down_proj", + "model.layers.36.mlp.experts.215.gate_proj", + "model.layers.36.mlp.experts.215.up_proj", + "model.layers.36.mlp.experts.216.down_proj", + "model.layers.36.mlp.experts.216.gate_proj", + "model.layers.36.mlp.experts.216.up_proj", + "model.layers.36.mlp.experts.217.down_proj", + "model.layers.36.mlp.experts.217.gate_proj", + "model.layers.36.mlp.experts.217.up_proj", + "model.layers.36.mlp.experts.218.down_proj", + "model.layers.36.mlp.experts.218.gate_proj", + "model.layers.36.mlp.experts.218.up_proj", + "model.layers.36.mlp.experts.219.down_proj", + "model.layers.36.mlp.experts.219.gate_proj", + "model.layers.36.mlp.experts.219.up_proj", + "model.layers.36.mlp.experts.22.down_proj", + "model.layers.36.mlp.experts.22.gate_proj", + "model.layers.36.mlp.experts.22.up_proj", + "model.layers.36.mlp.experts.220.down_proj", + "model.layers.36.mlp.experts.220.gate_proj", + "model.layers.36.mlp.experts.220.up_proj", + "model.layers.36.mlp.experts.221.down_proj", + "model.layers.36.mlp.experts.221.gate_proj", + "model.layers.36.mlp.experts.221.up_proj", + "model.layers.36.mlp.experts.222.down_proj", + "model.layers.36.mlp.experts.222.gate_proj", + "model.layers.36.mlp.experts.222.up_proj", + "model.layers.36.mlp.experts.223.down_proj", + "model.layers.36.mlp.experts.223.gate_proj", + "model.layers.36.mlp.experts.223.up_proj", + "model.layers.36.mlp.experts.224.down_proj", + "model.layers.36.mlp.experts.224.gate_proj", + "model.layers.36.mlp.experts.224.up_proj", + "model.layers.36.mlp.experts.225.down_proj", + "model.layers.36.mlp.experts.225.gate_proj", + "model.layers.36.mlp.experts.225.up_proj", + "model.layers.36.mlp.experts.226.down_proj", + "model.layers.36.mlp.experts.226.gate_proj", + "model.layers.36.mlp.experts.226.up_proj", + "model.layers.36.mlp.experts.227.down_proj", + "model.layers.36.mlp.experts.227.gate_proj", + "model.layers.36.mlp.experts.227.up_proj", + "model.layers.36.mlp.experts.228.down_proj", + "model.layers.36.mlp.experts.228.gate_proj", + "model.layers.36.mlp.experts.228.up_proj", + "model.layers.36.mlp.experts.229.down_proj", + "model.layers.36.mlp.experts.229.gate_proj", + "model.layers.36.mlp.experts.229.up_proj", + "model.layers.36.mlp.experts.23.down_proj", + "model.layers.36.mlp.experts.23.gate_proj", + "model.layers.36.mlp.experts.23.up_proj", + "model.layers.36.mlp.experts.230.down_proj", + "model.layers.36.mlp.experts.230.gate_proj", + "model.layers.36.mlp.experts.230.up_proj", + "model.layers.36.mlp.experts.231.down_proj", + "model.layers.36.mlp.experts.231.gate_proj", + "model.layers.36.mlp.experts.231.up_proj", + "model.layers.36.mlp.experts.232.down_proj", + "model.layers.36.mlp.experts.232.gate_proj", + "model.layers.36.mlp.experts.232.up_proj", + "model.layers.36.mlp.experts.233.down_proj", + "model.layers.36.mlp.experts.233.gate_proj", + "model.layers.36.mlp.experts.233.up_proj", + "model.layers.36.mlp.experts.234.down_proj", + "model.layers.36.mlp.experts.234.gate_proj", + "model.layers.36.mlp.experts.234.up_proj", + "model.layers.36.mlp.experts.235.down_proj", + "model.layers.36.mlp.experts.235.gate_proj", + "model.layers.36.mlp.experts.235.up_proj", + "model.layers.36.mlp.experts.236.down_proj", + "model.layers.36.mlp.experts.236.gate_proj", + "model.layers.36.mlp.experts.236.up_proj", + "model.layers.36.mlp.experts.237.down_proj", + "model.layers.36.mlp.experts.237.gate_proj", + "model.layers.36.mlp.experts.237.up_proj", + "model.layers.36.mlp.experts.238.down_proj", + "model.layers.36.mlp.experts.238.gate_proj", + "model.layers.36.mlp.experts.238.up_proj", + "model.layers.36.mlp.experts.239.down_proj", + "model.layers.36.mlp.experts.239.gate_proj", + "model.layers.36.mlp.experts.239.up_proj", + "model.layers.36.mlp.experts.24.down_proj", + "model.layers.36.mlp.experts.24.gate_proj", + "model.layers.36.mlp.experts.24.up_proj", + "model.layers.36.mlp.experts.240.down_proj", + "model.layers.36.mlp.experts.240.gate_proj", + "model.layers.36.mlp.experts.240.up_proj", + "model.layers.36.mlp.experts.241.down_proj", + "model.layers.36.mlp.experts.241.gate_proj", + "model.layers.36.mlp.experts.241.up_proj", + "model.layers.36.mlp.experts.242.down_proj", + "model.layers.36.mlp.experts.242.gate_proj", + "model.layers.36.mlp.experts.242.up_proj", + "model.layers.36.mlp.experts.243.down_proj", + "model.layers.36.mlp.experts.243.gate_proj", + "model.layers.36.mlp.experts.243.up_proj", + "model.layers.36.mlp.experts.244.down_proj", + "model.layers.36.mlp.experts.244.gate_proj", + "model.layers.36.mlp.experts.244.up_proj", + "model.layers.36.mlp.experts.245.down_proj", + "model.layers.36.mlp.experts.245.gate_proj", + "model.layers.36.mlp.experts.245.up_proj", + "model.layers.36.mlp.experts.246.down_proj", + "model.layers.36.mlp.experts.246.gate_proj", + "model.layers.36.mlp.experts.246.up_proj", + "model.layers.36.mlp.experts.247.down_proj", + "model.layers.36.mlp.experts.247.gate_proj", + "model.layers.36.mlp.experts.247.up_proj", + "model.layers.36.mlp.experts.248.down_proj", + "model.layers.36.mlp.experts.248.gate_proj", + "model.layers.36.mlp.experts.248.up_proj", + "model.layers.36.mlp.experts.249.down_proj", + "model.layers.36.mlp.experts.249.gate_proj", + "model.layers.36.mlp.experts.249.up_proj", + "model.layers.36.mlp.experts.25.down_proj", + "model.layers.36.mlp.experts.25.gate_proj", + "model.layers.36.mlp.experts.25.up_proj", + "model.layers.36.mlp.experts.250.down_proj", + "model.layers.36.mlp.experts.250.gate_proj", + "model.layers.36.mlp.experts.250.up_proj", + "model.layers.36.mlp.experts.251.down_proj", + "model.layers.36.mlp.experts.251.gate_proj", + "model.layers.36.mlp.experts.251.up_proj", + "model.layers.36.mlp.experts.252.down_proj", + "model.layers.36.mlp.experts.252.gate_proj", + "model.layers.36.mlp.experts.252.up_proj", + "model.layers.36.mlp.experts.253.down_proj", + "model.layers.36.mlp.experts.253.gate_proj", + "model.layers.36.mlp.experts.253.up_proj", + "model.layers.36.mlp.experts.254.down_proj", + "model.layers.36.mlp.experts.254.gate_proj", + "model.layers.36.mlp.experts.254.up_proj", + "model.layers.36.mlp.experts.255.down_proj", + "model.layers.36.mlp.experts.255.gate_proj", + "model.layers.36.mlp.experts.255.up_proj", + "model.layers.36.mlp.experts.26.down_proj", + "model.layers.36.mlp.experts.26.gate_proj", + "model.layers.36.mlp.experts.26.up_proj", + "model.layers.36.mlp.experts.27.down_proj", + "model.layers.36.mlp.experts.27.gate_proj", + "model.layers.36.mlp.experts.27.up_proj", + "model.layers.36.mlp.experts.28.down_proj", + "model.layers.36.mlp.experts.28.gate_proj", + "model.layers.36.mlp.experts.28.up_proj", + "model.layers.36.mlp.experts.29.down_proj", + "model.layers.36.mlp.experts.29.gate_proj", + "model.layers.36.mlp.experts.29.up_proj", + "model.layers.36.mlp.experts.3.down_proj", + "model.layers.36.mlp.experts.3.gate_proj", + "model.layers.36.mlp.experts.3.up_proj", + "model.layers.36.mlp.experts.30.down_proj", + "model.layers.36.mlp.experts.30.gate_proj", + "model.layers.36.mlp.experts.30.up_proj", + "model.layers.36.mlp.experts.31.down_proj", + "model.layers.36.mlp.experts.31.gate_proj", + "model.layers.36.mlp.experts.31.up_proj", + "model.layers.36.mlp.experts.32.down_proj", + "model.layers.36.mlp.experts.32.gate_proj", + "model.layers.36.mlp.experts.32.up_proj", + "model.layers.36.mlp.experts.33.down_proj", + "model.layers.36.mlp.experts.33.gate_proj", + "model.layers.36.mlp.experts.33.up_proj", + "model.layers.36.mlp.experts.34.down_proj", + "model.layers.36.mlp.experts.34.gate_proj", + "model.layers.36.mlp.experts.34.up_proj", + "model.layers.36.mlp.experts.35.down_proj", + "model.layers.36.mlp.experts.35.gate_proj", + "model.layers.36.mlp.experts.35.up_proj", + "model.layers.36.mlp.experts.36.down_proj", + "model.layers.36.mlp.experts.36.gate_proj", + "model.layers.36.mlp.experts.36.up_proj", + "model.layers.36.mlp.experts.37.down_proj", + "model.layers.36.mlp.experts.37.gate_proj", + "model.layers.36.mlp.experts.37.up_proj", + "model.layers.36.mlp.experts.38.down_proj", + "model.layers.36.mlp.experts.38.gate_proj", + "model.layers.36.mlp.experts.38.up_proj", + "model.layers.36.mlp.experts.39.down_proj", + "model.layers.36.mlp.experts.39.gate_proj", + "model.layers.36.mlp.experts.39.up_proj", + "model.layers.36.mlp.experts.4.down_proj", + "model.layers.36.mlp.experts.4.gate_proj", + "model.layers.36.mlp.experts.4.up_proj", + "model.layers.36.mlp.experts.40.down_proj", + "model.layers.36.mlp.experts.40.gate_proj", + "model.layers.36.mlp.experts.40.up_proj", + "model.layers.36.mlp.experts.41.down_proj", + "model.layers.36.mlp.experts.41.gate_proj", + "model.layers.36.mlp.experts.41.up_proj", + "model.layers.36.mlp.experts.42.down_proj", + "model.layers.36.mlp.experts.42.gate_proj", + "model.layers.36.mlp.experts.42.up_proj", + "model.layers.36.mlp.experts.43.down_proj", + "model.layers.36.mlp.experts.43.gate_proj", + "model.layers.36.mlp.experts.43.up_proj", + "model.layers.36.mlp.experts.44.down_proj", + "model.layers.36.mlp.experts.44.gate_proj", + "model.layers.36.mlp.experts.44.up_proj", + "model.layers.36.mlp.experts.45.down_proj", + "model.layers.36.mlp.experts.45.gate_proj", + "model.layers.36.mlp.experts.45.up_proj", + "model.layers.36.mlp.experts.46.down_proj", + "model.layers.36.mlp.experts.46.gate_proj", + "model.layers.36.mlp.experts.46.up_proj", + "model.layers.36.mlp.experts.47.down_proj", + "model.layers.36.mlp.experts.47.gate_proj", + "model.layers.36.mlp.experts.47.up_proj", + "model.layers.36.mlp.experts.48.down_proj", + "model.layers.36.mlp.experts.48.gate_proj", + "model.layers.36.mlp.experts.48.up_proj", + "model.layers.36.mlp.experts.49.down_proj", + "model.layers.36.mlp.experts.49.gate_proj", + "model.layers.36.mlp.experts.49.up_proj", + "model.layers.36.mlp.experts.5.down_proj", + "model.layers.36.mlp.experts.5.gate_proj", + "model.layers.36.mlp.experts.5.up_proj", + "model.layers.36.mlp.experts.50.down_proj", + "model.layers.36.mlp.experts.50.gate_proj", + "model.layers.36.mlp.experts.50.up_proj", + "model.layers.36.mlp.experts.51.down_proj", + "model.layers.36.mlp.experts.51.gate_proj", + "model.layers.36.mlp.experts.51.up_proj", + "model.layers.36.mlp.experts.52.down_proj", + "model.layers.36.mlp.experts.52.gate_proj", + "model.layers.36.mlp.experts.52.up_proj", + "model.layers.36.mlp.experts.53.down_proj", + "model.layers.36.mlp.experts.53.gate_proj", + "model.layers.36.mlp.experts.53.up_proj", + "model.layers.36.mlp.experts.54.down_proj", + "model.layers.36.mlp.experts.54.gate_proj", + "model.layers.36.mlp.experts.54.up_proj", + "model.layers.36.mlp.experts.55.down_proj", + "model.layers.36.mlp.experts.55.gate_proj", + "model.layers.36.mlp.experts.55.up_proj", + "model.layers.36.mlp.experts.56.down_proj", + "model.layers.36.mlp.experts.56.gate_proj", + "model.layers.36.mlp.experts.56.up_proj", + "model.layers.36.mlp.experts.57.down_proj", + "model.layers.36.mlp.experts.57.gate_proj", + "model.layers.36.mlp.experts.57.up_proj", + "model.layers.36.mlp.experts.58.down_proj", + "model.layers.36.mlp.experts.58.gate_proj", + "model.layers.36.mlp.experts.58.up_proj", + "model.layers.36.mlp.experts.59.down_proj", + "model.layers.36.mlp.experts.59.gate_proj", + "model.layers.36.mlp.experts.59.up_proj", + "model.layers.36.mlp.experts.6.down_proj", + "model.layers.36.mlp.experts.6.gate_proj", + "model.layers.36.mlp.experts.6.up_proj", + "model.layers.36.mlp.experts.60.down_proj", + "model.layers.36.mlp.experts.60.gate_proj", + "model.layers.36.mlp.experts.60.up_proj", + "model.layers.36.mlp.experts.61.down_proj", + "model.layers.36.mlp.experts.61.gate_proj", + "model.layers.36.mlp.experts.61.up_proj", + "model.layers.36.mlp.experts.62.down_proj", + "model.layers.36.mlp.experts.62.gate_proj", + "model.layers.36.mlp.experts.62.up_proj", + "model.layers.36.mlp.experts.63.down_proj", + "model.layers.36.mlp.experts.63.gate_proj", + "model.layers.36.mlp.experts.63.up_proj", + "model.layers.36.mlp.experts.64.down_proj", + "model.layers.36.mlp.experts.64.gate_proj", + "model.layers.36.mlp.experts.64.up_proj", + "model.layers.36.mlp.experts.65.down_proj", + "model.layers.36.mlp.experts.65.gate_proj", + "model.layers.36.mlp.experts.65.up_proj", + "model.layers.36.mlp.experts.66.down_proj", + "model.layers.36.mlp.experts.66.gate_proj", + "model.layers.36.mlp.experts.66.up_proj", + "model.layers.36.mlp.experts.67.down_proj", + "model.layers.36.mlp.experts.67.gate_proj", + "model.layers.36.mlp.experts.67.up_proj", + "model.layers.36.mlp.experts.68.down_proj", + "model.layers.36.mlp.experts.68.gate_proj", + "model.layers.36.mlp.experts.68.up_proj", + "model.layers.36.mlp.experts.69.down_proj", + "model.layers.36.mlp.experts.69.gate_proj", + "model.layers.36.mlp.experts.69.up_proj", + "model.layers.36.mlp.experts.7.down_proj", + "model.layers.36.mlp.experts.7.gate_proj", + "model.layers.36.mlp.experts.7.up_proj", + "model.layers.36.mlp.experts.70.down_proj", + "model.layers.36.mlp.experts.70.gate_proj", + "model.layers.36.mlp.experts.70.up_proj", + "model.layers.36.mlp.experts.71.down_proj", + "model.layers.36.mlp.experts.71.gate_proj", + "model.layers.36.mlp.experts.71.up_proj", + "model.layers.36.mlp.experts.72.down_proj", + "model.layers.36.mlp.experts.72.gate_proj", + "model.layers.36.mlp.experts.72.up_proj", + "model.layers.36.mlp.experts.73.down_proj", + "model.layers.36.mlp.experts.73.gate_proj", + "model.layers.36.mlp.experts.73.up_proj", + "model.layers.36.mlp.experts.74.down_proj", + "model.layers.36.mlp.experts.74.gate_proj", + "model.layers.36.mlp.experts.74.up_proj", + "model.layers.36.mlp.experts.75.down_proj", + "model.layers.36.mlp.experts.75.gate_proj", + "model.layers.36.mlp.experts.75.up_proj", + "model.layers.36.mlp.experts.76.down_proj", + "model.layers.36.mlp.experts.76.gate_proj", + "model.layers.36.mlp.experts.76.up_proj", + "model.layers.36.mlp.experts.77.down_proj", + "model.layers.36.mlp.experts.77.gate_proj", + "model.layers.36.mlp.experts.77.up_proj", + "model.layers.36.mlp.experts.78.down_proj", + "model.layers.36.mlp.experts.78.gate_proj", + "model.layers.36.mlp.experts.78.up_proj", + "model.layers.36.mlp.experts.79.down_proj", + "model.layers.36.mlp.experts.79.gate_proj", + "model.layers.36.mlp.experts.79.up_proj", + "model.layers.36.mlp.experts.8.down_proj", + "model.layers.36.mlp.experts.8.gate_proj", + "model.layers.36.mlp.experts.8.up_proj", + "model.layers.36.mlp.experts.80.down_proj", + "model.layers.36.mlp.experts.80.gate_proj", + "model.layers.36.mlp.experts.80.up_proj", + "model.layers.36.mlp.experts.81.down_proj", + "model.layers.36.mlp.experts.81.gate_proj", + "model.layers.36.mlp.experts.81.up_proj", + "model.layers.36.mlp.experts.82.down_proj", + "model.layers.36.mlp.experts.82.gate_proj", + "model.layers.36.mlp.experts.82.up_proj", + "model.layers.36.mlp.experts.83.down_proj", + "model.layers.36.mlp.experts.83.gate_proj", + "model.layers.36.mlp.experts.83.up_proj", + "model.layers.36.mlp.experts.84.down_proj", + "model.layers.36.mlp.experts.84.gate_proj", + "model.layers.36.mlp.experts.84.up_proj", + "model.layers.36.mlp.experts.85.down_proj", + "model.layers.36.mlp.experts.85.gate_proj", + "model.layers.36.mlp.experts.85.up_proj", + "model.layers.36.mlp.experts.86.down_proj", + "model.layers.36.mlp.experts.86.gate_proj", + "model.layers.36.mlp.experts.86.up_proj", + "model.layers.36.mlp.experts.87.down_proj", + "model.layers.36.mlp.experts.87.gate_proj", + "model.layers.36.mlp.experts.87.up_proj", + "model.layers.36.mlp.experts.88.down_proj", + "model.layers.36.mlp.experts.88.gate_proj", + "model.layers.36.mlp.experts.88.up_proj", + "model.layers.36.mlp.experts.89.down_proj", + "model.layers.36.mlp.experts.89.gate_proj", + "model.layers.36.mlp.experts.89.up_proj", + "model.layers.36.mlp.experts.9.down_proj", + "model.layers.36.mlp.experts.9.gate_proj", + "model.layers.36.mlp.experts.9.up_proj", + "model.layers.36.mlp.experts.90.down_proj", + "model.layers.36.mlp.experts.90.gate_proj", + "model.layers.36.mlp.experts.90.up_proj", + "model.layers.36.mlp.experts.91.down_proj", + "model.layers.36.mlp.experts.91.gate_proj", + "model.layers.36.mlp.experts.91.up_proj", + "model.layers.36.mlp.experts.92.down_proj", + "model.layers.36.mlp.experts.92.gate_proj", + "model.layers.36.mlp.experts.92.up_proj", + "model.layers.36.mlp.experts.93.down_proj", + "model.layers.36.mlp.experts.93.gate_proj", + "model.layers.36.mlp.experts.93.up_proj", + "model.layers.36.mlp.experts.94.down_proj", + "model.layers.36.mlp.experts.94.gate_proj", + "model.layers.36.mlp.experts.94.up_proj", + "model.layers.36.mlp.experts.95.down_proj", + "model.layers.36.mlp.experts.95.gate_proj", + "model.layers.36.mlp.experts.95.up_proj", + "model.layers.36.mlp.experts.96.down_proj", + "model.layers.36.mlp.experts.96.gate_proj", + "model.layers.36.mlp.experts.96.up_proj", + "model.layers.36.mlp.experts.97.down_proj", + "model.layers.36.mlp.experts.97.gate_proj", + "model.layers.36.mlp.experts.97.up_proj", + "model.layers.36.mlp.experts.98.down_proj", + "model.layers.36.mlp.experts.98.gate_proj", + "model.layers.36.mlp.experts.98.up_proj", + "model.layers.36.mlp.experts.99.down_proj", + "model.layers.36.mlp.experts.99.gate_proj", + "model.layers.36.mlp.experts.99.up_proj", + "model.layers.37.mlp.experts.0.down_proj", + "model.layers.37.mlp.experts.0.gate_proj", + "model.layers.37.mlp.experts.0.up_proj", + "model.layers.37.mlp.experts.1.down_proj", + "model.layers.37.mlp.experts.1.gate_proj", + "model.layers.37.mlp.experts.1.up_proj", + "model.layers.37.mlp.experts.10.down_proj", + "model.layers.37.mlp.experts.10.gate_proj", + "model.layers.37.mlp.experts.10.up_proj", + "model.layers.37.mlp.experts.100.down_proj", + "model.layers.37.mlp.experts.100.gate_proj", + "model.layers.37.mlp.experts.100.up_proj", + "model.layers.37.mlp.experts.101.down_proj", + "model.layers.37.mlp.experts.101.gate_proj", + "model.layers.37.mlp.experts.101.up_proj", + "model.layers.37.mlp.experts.102.down_proj", + "model.layers.37.mlp.experts.102.gate_proj", + "model.layers.37.mlp.experts.102.up_proj", + "model.layers.37.mlp.experts.103.down_proj", + "model.layers.37.mlp.experts.103.gate_proj", + "model.layers.37.mlp.experts.103.up_proj", + "model.layers.37.mlp.experts.104.down_proj", + "model.layers.37.mlp.experts.104.gate_proj", + "model.layers.37.mlp.experts.104.up_proj", + "model.layers.37.mlp.experts.105.down_proj", + "model.layers.37.mlp.experts.105.gate_proj", + "model.layers.37.mlp.experts.105.up_proj", + "model.layers.37.mlp.experts.106.down_proj", + "model.layers.37.mlp.experts.106.gate_proj", + "model.layers.37.mlp.experts.106.up_proj", + "model.layers.37.mlp.experts.107.down_proj", + "model.layers.37.mlp.experts.107.gate_proj", + "model.layers.37.mlp.experts.107.up_proj", + "model.layers.37.mlp.experts.108.down_proj", + "model.layers.37.mlp.experts.108.gate_proj", + "model.layers.37.mlp.experts.108.up_proj", + "model.layers.37.mlp.experts.109.down_proj", + "model.layers.37.mlp.experts.109.gate_proj", + "model.layers.37.mlp.experts.109.up_proj", + "model.layers.37.mlp.experts.11.down_proj", + "model.layers.37.mlp.experts.11.gate_proj", + "model.layers.37.mlp.experts.11.up_proj", + "model.layers.37.mlp.experts.110.down_proj", + "model.layers.37.mlp.experts.110.gate_proj", + "model.layers.37.mlp.experts.110.up_proj", + "model.layers.37.mlp.experts.111.down_proj", + "model.layers.37.mlp.experts.111.gate_proj", + "model.layers.37.mlp.experts.111.up_proj", + "model.layers.37.mlp.experts.112.down_proj", + "model.layers.37.mlp.experts.112.gate_proj", + "model.layers.37.mlp.experts.112.up_proj", + "model.layers.37.mlp.experts.113.down_proj", + "model.layers.37.mlp.experts.113.gate_proj", + "model.layers.37.mlp.experts.113.up_proj", + "model.layers.37.mlp.experts.114.down_proj", + "model.layers.37.mlp.experts.114.gate_proj", + "model.layers.37.mlp.experts.114.up_proj", + "model.layers.37.mlp.experts.115.down_proj", + "model.layers.37.mlp.experts.115.gate_proj", + "model.layers.37.mlp.experts.115.up_proj", + "model.layers.37.mlp.experts.116.down_proj", + "model.layers.37.mlp.experts.116.gate_proj", + "model.layers.37.mlp.experts.116.up_proj", + "model.layers.37.mlp.experts.117.down_proj", + "model.layers.37.mlp.experts.117.gate_proj", + "model.layers.37.mlp.experts.117.up_proj", + "model.layers.37.mlp.experts.118.down_proj", + "model.layers.37.mlp.experts.118.gate_proj", + "model.layers.37.mlp.experts.118.up_proj", + "model.layers.37.mlp.experts.119.down_proj", + "model.layers.37.mlp.experts.119.gate_proj", + "model.layers.37.mlp.experts.119.up_proj", + "model.layers.37.mlp.experts.12.down_proj", + "model.layers.37.mlp.experts.12.gate_proj", + "model.layers.37.mlp.experts.12.up_proj", + "model.layers.37.mlp.experts.120.down_proj", + "model.layers.37.mlp.experts.120.gate_proj", + "model.layers.37.mlp.experts.120.up_proj", + "model.layers.37.mlp.experts.121.down_proj", + "model.layers.37.mlp.experts.121.gate_proj", + "model.layers.37.mlp.experts.121.up_proj", + "model.layers.37.mlp.experts.122.down_proj", + "model.layers.37.mlp.experts.122.gate_proj", + "model.layers.37.mlp.experts.122.up_proj", + "model.layers.37.mlp.experts.123.down_proj", + "model.layers.37.mlp.experts.123.gate_proj", + "model.layers.37.mlp.experts.123.up_proj", + "model.layers.37.mlp.experts.124.down_proj", + "model.layers.37.mlp.experts.124.gate_proj", + "model.layers.37.mlp.experts.124.up_proj", + "model.layers.37.mlp.experts.125.down_proj", + "model.layers.37.mlp.experts.125.gate_proj", + "model.layers.37.mlp.experts.125.up_proj", + "model.layers.37.mlp.experts.126.down_proj", + "model.layers.37.mlp.experts.126.gate_proj", + "model.layers.37.mlp.experts.126.up_proj", + "model.layers.37.mlp.experts.127.down_proj", + "model.layers.37.mlp.experts.127.gate_proj", + "model.layers.37.mlp.experts.127.up_proj", + "model.layers.37.mlp.experts.128.down_proj", + "model.layers.37.mlp.experts.128.gate_proj", + "model.layers.37.mlp.experts.128.up_proj", + "model.layers.37.mlp.experts.129.down_proj", + "model.layers.37.mlp.experts.129.gate_proj", + "model.layers.37.mlp.experts.129.up_proj", + "model.layers.37.mlp.experts.13.down_proj", + "model.layers.37.mlp.experts.13.gate_proj", + "model.layers.37.mlp.experts.13.up_proj", + "model.layers.37.mlp.experts.130.down_proj", + "model.layers.37.mlp.experts.130.gate_proj", + "model.layers.37.mlp.experts.130.up_proj", + "model.layers.37.mlp.experts.131.down_proj", + "model.layers.37.mlp.experts.131.gate_proj", + "model.layers.37.mlp.experts.131.up_proj", + "model.layers.37.mlp.experts.132.down_proj", + "model.layers.37.mlp.experts.132.gate_proj", + "model.layers.37.mlp.experts.132.up_proj", + "model.layers.37.mlp.experts.133.down_proj", + "model.layers.37.mlp.experts.133.gate_proj", + "model.layers.37.mlp.experts.133.up_proj", + "model.layers.37.mlp.experts.134.down_proj", + "model.layers.37.mlp.experts.134.gate_proj", + "model.layers.37.mlp.experts.134.up_proj", + "model.layers.37.mlp.experts.135.down_proj", + "model.layers.37.mlp.experts.135.gate_proj", + "model.layers.37.mlp.experts.135.up_proj", + "model.layers.37.mlp.experts.136.down_proj", + "model.layers.37.mlp.experts.136.gate_proj", + "model.layers.37.mlp.experts.136.up_proj", + "model.layers.37.mlp.experts.137.down_proj", + "model.layers.37.mlp.experts.137.gate_proj", + "model.layers.37.mlp.experts.137.up_proj", + "model.layers.37.mlp.experts.138.down_proj", + "model.layers.37.mlp.experts.138.gate_proj", + "model.layers.37.mlp.experts.138.up_proj", + "model.layers.37.mlp.experts.139.down_proj", + "model.layers.37.mlp.experts.139.gate_proj", + "model.layers.37.mlp.experts.139.up_proj", + "model.layers.37.mlp.experts.14.down_proj", + "model.layers.37.mlp.experts.14.gate_proj", + "model.layers.37.mlp.experts.14.up_proj", + "model.layers.37.mlp.experts.140.down_proj", + "model.layers.37.mlp.experts.140.gate_proj", + "model.layers.37.mlp.experts.140.up_proj", + "model.layers.37.mlp.experts.141.down_proj", + "model.layers.37.mlp.experts.141.gate_proj", + "model.layers.37.mlp.experts.141.up_proj", + "model.layers.37.mlp.experts.142.down_proj", + "model.layers.37.mlp.experts.142.gate_proj", + "model.layers.37.mlp.experts.142.up_proj", + "model.layers.37.mlp.experts.143.down_proj", + "model.layers.37.mlp.experts.143.gate_proj", + "model.layers.37.mlp.experts.143.up_proj", + "model.layers.37.mlp.experts.144.down_proj", + "model.layers.37.mlp.experts.144.gate_proj", + "model.layers.37.mlp.experts.144.up_proj", + "model.layers.37.mlp.experts.145.down_proj", + "model.layers.37.mlp.experts.145.gate_proj", + "model.layers.37.mlp.experts.145.up_proj", + "model.layers.37.mlp.experts.146.down_proj", + "model.layers.37.mlp.experts.146.gate_proj", + "model.layers.37.mlp.experts.146.up_proj", + "model.layers.37.mlp.experts.147.down_proj", + "model.layers.37.mlp.experts.147.gate_proj", + "model.layers.37.mlp.experts.147.up_proj", + "model.layers.37.mlp.experts.148.down_proj", + "model.layers.37.mlp.experts.148.gate_proj", + "model.layers.37.mlp.experts.148.up_proj", + "model.layers.37.mlp.experts.149.down_proj", + "model.layers.37.mlp.experts.149.gate_proj", + "model.layers.37.mlp.experts.149.up_proj", + "model.layers.37.mlp.experts.15.down_proj", + "model.layers.37.mlp.experts.15.gate_proj", + "model.layers.37.mlp.experts.15.up_proj", + "model.layers.37.mlp.experts.150.down_proj", + "model.layers.37.mlp.experts.150.gate_proj", + "model.layers.37.mlp.experts.150.up_proj", + "model.layers.37.mlp.experts.151.down_proj", + "model.layers.37.mlp.experts.151.gate_proj", + "model.layers.37.mlp.experts.151.up_proj", + "model.layers.37.mlp.experts.152.down_proj", + "model.layers.37.mlp.experts.152.gate_proj", + "model.layers.37.mlp.experts.152.up_proj", + "model.layers.37.mlp.experts.153.down_proj", + "model.layers.37.mlp.experts.153.gate_proj", + "model.layers.37.mlp.experts.153.up_proj", + "model.layers.37.mlp.experts.154.down_proj", + "model.layers.37.mlp.experts.154.gate_proj", + "model.layers.37.mlp.experts.154.up_proj", + "model.layers.37.mlp.experts.155.down_proj", + "model.layers.37.mlp.experts.155.gate_proj", + "model.layers.37.mlp.experts.155.up_proj", + "model.layers.37.mlp.experts.156.down_proj", + "model.layers.37.mlp.experts.156.gate_proj", + "model.layers.37.mlp.experts.156.up_proj", + "model.layers.37.mlp.experts.157.down_proj", + "model.layers.37.mlp.experts.157.gate_proj", + "model.layers.37.mlp.experts.157.up_proj", + "model.layers.37.mlp.experts.158.down_proj", + "model.layers.37.mlp.experts.158.gate_proj", + "model.layers.37.mlp.experts.158.up_proj", + "model.layers.37.mlp.experts.159.down_proj", + "model.layers.37.mlp.experts.159.gate_proj", + "model.layers.37.mlp.experts.159.up_proj", + "model.layers.37.mlp.experts.16.down_proj", + "model.layers.37.mlp.experts.16.gate_proj", + "model.layers.37.mlp.experts.16.up_proj", + "model.layers.37.mlp.experts.160.down_proj", + "model.layers.37.mlp.experts.160.gate_proj", + "model.layers.37.mlp.experts.160.up_proj", + "model.layers.37.mlp.experts.161.down_proj", + "model.layers.37.mlp.experts.161.gate_proj", + "model.layers.37.mlp.experts.161.up_proj", + "model.layers.37.mlp.experts.162.down_proj", + "model.layers.37.mlp.experts.162.gate_proj", + "model.layers.37.mlp.experts.162.up_proj", + "model.layers.37.mlp.experts.163.down_proj", + "model.layers.37.mlp.experts.163.gate_proj", + "model.layers.37.mlp.experts.163.up_proj", + "model.layers.37.mlp.experts.164.down_proj", + "model.layers.37.mlp.experts.164.gate_proj", + "model.layers.37.mlp.experts.164.up_proj", + "model.layers.37.mlp.experts.165.down_proj", + "model.layers.37.mlp.experts.165.gate_proj", + "model.layers.37.mlp.experts.165.up_proj", + "model.layers.37.mlp.experts.166.down_proj", + "model.layers.37.mlp.experts.166.gate_proj", + "model.layers.37.mlp.experts.166.up_proj", + "model.layers.37.mlp.experts.167.down_proj", + "model.layers.37.mlp.experts.167.gate_proj", + "model.layers.37.mlp.experts.167.up_proj", + "model.layers.37.mlp.experts.168.down_proj", + "model.layers.37.mlp.experts.168.gate_proj", + "model.layers.37.mlp.experts.168.up_proj", + "model.layers.37.mlp.experts.169.down_proj", + "model.layers.37.mlp.experts.169.gate_proj", + "model.layers.37.mlp.experts.169.up_proj", + "model.layers.37.mlp.experts.17.down_proj", + "model.layers.37.mlp.experts.17.gate_proj", + "model.layers.37.mlp.experts.17.up_proj", + "model.layers.37.mlp.experts.170.down_proj", + "model.layers.37.mlp.experts.170.gate_proj", + "model.layers.37.mlp.experts.170.up_proj", + "model.layers.37.mlp.experts.171.down_proj", + "model.layers.37.mlp.experts.171.gate_proj", + "model.layers.37.mlp.experts.171.up_proj", + "model.layers.37.mlp.experts.172.down_proj", + "model.layers.37.mlp.experts.172.gate_proj", + "model.layers.37.mlp.experts.172.up_proj", + "model.layers.37.mlp.experts.173.down_proj", + "model.layers.37.mlp.experts.173.gate_proj", + "model.layers.37.mlp.experts.173.up_proj", + "model.layers.37.mlp.experts.174.down_proj", + "model.layers.37.mlp.experts.174.gate_proj", + "model.layers.37.mlp.experts.174.up_proj", + "model.layers.37.mlp.experts.175.down_proj", + "model.layers.37.mlp.experts.175.gate_proj", + "model.layers.37.mlp.experts.175.up_proj", + "model.layers.37.mlp.experts.176.down_proj", + "model.layers.37.mlp.experts.176.gate_proj", + "model.layers.37.mlp.experts.176.up_proj", + "model.layers.37.mlp.experts.177.down_proj", + "model.layers.37.mlp.experts.177.gate_proj", + "model.layers.37.mlp.experts.177.up_proj", + "model.layers.37.mlp.experts.178.down_proj", + "model.layers.37.mlp.experts.178.gate_proj", + "model.layers.37.mlp.experts.178.up_proj", + "model.layers.37.mlp.experts.179.down_proj", + "model.layers.37.mlp.experts.179.gate_proj", + "model.layers.37.mlp.experts.179.up_proj", + "model.layers.37.mlp.experts.18.down_proj", + "model.layers.37.mlp.experts.18.gate_proj", + "model.layers.37.mlp.experts.18.up_proj", + "model.layers.37.mlp.experts.180.down_proj", + "model.layers.37.mlp.experts.180.gate_proj", + "model.layers.37.mlp.experts.180.up_proj", + "model.layers.37.mlp.experts.181.down_proj", + "model.layers.37.mlp.experts.181.gate_proj", + "model.layers.37.mlp.experts.181.up_proj", + "model.layers.37.mlp.experts.182.down_proj", + "model.layers.37.mlp.experts.182.gate_proj", + "model.layers.37.mlp.experts.182.up_proj", + "model.layers.37.mlp.experts.183.down_proj", + "model.layers.37.mlp.experts.183.gate_proj", + "model.layers.37.mlp.experts.183.up_proj", + "model.layers.37.mlp.experts.184.down_proj", + "model.layers.37.mlp.experts.184.gate_proj", + "model.layers.37.mlp.experts.184.up_proj", + "model.layers.37.mlp.experts.185.down_proj", + "model.layers.37.mlp.experts.185.gate_proj", + "model.layers.37.mlp.experts.185.up_proj", + "model.layers.37.mlp.experts.186.down_proj", + "model.layers.37.mlp.experts.186.gate_proj", + "model.layers.37.mlp.experts.186.up_proj", + "model.layers.37.mlp.experts.187.down_proj", + "model.layers.37.mlp.experts.187.gate_proj", + "model.layers.37.mlp.experts.187.up_proj", + "model.layers.37.mlp.experts.188.down_proj", + "model.layers.37.mlp.experts.188.gate_proj", + "model.layers.37.mlp.experts.188.up_proj", + "model.layers.37.mlp.experts.189.down_proj", + "model.layers.37.mlp.experts.189.gate_proj", + "model.layers.37.mlp.experts.189.up_proj", + "model.layers.37.mlp.experts.19.down_proj", + "model.layers.37.mlp.experts.19.gate_proj", + "model.layers.37.mlp.experts.19.up_proj", + "model.layers.37.mlp.experts.190.down_proj", + "model.layers.37.mlp.experts.190.gate_proj", + "model.layers.37.mlp.experts.190.up_proj", + "model.layers.37.mlp.experts.191.down_proj", + "model.layers.37.mlp.experts.191.gate_proj", + "model.layers.37.mlp.experts.191.up_proj", + "model.layers.37.mlp.experts.192.down_proj", + "model.layers.37.mlp.experts.192.gate_proj", + "model.layers.37.mlp.experts.192.up_proj", + "model.layers.37.mlp.experts.193.down_proj", + "model.layers.37.mlp.experts.193.gate_proj", + "model.layers.37.mlp.experts.193.up_proj", + "model.layers.37.mlp.experts.194.down_proj", + "model.layers.37.mlp.experts.194.gate_proj", + "model.layers.37.mlp.experts.194.up_proj", + "model.layers.37.mlp.experts.195.down_proj", + "model.layers.37.mlp.experts.195.gate_proj", + "model.layers.37.mlp.experts.195.up_proj", + "model.layers.37.mlp.experts.196.down_proj", + "model.layers.37.mlp.experts.196.gate_proj", + "model.layers.37.mlp.experts.196.up_proj", + "model.layers.37.mlp.experts.197.down_proj", + "model.layers.37.mlp.experts.197.gate_proj", + "model.layers.37.mlp.experts.197.up_proj", + "model.layers.37.mlp.experts.198.down_proj", + "model.layers.37.mlp.experts.198.gate_proj", + "model.layers.37.mlp.experts.198.up_proj", + "model.layers.37.mlp.experts.199.down_proj", + "model.layers.37.mlp.experts.199.gate_proj", + "model.layers.37.mlp.experts.199.up_proj", + "model.layers.37.mlp.experts.2.down_proj", + "model.layers.37.mlp.experts.2.gate_proj", + "model.layers.37.mlp.experts.2.up_proj", + "model.layers.37.mlp.experts.20.down_proj", + "model.layers.37.mlp.experts.20.gate_proj", + "model.layers.37.mlp.experts.20.up_proj", + "model.layers.37.mlp.experts.200.down_proj", + "model.layers.37.mlp.experts.200.gate_proj", + "model.layers.37.mlp.experts.200.up_proj", + "model.layers.37.mlp.experts.201.down_proj", + "model.layers.37.mlp.experts.201.gate_proj", + "model.layers.37.mlp.experts.201.up_proj", + "model.layers.37.mlp.experts.202.down_proj", + "model.layers.37.mlp.experts.202.gate_proj", + "model.layers.37.mlp.experts.202.up_proj", + "model.layers.37.mlp.experts.203.down_proj", + "model.layers.37.mlp.experts.203.gate_proj", + "model.layers.37.mlp.experts.203.up_proj", + "model.layers.37.mlp.experts.204.down_proj", + "model.layers.37.mlp.experts.204.gate_proj", + "model.layers.37.mlp.experts.204.up_proj", + "model.layers.37.mlp.experts.205.down_proj", + "model.layers.37.mlp.experts.205.gate_proj", + "model.layers.37.mlp.experts.205.up_proj", + "model.layers.37.mlp.experts.206.down_proj", + "model.layers.37.mlp.experts.206.gate_proj", + "model.layers.37.mlp.experts.206.up_proj", + "model.layers.37.mlp.experts.207.down_proj", + "model.layers.37.mlp.experts.207.gate_proj", + "model.layers.37.mlp.experts.207.up_proj", + "model.layers.37.mlp.experts.208.down_proj", + "model.layers.37.mlp.experts.208.gate_proj", + "model.layers.37.mlp.experts.208.up_proj", + "model.layers.37.mlp.experts.209.down_proj", + "model.layers.37.mlp.experts.209.gate_proj", + "model.layers.37.mlp.experts.209.up_proj", + "model.layers.37.mlp.experts.21.down_proj", + "model.layers.37.mlp.experts.21.gate_proj", + "model.layers.37.mlp.experts.21.up_proj", + "model.layers.37.mlp.experts.210.down_proj", + "model.layers.37.mlp.experts.210.gate_proj", + "model.layers.37.mlp.experts.210.up_proj", + "model.layers.37.mlp.experts.211.down_proj", + "model.layers.37.mlp.experts.211.gate_proj", + "model.layers.37.mlp.experts.211.up_proj", + "model.layers.37.mlp.experts.212.down_proj", + "model.layers.37.mlp.experts.212.gate_proj", + "model.layers.37.mlp.experts.212.up_proj", + "model.layers.37.mlp.experts.213.down_proj", + "model.layers.37.mlp.experts.213.gate_proj", + "model.layers.37.mlp.experts.213.up_proj", + "model.layers.37.mlp.experts.214.down_proj", + "model.layers.37.mlp.experts.214.gate_proj", + "model.layers.37.mlp.experts.214.up_proj", + "model.layers.37.mlp.experts.215.down_proj", + "model.layers.37.mlp.experts.215.gate_proj", + "model.layers.37.mlp.experts.215.up_proj", + "model.layers.37.mlp.experts.216.down_proj", + "model.layers.37.mlp.experts.216.gate_proj", + "model.layers.37.mlp.experts.216.up_proj", + "model.layers.37.mlp.experts.217.down_proj", + "model.layers.37.mlp.experts.217.gate_proj", + "model.layers.37.mlp.experts.217.up_proj", + "model.layers.37.mlp.experts.218.down_proj", + "model.layers.37.mlp.experts.218.gate_proj", + "model.layers.37.mlp.experts.218.up_proj", + "model.layers.37.mlp.experts.219.down_proj", + "model.layers.37.mlp.experts.219.gate_proj", + "model.layers.37.mlp.experts.219.up_proj", + "model.layers.37.mlp.experts.22.down_proj", + "model.layers.37.mlp.experts.22.gate_proj", + "model.layers.37.mlp.experts.22.up_proj", + "model.layers.37.mlp.experts.220.down_proj", + "model.layers.37.mlp.experts.220.gate_proj", + "model.layers.37.mlp.experts.220.up_proj", + "model.layers.37.mlp.experts.221.down_proj", + "model.layers.37.mlp.experts.221.gate_proj", + "model.layers.37.mlp.experts.221.up_proj", + "model.layers.37.mlp.experts.222.down_proj", + "model.layers.37.mlp.experts.222.gate_proj", + "model.layers.37.mlp.experts.222.up_proj", + "model.layers.37.mlp.experts.223.down_proj", + "model.layers.37.mlp.experts.223.gate_proj", + "model.layers.37.mlp.experts.223.up_proj", + "model.layers.37.mlp.experts.224.down_proj", + "model.layers.37.mlp.experts.224.gate_proj", + "model.layers.37.mlp.experts.224.up_proj", + "model.layers.37.mlp.experts.225.down_proj", + "model.layers.37.mlp.experts.225.gate_proj", + "model.layers.37.mlp.experts.225.up_proj", + "model.layers.37.mlp.experts.226.down_proj", + "model.layers.37.mlp.experts.226.gate_proj", + "model.layers.37.mlp.experts.226.up_proj", + "model.layers.37.mlp.experts.227.down_proj", + "model.layers.37.mlp.experts.227.gate_proj", + "model.layers.37.mlp.experts.227.up_proj", + "model.layers.37.mlp.experts.228.down_proj", + "model.layers.37.mlp.experts.228.gate_proj", + "model.layers.37.mlp.experts.228.up_proj", + "model.layers.37.mlp.experts.229.down_proj", + "model.layers.37.mlp.experts.229.gate_proj", + "model.layers.37.mlp.experts.229.up_proj", + "model.layers.37.mlp.experts.23.down_proj", + "model.layers.37.mlp.experts.23.gate_proj", + "model.layers.37.mlp.experts.23.up_proj", + "model.layers.37.mlp.experts.230.down_proj", + "model.layers.37.mlp.experts.230.gate_proj", + "model.layers.37.mlp.experts.230.up_proj", + "model.layers.37.mlp.experts.231.down_proj", + "model.layers.37.mlp.experts.231.gate_proj", + "model.layers.37.mlp.experts.231.up_proj", + "model.layers.37.mlp.experts.232.down_proj", + "model.layers.37.mlp.experts.232.gate_proj", + "model.layers.37.mlp.experts.232.up_proj", + "model.layers.37.mlp.experts.233.down_proj", + "model.layers.37.mlp.experts.233.gate_proj", + "model.layers.37.mlp.experts.233.up_proj", + "model.layers.37.mlp.experts.234.down_proj", + "model.layers.37.mlp.experts.234.gate_proj", + "model.layers.37.mlp.experts.234.up_proj", + "model.layers.37.mlp.experts.235.down_proj", + "model.layers.37.mlp.experts.235.gate_proj", + "model.layers.37.mlp.experts.235.up_proj", + "model.layers.37.mlp.experts.236.down_proj", + "model.layers.37.mlp.experts.236.gate_proj", + "model.layers.37.mlp.experts.236.up_proj", + "model.layers.37.mlp.experts.237.down_proj", + "model.layers.37.mlp.experts.237.gate_proj", + "model.layers.37.mlp.experts.237.up_proj", + "model.layers.37.mlp.experts.238.down_proj", + "model.layers.37.mlp.experts.238.gate_proj", + "model.layers.37.mlp.experts.238.up_proj", + "model.layers.37.mlp.experts.239.down_proj", + "model.layers.37.mlp.experts.239.gate_proj", + "model.layers.37.mlp.experts.239.up_proj", + "model.layers.37.mlp.experts.24.down_proj", + "model.layers.37.mlp.experts.24.gate_proj", + "model.layers.37.mlp.experts.24.up_proj", + "model.layers.37.mlp.experts.240.down_proj", + "model.layers.37.mlp.experts.240.gate_proj", + "model.layers.37.mlp.experts.240.up_proj", + "model.layers.37.mlp.experts.241.down_proj", + "model.layers.37.mlp.experts.241.gate_proj", + "model.layers.37.mlp.experts.241.up_proj", + "model.layers.37.mlp.experts.242.down_proj", + "model.layers.37.mlp.experts.242.gate_proj", + "model.layers.37.mlp.experts.242.up_proj", + "model.layers.37.mlp.experts.243.down_proj", + "model.layers.37.mlp.experts.243.gate_proj", + "model.layers.37.mlp.experts.243.up_proj", + "model.layers.37.mlp.experts.244.down_proj", + "model.layers.37.mlp.experts.244.gate_proj", + "model.layers.37.mlp.experts.244.up_proj", + "model.layers.37.mlp.experts.245.down_proj", + "model.layers.37.mlp.experts.245.gate_proj", + "model.layers.37.mlp.experts.245.up_proj", + "model.layers.37.mlp.experts.246.down_proj", + "model.layers.37.mlp.experts.246.gate_proj", + "model.layers.37.mlp.experts.246.up_proj", + "model.layers.37.mlp.experts.247.down_proj", + "model.layers.37.mlp.experts.247.gate_proj", + "model.layers.37.mlp.experts.247.up_proj", + "model.layers.37.mlp.experts.248.down_proj", + "model.layers.37.mlp.experts.248.gate_proj", + "model.layers.37.mlp.experts.248.up_proj", + "model.layers.37.mlp.experts.249.down_proj", + "model.layers.37.mlp.experts.249.gate_proj", + "model.layers.37.mlp.experts.249.up_proj", + "model.layers.37.mlp.experts.25.down_proj", + "model.layers.37.mlp.experts.25.gate_proj", + "model.layers.37.mlp.experts.25.up_proj", + "model.layers.37.mlp.experts.250.down_proj", + "model.layers.37.mlp.experts.250.gate_proj", + "model.layers.37.mlp.experts.250.up_proj", + "model.layers.37.mlp.experts.251.down_proj", + "model.layers.37.mlp.experts.251.gate_proj", + "model.layers.37.mlp.experts.251.up_proj", + "model.layers.37.mlp.experts.252.down_proj", + "model.layers.37.mlp.experts.252.gate_proj", + "model.layers.37.mlp.experts.252.up_proj", + "model.layers.37.mlp.experts.253.down_proj", + "model.layers.37.mlp.experts.253.gate_proj", + "model.layers.37.mlp.experts.253.up_proj", + "model.layers.37.mlp.experts.254.down_proj", + "model.layers.37.mlp.experts.254.gate_proj", + "model.layers.37.mlp.experts.254.up_proj", + "model.layers.37.mlp.experts.255.down_proj", + "model.layers.37.mlp.experts.255.gate_proj", + "model.layers.37.mlp.experts.255.up_proj", + "model.layers.37.mlp.experts.26.down_proj", + "model.layers.37.mlp.experts.26.gate_proj", + "model.layers.37.mlp.experts.26.up_proj", + "model.layers.37.mlp.experts.27.down_proj", + "model.layers.37.mlp.experts.27.gate_proj", + "model.layers.37.mlp.experts.27.up_proj", + "model.layers.37.mlp.experts.28.down_proj", + "model.layers.37.mlp.experts.28.gate_proj", + "model.layers.37.mlp.experts.28.up_proj", + "model.layers.37.mlp.experts.29.down_proj", + "model.layers.37.mlp.experts.29.gate_proj", + "model.layers.37.mlp.experts.29.up_proj", + "model.layers.37.mlp.experts.3.down_proj", + "model.layers.37.mlp.experts.3.gate_proj", + "model.layers.37.mlp.experts.3.up_proj", + "model.layers.37.mlp.experts.30.down_proj", + "model.layers.37.mlp.experts.30.gate_proj", + "model.layers.37.mlp.experts.30.up_proj", + "model.layers.37.mlp.experts.31.down_proj", + "model.layers.37.mlp.experts.31.gate_proj", + "model.layers.37.mlp.experts.31.up_proj", + "model.layers.37.mlp.experts.32.down_proj", + "model.layers.37.mlp.experts.32.gate_proj", + "model.layers.37.mlp.experts.32.up_proj", + "model.layers.37.mlp.experts.33.down_proj", + "model.layers.37.mlp.experts.33.gate_proj", + "model.layers.37.mlp.experts.33.up_proj", + "model.layers.37.mlp.experts.34.down_proj", + "model.layers.37.mlp.experts.34.gate_proj", + "model.layers.37.mlp.experts.34.up_proj", + "model.layers.37.mlp.experts.35.down_proj", + "model.layers.37.mlp.experts.35.gate_proj", + "model.layers.37.mlp.experts.35.up_proj", + "model.layers.37.mlp.experts.36.down_proj", + "model.layers.37.mlp.experts.36.gate_proj", + "model.layers.37.mlp.experts.36.up_proj", + "model.layers.37.mlp.experts.37.down_proj", + "model.layers.37.mlp.experts.37.gate_proj", + "model.layers.37.mlp.experts.37.up_proj", + "model.layers.37.mlp.experts.38.down_proj", + "model.layers.37.mlp.experts.38.gate_proj", + "model.layers.37.mlp.experts.38.up_proj", + "model.layers.37.mlp.experts.39.down_proj", + "model.layers.37.mlp.experts.39.gate_proj", + "model.layers.37.mlp.experts.39.up_proj", + "model.layers.37.mlp.experts.4.down_proj", + "model.layers.37.mlp.experts.4.gate_proj", + "model.layers.37.mlp.experts.4.up_proj", + "model.layers.37.mlp.experts.40.down_proj", + "model.layers.37.mlp.experts.40.gate_proj", + "model.layers.37.mlp.experts.40.up_proj", + "model.layers.37.mlp.experts.41.down_proj", + "model.layers.37.mlp.experts.41.gate_proj", + "model.layers.37.mlp.experts.41.up_proj", + "model.layers.37.mlp.experts.42.down_proj", + "model.layers.37.mlp.experts.42.gate_proj", + "model.layers.37.mlp.experts.42.up_proj", + "model.layers.37.mlp.experts.43.down_proj", + "model.layers.37.mlp.experts.43.gate_proj", + "model.layers.37.mlp.experts.43.up_proj", + "model.layers.37.mlp.experts.44.down_proj", + "model.layers.37.mlp.experts.44.gate_proj", + "model.layers.37.mlp.experts.44.up_proj", + "model.layers.37.mlp.experts.45.down_proj", + "model.layers.37.mlp.experts.45.gate_proj", + "model.layers.37.mlp.experts.45.up_proj", + "model.layers.37.mlp.experts.46.down_proj", + "model.layers.37.mlp.experts.46.gate_proj", + "model.layers.37.mlp.experts.46.up_proj", + "model.layers.37.mlp.experts.47.down_proj", + "model.layers.37.mlp.experts.47.gate_proj", + "model.layers.37.mlp.experts.47.up_proj", + "model.layers.37.mlp.experts.48.down_proj", + "model.layers.37.mlp.experts.48.gate_proj", + "model.layers.37.mlp.experts.48.up_proj", + "model.layers.37.mlp.experts.49.down_proj", + "model.layers.37.mlp.experts.49.gate_proj", + "model.layers.37.mlp.experts.49.up_proj", + "model.layers.37.mlp.experts.5.down_proj", + "model.layers.37.mlp.experts.5.gate_proj", + "model.layers.37.mlp.experts.5.up_proj", + "model.layers.37.mlp.experts.50.down_proj", + "model.layers.37.mlp.experts.50.gate_proj", + "model.layers.37.mlp.experts.50.up_proj", + "model.layers.37.mlp.experts.51.down_proj", + "model.layers.37.mlp.experts.51.gate_proj", + "model.layers.37.mlp.experts.51.up_proj", + "model.layers.37.mlp.experts.52.down_proj", + "model.layers.37.mlp.experts.52.gate_proj", + "model.layers.37.mlp.experts.52.up_proj", + "model.layers.37.mlp.experts.53.down_proj", + "model.layers.37.mlp.experts.53.gate_proj", + "model.layers.37.mlp.experts.53.up_proj", + "model.layers.37.mlp.experts.54.down_proj", + "model.layers.37.mlp.experts.54.gate_proj", + "model.layers.37.mlp.experts.54.up_proj", + "model.layers.37.mlp.experts.55.down_proj", + "model.layers.37.mlp.experts.55.gate_proj", + "model.layers.37.mlp.experts.55.up_proj", + "model.layers.37.mlp.experts.56.down_proj", + "model.layers.37.mlp.experts.56.gate_proj", + "model.layers.37.mlp.experts.56.up_proj", + "model.layers.37.mlp.experts.57.down_proj", + "model.layers.37.mlp.experts.57.gate_proj", + "model.layers.37.mlp.experts.57.up_proj", + "model.layers.37.mlp.experts.58.down_proj", + "model.layers.37.mlp.experts.58.gate_proj", + "model.layers.37.mlp.experts.58.up_proj", + "model.layers.37.mlp.experts.59.down_proj", + "model.layers.37.mlp.experts.59.gate_proj", + "model.layers.37.mlp.experts.59.up_proj", + "model.layers.37.mlp.experts.6.down_proj", + "model.layers.37.mlp.experts.6.gate_proj", + "model.layers.37.mlp.experts.6.up_proj", + "model.layers.37.mlp.experts.60.down_proj", + "model.layers.37.mlp.experts.60.gate_proj", + "model.layers.37.mlp.experts.60.up_proj", + "model.layers.37.mlp.experts.61.down_proj", + "model.layers.37.mlp.experts.61.gate_proj", + "model.layers.37.mlp.experts.61.up_proj", + "model.layers.37.mlp.experts.62.down_proj", + "model.layers.37.mlp.experts.62.gate_proj", + "model.layers.37.mlp.experts.62.up_proj", + "model.layers.37.mlp.experts.63.down_proj", + "model.layers.37.mlp.experts.63.gate_proj", + "model.layers.37.mlp.experts.63.up_proj", + "model.layers.37.mlp.experts.64.down_proj", + "model.layers.37.mlp.experts.64.gate_proj", + "model.layers.37.mlp.experts.64.up_proj", + "model.layers.37.mlp.experts.65.down_proj", + "model.layers.37.mlp.experts.65.gate_proj", + "model.layers.37.mlp.experts.65.up_proj", + "model.layers.37.mlp.experts.66.down_proj", + "model.layers.37.mlp.experts.66.gate_proj", + "model.layers.37.mlp.experts.66.up_proj", + "model.layers.37.mlp.experts.67.down_proj", + "model.layers.37.mlp.experts.67.gate_proj", + "model.layers.37.mlp.experts.67.up_proj", + "model.layers.37.mlp.experts.68.down_proj", + "model.layers.37.mlp.experts.68.gate_proj", + "model.layers.37.mlp.experts.68.up_proj", + "model.layers.37.mlp.experts.69.down_proj", + "model.layers.37.mlp.experts.69.gate_proj", + "model.layers.37.mlp.experts.69.up_proj", + "model.layers.37.mlp.experts.7.down_proj", + "model.layers.37.mlp.experts.7.gate_proj", + "model.layers.37.mlp.experts.7.up_proj", + "model.layers.37.mlp.experts.70.down_proj", + "model.layers.37.mlp.experts.70.gate_proj", + "model.layers.37.mlp.experts.70.up_proj", + "model.layers.37.mlp.experts.71.down_proj", + "model.layers.37.mlp.experts.71.gate_proj", + "model.layers.37.mlp.experts.71.up_proj", + "model.layers.37.mlp.experts.72.down_proj", + "model.layers.37.mlp.experts.72.gate_proj", + "model.layers.37.mlp.experts.72.up_proj", + "model.layers.37.mlp.experts.73.down_proj", + "model.layers.37.mlp.experts.73.gate_proj", + "model.layers.37.mlp.experts.73.up_proj", + "model.layers.37.mlp.experts.74.down_proj", + "model.layers.37.mlp.experts.74.gate_proj", + "model.layers.37.mlp.experts.74.up_proj", + "model.layers.37.mlp.experts.75.down_proj", + "model.layers.37.mlp.experts.75.gate_proj", + "model.layers.37.mlp.experts.75.up_proj", + "model.layers.37.mlp.experts.76.down_proj", + "model.layers.37.mlp.experts.76.gate_proj", + "model.layers.37.mlp.experts.76.up_proj", + "model.layers.37.mlp.experts.77.down_proj", + "model.layers.37.mlp.experts.77.gate_proj", + "model.layers.37.mlp.experts.77.up_proj", + "model.layers.37.mlp.experts.78.down_proj", + "model.layers.37.mlp.experts.78.gate_proj", + "model.layers.37.mlp.experts.78.up_proj", + "model.layers.37.mlp.experts.79.down_proj", + "model.layers.37.mlp.experts.79.gate_proj", + "model.layers.37.mlp.experts.79.up_proj", + "model.layers.37.mlp.experts.8.down_proj", + "model.layers.37.mlp.experts.8.gate_proj", + "model.layers.37.mlp.experts.8.up_proj", + "model.layers.37.mlp.experts.80.down_proj", + "model.layers.37.mlp.experts.80.gate_proj", + "model.layers.37.mlp.experts.80.up_proj", + "model.layers.37.mlp.experts.81.down_proj", + "model.layers.37.mlp.experts.81.gate_proj", + "model.layers.37.mlp.experts.81.up_proj", + "model.layers.37.mlp.experts.82.down_proj", + "model.layers.37.mlp.experts.82.gate_proj", + "model.layers.37.mlp.experts.82.up_proj", + "model.layers.37.mlp.experts.83.down_proj", + "model.layers.37.mlp.experts.83.gate_proj", + "model.layers.37.mlp.experts.83.up_proj", + "model.layers.37.mlp.experts.84.down_proj", + "model.layers.37.mlp.experts.84.gate_proj", + "model.layers.37.mlp.experts.84.up_proj", + "model.layers.37.mlp.experts.85.down_proj", + "model.layers.37.mlp.experts.85.gate_proj", + "model.layers.37.mlp.experts.85.up_proj", + "model.layers.37.mlp.experts.86.down_proj", + "model.layers.37.mlp.experts.86.gate_proj", + "model.layers.37.mlp.experts.86.up_proj", + "model.layers.37.mlp.experts.87.down_proj", + "model.layers.37.mlp.experts.87.gate_proj", + "model.layers.37.mlp.experts.87.up_proj", + "model.layers.37.mlp.experts.88.down_proj", + "model.layers.37.mlp.experts.88.gate_proj", + "model.layers.37.mlp.experts.88.up_proj", + "model.layers.37.mlp.experts.89.down_proj", + "model.layers.37.mlp.experts.89.gate_proj", + "model.layers.37.mlp.experts.89.up_proj", + "model.layers.37.mlp.experts.9.down_proj", + "model.layers.37.mlp.experts.9.gate_proj", + "model.layers.37.mlp.experts.9.up_proj", + "model.layers.37.mlp.experts.90.down_proj", + "model.layers.37.mlp.experts.90.gate_proj", + "model.layers.37.mlp.experts.90.up_proj", + "model.layers.37.mlp.experts.91.down_proj", + "model.layers.37.mlp.experts.91.gate_proj", + "model.layers.37.mlp.experts.91.up_proj", + "model.layers.37.mlp.experts.92.down_proj", + "model.layers.37.mlp.experts.92.gate_proj", + "model.layers.37.mlp.experts.92.up_proj", + "model.layers.37.mlp.experts.93.down_proj", + "model.layers.37.mlp.experts.93.gate_proj", + "model.layers.37.mlp.experts.93.up_proj", + "model.layers.37.mlp.experts.94.down_proj", + "model.layers.37.mlp.experts.94.gate_proj", + "model.layers.37.mlp.experts.94.up_proj", + "model.layers.37.mlp.experts.95.down_proj", + "model.layers.37.mlp.experts.95.gate_proj", + "model.layers.37.mlp.experts.95.up_proj", + "model.layers.37.mlp.experts.96.down_proj", + "model.layers.37.mlp.experts.96.gate_proj", + "model.layers.37.mlp.experts.96.up_proj", + "model.layers.37.mlp.experts.97.down_proj", + "model.layers.37.mlp.experts.97.gate_proj", + "model.layers.37.mlp.experts.97.up_proj", + "model.layers.37.mlp.experts.98.down_proj", + "model.layers.37.mlp.experts.98.gate_proj", + "model.layers.37.mlp.experts.98.up_proj", + "model.layers.37.mlp.experts.99.down_proj", + "model.layers.37.mlp.experts.99.gate_proj", + "model.layers.37.mlp.experts.99.up_proj", + "model.layers.38.mlp.experts.0.down_proj", + "model.layers.38.mlp.experts.0.gate_proj", + "model.layers.38.mlp.experts.0.up_proj", + "model.layers.38.mlp.experts.1.down_proj", + "model.layers.38.mlp.experts.1.gate_proj", + "model.layers.38.mlp.experts.1.up_proj", + "model.layers.38.mlp.experts.10.down_proj", + "model.layers.38.mlp.experts.10.gate_proj", + "model.layers.38.mlp.experts.10.up_proj", + "model.layers.38.mlp.experts.100.down_proj", + "model.layers.38.mlp.experts.100.gate_proj", + "model.layers.38.mlp.experts.100.up_proj", + "model.layers.38.mlp.experts.101.down_proj", + "model.layers.38.mlp.experts.101.gate_proj", + "model.layers.38.mlp.experts.101.up_proj", + "model.layers.38.mlp.experts.102.down_proj", + "model.layers.38.mlp.experts.102.gate_proj", + "model.layers.38.mlp.experts.102.up_proj", + "model.layers.38.mlp.experts.103.down_proj", + "model.layers.38.mlp.experts.103.gate_proj", + "model.layers.38.mlp.experts.103.up_proj", + "model.layers.38.mlp.experts.104.down_proj", + "model.layers.38.mlp.experts.104.gate_proj", + "model.layers.38.mlp.experts.104.up_proj", + "model.layers.38.mlp.experts.105.down_proj", + "model.layers.38.mlp.experts.105.gate_proj", + "model.layers.38.mlp.experts.105.up_proj", + "model.layers.38.mlp.experts.106.down_proj", + "model.layers.38.mlp.experts.106.gate_proj", + "model.layers.38.mlp.experts.106.up_proj", + "model.layers.38.mlp.experts.107.down_proj", + "model.layers.38.mlp.experts.107.gate_proj", + "model.layers.38.mlp.experts.107.up_proj", + "model.layers.38.mlp.experts.108.down_proj", + "model.layers.38.mlp.experts.108.gate_proj", + "model.layers.38.mlp.experts.108.up_proj", + "model.layers.38.mlp.experts.109.down_proj", + "model.layers.38.mlp.experts.109.gate_proj", + "model.layers.38.mlp.experts.109.up_proj", + "model.layers.38.mlp.experts.11.down_proj", + "model.layers.38.mlp.experts.11.gate_proj", + "model.layers.38.mlp.experts.11.up_proj", + "model.layers.38.mlp.experts.110.down_proj", + "model.layers.38.mlp.experts.110.gate_proj", + "model.layers.38.mlp.experts.110.up_proj", + "model.layers.38.mlp.experts.111.down_proj", + "model.layers.38.mlp.experts.111.gate_proj", + "model.layers.38.mlp.experts.111.up_proj", + "model.layers.38.mlp.experts.112.down_proj", + "model.layers.38.mlp.experts.112.gate_proj", + "model.layers.38.mlp.experts.112.up_proj", + "model.layers.38.mlp.experts.113.down_proj", + "model.layers.38.mlp.experts.113.gate_proj", + "model.layers.38.mlp.experts.113.up_proj", + "model.layers.38.mlp.experts.114.down_proj", + "model.layers.38.mlp.experts.114.gate_proj", + "model.layers.38.mlp.experts.114.up_proj", + "model.layers.38.mlp.experts.115.down_proj", + "model.layers.38.mlp.experts.115.gate_proj", + "model.layers.38.mlp.experts.115.up_proj", + "model.layers.38.mlp.experts.116.down_proj", + "model.layers.38.mlp.experts.116.gate_proj", + "model.layers.38.mlp.experts.116.up_proj", + "model.layers.38.mlp.experts.117.down_proj", + "model.layers.38.mlp.experts.117.gate_proj", + "model.layers.38.mlp.experts.117.up_proj", + "model.layers.38.mlp.experts.118.down_proj", + "model.layers.38.mlp.experts.118.gate_proj", + "model.layers.38.mlp.experts.118.up_proj", + "model.layers.38.mlp.experts.119.down_proj", + "model.layers.38.mlp.experts.119.gate_proj", + "model.layers.38.mlp.experts.119.up_proj", + "model.layers.38.mlp.experts.12.down_proj", + "model.layers.38.mlp.experts.12.gate_proj", + "model.layers.38.mlp.experts.12.up_proj", + "model.layers.38.mlp.experts.120.down_proj", + "model.layers.38.mlp.experts.120.gate_proj", + "model.layers.38.mlp.experts.120.up_proj", + "model.layers.38.mlp.experts.121.down_proj", + "model.layers.38.mlp.experts.121.gate_proj", + "model.layers.38.mlp.experts.121.up_proj", + "model.layers.38.mlp.experts.122.down_proj", + "model.layers.38.mlp.experts.122.gate_proj", + "model.layers.38.mlp.experts.122.up_proj", + "model.layers.38.mlp.experts.123.down_proj", + "model.layers.38.mlp.experts.123.gate_proj", + "model.layers.38.mlp.experts.123.up_proj", + "model.layers.38.mlp.experts.124.down_proj", + "model.layers.38.mlp.experts.124.gate_proj", + "model.layers.38.mlp.experts.124.up_proj", + "model.layers.38.mlp.experts.125.down_proj", + "model.layers.38.mlp.experts.125.gate_proj", + "model.layers.38.mlp.experts.125.up_proj", + "model.layers.38.mlp.experts.126.down_proj", + "model.layers.38.mlp.experts.126.gate_proj", + "model.layers.38.mlp.experts.126.up_proj", + "model.layers.38.mlp.experts.127.down_proj", + "model.layers.38.mlp.experts.127.gate_proj", + "model.layers.38.mlp.experts.127.up_proj", + "model.layers.38.mlp.experts.128.down_proj", + "model.layers.38.mlp.experts.128.gate_proj", + "model.layers.38.mlp.experts.128.up_proj", + "model.layers.38.mlp.experts.129.down_proj", + "model.layers.38.mlp.experts.129.gate_proj", + "model.layers.38.mlp.experts.129.up_proj", + "model.layers.38.mlp.experts.13.down_proj", + "model.layers.38.mlp.experts.13.gate_proj", + "model.layers.38.mlp.experts.13.up_proj", + "model.layers.38.mlp.experts.130.down_proj", + "model.layers.38.mlp.experts.130.gate_proj", + "model.layers.38.mlp.experts.130.up_proj", + "model.layers.38.mlp.experts.131.down_proj", + "model.layers.38.mlp.experts.131.gate_proj", + "model.layers.38.mlp.experts.131.up_proj", + "model.layers.38.mlp.experts.132.down_proj", + "model.layers.38.mlp.experts.132.gate_proj", + "model.layers.38.mlp.experts.132.up_proj", + "model.layers.38.mlp.experts.133.down_proj", + "model.layers.38.mlp.experts.133.gate_proj", + "model.layers.38.mlp.experts.133.up_proj", + "model.layers.38.mlp.experts.134.down_proj", + "model.layers.38.mlp.experts.134.gate_proj", + "model.layers.38.mlp.experts.134.up_proj", + "model.layers.38.mlp.experts.135.down_proj", + "model.layers.38.mlp.experts.135.gate_proj", + "model.layers.38.mlp.experts.135.up_proj", + "model.layers.38.mlp.experts.136.down_proj", + "model.layers.38.mlp.experts.136.gate_proj", + "model.layers.38.mlp.experts.136.up_proj", + "model.layers.38.mlp.experts.137.down_proj", + "model.layers.38.mlp.experts.137.gate_proj", + "model.layers.38.mlp.experts.137.up_proj", + "model.layers.38.mlp.experts.138.down_proj", + "model.layers.38.mlp.experts.138.gate_proj", + "model.layers.38.mlp.experts.138.up_proj", + "model.layers.38.mlp.experts.139.down_proj", + "model.layers.38.mlp.experts.139.gate_proj", + "model.layers.38.mlp.experts.139.up_proj", + "model.layers.38.mlp.experts.14.down_proj", + "model.layers.38.mlp.experts.14.gate_proj", + "model.layers.38.mlp.experts.14.up_proj", + "model.layers.38.mlp.experts.140.down_proj", + "model.layers.38.mlp.experts.140.gate_proj", + "model.layers.38.mlp.experts.140.up_proj", + "model.layers.38.mlp.experts.141.down_proj", + "model.layers.38.mlp.experts.141.gate_proj", + "model.layers.38.mlp.experts.141.up_proj", + "model.layers.38.mlp.experts.142.down_proj", + "model.layers.38.mlp.experts.142.gate_proj", + "model.layers.38.mlp.experts.142.up_proj", + "model.layers.38.mlp.experts.143.down_proj", + "model.layers.38.mlp.experts.143.gate_proj", + "model.layers.38.mlp.experts.143.up_proj", + "model.layers.38.mlp.experts.144.down_proj", + "model.layers.38.mlp.experts.144.gate_proj", + "model.layers.38.mlp.experts.144.up_proj", + "model.layers.38.mlp.experts.145.down_proj", + "model.layers.38.mlp.experts.145.gate_proj", + "model.layers.38.mlp.experts.145.up_proj", + "model.layers.38.mlp.experts.146.down_proj", + "model.layers.38.mlp.experts.146.gate_proj", + "model.layers.38.mlp.experts.146.up_proj", + "model.layers.38.mlp.experts.147.down_proj", + "model.layers.38.mlp.experts.147.gate_proj", + "model.layers.38.mlp.experts.147.up_proj", + "model.layers.38.mlp.experts.148.down_proj", + "model.layers.38.mlp.experts.148.gate_proj", + "model.layers.38.mlp.experts.148.up_proj", + "model.layers.38.mlp.experts.149.down_proj", + "model.layers.38.mlp.experts.149.gate_proj", + "model.layers.38.mlp.experts.149.up_proj", + "model.layers.38.mlp.experts.15.down_proj", + "model.layers.38.mlp.experts.15.gate_proj", + "model.layers.38.mlp.experts.15.up_proj", + "model.layers.38.mlp.experts.150.down_proj", + "model.layers.38.mlp.experts.150.gate_proj", + "model.layers.38.mlp.experts.150.up_proj", + "model.layers.38.mlp.experts.151.down_proj", + "model.layers.38.mlp.experts.151.gate_proj", + "model.layers.38.mlp.experts.151.up_proj", + "model.layers.38.mlp.experts.152.down_proj", + "model.layers.38.mlp.experts.152.gate_proj", + "model.layers.38.mlp.experts.152.up_proj", + "model.layers.38.mlp.experts.153.down_proj", + "model.layers.38.mlp.experts.153.gate_proj", + "model.layers.38.mlp.experts.153.up_proj", + "model.layers.38.mlp.experts.154.down_proj", + "model.layers.38.mlp.experts.154.gate_proj", + "model.layers.38.mlp.experts.154.up_proj", + "model.layers.38.mlp.experts.155.down_proj", + "model.layers.38.mlp.experts.155.gate_proj", + "model.layers.38.mlp.experts.155.up_proj", + "model.layers.38.mlp.experts.156.down_proj", + "model.layers.38.mlp.experts.156.gate_proj", + "model.layers.38.mlp.experts.156.up_proj", + "model.layers.38.mlp.experts.157.down_proj", + "model.layers.38.mlp.experts.157.gate_proj", + "model.layers.38.mlp.experts.157.up_proj", + "model.layers.38.mlp.experts.158.down_proj", + "model.layers.38.mlp.experts.158.gate_proj", + "model.layers.38.mlp.experts.158.up_proj", + "model.layers.38.mlp.experts.159.down_proj", + "model.layers.38.mlp.experts.159.gate_proj", + "model.layers.38.mlp.experts.159.up_proj", + "model.layers.38.mlp.experts.16.down_proj", + "model.layers.38.mlp.experts.16.gate_proj", + "model.layers.38.mlp.experts.16.up_proj", + "model.layers.38.mlp.experts.160.down_proj", + "model.layers.38.mlp.experts.160.gate_proj", + "model.layers.38.mlp.experts.160.up_proj", + "model.layers.38.mlp.experts.161.down_proj", + "model.layers.38.mlp.experts.161.gate_proj", + "model.layers.38.mlp.experts.161.up_proj", + "model.layers.38.mlp.experts.162.down_proj", + "model.layers.38.mlp.experts.162.gate_proj", + "model.layers.38.mlp.experts.162.up_proj", + "model.layers.38.mlp.experts.163.down_proj", + "model.layers.38.mlp.experts.163.gate_proj", + "model.layers.38.mlp.experts.163.up_proj", + "model.layers.38.mlp.experts.164.down_proj", + "model.layers.38.mlp.experts.164.gate_proj", + "model.layers.38.mlp.experts.164.up_proj", + "model.layers.38.mlp.experts.165.down_proj", + "model.layers.38.mlp.experts.165.gate_proj", + "model.layers.38.mlp.experts.165.up_proj", + "model.layers.38.mlp.experts.166.down_proj", + "model.layers.38.mlp.experts.166.gate_proj", + "model.layers.38.mlp.experts.166.up_proj", + "model.layers.38.mlp.experts.167.down_proj", + "model.layers.38.mlp.experts.167.gate_proj", + "model.layers.38.mlp.experts.167.up_proj", + "model.layers.38.mlp.experts.168.down_proj", + "model.layers.38.mlp.experts.168.gate_proj", + "model.layers.38.mlp.experts.168.up_proj", + "model.layers.38.mlp.experts.169.down_proj", + "model.layers.38.mlp.experts.169.gate_proj", + "model.layers.38.mlp.experts.169.up_proj", + "model.layers.38.mlp.experts.17.down_proj", + "model.layers.38.mlp.experts.17.gate_proj", + "model.layers.38.mlp.experts.17.up_proj", + "model.layers.38.mlp.experts.170.down_proj", + "model.layers.38.mlp.experts.170.gate_proj", + "model.layers.38.mlp.experts.170.up_proj", + "model.layers.38.mlp.experts.171.down_proj", + "model.layers.38.mlp.experts.171.gate_proj", + "model.layers.38.mlp.experts.171.up_proj", + "model.layers.38.mlp.experts.172.down_proj", + "model.layers.38.mlp.experts.172.gate_proj", + "model.layers.38.mlp.experts.172.up_proj", + "model.layers.38.mlp.experts.173.down_proj", + "model.layers.38.mlp.experts.173.gate_proj", + "model.layers.38.mlp.experts.173.up_proj", + "model.layers.38.mlp.experts.174.down_proj", + "model.layers.38.mlp.experts.174.gate_proj", + "model.layers.38.mlp.experts.174.up_proj", + "model.layers.38.mlp.experts.175.down_proj", + "model.layers.38.mlp.experts.175.gate_proj", + "model.layers.38.mlp.experts.175.up_proj", + "model.layers.38.mlp.experts.176.down_proj", + "model.layers.38.mlp.experts.176.gate_proj", + "model.layers.38.mlp.experts.176.up_proj", + "model.layers.38.mlp.experts.177.down_proj", + "model.layers.38.mlp.experts.177.gate_proj", + "model.layers.38.mlp.experts.177.up_proj", + "model.layers.38.mlp.experts.178.down_proj", + "model.layers.38.mlp.experts.178.gate_proj", + "model.layers.38.mlp.experts.178.up_proj", + "model.layers.38.mlp.experts.179.down_proj", + "model.layers.38.mlp.experts.179.gate_proj", + "model.layers.38.mlp.experts.179.up_proj", + "model.layers.38.mlp.experts.18.down_proj", + "model.layers.38.mlp.experts.18.gate_proj", + "model.layers.38.mlp.experts.18.up_proj", + "model.layers.38.mlp.experts.180.down_proj", + "model.layers.38.mlp.experts.180.gate_proj", + "model.layers.38.mlp.experts.180.up_proj", + "model.layers.38.mlp.experts.181.down_proj", + "model.layers.38.mlp.experts.181.gate_proj", + "model.layers.38.mlp.experts.181.up_proj", + "model.layers.38.mlp.experts.182.down_proj", + "model.layers.38.mlp.experts.182.gate_proj", + "model.layers.38.mlp.experts.182.up_proj", + "model.layers.38.mlp.experts.183.down_proj", + "model.layers.38.mlp.experts.183.gate_proj", + "model.layers.38.mlp.experts.183.up_proj", + "model.layers.38.mlp.experts.184.down_proj", + "model.layers.38.mlp.experts.184.gate_proj", + "model.layers.38.mlp.experts.184.up_proj", + "model.layers.38.mlp.experts.185.down_proj", + "model.layers.38.mlp.experts.185.gate_proj", + "model.layers.38.mlp.experts.185.up_proj", + "model.layers.38.mlp.experts.186.down_proj", + "model.layers.38.mlp.experts.186.gate_proj", + "model.layers.38.mlp.experts.186.up_proj", + "model.layers.38.mlp.experts.187.down_proj", + "model.layers.38.mlp.experts.187.gate_proj", + "model.layers.38.mlp.experts.187.up_proj", + "model.layers.38.mlp.experts.188.down_proj", + "model.layers.38.mlp.experts.188.gate_proj", + "model.layers.38.mlp.experts.188.up_proj", + "model.layers.38.mlp.experts.189.down_proj", + "model.layers.38.mlp.experts.189.gate_proj", + "model.layers.38.mlp.experts.189.up_proj", + "model.layers.38.mlp.experts.19.down_proj", + "model.layers.38.mlp.experts.19.gate_proj", + "model.layers.38.mlp.experts.19.up_proj", + "model.layers.38.mlp.experts.190.down_proj", + "model.layers.38.mlp.experts.190.gate_proj", + "model.layers.38.mlp.experts.190.up_proj", + "model.layers.38.mlp.experts.191.down_proj", + "model.layers.38.mlp.experts.191.gate_proj", + "model.layers.38.mlp.experts.191.up_proj", + "model.layers.38.mlp.experts.192.down_proj", + "model.layers.38.mlp.experts.192.gate_proj", + "model.layers.38.mlp.experts.192.up_proj", + "model.layers.38.mlp.experts.193.down_proj", + "model.layers.38.mlp.experts.193.gate_proj", + "model.layers.38.mlp.experts.193.up_proj", + "model.layers.38.mlp.experts.194.down_proj", + "model.layers.38.mlp.experts.194.gate_proj", + "model.layers.38.mlp.experts.194.up_proj", + "model.layers.38.mlp.experts.195.down_proj", + "model.layers.38.mlp.experts.195.gate_proj", + "model.layers.38.mlp.experts.195.up_proj", + "model.layers.38.mlp.experts.196.down_proj", + "model.layers.38.mlp.experts.196.gate_proj", + "model.layers.38.mlp.experts.196.up_proj", + "model.layers.38.mlp.experts.197.down_proj", + "model.layers.38.mlp.experts.197.gate_proj", + "model.layers.38.mlp.experts.197.up_proj", + "model.layers.38.mlp.experts.198.down_proj", + "model.layers.38.mlp.experts.198.gate_proj", + "model.layers.38.mlp.experts.198.up_proj", + "model.layers.38.mlp.experts.199.down_proj", + "model.layers.38.mlp.experts.199.gate_proj", + "model.layers.38.mlp.experts.199.up_proj", + "model.layers.38.mlp.experts.2.down_proj", + "model.layers.38.mlp.experts.2.gate_proj", + "model.layers.38.mlp.experts.2.up_proj", + "model.layers.38.mlp.experts.20.down_proj", + "model.layers.38.mlp.experts.20.gate_proj", + "model.layers.38.mlp.experts.20.up_proj", + "model.layers.38.mlp.experts.200.down_proj", + "model.layers.38.mlp.experts.200.gate_proj", + "model.layers.38.mlp.experts.200.up_proj", + "model.layers.38.mlp.experts.201.down_proj", + "model.layers.38.mlp.experts.201.gate_proj", + "model.layers.38.mlp.experts.201.up_proj", + "model.layers.38.mlp.experts.202.down_proj", + "model.layers.38.mlp.experts.202.gate_proj", + "model.layers.38.mlp.experts.202.up_proj", + "model.layers.38.mlp.experts.203.down_proj", + "model.layers.38.mlp.experts.203.gate_proj", + "model.layers.38.mlp.experts.203.up_proj", + "model.layers.38.mlp.experts.204.down_proj", + "model.layers.38.mlp.experts.204.gate_proj", + "model.layers.38.mlp.experts.204.up_proj", + "model.layers.38.mlp.experts.205.down_proj", + "model.layers.38.mlp.experts.205.gate_proj", + "model.layers.38.mlp.experts.205.up_proj", + "model.layers.38.mlp.experts.206.down_proj", + "model.layers.38.mlp.experts.206.gate_proj", + "model.layers.38.mlp.experts.206.up_proj", + "model.layers.38.mlp.experts.207.down_proj", + "model.layers.38.mlp.experts.207.gate_proj", + "model.layers.38.mlp.experts.207.up_proj", + "model.layers.38.mlp.experts.208.down_proj", + "model.layers.38.mlp.experts.208.gate_proj", + "model.layers.38.mlp.experts.208.up_proj", + "model.layers.38.mlp.experts.209.down_proj", + "model.layers.38.mlp.experts.209.gate_proj", + "model.layers.38.mlp.experts.209.up_proj", + "model.layers.38.mlp.experts.21.down_proj", + "model.layers.38.mlp.experts.21.gate_proj", + "model.layers.38.mlp.experts.21.up_proj", + "model.layers.38.mlp.experts.210.down_proj", + "model.layers.38.mlp.experts.210.gate_proj", + "model.layers.38.mlp.experts.210.up_proj", + "model.layers.38.mlp.experts.211.down_proj", + "model.layers.38.mlp.experts.211.gate_proj", + "model.layers.38.mlp.experts.211.up_proj", + "model.layers.38.mlp.experts.212.down_proj", + "model.layers.38.mlp.experts.212.gate_proj", + "model.layers.38.mlp.experts.212.up_proj", + "model.layers.38.mlp.experts.213.down_proj", + "model.layers.38.mlp.experts.213.gate_proj", + "model.layers.38.mlp.experts.213.up_proj", + "model.layers.38.mlp.experts.214.down_proj", + "model.layers.38.mlp.experts.214.gate_proj", + "model.layers.38.mlp.experts.214.up_proj", + "model.layers.38.mlp.experts.215.down_proj", + "model.layers.38.mlp.experts.215.gate_proj", + "model.layers.38.mlp.experts.215.up_proj", + "model.layers.38.mlp.experts.216.down_proj", + "model.layers.38.mlp.experts.216.gate_proj", + "model.layers.38.mlp.experts.216.up_proj", + "model.layers.38.mlp.experts.217.down_proj", + "model.layers.38.mlp.experts.217.gate_proj", + "model.layers.38.mlp.experts.217.up_proj", + "model.layers.38.mlp.experts.218.down_proj", + "model.layers.38.mlp.experts.218.gate_proj", + "model.layers.38.mlp.experts.218.up_proj", + "model.layers.38.mlp.experts.219.down_proj", + "model.layers.38.mlp.experts.219.gate_proj", + "model.layers.38.mlp.experts.219.up_proj", + "model.layers.38.mlp.experts.22.down_proj", + "model.layers.38.mlp.experts.22.gate_proj", + "model.layers.38.mlp.experts.22.up_proj", + "model.layers.38.mlp.experts.220.down_proj", + "model.layers.38.mlp.experts.220.gate_proj", + "model.layers.38.mlp.experts.220.up_proj", + "model.layers.38.mlp.experts.221.down_proj", + "model.layers.38.mlp.experts.221.gate_proj", + "model.layers.38.mlp.experts.221.up_proj", + "model.layers.38.mlp.experts.222.down_proj", + "model.layers.38.mlp.experts.222.gate_proj", + "model.layers.38.mlp.experts.222.up_proj", + "model.layers.38.mlp.experts.223.down_proj", + "model.layers.38.mlp.experts.223.gate_proj", + "model.layers.38.mlp.experts.223.up_proj", + "model.layers.38.mlp.experts.224.down_proj", + "model.layers.38.mlp.experts.224.gate_proj", + "model.layers.38.mlp.experts.224.up_proj", + "model.layers.38.mlp.experts.225.down_proj", + "model.layers.38.mlp.experts.225.gate_proj", + "model.layers.38.mlp.experts.225.up_proj", + "model.layers.38.mlp.experts.226.down_proj", + "model.layers.38.mlp.experts.226.gate_proj", + "model.layers.38.mlp.experts.226.up_proj", + "model.layers.38.mlp.experts.227.down_proj", + "model.layers.38.mlp.experts.227.gate_proj", + "model.layers.38.mlp.experts.227.up_proj", + "model.layers.38.mlp.experts.228.down_proj", + "model.layers.38.mlp.experts.228.gate_proj", + "model.layers.38.mlp.experts.228.up_proj", + "model.layers.38.mlp.experts.229.down_proj", + "model.layers.38.mlp.experts.229.gate_proj", + "model.layers.38.mlp.experts.229.up_proj", + "model.layers.38.mlp.experts.23.down_proj", + "model.layers.38.mlp.experts.23.gate_proj", + "model.layers.38.mlp.experts.23.up_proj", + "model.layers.38.mlp.experts.230.down_proj", + "model.layers.38.mlp.experts.230.gate_proj", + "model.layers.38.mlp.experts.230.up_proj", + "model.layers.38.mlp.experts.231.down_proj", + "model.layers.38.mlp.experts.231.gate_proj", + "model.layers.38.mlp.experts.231.up_proj", + "model.layers.38.mlp.experts.232.down_proj", + "model.layers.38.mlp.experts.232.gate_proj", + "model.layers.38.mlp.experts.232.up_proj", + "model.layers.38.mlp.experts.233.down_proj", + "model.layers.38.mlp.experts.233.gate_proj", + "model.layers.38.mlp.experts.233.up_proj", + "model.layers.38.mlp.experts.234.down_proj", + "model.layers.38.mlp.experts.234.gate_proj", + "model.layers.38.mlp.experts.234.up_proj", + "model.layers.38.mlp.experts.235.down_proj", + "model.layers.38.mlp.experts.235.gate_proj", + "model.layers.38.mlp.experts.235.up_proj", + "model.layers.38.mlp.experts.236.down_proj", + "model.layers.38.mlp.experts.236.gate_proj", + "model.layers.38.mlp.experts.236.up_proj", + "model.layers.38.mlp.experts.237.down_proj", + "model.layers.38.mlp.experts.237.gate_proj", + "model.layers.38.mlp.experts.237.up_proj", + "model.layers.38.mlp.experts.238.down_proj", + "model.layers.38.mlp.experts.238.gate_proj", + "model.layers.38.mlp.experts.238.up_proj", + "model.layers.38.mlp.experts.239.down_proj", + "model.layers.38.mlp.experts.239.gate_proj", + "model.layers.38.mlp.experts.239.up_proj", + "model.layers.38.mlp.experts.24.down_proj", + "model.layers.38.mlp.experts.24.gate_proj", + "model.layers.38.mlp.experts.24.up_proj", + "model.layers.38.mlp.experts.240.down_proj", + "model.layers.38.mlp.experts.240.gate_proj", + "model.layers.38.mlp.experts.240.up_proj", + "model.layers.38.mlp.experts.241.down_proj", + "model.layers.38.mlp.experts.241.gate_proj", + "model.layers.38.mlp.experts.241.up_proj", + "model.layers.38.mlp.experts.242.down_proj", + "model.layers.38.mlp.experts.242.gate_proj", + "model.layers.38.mlp.experts.242.up_proj", + "model.layers.38.mlp.experts.243.down_proj", + "model.layers.38.mlp.experts.243.gate_proj", + "model.layers.38.mlp.experts.243.up_proj", + "model.layers.38.mlp.experts.244.down_proj", + "model.layers.38.mlp.experts.244.gate_proj", + "model.layers.38.mlp.experts.244.up_proj", + "model.layers.38.mlp.experts.245.down_proj", + "model.layers.38.mlp.experts.245.gate_proj", + "model.layers.38.mlp.experts.245.up_proj", + "model.layers.38.mlp.experts.246.down_proj", + "model.layers.38.mlp.experts.246.gate_proj", + "model.layers.38.mlp.experts.246.up_proj", + "model.layers.38.mlp.experts.247.down_proj", + "model.layers.38.mlp.experts.247.gate_proj", + "model.layers.38.mlp.experts.247.up_proj", + "model.layers.38.mlp.experts.248.down_proj", + "model.layers.38.mlp.experts.248.gate_proj", + "model.layers.38.mlp.experts.248.up_proj", + "model.layers.38.mlp.experts.249.down_proj", + "model.layers.38.mlp.experts.249.gate_proj", + "model.layers.38.mlp.experts.249.up_proj", + "model.layers.38.mlp.experts.25.down_proj", + "model.layers.38.mlp.experts.25.gate_proj", + "model.layers.38.mlp.experts.25.up_proj", + "model.layers.38.mlp.experts.250.down_proj", + "model.layers.38.mlp.experts.250.gate_proj", + "model.layers.38.mlp.experts.250.up_proj", + "model.layers.38.mlp.experts.251.down_proj", + "model.layers.38.mlp.experts.251.gate_proj", + "model.layers.38.mlp.experts.251.up_proj", + "model.layers.38.mlp.experts.252.down_proj", + "model.layers.38.mlp.experts.252.gate_proj", + "model.layers.38.mlp.experts.252.up_proj", + "model.layers.38.mlp.experts.253.down_proj", + "model.layers.38.mlp.experts.253.gate_proj", + "model.layers.38.mlp.experts.253.up_proj", + "model.layers.38.mlp.experts.254.down_proj", + "model.layers.38.mlp.experts.254.gate_proj", + "model.layers.38.mlp.experts.254.up_proj", + "model.layers.38.mlp.experts.255.down_proj", + "model.layers.38.mlp.experts.255.gate_proj", + "model.layers.38.mlp.experts.255.up_proj", + "model.layers.38.mlp.experts.26.down_proj", + "model.layers.38.mlp.experts.26.gate_proj", + "model.layers.38.mlp.experts.26.up_proj", + "model.layers.38.mlp.experts.27.down_proj", + "model.layers.38.mlp.experts.27.gate_proj", + "model.layers.38.mlp.experts.27.up_proj", + "model.layers.38.mlp.experts.28.down_proj", + "model.layers.38.mlp.experts.28.gate_proj", + "model.layers.38.mlp.experts.28.up_proj", + "model.layers.38.mlp.experts.29.down_proj", + "model.layers.38.mlp.experts.29.gate_proj", + "model.layers.38.mlp.experts.29.up_proj", + "model.layers.38.mlp.experts.3.down_proj", + "model.layers.38.mlp.experts.3.gate_proj", + "model.layers.38.mlp.experts.3.up_proj", + "model.layers.38.mlp.experts.30.down_proj", + "model.layers.38.mlp.experts.30.gate_proj", + "model.layers.38.mlp.experts.30.up_proj", + "model.layers.38.mlp.experts.31.down_proj", + "model.layers.38.mlp.experts.31.gate_proj", + "model.layers.38.mlp.experts.31.up_proj", + "model.layers.38.mlp.experts.32.down_proj", + "model.layers.38.mlp.experts.32.gate_proj", + "model.layers.38.mlp.experts.32.up_proj", + "model.layers.38.mlp.experts.33.down_proj", + "model.layers.38.mlp.experts.33.gate_proj", + "model.layers.38.mlp.experts.33.up_proj", + "model.layers.38.mlp.experts.34.down_proj", + "model.layers.38.mlp.experts.34.gate_proj", + "model.layers.38.mlp.experts.34.up_proj", + "model.layers.38.mlp.experts.35.down_proj", + "model.layers.38.mlp.experts.35.gate_proj", + "model.layers.38.mlp.experts.35.up_proj", + "model.layers.38.mlp.experts.36.down_proj", + "model.layers.38.mlp.experts.36.gate_proj", + "model.layers.38.mlp.experts.36.up_proj", + "model.layers.38.mlp.experts.37.down_proj", + "model.layers.38.mlp.experts.37.gate_proj", + "model.layers.38.mlp.experts.37.up_proj", + "model.layers.38.mlp.experts.38.down_proj", + "model.layers.38.mlp.experts.38.gate_proj", + "model.layers.38.mlp.experts.38.up_proj", + "model.layers.38.mlp.experts.39.down_proj", + "model.layers.38.mlp.experts.39.gate_proj", + "model.layers.38.mlp.experts.39.up_proj", + "model.layers.38.mlp.experts.4.down_proj", + "model.layers.38.mlp.experts.4.gate_proj", + "model.layers.38.mlp.experts.4.up_proj", + "model.layers.38.mlp.experts.40.down_proj", + "model.layers.38.mlp.experts.40.gate_proj", + "model.layers.38.mlp.experts.40.up_proj", + "model.layers.38.mlp.experts.41.down_proj", + "model.layers.38.mlp.experts.41.gate_proj", + "model.layers.38.mlp.experts.41.up_proj", + "model.layers.38.mlp.experts.42.down_proj", + "model.layers.38.mlp.experts.42.gate_proj", + "model.layers.38.mlp.experts.42.up_proj", + "model.layers.38.mlp.experts.43.down_proj", + "model.layers.38.mlp.experts.43.gate_proj", + "model.layers.38.mlp.experts.43.up_proj", + "model.layers.38.mlp.experts.44.down_proj", + "model.layers.38.mlp.experts.44.gate_proj", + "model.layers.38.mlp.experts.44.up_proj", + "model.layers.38.mlp.experts.45.down_proj", + "model.layers.38.mlp.experts.45.gate_proj", + "model.layers.38.mlp.experts.45.up_proj", + "model.layers.38.mlp.experts.46.down_proj", + "model.layers.38.mlp.experts.46.gate_proj", + "model.layers.38.mlp.experts.46.up_proj", + "model.layers.38.mlp.experts.47.down_proj", + "model.layers.38.mlp.experts.47.gate_proj", + "model.layers.38.mlp.experts.47.up_proj", + "model.layers.38.mlp.experts.48.down_proj", + "model.layers.38.mlp.experts.48.gate_proj", + "model.layers.38.mlp.experts.48.up_proj", + "model.layers.38.mlp.experts.49.down_proj", + "model.layers.38.mlp.experts.49.gate_proj", + "model.layers.38.mlp.experts.49.up_proj", + "model.layers.38.mlp.experts.5.down_proj", + "model.layers.38.mlp.experts.5.gate_proj", + "model.layers.38.mlp.experts.5.up_proj", + "model.layers.38.mlp.experts.50.down_proj", + "model.layers.38.mlp.experts.50.gate_proj", + "model.layers.38.mlp.experts.50.up_proj", + "model.layers.38.mlp.experts.51.down_proj", + "model.layers.38.mlp.experts.51.gate_proj", + "model.layers.38.mlp.experts.51.up_proj", + "model.layers.38.mlp.experts.52.down_proj", + "model.layers.38.mlp.experts.52.gate_proj", + "model.layers.38.mlp.experts.52.up_proj", + "model.layers.38.mlp.experts.53.down_proj", + "model.layers.38.mlp.experts.53.gate_proj", + "model.layers.38.mlp.experts.53.up_proj", + "model.layers.38.mlp.experts.54.down_proj", + "model.layers.38.mlp.experts.54.gate_proj", + "model.layers.38.mlp.experts.54.up_proj", + "model.layers.38.mlp.experts.55.down_proj", + "model.layers.38.mlp.experts.55.gate_proj", + "model.layers.38.mlp.experts.55.up_proj", + "model.layers.38.mlp.experts.56.down_proj", + "model.layers.38.mlp.experts.56.gate_proj", + "model.layers.38.mlp.experts.56.up_proj", + "model.layers.38.mlp.experts.57.down_proj", + "model.layers.38.mlp.experts.57.gate_proj", + "model.layers.38.mlp.experts.57.up_proj", + "model.layers.38.mlp.experts.58.down_proj", + "model.layers.38.mlp.experts.58.gate_proj", + "model.layers.38.mlp.experts.58.up_proj", + "model.layers.38.mlp.experts.59.down_proj", + "model.layers.38.mlp.experts.59.gate_proj", + "model.layers.38.mlp.experts.59.up_proj", + "model.layers.38.mlp.experts.6.down_proj", + "model.layers.38.mlp.experts.6.gate_proj", + "model.layers.38.mlp.experts.6.up_proj", + "model.layers.38.mlp.experts.60.down_proj", + "model.layers.38.mlp.experts.60.gate_proj", + "model.layers.38.mlp.experts.60.up_proj", + "model.layers.38.mlp.experts.61.down_proj", + "model.layers.38.mlp.experts.61.gate_proj", + "model.layers.38.mlp.experts.61.up_proj", + "model.layers.38.mlp.experts.62.down_proj", + "model.layers.38.mlp.experts.62.gate_proj", + "model.layers.38.mlp.experts.62.up_proj", + "model.layers.38.mlp.experts.63.down_proj", + "model.layers.38.mlp.experts.63.gate_proj", + "model.layers.38.mlp.experts.63.up_proj", + "model.layers.38.mlp.experts.64.down_proj", + "model.layers.38.mlp.experts.64.gate_proj", + "model.layers.38.mlp.experts.64.up_proj", + "model.layers.38.mlp.experts.65.down_proj", + "model.layers.38.mlp.experts.65.gate_proj", + "model.layers.38.mlp.experts.65.up_proj", + "model.layers.38.mlp.experts.66.down_proj", + "model.layers.38.mlp.experts.66.gate_proj", + "model.layers.38.mlp.experts.66.up_proj", + "model.layers.38.mlp.experts.67.down_proj", + "model.layers.38.mlp.experts.67.gate_proj", + "model.layers.38.mlp.experts.67.up_proj", + "model.layers.38.mlp.experts.68.down_proj", + "model.layers.38.mlp.experts.68.gate_proj", + "model.layers.38.mlp.experts.68.up_proj", + "model.layers.38.mlp.experts.69.down_proj", + "model.layers.38.mlp.experts.69.gate_proj", + "model.layers.38.mlp.experts.69.up_proj", + "model.layers.38.mlp.experts.7.down_proj", + "model.layers.38.mlp.experts.7.gate_proj", + "model.layers.38.mlp.experts.7.up_proj", + "model.layers.38.mlp.experts.70.down_proj", + "model.layers.38.mlp.experts.70.gate_proj", + "model.layers.38.mlp.experts.70.up_proj", + "model.layers.38.mlp.experts.71.down_proj", + "model.layers.38.mlp.experts.71.gate_proj", + "model.layers.38.mlp.experts.71.up_proj", + "model.layers.38.mlp.experts.72.down_proj", + "model.layers.38.mlp.experts.72.gate_proj", + "model.layers.38.mlp.experts.72.up_proj", + "model.layers.38.mlp.experts.73.down_proj", + "model.layers.38.mlp.experts.73.gate_proj", + "model.layers.38.mlp.experts.73.up_proj", + "model.layers.38.mlp.experts.74.down_proj", + "model.layers.38.mlp.experts.74.gate_proj", + "model.layers.38.mlp.experts.74.up_proj", + "model.layers.38.mlp.experts.75.down_proj", + "model.layers.38.mlp.experts.75.gate_proj", + "model.layers.38.mlp.experts.75.up_proj", + "model.layers.38.mlp.experts.76.down_proj", + "model.layers.38.mlp.experts.76.gate_proj", + "model.layers.38.mlp.experts.76.up_proj", + "model.layers.38.mlp.experts.77.down_proj", + "model.layers.38.mlp.experts.77.gate_proj", + "model.layers.38.mlp.experts.77.up_proj", + "model.layers.38.mlp.experts.78.down_proj", + "model.layers.38.mlp.experts.78.gate_proj", + "model.layers.38.mlp.experts.78.up_proj", + "model.layers.38.mlp.experts.79.down_proj", + "model.layers.38.mlp.experts.79.gate_proj", + "model.layers.38.mlp.experts.79.up_proj", + "model.layers.38.mlp.experts.8.down_proj", + "model.layers.38.mlp.experts.8.gate_proj", + "model.layers.38.mlp.experts.8.up_proj", + "model.layers.38.mlp.experts.80.down_proj", + "model.layers.38.mlp.experts.80.gate_proj", + "model.layers.38.mlp.experts.80.up_proj", + "model.layers.38.mlp.experts.81.down_proj", + "model.layers.38.mlp.experts.81.gate_proj", + "model.layers.38.mlp.experts.81.up_proj", + "model.layers.38.mlp.experts.82.down_proj", + "model.layers.38.mlp.experts.82.gate_proj", + "model.layers.38.mlp.experts.82.up_proj", + "model.layers.38.mlp.experts.83.down_proj", + "model.layers.38.mlp.experts.83.gate_proj", + "model.layers.38.mlp.experts.83.up_proj", + "model.layers.38.mlp.experts.84.down_proj", + "model.layers.38.mlp.experts.84.gate_proj", + "model.layers.38.mlp.experts.84.up_proj", + "model.layers.38.mlp.experts.85.down_proj", + "model.layers.38.mlp.experts.85.gate_proj", + "model.layers.38.mlp.experts.85.up_proj", + "model.layers.38.mlp.experts.86.down_proj", + "model.layers.38.mlp.experts.86.gate_proj", + "model.layers.38.mlp.experts.86.up_proj", + "model.layers.38.mlp.experts.87.down_proj", + "model.layers.38.mlp.experts.87.gate_proj", + "model.layers.38.mlp.experts.87.up_proj", + "model.layers.38.mlp.experts.88.down_proj", + "model.layers.38.mlp.experts.88.gate_proj", + "model.layers.38.mlp.experts.88.up_proj", + "model.layers.38.mlp.experts.89.down_proj", + "model.layers.38.mlp.experts.89.gate_proj", + "model.layers.38.mlp.experts.89.up_proj", + "model.layers.38.mlp.experts.9.down_proj", + "model.layers.38.mlp.experts.9.gate_proj", + "model.layers.38.mlp.experts.9.up_proj", + "model.layers.38.mlp.experts.90.down_proj", + "model.layers.38.mlp.experts.90.gate_proj", + "model.layers.38.mlp.experts.90.up_proj", + "model.layers.38.mlp.experts.91.down_proj", + "model.layers.38.mlp.experts.91.gate_proj", + "model.layers.38.mlp.experts.91.up_proj", + "model.layers.38.mlp.experts.92.down_proj", + "model.layers.38.mlp.experts.92.gate_proj", + "model.layers.38.mlp.experts.92.up_proj", + "model.layers.38.mlp.experts.93.down_proj", + "model.layers.38.mlp.experts.93.gate_proj", + "model.layers.38.mlp.experts.93.up_proj", + "model.layers.38.mlp.experts.94.down_proj", + "model.layers.38.mlp.experts.94.gate_proj", + "model.layers.38.mlp.experts.94.up_proj", + "model.layers.38.mlp.experts.95.down_proj", + "model.layers.38.mlp.experts.95.gate_proj", + "model.layers.38.mlp.experts.95.up_proj", + "model.layers.38.mlp.experts.96.down_proj", + "model.layers.38.mlp.experts.96.gate_proj", + "model.layers.38.mlp.experts.96.up_proj", + "model.layers.38.mlp.experts.97.down_proj", + "model.layers.38.mlp.experts.97.gate_proj", + "model.layers.38.mlp.experts.97.up_proj", + "model.layers.38.mlp.experts.98.down_proj", + "model.layers.38.mlp.experts.98.gate_proj", + "model.layers.38.mlp.experts.98.up_proj", + "model.layers.38.mlp.experts.99.down_proj", + "model.layers.38.mlp.experts.99.gate_proj", + "model.layers.38.mlp.experts.99.up_proj", + "model.layers.39.mlp.experts.0.down_proj", + "model.layers.39.mlp.experts.0.gate_proj", + "model.layers.39.mlp.experts.0.up_proj", + "model.layers.39.mlp.experts.1.down_proj", + "model.layers.39.mlp.experts.1.gate_proj", + "model.layers.39.mlp.experts.1.up_proj", + "model.layers.39.mlp.experts.10.down_proj", + "model.layers.39.mlp.experts.10.gate_proj", + "model.layers.39.mlp.experts.10.up_proj", + "model.layers.39.mlp.experts.100.down_proj", + "model.layers.39.mlp.experts.100.gate_proj", + "model.layers.39.mlp.experts.100.up_proj", + "model.layers.39.mlp.experts.101.down_proj", + "model.layers.39.mlp.experts.101.gate_proj", + "model.layers.39.mlp.experts.101.up_proj", + "model.layers.39.mlp.experts.102.down_proj", + "model.layers.39.mlp.experts.102.gate_proj", + "model.layers.39.mlp.experts.102.up_proj", + "model.layers.39.mlp.experts.103.down_proj", + "model.layers.39.mlp.experts.103.gate_proj", + "model.layers.39.mlp.experts.103.up_proj", + "model.layers.39.mlp.experts.104.down_proj", + "model.layers.39.mlp.experts.104.gate_proj", + "model.layers.39.mlp.experts.104.up_proj", + "model.layers.39.mlp.experts.105.down_proj", + "model.layers.39.mlp.experts.105.gate_proj", + "model.layers.39.mlp.experts.105.up_proj", + "model.layers.39.mlp.experts.106.down_proj", + "model.layers.39.mlp.experts.106.gate_proj", + "model.layers.39.mlp.experts.106.up_proj", + "model.layers.39.mlp.experts.107.down_proj", + "model.layers.39.mlp.experts.107.gate_proj", + "model.layers.39.mlp.experts.107.up_proj", + "model.layers.39.mlp.experts.108.down_proj", + "model.layers.39.mlp.experts.108.gate_proj", + "model.layers.39.mlp.experts.108.up_proj", + "model.layers.39.mlp.experts.109.down_proj", + "model.layers.39.mlp.experts.109.gate_proj", + "model.layers.39.mlp.experts.109.up_proj", + "model.layers.39.mlp.experts.11.down_proj", + "model.layers.39.mlp.experts.11.gate_proj", + "model.layers.39.mlp.experts.11.up_proj", + "model.layers.39.mlp.experts.110.down_proj", + "model.layers.39.mlp.experts.110.gate_proj", + "model.layers.39.mlp.experts.110.up_proj", + "model.layers.39.mlp.experts.111.down_proj", + "model.layers.39.mlp.experts.111.gate_proj", + "model.layers.39.mlp.experts.111.up_proj", + "model.layers.39.mlp.experts.112.down_proj", + "model.layers.39.mlp.experts.112.gate_proj", + "model.layers.39.mlp.experts.112.up_proj", + "model.layers.39.mlp.experts.113.down_proj", + "model.layers.39.mlp.experts.113.gate_proj", + "model.layers.39.mlp.experts.113.up_proj", + "model.layers.39.mlp.experts.114.down_proj", + "model.layers.39.mlp.experts.114.gate_proj", + "model.layers.39.mlp.experts.114.up_proj", + "model.layers.39.mlp.experts.115.down_proj", + "model.layers.39.mlp.experts.115.gate_proj", + "model.layers.39.mlp.experts.115.up_proj", + "model.layers.39.mlp.experts.116.down_proj", + "model.layers.39.mlp.experts.116.gate_proj", + "model.layers.39.mlp.experts.116.up_proj", + "model.layers.39.mlp.experts.117.down_proj", + "model.layers.39.mlp.experts.117.gate_proj", + "model.layers.39.mlp.experts.117.up_proj", + "model.layers.39.mlp.experts.118.down_proj", + "model.layers.39.mlp.experts.118.gate_proj", + "model.layers.39.mlp.experts.118.up_proj", + "model.layers.39.mlp.experts.119.down_proj", + "model.layers.39.mlp.experts.119.gate_proj", + "model.layers.39.mlp.experts.119.up_proj", + "model.layers.39.mlp.experts.12.down_proj", + "model.layers.39.mlp.experts.12.gate_proj", + "model.layers.39.mlp.experts.12.up_proj", + "model.layers.39.mlp.experts.120.down_proj", + "model.layers.39.mlp.experts.120.gate_proj", + "model.layers.39.mlp.experts.120.up_proj", + "model.layers.39.mlp.experts.121.down_proj", + "model.layers.39.mlp.experts.121.gate_proj", + "model.layers.39.mlp.experts.121.up_proj", + "model.layers.39.mlp.experts.122.down_proj", + "model.layers.39.mlp.experts.122.gate_proj", + "model.layers.39.mlp.experts.122.up_proj", + "model.layers.39.mlp.experts.123.down_proj", + "model.layers.39.mlp.experts.123.gate_proj", + "model.layers.39.mlp.experts.123.up_proj", + "model.layers.39.mlp.experts.124.down_proj", + "model.layers.39.mlp.experts.124.gate_proj", + "model.layers.39.mlp.experts.124.up_proj", + "model.layers.39.mlp.experts.125.down_proj", + "model.layers.39.mlp.experts.125.gate_proj", + "model.layers.39.mlp.experts.125.up_proj", + "model.layers.39.mlp.experts.126.down_proj", + "model.layers.39.mlp.experts.126.gate_proj", + "model.layers.39.mlp.experts.126.up_proj", + "model.layers.39.mlp.experts.127.down_proj", + "model.layers.39.mlp.experts.127.gate_proj", + "model.layers.39.mlp.experts.127.up_proj", + "model.layers.39.mlp.experts.128.down_proj", + "model.layers.39.mlp.experts.128.gate_proj", + "model.layers.39.mlp.experts.128.up_proj", + "model.layers.39.mlp.experts.129.down_proj", + "model.layers.39.mlp.experts.129.gate_proj", + "model.layers.39.mlp.experts.129.up_proj", + "model.layers.39.mlp.experts.13.down_proj", + "model.layers.39.mlp.experts.13.gate_proj", + "model.layers.39.mlp.experts.13.up_proj", + "model.layers.39.mlp.experts.130.down_proj", + "model.layers.39.mlp.experts.130.gate_proj", + "model.layers.39.mlp.experts.130.up_proj", + "model.layers.39.mlp.experts.131.down_proj", + "model.layers.39.mlp.experts.131.gate_proj", + "model.layers.39.mlp.experts.131.up_proj", + "model.layers.39.mlp.experts.132.down_proj", + "model.layers.39.mlp.experts.132.gate_proj", + "model.layers.39.mlp.experts.132.up_proj", + "model.layers.39.mlp.experts.133.down_proj", + "model.layers.39.mlp.experts.133.gate_proj", + "model.layers.39.mlp.experts.133.up_proj", + "model.layers.39.mlp.experts.134.down_proj", + "model.layers.39.mlp.experts.134.gate_proj", + "model.layers.39.mlp.experts.134.up_proj", + "model.layers.39.mlp.experts.135.down_proj", + "model.layers.39.mlp.experts.135.gate_proj", + "model.layers.39.mlp.experts.135.up_proj", + "model.layers.39.mlp.experts.136.down_proj", + "model.layers.39.mlp.experts.136.gate_proj", + "model.layers.39.mlp.experts.136.up_proj", + "model.layers.39.mlp.experts.137.down_proj", + "model.layers.39.mlp.experts.137.gate_proj", + "model.layers.39.mlp.experts.137.up_proj", + "model.layers.39.mlp.experts.138.down_proj", + "model.layers.39.mlp.experts.138.gate_proj", + "model.layers.39.mlp.experts.138.up_proj", + "model.layers.39.mlp.experts.139.down_proj", + "model.layers.39.mlp.experts.139.gate_proj", + "model.layers.39.mlp.experts.139.up_proj", + "model.layers.39.mlp.experts.14.down_proj", + "model.layers.39.mlp.experts.14.gate_proj", + "model.layers.39.mlp.experts.14.up_proj", + "model.layers.39.mlp.experts.140.down_proj", + "model.layers.39.mlp.experts.140.gate_proj", + "model.layers.39.mlp.experts.140.up_proj", + "model.layers.39.mlp.experts.141.down_proj", + "model.layers.39.mlp.experts.141.gate_proj", + "model.layers.39.mlp.experts.141.up_proj", + "model.layers.39.mlp.experts.142.down_proj", + "model.layers.39.mlp.experts.142.gate_proj", + "model.layers.39.mlp.experts.142.up_proj", + "model.layers.39.mlp.experts.143.down_proj", + "model.layers.39.mlp.experts.143.gate_proj", + "model.layers.39.mlp.experts.143.up_proj", + "model.layers.39.mlp.experts.144.down_proj", + "model.layers.39.mlp.experts.144.gate_proj", + "model.layers.39.mlp.experts.144.up_proj", + "model.layers.39.mlp.experts.145.down_proj", + "model.layers.39.mlp.experts.145.gate_proj", + "model.layers.39.mlp.experts.145.up_proj", + "model.layers.39.mlp.experts.146.down_proj", + "model.layers.39.mlp.experts.146.gate_proj", + "model.layers.39.mlp.experts.146.up_proj", + "model.layers.39.mlp.experts.147.down_proj", + "model.layers.39.mlp.experts.147.gate_proj", + "model.layers.39.mlp.experts.147.up_proj", + "model.layers.39.mlp.experts.148.down_proj", + "model.layers.39.mlp.experts.148.gate_proj", + "model.layers.39.mlp.experts.148.up_proj", + "model.layers.39.mlp.experts.149.down_proj", + "model.layers.39.mlp.experts.149.gate_proj", + "model.layers.39.mlp.experts.149.up_proj", + "model.layers.39.mlp.experts.15.down_proj", + "model.layers.39.mlp.experts.15.gate_proj", + "model.layers.39.mlp.experts.15.up_proj", + "model.layers.39.mlp.experts.150.down_proj", + "model.layers.39.mlp.experts.150.gate_proj", + "model.layers.39.mlp.experts.150.up_proj", + "model.layers.39.mlp.experts.151.down_proj", + "model.layers.39.mlp.experts.151.gate_proj", + "model.layers.39.mlp.experts.151.up_proj", + "model.layers.39.mlp.experts.152.down_proj", + "model.layers.39.mlp.experts.152.gate_proj", + "model.layers.39.mlp.experts.152.up_proj", + "model.layers.39.mlp.experts.153.down_proj", + "model.layers.39.mlp.experts.153.gate_proj", + "model.layers.39.mlp.experts.153.up_proj", + "model.layers.39.mlp.experts.154.down_proj", + "model.layers.39.mlp.experts.154.gate_proj", + "model.layers.39.mlp.experts.154.up_proj", + "model.layers.39.mlp.experts.155.down_proj", + "model.layers.39.mlp.experts.155.gate_proj", + "model.layers.39.mlp.experts.155.up_proj", + "model.layers.39.mlp.experts.156.down_proj", + "model.layers.39.mlp.experts.156.gate_proj", + "model.layers.39.mlp.experts.156.up_proj", + "model.layers.39.mlp.experts.157.down_proj", + "model.layers.39.mlp.experts.157.gate_proj", + "model.layers.39.mlp.experts.157.up_proj", + "model.layers.39.mlp.experts.158.down_proj", + "model.layers.39.mlp.experts.158.gate_proj", + "model.layers.39.mlp.experts.158.up_proj", + "model.layers.39.mlp.experts.159.down_proj", + "model.layers.39.mlp.experts.159.gate_proj", + "model.layers.39.mlp.experts.159.up_proj", + "model.layers.39.mlp.experts.16.down_proj", + "model.layers.39.mlp.experts.16.gate_proj", + "model.layers.39.mlp.experts.16.up_proj", + "model.layers.39.mlp.experts.160.down_proj", + "model.layers.39.mlp.experts.160.gate_proj", + "model.layers.39.mlp.experts.160.up_proj", + "model.layers.39.mlp.experts.161.down_proj", + "model.layers.39.mlp.experts.161.gate_proj", + "model.layers.39.mlp.experts.161.up_proj", + "model.layers.39.mlp.experts.162.down_proj", + "model.layers.39.mlp.experts.162.gate_proj", + "model.layers.39.mlp.experts.162.up_proj", + "model.layers.39.mlp.experts.163.down_proj", + "model.layers.39.mlp.experts.163.gate_proj", + "model.layers.39.mlp.experts.163.up_proj", + "model.layers.39.mlp.experts.164.down_proj", + "model.layers.39.mlp.experts.164.gate_proj", + "model.layers.39.mlp.experts.164.up_proj", + "model.layers.39.mlp.experts.165.down_proj", + "model.layers.39.mlp.experts.165.gate_proj", + "model.layers.39.mlp.experts.165.up_proj", + "model.layers.39.mlp.experts.166.down_proj", + "model.layers.39.mlp.experts.166.gate_proj", + "model.layers.39.mlp.experts.166.up_proj", + "model.layers.39.mlp.experts.167.down_proj", + "model.layers.39.mlp.experts.167.gate_proj", + "model.layers.39.mlp.experts.167.up_proj", + "model.layers.39.mlp.experts.168.down_proj", + "model.layers.39.mlp.experts.168.gate_proj", + "model.layers.39.mlp.experts.168.up_proj", + "model.layers.39.mlp.experts.169.down_proj", + "model.layers.39.mlp.experts.169.gate_proj", + "model.layers.39.mlp.experts.169.up_proj", + "model.layers.39.mlp.experts.17.down_proj", + "model.layers.39.mlp.experts.17.gate_proj", + "model.layers.39.mlp.experts.17.up_proj", + "model.layers.39.mlp.experts.170.down_proj", + "model.layers.39.mlp.experts.170.gate_proj", + "model.layers.39.mlp.experts.170.up_proj", + "model.layers.39.mlp.experts.171.down_proj", + "model.layers.39.mlp.experts.171.gate_proj", + "model.layers.39.mlp.experts.171.up_proj", + "model.layers.39.mlp.experts.172.down_proj", + "model.layers.39.mlp.experts.172.gate_proj", + "model.layers.39.mlp.experts.172.up_proj", + "model.layers.39.mlp.experts.173.down_proj", + "model.layers.39.mlp.experts.173.gate_proj", + "model.layers.39.mlp.experts.173.up_proj", + "model.layers.39.mlp.experts.174.down_proj", + "model.layers.39.mlp.experts.174.gate_proj", + "model.layers.39.mlp.experts.174.up_proj", + "model.layers.39.mlp.experts.175.down_proj", + "model.layers.39.mlp.experts.175.gate_proj", + "model.layers.39.mlp.experts.175.up_proj", + "model.layers.39.mlp.experts.176.down_proj", + "model.layers.39.mlp.experts.176.gate_proj", + "model.layers.39.mlp.experts.176.up_proj", + "model.layers.39.mlp.experts.177.down_proj", + "model.layers.39.mlp.experts.177.gate_proj", + "model.layers.39.mlp.experts.177.up_proj", + "model.layers.39.mlp.experts.178.down_proj", + "model.layers.39.mlp.experts.178.gate_proj", + "model.layers.39.mlp.experts.178.up_proj", + "model.layers.39.mlp.experts.179.down_proj", + "model.layers.39.mlp.experts.179.gate_proj", + "model.layers.39.mlp.experts.179.up_proj", + "model.layers.39.mlp.experts.18.down_proj", + "model.layers.39.mlp.experts.18.gate_proj", + "model.layers.39.mlp.experts.18.up_proj", + "model.layers.39.mlp.experts.180.down_proj", + "model.layers.39.mlp.experts.180.gate_proj", + "model.layers.39.mlp.experts.180.up_proj", + "model.layers.39.mlp.experts.181.down_proj", + "model.layers.39.mlp.experts.181.gate_proj", + "model.layers.39.mlp.experts.181.up_proj", + "model.layers.39.mlp.experts.182.down_proj", + "model.layers.39.mlp.experts.182.gate_proj", + "model.layers.39.mlp.experts.182.up_proj", + "model.layers.39.mlp.experts.183.down_proj", + "model.layers.39.mlp.experts.183.gate_proj", + "model.layers.39.mlp.experts.183.up_proj", + "model.layers.39.mlp.experts.184.down_proj", + "model.layers.39.mlp.experts.184.gate_proj", + "model.layers.39.mlp.experts.184.up_proj", + "model.layers.39.mlp.experts.185.down_proj", + "model.layers.39.mlp.experts.185.gate_proj", + "model.layers.39.mlp.experts.185.up_proj", + "model.layers.39.mlp.experts.186.down_proj", + "model.layers.39.mlp.experts.186.gate_proj", + "model.layers.39.mlp.experts.186.up_proj", + "model.layers.39.mlp.experts.187.down_proj", + "model.layers.39.mlp.experts.187.gate_proj", + "model.layers.39.mlp.experts.187.up_proj", + "model.layers.39.mlp.experts.188.down_proj", + "model.layers.39.mlp.experts.188.gate_proj", + "model.layers.39.mlp.experts.188.up_proj", + "model.layers.39.mlp.experts.189.down_proj", + "model.layers.39.mlp.experts.189.gate_proj", + "model.layers.39.mlp.experts.189.up_proj", + "model.layers.39.mlp.experts.19.down_proj", + "model.layers.39.mlp.experts.19.gate_proj", + "model.layers.39.mlp.experts.19.up_proj", + "model.layers.39.mlp.experts.190.down_proj", + "model.layers.39.mlp.experts.190.gate_proj", + "model.layers.39.mlp.experts.190.up_proj", + "model.layers.39.mlp.experts.191.down_proj", + "model.layers.39.mlp.experts.191.gate_proj", + "model.layers.39.mlp.experts.191.up_proj", + "model.layers.39.mlp.experts.192.down_proj", + "model.layers.39.mlp.experts.192.gate_proj", + "model.layers.39.mlp.experts.192.up_proj", + "model.layers.39.mlp.experts.193.down_proj", + "model.layers.39.mlp.experts.193.gate_proj", + "model.layers.39.mlp.experts.193.up_proj", + "model.layers.39.mlp.experts.194.down_proj", + "model.layers.39.mlp.experts.194.gate_proj", + "model.layers.39.mlp.experts.194.up_proj", + "model.layers.39.mlp.experts.195.down_proj", + "model.layers.39.mlp.experts.195.gate_proj", + "model.layers.39.mlp.experts.195.up_proj", + "model.layers.39.mlp.experts.196.down_proj", + "model.layers.39.mlp.experts.196.gate_proj", + "model.layers.39.mlp.experts.196.up_proj", + "model.layers.39.mlp.experts.197.down_proj", + "model.layers.39.mlp.experts.197.gate_proj", + "model.layers.39.mlp.experts.197.up_proj", + "model.layers.39.mlp.experts.198.down_proj", + "model.layers.39.mlp.experts.198.gate_proj", + "model.layers.39.mlp.experts.198.up_proj", + "model.layers.39.mlp.experts.199.down_proj", + "model.layers.39.mlp.experts.199.gate_proj", + "model.layers.39.mlp.experts.199.up_proj", + "model.layers.39.mlp.experts.2.down_proj", + "model.layers.39.mlp.experts.2.gate_proj", + "model.layers.39.mlp.experts.2.up_proj", + "model.layers.39.mlp.experts.20.down_proj", + "model.layers.39.mlp.experts.20.gate_proj", + "model.layers.39.mlp.experts.20.up_proj", + "model.layers.39.mlp.experts.200.down_proj", + "model.layers.39.mlp.experts.200.gate_proj", + "model.layers.39.mlp.experts.200.up_proj", + "model.layers.39.mlp.experts.201.down_proj", + "model.layers.39.mlp.experts.201.gate_proj", + "model.layers.39.mlp.experts.201.up_proj", + "model.layers.39.mlp.experts.202.down_proj", + "model.layers.39.mlp.experts.202.gate_proj", + "model.layers.39.mlp.experts.202.up_proj", + "model.layers.39.mlp.experts.203.down_proj", + "model.layers.39.mlp.experts.203.gate_proj", + "model.layers.39.mlp.experts.203.up_proj", + "model.layers.39.mlp.experts.204.down_proj", + "model.layers.39.mlp.experts.204.gate_proj", + "model.layers.39.mlp.experts.204.up_proj", + "model.layers.39.mlp.experts.205.down_proj", + "model.layers.39.mlp.experts.205.gate_proj", + "model.layers.39.mlp.experts.205.up_proj", + "model.layers.39.mlp.experts.206.down_proj", + "model.layers.39.mlp.experts.206.gate_proj", + "model.layers.39.mlp.experts.206.up_proj", + "model.layers.39.mlp.experts.207.down_proj", + "model.layers.39.mlp.experts.207.gate_proj", + "model.layers.39.mlp.experts.207.up_proj", + "model.layers.39.mlp.experts.208.down_proj", + "model.layers.39.mlp.experts.208.gate_proj", + "model.layers.39.mlp.experts.208.up_proj", + "model.layers.39.mlp.experts.209.down_proj", + "model.layers.39.mlp.experts.209.gate_proj", + "model.layers.39.mlp.experts.209.up_proj", + "model.layers.39.mlp.experts.21.down_proj", + "model.layers.39.mlp.experts.21.gate_proj", + "model.layers.39.mlp.experts.21.up_proj", + "model.layers.39.mlp.experts.210.down_proj", + "model.layers.39.mlp.experts.210.gate_proj", + "model.layers.39.mlp.experts.210.up_proj", + "model.layers.39.mlp.experts.211.down_proj", + "model.layers.39.mlp.experts.211.gate_proj", + "model.layers.39.mlp.experts.211.up_proj", + "model.layers.39.mlp.experts.212.down_proj", + "model.layers.39.mlp.experts.212.gate_proj", + "model.layers.39.mlp.experts.212.up_proj", + "model.layers.39.mlp.experts.213.down_proj", + "model.layers.39.mlp.experts.213.gate_proj", + "model.layers.39.mlp.experts.213.up_proj", + "model.layers.39.mlp.experts.214.down_proj", + "model.layers.39.mlp.experts.214.gate_proj", + "model.layers.39.mlp.experts.214.up_proj", + "model.layers.39.mlp.experts.215.down_proj", + "model.layers.39.mlp.experts.215.gate_proj", + "model.layers.39.mlp.experts.215.up_proj", + "model.layers.39.mlp.experts.216.down_proj", + "model.layers.39.mlp.experts.216.gate_proj", + "model.layers.39.mlp.experts.216.up_proj", + "model.layers.39.mlp.experts.217.down_proj", + "model.layers.39.mlp.experts.217.gate_proj", + "model.layers.39.mlp.experts.217.up_proj", + "model.layers.39.mlp.experts.218.down_proj", + "model.layers.39.mlp.experts.218.gate_proj", + "model.layers.39.mlp.experts.218.up_proj", + "model.layers.39.mlp.experts.219.down_proj", + "model.layers.39.mlp.experts.219.gate_proj", + "model.layers.39.mlp.experts.219.up_proj", + "model.layers.39.mlp.experts.22.down_proj", + "model.layers.39.mlp.experts.22.gate_proj", + "model.layers.39.mlp.experts.22.up_proj", + "model.layers.39.mlp.experts.220.down_proj", + "model.layers.39.mlp.experts.220.gate_proj", + "model.layers.39.mlp.experts.220.up_proj", + "model.layers.39.mlp.experts.221.down_proj", + "model.layers.39.mlp.experts.221.gate_proj", + "model.layers.39.mlp.experts.221.up_proj", + "model.layers.39.mlp.experts.222.down_proj", + "model.layers.39.mlp.experts.222.gate_proj", + "model.layers.39.mlp.experts.222.up_proj", + "model.layers.39.mlp.experts.223.down_proj", + "model.layers.39.mlp.experts.223.gate_proj", + "model.layers.39.mlp.experts.223.up_proj", + "model.layers.39.mlp.experts.224.down_proj", + "model.layers.39.mlp.experts.224.gate_proj", + "model.layers.39.mlp.experts.224.up_proj", + "model.layers.39.mlp.experts.225.down_proj", + "model.layers.39.mlp.experts.225.gate_proj", + "model.layers.39.mlp.experts.225.up_proj", + "model.layers.39.mlp.experts.226.down_proj", + "model.layers.39.mlp.experts.226.gate_proj", + "model.layers.39.mlp.experts.226.up_proj", + "model.layers.39.mlp.experts.227.down_proj", + "model.layers.39.mlp.experts.227.gate_proj", + "model.layers.39.mlp.experts.227.up_proj", + "model.layers.39.mlp.experts.228.down_proj", + "model.layers.39.mlp.experts.228.gate_proj", + "model.layers.39.mlp.experts.228.up_proj", + "model.layers.39.mlp.experts.229.down_proj", + "model.layers.39.mlp.experts.229.gate_proj", + "model.layers.39.mlp.experts.229.up_proj", + "model.layers.39.mlp.experts.23.down_proj", + "model.layers.39.mlp.experts.23.gate_proj", + "model.layers.39.mlp.experts.23.up_proj", + "model.layers.39.mlp.experts.230.down_proj", + "model.layers.39.mlp.experts.230.gate_proj", + "model.layers.39.mlp.experts.230.up_proj", + "model.layers.39.mlp.experts.231.down_proj", + "model.layers.39.mlp.experts.231.gate_proj", + "model.layers.39.mlp.experts.231.up_proj", + "model.layers.39.mlp.experts.232.down_proj", + "model.layers.39.mlp.experts.232.gate_proj", + "model.layers.39.mlp.experts.232.up_proj", + "model.layers.39.mlp.experts.233.down_proj", + "model.layers.39.mlp.experts.233.gate_proj", + "model.layers.39.mlp.experts.233.up_proj", + "model.layers.39.mlp.experts.234.down_proj", + "model.layers.39.mlp.experts.234.gate_proj", + "model.layers.39.mlp.experts.234.up_proj", + "model.layers.39.mlp.experts.235.down_proj", + "model.layers.39.mlp.experts.235.gate_proj", + "model.layers.39.mlp.experts.235.up_proj", + "model.layers.39.mlp.experts.236.down_proj", + "model.layers.39.mlp.experts.236.gate_proj", + "model.layers.39.mlp.experts.236.up_proj", + "model.layers.39.mlp.experts.237.down_proj", + "model.layers.39.mlp.experts.237.gate_proj", + "model.layers.39.mlp.experts.237.up_proj", + "model.layers.39.mlp.experts.238.down_proj", + "model.layers.39.mlp.experts.238.gate_proj", + "model.layers.39.mlp.experts.238.up_proj", + "model.layers.39.mlp.experts.239.down_proj", + "model.layers.39.mlp.experts.239.gate_proj", + "model.layers.39.mlp.experts.239.up_proj", + "model.layers.39.mlp.experts.24.down_proj", + "model.layers.39.mlp.experts.24.gate_proj", + "model.layers.39.mlp.experts.24.up_proj", + "model.layers.39.mlp.experts.240.down_proj", + "model.layers.39.mlp.experts.240.gate_proj", + "model.layers.39.mlp.experts.240.up_proj", + "model.layers.39.mlp.experts.241.down_proj", + "model.layers.39.mlp.experts.241.gate_proj", + "model.layers.39.mlp.experts.241.up_proj", + "model.layers.39.mlp.experts.242.down_proj", + "model.layers.39.mlp.experts.242.gate_proj", + "model.layers.39.mlp.experts.242.up_proj", + "model.layers.39.mlp.experts.243.down_proj", + "model.layers.39.mlp.experts.243.gate_proj", + "model.layers.39.mlp.experts.243.up_proj", + "model.layers.39.mlp.experts.244.down_proj", + "model.layers.39.mlp.experts.244.gate_proj", + "model.layers.39.mlp.experts.244.up_proj", + "model.layers.39.mlp.experts.245.down_proj", + "model.layers.39.mlp.experts.245.gate_proj", + "model.layers.39.mlp.experts.245.up_proj", + "model.layers.39.mlp.experts.246.down_proj", + "model.layers.39.mlp.experts.246.gate_proj", + "model.layers.39.mlp.experts.246.up_proj", + "model.layers.39.mlp.experts.247.down_proj", + "model.layers.39.mlp.experts.247.gate_proj", + "model.layers.39.mlp.experts.247.up_proj", + "model.layers.39.mlp.experts.248.down_proj", + "model.layers.39.mlp.experts.248.gate_proj", + "model.layers.39.mlp.experts.248.up_proj", + "model.layers.39.mlp.experts.249.down_proj", + "model.layers.39.mlp.experts.249.gate_proj", + "model.layers.39.mlp.experts.249.up_proj", + "model.layers.39.mlp.experts.25.down_proj", + "model.layers.39.mlp.experts.25.gate_proj", + "model.layers.39.mlp.experts.25.up_proj", + "model.layers.39.mlp.experts.250.down_proj", + "model.layers.39.mlp.experts.250.gate_proj", + "model.layers.39.mlp.experts.250.up_proj", + "model.layers.39.mlp.experts.251.down_proj", + "model.layers.39.mlp.experts.251.gate_proj", + "model.layers.39.mlp.experts.251.up_proj", + "model.layers.39.mlp.experts.252.down_proj", + "model.layers.39.mlp.experts.252.gate_proj", + "model.layers.39.mlp.experts.252.up_proj", + "model.layers.39.mlp.experts.253.down_proj", + "model.layers.39.mlp.experts.253.gate_proj", + "model.layers.39.mlp.experts.253.up_proj", + "model.layers.39.mlp.experts.254.down_proj", + "model.layers.39.mlp.experts.254.gate_proj", + "model.layers.39.mlp.experts.254.up_proj", + "model.layers.39.mlp.experts.255.down_proj", + "model.layers.39.mlp.experts.255.gate_proj", + "model.layers.39.mlp.experts.255.up_proj", + "model.layers.39.mlp.experts.26.down_proj", + "model.layers.39.mlp.experts.26.gate_proj", + "model.layers.39.mlp.experts.26.up_proj", + "model.layers.39.mlp.experts.27.down_proj", + "model.layers.39.mlp.experts.27.gate_proj", + "model.layers.39.mlp.experts.27.up_proj", + "model.layers.39.mlp.experts.28.down_proj", + "model.layers.39.mlp.experts.28.gate_proj", + "model.layers.39.mlp.experts.28.up_proj", + "model.layers.39.mlp.experts.29.down_proj", + "model.layers.39.mlp.experts.29.gate_proj", + "model.layers.39.mlp.experts.29.up_proj", + "model.layers.39.mlp.experts.3.down_proj", + "model.layers.39.mlp.experts.3.gate_proj", + "model.layers.39.mlp.experts.3.up_proj", + "model.layers.39.mlp.experts.30.down_proj", + "model.layers.39.mlp.experts.30.gate_proj", + "model.layers.39.mlp.experts.30.up_proj", + "model.layers.39.mlp.experts.31.down_proj", + "model.layers.39.mlp.experts.31.gate_proj", + "model.layers.39.mlp.experts.31.up_proj", + "model.layers.39.mlp.experts.32.down_proj", + "model.layers.39.mlp.experts.32.gate_proj", + "model.layers.39.mlp.experts.32.up_proj", + "model.layers.39.mlp.experts.33.down_proj", + "model.layers.39.mlp.experts.33.gate_proj", + "model.layers.39.mlp.experts.33.up_proj", + "model.layers.39.mlp.experts.34.down_proj", + "model.layers.39.mlp.experts.34.gate_proj", + "model.layers.39.mlp.experts.34.up_proj", + "model.layers.39.mlp.experts.35.down_proj", + "model.layers.39.mlp.experts.35.gate_proj", + "model.layers.39.mlp.experts.35.up_proj", + "model.layers.39.mlp.experts.36.down_proj", + "model.layers.39.mlp.experts.36.gate_proj", + "model.layers.39.mlp.experts.36.up_proj", + "model.layers.39.mlp.experts.37.down_proj", + "model.layers.39.mlp.experts.37.gate_proj", + "model.layers.39.mlp.experts.37.up_proj", + "model.layers.39.mlp.experts.38.down_proj", + "model.layers.39.mlp.experts.38.gate_proj", + "model.layers.39.mlp.experts.38.up_proj", + "model.layers.39.mlp.experts.39.down_proj", + "model.layers.39.mlp.experts.39.gate_proj", + "model.layers.39.mlp.experts.39.up_proj", + "model.layers.39.mlp.experts.4.down_proj", + "model.layers.39.mlp.experts.4.gate_proj", + "model.layers.39.mlp.experts.4.up_proj", + "model.layers.39.mlp.experts.40.down_proj", + "model.layers.39.mlp.experts.40.gate_proj", + "model.layers.39.mlp.experts.40.up_proj", + "model.layers.39.mlp.experts.41.down_proj", + "model.layers.39.mlp.experts.41.gate_proj", + "model.layers.39.mlp.experts.41.up_proj", + "model.layers.39.mlp.experts.42.down_proj", + "model.layers.39.mlp.experts.42.gate_proj", + "model.layers.39.mlp.experts.42.up_proj", + "model.layers.39.mlp.experts.43.down_proj", + "model.layers.39.mlp.experts.43.gate_proj", + "model.layers.39.mlp.experts.43.up_proj", + "model.layers.39.mlp.experts.44.down_proj", + "model.layers.39.mlp.experts.44.gate_proj", + "model.layers.39.mlp.experts.44.up_proj", + "model.layers.39.mlp.experts.45.down_proj", + "model.layers.39.mlp.experts.45.gate_proj", + "model.layers.39.mlp.experts.45.up_proj", + "model.layers.39.mlp.experts.46.down_proj", + "model.layers.39.mlp.experts.46.gate_proj", + "model.layers.39.mlp.experts.46.up_proj", + "model.layers.39.mlp.experts.47.down_proj", + "model.layers.39.mlp.experts.47.gate_proj", + "model.layers.39.mlp.experts.47.up_proj", + "model.layers.39.mlp.experts.48.down_proj", + "model.layers.39.mlp.experts.48.gate_proj", + "model.layers.39.mlp.experts.48.up_proj", + "model.layers.39.mlp.experts.49.down_proj", + "model.layers.39.mlp.experts.49.gate_proj", + "model.layers.39.mlp.experts.49.up_proj", + "model.layers.39.mlp.experts.5.down_proj", + "model.layers.39.mlp.experts.5.gate_proj", + "model.layers.39.mlp.experts.5.up_proj", + "model.layers.39.mlp.experts.50.down_proj", + "model.layers.39.mlp.experts.50.gate_proj", + "model.layers.39.mlp.experts.50.up_proj", + "model.layers.39.mlp.experts.51.down_proj", + "model.layers.39.mlp.experts.51.gate_proj", + "model.layers.39.mlp.experts.51.up_proj", + "model.layers.39.mlp.experts.52.down_proj", + "model.layers.39.mlp.experts.52.gate_proj", + "model.layers.39.mlp.experts.52.up_proj", + "model.layers.39.mlp.experts.53.down_proj", + "model.layers.39.mlp.experts.53.gate_proj", + "model.layers.39.mlp.experts.53.up_proj", + "model.layers.39.mlp.experts.54.down_proj", + "model.layers.39.mlp.experts.54.gate_proj", + "model.layers.39.mlp.experts.54.up_proj", + "model.layers.39.mlp.experts.55.down_proj", + "model.layers.39.mlp.experts.55.gate_proj", + "model.layers.39.mlp.experts.55.up_proj", + "model.layers.39.mlp.experts.56.down_proj", + "model.layers.39.mlp.experts.56.gate_proj", + "model.layers.39.mlp.experts.56.up_proj", + "model.layers.39.mlp.experts.57.down_proj", + "model.layers.39.mlp.experts.57.gate_proj", + "model.layers.39.mlp.experts.57.up_proj", + "model.layers.39.mlp.experts.58.down_proj", + "model.layers.39.mlp.experts.58.gate_proj", + "model.layers.39.mlp.experts.58.up_proj", + "model.layers.39.mlp.experts.59.down_proj", + "model.layers.39.mlp.experts.59.gate_proj", + "model.layers.39.mlp.experts.59.up_proj", + "model.layers.39.mlp.experts.6.down_proj", + "model.layers.39.mlp.experts.6.gate_proj", + "model.layers.39.mlp.experts.6.up_proj", + "model.layers.39.mlp.experts.60.down_proj", + "model.layers.39.mlp.experts.60.gate_proj", + "model.layers.39.mlp.experts.60.up_proj", + "model.layers.39.mlp.experts.61.down_proj", + "model.layers.39.mlp.experts.61.gate_proj", + "model.layers.39.mlp.experts.61.up_proj", + "model.layers.39.mlp.experts.62.down_proj", + "model.layers.39.mlp.experts.62.gate_proj", + "model.layers.39.mlp.experts.62.up_proj", + "model.layers.39.mlp.experts.63.down_proj", + "model.layers.39.mlp.experts.63.gate_proj", + "model.layers.39.mlp.experts.63.up_proj", + "model.layers.39.mlp.experts.64.down_proj", + "model.layers.39.mlp.experts.64.gate_proj", + "model.layers.39.mlp.experts.64.up_proj", + "model.layers.39.mlp.experts.65.down_proj", + "model.layers.39.mlp.experts.65.gate_proj", + "model.layers.39.mlp.experts.65.up_proj", + "model.layers.39.mlp.experts.66.down_proj", + "model.layers.39.mlp.experts.66.gate_proj", + "model.layers.39.mlp.experts.66.up_proj", + "model.layers.39.mlp.experts.67.down_proj", + "model.layers.39.mlp.experts.67.gate_proj", + "model.layers.39.mlp.experts.67.up_proj", + "model.layers.39.mlp.experts.68.down_proj", + "model.layers.39.mlp.experts.68.gate_proj", + "model.layers.39.mlp.experts.68.up_proj", + "model.layers.39.mlp.experts.69.down_proj", + "model.layers.39.mlp.experts.69.gate_proj", + "model.layers.39.mlp.experts.69.up_proj", + "model.layers.39.mlp.experts.7.down_proj", + "model.layers.39.mlp.experts.7.gate_proj", + "model.layers.39.mlp.experts.7.up_proj", + "model.layers.39.mlp.experts.70.down_proj", + "model.layers.39.mlp.experts.70.gate_proj", + "model.layers.39.mlp.experts.70.up_proj", + "model.layers.39.mlp.experts.71.down_proj", + "model.layers.39.mlp.experts.71.gate_proj", + "model.layers.39.mlp.experts.71.up_proj", + "model.layers.39.mlp.experts.72.down_proj", + "model.layers.39.mlp.experts.72.gate_proj", + "model.layers.39.mlp.experts.72.up_proj", + "model.layers.39.mlp.experts.73.down_proj", + "model.layers.39.mlp.experts.73.gate_proj", + "model.layers.39.mlp.experts.73.up_proj", + "model.layers.39.mlp.experts.74.down_proj", + "model.layers.39.mlp.experts.74.gate_proj", + "model.layers.39.mlp.experts.74.up_proj", + "model.layers.39.mlp.experts.75.down_proj", + "model.layers.39.mlp.experts.75.gate_proj", + "model.layers.39.mlp.experts.75.up_proj", + "model.layers.39.mlp.experts.76.down_proj", + "model.layers.39.mlp.experts.76.gate_proj", + "model.layers.39.mlp.experts.76.up_proj", + "model.layers.39.mlp.experts.77.down_proj", + "model.layers.39.mlp.experts.77.gate_proj", + "model.layers.39.mlp.experts.77.up_proj", + "model.layers.39.mlp.experts.78.down_proj", + "model.layers.39.mlp.experts.78.gate_proj", + "model.layers.39.mlp.experts.78.up_proj", + "model.layers.39.mlp.experts.79.down_proj", + "model.layers.39.mlp.experts.79.gate_proj", + "model.layers.39.mlp.experts.79.up_proj", + "model.layers.39.mlp.experts.8.down_proj", + "model.layers.39.mlp.experts.8.gate_proj", + "model.layers.39.mlp.experts.8.up_proj", + "model.layers.39.mlp.experts.80.down_proj", + "model.layers.39.mlp.experts.80.gate_proj", + "model.layers.39.mlp.experts.80.up_proj", + "model.layers.39.mlp.experts.81.down_proj", + "model.layers.39.mlp.experts.81.gate_proj", + "model.layers.39.mlp.experts.81.up_proj", + "model.layers.39.mlp.experts.82.down_proj", + "model.layers.39.mlp.experts.82.gate_proj", + "model.layers.39.mlp.experts.82.up_proj", + "model.layers.39.mlp.experts.83.down_proj", + "model.layers.39.mlp.experts.83.gate_proj", + "model.layers.39.mlp.experts.83.up_proj", + "model.layers.39.mlp.experts.84.down_proj", + "model.layers.39.mlp.experts.84.gate_proj", + "model.layers.39.mlp.experts.84.up_proj", + "model.layers.39.mlp.experts.85.down_proj", + "model.layers.39.mlp.experts.85.gate_proj", + "model.layers.39.mlp.experts.85.up_proj", + "model.layers.39.mlp.experts.86.down_proj", + "model.layers.39.mlp.experts.86.gate_proj", + "model.layers.39.mlp.experts.86.up_proj", + "model.layers.39.mlp.experts.87.down_proj", + "model.layers.39.mlp.experts.87.gate_proj", + "model.layers.39.mlp.experts.87.up_proj", + "model.layers.39.mlp.experts.88.down_proj", + "model.layers.39.mlp.experts.88.gate_proj", + "model.layers.39.mlp.experts.88.up_proj", + "model.layers.39.mlp.experts.89.down_proj", + "model.layers.39.mlp.experts.89.gate_proj", + "model.layers.39.mlp.experts.89.up_proj", + "model.layers.39.mlp.experts.9.down_proj", + "model.layers.39.mlp.experts.9.gate_proj", + "model.layers.39.mlp.experts.9.up_proj", + "model.layers.39.mlp.experts.90.down_proj", + "model.layers.39.mlp.experts.90.gate_proj", + "model.layers.39.mlp.experts.90.up_proj", + "model.layers.39.mlp.experts.91.down_proj", + "model.layers.39.mlp.experts.91.gate_proj", + "model.layers.39.mlp.experts.91.up_proj", + "model.layers.39.mlp.experts.92.down_proj", + "model.layers.39.mlp.experts.92.gate_proj", + "model.layers.39.mlp.experts.92.up_proj", + "model.layers.39.mlp.experts.93.down_proj", + "model.layers.39.mlp.experts.93.gate_proj", + "model.layers.39.mlp.experts.93.up_proj", + "model.layers.39.mlp.experts.94.down_proj", + "model.layers.39.mlp.experts.94.gate_proj", + "model.layers.39.mlp.experts.94.up_proj", + "model.layers.39.mlp.experts.95.down_proj", + "model.layers.39.mlp.experts.95.gate_proj", + "model.layers.39.mlp.experts.95.up_proj", + "model.layers.39.mlp.experts.96.down_proj", + "model.layers.39.mlp.experts.96.gate_proj", + "model.layers.39.mlp.experts.96.up_proj", + "model.layers.39.mlp.experts.97.down_proj", + "model.layers.39.mlp.experts.97.gate_proj", + "model.layers.39.mlp.experts.97.up_proj", + "model.layers.39.mlp.experts.98.down_proj", + "model.layers.39.mlp.experts.98.gate_proj", + "model.layers.39.mlp.experts.98.up_proj", + "model.layers.39.mlp.experts.99.down_proj", + "model.layers.39.mlp.experts.99.gate_proj", + "model.layers.39.mlp.experts.99.up_proj", + "model.layers.4.mlp.experts.0.down_proj", + "model.layers.4.mlp.experts.0.gate_proj", + "model.layers.4.mlp.experts.0.up_proj", + "model.layers.4.mlp.experts.1.down_proj", + "model.layers.4.mlp.experts.1.gate_proj", + "model.layers.4.mlp.experts.1.up_proj", + "model.layers.4.mlp.experts.10.down_proj", + "model.layers.4.mlp.experts.10.gate_proj", + "model.layers.4.mlp.experts.10.up_proj", + "model.layers.4.mlp.experts.100.down_proj", + "model.layers.4.mlp.experts.100.gate_proj", + "model.layers.4.mlp.experts.100.up_proj", + "model.layers.4.mlp.experts.101.down_proj", + "model.layers.4.mlp.experts.101.gate_proj", + "model.layers.4.mlp.experts.101.up_proj", + "model.layers.4.mlp.experts.102.down_proj", + "model.layers.4.mlp.experts.102.gate_proj", + "model.layers.4.mlp.experts.102.up_proj", + "model.layers.4.mlp.experts.103.down_proj", + "model.layers.4.mlp.experts.103.gate_proj", + "model.layers.4.mlp.experts.103.up_proj", + "model.layers.4.mlp.experts.104.down_proj", + "model.layers.4.mlp.experts.104.gate_proj", + "model.layers.4.mlp.experts.104.up_proj", + "model.layers.4.mlp.experts.105.down_proj", + "model.layers.4.mlp.experts.105.gate_proj", + "model.layers.4.mlp.experts.105.up_proj", + "model.layers.4.mlp.experts.106.down_proj", + "model.layers.4.mlp.experts.106.gate_proj", + "model.layers.4.mlp.experts.106.up_proj", + "model.layers.4.mlp.experts.107.down_proj", + "model.layers.4.mlp.experts.107.gate_proj", + "model.layers.4.mlp.experts.107.up_proj", + "model.layers.4.mlp.experts.108.down_proj", + "model.layers.4.mlp.experts.108.gate_proj", + "model.layers.4.mlp.experts.108.up_proj", + "model.layers.4.mlp.experts.109.down_proj", + "model.layers.4.mlp.experts.109.gate_proj", + "model.layers.4.mlp.experts.109.up_proj", + "model.layers.4.mlp.experts.11.down_proj", + "model.layers.4.mlp.experts.11.gate_proj", + "model.layers.4.mlp.experts.11.up_proj", + "model.layers.4.mlp.experts.110.down_proj", + "model.layers.4.mlp.experts.110.gate_proj", + "model.layers.4.mlp.experts.110.up_proj", + "model.layers.4.mlp.experts.111.down_proj", + "model.layers.4.mlp.experts.111.gate_proj", + "model.layers.4.mlp.experts.111.up_proj", + "model.layers.4.mlp.experts.112.down_proj", + "model.layers.4.mlp.experts.112.gate_proj", + "model.layers.4.mlp.experts.112.up_proj", + "model.layers.4.mlp.experts.113.down_proj", + "model.layers.4.mlp.experts.113.gate_proj", + "model.layers.4.mlp.experts.113.up_proj", + "model.layers.4.mlp.experts.114.down_proj", + "model.layers.4.mlp.experts.114.gate_proj", + "model.layers.4.mlp.experts.114.up_proj", + "model.layers.4.mlp.experts.115.down_proj", + "model.layers.4.mlp.experts.115.gate_proj", + "model.layers.4.mlp.experts.115.up_proj", + "model.layers.4.mlp.experts.116.down_proj", + "model.layers.4.mlp.experts.116.gate_proj", + "model.layers.4.mlp.experts.116.up_proj", + "model.layers.4.mlp.experts.117.down_proj", + "model.layers.4.mlp.experts.117.gate_proj", + "model.layers.4.mlp.experts.117.up_proj", + "model.layers.4.mlp.experts.118.down_proj", + "model.layers.4.mlp.experts.118.gate_proj", + "model.layers.4.mlp.experts.118.up_proj", + "model.layers.4.mlp.experts.119.down_proj", + "model.layers.4.mlp.experts.119.gate_proj", + "model.layers.4.mlp.experts.119.up_proj", + "model.layers.4.mlp.experts.12.down_proj", + "model.layers.4.mlp.experts.12.gate_proj", + "model.layers.4.mlp.experts.12.up_proj", + "model.layers.4.mlp.experts.120.down_proj", + "model.layers.4.mlp.experts.120.gate_proj", + "model.layers.4.mlp.experts.120.up_proj", + "model.layers.4.mlp.experts.121.down_proj", + "model.layers.4.mlp.experts.121.gate_proj", + "model.layers.4.mlp.experts.121.up_proj", + "model.layers.4.mlp.experts.122.down_proj", + "model.layers.4.mlp.experts.122.gate_proj", + "model.layers.4.mlp.experts.122.up_proj", + "model.layers.4.mlp.experts.123.down_proj", + "model.layers.4.mlp.experts.123.gate_proj", + "model.layers.4.mlp.experts.123.up_proj", + "model.layers.4.mlp.experts.124.down_proj", + "model.layers.4.mlp.experts.124.gate_proj", + "model.layers.4.mlp.experts.124.up_proj", + "model.layers.4.mlp.experts.125.down_proj", + "model.layers.4.mlp.experts.125.gate_proj", + "model.layers.4.mlp.experts.125.up_proj", + "model.layers.4.mlp.experts.126.down_proj", + "model.layers.4.mlp.experts.126.gate_proj", + "model.layers.4.mlp.experts.126.up_proj", + "model.layers.4.mlp.experts.127.down_proj", + "model.layers.4.mlp.experts.127.gate_proj", + "model.layers.4.mlp.experts.127.up_proj", + "model.layers.4.mlp.experts.128.down_proj", + "model.layers.4.mlp.experts.128.gate_proj", + "model.layers.4.mlp.experts.128.up_proj", + "model.layers.4.mlp.experts.129.down_proj", + "model.layers.4.mlp.experts.129.gate_proj", + "model.layers.4.mlp.experts.129.up_proj", + "model.layers.4.mlp.experts.13.down_proj", + "model.layers.4.mlp.experts.13.gate_proj", + "model.layers.4.mlp.experts.13.up_proj", + "model.layers.4.mlp.experts.130.down_proj", + "model.layers.4.mlp.experts.130.gate_proj", + "model.layers.4.mlp.experts.130.up_proj", + "model.layers.4.mlp.experts.131.down_proj", + "model.layers.4.mlp.experts.131.gate_proj", + "model.layers.4.mlp.experts.131.up_proj", + "model.layers.4.mlp.experts.132.down_proj", + "model.layers.4.mlp.experts.132.gate_proj", + "model.layers.4.mlp.experts.132.up_proj", + "model.layers.4.mlp.experts.133.down_proj", + "model.layers.4.mlp.experts.133.gate_proj", + "model.layers.4.mlp.experts.133.up_proj", + "model.layers.4.mlp.experts.134.down_proj", + "model.layers.4.mlp.experts.134.gate_proj", + "model.layers.4.mlp.experts.134.up_proj", + "model.layers.4.mlp.experts.135.down_proj", + "model.layers.4.mlp.experts.135.gate_proj", + "model.layers.4.mlp.experts.135.up_proj", + "model.layers.4.mlp.experts.136.down_proj", + "model.layers.4.mlp.experts.136.gate_proj", + "model.layers.4.mlp.experts.136.up_proj", + "model.layers.4.mlp.experts.137.down_proj", + "model.layers.4.mlp.experts.137.gate_proj", + "model.layers.4.mlp.experts.137.up_proj", + "model.layers.4.mlp.experts.138.down_proj", + "model.layers.4.mlp.experts.138.gate_proj", + "model.layers.4.mlp.experts.138.up_proj", + "model.layers.4.mlp.experts.139.down_proj", + "model.layers.4.mlp.experts.139.gate_proj", + "model.layers.4.mlp.experts.139.up_proj", + "model.layers.4.mlp.experts.14.down_proj", + "model.layers.4.mlp.experts.14.gate_proj", + "model.layers.4.mlp.experts.14.up_proj", + "model.layers.4.mlp.experts.140.down_proj", + "model.layers.4.mlp.experts.140.gate_proj", + "model.layers.4.mlp.experts.140.up_proj", + "model.layers.4.mlp.experts.141.down_proj", + "model.layers.4.mlp.experts.141.gate_proj", + "model.layers.4.mlp.experts.141.up_proj", + "model.layers.4.mlp.experts.142.down_proj", + "model.layers.4.mlp.experts.142.gate_proj", + "model.layers.4.mlp.experts.142.up_proj", + "model.layers.4.mlp.experts.143.down_proj", + "model.layers.4.mlp.experts.143.gate_proj", + "model.layers.4.mlp.experts.143.up_proj", + "model.layers.4.mlp.experts.144.down_proj", + "model.layers.4.mlp.experts.144.gate_proj", + "model.layers.4.mlp.experts.144.up_proj", + "model.layers.4.mlp.experts.145.down_proj", + "model.layers.4.mlp.experts.145.gate_proj", + "model.layers.4.mlp.experts.145.up_proj", + "model.layers.4.mlp.experts.146.down_proj", + "model.layers.4.mlp.experts.146.gate_proj", + "model.layers.4.mlp.experts.146.up_proj", + "model.layers.4.mlp.experts.147.down_proj", + "model.layers.4.mlp.experts.147.gate_proj", + "model.layers.4.mlp.experts.147.up_proj", + "model.layers.4.mlp.experts.148.down_proj", + "model.layers.4.mlp.experts.148.gate_proj", + "model.layers.4.mlp.experts.148.up_proj", + "model.layers.4.mlp.experts.149.down_proj", + "model.layers.4.mlp.experts.149.gate_proj", + "model.layers.4.mlp.experts.149.up_proj", + "model.layers.4.mlp.experts.15.down_proj", + "model.layers.4.mlp.experts.15.gate_proj", + "model.layers.4.mlp.experts.15.up_proj", + "model.layers.4.mlp.experts.150.down_proj", + "model.layers.4.mlp.experts.150.gate_proj", + "model.layers.4.mlp.experts.150.up_proj", + "model.layers.4.mlp.experts.151.down_proj", + "model.layers.4.mlp.experts.151.gate_proj", + "model.layers.4.mlp.experts.151.up_proj", + "model.layers.4.mlp.experts.152.down_proj", + "model.layers.4.mlp.experts.152.gate_proj", + "model.layers.4.mlp.experts.152.up_proj", + "model.layers.4.mlp.experts.153.down_proj", + "model.layers.4.mlp.experts.153.gate_proj", + "model.layers.4.mlp.experts.153.up_proj", + "model.layers.4.mlp.experts.154.down_proj", + "model.layers.4.mlp.experts.154.gate_proj", + "model.layers.4.mlp.experts.154.up_proj", + "model.layers.4.mlp.experts.155.down_proj", + "model.layers.4.mlp.experts.155.gate_proj", + "model.layers.4.mlp.experts.155.up_proj", + "model.layers.4.mlp.experts.156.down_proj", + "model.layers.4.mlp.experts.156.gate_proj", + "model.layers.4.mlp.experts.156.up_proj", + "model.layers.4.mlp.experts.157.down_proj", + "model.layers.4.mlp.experts.157.gate_proj", + "model.layers.4.mlp.experts.157.up_proj", + "model.layers.4.mlp.experts.158.down_proj", + "model.layers.4.mlp.experts.158.gate_proj", + "model.layers.4.mlp.experts.158.up_proj", + "model.layers.4.mlp.experts.159.down_proj", + "model.layers.4.mlp.experts.159.gate_proj", + "model.layers.4.mlp.experts.159.up_proj", + "model.layers.4.mlp.experts.16.down_proj", + "model.layers.4.mlp.experts.16.gate_proj", + "model.layers.4.mlp.experts.16.up_proj", + "model.layers.4.mlp.experts.160.down_proj", + "model.layers.4.mlp.experts.160.gate_proj", + "model.layers.4.mlp.experts.160.up_proj", + "model.layers.4.mlp.experts.161.down_proj", + "model.layers.4.mlp.experts.161.gate_proj", + "model.layers.4.mlp.experts.161.up_proj", + "model.layers.4.mlp.experts.162.down_proj", + "model.layers.4.mlp.experts.162.gate_proj", + "model.layers.4.mlp.experts.162.up_proj", + "model.layers.4.mlp.experts.163.down_proj", + "model.layers.4.mlp.experts.163.gate_proj", + "model.layers.4.mlp.experts.163.up_proj", + "model.layers.4.mlp.experts.164.down_proj", + "model.layers.4.mlp.experts.164.gate_proj", + "model.layers.4.mlp.experts.164.up_proj", + "model.layers.4.mlp.experts.165.down_proj", + "model.layers.4.mlp.experts.165.gate_proj", + "model.layers.4.mlp.experts.165.up_proj", + "model.layers.4.mlp.experts.166.down_proj", + "model.layers.4.mlp.experts.166.gate_proj", + "model.layers.4.mlp.experts.166.up_proj", + "model.layers.4.mlp.experts.167.down_proj", + "model.layers.4.mlp.experts.167.gate_proj", + "model.layers.4.mlp.experts.167.up_proj", + "model.layers.4.mlp.experts.168.down_proj", + "model.layers.4.mlp.experts.168.gate_proj", + "model.layers.4.mlp.experts.168.up_proj", + "model.layers.4.mlp.experts.169.down_proj", + "model.layers.4.mlp.experts.169.gate_proj", + "model.layers.4.mlp.experts.169.up_proj", + "model.layers.4.mlp.experts.17.down_proj", + "model.layers.4.mlp.experts.17.gate_proj", + "model.layers.4.mlp.experts.17.up_proj", + "model.layers.4.mlp.experts.170.down_proj", + "model.layers.4.mlp.experts.170.gate_proj", + "model.layers.4.mlp.experts.170.up_proj", + "model.layers.4.mlp.experts.171.down_proj", + "model.layers.4.mlp.experts.171.gate_proj", + "model.layers.4.mlp.experts.171.up_proj", + "model.layers.4.mlp.experts.172.down_proj", + "model.layers.4.mlp.experts.172.gate_proj", + "model.layers.4.mlp.experts.172.up_proj", + "model.layers.4.mlp.experts.173.down_proj", + "model.layers.4.mlp.experts.173.gate_proj", + "model.layers.4.mlp.experts.173.up_proj", + "model.layers.4.mlp.experts.174.down_proj", + "model.layers.4.mlp.experts.174.gate_proj", + "model.layers.4.mlp.experts.174.up_proj", + "model.layers.4.mlp.experts.175.down_proj", + "model.layers.4.mlp.experts.175.gate_proj", + "model.layers.4.mlp.experts.175.up_proj", + "model.layers.4.mlp.experts.176.down_proj", + "model.layers.4.mlp.experts.176.gate_proj", + "model.layers.4.mlp.experts.176.up_proj", + "model.layers.4.mlp.experts.177.down_proj", + "model.layers.4.mlp.experts.177.gate_proj", + "model.layers.4.mlp.experts.177.up_proj", + "model.layers.4.mlp.experts.178.down_proj", + "model.layers.4.mlp.experts.178.gate_proj", + "model.layers.4.mlp.experts.178.up_proj", + "model.layers.4.mlp.experts.179.down_proj", + "model.layers.4.mlp.experts.179.gate_proj", + "model.layers.4.mlp.experts.179.up_proj", + "model.layers.4.mlp.experts.18.down_proj", + "model.layers.4.mlp.experts.18.gate_proj", + "model.layers.4.mlp.experts.18.up_proj", + "model.layers.4.mlp.experts.180.down_proj", + "model.layers.4.mlp.experts.180.gate_proj", + "model.layers.4.mlp.experts.180.up_proj", + "model.layers.4.mlp.experts.181.down_proj", + "model.layers.4.mlp.experts.181.gate_proj", + "model.layers.4.mlp.experts.181.up_proj", + "model.layers.4.mlp.experts.182.down_proj", + "model.layers.4.mlp.experts.182.gate_proj", + "model.layers.4.mlp.experts.182.up_proj", + "model.layers.4.mlp.experts.183.down_proj", + "model.layers.4.mlp.experts.183.gate_proj", + "model.layers.4.mlp.experts.183.up_proj", + "model.layers.4.mlp.experts.184.down_proj", + "model.layers.4.mlp.experts.184.gate_proj", + "model.layers.4.mlp.experts.184.up_proj", + "model.layers.4.mlp.experts.185.down_proj", + "model.layers.4.mlp.experts.185.gate_proj", + "model.layers.4.mlp.experts.185.up_proj", + "model.layers.4.mlp.experts.186.down_proj", + "model.layers.4.mlp.experts.186.gate_proj", + "model.layers.4.mlp.experts.186.up_proj", + "model.layers.4.mlp.experts.187.down_proj", + "model.layers.4.mlp.experts.187.gate_proj", + "model.layers.4.mlp.experts.187.up_proj", + "model.layers.4.mlp.experts.188.down_proj", + "model.layers.4.mlp.experts.188.gate_proj", + "model.layers.4.mlp.experts.188.up_proj", + "model.layers.4.mlp.experts.189.down_proj", + "model.layers.4.mlp.experts.189.gate_proj", + "model.layers.4.mlp.experts.189.up_proj", + "model.layers.4.mlp.experts.19.down_proj", + "model.layers.4.mlp.experts.19.gate_proj", + "model.layers.4.mlp.experts.19.up_proj", + "model.layers.4.mlp.experts.190.down_proj", + "model.layers.4.mlp.experts.190.gate_proj", + "model.layers.4.mlp.experts.190.up_proj", + "model.layers.4.mlp.experts.191.down_proj", + "model.layers.4.mlp.experts.191.gate_proj", + "model.layers.4.mlp.experts.191.up_proj", + "model.layers.4.mlp.experts.192.down_proj", + "model.layers.4.mlp.experts.192.gate_proj", + "model.layers.4.mlp.experts.192.up_proj", + "model.layers.4.mlp.experts.193.down_proj", + "model.layers.4.mlp.experts.193.gate_proj", + "model.layers.4.mlp.experts.193.up_proj", + "model.layers.4.mlp.experts.194.down_proj", + "model.layers.4.mlp.experts.194.gate_proj", + "model.layers.4.mlp.experts.194.up_proj", + "model.layers.4.mlp.experts.195.down_proj", + "model.layers.4.mlp.experts.195.gate_proj", + "model.layers.4.mlp.experts.195.up_proj", + "model.layers.4.mlp.experts.196.down_proj", + "model.layers.4.mlp.experts.196.gate_proj", + "model.layers.4.mlp.experts.196.up_proj", + "model.layers.4.mlp.experts.197.down_proj", + "model.layers.4.mlp.experts.197.gate_proj", + "model.layers.4.mlp.experts.197.up_proj", + "model.layers.4.mlp.experts.198.down_proj", + "model.layers.4.mlp.experts.198.gate_proj", + "model.layers.4.mlp.experts.198.up_proj", + "model.layers.4.mlp.experts.199.down_proj", + "model.layers.4.mlp.experts.199.gate_proj", + "model.layers.4.mlp.experts.199.up_proj", + "model.layers.4.mlp.experts.2.down_proj", + "model.layers.4.mlp.experts.2.gate_proj", + "model.layers.4.mlp.experts.2.up_proj", + "model.layers.4.mlp.experts.20.down_proj", + "model.layers.4.mlp.experts.20.gate_proj", + "model.layers.4.mlp.experts.20.up_proj", + "model.layers.4.mlp.experts.200.down_proj", + "model.layers.4.mlp.experts.200.gate_proj", + "model.layers.4.mlp.experts.200.up_proj", + "model.layers.4.mlp.experts.201.down_proj", + "model.layers.4.mlp.experts.201.gate_proj", + "model.layers.4.mlp.experts.201.up_proj", + "model.layers.4.mlp.experts.202.down_proj", + "model.layers.4.mlp.experts.202.gate_proj", + "model.layers.4.mlp.experts.202.up_proj", + "model.layers.4.mlp.experts.203.down_proj", + "model.layers.4.mlp.experts.203.gate_proj", + "model.layers.4.mlp.experts.203.up_proj", + "model.layers.4.mlp.experts.204.down_proj", + "model.layers.4.mlp.experts.204.gate_proj", + "model.layers.4.mlp.experts.204.up_proj", + "model.layers.4.mlp.experts.205.down_proj", + "model.layers.4.mlp.experts.205.gate_proj", + "model.layers.4.mlp.experts.205.up_proj", + "model.layers.4.mlp.experts.206.down_proj", + "model.layers.4.mlp.experts.206.gate_proj", + "model.layers.4.mlp.experts.206.up_proj", + "model.layers.4.mlp.experts.207.down_proj", + "model.layers.4.mlp.experts.207.gate_proj", + "model.layers.4.mlp.experts.207.up_proj", + "model.layers.4.mlp.experts.208.down_proj", + "model.layers.4.mlp.experts.208.gate_proj", + "model.layers.4.mlp.experts.208.up_proj", + "model.layers.4.mlp.experts.209.down_proj", + "model.layers.4.mlp.experts.209.gate_proj", + "model.layers.4.mlp.experts.209.up_proj", + "model.layers.4.mlp.experts.21.down_proj", + "model.layers.4.mlp.experts.21.gate_proj", + "model.layers.4.mlp.experts.21.up_proj", + "model.layers.4.mlp.experts.210.down_proj", + "model.layers.4.mlp.experts.210.gate_proj", + "model.layers.4.mlp.experts.210.up_proj", + "model.layers.4.mlp.experts.211.down_proj", + "model.layers.4.mlp.experts.211.gate_proj", + "model.layers.4.mlp.experts.211.up_proj", + "model.layers.4.mlp.experts.212.down_proj", + "model.layers.4.mlp.experts.212.gate_proj", + "model.layers.4.mlp.experts.212.up_proj", + "model.layers.4.mlp.experts.213.down_proj", + "model.layers.4.mlp.experts.213.gate_proj", + "model.layers.4.mlp.experts.213.up_proj", + "model.layers.4.mlp.experts.214.down_proj", + "model.layers.4.mlp.experts.214.gate_proj", + "model.layers.4.mlp.experts.214.up_proj", + "model.layers.4.mlp.experts.215.down_proj", + "model.layers.4.mlp.experts.215.gate_proj", + "model.layers.4.mlp.experts.215.up_proj", + "model.layers.4.mlp.experts.216.down_proj", + "model.layers.4.mlp.experts.216.gate_proj", + "model.layers.4.mlp.experts.216.up_proj", + "model.layers.4.mlp.experts.217.down_proj", + "model.layers.4.mlp.experts.217.gate_proj", + "model.layers.4.mlp.experts.217.up_proj", + "model.layers.4.mlp.experts.218.down_proj", + "model.layers.4.mlp.experts.218.gate_proj", + "model.layers.4.mlp.experts.218.up_proj", + "model.layers.4.mlp.experts.219.down_proj", + "model.layers.4.mlp.experts.219.gate_proj", + "model.layers.4.mlp.experts.219.up_proj", + "model.layers.4.mlp.experts.22.down_proj", + "model.layers.4.mlp.experts.22.gate_proj", + "model.layers.4.mlp.experts.22.up_proj", + "model.layers.4.mlp.experts.220.down_proj", + "model.layers.4.mlp.experts.220.gate_proj", + "model.layers.4.mlp.experts.220.up_proj", + "model.layers.4.mlp.experts.221.down_proj", + "model.layers.4.mlp.experts.221.gate_proj", + "model.layers.4.mlp.experts.221.up_proj", + "model.layers.4.mlp.experts.222.down_proj", + "model.layers.4.mlp.experts.222.gate_proj", + "model.layers.4.mlp.experts.222.up_proj", + "model.layers.4.mlp.experts.223.down_proj", + "model.layers.4.mlp.experts.223.gate_proj", + "model.layers.4.mlp.experts.223.up_proj", + "model.layers.4.mlp.experts.224.down_proj", + "model.layers.4.mlp.experts.224.gate_proj", + "model.layers.4.mlp.experts.224.up_proj", + "model.layers.4.mlp.experts.225.down_proj", + "model.layers.4.mlp.experts.225.gate_proj", + "model.layers.4.mlp.experts.225.up_proj", + "model.layers.4.mlp.experts.226.down_proj", + "model.layers.4.mlp.experts.226.gate_proj", + "model.layers.4.mlp.experts.226.up_proj", + "model.layers.4.mlp.experts.227.down_proj", + "model.layers.4.mlp.experts.227.gate_proj", + "model.layers.4.mlp.experts.227.up_proj", + "model.layers.4.mlp.experts.228.down_proj", + "model.layers.4.mlp.experts.228.gate_proj", + "model.layers.4.mlp.experts.228.up_proj", + "model.layers.4.mlp.experts.229.down_proj", + "model.layers.4.mlp.experts.229.gate_proj", + "model.layers.4.mlp.experts.229.up_proj", + "model.layers.4.mlp.experts.23.down_proj", + "model.layers.4.mlp.experts.23.gate_proj", + "model.layers.4.mlp.experts.23.up_proj", + "model.layers.4.mlp.experts.230.down_proj", + "model.layers.4.mlp.experts.230.gate_proj", + "model.layers.4.mlp.experts.230.up_proj", + "model.layers.4.mlp.experts.231.down_proj", + "model.layers.4.mlp.experts.231.gate_proj", + "model.layers.4.mlp.experts.231.up_proj", + "model.layers.4.mlp.experts.232.down_proj", + "model.layers.4.mlp.experts.232.gate_proj", + "model.layers.4.mlp.experts.232.up_proj", + "model.layers.4.mlp.experts.233.down_proj", + "model.layers.4.mlp.experts.233.gate_proj", + "model.layers.4.mlp.experts.233.up_proj", + "model.layers.4.mlp.experts.234.down_proj", + "model.layers.4.mlp.experts.234.gate_proj", + "model.layers.4.mlp.experts.234.up_proj", + "model.layers.4.mlp.experts.235.down_proj", + "model.layers.4.mlp.experts.235.gate_proj", + "model.layers.4.mlp.experts.235.up_proj", + "model.layers.4.mlp.experts.236.down_proj", + "model.layers.4.mlp.experts.236.gate_proj", + "model.layers.4.mlp.experts.236.up_proj", + "model.layers.4.mlp.experts.237.down_proj", + "model.layers.4.mlp.experts.237.gate_proj", + "model.layers.4.mlp.experts.237.up_proj", + "model.layers.4.mlp.experts.238.down_proj", + "model.layers.4.mlp.experts.238.gate_proj", + "model.layers.4.mlp.experts.238.up_proj", + "model.layers.4.mlp.experts.239.down_proj", + "model.layers.4.mlp.experts.239.gate_proj", + "model.layers.4.mlp.experts.239.up_proj", + "model.layers.4.mlp.experts.24.down_proj", + "model.layers.4.mlp.experts.24.gate_proj", + "model.layers.4.mlp.experts.24.up_proj", + "model.layers.4.mlp.experts.240.down_proj", + "model.layers.4.mlp.experts.240.gate_proj", + "model.layers.4.mlp.experts.240.up_proj", + "model.layers.4.mlp.experts.241.down_proj", + "model.layers.4.mlp.experts.241.gate_proj", + "model.layers.4.mlp.experts.241.up_proj", + "model.layers.4.mlp.experts.242.down_proj", + "model.layers.4.mlp.experts.242.gate_proj", + "model.layers.4.mlp.experts.242.up_proj", + "model.layers.4.mlp.experts.243.down_proj", + "model.layers.4.mlp.experts.243.gate_proj", + "model.layers.4.mlp.experts.243.up_proj", + "model.layers.4.mlp.experts.244.down_proj", + "model.layers.4.mlp.experts.244.gate_proj", + "model.layers.4.mlp.experts.244.up_proj", + "model.layers.4.mlp.experts.245.down_proj", + "model.layers.4.mlp.experts.245.gate_proj", + "model.layers.4.mlp.experts.245.up_proj", + "model.layers.4.mlp.experts.246.down_proj", + "model.layers.4.mlp.experts.246.gate_proj", + "model.layers.4.mlp.experts.246.up_proj", + "model.layers.4.mlp.experts.247.down_proj", + "model.layers.4.mlp.experts.247.gate_proj", + "model.layers.4.mlp.experts.247.up_proj", + "model.layers.4.mlp.experts.248.down_proj", + "model.layers.4.mlp.experts.248.gate_proj", + "model.layers.4.mlp.experts.248.up_proj", + "model.layers.4.mlp.experts.249.down_proj", + "model.layers.4.mlp.experts.249.gate_proj", + "model.layers.4.mlp.experts.249.up_proj", + "model.layers.4.mlp.experts.25.down_proj", + "model.layers.4.mlp.experts.25.gate_proj", + "model.layers.4.mlp.experts.25.up_proj", + "model.layers.4.mlp.experts.250.down_proj", + "model.layers.4.mlp.experts.250.gate_proj", + "model.layers.4.mlp.experts.250.up_proj", + "model.layers.4.mlp.experts.251.down_proj", + "model.layers.4.mlp.experts.251.gate_proj", + "model.layers.4.mlp.experts.251.up_proj", + "model.layers.4.mlp.experts.252.down_proj", + "model.layers.4.mlp.experts.252.gate_proj", + "model.layers.4.mlp.experts.252.up_proj", + "model.layers.4.mlp.experts.253.down_proj", + "model.layers.4.mlp.experts.253.gate_proj", + "model.layers.4.mlp.experts.253.up_proj", + "model.layers.4.mlp.experts.254.down_proj", + "model.layers.4.mlp.experts.254.gate_proj", + "model.layers.4.mlp.experts.254.up_proj", + "model.layers.4.mlp.experts.255.down_proj", + "model.layers.4.mlp.experts.255.gate_proj", + "model.layers.4.mlp.experts.255.up_proj", + "model.layers.4.mlp.experts.26.down_proj", + "model.layers.4.mlp.experts.26.gate_proj", + "model.layers.4.mlp.experts.26.up_proj", + "model.layers.4.mlp.experts.27.down_proj", + "model.layers.4.mlp.experts.27.gate_proj", + "model.layers.4.mlp.experts.27.up_proj", + "model.layers.4.mlp.experts.28.down_proj", + "model.layers.4.mlp.experts.28.gate_proj", + "model.layers.4.mlp.experts.28.up_proj", + "model.layers.4.mlp.experts.29.down_proj", + "model.layers.4.mlp.experts.29.gate_proj", + "model.layers.4.mlp.experts.29.up_proj", + "model.layers.4.mlp.experts.3.down_proj", + "model.layers.4.mlp.experts.3.gate_proj", + "model.layers.4.mlp.experts.3.up_proj", + "model.layers.4.mlp.experts.30.down_proj", + "model.layers.4.mlp.experts.30.gate_proj", + "model.layers.4.mlp.experts.30.up_proj", + "model.layers.4.mlp.experts.31.down_proj", + "model.layers.4.mlp.experts.31.gate_proj", + "model.layers.4.mlp.experts.31.up_proj", + "model.layers.4.mlp.experts.32.down_proj", + "model.layers.4.mlp.experts.32.gate_proj", + "model.layers.4.mlp.experts.32.up_proj", + "model.layers.4.mlp.experts.33.down_proj", + "model.layers.4.mlp.experts.33.gate_proj", + "model.layers.4.mlp.experts.33.up_proj", + "model.layers.4.mlp.experts.34.down_proj", + "model.layers.4.mlp.experts.34.gate_proj", + "model.layers.4.mlp.experts.34.up_proj", + "model.layers.4.mlp.experts.35.down_proj", + "model.layers.4.mlp.experts.35.gate_proj", + "model.layers.4.mlp.experts.35.up_proj", + "model.layers.4.mlp.experts.36.down_proj", + "model.layers.4.mlp.experts.36.gate_proj", + "model.layers.4.mlp.experts.36.up_proj", + "model.layers.4.mlp.experts.37.down_proj", + "model.layers.4.mlp.experts.37.gate_proj", + "model.layers.4.mlp.experts.37.up_proj", + "model.layers.4.mlp.experts.38.down_proj", + "model.layers.4.mlp.experts.38.gate_proj", + "model.layers.4.mlp.experts.38.up_proj", + "model.layers.4.mlp.experts.39.down_proj", + "model.layers.4.mlp.experts.39.gate_proj", + "model.layers.4.mlp.experts.39.up_proj", + "model.layers.4.mlp.experts.4.down_proj", + "model.layers.4.mlp.experts.4.gate_proj", + "model.layers.4.mlp.experts.4.up_proj", + "model.layers.4.mlp.experts.40.down_proj", + "model.layers.4.mlp.experts.40.gate_proj", + "model.layers.4.mlp.experts.40.up_proj", + "model.layers.4.mlp.experts.41.down_proj", + "model.layers.4.mlp.experts.41.gate_proj", + "model.layers.4.mlp.experts.41.up_proj", + "model.layers.4.mlp.experts.42.down_proj", + "model.layers.4.mlp.experts.42.gate_proj", + "model.layers.4.mlp.experts.42.up_proj", + "model.layers.4.mlp.experts.43.down_proj", + "model.layers.4.mlp.experts.43.gate_proj", + "model.layers.4.mlp.experts.43.up_proj", + "model.layers.4.mlp.experts.44.down_proj", + "model.layers.4.mlp.experts.44.gate_proj", + "model.layers.4.mlp.experts.44.up_proj", + "model.layers.4.mlp.experts.45.down_proj", + "model.layers.4.mlp.experts.45.gate_proj", + "model.layers.4.mlp.experts.45.up_proj", + "model.layers.4.mlp.experts.46.down_proj", + "model.layers.4.mlp.experts.46.gate_proj", + "model.layers.4.mlp.experts.46.up_proj", + "model.layers.4.mlp.experts.47.down_proj", + "model.layers.4.mlp.experts.47.gate_proj", + "model.layers.4.mlp.experts.47.up_proj", + "model.layers.4.mlp.experts.48.down_proj", + "model.layers.4.mlp.experts.48.gate_proj", + "model.layers.4.mlp.experts.48.up_proj", + "model.layers.4.mlp.experts.49.down_proj", + "model.layers.4.mlp.experts.49.gate_proj", + "model.layers.4.mlp.experts.49.up_proj", + "model.layers.4.mlp.experts.5.down_proj", + "model.layers.4.mlp.experts.5.gate_proj", + "model.layers.4.mlp.experts.5.up_proj", + "model.layers.4.mlp.experts.50.down_proj", + "model.layers.4.mlp.experts.50.gate_proj", + "model.layers.4.mlp.experts.50.up_proj", + "model.layers.4.mlp.experts.51.down_proj", + "model.layers.4.mlp.experts.51.gate_proj", + "model.layers.4.mlp.experts.51.up_proj", + "model.layers.4.mlp.experts.52.down_proj", + "model.layers.4.mlp.experts.52.gate_proj", + "model.layers.4.mlp.experts.52.up_proj", + "model.layers.4.mlp.experts.53.down_proj", + "model.layers.4.mlp.experts.53.gate_proj", + "model.layers.4.mlp.experts.53.up_proj", + "model.layers.4.mlp.experts.54.down_proj", + "model.layers.4.mlp.experts.54.gate_proj", + "model.layers.4.mlp.experts.54.up_proj", + "model.layers.4.mlp.experts.55.down_proj", + "model.layers.4.mlp.experts.55.gate_proj", + "model.layers.4.mlp.experts.55.up_proj", + "model.layers.4.mlp.experts.56.down_proj", + "model.layers.4.mlp.experts.56.gate_proj", + "model.layers.4.mlp.experts.56.up_proj", + "model.layers.4.mlp.experts.57.down_proj", + "model.layers.4.mlp.experts.57.gate_proj", + "model.layers.4.mlp.experts.57.up_proj", + "model.layers.4.mlp.experts.58.down_proj", + "model.layers.4.mlp.experts.58.gate_proj", + "model.layers.4.mlp.experts.58.up_proj", + "model.layers.4.mlp.experts.59.down_proj", + "model.layers.4.mlp.experts.59.gate_proj", + "model.layers.4.mlp.experts.59.up_proj", + "model.layers.4.mlp.experts.6.down_proj", + "model.layers.4.mlp.experts.6.gate_proj", + "model.layers.4.mlp.experts.6.up_proj", + "model.layers.4.mlp.experts.60.down_proj", + "model.layers.4.mlp.experts.60.gate_proj", + "model.layers.4.mlp.experts.60.up_proj", + "model.layers.4.mlp.experts.61.down_proj", + "model.layers.4.mlp.experts.61.gate_proj", + "model.layers.4.mlp.experts.61.up_proj", + "model.layers.4.mlp.experts.62.down_proj", + "model.layers.4.mlp.experts.62.gate_proj", + "model.layers.4.mlp.experts.62.up_proj", + "model.layers.4.mlp.experts.63.down_proj", + "model.layers.4.mlp.experts.63.gate_proj", + "model.layers.4.mlp.experts.63.up_proj", + "model.layers.4.mlp.experts.64.down_proj", + "model.layers.4.mlp.experts.64.gate_proj", + "model.layers.4.mlp.experts.64.up_proj", + "model.layers.4.mlp.experts.65.down_proj", + "model.layers.4.mlp.experts.65.gate_proj", + "model.layers.4.mlp.experts.65.up_proj", + "model.layers.4.mlp.experts.66.down_proj", + "model.layers.4.mlp.experts.66.gate_proj", + "model.layers.4.mlp.experts.66.up_proj", + "model.layers.4.mlp.experts.67.down_proj", + "model.layers.4.mlp.experts.67.gate_proj", + "model.layers.4.mlp.experts.67.up_proj", + "model.layers.4.mlp.experts.68.down_proj", + "model.layers.4.mlp.experts.68.gate_proj", + "model.layers.4.mlp.experts.68.up_proj", + "model.layers.4.mlp.experts.69.down_proj", + "model.layers.4.mlp.experts.69.gate_proj", + "model.layers.4.mlp.experts.69.up_proj", + "model.layers.4.mlp.experts.7.down_proj", + "model.layers.4.mlp.experts.7.gate_proj", + "model.layers.4.mlp.experts.7.up_proj", + "model.layers.4.mlp.experts.70.down_proj", + "model.layers.4.mlp.experts.70.gate_proj", + "model.layers.4.mlp.experts.70.up_proj", + "model.layers.4.mlp.experts.71.down_proj", + "model.layers.4.mlp.experts.71.gate_proj", + "model.layers.4.mlp.experts.71.up_proj", + "model.layers.4.mlp.experts.72.down_proj", + "model.layers.4.mlp.experts.72.gate_proj", + "model.layers.4.mlp.experts.72.up_proj", + "model.layers.4.mlp.experts.73.down_proj", + "model.layers.4.mlp.experts.73.gate_proj", + "model.layers.4.mlp.experts.73.up_proj", + "model.layers.4.mlp.experts.74.down_proj", + "model.layers.4.mlp.experts.74.gate_proj", + "model.layers.4.mlp.experts.74.up_proj", + "model.layers.4.mlp.experts.75.down_proj", + "model.layers.4.mlp.experts.75.gate_proj", + "model.layers.4.mlp.experts.75.up_proj", + "model.layers.4.mlp.experts.76.down_proj", + "model.layers.4.mlp.experts.76.gate_proj", + "model.layers.4.mlp.experts.76.up_proj", + "model.layers.4.mlp.experts.77.down_proj", + "model.layers.4.mlp.experts.77.gate_proj", + "model.layers.4.mlp.experts.77.up_proj", + "model.layers.4.mlp.experts.78.down_proj", + "model.layers.4.mlp.experts.78.gate_proj", + "model.layers.4.mlp.experts.78.up_proj", + "model.layers.4.mlp.experts.79.down_proj", + "model.layers.4.mlp.experts.79.gate_proj", + "model.layers.4.mlp.experts.79.up_proj", + "model.layers.4.mlp.experts.8.down_proj", + "model.layers.4.mlp.experts.8.gate_proj", + "model.layers.4.mlp.experts.8.up_proj", + "model.layers.4.mlp.experts.80.down_proj", + "model.layers.4.mlp.experts.80.gate_proj", + "model.layers.4.mlp.experts.80.up_proj", + "model.layers.4.mlp.experts.81.down_proj", + "model.layers.4.mlp.experts.81.gate_proj", + "model.layers.4.mlp.experts.81.up_proj", + "model.layers.4.mlp.experts.82.down_proj", + "model.layers.4.mlp.experts.82.gate_proj", + "model.layers.4.mlp.experts.82.up_proj", + "model.layers.4.mlp.experts.83.down_proj", + "model.layers.4.mlp.experts.83.gate_proj", + "model.layers.4.mlp.experts.83.up_proj", + "model.layers.4.mlp.experts.84.down_proj", + "model.layers.4.mlp.experts.84.gate_proj", + "model.layers.4.mlp.experts.84.up_proj", + "model.layers.4.mlp.experts.85.down_proj", + "model.layers.4.mlp.experts.85.gate_proj", + "model.layers.4.mlp.experts.85.up_proj", + "model.layers.4.mlp.experts.86.down_proj", + "model.layers.4.mlp.experts.86.gate_proj", + "model.layers.4.mlp.experts.86.up_proj", + "model.layers.4.mlp.experts.87.down_proj", + "model.layers.4.mlp.experts.87.gate_proj", + "model.layers.4.mlp.experts.87.up_proj", + "model.layers.4.mlp.experts.88.down_proj", + "model.layers.4.mlp.experts.88.gate_proj", + "model.layers.4.mlp.experts.88.up_proj", + "model.layers.4.mlp.experts.89.down_proj", + "model.layers.4.mlp.experts.89.gate_proj", + "model.layers.4.mlp.experts.89.up_proj", + "model.layers.4.mlp.experts.9.down_proj", + "model.layers.4.mlp.experts.9.gate_proj", + "model.layers.4.mlp.experts.9.up_proj", + "model.layers.4.mlp.experts.90.down_proj", + "model.layers.4.mlp.experts.90.gate_proj", + "model.layers.4.mlp.experts.90.up_proj", + "model.layers.4.mlp.experts.91.down_proj", + "model.layers.4.mlp.experts.91.gate_proj", + "model.layers.4.mlp.experts.91.up_proj", + "model.layers.4.mlp.experts.92.down_proj", + "model.layers.4.mlp.experts.92.gate_proj", + "model.layers.4.mlp.experts.92.up_proj", + "model.layers.4.mlp.experts.93.down_proj", + "model.layers.4.mlp.experts.93.gate_proj", + "model.layers.4.mlp.experts.93.up_proj", + "model.layers.4.mlp.experts.94.down_proj", + "model.layers.4.mlp.experts.94.gate_proj", + "model.layers.4.mlp.experts.94.up_proj", + "model.layers.4.mlp.experts.95.down_proj", + "model.layers.4.mlp.experts.95.gate_proj", + "model.layers.4.mlp.experts.95.up_proj", + "model.layers.4.mlp.experts.96.down_proj", + "model.layers.4.mlp.experts.96.gate_proj", + "model.layers.4.mlp.experts.96.up_proj", + "model.layers.4.mlp.experts.97.down_proj", + "model.layers.4.mlp.experts.97.gate_proj", + "model.layers.4.mlp.experts.97.up_proj", + "model.layers.4.mlp.experts.98.down_proj", + "model.layers.4.mlp.experts.98.gate_proj", + "model.layers.4.mlp.experts.98.up_proj", + "model.layers.4.mlp.experts.99.down_proj", + "model.layers.4.mlp.experts.99.gate_proj", + "model.layers.4.mlp.experts.99.up_proj", + "model.layers.40.mlp.experts.0.down_proj", + "model.layers.40.mlp.experts.0.gate_proj", + "model.layers.40.mlp.experts.0.up_proj", + "model.layers.40.mlp.experts.1.down_proj", + "model.layers.40.mlp.experts.1.gate_proj", + "model.layers.40.mlp.experts.1.up_proj", + "model.layers.40.mlp.experts.10.down_proj", + "model.layers.40.mlp.experts.10.gate_proj", + "model.layers.40.mlp.experts.10.up_proj", + "model.layers.40.mlp.experts.100.down_proj", + "model.layers.40.mlp.experts.100.gate_proj", + "model.layers.40.mlp.experts.100.up_proj", + "model.layers.40.mlp.experts.101.down_proj", + "model.layers.40.mlp.experts.101.gate_proj", + "model.layers.40.mlp.experts.101.up_proj", + "model.layers.40.mlp.experts.102.down_proj", + "model.layers.40.mlp.experts.102.gate_proj", + "model.layers.40.mlp.experts.102.up_proj", + "model.layers.40.mlp.experts.103.down_proj", + "model.layers.40.mlp.experts.103.gate_proj", + "model.layers.40.mlp.experts.103.up_proj", + "model.layers.40.mlp.experts.104.down_proj", + "model.layers.40.mlp.experts.104.gate_proj", + "model.layers.40.mlp.experts.104.up_proj", + "model.layers.40.mlp.experts.105.down_proj", + "model.layers.40.mlp.experts.105.gate_proj", + "model.layers.40.mlp.experts.105.up_proj", + "model.layers.40.mlp.experts.106.down_proj", + "model.layers.40.mlp.experts.106.gate_proj", + "model.layers.40.mlp.experts.106.up_proj", + "model.layers.40.mlp.experts.107.down_proj", + "model.layers.40.mlp.experts.107.gate_proj", + "model.layers.40.mlp.experts.107.up_proj", + "model.layers.40.mlp.experts.108.down_proj", + "model.layers.40.mlp.experts.108.gate_proj", + "model.layers.40.mlp.experts.108.up_proj", + "model.layers.40.mlp.experts.109.down_proj", + "model.layers.40.mlp.experts.109.gate_proj", + "model.layers.40.mlp.experts.109.up_proj", + "model.layers.40.mlp.experts.11.down_proj", + "model.layers.40.mlp.experts.11.gate_proj", + "model.layers.40.mlp.experts.11.up_proj", + "model.layers.40.mlp.experts.110.down_proj", + "model.layers.40.mlp.experts.110.gate_proj", + "model.layers.40.mlp.experts.110.up_proj", + "model.layers.40.mlp.experts.111.down_proj", + "model.layers.40.mlp.experts.111.gate_proj", + "model.layers.40.mlp.experts.111.up_proj", + "model.layers.40.mlp.experts.112.down_proj", + "model.layers.40.mlp.experts.112.gate_proj", + "model.layers.40.mlp.experts.112.up_proj", + "model.layers.40.mlp.experts.113.down_proj", + "model.layers.40.mlp.experts.113.gate_proj", + "model.layers.40.mlp.experts.113.up_proj", + "model.layers.40.mlp.experts.114.down_proj", + "model.layers.40.mlp.experts.114.gate_proj", + "model.layers.40.mlp.experts.114.up_proj", + "model.layers.40.mlp.experts.115.down_proj", + "model.layers.40.mlp.experts.115.gate_proj", + "model.layers.40.mlp.experts.115.up_proj", + "model.layers.40.mlp.experts.116.down_proj", + "model.layers.40.mlp.experts.116.gate_proj", + "model.layers.40.mlp.experts.116.up_proj", + "model.layers.40.mlp.experts.117.down_proj", + "model.layers.40.mlp.experts.117.gate_proj", + "model.layers.40.mlp.experts.117.up_proj", + "model.layers.40.mlp.experts.118.down_proj", + "model.layers.40.mlp.experts.118.gate_proj", + "model.layers.40.mlp.experts.118.up_proj", + "model.layers.40.mlp.experts.119.down_proj", + "model.layers.40.mlp.experts.119.gate_proj", + "model.layers.40.mlp.experts.119.up_proj", + "model.layers.40.mlp.experts.12.down_proj", + "model.layers.40.mlp.experts.12.gate_proj", + "model.layers.40.mlp.experts.12.up_proj", + "model.layers.40.mlp.experts.120.down_proj", + "model.layers.40.mlp.experts.120.gate_proj", + "model.layers.40.mlp.experts.120.up_proj", + "model.layers.40.mlp.experts.121.down_proj", + "model.layers.40.mlp.experts.121.gate_proj", + "model.layers.40.mlp.experts.121.up_proj", + "model.layers.40.mlp.experts.122.down_proj", + "model.layers.40.mlp.experts.122.gate_proj", + "model.layers.40.mlp.experts.122.up_proj", + "model.layers.40.mlp.experts.123.down_proj", + "model.layers.40.mlp.experts.123.gate_proj", + "model.layers.40.mlp.experts.123.up_proj", + "model.layers.40.mlp.experts.124.down_proj", + "model.layers.40.mlp.experts.124.gate_proj", + "model.layers.40.mlp.experts.124.up_proj", + "model.layers.40.mlp.experts.125.down_proj", + "model.layers.40.mlp.experts.125.gate_proj", + "model.layers.40.mlp.experts.125.up_proj", + "model.layers.40.mlp.experts.126.down_proj", + "model.layers.40.mlp.experts.126.gate_proj", + "model.layers.40.mlp.experts.126.up_proj", + "model.layers.40.mlp.experts.127.down_proj", + "model.layers.40.mlp.experts.127.gate_proj", + "model.layers.40.mlp.experts.127.up_proj", + "model.layers.40.mlp.experts.128.down_proj", + "model.layers.40.mlp.experts.128.gate_proj", + "model.layers.40.mlp.experts.128.up_proj", + "model.layers.40.mlp.experts.129.down_proj", + "model.layers.40.mlp.experts.129.gate_proj", + "model.layers.40.mlp.experts.129.up_proj", + "model.layers.40.mlp.experts.13.down_proj", + "model.layers.40.mlp.experts.13.gate_proj", + "model.layers.40.mlp.experts.13.up_proj", + "model.layers.40.mlp.experts.130.down_proj", + "model.layers.40.mlp.experts.130.gate_proj", + "model.layers.40.mlp.experts.130.up_proj", + "model.layers.40.mlp.experts.131.down_proj", + "model.layers.40.mlp.experts.131.gate_proj", + "model.layers.40.mlp.experts.131.up_proj", + "model.layers.40.mlp.experts.132.down_proj", + "model.layers.40.mlp.experts.132.gate_proj", + "model.layers.40.mlp.experts.132.up_proj", + "model.layers.40.mlp.experts.133.down_proj", + "model.layers.40.mlp.experts.133.gate_proj", + "model.layers.40.mlp.experts.133.up_proj", + "model.layers.40.mlp.experts.134.down_proj", + "model.layers.40.mlp.experts.134.gate_proj", + "model.layers.40.mlp.experts.134.up_proj", + "model.layers.40.mlp.experts.135.down_proj", + "model.layers.40.mlp.experts.135.gate_proj", + "model.layers.40.mlp.experts.135.up_proj", + "model.layers.40.mlp.experts.136.down_proj", + "model.layers.40.mlp.experts.136.gate_proj", + "model.layers.40.mlp.experts.136.up_proj", + "model.layers.40.mlp.experts.137.down_proj", + "model.layers.40.mlp.experts.137.gate_proj", + "model.layers.40.mlp.experts.137.up_proj", + "model.layers.40.mlp.experts.138.down_proj", + "model.layers.40.mlp.experts.138.gate_proj", + "model.layers.40.mlp.experts.138.up_proj", + "model.layers.40.mlp.experts.139.down_proj", + "model.layers.40.mlp.experts.139.gate_proj", + "model.layers.40.mlp.experts.139.up_proj", + "model.layers.40.mlp.experts.14.down_proj", + "model.layers.40.mlp.experts.14.gate_proj", + "model.layers.40.mlp.experts.14.up_proj", + "model.layers.40.mlp.experts.140.down_proj", + "model.layers.40.mlp.experts.140.gate_proj", + "model.layers.40.mlp.experts.140.up_proj", + "model.layers.40.mlp.experts.141.down_proj", + "model.layers.40.mlp.experts.141.gate_proj", + "model.layers.40.mlp.experts.141.up_proj", + "model.layers.40.mlp.experts.142.down_proj", + "model.layers.40.mlp.experts.142.gate_proj", + "model.layers.40.mlp.experts.142.up_proj", + "model.layers.40.mlp.experts.143.down_proj", + "model.layers.40.mlp.experts.143.gate_proj", + "model.layers.40.mlp.experts.143.up_proj", + "model.layers.40.mlp.experts.144.down_proj", + "model.layers.40.mlp.experts.144.gate_proj", + "model.layers.40.mlp.experts.144.up_proj", + "model.layers.40.mlp.experts.145.down_proj", + "model.layers.40.mlp.experts.145.gate_proj", + "model.layers.40.mlp.experts.145.up_proj", + "model.layers.40.mlp.experts.146.down_proj", + "model.layers.40.mlp.experts.146.gate_proj", + "model.layers.40.mlp.experts.146.up_proj", + "model.layers.40.mlp.experts.147.down_proj", + "model.layers.40.mlp.experts.147.gate_proj", + "model.layers.40.mlp.experts.147.up_proj", + "model.layers.40.mlp.experts.148.down_proj", + "model.layers.40.mlp.experts.148.gate_proj", + "model.layers.40.mlp.experts.148.up_proj", + "model.layers.40.mlp.experts.149.down_proj", + "model.layers.40.mlp.experts.149.gate_proj", + "model.layers.40.mlp.experts.149.up_proj", + "model.layers.40.mlp.experts.15.down_proj", + "model.layers.40.mlp.experts.15.gate_proj", + "model.layers.40.mlp.experts.15.up_proj", + "model.layers.40.mlp.experts.150.down_proj", + "model.layers.40.mlp.experts.150.gate_proj", + "model.layers.40.mlp.experts.150.up_proj", + "model.layers.40.mlp.experts.151.down_proj", + "model.layers.40.mlp.experts.151.gate_proj", + "model.layers.40.mlp.experts.151.up_proj", + "model.layers.40.mlp.experts.152.down_proj", + "model.layers.40.mlp.experts.152.gate_proj", + "model.layers.40.mlp.experts.152.up_proj", + "model.layers.40.mlp.experts.153.down_proj", + "model.layers.40.mlp.experts.153.gate_proj", + "model.layers.40.mlp.experts.153.up_proj", + "model.layers.40.mlp.experts.154.down_proj", + "model.layers.40.mlp.experts.154.gate_proj", + "model.layers.40.mlp.experts.154.up_proj", + "model.layers.40.mlp.experts.155.down_proj", + "model.layers.40.mlp.experts.155.gate_proj", + "model.layers.40.mlp.experts.155.up_proj", + "model.layers.40.mlp.experts.156.down_proj", + "model.layers.40.mlp.experts.156.gate_proj", + "model.layers.40.mlp.experts.156.up_proj", + "model.layers.40.mlp.experts.157.down_proj", + "model.layers.40.mlp.experts.157.gate_proj", + "model.layers.40.mlp.experts.157.up_proj", + "model.layers.40.mlp.experts.158.down_proj", + "model.layers.40.mlp.experts.158.gate_proj", + "model.layers.40.mlp.experts.158.up_proj", + "model.layers.40.mlp.experts.159.down_proj", + "model.layers.40.mlp.experts.159.gate_proj", + "model.layers.40.mlp.experts.159.up_proj", + "model.layers.40.mlp.experts.16.down_proj", + "model.layers.40.mlp.experts.16.gate_proj", + "model.layers.40.mlp.experts.16.up_proj", + "model.layers.40.mlp.experts.160.down_proj", + "model.layers.40.mlp.experts.160.gate_proj", + "model.layers.40.mlp.experts.160.up_proj", + "model.layers.40.mlp.experts.161.down_proj", + "model.layers.40.mlp.experts.161.gate_proj", + "model.layers.40.mlp.experts.161.up_proj", + "model.layers.40.mlp.experts.162.down_proj", + "model.layers.40.mlp.experts.162.gate_proj", + "model.layers.40.mlp.experts.162.up_proj", + "model.layers.40.mlp.experts.163.down_proj", + "model.layers.40.mlp.experts.163.gate_proj", + "model.layers.40.mlp.experts.163.up_proj", + "model.layers.40.mlp.experts.164.down_proj", + "model.layers.40.mlp.experts.164.gate_proj", + "model.layers.40.mlp.experts.164.up_proj", + "model.layers.40.mlp.experts.165.down_proj", + "model.layers.40.mlp.experts.165.gate_proj", + "model.layers.40.mlp.experts.165.up_proj", + "model.layers.40.mlp.experts.166.down_proj", + "model.layers.40.mlp.experts.166.gate_proj", + "model.layers.40.mlp.experts.166.up_proj", + "model.layers.40.mlp.experts.167.down_proj", + "model.layers.40.mlp.experts.167.gate_proj", + "model.layers.40.mlp.experts.167.up_proj", + "model.layers.40.mlp.experts.168.down_proj", + "model.layers.40.mlp.experts.168.gate_proj", + "model.layers.40.mlp.experts.168.up_proj", + "model.layers.40.mlp.experts.169.down_proj", + "model.layers.40.mlp.experts.169.gate_proj", + "model.layers.40.mlp.experts.169.up_proj", + "model.layers.40.mlp.experts.17.down_proj", + "model.layers.40.mlp.experts.17.gate_proj", + "model.layers.40.mlp.experts.17.up_proj", + "model.layers.40.mlp.experts.170.down_proj", + "model.layers.40.mlp.experts.170.gate_proj", + "model.layers.40.mlp.experts.170.up_proj", + "model.layers.40.mlp.experts.171.down_proj", + "model.layers.40.mlp.experts.171.gate_proj", + "model.layers.40.mlp.experts.171.up_proj", + "model.layers.40.mlp.experts.172.down_proj", + "model.layers.40.mlp.experts.172.gate_proj", + "model.layers.40.mlp.experts.172.up_proj", + "model.layers.40.mlp.experts.173.down_proj", + "model.layers.40.mlp.experts.173.gate_proj", + "model.layers.40.mlp.experts.173.up_proj", + "model.layers.40.mlp.experts.174.down_proj", + "model.layers.40.mlp.experts.174.gate_proj", + "model.layers.40.mlp.experts.174.up_proj", + "model.layers.40.mlp.experts.175.down_proj", + "model.layers.40.mlp.experts.175.gate_proj", + "model.layers.40.mlp.experts.175.up_proj", + "model.layers.40.mlp.experts.176.down_proj", + "model.layers.40.mlp.experts.176.gate_proj", + "model.layers.40.mlp.experts.176.up_proj", + "model.layers.40.mlp.experts.177.down_proj", + "model.layers.40.mlp.experts.177.gate_proj", + "model.layers.40.mlp.experts.177.up_proj", + "model.layers.40.mlp.experts.178.down_proj", + "model.layers.40.mlp.experts.178.gate_proj", + "model.layers.40.mlp.experts.178.up_proj", + "model.layers.40.mlp.experts.179.down_proj", + "model.layers.40.mlp.experts.179.gate_proj", + "model.layers.40.mlp.experts.179.up_proj", + "model.layers.40.mlp.experts.18.down_proj", + "model.layers.40.mlp.experts.18.gate_proj", + "model.layers.40.mlp.experts.18.up_proj", + "model.layers.40.mlp.experts.180.down_proj", + "model.layers.40.mlp.experts.180.gate_proj", + "model.layers.40.mlp.experts.180.up_proj", + "model.layers.40.mlp.experts.181.down_proj", + "model.layers.40.mlp.experts.181.gate_proj", + "model.layers.40.mlp.experts.181.up_proj", + "model.layers.40.mlp.experts.182.down_proj", + "model.layers.40.mlp.experts.182.gate_proj", + "model.layers.40.mlp.experts.182.up_proj", + "model.layers.40.mlp.experts.183.down_proj", + "model.layers.40.mlp.experts.183.gate_proj", + "model.layers.40.mlp.experts.183.up_proj", + "model.layers.40.mlp.experts.184.down_proj", + "model.layers.40.mlp.experts.184.gate_proj", + "model.layers.40.mlp.experts.184.up_proj", + "model.layers.40.mlp.experts.185.down_proj", + "model.layers.40.mlp.experts.185.gate_proj", + "model.layers.40.mlp.experts.185.up_proj", + "model.layers.40.mlp.experts.186.down_proj", + "model.layers.40.mlp.experts.186.gate_proj", + "model.layers.40.mlp.experts.186.up_proj", + "model.layers.40.mlp.experts.187.down_proj", + "model.layers.40.mlp.experts.187.gate_proj", + "model.layers.40.mlp.experts.187.up_proj", + "model.layers.40.mlp.experts.188.down_proj", + "model.layers.40.mlp.experts.188.gate_proj", + "model.layers.40.mlp.experts.188.up_proj", + "model.layers.40.mlp.experts.189.down_proj", + "model.layers.40.mlp.experts.189.gate_proj", + "model.layers.40.mlp.experts.189.up_proj", + "model.layers.40.mlp.experts.19.down_proj", + "model.layers.40.mlp.experts.19.gate_proj", + "model.layers.40.mlp.experts.19.up_proj", + "model.layers.40.mlp.experts.190.down_proj", + "model.layers.40.mlp.experts.190.gate_proj", + "model.layers.40.mlp.experts.190.up_proj", + "model.layers.40.mlp.experts.191.down_proj", + "model.layers.40.mlp.experts.191.gate_proj", + "model.layers.40.mlp.experts.191.up_proj", + "model.layers.40.mlp.experts.192.down_proj", + "model.layers.40.mlp.experts.192.gate_proj", + "model.layers.40.mlp.experts.192.up_proj", + "model.layers.40.mlp.experts.193.down_proj", + "model.layers.40.mlp.experts.193.gate_proj", + "model.layers.40.mlp.experts.193.up_proj", + "model.layers.40.mlp.experts.194.down_proj", + "model.layers.40.mlp.experts.194.gate_proj", + "model.layers.40.mlp.experts.194.up_proj", + "model.layers.40.mlp.experts.195.down_proj", + "model.layers.40.mlp.experts.195.gate_proj", + "model.layers.40.mlp.experts.195.up_proj", + "model.layers.40.mlp.experts.196.down_proj", + "model.layers.40.mlp.experts.196.gate_proj", + "model.layers.40.mlp.experts.196.up_proj", + "model.layers.40.mlp.experts.197.down_proj", + "model.layers.40.mlp.experts.197.gate_proj", + "model.layers.40.mlp.experts.197.up_proj", + "model.layers.40.mlp.experts.198.down_proj", + "model.layers.40.mlp.experts.198.gate_proj", + "model.layers.40.mlp.experts.198.up_proj", + "model.layers.40.mlp.experts.199.down_proj", + "model.layers.40.mlp.experts.199.gate_proj", + "model.layers.40.mlp.experts.199.up_proj", + "model.layers.40.mlp.experts.2.down_proj", + "model.layers.40.mlp.experts.2.gate_proj", + "model.layers.40.mlp.experts.2.up_proj", + "model.layers.40.mlp.experts.20.down_proj", + "model.layers.40.mlp.experts.20.gate_proj", + "model.layers.40.mlp.experts.20.up_proj", + "model.layers.40.mlp.experts.200.down_proj", + "model.layers.40.mlp.experts.200.gate_proj", + "model.layers.40.mlp.experts.200.up_proj", + "model.layers.40.mlp.experts.201.down_proj", + "model.layers.40.mlp.experts.201.gate_proj", + "model.layers.40.mlp.experts.201.up_proj", + "model.layers.40.mlp.experts.202.down_proj", + "model.layers.40.mlp.experts.202.gate_proj", + "model.layers.40.mlp.experts.202.up_proj", + "model.layers.40.mlp.experts.203.down_proj", + "model.layers.40.mlp.experts.203.gate_proj", + "model.layers.40.mlp.experts.203.up_proj", + "model.layers.40.mlp.experts.204.down_proj", + "model.layers.40.mlp.experts.204.gate_proj", + "model.layers.40.mlp.experts.204.up_proj", + "model.layers.40.mlp.experts.205.down_proj", + "model.layers.40.mlp.experts.205.gate_proj", + "model.layers.40.mlp.experts.205.up_proj", + "model.layers.40.mlp.experts.206.down_proj", + "model.layers.40.mlp.experts.206.gate_proj", + "model.layers.40.mlp.experts.206.up_proj", + "model.layers.40.mlp.experts.207.down_proj", + "model.layers.40.mlp.experts.207.gate_proj", + "model.layers.40.mlp.experts.207.up_proj", + "model.layers.40.mlp.experts.208.down_proj", + "model.layers.40.mlp.experts.208.gate_proj", + "model.layers.40.mlp.experts.208.up_proj", + "model.layers.40.mlp.experts.209.down_proj", + "model.layers.40.mlp.experts.209.gate_proj", + "model.layers.40.mlp.experts.209.up_proj", + "model.layers.40.mlp.experts.21.down_proj", + "model.layers.40.mlp.experts.21.gate_proj", + "model.layers.40.mlp.experts.21.up_proj", + "model.layers.40.mlp.experts.210.down_proj", + "model.layers.40.mlp.experts.210.gate_proj", + "model.layers.40.mlp.experts.210.up_proj", + "model.layers.40.mlp.experts.211.down_proj", + "model.layers.40.mlp.experts.211.gate_proj", + "model.layers.40.mlp.experts.211.up_proj", + "model.layers.40.mlp.experts.212.down_proj", + "model.layers.40.mlp.experts.212.gate_proj", + "model.layers.40.mlp.experts.212.up_proj", + "model.layers.40.mlp.experts.213.down_proj", + "model.layers.40.mlp.experts.213.gate_proj", + "model.layers.40.mlp.experts.213.up_proj", + "model.layers.40.mlp.experts.214.down_proj", + "model.layers.40.mlp.experts.214.gate_proj", + "model.layers.40.mlp.experts.214.up_proj", + "model.layers.40.mlp.experts.215.down_proj", + "model.layers.40.mlp.experts.215.gate_proj", + "model.layers.40.mlp.experts.215.up_proj", + "model.layers.40.mlp.experts.216.down_proj", + "model.layers.40.mlp.experts.216.gate_proj", + "model.layers.40.mlp.experts.216.up_proj", + "model.layers.40.mlp.experts.217.down_proj", + "model.layers.40.mlp.experts.217.gate_proj", + "model.layers.40.mlp.experts.217.up_proj", + "model.layers.40.mlp.experts.218.down_proj", + "model.layers.40.mlp.experts.218.gate_proj", + "model.layers.40.mlp.experts.218.up_proj", + "model.layers.40.mlp.experts.219.down_proj", + "model.layers.40.mlp.experts.219.gate_proj", + "model.layers.40.mlp.experts.219.up_proj", + "model.layers.40.mlp.experts.22.down_proj", + "model.layers.40.mlp.experts.22.gate_proj", + "model.layers.40.mlp.experts.22.up_proj", + "model.layers.40.mlp.experts.220.down_proj", + "model.layers.40.mlp.experts.220.gate_proj", + "model.layers.40.mlp.experts.220.up_proj", + "model.layers.40.mlp.experts.221.down_proj", + "model.layers.40.mlp.experts.221.gate_proj", + "model.layers.40.mlp.experts.221.up_proj", + "model.layers.40.mlp.experts.222.down_proj", + "model.layers.40.mlp.experts.222.gate_proj", + "model.layers.40.mlp.experts.222.up_proj", + "model.layers.40.mlp.experts.223.down_proj", + "model.layers.40.mlp.experts.223.gate_proj", + "model.layers.40.mlp.experts.223.up_proj", + "model.layers.40.mlp.experts.224.down_proj", + "model.layers.40.mlp.experts.224.gate_proj", + "model.layers.40.mlp.experts.224.up_proj", + "model.layers.40.mlp.experts.225.down_proj", + "model.layers.40.mlp.experts.225.gate_proj", + "model.layers.40.mlp.experts.225.up_proj", + "model.layers.40.mlp.experts.226.down_proj", + "model.layers.40.mlp.experts.226.gate_proj", + "model.layers.40.mlp.experts.226.up_proj", + "model.layers.40.mlp.experts.227.down_proj", + "model.layers.40.mlp.experts.227.gate_proj", + "model.layers.40.mlp.experts.227.up_proj", + "model.layers.40.mlp.experts.228.down_proj", + "model.layers.40.mlp.experts.228.gate_proj", + "model.layers.40.mlp.experts.228.up_proj", + "model.layers.40.mlp.experts.229.down_proj", + "model.layers.40.mlp.experts.229.gate_proj", + "model.layers.40.mlp.experts.229.up_proj", + "model.layers.40.mlp.experts.23.down_proj", + "model.layers.40.mlp.experts.23.gate_proj", + "model.layers.40.mlp.experts.23.up_proj", + "model.layers.40.mlp.experts.230.down_proj", + "model.layers.40.mlp.experts.230.gate_proj", + "model.layers.40.mlp.experts.230.up_proj", + "model.layers.40.mlp.experts.231.down_proj", + "model.layers.40.mlp.experts.231.gate_proj", + "model.layers.40.mlp.experts.231.up_proj", + "model.layers.40.mlp.experts.232.down_proj", + "model.layers.40.mlp.experts.232.gate_proj", + "model.layers.40.mlp.experts.232.up_proj", + "model.layers.40.mlp.experts.233.down_proj", + "model.layers.40.mlp.experts.233.gate_proj", + "model.layers.40.mlp.experts.233.up_proj", + "model.layers.40.mlp.experts.234.down_proj", + "model.layers.40.mlp.experts.234.gate_proj", + "model.layers.40.mlp.experts.234.up_proj", + "model.layers.40.mlp.experts.235.down_proj", + "model.layers.40.mlp.experts.235.gate_proj", + "model.layers.40.mlp.experts.235.up_proj", + "model.layers.40.mlp.experts.236.down_proj", + "model.layers.40.mlp.experts.236.gate_proj", + "model.layers.40.mlp.experts.236.up_proj", + "model.layers.40.mlp.experts.237.down_proj", + "model.layers.40.mlp.experts.237.gate_proj", + "model.layers.40.mlp.experts.237.up_proj", + "model.layers.40.mlp.experts.238.down_proj", + "model.layers.40.mlp.experts.238.gate_proj", + "model.layers.40.mlp.experts.238.up_proj", + "model.layers.40.mlp.experts.239.down_proj", + "model.layers.40.mlp.experts.239.gate_proj", + "model.layers.40.mlp.experts.239.up_proj", + "model.layers.40.mlp.experts.24.down_proj", + "model.layers.40.mlp.experts.24.gate_proj", + "model.layers.40.mlp.experts.24.up_proj", + "model.layers.40.mlp.experts.240.down_proj", + "model.layers.40.mlp.experts.240.gate_proj", + "model.layers.40.mlp.experts.240.up_proj", + "model.layers.40.mlp.experts.241.down_proj", + "model.layers.40.mlp.experts.241.gate_proj", + "model.layers.40.mlp.experts.241.up_proj", + "model.layers.40.mlp.experts.242.down_proj", + "model.layers.40.mlp.experts.242.gate_proj", + "model.layers.40.mlp.experts.242.up_proj", + "model.layers.40.mlp.experts.243.down_proj", + "model.layers.40.mlp.experts.243.gate_proj", + "model.layers.40.mlp.experts.243.up_proj", + "model.layers.40.mlp.experts.244.down_proj", + "model.layers.40.mlp.experts.244.gate_proj", + "model.layers.40.mlp.experts.244.up_proj", + "model.layers.40.mlp.experts.245.down_proj", + "model.layers.40.mlp.experts.245.gate_proj", + "model.layers.40.mlp.experts.245.up_proj", + "model.layers.40.mlp.experts.246.down_proj", + "model.layers.40.mlp.experts.246.gate_proj", + "model.layers.40.mlp.experts.246.up_proj", + "model.layers.40.mlp.experts.247.down_proj", + "model.layers.40.mlp.experts.247.gate_proj", + "model.layers.40.mlp.experts.247.up_proj", + "model.layers.40.mlp.experts.248.down_proj", + "model.layers.40.mlp.experts.248.gate_proj", + "model.layers.40.mlp.experts.248.up_proj", + "model.layers.40.mlp.experts.249.down_proj", + "model.layers.40.mlp.experts.249.gate_proj", + "model.layers.40.mlp.experts.249.up_proj", + "model.layers.40.mlp.experts.25.down_proj", + "model.layers.40.mlp.experts.25.gate_proj", + "model.layers.40.mlp.experts.25.up_proj", + "model.layers.40.mlp.experts.250.down_proj", + "model.layers.40.mlp.experts.250.gate_proj", + "model.layers.40.mlp.experts.250.up_proj", + "model.layers.40.mlp.experts.251.down_proj", + "model.layers.40.mlp.experts.251.gate_proj", + "model.layers.40.mlp.experts.251.up_proj", + "model.layers.40.mlp.experts.252.down_proj", + "model.layers.40.mlp.experts.252.gate_proj", + "model.layers.40.mlp.experts.252.up_proj", + "model.layers.40.mlp.experts.253.down_proj", + "model.layers.40.mlp.experts.253.gate_proj", + "model.layers.40.mlp.experts.253.up_proj", + "model.layers.40.mlp.experts.254.down_proj", + "model.layers.40.mlp.experts.254.gate_proj", + "model.layers.40.mlp.experts.254.up_proj", + "model.layers.40.mlp.experts.255.down_proj", + "model.layers.40.mlp.experts.255.gate_proj", + "model.layers.40.mlp.experts.255.up_proj", + "model.layers.40.mlp.experts.26.down_proj", + "model.layers.40.mlp.experts.26.gate_proj", + "model.layers.40.mlp.experts.26.up_proj", + "model.layers.40.mlp.experts.27.down_proj", + "model.layers.40.mlp.experts.27.gate_proj", + "model.layers.40.mlp.experts.27.up_proj", + "model.layers.40.mlp.experts.28.down_proj", + "model.layers.40.mlp.experts.28.gate_proj", + "model.layers.40.mlp.experts.28.up_proj", + "model.layers.40.mlp.experts.29.down_proj", + "model.layers.40.mlp.experts.29.gate_proj", + "model.layers.40.mlp.experts.29.up_proj", + "model.layers.40.mlp.experts.3.down_proj", + "model.layers.40.mlp.experts.3.gate_proj", + "model.layers.40.mlp.experts.3.up_proj", + "model.layers.40.mlp.experts.30.down_proj", + "model.layers.40.mlp.experts.30.gate_proj", + "model.layers.40.mlp.experts.30.up_proj", + "model.layers.40.mlp.experts.31.down_proj", + "model.layers.40.mlp.experts.31.gate_proj", + "model.layers.40.mlp.experts.31.up_proj", + "model.layers.40.mlp.experts.32.down_proj", + "model.layers.40.mlp.experts.32.gate_proj", + "model.layers.40.mlp.experts.32.up_proj", + "model.layers.40.mlp.experts.33.down_proj", + "model.layers.40.mlp.experts.33.gate_proj", + "model.layers.40.mlp.experts.33.up_proj", + "model.layers.40.mlp.experts.34.down_proj", + "model.layers.40.mlp.experts.34.gate_proj", + "model.layers.40.mlp.experts.34.up_proj", + "model.layers.40.mlp.experts.35.down_proj", + "model.layers.40.mlp.experts.35.gate_proj", + "model.layers.40.mlp.experts.35.up_proj", + "model.layers.40.mlp.experts.36.down_proj", + "model.layers.40.mlp.experts.36.gate_proj", + "model.layers.40.mlp.experts.36.up_proj", + "model.layers.40.mlp.experts.37.down_proj", + "model.layers.40.mlp.experts.37.gate_proj", + "model.layers.40.mlp.experts.37.up_proj", + "model.layers.40.mlp.experts.38.down_proj", + "model.layers.40.mlp.experts.38.gate_proj", + "model.layers.40.mlp.experts.38.up_proj", + "model.layers.40.mlp.experts.39.down_proj", + "model.layers.40.mlp.experts.39.gate_proj", + "model.layers.40.mlp.experts.39.up_proj", + "model.layers.40.mlp.experts.4.down_proj", + "model.layers.40.mlp.experts.4.gate_proj", + "model.layers.40.mlp.experts.4.up_proj", + "model.layers.40.mlp.experts.40.down_proj", + "model.layers.40.mlp.experts.40.gate_proj", + "model.layers.40.mlp.experts.40.up_proj", + "model.layers.40.mlp.experts.41.down_proj", + "model.layers.40.mlp.experts.41.gate_proj", + "model.layers.40.mlp.experts.41.up_proj", + "model.layers.40.mlp.experts.42.down_proj", + "model.layers.40.mlp.experts.42.gate_proj", + "model.layers.40.mlp.experts.42.up_proj", + "model.layers.40.mlp.experts.43.down_proj", + "model.layers.40.mlp.experts.43.gate_proj", + "model.layers.40.mlp.experts.43.up_proj", + "model.layers.40.mlp.experts.44.down_proj", + "model.layers.40.mlp.experts.44.gate_proj", + "model.layers.40.mlp.experts.44.up_proj", + "model.layers.40.mlp.experts.45.down_proj", + "model.layers.40.mlp.experts.45.gate_proj", + "model.layers.40.mlp.experts.45.up_proj", + "model.layers.40.mlp.experts.46.down_proj", + "model.layers.40.mlp.experts.46.gate_proj", + "model.layers.40.mlp.experts.46.up_proj", + "model.layers.40.mlp.experts.47.down_proj", + "model.layers.40.mlp.experts.47.gate_proj", + "model.layers.40.mlp.experts.47.up_proj", + "model.layers.40.mlp.experts.48.down_proj", + "model.layers.40.mlp.experts.48.gate_proj", + "model.layers.40.mlp.experts.48.up_proj", + "model.layers.40.mlp.experts.49.down_proj", + "model.layers.40.mlp.experts.49.gate_proj", + "model.layers.40.mlp.experts.49.up_proj", + "model.layers.40.mlp.experts.5.down_proj", + "model.layers.40.mlp.experts.5.gate_proj", + "model.layers.40.mlp.experts.5.up_proj", + "model.layers.40.mlp.experts.50.down_proj", + "model.layers.40.mlp.experts.50.gate_proj", + "model.layers.40.mlp.experts.50.up_proj", + "model.layers.40.mlp.experts.51.down_proj", + "model.layers.40.mlp.experts.51.gate_proj", + "model.layers.40.mlp.experts.51.up_proj", + "model.layers.40.mlp.experts.52.down_proj", + "model.layers.40.mlp.experts.52.gate_proj", + "model.layers.40.mlp.experts.52.up_proj", + "model.layers.40.mlp.experts.53.down_proj", + "model.layers.40.mlp.experts.53.gate_proj", + "model.layers.40.mlp.experts.53.up_proj", + "model.layers.40.mlp.experts.54.down_proj", + "model.layers.40.mlp.experts.54.gate_proj", + "model.layers.40.mlp.experts.54.up_proj", + "model.layers.40.mlp.experts.55.down_proj", + "model.layers.40.mlp.experts.55.gate_proj", + "model.layers.40.mlp.experts.55.up_proj", + "model.layers.40.mlp.experts.56.down_proj", + "model.layers.40.mlp.experts.56.gate_proj", + "model.layers.40.mlp.experts.56.up_proj", + "model.layers.40.mlp.experts.57.down_proj", + "model.layers.40.mlp.experts.57.gate_proj", + "model.layers.40.mlp.experts.57.up_proj", + "model.layers.40.mlp.experts.58.down_proj", + "model.layers.40.mlp.experts.58.gate_proj", + "model.layers.40.mlp.experts.58.up_proj", + "model.layers.40.mlp.experts.59.down_proj", + "model.layers.40.mlp.experts.59.gate_proj", + "model.layers.40.mlp.experts.59.up_proj", + "model.layers.40.mlp.experts.6.down_proj", + "model.layers.40.mlp.experts.6.gate_proj", + "model.layers.40.mlp.experts.6.up_proj", + "model.layers.40.mlp.experts.60.down_proj", + "model.layers.40.mlp.experts.60.gate_proj", + "model.layers.40.mlp.experts.60.up_proj", + "model.layers.40.mlp.experts.61.down_proj", + "model.layers.40.mlp.experts.61.gate_proj", + "model.layers.40.mlp.experts.61.up_proj", + "model.layers.40.mlp.experts.62.down_proj", + "model.layers.40.mlp.experts.62.gate_proj", + "model.layers.40.mlp.experts.62.up_proj", + "model.layers.40.mlp.experts.63.down_proj", + "model.layers.40.mlp.experts.63.gate_proj", + "model.layers.40.mlp.experts.63.up_proj", + "model.layers.40.mlp.experts.64.down_proj", + "model.layers.40.mlp.experts.64.gate_proj", + "model.layers.40.mlp.experts.64.up_proj", + "model.layers.40.mlp.experts.65.down_proj", + "model.layers.40.mlp.experts.65.gate_proj", + "model.layers.40.mlp.experts.65.up_proj", + "model.layers.40.mlp.experts.66.down_proj", + "model.layers.40.mlp.experts.66.gate_proj", + "model.layers.40.mlp.experts.66.up_proj", + "model.layers.40.mlp.experts.67.down_proj", + "model.layers.40.mlp.experts.67.gate_proj", + "model.layers.40.mlp.experts.67.up_proj", + "model.layers.40.mlp.experts.68.down_proj", + "model.layers.40.mlp.experts.68.gate_proj", + "model.layers.40.mlp.experts.68.up_proj", + "model.layers.40.mlp.experts.69.down_proj", + "model.layers.40.mlp.experts.69.gate_proj", + "model.layers.40.mlp.experts.69.up_proj", + "model.layers.40.mlp.experts.7.down_proj", + "model.layers.40.mlp.experts.7.gate_proj", + "model.layers.40.mlp.experts.7.up_proj", + "model.layers.40.mlp.experts.70.down_proj", + "model.layers.40.mlp.experts.70.gate_proj", + "model.layers.40.mlp.experts.70.up_proj", + "model.layers.40.mlp.experts.71.down_proj", + "model.layers.40.mlp.experts.71.gate_proj", + "model.layers.40.mlp.experts.71.up_proj", + "model.layers.40.mlp.experts.72.down_proj", + "model.layers.40.mlp.experts.72.gate_proj", + "model.layers.40.mlp.experts.72.up_proj", + "model.layers.40.mlp.experts.73.down_proj", + "model.layers.40.mlp.experts.73.gate_proj", + "model.layers.40.mlp.experts.73.up_proj", + "model.layers.40.mlp.experts.74.down_proj", + "model.layers.40.mlp.experts.74.gate_proj", + "model.layers.40.mlp.experts.74.up_proj", + "model.layers.40.mlp.experts.75.down_proj", + "model.layers.40.mlp.experts.75.gate_proj", + "model.layers.40.mlp.experts.75.up_proj", + "model.layers.40.mlp.experts.76.down_proj", + "model.layers.40.mlp.experts.76.gate_proj", + "model.layers.40.mlp.experts.76.up_proj", + "model.layers.40.mlp.experts.77.down_proj", + "model.layers.40.mlp.experts.77.gate_proj", + "model.layers.40.mlp.experts.77.up_proj", + "model.layers.40.mlp.experts.78.down_proj", + "model.layers.40.mlp.experts.78.gate_proj", + "model.layers.40.mlp.experts.78.up_proj", + "model.layers.40.mlp.experts.79.down_proj", + "model.layers.40.mlp.experts.79.gate_proj", + "model.layers.40.mlp.experts.79.up_proj", + "model.layers.40.mlp.experts.8.down_proj", + "model.layers.40.mlp.experts.8.gate_proj", + "model.layers.40.mlp.experts.8.up_proj", + "model.layers.40.mlp.experts.80.down_proj", + "model.layers.40.mlp.experts.80.gate_proj", + "model.layers.40.mlp.experts.80.up_proj", + "model.layers.40.mlp.experts.81.down_proj", + "model.layers.40.mlp.experts.81.gate_proj", + "model.layers.40.mlp.experts.81.up_proj", + "model.layers.40.mlp.experts.82.down_proj", + "model.layers.40.mlp.experts.82.gate_proj", + "model.layers.40.mlp.experts.82.up_proj", + "model.layers.40.mlp.experts.83.down_proj", + "model.layers.40.mlp.experts.83.gate_proj", + "model.layers.40.mlp.experts.83.up_proj", + "model.layers.40.mlp.experts.84.down_proj", + "model.layers.40.mlp.experts.84.gate_proj", + "model.layers.40.mlp.experts.84.up_proj", + "model.layers.40.mlp.experts.85.down_proj", + "model.layers.40.mlp.experts.85.gate_proj", + "model.layers.40.mlp.experts.85.up_proj", + "model.layers.40.mlp.experts.86.down_proj", + "model.layers.40.mlp.experts.86.gate_proj", + "model.layers.40.mlp.experts.86.up_proj", + "model.layers.40.mlp.experts.87.down_proj", + "model.layers.40.mlp.experts.87.gate_proj", + "model.layers.40.mlp.experts.87.up_proj", + "model.layers.40.mlp.experts.88.down_proj", + "model.layers.40.mlp.experts.88.gate_proj", + "model.layers.40.mlp.experts.88.up_proj", + "model.layers.40.mlp.experts.89.down_proj", + "model.layers.40.mlp.experts.89.gate_proj", + "model.layers.40.mlp.experts.89.up_proj", + "model.layers.40.mlp.experts.9.down_proj", + "model.layers.40.mlp.experts.9.gate_proj", + "model.layers.40.mlp.experts.9.up_proj", + "model.layers.40.mlp.experts.90.down_proj", + "model.layers.40.mlp.experts.90.gate_proj", + "model.layers.40.mlp.experts.90.up_proj", + "model.layers.40.mlp.experts.91.down_proj", + "model.layers.40.mlp.experts.91.gate_proj", + "model.layers.40.mlp.experts.91.up_proj", + "model.layers.40.mlp.experts.92.down_proj", + "model.layers.40.mlp.experts.92.gate_proj", + "model.layers.40.mlp.experts.92.up_proj", + "model.layers.40.mlp.experts.93.down_proj", + "model.layers.40.mlp.experts.93.gate_proj", + "model.layers.40.mlp.experts.93.up_proj", + "model.layers.40.mlp.experts.94.down_proj", + "model.layers.40.mlp.experts.94.gate_proj", + "model.layers.40.mlp.experts.94.up_proj", + "model.layers.40.mlp.experts.95.down_proj", + "model.layers.40.mlp.experts.95.gate_proj", + "model.layers.40.mlp.experts.95.up_proj", + "model.layers.40.mlp.experts.96.down_proj", + "model.layers.40.mlp.experts.96.gate_proj", + "model.layers.40.mlp.experts.96.up_proj", + "model.layers.40.mlp.experts.97.down_proj", + "model.layers.40.mlp.experts.97.gate_proj", + "model.layers.40.mlp.experts.97.up_proj", + "model.layers.40.mlp.experts.98.down_proj", + "model.layers.40.mlp.experts.98.gate_proj", + "model.layers.40.mlp.experts.98.up_proj", + "model.layers.40.mlp.experts.99.down_proj", + "model.layers.40.mlp.experts.99.gate_proj", + "model.layers.40.mlp.experts.99.up_proj", + "model.layers.41.mlp.experts.0.down_proj", + "model.layers.41.mlp.experts.0.gate_proj", + "model.layers.41.mlp.experts.0.up_proj", + "model.layers.41.mlp.experts.1.down_proj", + "model.layers.41.mlp.experts.1.gate_proj", + "model.layers.41.mlp.experts.1.up_proj", + "model.layers.41.mlp.experts.10.down_proj", + "model.layers.41.mlp.experts.10.gate_proj", + "model.layers.41.mlp.experts.10.up_proj", + "model.layers.41.mlp.experts.100.down_proj", + "model.layers.41.mlp.experts.100.gate_proj", + "model.layers.41.mlp.experts.100.up_proj", + "model.layers.41.mlp.experts.101.down_proj", + "model.layers.41.mlp.experts.101.gate_proj", + "model.layers.41.mlp.experts.101.up_proj", + "model.layers.41.mlp.experts.102.down_proj", + "model.layers.41.mlp.experts.102.gate_proj", + "model.layers.41.mlp.experts.102.up_proj", + "model.layers.41.mlp.experts.103.down_proj", + "model.layers.41.mlp.experts.103.gate_proj", + "model.layers.41.mlp.experts.103.up_proj", + "model.layers.41.mlp.experts.104.down_proj", + "model.layers.41.mlp.experts.104.gate_proj", + "model.layers.41.mlp.experts.104.up_proj", + "model.layers.41.mlp.experts.105.down_proj", + "model.layers.41.mlp.experts.105.gate_proj", + "model.layers.41.mlp.experts.105.up_proj", + "model.layers.41.mlp.experts.106.down_proj", + "model.layers.41.mlp.experts.106.gate_proj", + "model.layers.41.mlp.experts.106.up_proj", + "model.layers.41.mlp.experts.107.down_proj", + "model.layers.41.mlp.experts.107.gate_proj", + "model.layers.41.mlp.experts.107.up_proj", + "model.layers.41.mlp.experts.108.down_proj", + "model.layers.41.mlp.experts.108.gate_proj", + "model.layers.41.mlp.experts.108.up_proj", + "model.layers.41.mlp.experts.109.down_proj", + "model.layers.41.mlp.experts.109.gate_proj", + "model.layers.41.mlp.experts.109.up_proj", + "model.layers.41.mlp.experts.11.down_proj", + "model.layers.41.mlp.experts.11.gate_proj", + "model.layers.41.mlp.experts.11.up_proj", + "model.layers.41.mlp.experts.110.down_proj", + "model.layers.41.mlp.experts.110.gate_proj", + "model.layers.41.mlp.experts.110.up_proj", + "model.layers.41.mlp.experts.111.down_proj", + "model.layers.41.mlp.experts.111.gate_proj", + "model.layers.41.mlp.experts.111.up_proj", + "model.layers.41.mlp.experts.112.down_proj", + "model.layers.41.mlp.experts.112.gate_proj", + "model.layers.41.mlp.experts.112.up_proj", + "model.layers.41.mlp.experts.113.down_proj", + "model.layers.41.mlp.experts.113.gate_proj", + "model.layers.41.mlp.experts.113.up_proj", + "model.layers.41.mlp.experts.114.down_proj", + "model.layers.41.mlp.experts.114.gate_proj", + "model.layers.41.mlp.experts.114.up_proj", + "model.layers.41.mlp.experts.115.down_proj", + "model.layers.41.mlp.experts.115.gate_proj", + "model.layers.41.mlp.experts.115.up_proj", + "model.layers.41.mlp.experts.116.down_proj", + "model.layers.41.mlp.experts.116.gate_proj", + "model.layers.41.mlp.experts.116.up_proj", + "model.layers.41.mlp.experts.117.down_proj", + "model.layers.41.mlp.experts.117.gate_proj", + "model.layers.41.mlp.experts.117.up_proj", + "model.layers.41.mlp.experts.118.down_proj", + "model.layers.41.mlp.experts.118.gate_proj", + "model.layers.41.mlp.experts.118.up_proj", + "model.layers.41.mlp.experts.119.down_proj", + "model.layers.41.mlp.experts.119.gate_proj", + "model.layers.41.mlp.experts.119.up_proj", + "model.layers.41.mlp.experts.12.down_proj", + "model.layers.41.mlp.experts.12.gate_proj", + "model.layers.41.mlp.experts.12.up_proj", + "model.layers.41.mlp.experts.120.down_proj", + "model.layers.41.mlp.experts.120.gate_proj", + "model.layers.41.mlp.experts.120.up_proj", + "model.layers.41.mlp.experts.121.down_proj", + "model.layers.41.mlp.experts.121.gate_proj", + "model.layers.41.mlp.experts.121.up_proj", + "model.layers.41.mlp.experts.122.down_proj", + "model.layers.41.mlp.experts.122.gate_proj", + "model.layers.41.mlp.experts.122.up_proj", + "model.layers.41.mlp.experts.123.down_proj", + "model.layers.41.mlp.experts.123.gate_proj", + "model.layers.41.mlp.experts.123.up_proj", + "model.layers.41.mlp.experts.124.down_proj", + "model.layers.41.mlp.experts.124.gate_proj", + "model.layers.41.mlp.experts.124.up_proj", + "model.layers.41.mlp.experts.125.down_proj", + "model.layers.41.mlp.experts.125.gate_proj", + "model.layers.41.mlp.experts.125.up_proj", + "model.layers.41.mlp.experts.126.down_proj", + "model.layers.41.mlp.experts.126.gate_proj", + "model.layers.41.mlp.experts.126.up_proj", + "model.layers.41.mlp.experts.127.down_proj", + "model.layers.41.mlp.experts.127.gate_proj", + "model.layers.41.mlp.experts.127.up_proj", + "model.layers.41.mlp.experts.128.down_proj", + "model.layers.41.mlp.experts.128.gate_proj", + "model.layers.41.mlp.experts.128.up_proj", + "model.layers.41.mlp.experts.129.down_proj", + "model.layers.41.mlp.experts.129.gate_proj", + "model.layers.41.mlp.experts.129.up_proj", + "model.layers.41.mlp.experts.13.down_proj", + "model.layers.41.mlp.experts.13.gate_proj", + "model.layers.41.mlp.experts.13.up_proj", + "model.layers.41.mlp.experts.130.down_proj", + "model.layers.41.mlp.experts.130.gate_proj", + "model.layers.41.mlp.experts.130.up_proj", + "model.layers.41.mlp.experts.131.down_proj", + "model.layers.41.mlp.experts.131.gate_proj", + "model.layers.41.mlp.experts.131.up_proj", + "model.layers.41.mlp.experts.132.down_proj", + "model.layers.41.mlp.experts.132.gate_proj", + "model.layers.41.mlp.experts.132.up_proj", + "model.layers.41.mlp.experts.133.down_proj", + "model.layers.41.mlp.experts.133.gate_proj", + "model.layers.41.mlp.experts.133.up_proj", + "model.layers.41.mlp.experts.134.down_proj", + "model.layers.41.mlp.experts.134.gate_proj", + "model.layers.41.mlp.experts.134.up_proj", + "model.layers.41.mlp.experts.135.down_proj", + "model.layers.41.mlp.experts.135.gate_proj", + "model.layers.41.mlp.experts.135.up_proj", + "model.layers.41.mlp.experts.136.down_proj", + "model.layers.41.mlp.experts.136.gate_proj", + "model.layers.41.mlp.experts.136.up_proj", + "model.layers.41.mlp.experts.137.down_proj", + "model.layers.41.mlp.experts.137.gate_proj", + "model.layers.41.mlp.experts.137.up_proj", + "model.layers.41.mlp.experts.138.down_proj", + "model.layers.41.mlp.experts.138.gate_proj", + "model.layers.41.mlp.experts.138.up_proj", + "model.layers.41.mlp.experts.139.down_proj", + "model.layers.41.mlp.experts.139.gate_proj", + "model.layers.41.mlp.experts.139.up_proj", + "model.layers.41.mlp.experts.14.down_proj", + "model.layers.41.mlp.experts.14.gate_proj", + "model.layers.41.mlp.experts.14.up_proj", + "model.layers.41.mlp.experts.140.down_proj", + "model.layers.41.mlp.experts.140.gate_proj", + "model.layers.41.mlp.experts.140.up_proj", + "model.layers.41.mlp.experts.141.down_proj", + "model.layers.41.mlp.experts.141.gate_proj", + "model.layers.41.mlp.experts.141.up_proj", + "model.layers.41.mlp.experts.142.down_proj", + "model.layers.41.mlp.experts.142.gate_proj", + "model.layers.41.mlp.experts.142.up_proj", + "model.layers.41.mlp.experts.143.down_proj", + "model.layers.41.mlp.experts.143.gate_proj", + "model.layers.41.mlp.experts.143.up_proj", + "model.layers.41.mlp.experts.144.down_proj", + "model.layers.41.mlp.experts.144.gate_proj", + "model.layers.41.mlp.experts.144.up_proj", + "model.layers.41.mlp.experts.145.down_proj", + "model.layers.41.mlp.experts.145.gate_proj", + "model.layers.41.mlp.experts.145.up_proj", + "model.layers.41.mlp.experts.146.down_proj", + "model.layers.41.mlp.experts.146.gate_proj", + "model.layers.41.mlp.experts.146.up_proj", + "model.layers.41.mlp.experts.147.down_proj", + "model.layers.41.mlp.experts.147.gate_proj", + "model.layers.41.mlp.experts.147.up_proj", + "model.layers.41.mlp.experts.148.down_proj", + "model.layers.41.mlp.experts.148.gate_proj", + "model.layers.41.mlp.experts.148.up_proj", + "model.layers.41.mlp.experts.149.down_proj", + "model.layers.41.mlp.experts.149.gate_proj", + "model.layers.41.mlp.experts.149.up_proj", + "model.layers.41.mlp.experts.15.down_proj", + "model.layers.41.mlp.experts.15.gate_proj", + "model.layers.41.mlp.experts.15.up_proj", + "model.layers.41.mlp.experts.150.down_proj", + "model.layers.41.mlp.experts.150.gate_proj", + "model.layers.41.mlp.experts.150.up_proj", + "model.layers.41.mlp.experts.151.down_proj", + "model.layers.41.mlp.experts.151.gate_proj", + "model.layers.41.mlp.experts.151.up_proj", + "model.layers.41.mlp.experts.152.down_proj", + "model.layers.41.mlp.experts.152.gate_proj", + "model.layers.41.mlp.experts.152.up_proj", + "model.layers.41.mlp.experts.153.down_proj", + "model.layers.41.mlp.experts.153.gate_proj", + "model.layers.41.mlp.experts.153.up_proj", + "model.layers.41.mlp.experts.154.down_proj", + "model.layers.41.mlp.experts.154.gate_proj", + "model.layers.41.mlp.experts.154.up_proj", + "model.layers.41.mlp.experts.155.down_proj", + "model.layers.41.mlp.experts.155.gate_proj", + "model.layers.41.mlp.experts.155.up_proj", + "model.layers.41.mlp.experts.156.down_proj", + "model.layers.41.mlp.experts.156.gate_proj", + "model.layers.41.mlp.experts.156.up_proj", + "model.layers.41.mlp.experts.157.down_proj", + "model.layers.41.mlp.experts.157.gate_proj", + "model.layers.41.mlp.experts.157.up_proj", + "model.layers.41.mlp.experts.158.down_proj", + "model.layers.41.mlp.experts.158.gate_proj", + "model.layers.41.mlp.experts.158.up_proj", + "model.layers.41.mlp.experts.159.down_proj", + "model.layers.41.mlp.experts.159.gate_proj", + "model.layers.41.mlp.experts.159.up_proj", + "model.layers.41.mlp.experts.16.down_proj", + "model.layers.41.mlp.experts.16.gate_proj", + "model.layers.41.mlp.experts.16.up_proj", + "model.layers.41.mlp.experts.160.down_proj", + "model.layers.41.mlp.experts.160.gate_proj", + "model.layers.41.mlp.experts.160.up_proj", + "model.layers.41.mlp.experts.161.down_proj", + "model.layers.41.mlp.experts.161.gate_proj", + "model.layers.41.mlp.experts.161.up_proj", + "model.layers.41.mlp.experts.162.down_proj", + "model.layers.41.mlp.experts.162.gate_proj", + "model.layers.41.mlp.experts.162.up_proj", + "model.layers.41.mlp.experts.163.down_proj", + "model.layers.41.mlp.experts.163.gate_proj", + "model.layers.41.mlp.experts.163.up_proj", + "model.layers.41.mlp.experts.164.down_proj", + "model.layers.41.mlp.experts.164.gate_proj", + "model.layers.41.mlp.experts.164.up_proj", + "model.layers.41.mlp.experts.165.down_proj", + "model.layers.41.mlp.experts.165.gate_proj", + "model.layers.41.mlp.experts.165.up_proj", + "model.layers.41.mlp.experts.166.down_proj", + "model.layers.41.mlp.experts.166.gate_proj", + "model.layers.41.mlp.experts.166.up_proj", + "model.layers.41.mlp.experts.167.down_proj", + "model.layers.41.mlp.experts.167.gate_proj", + "model.layers.41.mlp.experts.167.up_proj", + "model.layers.41.mlp.experts.168.down_proj", + "model.layers.41.mlp.experts.168.gate_proj", + "model.layers.41.mlp.experts.168.up_proj", + "model.layers.41.mlp.experts.169.down_proj", + "model.layers.41.mlp.experts.169.gate_proj", + "model.layers.41.mlp.experts.169.up_proj", + "model.layers.41.mlp.experts.17.down_proj", + "model.layers.41.mlp.experts.17.gate_proj", + "model.layers.41.mlp.experts.17.up_proj", + "model.layers.41.mlp.experts.170.down_proj", + "model.layers.41.mlp.experts.170.gate_proj", + "model.layers.41.mlp.experts.170.up_proj", + "model.layers.41.mlp.experts.171.down_proj", + "model.layers.41.mlp.experts.171.gate_proj", + "model.layers.41.mlp.experts.171.up_proj", + "model.layers.41.mlp.experts.172.down_proj", + "model.layers.41.mlp.experts.172.gate_proj", + "model.layers.41.mlp.experts.172.up_proj", + "model.layers.41.mlp.experts.173.down_proj", + "model.layers.41.mlp.experts.173.gate_proj", + "model.layers.41.mlp.experts.173.up_proj", + "model.layers.41.mlp.experts.174.down_proj", + "model.layers.41.mlp.experts.174.gate_proj", + "model.layers.41.mlp.experts.174.up_proj", + "model.layers.41.mlp.experts.175.down_proj", + "model.layers.41.mlp.experts.175.gate_proj", + "model.layers.41.mlp.experts.175.up_proj", + "model.layers.41.mlp.experts.176.down_proj", + "model.layers.41.mlp.experts.176.gate_proj", + "model.layers.41.mlp.experts.176.up_proj", + "model.layers.41.mlp.experts.177.down_proj", + "model.layers.41.mlp.experts.177.gate_proj", + "model.layers.41.mlp.experts.177.up_proj", + "model.layers.41.mlp.experts.178.down_proj", + "model.layers.41.mlp.experts.178.gate_proj", + "model.layers.41.mlp.experts.178.up_proj", + "model.layers.41.mlp.experts.179.down_proj", + "model.layers.41.mlp.experts.179.gate_proj", + "model.layers.41.mlp.experts.179.up_proj", + "model.layers.41.mlp.experts.18.down_proj", + "model.layers.41.mlp.experts.18.gate_proj", + "model.layers.41.mlp.experts.18.up_proj", + "model.layers.41.mlp.experts.180.down_proj", + "model.layers.41.mlp.experts.180.gate_proj", + "model.layers.41.mlp.experts.180.up_proj", + "model.layers.41.mlp.experts.181.down_proj", + "model.layers.41.mlp.experts.181.gate_proj", + "model.layers.41.mlp.experts.181.up_proj", + "model.layers.41.mlp.experts.182.down_proj", + "model.layers.41.mlp.experts.182.gate_proj", + "model.layers.41.mlp.experts.182.up_proj", + "model.layers.41.mlp.experts.183.down_proj", + "model.layers.41.mlp.experts.183.gate_proj", + "model.layers.41.mlp.experts.183.up_proj", + "model.layers.41.mlp.experts.184.down_proj", + "model.layers.41.mlp.experts.184.gate_proj", + "model.layers.41.mlp.experts.184.up_proj", + "model.layers.41.mlp.experts.185.down_proj", + "model.layers.41.mlp.experts.185.gate_proj", + "model.layers.41.mlp.experts.185.up_proj", + "model.layers.41.mlp.experts.186.down_proj", + "model.layers.41.mlp.experts.186.gate_proj", + "model.layers.41.mlp.experts.186.up_proj", + "model.layers.41.mlp.experts.187.down_proj", + "model.layers.41.mlp.experts.187.gate_proj", + "model.layers.41.mlp.experts.187.up_proj", + "model.layers.41.mlp.experts.188.down_proj", + "model.layers.41.mlp.experts.188.gate_proj", + "model.layers.41.mlp.experts.188.up_proj", + "model.layers.41.mlp.experts.189.down_proj", + "model.layers.41.mlp.experts.189.gate_proj", + "model.layers.41.mlp.experts.189.up_proj", + "model.layers.41.mlp.experts.19.down_proj", + "model.layers.41.mlp.experts.19.gate_proj", + "model.layers.41.mlp.experts.19.up_proj", + "model.layers.41.mlp.experts.190.down_proj", + "model.layers.41.mlp.experts.190.gate_proj", + "model.layers.41.mlp.experts.190.up_proj", + "model.layers.41.mlp.experts.191.down_proj", + "model.layers.41.mlp.experts.191.gate_proj", + "model.layers.41.mlp.experts.191.up_proj", + "model.layers.41.mlp.experts.192.down_proj", + "model.layers.41.mlp.experts.192.gate_proj", + "model.layers.41.mlp.experts.192.up_proj", + "model.layers.41.mlp.experts.193.down_proj", + "model.layers.41.mlp.experts.193.gate_proj", + "model.layers.41.mlp.experts.193.up_proj", + "model.layers.41.mlp.experts.194.down_proj", + "model.layers.41.mlp.experts.194.gate_proj", + "model.layers.41.mlp.experts.194.up_proj", + "model.layers.41.mlp.experts.195.down_proj", + "model.layers.41.mlp.experts.195.gate_proj", + "model.layers.41.mlp.experts.195.up_proj", + "model.layers.41.mlp.experts.196.down_proj", + "model.layers.41.mlp.experts.196.gate_proj", + "model.layers.41.mlp.experts.196.up_proj", + "model.layers.41.mlp.experts.197.down_proj", + "model.layers.41.mlp.experts.197.gate_proj", + "model.layers.41.mlp.experts.197.up_proj", + "model.layers.41.mlp.experts.198.down_proj", + "model.layers.41.mlp.experts.198.gate_proj", + "model.layers.41.mlp.experts.198.up_proj", + "model.layers.41.mlp.experts.199.down_proj", + "model.layers.41.mlp.experts.199.gate_proj", + "model.layers.41.mlp.experts.199.up_proj", + "model.layers.41.mlp.experts.2.down_proj", + "model.layers.41.mlp.experts.2.gate_proj", + "model.layers.41.mlp.experts.2.up_proj", + "model.layers.41.mlp.experts.20.down_proj", + "model.layers.41.mlp.experts.20.gate_proj", + "model.layers.41.mlp.experts.20.up_proj", + "model.layers.41.mlp.experts.200.down_proj", + "model.layers.41.mlp.experts.200.gate_proj", + "model.layers.41.mlp.experts.200.up_proj", + "model.layers.41.mlp.experts.201.down_proj", + "model.layers.41.mlp.experts.201.gate_proj", + "model.layers.41.mlp.experts.201.up_proj", + "model.layers.41.mlp.experts.202.down_proj", + "model.layers.41.mlp.experts.202.gate_proj", + "model.layers.41.mlp.experts.202.up_proj", + "model.layers.41.mlp.experts.203.down_proj", + "model.layers.41.mlp.experts.203.gate_proj", + "model.layers.41.mlp.experts.203.up_proj", + "model.layers.41.mlp.experts.204.down_proj", + "model.layers.41.mlp.experts.204.gate_proj", + "model.layers.41.mlp.experts.204.up_proj", + "model.layers.41.mlp.experts.205.down_proj", + "model.layers.41.mlp.experts.205.gate_proj", + "model.layers.41.mlp.experts.205.up_proj", + "model.layers.41.mlp.experts.206.down_proj", + "model.layers.41.mlp.experts.206.gate_proj", + "model.layers.41.mlp.experts.206.up_proj", + "model.layers.41.mlp.experts.207.down_proj", + "model.layers.41.mlp.experts.207.gate_proj", + "model.layers.41.mlp.experts.207.up_proj", + "model.layers.41.mlp.experts.208.down_proj", + "model.layers.41.mlp.experts.208.gate_proj", + "model.layers.41.mlp.experts.208.up_proj", + "model.layers.41.mlp.experts.209.down_proj", + "model.layers.41.mlp.experts.209.gate_proj", + "model.layers.41.mlp.experts.209.up_proj", + "model.layers.41.mlp.experts.21.down_proj", + "model.layers.41.mlp.experts.21.gate_proj", + "model.layers.41.mlp.experts.21.up_proj", + "model.layers.41.mlp.experts.210.down_proj", + "model.layers.41.mlp.experts.210.gate_proj", + "model.layers.41.mlp.experts.210.up_proj", + "model.layers.41.mlp.experts.211.down_proj", + "model.layers.41.mlp.experts.211.gate_proj", + "model.layers.41.mlp.experts.211.up_proj", + "model.layers.41.mlp.experts.212.down_proj", + "model.layers.41.mlp.experts.212.gate_proj", + "model.layers.41.mlp.experts.212.up_proj", + "model.layers.41.mlp.experts.213.down_proj", + "model.layers.41.mlp.experts.213.gate_proj", + "model.layers.41.mlp.experts.213.up_proj", + "model.layers.41.mlp.experts.214.down_proj", + "model.layers.41.mlp.experts.214.gate_proj", + "model.layers.41.mlp.experts.214.up_proj", + "model.layers.41.mlp.experts.215.down_proj", + "model.layers.41.mlp.experts.215.gate_proj", + "model.layers.41.mlp.experts.215.up_proj", + "model.layers.41.mlp.experts.216.down_proj", + "model.layers.41.mlp.experts.216.gate_proj", + "model.layers.41.mlp.experts.216.up_proj", + "model.layers.41.mlp.experts.217.down_proj", + "model.layers.41.mlp.experts.217.gate_proj", + "model.layers.41.mlp.experts.217.up_proj", + "model.layers.41.mlp.experts.218.down_proj", + "model.layers.41.mlp.experts.218.gate_proj", + "model.layers.41.mlp.experts.218.up_proj", + "model.layers.41.mlp.experts.219.down_proj", + "model.layers.41.mlp.experts.219.gate_proj", + "model.layers.41.mlp.experts.219.up_proj", + "model.layers.41.mlp.experts.22.down_proj", + "model.layers.41.mlp.experts.22.gate_proj", + "model.layers.41.mlp.experts.22.up_proj", + "model.layers.41.mlp.experts.220.down_proj", + "model.layers.41.mlp.experts.220.gate_proj", + "model.layers.41.mlp.experts.220.up_proj", + "model.layers.41.mlp.experts.221.down_proj", + "model.layers.41.mlp.experts.221.gate_proj", + "model.layers.41.mlp.experts.221.up_proj", + "model.layers.41.mlp.experts.222.down_proj", + "model.layers.41.mlp.experts.222.gate_proj", + "model.layers.41.mlp.experts.222.up_proj", + "model.layers.41.mlp.experts.223.down_proj", + "model.layers.41.mlp.experts.223.gate_proj", + "model.layers.41.mlp.experts.223.up_proj", + "model.layers.41.mlp.experts.224.down_proj", + "model.layers.41.mlp.experts.224.gate_proj", + "model.layers.41.mlp.experts.224.up_proj", + "model.layers.41.mlp.experts.225.down_proj", + "model.layers.41.mlp.experts.225.gate_proj", + "model.layers.41.mlp.experts.225.up_proj", + "model.layers.41.mlp.experts.226.down_proj", + "model.layers.41.mlp.experts.226.gate_proj", + "model.layers.41.mlp.experts.226.up_proj", + "model.layers.41.mlp.experts.227.down_proj", + "model.layers.41.mlp.experts.227.gate_proj", + "model.layers.41.mlp.experts.227.up_proj", + "model.layers.41.mlp.experts.228.down_proj", + "model.layers.41.mlp.experts.228.gate_proj", + "model.layers.41.mlp.experts.228.up_proj", + "model.layers.41.mlp.experts.229.down_proj", + "model.layers.41.mlp.experts.229.gate_proj", + "model.layers.41.mlp.experts.229.up_proj", + "model.layers.41.mlp.experts.23.down_proj", + "model.layers.41.mlp.experts.23.gate_proj", + "model.layers.41.mlp.experts.23.up_proj", + "model.layers.41.mlp.experts.230.down_proj", + "model.layers.41.mlp.experts.230.gate_proj", + "model.layers.41.mlp.experts.230.up_proj", + "model.layers.41.mlp.experts.231.down_proj", + "model.layers.41.mlp.experts.231.gate_proj", + "model.layers.41.mlp.experts.231.up_proj", + "model.layers.41.mlp.experts.232.down_proj", + "model.layers.41.mlp.experts.232.gate_proj", + "model.layers.41.mlp.experts.232.up_proj", + "model.layers.41.mlp.experts.233.down_proj", + "model.layers.41.mlp.experts.233.gate_proj", + "model.layers.41.mlp.experts.233.up_proj", + "model.layers.41.mlp.experts.234.down_proj", + "model.layers.41.mlp.experts.234.gate_proj", + "model.layers.41.mlp.experts.234.up_proj", + "model.layers.41.mlp.experts.235.down_proj", + "model.layers.41.mlp.experts.235.gate_proj", + "model.layers.41.mlp.experts.235.up_proj", + "model.layers.41.mlp.experts.236.down_proj", + "model.layers.41.mlp.experts.236.gate_proj", + "model.layers.41.mlp.experts.236.up_proj", + "model.layers.41.mlp.experts.237.down_proj", + "model.layers.41.mlp.experts.237.gate_proj", + "model.layers.41.mlp.experts.237.up_proj", + "model.layers.41.mlp.experts.238.down_proj", + "model.layers.41.mlp.experts.238.gate_proj", + "model.layers.41.mlp.experts.238.up_proj", + "model.layers.41.mlp.experts.239.down_proj", + "model.layers.41.mlp.experts.239.gate_proj", + "model.layers.41.mlp.experts.239.up_proj", + "model.layers.41.mlp.experts.24.down_proj", + "model.layers.41.mlp.experts.24.gate_proj", + "model.layers.41.mlp.experts.24.up_proj", + "model.layers.41.mlp.experts.240.down_proj", + "model.layers.41.mlp.experts.240.gate_proj", + "model.layers.41.mlp.experts.240.up_proj", + "model.layers.41.mlp.experts.241.down_proj", + "model.layers.41.mlp.experts.241.gate_proj", + "model.layers.41.mlp.experts.241.up_proj", + "model.layers.41.mlp.experts.242.down_proj", + "model.layers.41.mlp.experts.242.gate_proj", + "model.layers.41.mlp.experts.242.up_proj", + "model.layers.41.mlp.experts.243.down_proj", + "model.layers.41.mlp.experts.243.gate_proj", + "model.layers.41.mlp.experts.243.up_proj", + "model.layers.41.mlp.experts.244.down_proj", + "model.layers.41.mlp.experts.244.gate_proj", + "model.layers.41.mlp.experts.244.up_proj", + "model.layers.41.mlp.experts.245.down_proj", + "model.layers.41.mlp.experts.245.gate_proj", + "model.layers.41.mlp.experts.245.up_proj", + "model.layers.41.mlp.experts.246.down_proj", + "model.layers.41.mlp.experts.246.gate_proj", + "model.layers.41.mlp.experts.246.up_proj", + "model.layers.41.mlp.experts.247.down_proj", + "model.layers.41.mlp.experts.247.gate_proj", + "model.layers.41.mlp.experts.247.up_proj", + "model.layers.41.mlp.experts.248.down_proj", + "model.layers.41.mlp.experts.248.gate_proj", + "model.layers.41.mlp.experts.248.up_proj", + "model.layers.41.mlp.experts.249.down_proj", + "model.layers.41.mlp.experts.249.gate_proj", + "model.layers.41.mlp.experts.249.up_proj", + "model.layers.41.mlp.experts.25.down_proj", + "model.layers.41.mlp.experts.25.gate_proj", + "model.layers.41.mlp.experts.25.up_proj", + "model.layers.41.mlp.experts.250.down_proj", + "model.layers.41.mlp.experts.250.gate_proj", + "model.layers.41.mlp.experts.250.up_proj", + "model.layers.41.mlp.experts.251.down_proj", + "model.layers.41.mlp.experts.251.gate_proj", + "model.layers.41.mlp.experts.251.up_proj", + "model.layers.41.mlp.experts.252.down_proj", + "model.layers.41.mlp.experts.252.gate_proj", + "model.layers.41.mlp.experts.252.up_proj", + "model.layers.41.mlp.experts.253.down_proj", + "model.layers.41.mlp.experts.253.gate_proj", + "model.layers.41.mlp.experts.253.up_proj", + "model.layers.41.mlp.experts.254.down_proj", + "model.layers.41.mlp.experts.254.gate_proj", + "model.layers.41.mlp.experts.254.up_proj", + "model.layers.41.mlp.experts.255.down_proj", + "model.layers.41.mlp.experts.255.gate_proj", + "model.layers.41.mlp.experts.255.up_proj", + "model.layers.41.mlp.experts.26.down_proj", + "model.layers.41.mlp.experts.26.gate_proj", + "model.layers.41.mlp.experts.26.up_proj", + "model.layers.41.mlp.experts.27.down_proj", + "model.layers.41.mlp.experts.27.gate_proj", + "model.layers.41.mlp.experts.27.up_proj", + "model.layers.41.mlp.experts.28.down_proj", + "model.layers.41.mlp.experts.28.gate_proj", + "model.layers.41.mlp.experts.28.up_proj", + "model.layers.41.mlp.experts.29.down_proj", + "model.layers.41.mlp.experts.29.gate_proj", + "model.layers.41.mlp.experts.29.up_proj", + "model.layers.41.mlp.experts.3.down_proj", + "model.layers.41.mlp.experts.3.gate_proj", + "model.layers.41.mlp.experts.3.up_proj", + "model.layers.41.mlp.experts.30.down_proj", + "model.layers.41.mlp.experts.30.gate_proj", + "model.layers.41.mlp.experts.30.up_proj", + "model.layers.41.mlp.experts.31.down_proj", + "model.layers.41.mlp.experts.31.gate_proj", + "model.layers.41.mlp.experts.31.up_proj", + "model.layers.41.mlp.experts.32.down_proj", + "model.layers.41.mlp.experts.32.gate_proj", + "model.layers.41.mlp.experts.32.up_proj", + "model.layers.41.mlp.experts.33.down_proj", + "model.layers.41.mlp.experts.33.gate_proj", + "model.layers.41.mlp.experts.33.up_proj", + "model.layers.41.mlp.experts.34.down_proj", + "model.layers.41.mlp.experts.34.gate_proj", + "model.layers.41.mlp.experts.34.up_proj", + "model.layers.41.mlp.experts.35.down_proj", + "model.layers.41.mlp.experts.35.gate_proj", + "model.layers.41.mlp.experts.35.up_proj", + "model.layers.41.mlp.experts.36.down_proj", + "model.layers.41.mlp.experts.36.gate_proj", + "model.layers.41.mlp.experts.36.up_proj", + "model.layers.41.mlp.experts.37.down_proj", + "model.layers.41.mlp.experts.37.gate_proj", + "model.layers.41.mlp.experts.37.up_proj", + "model.layers.41.mlp.experts.38.down_proj", + "model.layers.41.mlp.experts.38.gate_proj", + "model.layers.41.mlp.experts.38.up_proj", + "model.layers.41.mlp.experts.39.down_proj", + "model.layers.41.mlp.experts.39.gate_proj", + "model.layers.41.mlp.experts.39.up_proj", + "model.layers.41.mlp.experts.4.down_proj", + "model.layers.41.mlp.experts.4.gate_proj", + "model.layers.41.mlp.experts.4.up_proj", + "model.layers.41.mlp.experts.40.down_proj", + "model.layers.41.mlp.experts.40.gate_proj", + "model.layers.41.mlp.experts.40.up_proj", + "model.layers.41.mlp.experts.41.down_proj", + "model.layers.41.mlp.experts.41.gate_proj", + "model.layers.41.mlp.experts.41.up_proj", + "model.layers.41.mlp.experts.42.down_proj", + "model.layers.41.mlp.experts.42.gate_proj", + "model.layers.41.mlp.experts.42.up_proj", + "model.layers.41.mlp.experts.43.down_proj", + "model.layers.41.mlp.experts.43.gate_proj", + "model.layers.41.mlp.experts.43.up_proj", + "model.layers.41.mlp.experts.44.down_proj", + "model.layers.41.mlp.experts.44.gate_proj", + "model.layers.41.mlp.experts.44.up_proj", + "model.layers.41.mlp.experts.45.down_proj", + "model.layers.41.mlp.experts.45.gate_proj", + "model.layers.41.mlp.experts.45.up_proj", + "model.layers.41.mlp.experts.46.down_proj", + "model.layers.41.mlp.experts.46.gate_proj", + "model.layers.41.mlp.experts.46.up_proj", + "model.layers.41.mlp.experts.47.down_proj", + "model.layers.41.mlp.experts.47.gate_proj", + "model.layers.41.mlp.experts.47.up_proj", + "model.layers.41.mlp.experts.48.down_proj", + "model.layers.41.mlp.experts.48.gate_proj", + "model.layers.41.mlp.experts.48.up_proj", + "model.layers.41.mlp.experts.49.down_proj", + "model.layers.41.mlp.experts.49.gate_proj", + "model.layers.41.mlp.experts.49.up_proj", + "model.layers.41.mlp.experts.5.down_proj", + "model.layers.41.mlp.experts.5.gate_proj", + "model.layers.41.mlp.experts.5.up_proj", + "model.layers.41.mlp.experts.50.down_proj", + "model.layers.41.mlp.experts.50.gate_proj", + "model.layers.41.mlp.experts.50.up_proj", + "model.layers.41.mlp.experts.51.down_proj", + "model.layers.41.mlp.experts.51.gate_proj", + "model.layers.41.mlp.experts.51.up_proj", + "model.layers.41.mlp.experts.52.down_proj", + "model.layers.41.mlp.experts.52.gate_proj", + "model.layers.41.mlp.experts.52.up_proj", + "model.layers.41.mlp.experts.53.down_proj", + "model.layers.41.mlp.experts.53.gate_proj", + "model.layers.41.mlp.experts.53.up_proj", + "model.layers.41.mlp.experts.54.down_proj", + "model.layers.41.mlp.experts.54.gate_proj", + "model.layers.41.mlp.experts.54.up_proj", + "model.layers.41.mlp.experts.55.down_proj", + "model.layers.41.mlp.experts.55.gate_proj", + "model.layers.41.mlp.experts.55.up_proj", + "model.layers.41.mlp.experts.56.down_proj", + "model.layers.41.mlp.experts.56.gate_proj", + "model.layers.41.mlp.experts.56.up_proj", + "model.layers.41.mlp.experts.57.down_proj", + "model.layers.41.mlp.experts.57.gate_proj", + "model.layers.41.mlp.experts.57.up_proj", + "model.layers.41.mlp.experts.58.down_proj", + "model.layers.41.mlp.experts.58.gate_proj", + "model.layers.41.mlp.experts.58.up_proj", + "model.layers.41.mlp.experts.59.down_proj", + "model.layers.41.mlp.experts.59.gate_proj", + "model.layers.41.mlp.experts.59.up_proj", + "model.layers.41.mlp.experts.6.down_proj", + "model.layers.41.mlp.experts.6.gate_proj", + "model.layers.41.mlp.experts.6.up_proj", + "model.layers.41.mlp.experts.60.down_proj", + "model.layers.41.mlp.experts.60.gate_proj", + "model.layers.41.mlp.experts.60.up_proj", + "model.layers.41.mlp.experts.61.down_proj", + "model.layers.41.mlp.experts.61.gate_proj", + "model.layers.41.mlp.experts.61.up_proj", + "model.layers.41.mlp.experts.62.down_proj", + "model.layers.41.mlp.experts.62.gate_proj", + "model.layers.41.mlp.experts.62.up_proj", + "model.layers.41.mlp.experts.63.down_proj", + "model.layers.41.mlp.experts.63.gate_proj", + "model.layers.41.mlp.experts.63.up_proj", + "model.layers.41.mlp.experts.64.down_proj", + "model.layers.41.mlp.experts.64.gate_proj", + "model.layers.41.mlp.experts.64.up_proj", + "model.layers.41.mlp.experts.65.down_proj", + "model.layers.41.mlp.experts.65.gate_proj", + "model.layers.41.mlp.experts.65.up_proj", + "model.layers.41.mlp.experts.66.down_proj", + "model.layers.41.mlp.experts.66.gate_proj", + "model.layers.41.mlp.experts.66.up_proj", + "model.layers.41.mlp.experts.67.down_proj", + "model.layers.41.mlp.experts.67.gate_proj", + "model.layers.41.mlp.experts.67.up_proj", + "model.layers.41.mlp.experts.68.down_proj", + "model.layers.41.mlp.experts.68.gate_proj", + "model.layers.41.mlp.experts.68.up_proj", + "model.layers.41.mlp.experts.69.down_proj", + "model.layers.41.mlp.experts.69.gate_proj", + "model.layers.41.mlp.experts.69.up_proj", + "model.layers.41.mlp.experts.7.down_proj", + "model.layers.41.mlp.experts.7.gate_proj", + "model.layers.41.mlp.experts.7.up_proj", + "model.layers.41.mlp.experts.70.down_proj", + "model.layers.41.mlp.experts.70.gate_proj", + "model.layers.41.mlp.experts.70.up_proj", + "model.layers.41.mlp.experts.71.down_proj", + "model.layers.41.mlp.experts.71.gate_proj", + "model.layers.41.mlp.experts.71.up_proj", + "model.layers.41.mlp.experts.72.down_proj", + "model.layers.41.mlp.experts.72.gate_proj", + "model.layers.41.mlp.experts.72.up_proj", + "model.layers.41.mlp.experts.73.down_proj", + "model.layers.41.mlp.experts.73.gate_proj", + "model.layers.41.mlp.experts.73.up_proj", + "model.layers.41.mlp.experts.74.down_proj", + "model.layers.41.mlp.experts.74.gate_proj", + "model.layers.41.mlp.experts.74.up_proj", + "model.layers.41.mlp.experts.75.down_proj", + "model.layers.41.mlp.experts.75.gate_proj", + "model.layers.41.mlp.experts.75.up_proj", + "model.layers.41.mlp.experts.76.down_proj", + "model.layers.41.mlp.experts.76.gate_proj", + "model.layers.41.mlp.experts.76.up_proj", + "model.layers.41.mlp.experts.77.down_proj", + "model.layers.41.mlp.experts.77.gate_proj", + "model.layers.41.mlp.experts.77.up_proj", + "model.layers.41.mlp.experts.78.down_proj", + "model.layers.41.mlp.experts.78.gate_proj", + "model.layers.41.mlp.experts.78.up_proj", + "model.layers.41.mlp.experts.79.down_proj", + "model.layers.41.mlp.experts.79.gate_proj", + "model.layers.41.mlp.experts.79.up_proj", + "model.layers.41.mlp.experts.8.down_proj", + "model.layers.41.mlp.experts.8.gate_proj", + "model.layers.41.mlp.experts.8.up_proj", + "model.layers.41.mlp.experts.80.down_proj", + "model.layers.41.mlp.experts.80.gate_proj", + "model.layers.41.mlp.experts.80.up_proj", + "model.layers.41.mlp.experts.81.down_proj", + "model.layers.41.mlp.experts.81.gate_proj", + "model.layers.41.mlp.experts.81.up_proj", + "model.layers.41.mlp.experts.82.down_proj", + "model.layers.41.mlp.experts.82.gate_proj", + "model.layers.41.mlp.experts.82.up_proj", + "model.layers.41.mlp.experts.83.down_proj", + "model.layers.41.mlp.experts.83.gate_proj", + "model.layers.41.mlp.experts.83.up_proj", + "model.layers.41.mlp.experts.84.down_proj", + "model.layers.41.mlp.experts.84.gate_proj", + "model.layers.41.mlp.experts.84.up_proj", + "model.layers.41.mlp.experts.85.down_proj", + "model.layers.41.mlp.experts.85.gate_proj", + "model.layers.41.mlp.experts.85.up_proj", + "model.layers.41.mlp.experts.86.down_proj", + "model.layers.41.mlp.experts.86.gate_proj", + "model.layers.41.mlp.experts.86.up_proj", + "model.layers.41.mlp.experts.87.down_proj", + "model.layers.41.mlp.experts.87.gate_proj", + "model.layers.41.mlp.experts.87.up_proj", + "model.layers.41.mlp.experts.88.down_proj", + "model.layers.41.mlp.experts.88.gate_proj", + "model.layers.41.mlp.experts.88.up_proj", + "model.layers.41.mlp.experts.89.down_proj", + "model.layers.41.mlp.experts.89.gate_proj", + "model.layers.41.mlp.experts.89.up_proj", + "model.layers.41.mlp.experts.9.down_proj", + "model.layers.41.mlp.experts.9.gate_proj", + "model.layers.41.mlp.experts.9.up_proj", + "model.layers.41.mlp.experts.90.down_proj", + "model.layers.41.mlp.experts.90.gate_proj", + "model.layers.41.mlp.experts.90.up_proj", + "model.layers.41.mlp.experts.91.down_proj", + "model.layers.41.mlp.experts.91.gate_proj", + "model.layers.41.mlp.experts.91.up_proj", + "model.layers.41.mlp.experts.92.down_proj", + "model.layers.41.mlp.experts.92.gate_proj", + "model.layers.41.mlp.experts.92.up_proj", + "model.layers.41.mlp.experts.93.down_proj", + "model.layers.41.mlp.experts.93.gate_proj", + "model.layers.41.mlp.experts.93.up_proj", + "model.layers.41.mlp.experts.94.down_proj", + "model.layers.41.mlp.experts.94.gate_proj", + "model.layers.41.mlp.experts.94.up_proj", + "model.layers.41.mlp.experts.95.down_proj", + "model.layers.41.mlp.experts.95.gate_proj", + "model.layers.41.mlp.experts.95.up_proj", + "model.layers.41.mlp.experts.96.down_proj", + "model.layers.41.mlp.experts.96.gate_proj", + "model.layers.41.mlp.experts.96.up_proj", + "model.layers.41.mlp.experts.97.down_proj", + "model.layers.41.mlp.experts.97.gate_proj", + "model.layers.41.mlp.experts.97.up_proj", + "model.layers.41.mlp.experts.98.down_proj", + "model.layers.41.mlp.experts.98.gate_proj", + "model.layers.41.mlp.experts.98.up_proj", + "model.layers.41.mlp.experts.99.down_proj", + "model.layers.41.mlp.experts.99.gate_proj", + "model.layers.41.mlp.experts.99.up_proj", + "model.layers.42.mlp.experts.0.down_proj", + "model.layers.42.mlp.experts.0.gate_proj", + "model.layers.42.mlp.experts.0.up_proj", + "model.layers.42.mlp.experts.1.down_proj", + "model.layers.42.mlp.experts.1.gate_proj", + "model.layers.42.mlp.experts.1.up_proj", + "model.layers.42.mlp.experts.10.down_proj", + "model.layers.42.mlp.experts.10.gate_proj", + "model.layers.42.mlp.experts.10.up_proj", + "model.layers.42.mlp.experts.100.down_proj", + "model.layers.42.mlp.experts.100.gate_proj", + "model.layers.42.mlp.experts.100.up_proj", + "model.layers.42.mlp.experts.101.down_proj", + "model.layers.42.mlp.experts.101.gate_proj", + "model.layers.42.mlp.experts.101.up_proj", + "model.layers.42.mlp.experts.102.down_proj", + "model.layers.42.mlp.experts.102.gate_proj", + "model.layers.42.mlp.experts.102.up_proj", + "model.layers.42.mlp.experts.103.down_proj", + "model.layers.42.mlp.experts.103.gate_proj", + "model.layers.42.mlp.experts.103.up_proj", + "model.layers.42.mlp.experts.104.down_proj", + "model.layers.42.mlp.experts.104.gate_proj", + "model.layers.42.mlp.experts.104.up_proj", + "model.layers.42.mlp.experts.105.down_proj", + "model.layers.42.mlp.experts.105.gate_proj", + "model.layers.42.mlp.experts.105.up_proj", + "model.layers.42.mlp.experts.106.down_proj", + "model.layers.42.mlp.experts.106.gate_proj", + "model.layers.42.mlp.experts.106.up_proj", + "model.layers.42.mlp.experts.107.down_proj", + "model.layers.42.mlp.experts.107.gate_proj", + "model.layers.42.mlp.experts.107.up_proj", + "model.layers.42.mlp.experts.108.down_proj", + "model.layers.42.mlp.experts.108.gate_proj", + "model.layers.42.mlp.experts.108.up_proj", + "model.layers.42.mlp.experts.109.down_proj", + "model.layers.42.mlp.experts.109.gate_proj", + "model.layers.42.mlp.experts.109.up_proj", + "model.layers.42.mlp.experts.11.down_proj", + "model.layers.42.mlp.experts.11.gate_proj", + "model.layers.42.mlp.experts.11.up_proj", + "model.layers.42.mlp.experts.110.down_proj", + "model.layers.42.mlp.experts.110.gate_proj", + "model.layers.42.mlp.experts.110.up_proj", + "model.layers.42.mlp.experts.111.down_proj", + "model.layers.42.mlp.experts.111.gate_proj", + "model.layers.42.mlp.experts.111.up_proj", + "model.layers.42.mlp.experts.112.down_proj", + "model.layers.42.mlp.experts.112.gate_proj", + "model.layers.42.mlp.experts.112.up_proj", + "model.layers.42.mlp.experts.113.down_proj", + "model.layers.42.mlp.experts.113.gate_proj", + "model.layers.42.mlp.experts.113.up_proj", + "model.layers.42.mlp.experts.114.down_proj", + "model.layers.42.mlp.experts.114.gate_proj", + "model.layers.42.mlp.experts.114.up_proj", + "model.layers.42.mlp.experts.115.down_proj", + "model.layers.42.mlp.experts.115.gate_proj", + "model.layers.42.mlp.experts.115.up_proj", + "model.layers.42.mlp.experts.116.down_proj", + "model.layers.42.mlp.experts.116.gate_proj", + "model.layers.42.mlp.experts.116.up_proj", + "model.layers.42.mlp.experts.117.down_proj", + "model.layers.42.mlp.experts.117.gate_proj", + "model.layers.42.mlp.experts.117.up_proj", + "model.layers.42.mlp.experts.118.down_proj", + "model.layers.42.mlp.experts.118.gate_proj", + "model.layers.42.mlp.experts.118.up_proj", + "model.layers.42.mlp.experts.119.down_proj", + "model.layers.42.mlp.experts.119.gate_proj", + "model.layers.42.mlp.experts.119.up_proj", + "model.layers.42.mlp.experts.12.down_proj", + "model.layers.42.mlp.experts.12.gate_proj", + "model.layers.42.mlp.experts.12.up_proj", + "model.layers.42.mlp.experts.120.down_proj", + "model.layers.42.mlp.experts.120.gate_proj", + "model.layers.42.mlp.experts.120.up_proj", + "model.layers.42.mlp.experts.121.down_proj", + "model.layers.42.mlp.experts.121.gate_proj", + "model.layers.42.mlp.experts.121.up_proj", + "model.layers.42.mlp.experts.122.down_proj", + "model.layers.42.mlp.experts.122.gate_proj", + "model.layers.42.mlp.experts.122.up_proj", + "model.layers.42.mlp.experts.123.down_proj", + "model.layers.42.mlp.experts.123.gate_proj", + "model.layers.42.mlp.experts.123.up_proj", + "model.layers.42.mlp.experts.124.down_proj", + "model.layers.42.mlp.experts.124.gate_proj", + "model.layers.42.mlp.experts.124.up_proj", + "model.layers.42.mlp.experts.125.down_proj", + "model.layers.42.mlp.experts.125.gate_proj", + "model.layers.42.mlp.experts.125.up_proj", + "model.layers.42.mlp.experts.126.down_proj", + "model.layers.42.mlp.experts.126.gate_proj", + "model.layers.42.mlp.experts.126.up_proj", + "model.layers.42.mlp.experts.127.down_proj", + "model.layers.42.mlp.experts.127.gate_proj", + "model.layers.42.mlp.experts.127.up_proj", + "model.layers.42.mlp.experts.128.down_proj", + "model.layers.42.mlp.experts.128.gate_proj", + "model.layers.42.mlp.experts.128.up_proj", + "model.layers.42.mlp.experts.129.down_proj", + "model.layers.42.mlp.experts.129.gate_proj", + "model.layers.42.mlp.experts.129.up_proj", + "model.layers.42.mlp.experts.13.down_proj", + "model.layers.42.mlp.experts.13.gate_proj", + "model.layers.42.mlp.experts.13.up_proj", + "model.layers.42.mlp.experts.130.down_proj", + "model.layers.42.mlp.experts.130.gate_proj", + "model.layers.42.mlp.experts.130.up_proj", + "model.layers.42.mlp.experts.131.down_proj", + "model.layers.42.mlp.experts.131.gate_proj", + "model.layers.42.mlp.experts.131.up_proj", + "model.layers.42.mlp.experts.132.down_proj", + "model.layers.42.mlp.experts.132.gate_proj", + "model.layers.42.mlp.experts.132.up_proj", + "model.layers.42.mlp.experts.133.down_proj", + "model.layers.42.mlp.experts.133.gate_proj", + "model.layers.42.mlp.experts.133.up_proj", + "model.layers.42.mlp.experts.134.down_proj", + "model.layers.42.mlp.experts.134.gate_proj", + "model.layers.42.mlp.experts.134.up_proj", + "model.layers.42.mlp.experts.135.down_proj", + "model.layers.42.mlp.experts.135.gate_proj", + "model.layers.42.mlp.experts.135.up_proj", + "model.layers.42.mlp.experts.136.down_proj", + "model.layers.42.mlp.experts.136.gate_proj", + "model.layers.42.mlp.experts.136.up_proj", + "model.layers.42.mlp.experts.137.down_proj", + "model.layers.42.mlp.experts.137.gate_proj", + "model.layers.42.mlp.experts.137.up_proj", + "model.layers.42.mlp.experts.138.down_proj", + "model.layers.42.mlp.experts.138.gate_proj", + "model.layers.42.mlp.experts.138.up_proj", + "model.layers.42.mlp.experts.139.down_proj", + "model.layers.42.mlp.experts.139.gate_proj", + "model.layers.42.mlp.experts.139.up_proj", + "model.layers.42.mlp.experts.14.down_proj", + "model.layers.42.mlp.experts.14.gate_proj", + "model.layers.42.mlp.experts.14.up_proj", + "model.layers.42.mlp.experts.140.down_proj", + "model.layers.42.mlp.experts.140.gate_proj", + "model.layers.42.mlp.experts.140.up_proj", + "model.layers.42.mlp.experts.141.down_proj", + "model.layers.42.mlp.experts.141.gate_proj", + "model.layers.42.mlp.experts.141.up_proj", + "model.layers.42.mlp.experts.142.down_proj", + "model.layers.42.mlp.experts.142.gate_proj", + "model.layers.42.mlp.experts.142.up_proj", + "model.layers.42.mlp.experts.143.down_proj", + "model.layers.42.mlp.experts.143.gate_proj", + "model.layers.42.mlp.experts.143.up_proj", + "model.layers.42.mlp.experts.144.down_proj", + "model.layers.42.mlp.experts.144.gate_proj", + "model.layers.42.mlp.experts.144.up_proj", + "model.layers.42.mlp.experts.145.down_proj", + "model.layers.42.mlp.experts.145.gate_proj", + "model.layers.42.mlp.experts.145.up_proj", + "model.layers.42.mlp.experts.146.down_proj", + "model.layers.42.mlp.experts.146.gate_proj", + "model.layers.42.mlp.experts.146.up_proj", + "model.layers.42.mlp.experts.147.down_proj", + "model.layers.42.mlp.experts.147.gate_proj", + "model.layers.42.mlp.experts.147.up_proj", + "model.layers.42.mlp.experts.148.down_proj", + "model.layers.42.mlp.experts.148.gate_proj", + "model.layers.42.mlp.experts.148.up_proj", + "model.layers.42.mlp.experts.149.down_proj", + "model.layers.42.mlp.experts.149.gate_proj", + "model.layers.42.mlp.experts.149.up_proj", + "model.layers.42.mlp.experts.15.down_proj", + "model.layers.42.mlp.experts.15.gate_proj", + "model.layers.42.mlp.experts.15.up_proj", + "model.layers.42.mlp.experts.150.down_proj", + "model.layers.42.mlp.experts.150.gate_proj", + "model.layers.42.mlp.experts.150.up_proj", + "model.layers.42.mlp.experts.151.down_proj", + "model.layers.42.mlp.experts.151.gate_proj", + "model.layers.42.mlp.experts.151.up_proj", + "model.layers.42.mlp.experts.152.down_proj", + "model.layers.42.mlp.experts.152.gate_proj", + "model.layers.42.mlp.experts.152.up_proj", + "model.layers.42.mlp.experts.153.down_proj", + "model.layers.42.mlp.experts.153.gate_proj", + "model.layers.42.mlp.experts.153.up_proj", + "model.layers.42.mlp.experts.154.down_proj", + "model.layers.42.mlp.experts.154.gate_proj", + "model.layers.42.mlp.experts.154.up_proj", + "model.layers.42.mlp.experts.155.down_proj", + "model.layers.42.mlp.experts.155.gate_proj", + "model.layers.42.mlp.experts.155.up_proj", + "model.layers.42.mlp.experts.156.down_proj", + "model.layers.42.mlp.experts.156.gate_proj", + "model.layers.42.mlp.experts.156.up_proj", + "model.layers.42.mlp.experts.157.down_proj", + "model.layers.42.mlp.experts.157.gate_proj", + "model.layers.42.mlp.experts.157.up_proj", + "model.layers.42.mlp.experts.158.down_proj", + "model.layers.42.mlp.experts.158.gate_proj", + "model.layers.42.mlp.experts.158.up_proj", + "model.layers.42.mlp.experts.159.down_proj", + "model.layers.42.mlp.experts.159.gate_proj", + "model.layers.42.mlp.experts.159.up_proj", + "model.layers.42.mlp.experts.16.down_proj", + "model.layers.42.mlp.experts.16.gate_proj", + "model.layers.42.mlp.experts.16.up_proj", + "model.layers.42.mlp.experts.160.down_proj", + "model.layers.42.mlp.experts.160.gate_proj", + "model.layers.42.mlp.experts.160.up_proj", + "model.layers.42.mlp.experts.161.down_proj", + "model.layers.42.mlp.experts.161.gate_proj", + "model.layers.42.mlp.experts.161.up_proj", + "model.layers.42.mlp.experts.162.down_proj", + "model.layers.42.mlp.experts.162.gate_proj", + "model.layers.42.mlp.experts.162.up_proj", + "model.layers.42.mlp.experts.163.down_proj", + "model.layers.42.mlp.experts.163.gate_proj", + "model.layers.42.mlp.experts.163.up_proj", + "model.layers.42.mlp.experts.164.down_proj", + "model.layers.42.mlp.experts.164.gate_proj", + "model.layers.42.mlp.experts.164.up_proj", + "model.layers.42.mlp.experts.165.down_proj", + "model.layers.42.mlp.experts.165.gate_proj", + "model.layers.42.mlp.experts.165.up_proj", + "model.layers.42.mlp.experts.166.down_proj", + "model.layers.42.mlp.experts.166.gate_proj", + "model.layers.42.mlp.experts.166.up_proj", + "model.layers.42.mlp.experts.167.down_proj", + "model.layers.42.mlp.experts.167.gate_proj", + "model.layers.42.mlp.experts.167.up_proj", + "model.layers.42.mlp.experts.168.down_proj", + "model.layers.42.mlp.experts.168.gate_proj", + "model.layers.42.mlp.experts.168.up_proj", + "model.layers.42.mlp.experts.169.down_proj", + "model.layers.42.mlp.experts.169.gate_proj", + "model.layers.42.mlp.experts.169.up_proj", + "model.layers.42.mlp.experts.17.down_proj", + "model.layers.42.mlp.experts.17.gate_proj", + "model.layers.42.mlp.experts.17.up_proj", + "model.layers.42.mlp.experts.170.down_proj", + "model.layers.42.mlp.experts.170.gate_proj", + "model.layers.42.mlp.experts.170.up_proj", + "model.layers.42.mlp.experts.171.down_proj", + "model.layers.42.mlp.experts.171.gate_proj", + "model.layers.42.mlp.experts.171.up_proj", + "model.layers.42.mlp.experts.172.down_proj", + "model.layers.42.mlp.experts.172.gate_proj", + "model.layers.42.mlp.experts.172.up_proj", + "model.layers.42.mlp.experts.173.down_proj", + "model.layers.42.mlp.experts.173.gate_proj", + "model.layers.42.mlp.experts.173.up_proj", + "model.layers.42.mlp.experts.174.down_proj", + "model.layers.42.mlp.experts.174.gate_proj", + "model.layers.42.mlp.experts.174.up_proj", + "model.layers.42.mlp.experts.175.down_proj", + "model.layers.42.mlp.experts.175.gate_proj", + "model.layers.42.mlp.experts.175.up_proj", + "model.layers.42.mlp.experts.176.down_proj", + "model.layers.42.mlp.experts.176.gate_proj", + "model.layers.42.mlp.experts.176.up_proj", + "model.layers.42.mlp.experts.177.down_proj", + "model.layers.42.mlp.experts.177.gate_proj", + "model.layers.42.mlp.experts.177.up_proj", + "model.layers.42.mlp.experts.178.down_proj", + "model.layers.42.mlp.experts.178.gate_proj", + "model.layers.42.mlp.experts.178.up_proj", + "model.layers.42.mlp.experts.179.down_proj", + "model.layers.42.mlp.experts.179.gate_proj", + "model.layers.42.mlp.experts.179.up_proj", + "model.layers.42.mlp.experts.18.down_proj", + "model.layers.42.mlp.experts.18.gate_proj", + "model.layers.42.mlp.experts.18.up_proj", + "model.layers.42.mlp.experts.180.down_proj", + "model.layers.42.mlp.experts.180.gate_proj", + "model.layers.42.mlp.experts.180.up_proj", + "model.layers.42.mlp.experts.181.down_proj", + "model.layers.42.mlp.experts.181.gate_proj", + "model.layers.42.mlp.experts.181.up_proj", + "model.layers.42.mlp.experts.182.down_proj", + "model.layers.42.mlp.experts.182.gate_proj", + "model.layers.42.mlp.experts.182.up_proj", + "model.layers.42.mlp.experts.183.down_proj", + "model.layers.42.mlp.experts.183.gate_proj", + "model.layers.42.mlp.experts.183.up_proj", + "model.layers.42.mlp.experts.184.down_proj", + "model.layers.42.mlp.experts.184.gate_proj", + "model.layers.42.mlp.experts.184.up_proj", + "model.layers.42.mlp.experts.185.down_proj", + "model.layers.42.mlp.experts.185.gate_proj", + "model.layers.42.mlp.experts.185.up_proj", + "model.layers.42.mlp.experts.186.down_proj", + "model.layers.42.mlp.experts.186.gate_proj", + "model.layers.42.mlp.experts.186.up_proj", + "model.layers.42.mlp.experts.187.down_proj", + "model.layers.42.mlp.experts.187.gate_proj", + "model.layers.42.mlp.experts.187.up_proj", + "model.layers.42.mlp.experts.188.down_proj", + "model.layers.42.mlp.experts.188.gate_proj", + "model.layers.42.mlp.experts.188.up_proj", + "model.layers.42.mlp.experts.189.down_proj", + "model.layers.42.mlp.experts.189.gate_proj", + "model.layers.42.mlp.experts.189.up_proj", + "model.layers.42.mlp.experts.19.down_proj", + "model.layers.42.mlp.experts.19.gate_proj", + "model.layers.42.mlp.experts.19.up_proj", + "model.layers.42.mlp.experts.190.down_proj", + "model.layers.42.mlp.experts.190.gate_proj", + "model.layers.42.mlp.experts.190.up_proj", + "model.layers.42.mlp.experts.191.down_proj", + "model.layers.42.mlp.experts.191.gate_proj", + "model.layers.42.mlp.experts.191.up_proj", + "model.layers.42.mlp.experts.192.down_proj", + "model.layers.42.mlp.experts.192.gate_proj", + "model.layers.42.mlp.experts.192.up_proj", + "model.layers.42.mlp.experts.193.down_proj", + "model.layers.42.mlp.experts.193.gate_proj", + "model.layers.42.mlp.experts.193.up_proj", + "model.layers.42.mlp.experts.194.down_proj", + "model.layers.42.mlp.experts.194.gate_proj", + "model.layers.42.mlp.experts.194.up_proj", + "model.layers.42.mlp.experts.195.down_proj", + "model.layers.42.mlp.experts.195.gate_proj", + "model.layers.42.mlp.experts.195.up_proj", + "model.layers.42.mlp.experts.196.down_proj", + "model.layers.42.mlp.experts.196.gate_proj", + "model.layers.42.mlp.experts.196.up_proj", + "model.layers.42.mlp.experts.197.down_proj", + "model.layers.42.mlp.experts.197.gate_proj", + "model.layers.42.mlp.experts.197.up_proj", + "model.layers.42.mlp.experts.198.down_proj", + "model.layers.42.mlp.experts.198.gate_proj", + "model.layers.42.mlp.experts.198.up_proj", + "model.layers.42.mlp.experts.199.down_proj", + "model.layers.42.mlp.experts.199.gate_proj", + "model.layers.42.mlp.experts.199.up_proj", + "model.layers.42.mlp.experts.2.down_proj", + "model.layers.42.mlp.experts.2.gate_proj", + "model.layers.42.mlp.experts.2.up_proj", + "model.layers.42.mlp.experts.20.down_proj", + "model.layers.42.mlp.experts.20.gate_proj", + "model.layers.42.mlp.experts.20.up_proj", + "model.layers.42.mlp.experts.200.down_proj", + "model.layers.42.mlp.experts.200.gate_proj", + "model.layers.42.mlp.experts.200.up_proj", + "model.layers.42.mlp.experts.201.down_proj", + "model.layers.42.mlp.experts.201.gate_proj", + "model.layers.42.mlp.experts.201.up_proj", + "model.layers.42.mlp.experts.202.down_proj", + "model.layers.42.mlp.experts.202.gate_proj", + "model.layers.42.mlp.experts.202.up_proj", + "model.layers.42.mlp.experts.203.down_proj", + "model.layers.42.mlp.experts.203.gate_proj", + "model.layers.42.mlp.experts.203.up_proj", + "model.layers.42.mlp.experts.204.down_proj", + "model.layers.42.mlp.experts.204.gate_proj", + "model.layers.42.mlp.experts.204.up_proj", + "model.layers.42.mlp.experts.205.down_proj", + "model.layers.42.mlp.experts.205.gate_proj", + "model.layers.42.mlp.experts.205.up_proj", + "model.layers.42.mlp.experts.206.down_proj", + "model.layers.42.mlp.experts.206.gate_proj", + "model.layers.42.mlp.experts.206.up_proj", + "model.layers.42.mlp.experts.207.down_proj", + "model.layers.42.mlp.experts.207.gate_proj", + "model.layers.42.mlp.experts.207.up_proj", + "model.layers.42.mlp.experts.208.down_proj", + "model.layers.42.mlp.experts.208.gate_proj", + "model.layers.42.mlp.experts.208.up_proj", + "model.layers.42.mlp.experts.209.down_proj", + "model.layers.42.mlp.experts.209.gate_proj", + "model.layers.42.mlp.experts.209.up_proj", + "model.layers.42.mlp.experts.21.down_proj", + "model.layers.42.mlp.experts.21.gate_proj", + "model.layers.42.mlp.experts.21.up_proj", + "model.layers.42.mlp.experts.210.down_proj", + "model.layers.42.mlp.experts.210.gate_proj", + "model.layers.42.mlp.experts.210.up_proj", + "model.layers.42.mlp.experts.211.down_proj", + "model.layers.42.mlp.experts.211.gate_proj", + "model.layers.42.mlp.experts.211.up_proj", + "model.layers.42.mlp.experts.212.down_proj", + "model.layers.42.mlp.experts.212.gate_proj", + "model.layers.42.mlp.experts.212.up_proj", + "model.layers.42.mlp.experts.213.down_proj", + "model.layers.42.mlp.experts.213.gate_proj", + "model.layers.42.mlp.experts.213.up_proj", + "model.layers.42.mlp.experts.214.down_proj", + "model.layers.42.mlp.experts.214.gate_proj", + "model.layers.42.mlp.experts.214.up_proj", + "model.layers.42.mlp.experts.215.down_proj", + "model.layers.42.mlp.experts.215.gate_proj", + "model.layers.42.mlp.experts.215.up_proj", + "model.layers.42.mlp.experts.216.down_proj", + "model.layers.42.mlp.experts.216.gate_proj", + "model.layers.42.mlp.experts.216.up_proj", + "model.layers.42.mlp.experts.217.down_proj", + "model.layers.42.mlp.experts.217.gate_proj", + "model.layers.42.mlp.experts.217.up_proj", + "model.layers.42.mlp.experts.218.down_proj", + "model.layers.42.mlp.experts.218.gate_proj", + "model.layers.42.mlp.experts.218.up_proj", + "model.layers.42.mlp.experts.219.down_proj", + "model.layers.42.mlp.experts.219.gate_proj", + "model.layers.42.mlp.experts.219.up_proj", + "model.layers.42.mlp.experts.22.down_proj", + "model.layers.42.mlp.experts.22.gate_proj", + "model.layers.42.mlp.experts.22.up_proj", + "model.layers.42.mlp.experts.220.down_proj", + "model.layers.42.mlp.experts.220.gate_proj", + "model.layers.42.mlp.experts.220.up_proj", + "model.layers.42.mlp.experts.221.down_proj", + "model.layers.42.mlp.experts.221.gate_proj", + "model.layers.42.mlp.experts.221.up_proj", + "model.layers.42.mlp.experts.222.down_proj", + "model.layers.42.mlp.experts.222.gate_proj", + "model.layers.42.mlp.experts.222.up_proj", + "model.layers.42.mlp.experts.223.down_proj", + "model.layers.42.mlp.experts.223.gate_proj", + "model.layers.42.mlp.experts.223.up_proj", + "model.layers.42.mlp.experts.224.down_proj", + "model.layers.42.mlp.experts.224.gate_proj", + "model.layers.42.mlp.experts.224.up_proj", + "model.layers.42.mlp.experts.225.down_proj", + "model.layers.42.mlp.experts.225.gate_proj", + "model.layers.42.mlp.experts.225.up_proj", + "model.layers.42.mlp.experts.226.down_proj", + "model.layers.42.mlp.experts.226.gate_proj", + "model.layers.42.mlp.experts.226.up_proj", + "model.layers.42.mlp.experts.227.down_proj", + "model.layers.42.mlp.experts.227.gate_proj", + "model.layers.42.mlp.experts.227.up_proj", + "model.layers.42.mlp.experts.228.down_proj", + "model.layers.42.mlp.experts.228.gate_proj", + "model.layers.42.mlp.experts.228.up_proj", + "model.layers.42.mlp.experts.229.down_proj", + "model.layers.42.mlp.experts.229.gate_proj", + "model.layers.42.mlp.experts.229.up_proj", + "model.layers.42.mlp.experts.23.down_proj", + "model.layers.42.mlp.experts.23.gate_proj", + "model.layers.42.mlp.experts.23.up_proj", + "model.layers.42.mlp.experts.230.down_proj", + "model.layers.42.mlp.experts.230.gate_proj", + "model.layers.42.mlp.experts.230.up_proj", + "model.layers.42.mlp.experts.231.down_proj", + "model.layers.42.mlp.experts.231.gate_proj", + "model.layers.42.mlp.experts.231.up_proj", + "model.layers.42.mlp.experts.232.down_proj", + "model.layers.42.mlp.experts.232.gate_proj", + "model.layers.42.mlp.experts.232.up_proj", + "model.layers.42.mlp.experts.233.down_proj", + "model.layers.42.mlp.experts.233.gate_proj", + "model.layers.42.mlp.experts.233.up_proj", + "model.layers.42.mlp.experts.234.down_proj", + "model.layers.42.mlp.experts.234.gate_proj", + "model.layers.42.mlp.experts.234.up_proj", + "model.layers.42.mlp.experts.235.down_proj", + "model.layers.42.mlp.experts.235.gate_proj", + "model.layers.42.mlp.experts.235.up_proj", + "model.layers.42.mlp.experts.236.down_proj", + "model.layers.42.mlp.experts.236.gate_proj", + "model.layers.42.mlp.experts.236.up_proj", + "model.layers.42.mlp.experts.237.down_proj", + "model.layers.42.mlp.experts.237.gate_proj", + "model.layers.42.mlp.experts.237.up_proj", + "model.layers.42.mlp.experts.238.down_proj", + "model.layers.42.mlp.experts.238.gate_proj", + "model.layers.42.mlp.experts.238.up_proj", + "model.layers.42.mlp.experts.239.down_proj", + "model.layers.42.mlp.experts.239.gate_proj", + "model.layers.42.mlp.experts.239.up_proj", + "model.layers.42.mlp.experts.24.down_proj", + "model.layers.42.mlp.experts.24.gate_proj", + "model.layers.42.mlp.experts.24.up_proj", + "model.layers.42.mlp.experts.240.down_proj", + "model.layers.42.mlp.experts.240.gate_proj", + "model.layers.42.mlp.experts.240.up_proj", + "model.layers.42.mlp.experts.241.down_proj", + "model.layers.42.mlp.experts.241.gate_proj", + "model.layers.42.mlp.experts.241.up_proj", + "model.layers.42.mlp.experts.242.down_proj", + "model.layers.42.mlp.experts.242.gate_proj", + "model.layers.42.mlp.experts.242.up_proj", + "model.layers.42.mlp.experts.243.down_proj", + "model.layers.42.mlp.experts.243.gate_proj", + "model.layers.42.mlp.experts.243.up_proj", + "model.layers.42.mlp.experts.244.down_proj", + "model.layers.42.mlp.experts.244.gate_proj", + "model.layers.42.mlp.experts.244.up_proj", + "model.layers.42.mlp.experts.245.down_proj", + "model.layers.42.mlp.experts.245.gate_proj", + "model.layers.42.mlp.experts.245.up_proj", + "model.layers.42.mlp.experts.246.down_proj", + "model.layers.42.mlp.experts.246.gate_proj", + "model.layers.42.mlp.experts.246.up_proj", + "model.layers.42.mlp.experts.247.down_proj", + "model.layers.42.mlp.experts.247.gate_proj", + "model.layers.42.mlp.experts.247.up_proj", + "model.layers.42.mlp.experts.248.down_proj", + "model.layers.42.mlp.experts.248.gate_proj", + "model.layers.42.mlp.experts.248.up_proj", + "model.layers.42.mlp.experts.249.down_proj", + "model.layers.42.mlp.experts.249.gate_proj", + "model.layers.42.mlp.experts.249.up_proj", + "model.layers.42.mlp.experts.25.down_proj", + "model.layers.42.mlp.experts.25.gate_proj", + "model.layers.42.mlp.experts.25.up_proj", + "model.layers.42.mlp.experts.250.down_proj", + "model.layers.42.mlp.experts.250.gate_proj", + "model.layers.42.mlp.experts.250.up_proj", + "model.layers.42.mlp.experts.251.down_proj", + "model.layers.42.mlp.experts.251.gate_proj", + "model.layers.42.mlp.experts.251.up_proj", + "model.layers.42.mlp.experts.252.down_proj", + "model.layers.42.mlp.experts.252.gate_proj", + "model.layers.42.mlp.experts.252.up_proj", + "model.layers.42.mlp.experts.253.down_proj", + "model.layers.42.mlp.experts.253.gate_proj", + "model.layers.42.mlp.experts.253.up_proj", + "model.layers.42.mlp.experts.254.down_proj", + "model.layers.42.mlp.experts.254.gate_proj", + "model.layers.42.mlp.experts.254.up_proj", + "model.layers.42.mlp.experts.255.down_proj", + "model.layers.42.mlp.experts.255.gate_proj", + "model.layers.42.mlp.experts.255.up_proj", + "model.layers.42.mlp.experts.26.down_proj", + "model.layers.42.mlp.experts.26.gate_proj", + "model.layers.42.mlp.experts.26.up_proj", + "model.layers.42.mlp.experts.27.down_proj", + "model.layers.42.mlp.experts.27.gate_proj", + "model.layers.42.mlp.experts.27.up_proj", + "model.layers.42.mlp.experts.28.down_proj", + "model.layers.42.mlp.experts.28.gate_proj", + "model.layers.42.mlp.experts.28.up_proj", + "model.layers.42.mlp.experts.29.down_proj", + "model.layers.42.mlp.experts.29.gate_proj", + "model.layers.42.mlp.experts.29.up_proj", + "model.layers.42.mlp.experts.3.down_proj", + "model.layers.42.mlp.experts.3.gate_proj", + "model.layers.42.mlp.experts.3.up_proj", + "model.layers.42.mlp.experts.30.down_proj", + "model.layers.42.mlp.experts.30.gate_proj", + "model.layers.42.mlp.experts.30.up_proj", + "model.layers.42.mlp.experts.31.down_proj", + "model.layers.42.mlp.experts.31.gate_proj", + "model.layers.42.mlp.experts.31.up_proj", + "model.layers.42.mlp.experts.32.down_proj", + "model.layers.42.mlp.experts.32.gate_proj", + "model.layers.42.mlp.experts.32.up_proj", + "model.layers.42.mlp.experts.33.down_proj", + "model.layers.42.mlp.experts.33.gate_proj", + "model.layers.42.mlp.experts.33.up_proj", + "model.layers.42.mlp.experts.34.down_proj", + "model.layers.42.mlp.experts.34.gate_proj", + "model.layers.42.mlp.experts.34.up_proj", + "model.layers.42.mlp.experts.35.down_proj", + "model.layers.42.mlp.experts.35.gate_proj", + "model.layers.42.mlp.experts.35.up_proj", + "model.layers.42.mlp.experts.36.down_proj", + "model.layers.42.mlp.experts.36.gate_proj", + "model.layers.42.mlp.experts.36.up_proj", + "model.layers.42.mlp.experts.37.down_proj", + "model.layers.42.mlp.experts.37.gate_proj", + "model.layers.42.mlp.experts.37.up_proj", + "model.layers.42.mlp.experts.38.down_proj", + "model.layers.42.mlp.experts.38.gate_proj", + "model.layers.42.mlp.experts.38.up_proj", + "model.layers.42.mlp.experts.39.down_proj", + "model.layers.42.mlp.experts.39.gate_proj", + "model.layers.42.mlp.experts.39.up_proj", + "model.layers.42.mlp.experts.4.down_proj", + "model.layers.42.mlp.experts.4.gate_proj", + "model.layers.42.mlp.experts.4.up_proj", + "model.layers.42.mlp.experts.40.down_proj", + "model.layers.42.mlp.experts.40.gate_proj", + "model.layers.42.mlp.experts.40.up_proj", + "model.layers.42.mlp.experts.41.down_proj", + "model.layers.42.mlp.experts.41.gate_proj", + "model.layers.42.mlp.experts.41.up_proj", + "model.layers.42.mlp.experts.42.down_proj", + "model.layers.42.mlp.experts.42.gate_proj", + "model.layers.42.mlp.experts.42.up_proj", + "model.layers.42.mlp.experts.43.down_proj", + "model.layers.42.mlp.experts.43.gate_proj", + "model.layers.42.mlp.experts.43.up_proj", + "model.layers.42.mlp.experts.44.down_proj", + "model.layers.42.mlp.experts.44.gate_proj", + "model.layers.42.mlp.experts.44.up_proj", + "model.layers.42.mlp.experts.45.down_proj", + "model.layers.42.mlp.experts.45.gate_proj", + "model.layers.42.mlp.experts.45.up_proj", + "model.layers.42.mlp.experts.46.down_proj", + "model.layers.42.mlp.experts.46.gate_proj", + "model.layers.42.mlp.experts.46.up_proj", + "model.layers.42.mlp.experts.47.down_proj", + "model.layers.42.mlp.experts.47.gate_proj", + "model.layers.42.mlp.experts.47.up_proj", + "model.layers.42.mlp.experts.48.down_proj", + "model.layers.42.mlp.experts.48.gate_proj", + "model.layers.42.mlp.experts.48.up_proj", + "model.layers.42.mlp.experts.49.down_proj", + "model.layers.42.mlp.experts.49.gate_proj", + "model.layers.42.mlp.experts.49.up_proj", + "model.layers.42.mlp.experts.5.down_proj", + "model.layers.42.mlp.experts.5.gate_proj", + "model.layers.42.mlp.experts.5.up_proj", + "model.layers.42.mlp.experts.50.down_proj", + "model.layers.42.mlp.experts.50.gate_proj", + "model.layers.42.mlp.experts.50.up_proj", + "model.layers.42.mlp.experts.51.down_proj", + "model.layers.42.mlp.experts.51.gate_proj", + "model.layers.42.mlp.experts.51.up_proj", + "model.layers.42.mlp.experts.52.down_proj", + "model.layers.42.mlp.experts.52.gate_proj", + "model.layers.42.mlp.experts.52.up_proj", + "model.layers.42.mlp.experts.53.down_proj", + "model.layers.42.mlp.experts.53.gate_proj", + "model.layers.42.mlp.experts.53.up_proj", + "model.layers.42.mlp.experts.54.down_proj", + "model.layers.42.mlp.experts.54.gate_proj", + "model.layers.42.mlp.experts.54.up_proj", + "model.layers.42.mlp.experts.55.down_proj", + "model.layers.42.mlp.experts.55.gate_proj", + "model.layers.42.mlp.experts.55.up_proj", + "model.layers.42.mlp.experts.56.down_proj", + "model.layers.42.mlp.experts.56.gate_proj", + "model.layers.42.mlp.experts.56.up_proj", + "model.layers.42.mlp.experts.57.down_proj", + "model.layers.42.mlp.experts.57.gate_proj", + "model.layers.42.mlp.experts.57.up_proj", + "model.layers.42.mlp.experts.58.down_proj", + "model.layers.42.mlp.experts.58.gate_proj", + "model.layers.42.mlp.experts.58.up_proj", + "model.layers.42.mlp.experts.59.down_proj", + "model.layers.42.mlp.experts.59.gate_proj", + "model.layers.42.mlp.experts.59.up_proj", + "model.layers.42.mlp.experts.6.down_proj", + "model.layers.42.mlp.experts.6.gate_proj", + "model.layers.42.mlp.experts.6.up_proj", + "model.layers.42.mlp.experts.60.down_proj", + "model.layers.42.mlp.experts.60.gate_proj", + "model.layers.42.mlp.experts.60.up_proj", + "model.layers.42.mlp.experts.61.down_proj", + "model.layers.42.mlp.experts.61.gate_proj", + "model.layers.42.mlp.experts.61.up_proj", + "model.layers.42.mlp.experts.62.down_proj", + "model.layers.42.mlp.experts.62.gate_proj", + "model.layers.42.mlp.experts.62.up_proj", + "model.layers.42.mlp.experts.63.down_proj", + "model.layers.42.mlp.experts.63.gate_proj", + "model.layers.42.mlp.experts.63.up_proj", + "model.layers.42.mlp.experts.64.down_proj", + "model.layers.42.mlp.experts.64.gate_proj", + "model.layers.42.mlp.experts.64.up_proj", + "model.layers.42.mlp.experts.65.down_proj", + "model.layers.42.mlp.experts.65.gate_proj", + "model.layers.42.mlp.experts.65.up_proj", + "model.layers.42.mlp.experts.66.down_proj", + "model.layers.42.mlp.experts.66.gate_proj", + "model.layers.42.mlp.experts.66.up_proj", + "model.layers.42.mlp.experts.67.down_proj", + "model.layers.42.mlp.experts.67.gate_proj", + "model.layers.42.mlp.experts.67.up_proj", + "model.layers.42.mlp.experts.68.down_proj", + "model.layers.42.mlp.experts.68.gate_proj", + "model.layers.42.mlp.experts.68.up_proj", + "model.layers.42.mlp.experts.69.down_proj", + "model.layers.42.mlp.experts.69.gate_proj", + "model.layers.42.mlp.experts.69.up_proj", + "model.layers.42.mlp.experts.7.down_proj", + "model.layers.42.mlp.experts.7.gate_proj", + "model.layers.42.mlp.experts.7.up_proj", + "model.layers.42.mlp.experts.70.down_proj", + "model.layers.42.mlp.experts.70.gate_proj", + "model.layers.42.mlp.experts.70.up_proj", + "model.layers.42.mlp.experts.71.down_proj", + "model.layers.42.mlp.experts.71.gate_proj", + "model.layers.42.mlp.experts.71.up_proj", + "model.layers.42.mlp.experts.72.down_proj", + "model.layers.42.mlp.experts.72.gate_proj", + "model.layers.42.mlp.experts.72.up_proj", + "model.layers.42.mlp.experts.73.down_proj", + "model.layers.42.mlp.experts.73.gate_proj", + "model.layers.42.mlp.experts.73.up_proj", + "model.layers.42.mlp.experts.74.down_proj", + "model.layers.42.mlp.experts.74.gate_proj", + "model.layers.42.mlp.experts.74.up_proj", + "model.layers.42.mlp.experts.75.down_proj", + "model.layers.42.mlp.experts.75.gate_proj", + "model.layers.42.mlp.experts.75.up_proj", + "model.layers.42.mlp.experts.76.down_proj", + "model.layers.42.mlp.experts.76.gate_proj", + "model.layers.42.mlp.experts.76.up_proj", + "model.layers.42.mlp.experts.77.down_proj", + "model.layers.42.mlp.experts.77.gate_proj", + "model.layers.42.mlp.experts.77.up_proj", + "model.layers.42.mlp.experts.78.down_proj", + "model.layers.42.mlp.experts.78.gate_proj", + "model.layers.42.mlp.experts.78.up_proj", + "model.layers.42.mlp.experts.79.down_proj", + "model.layers.42.mlp.experts.79.gate_proj", + "model.layers.42.mlp.experts.79.up_proj", + "model.layers.42.mlp.experts.8.down_proj", + "model.layers.42.mlp.experts.8.gate_proj", + "model.layers.42.mlp.experts.8.up_proj", + "model.layers.42.mlp.experts.80.down_proj", + "model.layers.42.mlp.experts.80.gate_proj", + "model.layers.42.mlp.experts.80.up_proj", + "model.layers.42.mlp.experts.81.down_proj", + "model.layers.42.mlp.experts.81.gate_proj", + "model.layers.42.mlp.experts.81.up_proj", + "model.layers.42.mlp.experts.82.down_proj", + "model.layers.42.mlp.experts.82.gate_proj", + "model.layers.42.mlp.experts.82.up_proj", + "model.layers.42.mlp.experts.83.down_proj", + "model.layers.42.mlp.experts.83.gate_proj", + "model.layers.42.mlp.experts.83.up_proj", + "model.layers.42.mlp.experts.84.down_proj", + "model.layers.42.mlp.experts.84.gate_proj", + "model.layers.42.mlp.experts.84.up_proj", + "model.layers.42.mlp.experts.85.down_proj", + "model.layers.42.mlp.experts.85.gate_proj", + "model.layers.42.mlp.experts.85.up_proj", + "model.layers.42.mlp.experts.86.down_proj", + "model.layers.42.mlp.experts.86.gate_proj", + "model.layers.42.mlp.experts.86.up_proj", + "model.layers.42.mlp.experts.87.down_proj", + "model.layers.42.mlp.experts.87.gate_proj", + "model.layers.42.mlp.experts.87.up_proj", + "model.layers.42.mlp.experts.88.down_proj", + "model.layers.42.mlp.experts.88.gate_proj", + "model.layers.42.mlp.experts.88.up_proj", + "model.layers.42.mlp.experts.89.down_proj", + "model.layers.42.mlp.experts.89.gate_proj", + "model.layers.42.mlp.experts.89.up_proj", + "model.layers.42.mlp.experts.9.down_proj", + "model.layers.42.mlp.experts.9.gate_proj", + "model.layers.42.mlp.experts.9.up_proj", + "model.layers.42.mlp.experts.90.down_proj", + "model.layers.42.mlp.experts.90.gate_proj", + "model.layers.42.mlp.experts.90.up_proj", + "model.layers.42.mlp.experts.91.down_proj", + "model.layers.42.mlp.experts.91.gate_proj", + "model.layers.42.mlp.experts.91.up_proj", + "model.layers.42.mlp.experts.92.down_proj", + "model.layers.42.mlp.experts.92.gate_proj", + "model.layers.42.mlp.experts.92.up_proj", + "model.layers.42.mlp.experts.93.down_proj", + "model.layers.42.mlp.experts.93.gate_proj", + "model.layers.42.mlp.experts.93.up_proj", + "model.layers.42.mlp.experts.94.down_proj", + "model.layers.42.mlp.experts.94.gate_proj", + "model.layers.42.mlp.experts.94.up_proj", + "model.layers.42.mlp.experts.95.down_proj", + "model.layers.42.mlp.experts.95.gate_proj", + "model.layers.42.mlp.experts.95.up_proj", + "model.layers.42.mlp.experts.96.down_proj", + "model.layers.42.mlp.experts.96.gate_proj", + "model.layers.42.mlp.experts.96.up_proj", + "model.layers.42.mlp.experts.97.down_proj", + "model.layers.42.mlp.experts.97.gate_proj", + "model.layers.42.mlp.experts.97.up_proj", + "model.layers.42.mlp.experts.98.down_proj", + "model.layers.42.mlp.experts.98.gate_proj", + "model.layers.42.mlp.experts.98.up_proj", + "model.layers.42.mlp.experts.99.down_proj", + "model.layers.42.mlp.experts.99.gate_proj", + "model.layers.42.mlp.experts.99.up_proj", + "model.layers.43.mlp.experts.0.down_proj", + "model.layers.43.mlp.experts.0.gate_proj", + "model.layers.43.mlp.experts.0.up_proj", + "model.layers.43.mlp.experts.1.down_proj", + "model.layers.43.mlp.experts.1.gate_proj", + "model.layers.43.mlp.experts.1.up_proj", + "model.layers.43.mlp.experts.10.down_proj", + "model.layers.43.mlp.experts.10.gate_proj", + "model.layers.43.mlp.experts.10.up_proj", + "model.layers.43.mlp.experts.100.down_proj", + "model.layers.43.mlp.experts.100.gate_proj", + "model.layers.43.mlp.experts.100.up_proj", + "model.layers.43.mlp.experts.101.down_proj", + "model.layers.43.mlp.experts.101.gate_proj", + "model.layers.43.mlp.experts.101.up_proj", + "model.layers.43.mlp.experts.102.down_proj", + "model.layers.43.mlp.experts.102.gate_proj", + "model.layers.43.mlp.experts.102.up_proj", + "model.layers.43.mlp.experts.103.down_proj", + "model.layers.43.mlp.experts.103.gate_proj", + "model.layers.43.mlp.experts.103.up_proj", + "model.layers.43.mlp.experts.104.down_proj", + "model.layers.43.mlp.experts.104.gate_proj", + "model.layers.43.mlp.experts.104.up_proj", + "model.layers.43.mlp.experts.105.down_proj", + "model.layers.43.mlp.experts.105.gate_proj", + "model.layers.43.mlp.experts.105.up_proj", + "model.layers.43.mlp.experts.106.down_proj", + "model.layers.43.mlp.experts.106.gate_proj", + "model.layers.43.mlp.experts.106.up_proj", + "model.layers.43.mlp.experts.107.down_proj", + "model.layers.43.mlp.experts.107.gate_proj", + "model.layers.43.mlp.experts.107.up_proj", + "model.layers.43.mlp.experts.108.down_proj", + "model.layers.43.mlp.experts.108.gate_proj", + "model.layers.43.mlp.experts.108.up_proj", + "model.layers.43.mlp.experts.109.down_proj", + "model.layers.43.mlp.experts.109.gate_proj", + "model.layers.43.mlp.experts.109.up_proj", + "model.layers.43.mlp.experts.11.down_proj", + "model.layers.43.mlp.experts.11.gate_proj", + "model.layers.43.mlp.experts.11.up_proj", + "model.layers.43.mlp.experts.110.down_proj", + "model.layers.43.mlp.experts.110.gate_proj", + "model.layers.43.mlp.experts.110.up_proj", + "model.layers.43.mlp.experts.111.down_proj", + "model.layers.43.mlp.experts.111.gate_proj", + "model.layers.43.mlp.experts.111.up_proj", + "model.layers.43.mlp.experts.112.down_proj", + "model.layers.43.mlp.experts.112.gate_proj", + "model.layers.43.mlp.experts.112.up_proj", + "model.layers.43.mlp.experts.113.down_proj", + "model.layers.43.mlp.experts.113.gate_proj", + "model.layers.43.mlp.experts.113.up_proj", + "model.layers.43.mlp.experts.114.down_proj", + "model.layers.43.mlp.experts.114.gate_proj", + "model.layers.43.mlp.experts.114.up_proj", + "model.layers.43.mlp.experts.115.down_proj", + "model.layers.43.mlp.experts.115.gate_proj", + "model.layers.43.mlp.experts.115.up_proj", + "model.layers.43.mlp.experts.116.down_proj", + "model.layers.43.mlp.experts.116.gate_proj", + "model.layers.43.mlp.experts.116.up_proj", + "model.layers.43.mlp.experts.117.down_proj", + "model.layers.43.mlp.experts.117.gate_proj", + "model.layers.43.mlp.experts.117.up_proj", + "model.layers.43.mlp.experts.118.down_proj", + "model.layers.43.mlp.experts.118.gate_proj", + "model.layers.43.mlp.experts.118.up_proj", + "model.layers.43.mlp.experts.119.down_proj", + "model.layers.43.mlp.experts.119.gate_proj", + "model.layers.43.mlp.experts.119.up_proj", + "model.layers.43.mlp.experts.12.down_proj", + "model.layers.43.mlp.experts.12.gate_proj", + "model.layers.43.mlp.experts.12.up_proj", + "model.layers.43.mlp.experts.120.down_proj", + "model.layers.43.mlp.experts.120.gate_proj", + "model.layers.43.mlp.experts.120.up_proj", + "model.layers.43.mlp.experts.121.down_proj", + "model.layers.43.mlp.experts.121.gate_proj", + "model.layers.43.mlp.experts.121.up_proj", + "model.layers.43.mlp.experts.122.down_proj", + "model.layers.43.mlp.experts.122.gate_proj", + "model.layers.43.mlp.experts.122.up_proj", + "model.layers.43.mlp.experts.123.down_proj", + "model.layers.43.mlp.experts.123.gate_proj", + "model.layers.43.mlp.experts.123.up_proj", + "model.layers.43.mlp.experts.124.down_proj", + "model.layers.43.mlp.experts.124.gate_proj", + "model.layers.43.mlp.experts.124.up_proj", + "model.layers.43.mlp.experts.125.down_proj", + "model.layers.43.mlp.experts.125.gate_proj", + "model.layers.43.mlp.experts.125.up_proj", + "model.layers.43.mlp.experts.126.down_proj", + "model.layers.43.mlp.experts.126.gate_proj", + "model.layers.43.mlp.experts.126.up_proj", + "model.layers.43.mlp.experts.127.down_proj", + "model.layers.43.mlp.experts.127.gate_proj", + "model.layers.43.mlp.experts.127.up_proj", + "model.layers.43.mlp.experts.128.down_proj", + "model.layers.43.mlp.experts.128.gate_proj", + "model.layers.43.mlp.experts.128.up_proj", + "model.layers.43.mlp.experts.129.down_proj", + "model.layers.43.mlp.experts.129.gate_proj", + "model.layers.43.mlp.experts.129.up_proj", + "model.layers.43.mlp.experts.13.down_proj", + "model.layers.43.mlp.experts.13.gate_proj", + "model.layers.43.mlp.experts.13.up_proj", + "model.layers.43.mlp.experts.130.down_proj", + "model.layers.43.mlp.experts.130.gate_proj", + "model.layers.43.mlp.experts.130.up_proj", + "model.layers.43.mlp.experts.131.down_proj", + "model.layers.43.mlp.experts.131.gate_proj", + "model.layers.43.mlp.experts.131.up_proj", + "model.layers.43.mlp.experts.132.down_proj", + "model.layers.43.mlp.experts.132.gate_proj", + "model.layers.43.mlp.experts.132.up_proj", + "model.layers.43.mlp.experts.133.down_proj", + "model.layers.43.mlp.experts.133.gate_proj", + "model.layers.43.mlp.experts.133.up_proj", + "model.layers.43.mlp.experts.134.down_proj", + "model.layers.43.mlp.experts.134.gate_proj", + "model.layers.43.mlp.experts.134.up_proj", + "model.layers.43.mlp.experts.135.down_proj", + "model.layers.43.mlp.experts.135.gate_proj", + "model.layers.43.mlp.experts.135.up_proj", + "model.layers.43.mlp.experts.136.down_proj", + "model.layers.43.mlp.experts.136.gate_proj", + "model.layers.43.mlp.experts.136.up_proj", + "model.layers.43.mlp.experts.137.down_proj", + "model.layers.43.mlp.experts.137.gate_proj", + "model.layers.43.mlp.experts.137.up_proj", + "model.layers.43.mlp.experts.138.down_proj", + "model.layers.43.mlp.experts.138.gate_proj", + "model.layers.43.mlp.experts.138.up_proj", + "model.layers.43.mlp.experts.139.down_proj", + "model.layers.43.mlp.experts.139.gate_proj", + "model.layers.43.mlp.experts.139.up_proj", + "model.layers.43.mlp.experts.14.down_proj", + "model.layers.43.mlp.experts.14.gate_proj", + "model.layers.43.mlp.experts.14.up_proj", + "model.layers.43.mlp.experts.140.down_proj", + "model.layers.43.mlp.experts.140.gate_proj", + "model.layers.43.mlp.experts.140.up_proj", + "model.layers.43.mlp.experts.141.down_proj", + "model.layers.43.mlp.experts.141.gate_proj", + "model.layers.43.mlp.experts.141.up_proj", + "model.layers.43.mlp.experts.142.down_proj", + "model.layers.43.mlp.experts.142.gate_proj", + "model.layers.43.mlp.experts.142.up_proj", + "model.layers.43.mlp.experts.143.down_proj", + "model.layers.43.mlp.experts.143.gate_proj", + "model.layers.43.mlp.experts.143.up_proj", + "model.layers.43.mlp.experts.144.down_proj", + "model.layers.43.mlp.experts.144.gate_proj", + "model.layers.43.mlp.experts.144.up_proj", + "model.layers.43.mlp.experts.145.down_proj", + "model.layers.43.mlp.experts.145.gate_proj", + "model.layers.43.mlp.experts.145.up_proj", + "model.layers.43.mlp.experts.146.down_proj", + "model.layers.43.mlp.experts.146.gate_proj", + "model.layers.43.mlp.experts.146.up_proj", + "model.layers.43.mlp.experts.147.down_proj", + "model.layers.43.mlp.experts.147.gate_proj", + "model.layers.43.mlp.experts.147.up_proj", + "model.layers.43.mlp.experts.148.down_proj", + "model.layers.43.mlp.experts.148.gate_proj", + "model.layers.43.mlp.experts.148.up_proj", + "model.layers.43.mlp.experts.149.down_proj", + "model.layers.43.mlp.experts.149.gate_proj", + "model.layers.43.mlp.experts.149.up_proj", + "model.layers.43.mlp.experts.15.down_proj", + "model.layers.43.mlp.experts.15.gate_proj", + "model.layers.43.mlp.experts.15.up_proj", + "model.layers.43.mlp.experts.150.down_proj", + "model.layers.43.mlp.experts.150.gate_proj", + "model.layers.43.mlp.experts.150.up_proj", + "model.layers.43.mlp.experts.151.down_proj", + "model.layers.43.mlp.experts.151.gate_proj", + "model.layers.43.mlp.experts.151.up_proj", + "model.layers.43.mlp.experts.152.down_proj", + "model.layers.43.mlp.experts.152.gate_proj", + "model.layers.43.mlp.experts.152.up_proj", + "model.layers.43.mlp.experts.153.down_proj", + "model.layers.43.mlp.experts.153.gate_proj", + "model.layers.43.mlp.experts.153.up_proj", + "model.layers.43.mlp.experts.154.down_proj", + "model.layers.43.mlp.experts.154.gate_proj", + "model.layers.43.mlp.experts.154.up_proj", + "model.layers.43.mlp.experts.155.down_proj", + "model.layers.43.mlp.experts.155.gate_proj", + "model.layers.43.mlp.experts.155.up_proj", + "model.layers.43.mlp.experts.156.down_proj", + "model.layers.43.mlp.experts.156.gate_proj", + "model.layers.43.mlp.experts.156.up_proj", + "model.layers.43.mlp.experts.157.down_proj", + "model.layers.43.mlp.experts.157.gate_proj", + "model.layers.43.mlp.experts.157.up_proj", + "model.layers.43.mlp.experts.158.down_proj", + "model.layers.43.mlp.experts.158.gate_proj", + "model.layers.43.mlp.experts.158.up_proj", + "model.layers.43.mlp.experts.159.down_proj", + "model.layers.43.mlp.experts.159.gate_proj", + "model.layers.43.mlp.experts.159.up_proj", + "model.layers.43.mlp.experts.16.down_proj", + "model.layers.43.mlp.experts.16.gate_proj", + "model.layers.43.mlp.experts.16.up_proj", + "model.layers.43.mlp.experts.160.down_proj", + "model.layers.43.mlp.experts.160.gate_proj", + "model.layers.43.mlp.experts.160.up_proj", + "model.layers.43.mlp.experts.161.down_proj", + "model.layers.43.mlp.experts.161.gate_proj", + "model.layers.43.mlp.experts.161.up_proj", + "model.layers.43.mlp.experts.162.down_proj", + "model.layers.43.mlp.experts.162.gate_proj", + "model.layers.43.mlp.experts.162.up_proj", + "model.layers.43.mlp.experts.163.down_proj", + "model.layers.43.mlp.experts.163.gate_proj", + "model.layers.43.mlp.experts.163.up_proj", + "model.layers.43.mlp.experts.164.down_proj", + "model.layers.43.mlp.experts.164.gate_proj", + "model.layers.43.mlp.experts.164.up_proj", + "model.layers.43.mlp.experts.165.down_proj", + "model.layers.43.mlp.experts.165.gate_proj", + "model.layers.43.mlp.experts.165.up_proj", + "model.layers.43.mlp.experts.166.down_proj", + "model.layers.43.mlp.experts.166.gate_proj", + "model.layers.43.mlp.experts.166.up_proj", + "model.layers.43.mlp.experts.167.down_proj", + "model.layers.43.mlp.experts.167.gate_proj", + "model.layers.43.mlp.experts.167.up_proj", + "model.layers.43.mlp.experts.168.down_proj", + "model.layers.43.mlp.experts.168.gate_proj", + "model.layers.43.mlp.experts.168.up_proj", + "model.layers.43.mlp.experts.169.down_proj", + "model.layers.43.mlp.experts.169.gate_proj", + "model.layers.43.mlp.experts.169.up_proj", + "model.layers.43.mlp.experts.17.down_proj", + "model.layers.43.mlp.experts.17.gate_proj", + "model.layers.43.mlp.experts.17.up_proj", + "model.layers.43.mlp.experts.170.down_proj", + "model.layers.43.mlp.experts.170.gate_proj", + "model.layers.43.mlp.experts.170.up_proj", + "model.layers.43.mlp.experts.171.down_proj", + "model.layers.43.mlp.experts.171.gate_proj", + "model.layers.43.mlp.experts.171.up_proj", + "model.layers.43.mlp.experts.172.down_proj", + "model.layers.43.mlp.experts.172.gate_proj", + "model.layers.43.mlp.experts.172.up_proj", + "model.layers.43.mlp.experts.173.down_proj", + "model.layers.43.mlp.experts.173.gate_proj", + "model.layers.43.mlp.experts.173.up_proj", + "model.layers.43.mlp.experts.174.down_proj", + "model.layers.43.mlp.experts.174.gate_proj", + "model.layers.43.mlp.experts.174.up_proj", + "model.layers.43.mlp.experts.175.down_proj", + "model.layers.43.mlp.experts.175.gate_proj", + "model.layers.43.mlp.experts.175.up_proj", + "model.layers.43.mlp.experts.176.down_proj", + "model.layers.43.mlp.experts.176.gate_proj", + "model.layers.43.mlp.experts.176.up_proj", + "model.layers.43.mlp.experts.177.down_proj", + "model.layers.43.mlp.experts.177.gate_proj", + "model.layers.43.mlp.experts.177.up_proj", + "model.layers.43.mlp.experts.178.down_proj", + "model.layers.43.mlp.experts.178.gate_proj", + "model.layers.43.mlp.experts.178.up_proj", + "model.layers.43.mlp.experts.179.down_proj", + "model.layers.43.mlp.experts.179.gate_proj", + "model.layers.43.mlp.experts.179.up_proj", + "model.layers.43.mlp.experts.18.down_proj", + "model.layers.43.mlp.experts.18.gate_proj", + "model.layers.43.mlp.experts.18.up_proj", + "model.layers.43.mlp.experts.180.down_proj", + "model.layers.43.mlp.experts.180.gate_proj", + "model.layers.43.mlp.experts.180.up_proj", + "model.layers.43.mlp.experts.181.down_proj", + "model.layers.43.mlp.experts.181.gate_proj", + "model.layers.43.mlp.experts.181.up_proj", + "model.layers.43.mlp.experts.182.down_proj", + "model.layers.43.mlp.experts.182.gate_proj", + "model.layers.43.mlp.experts.182.up_proj", + "model.layers.43.mlp.experts.183.down_proj", + "model.layers.43.mlp.experts.183.gate_proj", + "model.layers.43.mlp.experts.183.up_proj", + "model.layers.43.mlp.experts.184.down_proj", + "model.layers.43.mlp.experts.184.gate_proj", + "model.layers.43.mlp.experts.184.up_proj", + "model.layers.43.mlp.experts.185.down_proj", + "model.layers.43.mlp.experts.185.gate_proj", + "model.layers.43.mlp.experts.185.up_proj", + "model.layers.43.mlp.experts.186.down_proj", + "model.layers.43.mlp.experts.186.gate_proj", + "model.layers.43.mlp.experts.186.up_proj", + "model.layers.43.mlp.experts.187.down_proj", + "model.layers.43.mlp.experts.187.gate_proj", + "model.layers.43.mlp.experts.187.up_proj", + "model.layers.43.mlp.experts.188.down_proj", + "model.layers.43.mlp.experts.188.gate_proj", + "model.layers.43.mlp.experts.188.up_proj", + "model.layers.43.mlp.experts.189.down_proj", + "model.layers.43.mlp.experts.189.gate_proj", + "model.layers.43.mlp.experts.189.up_proj", + "model.layers.43.mlp.experts.19.down_proj", + "model.layers.43.mlp.experts.19.gate_proj", + "model.layers.43.mlp.experts.19.up_proj", + "model.layers.43.mlp.experts.190.down_proj", + "model.layers.43.mlp.experts.190.gate_proj", + "model.layers.43.mlp.experts.190.up_proj", + "model.layers.43.mlp.experts.191.down_proj", + "model.layers.43.mlp.experts.191.gate_proj", + "model.layers.43.mlp.experts.191.up_proj", + "model.layers.43.mlp.experts.192.down_proj", + "model.layers.43.mlp.experts.192.gate_proj", + "model.layers.43.mlp.experts.192.up_proj", + "model.layers.43.mlp.experts.193.down_proj", + "model.layers.43.mlp.experts.193.gate_proj", + "model.layers.43.mlp.experts.193.up_proj", + "model.layers.43.mlp.experts.194.down_proj", + "model.layers.43.mlp.experts.194.gate_proj", + "model.layers.43.mlp.experts.194.up_proj", + "model.layers.43.mlp.experts.195.down_proj", + "model.layers.43.mlp.experts.195.gate_proj", + "model.layers.43.mlp.experts.195.up_proj", + "model.layers.43.mlp.experts.196.down_proj", + "model.layers.43.mlp.experts.196.gate_proj", + "model.layers.43.mlp.experts.196.up_proj", + "model.layers.43.mlp.experts.197.down_proj", + "model.layers.43.mlp.experts.197.gate_proj", + "model.layers.43.mlp.experts.197.up_proj", + "model.layers.43.mlp.experts.198.down_proj", + "model.layers.43.mlp.experts.198.gate_proj", + "model.layers.43.mlp.experts.198.up_proj", + "model.layers.43.mlp.experts.199.down_proj", + "model.layers.43.mlp.experts.199.gate_proj", + "model.layers.43.mlp.experts.199.up_proj", + "model.layers.43.mlp.experts.2.down_proj", + "model.layers.43.mlp.experts.2.gate_proj", + "model.layers.43.mlp.experts.2.up_proj", + "model.layers.43.mlp.experts.20.down_proj", + "model.layers.43.mlp.experts.20.gate_proj", + "model.layers.43.mlp.experts.20.up_proj", + "model.layers.43.mlp.experts.200.down_proj", + "model.layers.43.mlp.experts.200.gate_proj", + "model.layers.43.mlp.experts.200.up_proj", + "model.layers.43.mlp.experts.201.down_proj", + "model.layers.43.mlp.experts.201.gate_proj", + "model.layers.43.mlp.experts.201.up_proj", + "model.layers.43.mlp.experts.202.down_proj", + "model.layers.43.mlp.experts.202.gate_proj", + "model.layers.43.mlp.experts.202.up_proj", + "model.layers.43.mlp.experts.203.down_proj", + "model.layers.43.mlp.experts.203.gate_proj", + "model.layers.43.mlp.experts.203.up_proj", + "model.layers.43.mlp.experts.204.down_proj", + "model.layers.43.mlp.experts.204.gate_proj", + "model.layers.43.mlp.experts.204.up_proj", + "model.layers.43.mlp.experts.205.down_proj", + "model.layers.43.mlp.experts.205.gate_proj", + "model.layers.43.mlp.experts.205.up_proj", + "model.layers.43.mlp.experts.206.down_proj", + "model.layers.43.mlp.experts.206.gate_proj", + "model.layers.43.mlp.experts.206.up_proj", + "model.layers.43.mlp.experts.207.down_proj", + "model.layers.43.mlp.experts.207.gate_proj", + "model.layers.43.mlp.experts.207.up_proj", + "model.layers.43.mlp.experts.208.down_proj", + "model.layers.43.mlp.experts.208.gate_proj", + "model.layers.43.mlp.experts.208.up_proj", + "model.layers.43.mlp.experts.209.down_proj", + "model.layers.43.mlp.experts.209.gate_proj", + "model.layers.43.mlp.experts.209.up_proj", + "model.layers.43.mlp.experts.21.down_proj", + "model.layers.43.mlp.experts.21.gate_proj", + "model.layers.43.mlp.experts.21.up_proj", + "model.layers.43.mlp.experts.210.down_proj", + "model.layers.43.mlp.experts.210.gate_proj", + "model.layers.43.mlp.experts.210.up_proj", + "model.layers.43.mlp.experts.211.down_proj", + "model.layers.43.mlp.experts.211.gate_proj", + "model.layers.43.mlp.experts.211.up_proj", + "model.layers.43.mlp.experts.212.down_proj", + "model.layers.43.mlp.experts.212.gate_proj", + "model.layers.43.mlp.experts.212.up_proj", + "model.layers.43.mlp.experts.213.down_proj", + "model.layers.43.mlp.experts.213.gate_proj", + "model.layers.43.mlp.experts.213.up_proj", + "model.layers.43.mlp.experts.214.down_proj", + "model.layers.43.mlp.experts.214.gate_proj", + "model.layers.43.mlp.experts.214.up_proj", + "model.layers.43.mlp.experts.215.down_proj", + "model.layers.43.mlp.experts.215.gate_proj", + "model.layers.43.mlp.experts.215.up_proj", + "model.layers.43.mlp.experts.216.down_proj", + "model.layers.43.mlp.experts.216.gate_proj", + "model.layers.43.mlp.experts.216.up_proj", + "model.layers.43.mlp.experts.217.down_proj", + "model.layers.43.mlp.experts.217.gate_proj", + "model.layers.43.mlp.experts.217.up_proj", + "model.layers.43.mlp.experts.218.down_proj", + "model.layers.43.mlp.experts.218.gate_proj", + "model.layers.43.mlp.experts.218.up_proj", + "model.layers.43.mlp.experts.219.down_proj", + "model.layers.43.mlp.experts.219.gate_proj", + "model.layers.43.mlp.experts.219.up_proj", + "model.layers.43.mlp.experts.22.down_proj", + "model.layers.43.mlp.experts.22.gate_proj", + "model.layers.43.mlp.experts.22.up_proj", + "model.layers.43.mlp.experts.220.down_proj", + "model.layers.43.mlp.experts.220.gate_proj", + "model.layers.43.mlp.experts.220.up_proj", + "model.layers.43.mlp.experts.221.down_proj", + "model.layers.43.mlp.experts.221.gate_proj", + "model.layers.43.mlp.experts.221.up_proj", + "model.layers.43.mlp.experts.222.down_proj", + "model.layers.43.mlp.experts.222.gate_proj", + "model.layers.43.mlp.experts.222.up_proj", + "model.layers.43.mlp.experts.223.down_proj", + "model.layers.43.mlp.experts.223.gate_proj", + "model.layers.43.mlp.experts.223.up_proj", + "model.layers.43.mlp.experts.224.down_proj", + "model.layers.43.mlp.experts.224.gate_proj", + "model.layers.43.mlp.experts.224.up_proj", + "model.layers.43.mlp.experts.225.down_proj", + "model.layers.43.mlp.experts.225.gate_proj", + "model.layers.43.mlp.experts.225.up_proj", + "model.layers.43.mlp.experts.226.down_proj", + "model.layers.43.mlp.experts.226.gate_proj", + "model.layers.43.mlp.experts.226.up_proj", + "model.layers.43.mlp.experts.227.down_proj", + "model.layers.43.mlp.experts.227.gate_proj", + "model.layers.43.mlp.experts.227.up_proj", + "model.layers.43.mlp.experts.228.down_proj", + "model.layers.43.mlp.experts.228.gate_proj", + "model.layers.43.mlp.experts.228.up_proj", + "model.layers.43.mlp.experts.229.down_proj", + "model.layers.43.mlp.experts.229.gate_proj", + "model.layers.43.mlp.experts.229.up_proj", + "model.layers.43.mlp.experts.23.down_proj", + "model.layers.43.mlp.experts.23.gate_proj", + "model.layers.43.mlp.experts.23.up_proj", + "model.layers.43.mlp.experts.230.down_proj", + "model.layers.43.mlp.experts.230.gate_proj", + "model.layers.43.mlp.experts.230.up_proj", + "model.layers.43.mlp.experts.231.down_proj", + "model.layers.43.mlp.experts.231.gate_proj", + "model.layers.43.mlp.experts.231.up_proj", + "model.layers.43.mlp.experts.232.down_proj", + "model.layers.43.mlp.experts.232.gate_proj", + "model.layers.43.mlp.experts.232.up_proj", + "model.layers.43.mlp.experts.233.down_proj", + "model.layers.43.mlp.experts.233.gate_proj", + "model.layers.43.mlp.experts.233.up_proj", + "model.layers.43.mlp.experts.234.down_proj", + "model.layers.43.mlp.experts.234.gate_proj", + "model.layers.43.mlp.experts.234.up_proj", + "model.layers.43.mlp.experts.235.down_proj", + "model.layers.43.mlp.experts.235.gate_proj", + "model.layers.43.mlp.experts.235.up_proj", + "model.layers.43.mlp.experts.236.down_proj", + "model.layers.43.mlp.experts.236.gate_proj", + "model.layers.43.mlp.experts.236.up_proj", + "model.layers.43.mlp.experts.237.down_proj", + "model.layers.43.mlp.experts.237.gate_proj", + "model.layers.43.mlp.experts.237.up_proj", + "model.layers.43.mlp.experts.238.down_proj", + "model.layers.43.mlp.experts.238.gate_proj", + "model.layers.43.mlp.experts.238.up_proj", + "model.layers.43.mlp.experts.239.down_proj", + "model.layers.43.mlp.experts.239.gate_proj", + "model.layers.43.mlp.experts.239.up_proj", + "model.layers.43.mlp.experts.24.down_proj", + "model.layers.43.mlp.experts.24.gate_proj", + "model.layers.43.mlp.experts.24.up_proj", + "model.layers.43.mlp.experts.240.down_proj", + "model.layers.43.mlp.experts.240.gate_proj", + "model.layers.43.mlp.experts.240.up_proj", + "model.layers.43.mlp.experts.241.down_proj", + "model.layers.43.mlp.experts.241.gate_proj", + "model.layers.43.mlp.experts.241.up_proj", + "model.layers.43.mlp.experts.242.down_proj", + "model.layers.43.mlp.experts.242.gate_proj", + "model.layers.43.mlp.experts.242.up_proj", + "model.layers.43.mlp.experts.243.down_proj", + "model.layers.43.mlp.experts.243.gate_proj", + "model.layers.43.mlp.experts.243.up_proj", + "model.layers.43.mlp.experts.244.down_proj", + "model.layers.43.mlp.experts.244.gate_proj", + "model.layers.43.mlp.experts.244.up_proj", + "model.layers.43.mlp.experts.245.down_proj", + "model.layers.43.mlp.experts.245.gate_proj", + "model.layers.43.mlp.experts.245.up_proj", + "model.layers.43.mlp.experts.246.down_proj", + "model.layers.43.mlp.experts.246.gate_proj", + "model.layers.43.mlp.experts.246.up_proj", + "model.layers.43.mlp.experts.247.down_proj", + "model.layers.43.mlp.experts.247.gate_proj", + "model.layers.43.mlp.experts.247.up_proj", + "model.layers.43.mlp.experts.248.down_proj", + "model.layers.43.mlp.experts.248.gate_proj", + "model.layers.43.mlp.experts.248.up_proj", + "model.layers.43.mlp.experts.249.down_proj", + "model.layers.43.mlp.experts.249.gate_proj", + "model.layers.43.mlp.experts.249.up_proj", + "model.layers.43.mlp.experts.25.down_proj", + "model.layers.43.mlp.experts.25.gate_proj", + "model.layers.43.mlp.experts.25.up_proj", + "model.layers.43.mlp.experts.250.down_proj", + "model.layers.43.mlp.experts.250.gate_proj", + "model.layers.43.mlp.experts.250.up_proj", + "model.layers.43.mlp.experts.251.down_proj", + "model.layers.43.mlp.experts.251.gate_proj", + "model.layers.43.mlp.experts.251.up_proj", + "model.layers.43.mlp.experts.252.down_proj", + "model.layers.43.mlp.experts.252.gate_proj", + "model.layers.43.mlp.experts.252.up_proj", + "model.layers.43.mlp.experts.253.down_proj", + "model.layers.43.mlp.experts.253.gate_proj", + "model.layers.43.mlp.experts.253.up_proj", + "model.layers.43.mlp.experts.254.down_proj", + "model.layers.43.mlp.experts.254.gate_proj", + "model.layers.43.mlp.experts.254.up_proj", + "model.layers.43.mlp.experts.255.down_proj", + "model.layers.43.mlp.experts.255.gate_proj", + "model.layers.43.mlp.experts.255.up_proj", + "model.layers.43.mlp.experts.26.down_proj", + "model.layers.43.mlp.experts.26.gate_proj", + "model.layers.43.mlp.experts.26.up_proj", + "model.layers.43.mlp.experts.27.down_proj", + "model.layers.43.mlp.experts.27.gate_proj", + "model.layers.43.mlp.experts.27.up_proj", + "model.layers.43.mlp.experts.28.down_proj", + "model.layers.43.mlp.experts.28.gate_proj", + "model.layers.43.mlp.experts.28.up_proj", + "model.layers.43.mlp.experts.29.down_proj", + "model.layers.43.mlp.experts.29.gate_proj", + "model.layers.43.mlp.experts.29.up_proj", + "model.layers.43.mlp.experts.3.down_proj", + "model.layers.43.mlp.experts.3.gate_proj", + "model.layers.43.mlp.experts.3.up_proj", + "model.layers.43.mlp.experts.30.down_proj", + "model.layers.43.mlp.experts.30.gate_proj", + "model.layers.43.mlp.experts.30.up_proj", + "model.layers.43.mlp.experts.31.down_proj", + "model.layers.43.mlp.experts.31.gate_proj", + "model.layers.43.mlp.experts.31.up_proj", + "model.layers.43.mlp.experts.32.down_proj", + "model.layers.43.mlp.experts.32.gate_proj", + "model.layers.43.mlp.experts.32.up_proj", + "model.layers.43.mlp.experts.33.down_proj", + "model.layers.43.mlp.experts.33.gate_proj", + "model.layers.43.mlp.experts.33.up_proj", + "model.layers.43.mlp.experts.34.down_proj", + "model.layers.43.mlp.experts.34.gate_proj", + "model.layers.43.mlp.experts.34.up_proj", + "model.layers.43.mlp.experts.35.down_proj", + "model.layers.43.mlp.experts.35.gate_proj", + "model.layers.43.mlp.experts.35.up_proj", + "model.layers.43.mlp.experts.36.down_proj", + "model.layers.43.mlp.experts.36.gate_proj", + "model.layers.43.mlp.experts.36.up_proj", + "model.layers.43.mlp.experts.37.down_proj", + "model.layers.43.mlp.experts.37.gate_proj", + "model.layers.43.mlp.experts.37.up_proj", + "model.layers.43.mlp.experts.38.down_proj", + "model.layers.43.mlp.experts.38.gate_proj", + "model.layers.43.mlp.experts.38.up_proj", + "model.layers.43.mlp.experts.39.down_proj", + "model.layers.43.mlp.experts.39.gate_proj", + "model.layers.43.mlp.experts.39.up_proj", + "model.layers.43.mlp.experts.4.down_proj", + "model.layers.43.mlp.experts.4.gate_proj", + "model.layers.43.mlp.experts.4.up_proj", + "model.layers.43.mlp.experts.40.down_proj", + "model.layers.43.mlp.experts.40.gate_proj", + "model.layers.43.mlp.experts.40.up_proj", + "model.layers.43.mlp.experts.41.down_proj", + "model.layers.43.mlp.experts.41.gate_proj", + "model.layers.43.mlp.experts.41.up_proj", + "model.layers.43.mlp.experts.42.down_proj", + "model.layers.43.mlp.experts.42.gate_proj", + "model.layers.43.mlp.experts.42.up_proj", + "model.layers.43.mlp.experts.43.down_proj", + "model.layers.43.mlp.experts.43.gate_proj", + "model.layers.43.mlp.experts.43.up_proj", + "model.layers.43.mlp.experts.44.down_proj", + "model.layers.43.mlp.experts.44.gate_proj", + "model.layers.43.mlp.experts.44.up_proj", + "model.layers.43.mlp.experts.45.down_proj", + "model.layers.43.mlp.experts.45.gate_proj", + "model.layers.43.mlp.experts.45.up_proj", + "model.layers.43.mlp.experts.46.down_proj", + "model.layers.43.mlp.experts.46.gate_proj", + "model.layers.43.mlp.experts.46.up_proj", + "model.layers.43.mlp.experts.47.down_proj", + "model.layers.43.mlp.experts.47.gate_proj", + "model.layers.43.mlp.experts.47.up_proj", + "model.layers.43.mlp.experts.48.down_proj", + "model.layers.43.mlp.experts.48.gate_proj", + "model.layers.43.mlp.experts.48.up_proj", + "model.layers.43.mlp.experts.49.down_proj", + "model.layers.43.mlp.experts.49.gate_proj", + "model.layers.43.mlp.experts.49.up_proj", + "model.layers.43.mlp.experts.5.down_proj", + "model.layers.43.mlp.experts.5.gate_proj", + "model.layers.43.mlp.experts.5.up_proj", + "model.layers.43.mlp.experts.50.down_proj", + "model.layers.43.mlp.experts.50.gate_proj", + "model.layers.43.mlp.experts.50.up_proj", + "model.layers.43.mlp.experts.51.down_proj", + "model.layers.43.mlp.experts.51.gate_proj", + "model.layers.43.mlp.experts.51.up_proj", + "model.layers.43.mlp.experts.52.down_proj", + "model.layers.43.mlp.experts.52.gate_proj", + "model.layers.43.mlp.experts.52.up_proj", + "model.layers.43.mlp.experts.53.down_proj", + "model.layers.43.mlp.experts.53.gate_proj", + "model.layers.43.mlp.experts.53.up_proj", + "model.layers.43.mlp.experts.54.down_proj", + "model.layers.43.mlp.experts.54.gate_proj", + "model.layers.43.mlp.experts.54.up_proj", + "model.layers.43.mlp.experts.55.down_proj", + "model.layers.43.mlp.experts.55.gate_proj", + "model.layers.43.mlp.experts.55.up_proj", + "model.layers.43.mlp.experts.56.down_proj", + "model.layers.43.mlp.experts.56.gate_proj", + "model.layers.43.mlp.experts.56.up_proj", + "model.layers.43.mlp.experts.57.down_proj", + "model.layers.43.mlp.experts.57.gate_proj", + "model.layers.43.mlp.experts.57.up_proj", + "model.layers.43.mlp.experts.58.down_proj", + "model.layers.43.mlp.experts.58.gate_proj", + "model.layers.43.mlp.experts.58.up_proj", + "model.layers.43.mlp.experts.59.down_proj", + "model.layers.43.mlp.experts.59.gate_proj", + "model.layers.43.mlp.experts.59.up_proj", + "model.layers.43.mlp.experts.6.down_proj", + "model.layers.43.mlp.experts.6.gate_proj", + "model.layers.43.mlp.experts.6.up_proj", + "model.layers.43.mlp.experts.60.down_proj", + "model.layers.43.mlp.experts.60.gate_proj", + "model.layers.43.mlp.experts.60.up_proj", + "model.layers.43.mlp.experts.61.down_proj", + "model.layers.43.mlp.experts.61.gate_proj", + "model.layers.43.mlp.experts.61.up_proj", + "model.layers.43.mlp.experts.62.down_proj", + "model.layers.43.mlp.experts.62.gate_proj", + "model.layers.43.mlp.experts.62.up_proj", + "model.layers.43.mlp.experts.63.down_proj", + "model.layers.43.mlp.experts.63.gate_proj", + "model.layers.43.mlp.experts.63.up_proj", + "model.layers.43.mlp.experts.64.down_proj", + "model.layers.43.mlp.experts.64.gate_proj", + "model.layers.43.mlp.experts.64.up_proj", + "model.layers.43.mlp.experts.65.down_proj", + "model.layers.43.mlp.experts.65.gate_proj", + "model.layers.43.mlp.experts.65.up_proj", + "model.layers.43.mlp.experts.66.down_proj", + "model.layers.43.mlp.experts.66.gate_proj", + "model.layers.43.mlp.experts.66.up_proj", + "model.layers.43.mlp.experts.67.down_proj", + "model.layers.43.mlp.experts.67.gate_proj", + "model.layers.43.mlp.experts.67.up_proj", + "model.layers.43.mlp.experts.68.down_proj", + "model.layers.43.mlp.experts.68.gate_proj", + "model.layers.43.mlp.experts.68.up_proj", + "model.layers.43.mlp.experts.69.down_proj", + "model.layers.43.mlp.experts.69.gate_proj", + "model.layers.43.mlp.experts.69.up_proj", + "model.layers.43.mlp.experts.7.down_proj", + "model.layers.43.mlp.experts.7.gate_proj", + "model.layers.43.mlp.experts.7.up_proj", + "model.layers.43.mlp.experts.70.down_proj", + "model.layers.43.mlp.experts.70.gate_proj", + "model.layers.43.mlp.experts.70.up_proj", + "model.layers.43.mlp.experts.71.down_proj", + "model.layers.43.mlp.experts.71.gate_proj", + "model.layers.43.mlp.experts.71.up_proj", + "model.layers.43.mlp.experts.72.down_proj", + "model.layers.43.mlp.experts.72.gate_proj", + "model.layers.43.mlp.experts.72.up_proj", + "model.layers.43.mlp.experts.73.down_proj", + "model.layers.43.mlp.experts.73.gate_proj", + "model.layers.43.mlp.experts.73.up_proj", + "model.layers.43.mlp.experts.74.down_proj", + "model.layers.43.mlp.experts.74.gate_proj", + "model.layers.43.mlp.experts.74.up_proj", + "model.layers.43.mlp.experts.75.down_proj", + "model.layers.43.mlp.experts.75.gate_proj", + "model.layers.43.mlp.experts.75.up_proj", + "model.layers.43.mlp.experts.76.down_proj", + "model.layers.43.mlp.experts.76.gate_proj", + "model.layers.43.mlp.experts.76.up_proj", + "model.layers.43.mlp.experts.77.down_proj", + "model.layers.43.mlp.experts.77.gate_proj", + "model.layers.43.mlp.experts.77.up_proj", + "model.layers.43.mlp.experts.78.down_proj", + "model.layers.43.mlp.experts.78.gate_proj", + "model.layers.43.mlp.experts.78.up_proj", + "model.layers.43.mlp.experts.79.down_proj", + "model.layers.43.mlp.experts.79.gate_proj", + "model.layers.43.mlp.experts.79.up_proj", + "model.layers.43.mlp.experts.8.down_proj", + "model.layers.43.mlp.experts.8.gate_proj", + "model.layers.43.mlp.experts.8.up_proj", + "model.layers.43.mlp.experts.80.down_proj", + "model.layers.43.mlp.experts.80.gate_proj", + "model.layers.43.mlp.experts.80.up_proj", + "model.layers.43.mlp.experts.81.down_proj", + "model.layers.43.mlp.experts.81.gate_proj", + "model.layers.43.mlp.experts.81.up_proj", + "model.layers.43.mlp.experts.82.down_proj", + "model.layers.43.mlp.experts.82.gate_proj", + "model.layers.43.mlp.experts.82.up_proj", + "model.layers.43.mlp.experts.83.down_proj", + "model.layers.43.mlp.experts.83.gate_proj", + "model.layers.43.mlp.experts.83.up_proj", + "model.layers.43.mlp.experts.84.down_proj", + "model.layers.43.mlp.experts.84.gate_proj", + "model.layers.43.mlp.experts.84.up_proj", + "model.layers.43.mlp.experts.85.down_proj", + "model.layers.43.mlp.experts.85.gate_proj", + "model.layers.43.mlp.experts.85.up_proj", + "model.layers.43.mlp.experts.86.down_proj", + "model.layers.43.mlp.experts.86.gate_proj", + "model.layers.43.mlp.experts.86.up_proj", + "model.layers.43.mlp.experts.87.down_proj", + "model.layers.43.mlp.experts.87.gate_proj", + "model.layers.43.mlp.experts.87.up_proj", + "model.layers.43.mlp.experts.88.down_proj", + "model.layers.43.mlp.experts.88.gate_proj", + "model.layers.43.mlp.experts.88.up_proj", + "model.layers.43.mlp.experts.89.down_proj", + "model.layers.43.mlp.experts.89.gate_proj", + "model.layers.43.mlp.experts.89.up_proj", + "model.layers.43.mlp.experts.9.down_proj", + "model.layers.43.mlp.experts.9.gate_proj", + "model.layers.43.mlp.experts.9.up_proj", + "model.layers.43.mlp.experts.90.down_proj", + "model.layers.43.mlp.experts.90.gate_proj", + "model.layers.43.mlp.experts.90.up_proj", + "model.layers.43.mlp.experts.91.down_proj", + "model.layers.43.mlp.experts.91.gate_proj", + "model.layers.43.mlp.experts.91.up_proj", + "model.layers.43.mlp.experts.92.down_proj", + "model.layers.43.mlp.experts.92.gate_proj", + "model.layers.43.mlp.experts.92.up_proj", + "model.layers.43.mlp.experts.93.down_proj", + "model.layers.43.mlp.experts.93.gate_proj", + "model.layers.43.mlp.experts.93.up_proj", + "model.layers.43.mlp.experts.94.down_proj", + "model.layers.43.mlp.experts.94.gate_proj", + "model.layers.43.mlp.experts.94.up_proj", + "model.layers.43.mlp.experts.95.down_proj", + "model.layers.43.mlp.experts.95.gate_proj", + "model.layers.43.mlp.experts.95.up_proj", + "model.layers.43.mlp.experts.96.down_proj", + "model.layers.43.mlp.experts.96.gate_proj", + "model.layers.43.mlp.experts.96.up_proj", + "model.layers.43.mlp.experts.97.down_proj", + "model.layers.43.mlp.experts.97.gate_proj", + "model.layers.43.mlp.experts.97.up_proj", + "model.layers.43.mlp.experts.98.down_proj", + "model.layers.43.mlp.experts.98.gate_proj", + "model.layers.43.mlp.experts.98.up_proj", + "model.layers.43.mlp.experts.99.down_proj", + "model.layers.43.mlp.experts.99.gate_proj", + "model.layers.43.mlp.experts.99.up_proj", + "model.layers.44.mlp.experts.0.down_proj", + "model.layers.44.mlp.experts.0.gate_proj", + "model.layers.44.mlp.experts.0.up_proj", + "model.layers.44.mlp.experts.1.down_proj", + "model.layers.44.mlp.experts.1.gate_proj", + "model.layers.44.mlp.experts.1.up_proj", + "model.layers.44.mlp.experts.10.down_proj", + "model.layers.44.mlp.experts.10.gate_proj", + "model.layers.44.mlp.experts.10.up_proj", + "model.layers.44.mlp.experts.100.down_proj", + "model.layers.44.mlp.experts.100.gate_proj", + "model.layers.44.mlp.experts.100.up_proj", + "model.layers.44.mlp.experts.101.down_proj", + "model.layers.44.mlp.experts.101.gate_proj", + "model.layers.44.mlp.experts.101.up_proj", + "model.layers.44.mlp.experts.102.down_proj", + "model.layers.44.mlp.experts.102.gate_proj", + "model.layers.44.mlp.experts.102.up_proj", + "model.layers.44.mlp.experts.103.down_proj", + "model.layers.44.mlp.experts.103.gate_proj", + "model.layers.44.mlp.experts.103.up_proj", + "model.layers.44.mlp.experts.104.down_proj", + "model.layers.44.mlp.experts.104.gate_proj", + "model.layers.44.mlp.experts.104.up_proj", + "model.layers.44.mlp.experts.105.down_proj", + "model.layers.44.mlp.experts.105.gate_proj", + "model.layers.44.mlp.experts.105.up_proj", + "model.layers.44.mlp.experts.106.down_proj", + "model.layers.44.mlp.experts.106.gate_proj", + "model.layers.44.mlp.experts.106.up_proj", + "model.layers.44.mlp.experts.107.down_proj", + "model.layers.44.mlp.experts.107.gate_proj", + "model.layers.44.mlp.experts.107.up_proj", + "model.layers.44.mlp.experts.108.down_proj", + "model.layers.44.mlp.experts.108.gate_proj", + "model.layers.44.mlp.experts.108.up_proj", + "model.layers.44.mlp.experts.109.down_proj", + "model.layers.44.mlp.experts.109.gate_proj", + "model.layers.44.mlp.experts.109.up_proj", + "model.layers.44.mlp.experts.11.down_proj", + "model.layers.44.mlp.experts.11.gate_proj", + "model.layers.44.mlp.experts.11.up_proj", + "model.layers.44.mlp.experts.110.down_proj", + "model.layers.44.mlp.experts.110.gate_proj", + "model.layers.44.mlp.experts.110.up_proj", + "model.layers.44.mlp.experts.111.down_proj", + "model.layers.44.mlp.experts.111.gate_proj", + "model.layers.44.mlp.experts.111.up_proj", + "model.layers.44.mlp.experts.112.down_proj", + "model.layers.44.mlp.experts.112.gate_proj", + "model.layers.44.mlp.experts.112.up_proj", + "model.layers.44.mlp.experts.113.down_proj", + "model.layers.44.mlp.experts.113.gate_proj", + "model.layers.44.mlp.experts.113.up_proj", + "model.layers.44.mlp.experts.114.down_proj", + "model.layers.44.mlp.experts.114.gate_proj", + "model.layers.44.mlp.experts.114.up_proj", + "model.layers.44.mlp.experts.115.down_proj", + "model.layers.44.mlp.experts.115.gate_proj", + "model.layers.44.mlp.experts.115.up_proj", + "model.layers.44.mlp.experts.116.down_proj", + "model.layers.44.mlp.experts.116.gate_proj", + "model.layers.44.mlp.experts.116.up_proj", + "model.layers.44.mlp.experts.117.down_proj", + "model.layers.44.mlp.experts.117.gate_proj", + "model.layers.44.mlp.experts.117.up_proj", + "model.layers.44.mlp.experts.118.down_proj", + "model.layers.44.mlp.experts.118.gate_proj", + "model.layers.44.mlp.experts.118.up_proj", + "model.layers.44.mlp.experts.119.down_proj", + "model.layers.44.mlp.experts.119.gate_proj", + "model.layers.44.mlp.experts.119.up_proj", + "model.layers.44.mlp.experts.12.down_proj", + "model.layers.44.mlp.experts.12.gate_proj", + "model.layers.44.mlp.experts.12.up_proj", + "model.layers.44.mlp.experts.120.down_proj", + "model.layers.44.mlp.experts.120.gate_proj", + "model.layers.44.mlp.experts.120.up_proj", + "model.layers.44.mlp.experts.121.down_proj", + "model.layers.44.mlp.experts.121.gate_proj", + "model.layers.44.mlp.experts.121.up_proj", + "model.layers.44.mlp.experts.122.down_proj", + "model.layers.44.mlp.experts.122.gate_proj", + "model.layers.44.mlp.experts.122.up_proj", + "model.layers.44.mlp.experts.123.down_proj", + "model.layers.44.mlp.experts.123.gate_proj", + "model.layers.44.mlp.experts.123.up_proj", + "model.layers.44.mlp.experts.124.down_proj", + "model.layers.44.mlp.experts.124.gate_proj", + "model.layers.44.mlp.experts.124.up_proj", + "model.layers.44.mlp.experts.125.down_proj", + "model.layers.44.mlp.experts.125.gate_proj", + "model.layers.44.mlp.experts.125.up_proj", + "model.layers.44.mlp.experts.126.down_proj", + "model.layers.44.mlp.experts.126.gate_proj", + "model.layers.44.mlp.experts.126.up_proj", + "model.layers.44.mlp.experts.127.down_proj", + "model.layers.44.mlp.experts.127.gate_proj", + "model.layers.44.mlp.experts.127.up_proj", + "model.layers.44.mlp.experts.128.down_proj", + "model.layers.44.mlp.experts.128.gate_proj", + "model.layers.44.mlp.experts.128.up_proj", + "model.layers.44.mlp.experts.129.down_proj", + "model.layers.44.mlp.experts.129.gate_proj", + "model.layers.44.mlp.experts.129.up_proj", + "model.layers.44.mlp.experts.13.down_proj", + "model.layers.44.mlp.experts.13.gate_proj", + "model.layers.44.mlp.experts.13.up_proj", + "model.layers.44.mlp.experts.130.down_proj", + "model.layers.44.mlp.experts.130.gate_proj", + "model.layers.44.mlp.experts.130.up_proj", + "model.layers.44.mlp.experts.131.down_proj", + "model.layers.44.mlp.experts.131.gate_proj", + "model.layers.44.mlp.experts.131.up_proj", + "model.layers.44.mlp.experts.132.down_proj", + "model.layers.44.mlp.experts.132.gate_proj", + "model.layers.44.mlp.experts.132.up_proj", + "model.layers.44.mlp.experts.133.down_proj", + "model.layers.44.mlp.experts.133.gate_proj", + "model.layers.44.mlp.experts.133.up_proj", + "model.layers.44.mlp.experts.134.down_proj", + "model.layers.44.mlp.experts.134.gate_proj", + "model.layers.44.mlp.experts.134.up_proj", + "model.layers.44.mlp.experts.135.down_proj", + "model.layers.44.mlp.experts.135.gate_proj", + "model.layers.44.mlp.experts.135.up_proj", + "model.layers.44.mlp.experts.136.down_proj", + "model.layers.44.mlp.experts.136.gate_proj", + "model.layers.44.mlp.experts.136.up_proj", + "model.layers.44.mlp.experts.137.down_proj", + "model.layers.44.mlp.experts.137.gate_proj", + "model.layers.44.mlp.experts.137.up_proj", + "model.layers.44.mlp.experts.138.down_proj", + "model.layers.44.mlp.experts.138.gate_proj", + "model.layers.44.mlp.experts.138.up_proj", + "model.layers.44.mlp.experts.139.down_proj", + "model.layers.44.mlp.experts.139.gate_proj", + "model.layers.44.mlp.experts.139.up_proj", + "model.layers.44.mlp.experts.14.down_proj", + "model.layers.44.mlp.experts.14.gate_proj", + "model.layers.44.mlp.experts.14.up_proj", + "model.layers.44.mlp.experts.140.down_proj", + "model.layers.44.mlp.experts.140.gate_proj", + "model.layers.44.mlp.experts.140.up_proj", + "model.layers.44.mlp.experts.141.down_proj", + "model.layers.44.mlp.experts.141.gate_proj", + "model.layers.44.mlp.experts.141.up_proj", + "model.layers.44.mlp.experts.142.down_proj", + "model.layers.44.mlp.experts.142.gate_proj", + "model.layers.44.mlp.experts.142.up_proj", + "model.layers.44.mlp.experts.143.down_proj", + "model.layers.44.mlp.experts.143.gate_proj", + "model.layers.44.mlp.experts.143.up_proj", + "model.layers.44.mlp.experts.144.down_proj", + "model.layers.44.mlp.experts.144.gate_proj", + "model.layers.44.mlp.experts.144.up_proj", + "model.layers.44.mlp.experts.145.down_proj", + "model.layers.44.mlp.experts.145.gate_proj", + "model.layers.44.mlp.experts.145.up_proj", + "model.layers.44.mlp.experts.146.down_proj", + "model.layers.44.mlp.experts.146.gate_proj", + "model.layers.44.mlp.experts.146.up_proj", + "model.layers.44.mlp.experts.147.down_proj", + "model.layers.44.mlp.experts.147.gate_proj", + "model.layers.44.mlp.experts.147.up_proj", + "model.layers.44.mlp.experts.148.down_proj", + "model.layers.44.mlp.experts.148.gate_proj", + "model.layers.44.mlp.experts.148.up_proj", + "model.layers.44.mlp.experts.149.down_proj", + "model.layers.44.mlp.experts.149.gate_proj", + "model.layers.44.mlp.experts.149.up_proj", + "model.layers.44.mlp.experts.15.down_proj", + "model.layers.44.mlp.experts.15.gate_proj", + "model.layers.44.mlp.experts.15.up_proj", + "model.layers.44.mlp.experts.150.down_proj", + "model.layers.44.mlp.experts.150.gate_proj", + "model.layers.44.mlp.experts.150.up_proj", + "model.layers.44.mlp.experts.151.down_proj", + "model.layers.44.mlp.experts.151.gate_proj", + "model.layers.44.mlp.experts.151.up_proj", + "model.layers.44.mlp.experts.152.down_proj", + "model.layers.44.mlp.experts.152.gate_proj", + "model.layers.44.mlp.experts.152.up_proj", + "model.layers.44.mlp.experts.153.down_proj", + "model.layers.44.mlp.experts.153.gate_proj", + "model.layers.44.mlp.experts.153.up_proj", + "model.layers.44.mlp.experts.154.down_proj", + "model.layers.44.mlp.experts.154.gate_proj", + "model.layers.44.mlp.experts.154.up_proj", + "model.layers.44.mlp.experts.155.down_proj", + "model.layers.44.mlp.experts.155.gate_proj", + "model.layers.44.mlp.experts.155.up_proj", + "model.layers.44.mlp.experts.156.down_proj", + "model.layers.44.mlp.experts.156.gate_proj", + "model.layers.44.mlp.experts.156.up_proj", + "model.layers.44.mlp.experts.157.down_proj", + "model.layers.44.mlp.experts.157.gate_proj", + "model.layers.44.mlp.experts.157.up_proj", + "model.layers.44.mlp.experts.158.down_proj", + "model.layers.44.mlp.experts.158.gate_proj", + "model.layers.44.mlp.experts.158.up_proj", + "model.layers.44.mlp.experts.159.down_proj", + "model.layers.44.mlp.experts.159.gate_proj", + "model.layers.44.mlp.experts.159.up_proj", + "model.layers.44.mlp.experts.16.down_proj", + "model.layers.44.mlp.experts.16.gate_proj", + "model.layers.44.mlp.experts.16.up_proj", + "model.layers.44.mlp.experts.160.down_proj", + "model.layers.44.mlp.experts.160.gate_proj", + "model.layers.44.mlp.experts.160.up_proj", + "model.layers.44.mlp.experts.161.down_proj", + "model.layers.44.mlp.experts.161.gate_proj", + "model.layers.44.mlp.experts.161.up_proj", + "model.layers.44.mlp.experts.162.down_proj", + "model.layers.44.mlp.experts.162.gate_proj", + "model.layers.44.mlp.experts.162.up_proj", + "model.layers.44.mlp.experts.163.down_proj", + "model.layers.44.mlp.experts.163.gate_proj", + "model.layers.44.mlp.experts.163.up_proj", + "model.layers.44.mlp.experts.164.down_proj", + "model.layers.44.mlp.experts.164.gate_proj", + "model.layers.44.mlp.experts.164.up_proj", + "model.layers.44.mlp.experts.165.down_proj", + "model.layers.44.mlp.experts.165.gate_proj", + "model.layers.44.mlp.experts.165.up_proj", + "model.layers.44.mlp.experts.166.down_proj", + "model.layers.44.mlp.experts.166.gate_proj", + "model.layers.44.mlp.experts.166.up_proj", + "model.layers.44.mlp.experts.167.down_proj", + "model.layers.44.mlp.experts.167.gate_proj", + "model.layers.44.mlp.experts.167.up_proj", + "model.layers.44.mlp.experts.168.down_proj", + "model.layers.44.mlp.experts.168.gate_proj", + "model.layers.44.mlp.experts.168.up_proj", + "model.layers.44.mlp.experts.169.down_proj", + "model.layers.44.mlp.experts.169.gate_proj", + "model.layers.44.mlp.experts.169.up_proj", + "model.layers.44.mlp.experts.17.down_proj", + "model.layers.44.mlp.experts.17.gate_proj", + "model.layers.44.mlp.experts.17.up_proj", + "model.layers.44.mlp.experts.170.down_proj", + "model.layers.44.mlp.experts.170.gate_proj", + "model.layers.44.mlp.experts.170.up_proj", + "model.layers.44.mlp.experts.171.down_proj", + "model.layers.44.mlp.experts.171.gate_proj", + "model.layers.44.mlp.experts.171.up_proj", + "model.layers.44.mlp.experts.172.down_proj", + "model.layers.44.mlp.experts.172.gate_proj", + "model.layers.44.mlp.experts.172.up_proj", + "model.layers.44.mlp.experts.173.down_proj", + "model.layers.44.mlp.experts.173.gate_proj", + "model.layers.44.mlp.experts.173.up_proj", + "model.layers.44.mlp.experts.174.down_proj", + "model.layers.44.mlp.experts.174.gate_proj", + "model.layers.44.mlp.experts.174.up_proj", + "model.layers.44.mlp.experts.175.down_proj", + "model.layers.44.mlp.experts.175.gate_proj", + "model.layers.44.mlp.experts.175.up_proj", + "model.layers.44.mlp.experts.176.down_proj", + "model.layers.44.mlp.experts.176.gate_proj", + "model.layers.44.mlp.experts.176.up_proj", + "model.layers.44.mlp.experts.177.down_proj", + "model.layers.44.mlp.experts.177.gate_proj", + "model.layers.44.mlp.experts.177.up_proj", + "model.layers.44.mlp.experts.178.down_proj", + "model.layers.44.mlp.experts.178.gate_proj", + "model.layers.44.mlp.experts.178.up_proj", + "model.layers.44.mlp.experts.179.down_proj", + "model.layers.44.mlp.experts.179.gate_proj", + "model.layers.44.mlp.experts.179.up_proj", + "model.layers.44.mlp.experts.18.down_proj", + "model.layers.44.mlp.experts.18.gate_proj", + "model.layers.44.mlp.experts.18.up_proj", + "model.layers.44.mlp.experts.180.down_proj", + "model.layers.44.mlp.experts.180.gate_proj", + "model.layers.44.mlp.experts.180.up_proj", + "model.layers.44.mlp.experts.181.down_proj", + "model.layers.44.mlp.experts.181.gate_proj", + "model.layers.44.mlp.experts.181.up_proj", + "model.layers.44.mlp.experts.182.down_proj", + "model.layers.44.mlp.experts.182.gate_proj", + "model.layers.44.mlp.experts.182.up_proj", + "model.layers.44.mlp.experts.183.down_proj", + "model.layers.44.mlp.experts.183.gate_proj", + "model.layers.44.mlp.experts.183.up_proj", + "model.layers.44.mlp.experts.184.down_proj", + "model.layers.44.mlp.experts.184.gate_proj", + "model.layers.44.mlp.experts.184.up_proj", + "model.layers.44.mlp.experts.185.down_proj", + "model.layers.44.mlp.experts.185.gate_proj", + "model.layers.44.mlp.experts.185.up_proj", + "model.layers.44.mlp.experts.186.down_proj", + "model.layers.44.mlp.experts.186.gate_proj", + "model.layers.44.mlp.experts.186.up_proj", + "model.layers.44.mlp.experts.187.down_proj", + "model.layers.44.mlp.experts.187.gate_proj", + "model.layers.44.mlp.experts.187.up_proj", + "model.layers.44.mlp.experts.188.down_proj", + "model.layers.44.mlp.experts.188.gate_proj", + "model.layers.44.mlp.experts.188.up_proj", + "model.layers.44.mlp.experts.189.down_proj", + "model.layers.44.mlp.experts.189.gate_proj", + "model.layers.44.mlp.experts.189.up_proj", + "model.layers.44.mlp.experts.19.down_proj", + "model.layers.44.mlp.experts.19.gate_proj", + "model.layers.44.mlp.experts.19.up_proj", + "model.layers.44.mlp.experts.190.down_proj", + "model.layers.44.mlp.experts.190.gate_proj", + "model.layers.44.mlp.experts.190.up_proj", + "model.layers.44.mlp.experts.191.down_proj", + "model.layers.44.mlp.experts.191.gate_proj", + "model.layers.44.mlp.experts.191.up_proj", + "model.layers.44.mlp.experts.192.down_proj", + "model.layers.44.mlp.experts.192.gate_proj", + "model.layers.44.mlp.experts.192.up_proj", + "model.layers.44.mlp.experts.193.down_proj", + "model.layers.44.mlp.experts.193.gate_proj", + "model.layers.44.mlp.experts.193.up_proj", + "model.layers.44.mlp.experts.194.down_proj", + "model.layers.44.mlp.experts.194.gate_proj", + "model.layers.44.mlp.experts.194.up_proj", + "model.layers.44.mlp.experts.195.down_proj", + "model.layers.44.mlp.experts.195.gate_proj", + "model.layers.44.mlp.experts.195.up_proj", + "model.layers.44.mlp.experts.196.down_proj", + "model.layers.44.mlp.experts.196.gate_proj", + "model.layers.44.mlp.experts.196.up_proj", + "model.layers.44.mlp.experts.197.down_proj", + "model.layers.44.mlp.experts.197.gate_proj", + "model.layers.44.mlp.experts.197.up_proj", + "model.layers.44.mlp.experts.198.down_proj", + "model.layers.44.mlp.experts.198.gate_proj", + "model.layers.44.mlp.experts.198.up_proj", + "model.layers.44.mlp.experts.199.down_proj", + "model.layers.44.mlp.experts.199.gate_proj", + "model.layers.44.mlp.experts.199.up_proj", + "model.layers.44.mlp.experts.2.down_proj", + "model.layers.44.mlp.experts.2.gate_proj", + "model.layers.44.mlp.experts.2.up_proj", + "model.layers.44.mlp.experts.20.down_proj", + "model.layers.44.mlp.experts.20.gate_proj", + "model.layers.44.mlp.experts.20.up_proj", + "model.layers.44.mlp.experts.200.down_proj", + "model.layers.44.mlp.experts.200.gate_proj", + "model.layers.44.mlp.experts.200.up_proj", + "model.layers.44.mlp.experts.201.down_proj", + "model.layers.44.mlp.experts.201.gate_proj", + "model.layers.44.mlp.experts.201.up_proj", + "model.layers.44.mlp.experts.202.down_proj", + "model.layers.44.mlp.experts.202.gate_proj", + "model.layers.44.mlp.experts.202.up_proj", + "model.layers.44.mlp.experts.203.down_proj", + "model.layers.44.mlp.experts.203.gate_proj", + "model.layers.44.mlp.experts.203.up_proj", + "model.layers.44.mlp.experts.204.down_proj", + "model.layers.44.mlp.experts.204.gate_proj", + "model.layers.44.mlp.experts.204.up_proj", + "model.layers.44.mlp.experts.205.down_proj", + "model.layers.44.mlp.experts.205.gate_proj", + "model.layers.44.mlp.experts.205.up_proj", + "model.layers.44.mlp.experts.206.down_proj", + "model.layers.44.mlp.experts.206.gate_proj", + "model.layers.44.mlp.experts.206.up_proj", + "model.layers.44.mlp.experts.207.down_proj", + "model.layers.44.mlp.experts.207.gate_proj", + "model.layers.44.mlp.experts.207.up_proj", + "model.layers.44.mlp.experts.208.down_proj", + "model.layers.44.mlp.experts.208.gate_proj", + "model.layers.44.mlp.experts.208.up_proj", + "model.layers.44.mlp.experts.209.down_proj", + "model.layers.44.mlp.experts.209.gate_proj", + "model.layers.44.mlp.experts.209.up_proj", + "model.layers.44.mlp.experts.21.down_proj", + "model.layers.44.mlp.experts.21.gate_proj", + "model.layers.44.mlp.experts.21.up_proj", + "model.layers.44.mlp.experts.210.down_proj", + "model.layers.44.mlp.experts.210.gate_proj", + "model.layers.44.mlp.experts.210.up_proj", + "model.layers.44.mlp.experts.211.down_proj", + "model.layers.44.mlp.experts.211.gate_proj", + "model.layers.44.mlp.experts.211.up_proj", + "model.layers.44.mlp.experts.212.down_proj", + "model.layers.44.mlp.experts.212.gate_proj", + "model.layers.44.mlp.experts.212.up_proj", + "model.layers.44.mlp.experts.213.down_proj", + "model.layers.44.mlp.experts.213.gate_proj", + "model.layers.44.mlp.experts.213.up_proj", + "model.layers.44.mlp.experts.214.down_proj", + "model.layers.44.mlp.experts.214.gate_proj", + "model.layers.44.mlp.experts.214.up_proj", + "model.layers.44.mlp.experts.215.down_proj", + "model.layers.44.mlp.experts.215.gate_proj", + "model.layers.44.mlp.experts.215.up_proj", + "model.layers.44.mlp.experts.216.down_proj", + "model.layers.44.mlp.experts.216.gate_proj", + "model.layers.44.mlp.experts.216.up_proj", + "model.layers.44.mlp.experts.217.down_proj", + "model.layers.44.mlp.experts.217.gate_proj", + "model.layers.44.mlp.experts.217.up_proj", + "model.layers.44.mlp.experts.218.down_proj", + "model.layers.44.mlp.experts.218.gate_proj", + "model.layers.44.mlp.experts.218.up_proj", + "model.layers.44.mlp.experts.219.down_proj", + "model.layers.44.mlp.experts.219.gate_proj", + "model.layers.44.mlp.experts.219.up_proj", + "model.layers.44.mlp.experts.22.down_proj", + "model.layers.44.mlp.experts.22.gate_proj", + "model.layers.44.mlp.experts.22.up_proj", + "model.layers.44.mlp.experts.220.down_proj", + "model.layers.44.mlp.experts.220.gate_proj", + "model.layers.44.mlp.experts.220.up_proj", + "model.layers.44.mlp.experts.221.down_proj", + "model.layers.44.mlp.experts.221.gate_proj", + "model.layers.44.mlp.experts.221.up_proj", + "model.layers.44.mlp.experts.222.down_proj", + "model.layers.44.mlp.experts.222.gate_proj", + "model.layers.44.mlp.experts.222.up_proj", + "model.layers.44.mlp.experts.223.down_proj", + "model.layers.44.mlp.experts.223.gate_proj", + "model.layers.44.mlp.experts.223.up_proj", + "model.layers.44.mlp.experts.224.down_proj", + "model.layers.44.mlp.experts.224.gate_proj", + "model.layers.44.mlp.experts.224.up_proj", + "model.layers.44.mlp.experts.225.down_proj", + "model.layers.44.mlp.experts.225.gate_proj", + "model.layers.44.mlp.experts.225.up_proj", + "model.layers.44.mlp.experts.226.down_proj", + "model.layers.44.mlp.experts.226.gate_proj", + "model.layers.44.mlp.experts.226.up_proj", + "model.layers.44.mlp.experts.227.down_proj", + "model.layers.44.mlp.experts.227.gate_proj", + "model.layers.44.mlp.experts.227.up_proj", + "model.layers.44.mlp.experts.228.down_proj", + "model.layers.44.mlp.experts.228.gate_proj", + "model.layers.44.mlp.experts.228.up_proj", + "model.layers.44.mlp.experts.229.down_proj", + "model.layers.44.mlp.experts.229.gate_proj", + "model.layers.44.mlp.experts.229.up_proj", + "model.layers.44.mlp.experts.23.down_proj", + "model.layers.44.mlp.experts.23.gate_proj", + "model.layers.44.mlp.experts.23.up_proj", + "model.layers.44.mlp.experts.230.down_proj", + "model.layers.44.mlp.experts.230.gate_proj", + "model.layers.44.mlp.experts.230.up_proj", + "model.layers.44.mlp.experts.231.down_proj", + "model.layers.44.mlp.experts.231.gate_proj", + "model.layers.44.mlp.experts.231.up_proj", + "model.layers.44.mlp.experts.232.down_proj", + "model.layers.44.mlp.experts.232.gate_proj", + "model.layers.44.mlp.experts.232.up_proj", + "model.layers.44.mlp.experts.233.down_proj", + "model.layers.44.mlp.experts.233.gate_proj", + "model.layers.44.mlp.experts.233.up_proj", + "model.layers.44.mlp.experts.234.down_proj", + "model.layers.44.mlp.experts.234.gate_proj", + "model.layers.44.mlp.experts.234.up_proj", + "model.layers.44.mlp.experts.235.down_proj", + "model.layers.44.mlp.experts.235.gate_proj", + "model.layers.44.mlp.experts.235.up_proj", + "model.layers.44.mlp.experts.236.down_proj", + "model.layers.44.mlp.experts.236.gate_proj", + "model.layers.44.mlp.experts.236.up_proj", + "model.layers.44.mlp.experts.237.down_proj", + "model.layers.44.mlp.experts.237.gate_proj", + "model.layers.44.mlp.experts.237.up_proj", + "model.layers.44.mlp.experts.238.down_proj", + "model.layers.44.mlp.experts.238.gate_proj", + "model.layers.44.mlp.experts.238.up_proj", + "model.layers.44.mlp.experts.239.down_proj", + "model.layers.44.mlp.experts.239.gate_proj", + "model.layers.44.mlp.experts.239.up_proj", + "model.layers.44.mlp.experts.24.down_proj", + "model.layers.44.mlp.experts.24.gate_proj", + "model.layers.44.mlp.experts.24.up_proj", + "model.layers.44.mlp.experts.240.down_proj", + "model.layers.44.mlp.experts.240.gate_proj", + "model.layers.44.mlp.experts.240.up_proj", + "model.layers.44.mlp.experts.241.down_proj", + "model.layers.44.mlp.experts.241.gate_proj", + "model.layers.44.mlp.experts.241.up_proj", + "model.layers.44.mlp.experts.242.down_proj", + "model.layers.44.mlp.experts.242.gate_proj", + "model.layers.44.mlp.experts.242.up_proj", + "model.layers.44.mlp.experts.243.down_proj", + "model.layers.44.mlp.experts.243.gate_proj", + "model.layers.44.mlp.experts.243.up_proj", + "model.layers.44.mlp.experts.244.down_proj", + "model.layers.44.mlp.experts.244.gate_proj", + "model.layers.44.mlp.experts.244.up_proj", + "model.layers.44.mlp.experts.245.down_proj", + "model.layers.44.mlp.experts.245.gate_proj", + "model.layers.44.mlp.experts.245.up_proj", + "model.layers.44.mlp.experts.246.down_proj", + "model.layers.44.mlp.experts.246.gate_proj", + "model.layers.44.mlp.experts.246.up_proj", + "model.layers.44.mlp.experts.247.down_proj", + "model.layers.44.mlp.experts.247.gate_proj", + "model.layers.44.mlp.experts.247.up_proj", + "model.layers.44.mlp.experts.248.down_proj", + "model.layers.44.mlp.experts.248.gate_proj", + "model.layers.44.mlp.experts.248.up_proj", + "model.layers.44.mlp.experts.249.down_proj", + "model.layers.44.mlp.experts.249.gate_proj", + "model.layers.44.mlp.experts.249.up_proj", + "model.layers.44.mlp.experts.25.down_proj", + "model.layers.44.mlp.experts.25.gate_proj", + "model.layers.44.mlp.experts.25.up_proj", + "model.layers.44.mlp.experts.250.down_proj", + "model.layers.44.mlp.experts.250.gate_proj", + "model.layers.44.mlp.experts.250.up_proj", + "model.layers.44.mlp.experts.251.down_proj", + "model.layers.44.mlp.experts.251.gate_proj", + "model.layers.44.mlp.experts.251.up_proj", + "model.layers.44.mlp.experts.252.down_proj", + "model.layers.44.mlp.experts.252.gate_proj", + "model.layers.44.mlp.experts.252.up_proj", + "model.layers.44.mlp.experts.253.down_proj", + "model.layers.44.mlp.experts.253.gate_proj", + "model.layers.44.mlp.experts.253.up_proj", + "model.layers.44.mlp.experts.254.down_proj", + "model.layers.44.mlp.experts.254.gate_proj", + "model.layers.44.mlp.experts.254.up_proj", + "model.layers.44.mlp.experts.255.down_proj", + "model.layers.44.mlp.experts.255.gate_proj", + "model.layers.44.mlp.experts.255.up_proj", + "model.layers.44.mlp.experts.26.down_proj", + "model.layers.44.mlp.experts.26.gate_proj", + "model.layers.44.mlp.experts.26.up_proj", + "model.layers.44.mlp.experts.27.down_proj", + "model.layers.44.mlp.experts.27.gate_proj", + "model.layers.44.mlp.experts.27.up_proj", + "model.layers.44.mlp.experts.28.down_proj", + "model.layers.44.mlp.experts.28.gate_proj", + "model.layers.44.mlp.experts.28.up_proj", + "model.layers.44.mlp.experts.29.down_proj", + "model.layers.44.mlp.experts.29.gate_proj", + "model.layers.44.mlp.experts.29.up_proj", + "model.layers.44.mlp.experts.3.down_proj", + "model.layers.44.mlp.experts.3.gate_proj", + "model.layers.44.mlp.experts.3.up_proj", + "model.layers.44.mlp.experts.30.down_proj", + "model.layers.44.mlp.experts.30.gate_proj", + "model.layers.44.mlp.experts.30.up_proj", + "model.layers.44.mlp.experts.31.down_proj", + "model.layers.44.mlp.experts.31.gate_proj", + "model.layers.44.mlp.experts.31.up_proj", + "model.layers.44.mlp.experts.32.down_proj", + "model.layers.44.mlp.experts.32.gate_proj", + "model.layers.44.mlp.experts.32.up_proj", + "model.layers.44.mlp.experts.33.down_proj", + "model.layers.44.mlp.experts.33.gate_proj", + "model.layers.44.mlp.experts.33.up_proj", + "model.layers.44.mlp.experts.34.down_proj", + "model.layers.44.mlp.experts.34.gate_proj", + "model.layers.44.mlp.experts.34.up_proj", + "model.layers.44.mlp.experts.35.down_proj", + "model.layers.44.mlp.experts.35.gate_proj", + "model.layers.44.mlp.experts.35.up_proj", + "model.layers.44.mlp.experts.36.down_proj", + "model.layers.44.mlp.experts.36.gate_proj", + "model.layers.44.mlp.experts.36.up_proj", + "model.layers.44.mlp.experts.37.down_proj", + "model.layers.44.mlp.experts.37.gate_proj", + "model.layers.44.mlp.experts.37.up_proj", + "model.layers.44.mlp.experts.38.down_proj", + "model.layers.44.mlp.experts.38.gate_proj", + "model.layers.44.mlp.experts.38.up_proj", + "model.layers.44.mlp.experts.39.down_proj", + "model.layers.44.mlp.experts.39.gate_proj", + "model.layers.44.mlp.experts.39.up_proj", + "model.layers.44.mlp.experts.4.down_proj", + "model.layers.44.mlp.experts.4.gate_proj", + "model.layers.44.mlp.experts.4.up_proj", + "model.layers.44.mlp.experts.40.down_proj", + "model.layers.44.mlp.experts.40.gate_proj", + "model.layers.44.mlp.experts.40.up_proj", + "model.layers.44.mlp.experts.41.down_proj", + "model.layers.44.mlp.experts.41.gate_proj", + "model.layers.44.mlp.experts.41.up_proj", + "model.layers.44.mlp.experts.42.down_proj", + "model.layers.44.mlp.experts.42.gate_proj", + "model.layers.44.mlp.experts.42.up_proj", + "model.layers.44.mlp.experts.43.down_proj", + "model.layers.44.mlp.experts.43.gate_proj", + "model.layers.44.mlp.experts.43.up_proj", + "model.layers.44.mlp.experts.44.down_proj", + "model.layers.44.mlp.experts.44.gate_proj", + "model.layers.44.mlp.experts.44.up_proj", + "model.layers.44.mlp.experts.45.down_proj", + "model.layers.44.mlp.experts.45.gate_proj", + "model.layers.44.mlp.experts.45.up_proj", + "model.layers.44.mlp.experts.46.down_proj", + "model.layers.44.mlp.experts.46.gate_proj", + "model.layers.44.mlp.experts.46.up_proj", + "model.layers.44.mlp.experts.47.down_proj", + "model.layers.44.mlp.experts.47.gate_proj", + "model.layers.44.mlp.experts.47.up_proj", + "model.layers.44.mlp.experts.48.down_proj", + "model.layers.44.mlp.experts.48.gate_proj", + "model.layers.44.mlp.experts.48.up_proj", + "model.layers.44.mlp.experts.49.down_proj", + "model.layers.44.mlp.experts.49.gate_proj", + "model.layers.44.mlp.experts.49.up_proj", + "model.layers.44.mlp.experts.5.down_proj", + "model.layers.44.mlp.experts.5.gate_proj", + "model.layers.44.mlp.experts.5.up_proj", + "model.layers.44.mlp.experts.50.down_proj", + "model.layers.44.mlp.experts.50.gate_proj", + "model.layers.44.mlp.experts.50.up_proj", + "model.layers.44.mlp.experts.51.down_proj", + "model.layers.44.mlp.experts.51.gate_proj", + "model.layers.44.mlp.experts.51.up_proj", + "model.layers.44.mlp.experts.52.down_proj", + "model.layers.44.mlp.experts.52.gate_proj", + "model.layers.44.mlp.experts.52.up_proj", + "model.layers.44.mlp.experts.53.down_proj", + "model.layers.44.mlp.experts.53.gate_proj", + "model.layers.44.mlp.experts.53.up_proj", + "model.layers.44.mlp.experts.54.down_proj", + "model.layers.44.mlp.experts.54.gate_proj", + "model.layers.44.mlp.experts.54.up_proj", + "model.layers.44.mlp.experts.55.down_proj", + "model.layers.44.mlp.experts.55.gate_proj", + "model.layers.44.mlp.experts.55.up_proj", + "model.layers.44.mlp.experts.56.down_proj", + "model.layers.44.mlp.experts.56.gate_proj", + "model.layers.44.mlp.experts.56.up_proj", + "model.layers.44.mlp.experts.57.down_proj", + "model.layers.44.mlp.experts.57.gate_proj", + "model.layers.44.mlp.experts.57.up_proj", + "model.layers.44.mlp.experts.58.down_proj", + "model.layers.44.mlp.experts.58.gate_proj", + "model.layers.44.mlp.experts.58.up_proj", + "model.layers.44.mlp.experts.59.down_proj", + "model.layers.44.mlp.experts.59.gate_proj", + "model.layers.44.mlp.experts.59.up_proj", + "model.layers.44.mlp.experts.6.down_proj", + "model.layers.44.mlp.experts.6.gate_proj", + "model.layers.44.mlp.experts.6.up_proj", + "model.layers.44.mlp.experts.60.down_proj", + "model.layers.44.mlp.experts.60.gate_proj", + "model.layers.44.mlp.experts.60.up_proj", + "model.layers.44.mlp.experts.61.down_proj", + "model.layers.44.mlp.experts.61.gate_proj", + "model.layers.44.mlp.experts.61.up_proj", + "model.layers.44.mlp.experts.62.down_proj", + "model.layers.44.mlp.experts.62.gate_proj", + "model.layers.44.mlp.experts.62.up_proj", + "model.layers.44.mlp.experts.63.down_proj", + "model.layers.44.mlp.experts.63.gate_proj", + "model.layers.44.mlp.experts.63.up_proj", + "model.layers.44.mlp.experts.64.down_proj", + "model.layers.44.mlp.experts.64.gate_proj", + "model.layers.44.mlp.experts.64.up_proj", + "model.layers.44.mlp.experts.65.down_proj", + "model.layers.44.mlp.experts.65.gate_proj", + "model.layers.44.mlp.experts.65.up_proj", + "model.layers.44.mlp.experts.66.down_proj", + "model.layers.44.mlp.experts.66.gate_proj", + "model.layers.44.mlp.experts.66.up_proj", + "model.layers.44.mlp.experts.67.down_proj", + "model.layers.44.mlp.experts.67.gate_proj", + "model.layers.44.mlp.experts.67.up_proj", + "model.layers.44.mlp.experts.68.down_proj", + "model.layers.44.mlp.experts.68.gate_proj", + "model.layers.44.mlp.experts.68.up_proj", + "model.layers.44.mlp.experts.69.down_proj", + "model.layers.44.mlp.experts.69.gate_proj", + "model.layers.44.mlp.experts.69.up_proj", + "model.layers.44.mlp.experts.7.down_proj", + "model.layers.44.mlp.experts.7.gate_proj", + "model.layers.44.mlp.experts.7.up_proj", + "model.layers.44.mlp.experts.70.down_proj", + "model.layers.44.mlp.experts.70.gate_proj", + "model.layers.44.mlp.experts.70.up_proj", + "model.layers.44.mlp.experts.71.down_proj", + "model.layers.44.mlp.experts.71.gate_proj", + "model.layers.44.mlp.experts.71.up_proj", + "model.layers.44.mlp.experts.72.down_proj", + "model.layers.44.mlp.experts.72.gate_proj", + "model.layers.44.mlp.experts.72.up_proj", + "model.layers.44.mlp.experts.73.down_proj", + "model.layers.44.mlp.experts.73.gate_proj", + "model.layers.44.mlp.experts.73.up_proj", + "model.layers.44.mlp.experts.74.down_proj", + "model.layers.44.mlp.experts.74.gate_proj", + "model.layers.44.mlp.experts.74.up_proj", + "model.layers.44.mlp.experts.75.down_proj", + "model.layers.44.mlp.experts.75.gate_proj", + "model.layers.44.mlp.experts.75.up_proj", + "model.layers.44.mlp.experts.76.down_proj", + "model.layers.44.mlp.experts.76.gate_proj", + "model.layers.44.mlp.experts.76.up_proj", + "model.layers.44.mlp.experts.77.down_proj", + "model.layers.44.mlp.experts.77.gate_proj", + "model.layers.44.mlp.experts.77.up_proj", + "model.layers.44.mlp.experts.78.down_proj", + "model.layers.44.mlp.experts.78.gate_proj", + "model.layers.44.mlp.experts.78.up_proj", + "model.layers.44.mlp.experts.79.down_proj", + "model.layers.44.mlp.experts.79.gate_proj", + "model.layers.44.mlp.experts.79.up_proj", + "model.layers.44.mlp.experts.8.down_proj", + "model.layers.44.mlp.experts.8.gate_proj", + "model.layers.44.mlp.experts.8.up_proj", + "model.layers.44.mlp.experts.80.down_proj", + "model.layers.44.mlp.experts.80.gate_proj", + "model.layers.44.mlp.experts.80.up_proj", + "model.layers.44.mlp.experts.81.down_proj", + "model.layers.44.mlp.experts.81.gate_proj", + "model.layers.44.mlp.experts.81.up_proj", + "model.layers.44.mlp.experts.82.down_proj", + "model.layers.44.mlp.experts.82.gate_proj", + "model.layers.44.mlp.experts.82.up_proj", + "model.layers.44.mlp.experts.83.down_proj", + "model.layers.44.mlp.experts.83.gate_proj", + "model.layers.44.mlp.experts.83.up_proj", + "model.layers.44.mlp.experts.84.down_proj", + "model.layers.44.mlp.experts.84.gate_proj", + "model.layers.44.mlp.experts.84.up_proj", + "model.layers.44.mlp.experts.85.down_proj", + "model.layers.44.mlp.experts.85.gate_proj", + "model.layers.44.mlp.experts.85.up_proj", + "model.layers.44.mlp.experts.86.down_proj", + "model.layers.44.mlp.experts.86.gate_proj", + "model.layers.44.mlp.experts.86.up_proj", + "model.layers.44.mlp.experts.87.down_proj", + "model.layers.44.mlp.experts.87.gate_proj", + "model.layers.44.mlp.experts.87.up_proj", + "model.layers.44.mlp.experts.88.down_proj", + "model.layers.44.mlp.experts.88.gate_proj", + "model.layers.44.mlp.experts.88.up_proj", + "model.layers.44.mlp.experts.89.down_proj", + "model.layers.44.mlp.experts.89.gate_proj", + "model.layers.44.mlp.experts.89.up_proj", + "model.layers.44.mlp.experts.9.down_proj", + "model.layers.44.mlp.experts.9.gate_proj", + "model.layers.44.mlp.experts.9.up_proj", + "model.layers.44.mlp.experts.90.down_proj", + "model.layers.44.mlp.experts.90.gate_proj", + "model.layers.44.mlp.experts.90.up_proj", + "model.layers.44.mlp.experts.91.down_proj", + "model.layers.44.mlp.experts.91.gate_proj", + "model.layers.44.mlp.experts.91.up_proj", + "model.layers.44.mlp.experts.92.down_proj", + "model.layers.44.mlp.experts.92.gate_proj", + "model.layers.44.mlp.experts.92.up_proj", + "model.layers.44.mlp.experts.93.down_proj", + "model.layers.44.mlp.experts.93.gate_proj", + "model.layers.44.mlp.experts.93.up_proj", + "model.layers.44.mlp.experts.94.down_proj", + "model.layers.44.mlp.experts.94.gate_proj", + "model.layers.44.mlp.experts.94.up_proj", + "model.layers.44.mlp.experts.95.down_proj", + "model.layers.44.mlp.experts.95.gate_proj", + "model.layers.44.mlp.experts.95.up_proj", + "model.layers.44.mlp.experts.96.down_proj", + "model.layers.44.mlp.experts.96.gate_proj", + "model.layers.44.mlp.experts.96.up_proj", + "model.layers.44.mlp.experts.97.down_proj", + "model.layers.44.mlp.experts.97.gate_proj", + "model.layers.44.mlp.experts.97.up_proj", + "model.layers.44.mlp.experts.98.down_proj", + "model.layers.44.mlp.experts.98.gate_proj", + "model.layers.44.mlp.experts.98.up_proj", + "model.layers.44.mlp.experts.99.down_proj", + "model.layers.44.mlp.experts.99.gate_proj", + "model.layers.44.mlp.experts.99.up_proj", + "model.layers.45.mlp.experts.0.down_proj", + "model.layers.45.mlp.experts.0.gate_proj", + "model.layers.45.mlp.experts.0.up_proj", + "model.layers.45.mlp.experts.1.down_proj", + "model.layers.45.mlp.experts.1.gate_proj", + "model.layers.45.mlp.experts.1.up_proj", + "model.layers.45.mlp.experts.10.down_proj", + "model.layers.45.mlp.experts.10.gate_proj", + "model.layers.45.mlp.experts.10.up_proj", + "model.layers.45.mlp.experts.100.down_proj", + "model.layers.45.mlp.experts.100.gate_proj", + "model.layers.45.mlp.experts.100.up_proj", + "model.layers.45.mlp.experts.101.down_proj", + "model.layers.45.mlp.experts.101.gate_proj", + "model.layers.45.mlp.experts.101.up_proj", + "model.layers.45.mlp.experts.102.down_proj", + "model.layers.45.mlp.experts.102.gate_proj", + "model.layers.45.mlp.experts.102.up_proj", + "model.layers.45.mlp.experts.103.down_proj", + "model.layers.45.mlp.experts.103.gate_proj", + "model.layers.45.mlp.experts.103.up_proj", + "model.layers.45.mlp.experts.104.down_proj", + "model.layers.45.mlp.experts.104.gate_proj", + "model.layers.45.mlp.experts.104.up_proj", + "model.layers.45.mlp.experts.105.down_proj", + "model.layers.45.mlp.experts.105.gate_proj", + "model.layers.45.mlp.experts.105.up_proj", + "model.layers.45.mlp.experts.106.down_proj", + "model.layers.45.mlp.experts.106.gate_proj", + "model.layers.45.mlp.experts.106.up_proj", + "model.layers.45.mlp.experts.107.down_proj", + "model.layers.45.mlp.experts.107.gate_proj", + "model.layers.45.mlp.experts.107.up_proj", + "model.layers.45.mlp.experts.108.down_proj", + "model.layers.45.mlp.experts.108.gate_proj", + "model.layers.45.mlp.experts.108.up_proj", + "model.layers.45.mlp.experts.109.down_proj", + "model.layers.45.mlp.experts.109.gate_proj", + "model.layers.45.mlp.experts.109.up_proj", + "model.layers.45.mlp.experts.11.down_proj", + "model.layers.45.mlp.experts.11.gate_proj", + "model.layers.45.mlp.experts.11.up_proj", + "model.layers.45.mlp.experts.110.down_proj", + "model.layers.45.mlp.experts.110.gate_proj", + "model.layers.45.mlp.experts.110.up_proj", + "model.layers.45.mlp.experts.111.down_proj", + "model.layers.45.mlp.experts.111.gate_proj", + "model.layers.45.mlp.experts.111.up_proj", + "model.layers.45.mlp.experts.112.down_proj", + "model.layers.45.mlp.experts.112.gate_proj", + "model.layers.45.mlp.experts.112.up_proj", + "model.layers.45.mlp.experts.113.down_proj", + "model.layers.45.mlp.experts.113.gate_proj", + "model.layers.45.mlp.experts.113.up_proj", + "model.layers.45.mlp.experts.114.down_proj", + "model.layers.45.mlp.experts.114.gate_proj", + "model.layers.45.mlp.experts.114.up_proj", + "model.layers.45.mlp.experts.115.down_proj", + "model.layers.45.mlp.experts.115.gate_proj", + "model.layers.45.mlp.experts.115.up_proj", + "model.layers.45.mlp.experts.116.down_proj", + "model.layers.45.mlp.experts.116.gate_proj", + "model.layers.45.mlp.experts.116.up_proj", + "model.layers.45.mlp.experts.117.down_proj", + "model.layers.45.mlp.experts.117.gate_proj", + "model.layers.45.mlp.experts.117.up_proj", + "model.layers.45.mlp.experts.118.down_proj", + "model.layers.45.mlp.experts.118.gate_proj", + "model.layers.45.mlp.experts.118.up_proj", + "model.layers.45.mlp.experts.119.down_proj", + "model.layers.45.mlp.experts.119.gate_proj", + "model.layers.45.mlp.experts.119.up_proj", + "model.layers.45.mlp.experts.12.down_proj", + "model.layers.45.mlp.experts.12.gate_proj", + "model.layers.45.mlp.experts.12.up_proj", + "model.layers.45.mlp.experts.120.down_proj", + "model.layers.45.mlp.experts.120.gate_proj", + "model.layers.45.mlp.experts.120.up_proj", + "model.layers.45.mlp.experts.121.down_proj", + "model.layers.45.mlp.experts.121.gate_proj", + "model.layers.45.mlp.experts.121.up_proj", + "model.layers.45.mlp.experts.122.down_proj", + "model.layers.45.mlp.experts.122.gate_proj", + "model.layers.45.mlp.experts.122.up_proj", + "model.layers.45.mlp.experts.123.down_proj", + "model.layers.45.mlp.experts.123.gate_proj", + "model.layers.45.mlp.experts.123.up_proj", + "model.layers.45.mlp.experts.124.down_proj", + "model.layers.45.mlp.experts.124.gate_proj", + "model.layers.45.mlp.experts.124.up_proj", + "model.layers.45.mlp.experts.125.down_proj", + "model.layers.45.mlp.experts.125.gate_proj", + "model.layers.45.mlp.experts.125.up_proj", + "model.layers.45.mlp.experts.126.down_proj", + "model.layers.45.mlp.experts.126.gate_proj", + "model.layers.45.mlp.experts.126.up_proj", + "model.layers.45.mlp.experts.127.down_proj", + "model.layers.45.mlp.experts.127.gate_proj", + "model.layers.45.mlp.experts.127.up_proj", + "model.layers.45.mlp.experts.128.down_proj", + "model.layers.45.mlp.experts.128.gate_proj", + "model.layers.45.mlp.experts.128.up_proj", + "model.layers.45.mlp.experts.129.down_proj", + "model.layers.45.mlp.experts.129.gate_proj", + "model.layers.45.mlp.experts.129.up_proj", + "model.layers.45.mlp.experts.13.down_proj", + "model.layers.45.mlp.experts.13.gate_proj", + "model.layers.45.mlp.experts.13.up_proj", + "model.layers.45.mlp.experts.130.down_proj", + "model.layers.45.mlp.experts.130.gate_proj", + "model.layers.45.mlp.experts.130.up_proj", + "model.layers.45.mlp.experts.131.down_proj", + "model.layers.45.mlp.experts.131.gate_proj", + "model.layers.45.mlp.experts.131.up_proj", + "model.layers.45.mlp.experts.132.down_proj", + "model.layers.45.mlp.experts.132.gate_proj", + "model.layers.45.mlp.experts.132.up_proj", + "model.layers.45.mlp.experts.133.down_proj", + "model.layers.45.mlp.experts.133.gate_proj", + "model.layers.45.mlp.experts.133.up_proj", + "model.layers.45.mlp.experts.134.down_proj", + "model.layers.45.mlp.experts.134.gate_proj", + "model.layers.45.mlp.experts.134.up_proj", + "model.layers.45.mlp.experts.135.down_proj", + "model.layers.45.mlp.experts.135.gate_proj", + "model.layers.45.mlp.experts.135.up_proj", + "model.layers.45.mlp.experts.136.down_proj", + "model.layers.45.mlp.experts.136.gate_proj", + "model.layers.45.mlp.experts.136.up_proj", + "model.layers.45.mlp.experts.137.down_proj", + "model.layers.45.mlp.experts.137.gate_proj", + "model.layers.45.mlp.experts.137.up_proj", + "model.layers.45.mlp.experts.138.down_proj", + "model.layers.45.mlp.experts.138.gate_proj", + "model.layers.45.mlp.experts.138.up_proj", + "model.layers.45.mlp.experts.139.down_proj", + "model.layers.45.mlp.experts.139.gate_proj", + "model.layers.45.mlp.experts.139.up_proj", + "model.layers.45.mlp.experts.14.down_proj", + "model.layers.45.mlp.experts.14.gate_proj", + "model.layers.45.mlp.experts.14.up_proj", + "model.layers.45.mlp.experts.140.down_proj", + "model.layers.45.mlp.experts.140.gate_proj", + "model.layers.45.mlp.experts.140.up_proj", + "model.layers.45.mlp.experts.141.down_proj", + "model.layers.45.mlp.experts.141.gate_proj", + "model.layers.45.mlp.experts.141.up_proj", + "model.layers.45.mlp.experts.142.down_proj", + "model.layers.45.mlp.experts.142.gate_proj", + "model.layers.45.mlp.experts.142.up_proj", + "model.layers.45.mlp.experts.143.down_proj", + "model.layers.45.mlp.experts.143.gate_proj", + "model.layers.45.mlp.experts.143.up_proj", + "model.layers.45.mlp.experts.144.down_proj", + "model.layers.45.mlp.experts.144.gate_proj", + "model.layers.45.mlp.experts.144.up_proj", + "model.layers.45.mlp.experts.145.down_proj", + "model.layers.45.mlp.experts.145.gate_proj", + "model.layers.45.mlp.experts.145.up_proj", + "model.layers.45.mlp.experts.146.down_proj", + "model.layers.45.mlp.experts.146.gate_proj", + "model.layers.45.mlp.experts.146.up_proj", + "model.layers.45.mlp.experts.147.down_proj", + "model.layers.45.mlp.experts.147.gate_proj", + "model.layers.45.mlp.experts.147.up_proj", + "model.layers.45.mlp.experts.148.down_proj", + "model.layers.45.mlp.experts.148.gate_proj", + "model.layers.45.mlp.experts.148.up_proj", + "model.layers.45.mlp.experts.149.down_proj", + "model.layers.45.mlp.experts.149.gate_proj", + "model.layers.45.mlp.experts.149.up_proj", + "model.layers.45.mlp.experts.15.down_proj", + "model.layers.45.mlp.experts.15.gate_proj", + "model.layers.45.mlp.experts.15.up_proj", + "model.layers.45.mlp.experts.150.down_proj", + "model.layers.45.mlp.experts.150.gate_proj", + "model.layers.45.mlp.experts.150.up_proj", + "model.layers.45.mlp.experts.151.down_proj", + "model.layers.45.mlp.experts.151.gate_proj", + "model.layers.45.mlp.experts.151.up_proj", + "model.layers.45.mlp.experts.152.down_proj", + "model.layers.45.mlp.experts.152.gate_proj", + "model.layers.45.mlp.experts.152.up_proj", + "model.layers.45.mlp.experts.153.down_proj", + "model.layers.45.mlp.experts.153.gate_proj", + "model.layers.45.mlp.experts.153.up_proj", + "model.layers.45.mlp.experts.154.down_proj", + "model.layers.45.mlp.experts.154.gate_proj", + "model.layers.45.mlp.experts.154.up_proj", + "model.layers.45.mlp.experts.155.down_proj", + "model.layers.45.mlp.experts.155.gate_proj", + "model.layers.45.mlp.experts.155.up_proj", + "model.layers.45.mlp.experts.156.down_proj", + "model.layers.45.mlp.experts.156.gate_proj", + "model.layers.45.mlp.experts.156.up_proj", + "model.layers.45.mlp.experts.157.down_proj", + "model.layers.45.mlp.experts.157.gate_proj", + "model.layers.45.mlp.experts.157.up_proj", + "model.layers.45.mlp.experts.158.down_proj", + "model.layers.45.mlp.experts.158.gate_proj", + "model.layers.45.mlp.experts.158.up_proj", + "model.layers.45.mlp.experts.159.down_proj", + "model.layers.45.mlp.experts.159.gate_proj", + "model.layers.45.mlp.experts.159.up_proj", + "model.layers.45.mlp.experts.16.down_proj", + "model.layers.45.mlp.experts.16.gate_proj", + "model.layers.45.mlp.experts.16.up_proj", + "model.layers.45.mlp.experts.160.down_proj", + "model.layers.45.mlp.experts.160.gate_proj", + "model.layers.45.mlp.experts.160.up_proj", + "model.layers.45.mlp.experts.161.down_proj", + "model.layers.45.mlp.experts.161.gate_proj", + "model.layers.45.mlp.experts.161.up_proj", + "model.layers.45.mlp.experts.162.down_proj", + "model.layers.45.mlp.experts.162.gate_proj", + "model.layers.45.mlp.experts.162.up_proj", + "model.layers.45.mlp.experts.163.down_proj", + "model.layers.45.mlp.experts.163.gate_proj", + "model.layers.45.mlp.experts.163.up_proj", + "model.layers.45.mlp.experts.164.down_proj", + "model.layers.45.mlp.experts.164.gate_proj", + "model.layers.45.mlp.experts.164.up_proj", + "model.layers.45.mlp.experts.165.down_proj", + "model.layers.45.mlp.experts.165.gate_proj", + "model.layers.45.mlp.experts.165.up_proj", + "model.layers.45.mlp.experts.166.down_proj", + "model.layers.45.mlp.experts.166.gate_proj", + "model.layers.45.mlp.experts.166.up_proj", + "model.layers.45.mlp.experts.167.down_proj", + "model.layers.45.mlp.experts.167.gate_proj", + "model.layers.45.mlp.experts.167.up_proj", + "model.layers.45.mlp.experts.168.down_proj", + "model.layers.45.mlp.experts.168.gate_proj", + "model.layers.45.mlp.experts.168.up_proj", + "model.layers.45.mlp.experts.169.down_proj", + "model.layers.45.mlp.experts.169.gate_proj", + "model.layers.45.mlp.experts.169.up_proj", + "model.layers.45.mlp.experts.17.down_proj", + "model.layers.45.mlp.experts.17.gate_proj", + "model.layers.45.mlp.experts.17.up_proj", + "model.layers.45.mlp.experts.170.down_proj", + "model.layers.45.mlp.experts.170.gate_proj", + "model.layers.45.mlp.experts.170.up_proj", + "model.layers.45.mlp.experts.171.down_proj", + "model.layers.45.mlp.experts.171.gate_proj", + "model.layers.45.mlp.experts.171.up_proj", + "model.layers.45.mlp.experts.172.down_proj", + "model.layers.45.mlp.experts.172.gate_proj", + "model.layers.45.mlp.experts.172.up_proj", + "model.layers.45.mlp.experts.173.down_proj", + "model.layers.45.mlp.experts.173.gate_proj", + "model.layers.45.mlp.experts.173.up_proj", + "model.layers.45.mlp.experts.174.down_proj", + "model.layers.45.mlp.experts.174.gate_proj", + "model.layers.45.mlp.experts.174.up_proj", + "model.layers.45.mlp.experts.175.down_proj", + "model.layers.45.mlp.experts.175.gate_proj", + "model.layers.45.mlp.experts.175.up_proj", + "model.layers.45.mlp.experts.176.down_proj", + "model.layers.45.mlp.experts.176.gate_proj", + "model.layers.45.mlp.experts.176.up_proj", + "model.layers.45.mlp.experts.177.down_proj", + "model.layers.45.mlp.experts.177.gate_proj", + "model.layers.45.mlp.experts.177.up_proj", + "model.layers.45.mlp.experts.178.down_proj", + "model.layers.45.mlp.experts.178.gate_proj", + "model.layers.45.mlp.experts.178.up_proj", + "model.layers.45.mlp.experts.179.down_proj", + "model.layers.45.mlp.experts.179.gate_proj", + "model.layers.45.mlp.experts.179.up_proj", + "model.layers.45.mlp.experts.18.down_proj", + "model.layers.45.mlp.experts.18.gate_proj", + "model.layers.45.mlp.experts.18.up_proj", + "model.layers.45.mlp.experts.180.down_proj", + "model.layers.45.mlp.experts.180.gate_proj", + "model.layers.45.mlp.experts.180.up_proj", + "model.layers.45.mlp.experts.181.down_proj", + "model.layers.45.mlp.experts.181.gate_proj", + "model.layers.45.mlp.experts.181.up_proj", + "model.layers.45.mlp.experts.182.down_proj", + "model.layers.45.mlp.experts.182.gate_proj", + "model.layers.45.mlp.experts.182.up_proj", + "model.layers.45.mlp.experts.183.down_proj", + "model.layers.45.mlp.experts.183.gate_proj", + "model.layers.45.mlp.experts.183.up_proj", + "model.layers.45.mlp.experts.184.down_proj", + "model.layers.45.mlp.experts.184.gate_proj", + "model.layers.45.mlp.experts.184.up_proj", + "model.layers.45.mlp.experts.185.down_proj", + "model.layers.45.mlp.experts.185.gate_proj", + "model.layers.45.mlp.experts.185.up_proj", + "model.layers.45.mlp.experts.186.down_proj", + "model.layers.45.mlp.experts.186.gate_proj", + "model.layers.45.mlp.experts.186.up_proj", + "model.layers.45.mlp.experts.187.down_proj", + "model.layers.45.mlp.experts.187.gate_proj", + "model.layers.45.mlp.experts.187.up_proj", + "model.layers.45.mlp.experts.188.down_proj", + "model.layers.45.mlp.experts.188.gate_proj", + "model.layers.45.mlp.experts.188.up_proj", + "model.layers.45.mlp.experts.189.down_proj", + "model.layers.45.mlp.experts.189.gate_proj", + "model.layers.45.mlp.experts.189.up_proj", + "model.layers.45.mlp.experts.19.down_proj", + "model.layers.45.mlp.experts.19.gate_proj", + "model.layers.45.mlp.experts.19.up_proj", + "model.layers.45.mlp.experts.190.down_proj", + "model.layers.45.mlp.experts.190.gate_proj", + "model.layers.45.mlp.experts.190.up_proj", + "model.layers.45.mlp.experts.191.down_proj", + "model.layers.45.mlp.experts.191.gate_proj", + "model.layers.45.mlp.experts.191.up_proj", + "model.layers.45.mlp.experts.192.down_proj", + "model.layers.45.mlp.experts.192.gate_proj", + "model.layers.45.mlp.experts.192.up_proj", + "model.layers.45.mlp.experts.193.down_proj", + "model.layers.45.mlp.experts.193.gate_proj", + "model.layers.45.mlp.experts.193.up_proj", + "model.layers.45.mlp.experts.194.down_proj", + "model.layers.45.mlp.experts.194.gate_proj", + "model.layers.45.mlp.experts.194.up_proj", + "model.layers.45.mlp.experts.195.down_proj", + "model.layers.45.mlp.experts.195.gate_proj", + "model.layers.45.mlp.experts.195.up_proj", + "model.layers.45.mlp.experts.196.down_proj", + "model.layers.45.mlp.experts.196.gate_proj", + "model.layers.45.mlp.experts.196.up_proj", + "model.layers.45.mlp.experts.197.down_proj", + "model.layers.45.mlp.experts.197.gate_proj", + "model.layers.45.mlp.experts.197.up_proj", + "model.layers.45.mlp.experts.198.down_proj", + "model.layers.45.mlp.experts.198.gate_proj", + "model.layers.45.mlp.experts.198.up_proj", + "model.layers.45.mlp.experts.199.down_proj", + "model.layers.45.mlp.experts.199.gate_proj", + "model.layers.45.mlp.experts.199.up_proj", + "model.layers.45.mlp.experts.2.down_proj", + "model.layers.45.mlp.experts.2.gate_proj", + "model.layers.45.mlp.experts.2.up_proj", + "model.layers.45.mlp.experts.20.down_proj", + "model.layers.45.mlp.experts.20.gate_proj", + "model.layers.45.mlp.experts.20.up_proj", + "model.layers.45.mlp.experts.200.down_proj", + "model.layers.45.mlp.experts.200.gate_proj", + "model.layers.45.mlp.experts.200.up_proj", + "model.layers.45.mlp.experts.201.down_proj", + "model.layers.45.mlp.experts.201.gate_proj", + "model.layers.45.mlp.experts.201.up_proj", + "model.layers.45.mlp.experts.202.down_proj", + "model.layers.45.mlp.experts.202.gate_proj", + "model.layers.45.mlp.experts.202.up_proj", + "model.layers.45.mlp.experts.203.down_proj", + "model.layers.45.mlp.experts.203.gate_proj", + "model.layers.45.mlp.experts.203.up_proj", + "model.layers.45.mlp.experts.204.down_proj", + "model.layers.45.mlp.experts.204.gate_proj", + "model.layers.45.mlp.experts.204.up_proj", + "model.layers.45.mlp.experts.205.down_proj", + "model.layers.45.mlp.experts.205.gate_proj", + "model.layers.45.mlp.experts.205.up_proj", + "model.layers.45.mlp.experts.206.down_proj", + "model.layers.45.mlp.experts.206.gate_proj", + "model.layers.45.mlp.experts.206.up_proj", + "model.layers.45.mlp.experts.207.down_proj", + "model.layers.45.mlp.experts.207.gate_proj", + "model.layers.45.mlp.experts.207.up_proj", + "model.layers.45.mlp.experts.208.down_proj", + "model.layers.45.mlp.experts.208.gate_proj", + "model.layers.45.mlp.experts.208.up_proj", + "model.layers.45.mlp.experts.209.down_proj", + "model.layers.45.mlp.experts.209.gate_proj", + "model.layers.45.mlp.experts.209.up_proj", + "model.layers.45.mlp.experts.21.down_proj", + "model.layers.45.mlp.experts.21.gate_proj", + "model.layers.45.mlp.experts.21.up_proj", + "model.layers.45.mlp.experts.210.down_proj", + "model.layers.45.mlp.experts.210.gate_proj", + "model.layers.45.mlp.experts.210.up_proj", + "model.layers.45.mlp.experts.211.down_proj", + "model.layers.45.mlp.experts.211.gate_proj", + "model.layers.45.mlp.experts.211.up_proj", + "model.layers.45.mlp.experts.212.down_proj", + "model.layers.45.mlp.experts.212.gate_proj", + "model.layers.45.mlp.experts.212.up_proj", + "model.layers.45.mlp.experts.213.down_proj", + "model.layers.45.mlp.experts.213.gate_proj", + "model.layers.45.mlp.experts.213.up_proj", + "model.layers.45.mlp.experts.214.down_proj", + "model.layers.45.mlp.experts.214.gate_proj", + "model.layers.45.mlp.experts.214.up_proj", + "model.layers.45.mlp.experts.215.down_proj", + "model.layers.45.mlp.experts.215.gate_proj", + "model.layers.45.mlp.experts.215.up_proj", + "model.layers.45.mlp.experts.216.down_proj", + "model.layers.45.mlp.experts.216.gate_proj", + "model.layers.45.mlp.experts.216.up_proj", + "model.layers.45.mlp.experts.217.down_proj", + "model.layers.45.mlp.experts.217.gate_proj", + "model.layers.45.mlp.experts.217.up_proj", + "model.layers.45.mlp.experts.218.down_proj", + "model.layers.45.mlp.experts.218.gate_proj", + "model.layers.45.mlp.experts.218.up_proj", + "model.layers.45.mlp.experts.219.down_proj", + "model.layers.45.mlp.experts.219.gate_proj", + "model.layers.45.mlp.experts.219.up_proj", + "model.layers.45.mlp.experts.22.down_proj", + "model.layers.45.mlp.experts.22.gate_proj", + "model.layers.45.mlp.experts.22.up_proj", + "model.layers.45.mlp.experts.220.down_proj", + "model.layers.45.mlp.experts.220.gate_proj", + "model.layers.45.mlp.experts.220.up_proj", + "model.layers.45.mlp.experts.221.down_proj", + "model.layers.45.mlp.experts.221.gate_proj", + "model.layers.45.mlp.experts.221.up_proj", + "model.layers.45.mlp.experts.222.down_proj", + "model.layers.45.mlp.experts.222.gate_proj", + "model.layers.45.mlp.experts.222.up_proj", + "model.layers.45.mlp.experts.223.down_proj", + "model.layers.45.mlp.experts.223.gate_proj", + "model.layers.45.mlp.experts.223.up_proj", + "model.layers.45.mlp.experts.224.down_proj", + "model.layers.45.mlp.experts.224.gate_proj", + "model.layers.45.mlp.experts.224.up_proj", + "model.layers.45.mlp.experts.225.down_proj", + "model.layers.45.mlp.experts.225.gate_proj", + "model.layers.45.mlp.experts.225.up_proj", + "model.layers.45.mlp.experts.226.down_proj", + "model.layers.45.mlp.experts.226.gate_proj", + "model.layers.45.mlp.experts.226.up_proj", + "model.layers.45.mlp.experts.227.down_proj", + "model.layers.45.mlp.experts.227.gate_proj", + "model.layers.45.mlp.experts.227.up_proj", + "model.layers.45.mlp.experts.228.down_proj", + "model.layers.45.mlp.experts.228.gate_proj", + "model.layers.45.mlp.experts.228.up_proj", + "model.layers.45.mlp.experts.229.down_proj", + "model.layers.45.mlp.experts.229.gate_proj", + "model.layers.45.mlp.experts.229.up_proj", + "model.layers.45.mlp.experts.23.down_proj", + "model.layers.45.mlp.experts.23.gate_proj", + "model.layers.45.mlp.experts.23.up_proj", + "model.layers.45.mlp.experts.230.down_proj", + "model.layers.45.mlp.experts.230.gate_proj", + "model.layers.45.mlp.experts.230.up_proj", + "model.layers.45.mlp.experts.231.down_proj", + "model.layers.45.mlp.experts.231.gate_proj", + "model.layers.45.mlp.experts.231.up_proj", + "model.layers.45.mlp.experts.232.down_proj", + "model.layers.45.mlp.experts.232.gate_proj", + "model.layers.45.mlp.experts.232.up_proj", + "model.layers.45.mlp.experts.233.down_proj", + "model.layers.45.mlp.experts.233.gate_proj", + "model.layers.45.mlp.experts.233.up_proj", + "model.layers.45.mlp.experts.234.down_proj", + "model.layers.45.mlp.experts.234.gate_proj", + "model.layers.45.mlp.experts.234.up_proj", + "model.layers.45.mlp.experts.235.down_proj", + "model.layers.45.mlp.experts.235.gate_proj", + "model.layers.45.mlp.experts.235.up_proj", + "model.layers.45.mlp.experts.236.down_proj", + "model.layers.45.mlp.experts.236.gate_proj", + "model.layers.45.mlp.experts.236.up_proj", + "model.layers.45.mlp.experts.237.down_proj", + "model.layers.45.mlp.experts.237.gate_proj", + "model.layers.45.mlp.experts.237.up_proj", + "model.layers.45.mlp.experts.238.down_proj", + "model.layers.45.mlp.experts.238.gate_proj", + "model.layers.45.mlp.experts.238.up_proj", + "model.layers.45.mlp.experts.239.down_proj", + "model.layers.45.mlp.experts.239.gate_proj", + "model.layers.45.mlp.experts.239.up_proj", + "model.layers.45.mlp.experts.24.down_proj", + "model.layers.45.mlp.experts.24.gate_proj", + "model.layers.45.mlp.experts.24.up_proj", + "model.layers.45.mlp.experts.240.down_proj", + "model.layers.45.mlp.experts.240.gate_proj", + "model.layers.45.mlp.experts.240.up_proj", + "model.layers.45.mlp.experts.241.down_proj", + "model.layers.45.mlp.experts.241.gate_proj", + "model.layers.45.mlp.experts.241.up_proj", + "model.layers.45.mlp.experts.242.down_proj", + "model.layers.45.mlp.experts.242.gate_proj", + "model.layers.45.mlp.experts.242.up_proj", + "model.layers.45.mlp.experts.243.down_proj", + "model.layers.45.mlp.experts.243.gate_proj", + "model.layers.45.mlp.experts.243.up_proj", + "model.layers.45.mlp.experts.244.down_proj", + "model.layers.45.mlp.experts.244.gate_proj", + "model.layers.45.mlp.experts.244.up_proj", + "model.layers.45.mlp.experts.245.down_proj", + "model.layers.45.mlp.experts.245.gate_proj", + "model.layers.45.mlp.experts.245.up_proj", + "model.layers.45.mlp.experts.246.down_proj", + "model.layers.45.mlp.experts.246.gate_proj", + "model.layers.45.mlp.experts.246.up_proj", + "model.layers.45.mlp.experts.247.down_proj", + "model.layers.45.mlp.experts.247.gate_proj", + "model.layers.45.mlp.experts.247.up_proj", + "model.layers.45.mlp.experts.248.down_proj", + "model.layers.45.mlp.experts.248.gate_proj", + "model.layers.45.mlp.experts.248.up_proj", + "model.layers.45.mlp.experts.249.down_proj", + "model.layers.45.mlp.experts.249.gate_proj", + "model.layers.45.mlp.experts.249.up_proj", + "model.layers.45.mlp.experts.25.down_proj", + "model.layers.45.mlp.experts.25.gate_proj", + "model.layers.45.mlp.experts.25.up_proj", + "model.layers.45.mlp.experts.250.down_proj", + "model.layers.45.mlp.experts.250.gate_proj", + "model.layers.45.mlp.experts.250.up_proj", + "model.layers.45.mlp.experts.251.down_proj", + "model.layers.45.mlp.experts.251.gate_proj", + "model.layers.45.mlp.experts.251.up_proj", + "model.layers.45.mlp.experts.252.down_proj", + "model.layers.45.mlp.experts.252.gate_proj", + "model.layers.45.mlp.experts.252.up_proj", + "model.layers.45.mlp.experts.253.down_proj", + "model.layers.45.mlp.experts.253.gate_proj", + "model.layers.45.mlp.experts.253.up_proj", + "model.layers.45.mlp.experts.254.down_proj", + "model.layers.45.mlp.experts.254.gate_proj", + "model.layers.45.mlp.experts.254.up_proj", + "model.layers.45.mlp.experts.255.down_proj", + "model.layers.45.mlp.experts.255.gate_proj", + "model.layers.45.mlp.experts.255.up_proj", + "model.layers.45.mlp.experts.26.down_proj", + "model.layers.45.mlp.experts.26.gate_proj", + "model.layers.45.mlp.experts.26.up_proj", + "model.layers.45.mlp.experts.27.down_proj", + "model.layers.45.mlp.experts.27.gate_proj", + "model.layers.45.mlp.experts.27.up_proj", + "model.layers.45.mlp.experts.28.down_proj", + "model.layers.45.mlp.experts.28.gate_proj", + "model.layers.45.mlp.experts.28.up_proj", + "model.layers.45.mlp.experts.29.down_proj", + "model.layers.45.mlp.experts.29.gate_proj", + "model.layers.45.mlp.experts.29.up_proj", + "model.layers.45.mlp.experts.3.down_proj", + "model.layers.45.mlp.experts.3.gate_proj", + "model.layers.45.mlp.experts.3.up_proj", + "model.layers.45.mlp.experts.30.down_proj", + "model.layers.45.mlp.experts.30.gate_proj", + "model.layers.45.mlp.experts.30.up_proj", + "model.layers.45.mlp.experts.31.down_proj", + "model.layers.45.mlp.experts.31.gate_proj", + "model.layers.45.mlp.experts.31.up_proj", + "model.layers.45.mlp.experts.32.down_proj", + "model.layers.45.mlp.experts.32.gate_proj", + "model.layers.45.mlp.experts.32.up_proj", + "model.layers.45.mlp.experts.33.down_proj", + "model.layers.45.mlp.experts.33.gate_proj", + "model.layers.45.mlp.experts.33.up_proj", + "model.layers.45.mlp.experts.34.down_proj", + "model.layers.45.mlp.experts.34.gate_proj", + "model.layers.45.mlp.experts.34.up_proj", + "model.layers.45.mlp.experts.35.down_proj", + "model.layers.45.mlp.experts.35.gate_proj", + "model.layers.45.mlp.experts.35.up_proj", + "model.layers.45.mlp.experts.36.down_proj", + "model.layers.45.mlp.experts.36.gate_proj", + "model.layers.45.mlp.experts.36.up_proj", + "model.layers.45.mlp.experts.37.down_proj", + "model.layers.45.mlp.experts.37.gate_proj", + "model.layers.45.mlp.experts.37.up_proj", + "model.layers.45.mlp.experts.38.down_proj", + "model.layers.45.mlp.experts.38.gate_proj", + "model.layers.45.mlp.experts.38.up_proj", + "model.layers.45.mlp.experts.39.down_proj", + "model.layers.45.mlp.experts.39.gate_proj", + "model.layers.45.mlp.experts.39.up_proj", + "model.layers.45.mlp.experts.4.down_proj", + "model.layers.45.mlp.experts.4.gate_proj", + "model.layers.45.mlp.experts.4.up_proj", + "model.layers.45.mlp.experts.40.down_proj", + "model.layers.45.mlp.experts.40.gate_proj", + "model.layers.45.mlp.experts.40.up_proj", + "model.layers.45.mlp.experts.41.down_proj", + "model.layers.45.mlp.experts.41.gate_proj", + "model.layers.45.mlp.experts.41.up_proj", + "model.layers.45.mlp.experts.42.down_proj", + "model.layers.45.mlp.experts.42.gate_proj", + "model.layers.45.mlp.experts.42.up_proj", + "model.layers.45.mlp.experts.43.down_proj", + "model.layers.45.mlp.experts.43.gate_proj", + "model.layers.45.mlp.experts.43.up_proj", + "model.layers.45.mlp.experts.44.down_proj", + "model.layers.45.mlp.experts.44.gate_proj", + "model.layers.45.mlp.experts.44.up_proj", + "model.layers.45.mlp.experts.45.down_proj", + "model.layers.45.mlp.experts.45.gate_proj", + "model.layers.45.mlp.experts.45.up_proj", + "model.layers.45.mlp.experts.46.down_proj", + "model.layers.45.mlp.experts.46.gate_proj", + "model.layers.45.mlp.experts.46.up_proj", + "model.layers.45.mlp.experts.47.down_proj", + "model.layers.45.mlp.experts.47.gate_proj", + "model.layers.45.mlp.experts.47.up_proj", + "model.layers.45.mlp.experts.48.down_proj", + "model.layers.45.mlp.experts.48.gate_proj", + "model.layers.45.mlp.experts.48.up_proj", + "model.layers.45.mlp.experts.49.down_proj", + "model.layers.45.mlp.experts.49.gate_proj", + "model.layers.45.mlp.experts.49.up_proj", + "model.layers.45.mlp.experts.5.down_proj", + "model.layers.45.mlp.experts.5.gate_proj", + "model.layers.45.mlp.experts.5.up_proj", + "model.layers.45.mlp.experts.50.down_proj", + "model.layers.45.mlp.experts.50.gate_proj", + "model.layers.45.mlp.experts.50.up_proj", + "model.layers.45.mlp.experts.51.down_proj", + "model.layers.45.mlp.experts.51.gate_proj", + "model.layers.45.mlp.experts.51.up_proj", + "model.layers.45.mlp.experts.52.down_proj", + "model.layers.45.mlp.experts.52.gate_proj", + "model.layers.45.mlp.experts.52.up_proj", + "model.layers.45.mlp.experts.53.down_proj", + "model.layers.45.mlp.experts.53.gate_proj", + "model.layers.45.mlp.experts.53.up_proj", + "model.layers.45.mlp.experts.54.down_proj", + "model.layers.45.mlp.experts.54.gate_proj", + "model.layers.45.mlp.experts.54.up_proj", + "model.layers.45.mlp.experts.55.down_proj", + "model.layers.45.mlp.experts.55.gate_proj", + "model.layers.45.mlp.experts.55.up_proj", + "model.layers.45.mlp.experts.56.down_proj", + "model.layers.45.mlp.experts.56.gate_proj", + "model.layers.45.mlp.experts.56.up_proj", + "model.layers.45.mlp.experts.57.down_proj", + "model.layers.45.mlp.experts.57.gate_proj", + "model.layers.45.mlp.experts.57.up_proj", + "model.layers.45.mlp.experts.58.down_proj", + "model.layers.45.mlp.experts.58.gate_proj", + "model.layers.45.mlp.experts.58.up_proj", + "model.layers.45.mlp.experts.59.down_proj", + "model.layers.45.mlp.experts.59.gate_proj", + "model.layers.45.mlp.experts.59.up_proj", + "model.layers.45.mlp.experts.6.down_proj", + "model.layers.45.mlp.experts.6.gate_proj", + "model.layers.45.mlp.experts.6.up_proj", + "model.layers.45.mlp.experts.60.down_proj", + "model.layers.45.mlp.experts.60.gate_proj", + "model.layers.45.mlp.experts.60.up_proj", + "model.layers.45.mlp.experts.61.down_proj", + "model.layers.45.mlp.experts.61.gate_proj", + "model.layers.45.mlp.experts.61.up_proj", + "model.layers.45.mlp.experts.62.down_proj", + "model.layers.45.mlp.experts.62.gate_proj", + "model.layers.45.mlp.experts.62.up_proj", + "model.layers.45.mlp.experts.63.down_proj", + "model.layers.45.mlp.experts.63.gate_proj", + "model.layers.45.mlp.experts.63.up_proj", + "model.layers.45.mlp.experts.64.down_proj", + "model.layers.45.mlp.experts.64.gate_proj", + "model.layers.45.mlp.experts.64.up_proj", + "model.layers.45.mlp.experts.65.down_proj", + "model.layers.45.mlp.experts.65.gate_proj", + "model.layers.45.mlp.experts.65.up_proj", + "model.layers.45.mlp.experts.66.down_proj", + "model.layers.45.mlp.experts.66.gate_proj", + "model.layers.45.mlp.experts.66.up_proj", + "model.layers.45.mlp.experts.67.down_proj", + "model.layers.45.mlp.experts.67.gate_proj", + "model.layers.45.mlp.experts.67.up_proj", + "model.layers.45.mlp.experts.68.down_proj", + "model.layers.45.mlp.experts.68.gate_proj", + "model.layers.45.mlp.experts.68.up_proj", + "model.layers.45.mlp.experts.69.down_proj", + "model.layers.45.mlp.experts.69.gate_proj", + "model.layers.45.mlp.experts.69.up_proj", + "model.layers.45.mlp.experts.7.down_proj", + "model.layers.45.mlp.experts.7.gate_proj", + "model.layers.45.mlp.experts.7.up_proj", + "model.layers.45.mlp.experts.70.down_proj", + "model.layers.45.mlp.experts.70.gate_proj", + "model.layers.45.mlp.experts.70.up_proj", + "model.layers.45.mlp.experts.71.down_proj", + "model.layers.45.mlp.experts.71.gate_proj", + "model.layers.45.mlp.experts.71.up_proj", + "model.layers.45.mlp.experts.72.down_proj", + "model.layers.45.mlp.experts.72.gate_proj", + "model.layers.45.mlp.experts.72.up_proj", + "model.layers.45.mlp.experts.73.down_proj", + "model.layers.45.mlp.experts.73.gate_proj", + "model.layers.45.mlp.experts.73.up_proj", + "model.layers.45.mlp.experts.74.down_proj", + "model.layers.45.mlp.experts.74.gate_proj", + "model.layers.45.mlp.experts.74.up_proj", + "model.layers.45.mlp.experts.75.down_proj", + "model.layers.45.mlp.experts.75.gate_proj", + "model.layers.45.mlp.experts.75.up_proj", + "model.layers.45.mlp.experts.76.down_proj", + "model.layers.45.mlp.experts.76.gate_proj", + "model.layers.45.mlp.experts.76.up_proj", + "model.layers.45.mlp.experts.77.down_proj", + "model.layers.45.mlp.experts.77.gate_proj", + "model.layers.45.mlp.experts.77.up_proj", + "model.layers.45.mlp.experts.78.down_proj", + "model.layers.45.mlp.experts.78.gate_proj", + "model.layers.45.mlp.experts.78.up_proj", + "model.layers.45.mlp.experts.79.down_proj", + "model.layers.45.mlp.experts.79.gate_proj", + "model.layers.45.mlp.experts.79.up_proj", + "model.layers.45.mlp.experts.8.down_proj", + "model.layers.45.mlp.experts.8.gate_proj", + "model.layers.45.mlp.experts.8.up_proj", + "model.layers.45.mlp.experts.80.down_proj", + "model.layers.45.mlp.experts.80.gate_proj", + "model.layers.45.mlp.experts.80.up_proj", + "model.layers.45.mlp.experts.81.down_proj", + "model.layers.45.mlp.experts.81.gate_proj", + "model.layers.45.mlp.experts.81.up_proj", + "model.layers.45.mlp.experts.82.down_proj", + "model.layers.45.mlp.experts.82.gate_proj", + "model.layers.45.mlp.experts.82.up_proj", + "model.layers.45.mlp.experts.83.down_proj", + "model.layers.45.mlp.experts.83.gate_proj", + "model.layers.45.mlp.experts.83.up_proj", + "model.layers.45.mlp.experts.84.down_proj", + "model.layers.45.mlp.experts.84.gate_proj", + "model.layers.45.mlp.experts.84.up_proj", + "model.layers.45.mlp.experts.85.down_proj", + "model.layers.45.mlp.experts.85.gate_proj", + "model.layers.45.mlp.experts.85.up_proj", + "model.layers.45.mlp.experts.86.down_proj", + "model.layers.45.mlp.experts.86.gate_proj", + "model.layers.45.mlp.experts.86.up_proj", + "model.layers.45.mlp.experts.87.down_proj", + "model.layers.45.mlp.experts.87.gate_proj", + "model.layers.45.mlp.experts.87.up_proj", + "model.layers.45.mlp.experts.88.down_proj", + "model.layers.45.mlp.experts.88.gate_proj", + "model.layers.45.mlp.experts.88.up_proj", + "model.layers.45.mlp.experts.89.down_proj", + "model.layers.45.mlp.experts.89.gate_proj", + "model.layers.45.mlp.experts.89.up_proj", + "model.layers.45.mlp.experts.9.down_proj", + "model.layers.45.mlp.experts.9.gate_proj", + "model.layers.45.mlp.experts.9.up_proj", + "model.layers.45.mlp.experts.90.down_proj", + "model.layers.45.mlp.experts.90.gate_proj", + "model.layers.45.mlp.experts.90.up_proj", + "model.layers.45.mlp.experts.91.down_proj", + "model.layers.45.mlp.experts.91.gate_proj", + "model.layers.45.mlp.experts.91.up_proj", + "model.layers.45.mlp.experts.92.down_proj", + "model.layers.45.mlp.experts.92.gate_proj", + "model.layers.45.mlp.experts.92.up_proj", + "model.layers.45.mlp.experts.93.down_proj", + "model.layers.45.mlp.experts.93.gate_proj", + "model.layers.45.mlp.experts.93.up_proj", + "model.layers.45.mlp.experts.94.down_proj", + "model.layers.45.mlp.experts.94.gate_proj", + "model.layers.45.mlp.experts.94.up_proj", + "model.layers.45.mlp.experts.95.down_proj", + "model.layers.45.mlp.experts.95.gate_proj", + "model.layers.45.mlp.experts.95.up_proj", + "model.layers.45.mlp.experts.96.down_proj", + "model.layers.45.mlp.experts.96.gate_proj", + "model.layers.45.mlp.experts.96.up_proj", + "model.layers.45.mlp.experts.97.down_proj", + "model.layers.45.mlp.experts.97.gate_proj", + "model.layers.45.mlp.experts.97.up_proj", + "model.layers.45.mlp.experts.98.down_proj", + "model.layers.45.mlp.experts.98.gate_proj", + "model.layers.45.mlp.experts.98.up_proj", + "model.layers.45.mlp.experts.99.down_proj", + "model.layers.45.mlp.experts.99.gate_proj", + "model.layers.45.mlp.experts.99.up_proj", + "model.layers.46.mlp.experts.0.down_proj", + "model.layers.46.mlp.experts.0.gate_proj", + "model.layers.46.mlp.experts.0.up_proj", + "model.layers.46.mlp.experts.1.down_proj", + "model.layers.46.mlp.experts.1.gate_proj", + "model.layers.46.mlp.experts.1.up_proj", + "model.layers.46.mlp.experts.10.down_proj", + "model.layers.46.mlp.experts.10.gate_proj", + "model.layers.46.mlp.experts.10.up_proj", + "model.layers.46.mlp.experts.100.down_proj", + "model.layers.46.mlp.experts.100.gate_proj", + "model.layers.46.mlp.experts.100.up_proj", + "model.layers.46.mlp.experts.101.down_proj", + "model.layers.46.mlp.experts.101.gate_proj", + "model.layers.46.mlp.experts.101.up_proj", + "model.layers.46.mlp.experts.102.down_proj", + "model.layers.46.mlp.experts.102.gate_proj", + "model.layers.46.mlp.experts.102.up_proj", + "model.layers.46.mlp.experts.103.down_proj", + "model.layers.46.mlp.experts.103.gate_proj", + "model.layers.46.mlp.experts.103.up_proj", + "model.layers.46.mlp.experts.104.down_proj", + "model.layers.46.mlp.experts.104.gate_proj", + "model.layers.46.mlp.experts.104.up_proj", + "model.layers.46.mlp.experts.105.down_proj", + "model.layers.46.mlp.experts.105.gate_proj", + "model.layers.46.mlp.experts.105.up_proj", + "model.layers.46.mlp.experts.106.down_proj", + "model.layers.46.mlp.experts.106.gate_proj", + "model.layers.46.mlp.experts.106.up_proj", + "model.layers.46.mlp.experts.107.down_proj", + "model.layers.46.mlp.experts.107.gate_proj", + "model.layers.46.mlp.experts.107.up_proj", + "model.layers.46.mlp.experts.108.down_proj", + "model.layers.46.mlp.experts.108.gate_proj", + "model.layers.46.mlp.experts.108.up_proj", + "model.layers.46.mlp.experts.109.down_proj", + "model.layers.46.mlp.experts.109.gate_proj", + "model.layers.46.mlp.experts.109.up_proj", + "model.layers.46.mlp.experts.11.down_proj", + "model.layers.46.mlp.experts.11.gate_proj", + "model.layers.46.mlp.experts.11.up_proj", + "model.layers.46.mlp.experts.110.down_proj", + "model.layers.46.mlp.experts.110.gate_proj", + "model.layers.46.mlp.experts.110.up_proj", + "model.layers.46.mlp.experts.111.down_proj", + "model.layers.46.mlp.experts.111.gate_proj", + "model.layers.46.mlp.experts.111.up_proj", + "model.layers.46.mlp.experts.112.down_proj", + "model.layers.46.mlp.experts.112.gate_proj", + "model.layers.46.mlp.experts.112.up_proj", + "model.layers.46.mlp.experts.113.down_proj", + "model.layers.46.mlp.experts.113.gate_proj", + "model.layers.46.mlp.experts.113.up_proj", + "model.layers.46.mlp.experts.114.down_proj", + "model.layers.46.mlp.experts.114.gate_proj", + "model.layers.46.mlp.experts.114.up_proj", + "model.layers.46.mlp.experts.115.down_proj", + "model.layers.46.mlp.experts.115.gate_proj", + "model.layers.46.mlp.experts.115.up_proj", + "model.layers.46.mlp.experts.116.down_proj", + "model.layers.46.mlp.experts.116.gate_proj", + "model.layers.46.mlp.experts.116.up_proj", + "model.layers.46.mlp.experts.117.down_proj", + "model.layers.46.mlp.experts.117.gate_proj", + "model.layers.46.mlp.experts.117.up_proj", + "model.layers.46.mlp.experts.118.down_proj", + "model.layers.46.mlp.experts.118.gate_proj", + "model.layers.46.mlp.experts.118.up_proj", + "model.layers.46.mlp.experts.119.down_proj", + "model.layers.46.mlp.experts.119.gate_proj", + "model.layers.46.mlp.experts.119.up_proj", + "model.layers.46.mlp.experts.12.down_proj", + "model.layers.46.mlp.experts.12.gate_proj", + "model.layers.46.mlp.experts.12.up_proj", + "model.layers.46.mlp.experts.120.down_proj", + "model.layers.46.mlp.experts.120.gate_proj", + "model.layers.46.mlp.experts.120.up_proj", + "model.layers.46.mlp.experts.121.down_proj", + "model.layers.46.mlp.experts.121.gate_proj", + "model.layers.46.mlp.experts.121.up_proj", + "model.layers.46.mlp.experts.122.down_proj", + "model.layers.46.mlp.experts.122.gate_proj", + "model.layers.46.mlp.experts.122.up_proj", + "model.layers.46.mlp.experts.123.down_proj", + "model.layers.46.mlp.experts.123.gate_proj", + "model.layers.46.mlp.experts.123.up_proj", + "model.layers.46.mlp.experts.124.down_proj", + "model.layers.46.mlp.experts.124.gate_proj", + "model.layers.46.mlp.experts.124.up_proj", + "model.layers.46.mlp.experts.125.down_proj", + "model.layers.46.mlp.experts.125.gate_proj", + "model.layers.46.mlp.experts.125.up_proj", + "model.layers.46.mlp.experts.126.down_proj", + "model.layers.46.mlp.experts.126.gate_proj", + "model.layers.46.mlp.experts.126.up_proj", + "model.layers.46.mlp.experts.127.down_proj", + "model.layers.46.mlp.experts.127.gate_proj", + "model.layers.46.mlp.experts.127.up_proj", + "model.layers.46.mlp.experts.128.down_proj", + "model.layers.46.mlp.experts.128.gate_proj", + "model.layers.46.mlp.experts.128.up_proj", + "model.layers.46.mlp.experts.129.down_proj", + "model.layers.46.mlp.experts.129.gate_proj", + "model.layers.46.mlp.experts.129.up_proj", + "model.layers.46.mlp.experts.13.down_proj", + "model.layers.46.mlp.experts.13.gate_proj", + "model.layers.46.mlp.experts.13.up_proj", + "model.layers.46.mlp.experts.130.down_proj", + "model.layers.46.mlp.experts.130.gate_proj", + "model.layers.46.mlp.experts.130.up_proj", + "model.layers.46.mlp.experts.131.down_proj", + "model.layers.46.mlp.experts.131.gate_proj", + "model.layers.46.mlp.experts.131.up_proj", + "model.layers.46.mlp.experts.132.down_proj", + "model.layers.46.mlp.experts.132.gate_proj", + "model.layers.46.mlp.experts.132.up_proj", + "model.layers.46.mlp.experts.133.down_proj", + "model.layers.46.mlp.experts.133.gate_proj", + "model.layers.46.mlp.experts.133.up_proj", + "model.layers.46.mlp.experts.134.down_proj", + "model.layers.46.mlp.experts.134.gate_proj", + "model.layers.46.mlp.experts.134.up_proj", + "model.layers.46.mlp.experts.135.down_proj", + "model.layers.46.mlp.experts.135.gate_proj", + "model.layers.46.mlp.experts.135.up_proj", + "model.layers.46.mlp.experts.136.down_proj", + "model.layers.46.mlp.experts.136.gate_proj", + "model.layers.46.mlp.experts.136.up_proj", + "model.layers.46.mlp.experts.137.down_proj", + "model.layers.46.mlp.experts.137.gate_proj", + "model.layers.46.mlp.experts.137.up_proj", + "model.layers.46.mlp.experts.138.down_proj", + "model.layers.46.mlp.experts.138.gate_proj", + "model.layers.46.mlp.experts.138.up_proj", + "model.layers.46.mlp.experts.139.down_proj", + "model.layers.46.mlp.experts.139.gate_proj", + "model.layers.46.mlp.experts.139.up_proj", + "model.layers.46.mlp.experts.14.down_proj", + "model.layers.46.mlp.experts.14.gate_proj", + "model.layers.46.mlp.experts.14.up_proj", + "model.layers.46.mlp.experts.140.down_proj", + "model.layers.46.mlp.experts.140.gate_proj", + "model.layers.46.mlp.experts.140.up_proj", + "model.layers.46.mlp.experts.141.down_proj", + "model.layers.46.mlp.experts.141.gate_proj", + "model.layers.46.mlp.experts.141.up_proj", + "model.layers.46.mlp.experts.142.down_proj", + "model.layers.46.mlp.experts.142.gate_proj", + "model.layers.46.mlp.experts.142.up_proj", + "model.layers.46.mlp.experts.143.down_proj", + "model.layers.46.mlp.experts.143.gate_proj", + "model.layers.46.mlp.experts.143.up_proj", + "model.layers.46.mlp.experts.144.down_proj", + "model.layers.46.mlp.experts.144.gate_proj", + "model.layers.46.mlp.experts.144.up_proj", + "model.layers.46.mlp.experts.145.down_proj", + "model.layers.46.mlp.experts.145.gate_proj", + "model.layers.46.mlp.experts.145.up_proj", + "model.layers.46.mlp.experts.146.down_proj", + "model.layers.46.mlp.experts.146.gate_proj", + "model.layers.46.mlp.experts.146.up_proj", + "model.layers.46.mlp.experts.147.down_proj", + "model.layers.46.mlp.experts.147.gate_proj", + "model.layers.46.mlp.experts.147.up_proj", + "model.layers.46.mlp.experts.148.down_proj", + "model.layers.46.mlp.experts.148.gate_proj", + "model.layers.46.mlp.experts.148.up_proj", + "model.layers.46.mlp.experts.149.down_proj", + "model.layers.46.mlp.experts.149.gate_proj", + "model.layers.46.mlp.experts.149.up_proj", + "model.layers.46.mlp.experts.15.down_proj", + "model.layers.46.mlp.experts.15.gate_proj", + "model.layers.46.mlp.experts.15.up_proj", + "model.layers.46.mlp.experts.150.down_proj", + "model.layers.46.mlp.experts.150.gate_proj", + "model.layers.46.mlp.experts.150.up_proj", + "model.layers.46.mlp.experts.151.down_proj", + "model.layers.46.mlp.experts.151.gate_proj", + "model.layers.46.mlp.experts.151.up_proj", + "model.layers.46.mlp.experts.152.down_proj", + "model.layers.46.mlp.experts.152.gate_proj", + "model.layers.46.mlp.experts.152.up_proj", + "model.layers.46.mlp.experts.153.down_proj", + "model.layers.46.mlp.experts.153.gate_proj", + "model.layers.46.mlp.experts.153.up_proj", + "model.layers.46.mlp.experts.154.down_proj", + "model.layers.46.mlp.experts.154.gate_proj", + "model.layers.46.mlp.experts.154.up_proj", + "model.layers.46.mlp.experts.155.down_proj", + "model.layers.46.mlp.experts.155.gate_proj", + "model.layers.46.mlp.experts.155.up_proj", + "model.layers.46.mlp.experts.156.down_proj", + "model.layers.46.mlp.experts.156.gate_proj", + "model.layers.46.mlp.experts.156.up_proj", + "model.layers.46.mlp.experts.157.down_proj", + "model.layers.46.mlp.experts.157.gate_proj", + "model.layers.46.mlp.experts.157.up_proj", + "model.layers.46.mlp.experts.158.down_proj", + "model.layers.46.mlp.experts.158.gate_proj", + "model.layers.46.mlp.experts.158.up_proj", + "model.layers.46.mlp.experts.159.down_proj", + "model.layers.46.mlp.experts.159.gate_proj", + "model.layers.46.mlp.experts.159.up_proj", + "model.layers.46.mlp.experts.16.down_proj", + "model.layers.46.mlp.experts.16.gate_proj", + "model.layers.46.mlp.experts.16.up_proj", + "model.layers.46.mlp.experts.160.down_proj", + "model.layers.46.mlp.experts.160.gate_proj", + "model.layers.46.mlp.experts.160.up_proj", + "model.layers.46.mlp.experts.161.down_proj", + "model.layers.46.mlp.experts.161.gate_proj", + "model.layers.46.mlp.experts.161.up_proj", + "model.layers.46.mlp.experts.162.down_proj", + "model.layers.46.mlp.experts.162.gate_proj", + "model.layers.46.mlp.experts.162.up_proj", + "model.layers.46.mlp.experts.163.down_proj", + "model.layers.46.mlp.experts.163.gate_proj", + "model.layers.46.mlp.experts.163.up_proj", + "model.layers.46.mlp.experts.164.down_proj", + "model.layers.46.mlp.experts.164.gate_proj", + "model.layers.46.mlp.experts.164.up_proj", + "model.layers.46.mlp.experts.165.down_proj", + "model.layers.46.mlp.experts.165.gate_proj", + "model.layers.46.mlp.experts.165.up_proj", + "model.layers.46.mlp.experts.166.down_proj", + "model.layers.46.mlp.experts.166.gate_proj", + "model.layers.46.mlp.experts.166.up_proj", + "model.layers.46.mlp.experts.167.down_proj", + "model.layers.46.mlp.experts.167.gate_proj", + "model.layers.46.mlp.experts.167.up_proj", + "model.layers.46.mlp.experts.168.down_proj", + "model.layers.46.mlp.experts.168.gate_proj", + "model.layers.46.mlp.experts.168.up_proj", + "model.layers.46.mlp.experts.169.down_proj", + "model.layers.46.mlp.experts.169.gate_proj", + "model.layers.46.mlp.experts.169.up_proj", + "model.layers.46.mlp.experts.17.down_proj", + "model.layers.46.mlp.experts.17.gate_proj", + "model.layers.46.mlp.experts.17.up_proj", + "model.layers.46.mlp.experts.170.down_proj", + "model.layers.46.mlp.experts.170.gate_proj", + "model.layers.46.mlp.experts.170.up_proj", + "model.layers.46.mlp.experts.171.down_proj", + "model.layers.46.mlp.experts.171.gate_proj", + "model.layers.46.mlp.experts.171.up_proj", + "model.layers.46.mlp.experts.172.down_proj", + "model.layers.46.mlp.experts.172.gate_proj", + "model.layers.46.mlp.experts.172.up_proj", + "model.layers.46.mlp.experts.173.down_proj", + "model.layers.46.mlp.experts.173.gate_proj", + "model.layers.46.mlp.experts.173.up_proj", + "model.layers.46.mlp.experts.174.down_proj", + "model.layers.46.mlp.experts.174.gate_proj", + "model.layers.46.mlp.experts.174.up_proj", + "model.layers.46.mlp.experts.175.down_proj", + "model.layers.46.mlp.experts.175.gate_proj", + "model.layers.46.mlp.experts.175.up_proj", + "model.layers.46.mlp.experts.176.down_proj", + "model.layers.46.mlp.experts.176.gate_proj", + "model.layers.46.mlp.experts.176.up_proj", + "model.layers.46.mlp.experts.177.down_proj", + "model.layers.46.mlp.experts.177.gate_proj", + "model.layers.46.mlp.experts.177.up_proj", + "model.layers.46.mlp.experts.178.down_proj", + "model.layers.46.mlp.experts.178.gate_proj", + "model.layers.46.mlp.experts.178.up_proj", + "model.layers.46.mlp.experts.179.down_proj", + "model.layers.46.mlp.experts.179.gate_proj", + "model.layers.46.mlp.experts.179.up_proj", + "model.layers.46.mlp.experts.18.down_proj", + "model.layers.46.mlp.experts.18.gate_proj", + "model.layers.46.mlp.experts.18.up_proj", + "model.layers.46.mlp.experts.180.down_proj", + "model.layers.46.mlp.experts.180.gate_proj", + "model.layers.46.mlp.experts.180.up_proj", + "model.layers.46.mlp.experts.181.down_proj", + "model.layers.46.mlp.experts.181.gate_proj", + "model.layers.46.mlp.experts.181.up_proj", + "model.layers.46.mlp.experts.182.down_proj", + "model.layers.46.mlp.experts.182.gate_proj", + "model.layers.46.mlp.experts.182.up_proj", + "model.layers.46.mlp.experts.183.down_proj", + "model.layers.46.mlp.experts.183.gate_proj", + "model.layers.46.mlp.experts.183.up_proj", + "model.layers.46.mlp.experts.184.down_proj", + "model.layers.46.mlp.experts.184.gate_proj", + "model.layers.46.mlp.experts.184.up_proj", + "model.layers.46.mlp.experts.185.down_proj", + "model.layers.46.mlp.experts.185.gate_proj", + "model.layers.46.mlp.experts.185.up_proj", + "model.layers.46.mlp.experts.186.down_proj", + "model.layers.46.mlp.experts.186.gate_proj", + "model.layers.46.mlp.experts.186.up_proj", + "model.layers.46.mlp.experts.187.down_proj", + "model.layers.46.mlp.experts.187.gate_proj", + "model.layers.46.mlp.experts.187.up_proj", + "model.layers.46.mlp.experts.188.down_proj", + "model.layers.46.mlp.experts.188.gate_proj", + "model.layers.46.mlp.experts.188.up_proj", + "model.layers.46.mlp.experts.189.down_proj", + "model.layers.46.mlp.experts.189.gate_proj", + "model.layers.46.mlp.experts.189.up_proj", + "model.layers.46.mlp.experts.19.down_proj", + "model.layers.46.mlp.experts.19.gate_proj", + "model.layers.46.mlp.experts.19.up_proj", + "model.layers.46.mlp.experts.190.down_proj", + "model.layers.46.mlp.experts.190.gate_proj", + "model.layers.46.mlp.experts.190.up_proj", + "model.layers.46.mlp.experts.191.down_proj", + "model.layers.46.mlp.experts.191.gate_proj", + "model.layers.46.mlp.experts.191.up_proj", + "model.layers.46.mlp.experts.192.down_proj", + "model.layers.46.mlp.experts.192.gate_proj", + "model.layers.46.mlp.experts.192.up_proj", + "model.layers.46.mlp.experts.193.down_proj", + "model.layers.46.mlp.experts.193.gate_proj", + "model.layers.46.mlp.experts.193.up_proj", + "model.layers.46.mlp.experts.194.down_proj", + "model.layers.46.mlp.experts.194.gate_proj", + "model.layers.46.mlp.experts.194.up_proj", + "model.layers.46.mlp.experts.195.down_proj", + "model.layers.46.mlp.experts.195.gate_proj", + "model.layers.46.mlp.experts.195.up_proj", + "model.layers.46.mlp.experts.196.down_proj", + "model.layers.46.mlp.experts.196.gate_proj", + "model.layers.46.mlp.experts.196.up_proj", + "model.layers.46.mlp.experts.197.down_proj", + "model.layers.46.mlp.experts.197.gate_proj", + "model.layers.46.mlp.experts.197.up_proj", + "model.layers.46.mlp.experts.198.down_proj", + "model.layers.46.mlp.experts.198.gate_proj", + "model.layers.46.mlp.experts.198.up_proj", + "model.layers.46.mlp.experts.199.down_proj", + "model.layers.46.mlp.experts.199.gate_proj", + "model.layers.46.mlp.experts.199.up_proj", + "model.layers.46.mlp.experts.2.down_proj", + "model.layers.46.mlp.experts.2.gate_proj", + "model.layers.46.mlp.experts.2.up_proj", + "model.layers.46.mlp.experts.20.down_proj", + "model.layers.46.mlp.experts.20.gate_proj", + "model.layers.46.mlp.experts.20.up_proj", + "model.layers.46.mlp.experts.200.down_proj", + "model.layers.46.mlp.experts.200.gate_proj", + "model.layers.46.mlp.experts.200.up_proj", + "model.layers.46.mlp.experts.201.down_proj", + "model.layers.46.mlp.experts.201.gate_proj", + "model.layers.46.mlp.experts.201.up_proj", + "model.layers.46.mlp.experts.202.down_proj", + "model.layers.46.mlp.experts.202.gate_proj", + "model.layers.46.mlp.experts.202.up_proj", + "model.layers.46.mlp.experts.203.down_proj", + "model.layers.46.mlp.experts.203.gate_proj", + "model.layers.46.mlp.experts.203.up_proj", + "model.layers.46.mlp.experts.204.down_proj", + "model.layers.46.mlp.experts.204.gate_proj", + "model.layers.46.mlp.experts.204.up_proj", + "model.layers.46.mlp.experts.205.down_proj", + "model.layers.46.mlp.experts.205.gate_proj", + "model.layers.46.mlp.experts.205.up_proj", + "model.layers.46.mlp.experts.206.down_proj", + "model.layers.46.mlp.experts.206.gate_proj", + "model.layers.46.mlp.experts.206.up_proj", + "model.layers.46.mlp.experts.207.down_proj", + "model.layers.46.mlp.experts.207.gate_proj", + "model.layers.46.mlp.experts.207.up_proj", + "model.layers.46.mlp.experts.208.down_proj", + "model.layers.46.mlp.experts.208.gate_proj", + "model.layers.46.mlp.experts.208.up_proj", + "model.layers.46.mlp.experts.209.down_proj", + "model.layers.46.mlp.experts.209.gate_proj", + "model.layers.46.mlp.experts.209.up_proj", + "model.layers.46.mlp.experts.21.down_proj", + "model.layers.46.mlp.experts.21.gate_proj", + "model.layers.46.mlp.experts.21.up_proj", + "model.layers.46.mlp.experts.210.down_proj", + "model.layers.46.mlp.experts.210.gate_proj", + "model.layers.46.mlp.experts.210.up_proj", + "model.layers.46.mlp.experts.211.down_proj", + "model.layers.46.mlp.experts.211.gate_proj", + "model.layers.46.mlp.experts.211.up_proj", + "model.layers.46.mlp.experts.212.down_proj", + "model.layers.46.mlp.experts.212.gate_proj", + "model.layers.46.mlp.experts.212.up_proj", + "model.layers.46.mlp.experts.213.down_proj", + "model.layers.46.mlp.experts.213.gate_proj", + "model.layers.46.mlp.experts.213.up_proj", + "model.layers.46.mlp.experts.214.down_proj", + "model.layers.46.mlp.experts.214.gate_proj", + "model.layers.46.mlp.experts.214.up_proj", + "model.layers.46.mlp.experts.215.down_proj", + "model.layers.46.mlp.experts.215.gate_proj", + "model.layers.46.mlp.experts.215.up_proj", + "model.layers.46.mlp.experts.216.down_proj", + "model.layers.46.mlp.experts.216.gate_proj", + "model.layers.46.mlp.experts.216.up_proj", + "model.layers.46.mlp.experts.217.down_proj", + "model.layers.46.mlp.experts.217.gate_proj", + "model.layers.46.mlp.experts.217.up_proj", + "model.layers.46.mlp.experts.218.down_proj", + "model.layers.46.mlp.experts.218.gate_proj", + "model.layers.46.mlp.experts.218.up_proj", + "model.layers.46.mlp.experts.219.down_proj", + "model.layers.46.mlp.experts.219.gate_proj", + "model.layers.46.mlp.experts.219.up_proj", + "model.layers.46.mlp.experts.22.down_proj", + "model.layers.46.mlp.experts.22.gate_proj", + "model.layers.46.mlp.experts.22.up_proj", + "model.layers.46.mlp.experts.220.down_proj", + "model.layers.46.mlp.experts.220.gate_proj", + "model.layers.46.mlp.experts.220.up_proj", + "model.layers.46.mlp.experts.221.down_proj", + "model.layers.46.mlp.experts.221.gate_proj", + "model.layers.46.mlp.experts.221.up_proj", + "model.layers.46.mlp.experts.222.down_proj", + "model.layers.46.mlp.experts.222.gate_proj", + "model.layers.46.mlp.experts.222.up_proj", + "model.layers.46.mlp.experts.223.down_proj", + "model.layers.46.mlp.experts.223.gate_proj", + "model.layers.46.mlp.experts.223.up_proj", + "model.layers.46.mlp.experts.224.down_proj", + "model.layers.46.mlp.experts.224.gate_proj", + "model.layers.46.mlp.experts.224.up_proj", + "model.layers.46.mlp.experts.225.down_proj", + "model.layers.46.mlp.experts.225.gate_proj", + "model.layers.46.mlp.experts.225.up_proj", + "model.layers.46.mlp.experts.226.down_proj", + "model.layers.46.mlp.experts.226.gate_proj", + "model.layers.46.mlp.experts.226.up_proj", + "model.layers.46.mlp.experts.227.down_proj", + "model.layers.46.mlp.experts.227.gate_proj", + "model.layers.46.mlp.experts.227.up_proj", + "model.layers.46.mlp.experts.228.down_proj", + "model.layers.46.mlp.experts.228.gate_proj", + "model.layers.46.mlp.experts.228.up_proj", + "model.layers.46.mlp.experts.229.down_proj", + "model.layers.46.mlp.experts.229.gate_proj", + "model.layers.46.mlp.experts.229.up_proj", + "model.layers.46.mlp.experts.23.down_proj", + "model.layers.46.mlp.experts.23.gate_proj", + "model.layers.46.mlp.experts.23.up_proj", + "model.layers.46.mlp.experts.230.down_proj", + "model.layers.46.mlp.experts.230.gate_proj", + "model.layers.46.mlp.experts.230.up_proj", + "model.layers.46.mlp.experts.231.down_proj", + "model.layers.46.mlp.experts.231.gate_proj", + "model.layers.46.mlp.experts.231.up_proj", + "model.layers.46.mlp.experts.232.down_proj", + "model.layers.46.mlp.experts.232.gate_proj", + "model.layers.46.mlp.experts.232.up_proj", + "model.layers.46.mlp.experts.233.down_proj", + "model.layers.46.mlp.experts.233.gate_proj", + "model.layers.46.mlp.experts.233.up_proj", + "model.layers.46.mlp.experts.234.down_proj", + "model.layers.46.mlp.experts.234.gate_proj", + "model.layers.46.mlp.experts.234.up_proj", + "model.layers.46.mlp.experts.235.down_proj", + "model.layers.46.mlp.experts.235.gate_proj", + "model.layers.46.mlp.experts.235.up_proj", + "model.layers.46.mlp.experts.236.down_proj", + "model.layers.46.mlp.experts.236.gate_proj", + "model.layers.46.mlp.experts.236.up_proj", + "model.layers.46.mlp.experts.237.down_proj", + "model.layers.46.mlp.experts.237.gate_proj", + "model.layers.46.mlp.experts.237.up_proj", + "model.layers.46.mlp.experts.238.down_proj", + "model.layers.46.mlp.experts.238.gate_proj", + "model.layers.46.mlp.experts.238.up_proj", + "model.layers.46.mlp.experts.239.down_proj", + "model.layers.46.mlp.experts.239.gate_proj", + "model.layers.46.mlp.experts.239.up_proj", + "model.layers.46.mlp.experts.24.down_proj", + "model.layers.46.mlp.experts.24.gate_proj", + "model.layers.46.mlp.experts.24.up_proj", + "model.layers.46.mlp.experts.240.down_proj", + "model.layers.46.mlp.experts.240.gate_proj", + "model.layers.46.mlp.experts.240.up_proj", + "model.layers.46.mlp.experts.241.down_proj", + "model.layers.46.mlp.experts.241.gate_proj", + "model.layers.46.mlp.experts.241.up_proj", + "model.layers.46.mlp.experts.242.down_proj", + "model.layers.46.mlp.experts.242.gate_proj", + "model.layers.46.mlp.experts.242.up_proj", + "model.layers.46.mlp.experts.243.down_proj", + "model.layers.46.mlp.experts.243.gate_proj", + "model.layers.46.mlp.experts.243.up_proj", + "model.layers.46.mlp.experts.244.down_proj", + "model.layers.46.mlp.experts.244.gate_proj", + "model.layers.46.mlp.experts.244.up_proj", + "model.layers.46.mlp.experts.245.down_proj", + "model.layers.46.mlp.experts.245.gate_proj", + "model.layers.46.mlp.experts.245.up_proj", + "model.layers.46.mlp.experts.246.down_proj", + "model.layers.46.mlp.experts.246.gate_proj", + "model.layers.46.mlp.experts.246.up_proj", + "model.layers.46.mlp.experts.247.down_proj", + "model.layers.46.mlp.experts.247.gate_proj", + "model.layers.46.mlp.experts.247.up_proj", + "model.layers.46.mlp.experts.248.down_proj", + "model.layers.46.mlp.experts.248.gate_proj", + "model.layers.46.mlp.experts.248.up_proj", + "model.layers.46.mlp.experts.249.down_proj", + "model.layers.46.mlp.experts.249.gate_proj", + "model.layers.46.mlp.experts.249.up_proj", + "model.layers.46.mlp.experts.25.down_proj", + "model.layers.46.mlp.experts.25.gate_proj", + "model.layers.46.mlp.experts.25.up_proj", + "model.layers.46.mlp.experts.250.down_proj", + "model.layers.46.mlp.experts.250.gate_proj", + "model.layers.46.mlp.experts.250.up_proj", + "model.layers.46.mlp.experts.251.down_proj", + "model.layers.46.mlp.experts.251.gate_proj", + "model.layers.46.mlp.experts.251.up_proj", + "model.layers.46.mlp.experts.252.down_proj", + "model.layers.46.mlp.experts.252.gate_proj", + "model.layers.46.mlp.experts.252.up_proj", + "model.layers.46.mlp.experts.253.down_proj", + "model.layers.46.mlp.experts.253.gate_proj", + "model.layers.46.mlp.experts.253.up_proj", + "model.layers.46.mlp.experts.254.down_proj", + "model.layers.46.mlp.experts.254.gate_proj", + "model.layers.46.mlp.experts.254.up_proj", + "model.layers.46.mlp.experts.255.down_proj", + "model.layers.46.mlp.experts.255.gate_proj", + "model.layers.46.mlp.experts.255.up_proj", + "model.layers.46.mlp.experts.26.down_proj", + "model.layers.46.mlp.experts.26.gate_proj", + "model.layers.46.mlp.experts.26.up_proj", + "model.layers.46.mlp.experts.27.down_proj", + "model.layers.46.mlp.experts.27.gate_proj", + "model.layers.46.mlp.experts.27.up_proj", + "model.layers.46.mlp.experts.28.down_proj", + "model.layers.46.mlp.experts.28.gate_proj", + "model.layers.46.mlp.experts.28.up_proj", + "model.layers.46.mlp.experts.29.down_proj", + "model.layers.46.mlp.experts.29.gate_proj", + "model.layers.46.mlp.experts.29.up_proj", + "model.layers.46.mlp.experts.3.down_proj", + "model.layers.46.mlp.experts.3.gate_proj", + "model.layers.46.mlp.experts.3.up_proj", + "model.layers.46.mlp.experts.30.down_proj", + "model.layers.46.mlp.experts.30.gate_proj", + "model.layers.46.mlp.experts.30.up_proj", + "model.layers.46.mlp.experts.31.down_proj", + "model.layers.46.mlp.experts.31.gate_proj", + "model.layers.46.mlp.experts.31.up_proj", + "model.layers.46.mlp.experts.32.down_proj", + "model.layers.46.mlp.experts.32.gate_proj", + "model.layers.46.mlp.experts.32.up_proj", + "model.layers.46.mlp.experts.33.down_proj", + "model.layers.46.mlp.experts.33.gate_proj", + "model.layers.46.mlp.experts.33.up_proj", + "model.layers.46.mlp.experts.34.down_proj", + "model.layers.46.mlp.experts.34.gate_proj", + "model.layers.46.mlp.experts.34.up_proj", + "model.layers.46.mlp.experts.35.down_proj", + "model.layers.46.mlp.experts.35.gate_proj", + "model.layers.46.mlp.experts.35.up_proj", + "model.layers.46.mlp.experts.36.down_proj", + "model.layers.46.mlp.experts.36.gate_proj", + "model.layers.46.mlp.experts.36.up_proj", + "model.layers.46.mlp.experts.37.down_proj", + "model.layers.46.mlp.experts.37.gate_proj", + "model.layers.46.mlp.experts.37.up_proj", + "model.layers.46.mlp.experts.38.down_proj", + "model.layers.46.mlp.experts.38.gate_proj", + "model.layers.46.mlp.experts.38.up_proj", + "model.layers.46.mlp.experts.39.down_proj", + "model.layers.46.mlp.experts.39.gate_proj", + "model.layers.46.mlp.experts.39.up_proj", + "model.layers.46.mlp.experts.4.down_proj", + "model.layers.46.mlp.experts.4.gate_proj", + "model.layers.46.mlp.experts.4.up_proj", + "model.layers.46.mlp.experts.40.down_proj", + "model.layers.46.mlp.experts.40.gate_proj", + "model.layers.46.mlp.experts.40.up_proj", + "model.layers.46.mlp.experts.41.down_proj", + "model.layers.46.mlp.experts.41.gate_proj", + "model.layers.46.mlp.experts.41.up_proj", + "model.layers.46.mlp.experts.42.down_proj", + "model.layers.46.mlp.experts.42.gate_proj", + "model.layers.46.mlp.experts.42.up_proj", + "model.layers.46.mlp.experts.43.down_proj", + "model.layers.46.mlp.experts.43.gate_proj", + "model.layers.46.mlp.experts.43.up_proj", + "model.layers.46.mlp.experts.44.down_proj", + "model.layers.46.mlp.experts.44.gate_proj", + "model.layers.46.mlp.experts.44.up_proj", + "model.layers.46.mlp.experts.45.down_proj", + "model.layers.46.mlp.experts.45.gate_proj", + "model.layers.46.mlp.experts.45.up_proj", + "model.layers.46.mlp.experts.46.down_proj", + "model.layers.46.mlp.experts.46.gate_proj", + "model.layers.46.mlp.experts.46.up_proj", + "model.layers.46.mlp.experts.47.down_proj", + "model.layers.46.mlp.experts.47.gate_proj", + "model.layers.46.mlp.experts.47.up_proj", + "model.layers.46.mlp.experts.48.down_proj", + "model.layers.46.mlp.experts.48.gate_proj", + "model.layers.46.mlp.experts.48.up_proj", + "model.layers.46.mlp.experts.49.down_proj", + "model.layers.46.mlp.experts.49.gate_proj", + "model.layers.46.mlp.experts.49.up_proj", + "model.layers.46.mlp.experts.5.down_proj", + "model.layers.46.mlp.experts.5.gate_proj", + "model.layers.46.mlp.experts.5.up_proj", + "model.layers.46.mlp.experts.50.down_proj", + "model.layers.46.mlp.experts.50.gate_proj", + "model.layers.46.mlp.experts.50.up_proj", + "model.layers.46.mlp.experts.51.down_proj", + "model.layers.46.mlp.experts.51.gate_proj", + "model.layers.46.mlp.experts.51.up_proj", + "model.layers.46.mlp.experts.52.down_proj", + "model.layers.46.mlp.experts.52.gate_proj", + "model.layers.46.mlp.experts.52.up_proj", + "model.layers.46.mlp.experts.53.down_proj", + "model.layers.46.mlp.experts.53.gate_proj", + "model.layers.46.mlp.experts.53.up_proj", + "model.layers.46.mlp.experts.54.down_proj", + "model.layers.46.mlp.experts.54.gate_proj", + "model.layers.46.mlp.experts.54.up_proj", + "model.layers.46.mlp.experts.55.down_proj", + "model.layers.46.mlp.experts.55.gate_proj", + "model.layers.46.mlp.experts.55.up_proj", + "model.layers.46.mlp.experts.56.down_proj", + "model.layers.46.mlp.experts.56.gate_proj", + "model.layers.46.mlp.experts.56.up_proj", + "model.layers.46.mlp.experts.57.down_proj", + "model.layers.46.mlp.experts.57.gate_proj", + "model.layers.46.mlp.experts.57.up_proj", + "model.layers.46.mlp.experts.58.down_proj", + "model.layers.46.mlp.experts.58.gate_proj", + "model.layers.46.mlp.experts.58.up_proj", + "model.layers.46.mlp.experts.59.down_proj", + "model.layers.46.mlp.experts.59.gate_proj", + "model.layers.46.mlp.experts.59.up_proj", + "model.layers.46.mlp.experts.6.down_proj", + "model.layers.46.mlp.experts.6.gate_proj", + "model.layers.46.mlp.experts.6.up_proj", + "model.layers.46.mlp.experts.60.down_proj", + "model.layers.46.mlp.experts.60.gate_proj", + "model.layers.46.mlp.experts.60.up_proj", + "model.layers.46.mlp.experts.61.down_proj", + "model.layers.46.mlp.experts.61.gate_proj", + "model.layers.46.mlp.experts.61.up_proj", + "model.layers.46.mlp.experts.62.down_proj", + "model.layers.46.mlp.experts.62.gate_proj", + "model.layers.46.mlp.experts.62.up_proj", + "model.layers.46.mlp.experts.63.down_proj", + "model.layers.46.mlp.experts.63.gate_proj", + "model.layers.46.mlp.experts.63.up_proj", + "model.layers.46.mlp.experts.64.down_proj", + "model.layers.46.mlp.experts.64.gate_proj", + "model.layers.46.mlp.experts.64.up_proj", + "model.layers.46.mlp.experts.65.down_proj", + "model.layers.46.mlp.experts.65.gate_proj", + "model.layers.46.mlp.experts.65.up_proj", + "model.layers.46.mlp.experts.66.down_proj", + "model.layers.46.mlp.experts.66.gate_proj", + "model.layers.46.mlp.experts.66.up_proj", + "model.layers.46.mlp.experts.67.down_proj", + "model.layers.46.mlp.experts.67.gate_proj", + "model.layers.46.mlp.experts.67.up_proj", + "model.layers.46.mlp.experts.68.down_proj", + "model.layers.46.mlp.experts.68.gate_proj", + "model.layers.46.mlp.experts.68.up_proj", + "model.layers.46.mlp.experts.69.down_proj", + "model.layers.46.mlp.experts.69.gate_proj", + "model.layers.46.mlp.experts.69.up_proj", + "model.layers.46.mlp.experts.7.down_proj", + "model.layers.46.mlp.experts.7.gate_proj", + "model.layers.46.mlp.experts.7.up_proj", + "model.layers.46.mlp.experts.70.down_proj", + "model.layers.46.mlp.experts.70.gate_proj", + "model.layers.46.mlp.experts.70.up_proj", + "model.layers.46.mlp.experts.71.down_proj", + "model.layers.46.mlp.experts.71.gate_proj", + "model.layers.46.mlp.experts.71.up_proj", + "model.layers.46.mlp.experts.72.down_proj", + "model.layers.46.mlp.experts.72.gate_proj", + "model.layers.46.mlp.experts.72.up_proj", + "model.layers.46.mlp.experts.73.down_proj", + "model.layers.46.mlp.experts.73.gate_proj", + "model.layers.46.mlp.experts.73.up_proj", + "model.layers.46.mlp.experts.74.down_proj", + "model.layers.46.mlp.experts.74.gate_proj", + "model.layers.46.mlp.experts.74.up_proj", + "model.layers.46.mlp.experts.75.down_proj", + "model.layers.46.mlp.experts.75.gate_proj", + "model.layers.46.mlp.experts.75.up_proj", + "model.layers.46.mlp.experts.76.down_proj", + "model.layers.46.mlp.experts.76.gate_proj", + "model.layers.46.mlp.experts.76.up_proj", + "model.layers.46.mlp.experts.77.down_proj", + "model.layers.46.mlp.experts.77.gate_proj", + "model.layers.46.mlp.experts.77.up_proj", + "model.layers.46.mlp.experts.78.down_proj", + "model.layers.46.mlp.experts.78.gate_proj", + "model.layers.46.mlp.experts.78.up_proj", + "model.layers.46.mlp.experts.79.down_proj", + "model.layers.46.mlp.experts.79.gate_proj", + "model.layers.46.mlp.experts.79.up_proj", + "model.layers.46.mlp.experts.8.down_proj", + "model.layers.46.mlp.experts.8.gate_proj", + "model.layers.46.mlp.experts.8.up_proj", + "model.layers.46.mlp.experts.80.down_proj", + "model.layers.46.mlp.experts.80.gate_proj", + "model.layers.46.mlp.experts.80.up_proj", + "model.layers.46.mlp.experts.81.down_proj", + "model.layers.46.mlp.experts.81.gate_proj", + "model.layers.46.mlp.experts.81.up_proj", + "model.layers.46.mlp.experts.82.down_proj", + "model.layers.46.mlp.experts.82.gate_proj", + "model.layers.46.mlp.experts.82.up_proj", + "model.layers.46.mlp.experts.83.down_proj", + "model.layers.46.mlp.experts.83.gate_proj", + "model.layers.46.mlp.experts.83.up_proj", + "model.layers.46.mlp.experts.84.down_proj", + "model.layers.46.mlp.experts.84.gate_proj", + "model.layers.46.mlp.experts.84.up_proj", + "model.layers.46.mlp.experts.85.down_proj", + "model.layers.46.mlp.experts.85.gate_proj", + "model.layers.46.mlp.experts.85.up_proj", + "model.layers.46.mlp.experts.86.down_proj", + "model.layers.46.mlp.experts.86.gate_proj", + "model.layers.46.mlp.experts.86.up_proj", + "model.layers.46.mlp.experts.87.down_proj", + "model.layers.46.mlp.experts.87.gate_proj", + "model.layers.46.mlp.experts.87.up_proj", + "model.layers.46.mlp.experts.88.down_proj", + "model.layers.46.mlp.experts.88.gate_proj", + "model.layers.46.mlp.experts.88.up_proj", + "model.layers.46.mlp.experts.89.down_proj", + "model.layers.46.mlp.experts.89.gate_proj", + "model.layers.46.mlp.experts.89.up_proj", + "model.layers.46.mlp.experts.9.down_proj", + "model.layers.46.mlp.experts.9.gate_proj", + "model.layers.46.mlp.experts.9.up_proj", + "model.layers.46.mlp.experts.90.down_proj", + "model.layers.46.mlp.experts.90.gate_proj", + "model.layers.46.mlp.experts.90.up_proj", + "model.layers.46.mlp.experts.91.down_proj", + "model.layers.46.mlp.experts.91.gate_proj", + "model.layers.46.mlp.experts.91.up_proj", + "model.layers.46.mlp.experts.92.down_proj", + "model.layers.46.mlp.experts.92.gate_proj", + "model.layers.46.mlp.experts.92.up_proj", + "model.layers.46.mlp.experts.93.down_proj", + "model.layers.46.mlp.experts.93.gate_proj", + "model.layers.46.mlp.experts.93.up_proj", + "model.layers.46.mlp.experts.94.down_proj", + "model.layers.46.mlp.experts.94.gate_proj", + "model.layers.46.mlp.experts.94.up_proj", + "model.layers.46.mlp.experts.95.down_proj", + "model.layers.46.mlp.experts.95.gate_proj", + "model.layers.46.mlp.experts.95.up_proj", + "model.layers.46.mlp.experts.96.down_proj", + "model.layers.46.mlp.experts.96.gate_proj", + "model.layers.46.mlp.experts.96.up_proj", + "model.layers.46.mlp.experts.97.down_proj", + "model.layers.46.mlp.experts.97.gate_proj", + "model.layers.46.mlp.experts.97.up_proj", + "model.layers.46.mlp.experts.98.down_proj", + "model.layers.46.mlp.experts.98.gate_proj", + "model.layers.46.mlp.experts.98.up_proj", + "model.layers.46.mlp.experts.99.down_proj", + "model.layers.46.mlp.experts.99.gate_proj", + "model.layers.46.mlp.experts.99.up_proj", + "model.layers.47.mlp.experts.0.down_proj", + "model.layers.47.mlp.experts.0.gate_proj", + "model.layers.47.mlp.experts.0.up_proj", + "model.layers.47.mlp.experts.1.down_proj", + "model.layers.47.mlp.experts.1.gate_proj", + "model.layers.47.mlp.experts.1.up_proj", + "model.layers.47.mlp.experts.10.down_proj", + "model.layers.47.mlp.experts.10.gate_proj", + "model.layers.47.mlp.experts.10.up_proj", + "model.layers.47.mlp.experts.100.down_proj", + "model.layers.47.mlp.experts.100.gate_proj", + "model.layers.47.mlp.experts.100.up_proj", + "model.layers.47.mlp.experts.101.down_proj", + "model.layers.47.mlp.experts.101.gate_proj", + "model.layers.47.mlp.experts.101.up_proj", + "model.layers.47.mlp.experts.102.down_proj", + "model.layers.47.mlp.experts.102.gate_proj", + "model.layers.47.mlp.experts.102.up_proj", + "model.layers.47.mlp.experts.103.down_proj", + "model.layers.47.mlp.experts.103.gate_proj", + "model.layers.47.mlp.experts.103.up_proj", + "model.layers.47.mlp.experts.104.down_proj", + "model.layers.47.mlp.experts.104.gate_proj", + "model.layers.47.mlp.experts.104.up_proj", + "model.layers.47.mlp.experts.105.down_proj", + "model.layers.47.mlp.experts.105.gate_proj", + "model.layers.47.mlp.experts.105.up_proj", + "model.layers.47.mlp.experts.106.down_proj", + "model.layers.47.mlp.experts.106.gate_proj", + "model.layers.47.mlp.experts.106.up_proj", + "model.layers.47.mlp.experts.107.down_proj", + "model.layers.47.mlp.experts.107.gate_proj", + "model.layers.47.mlp.experts.107.up_proj", + "model.layers.47.mlp.experts.108.down_proj", + "model.layers.47.mlp.experts.108.gate_proj", + "model.layers.47.mlp.experts.108.up_proj", + "model.layers.47.mlp.experts.109.down_proj", + "model.layers.47.mlp.experts.109.gate_proj", + "model.layers.47.mlp.experts.109.up_proj", + "model.layers.47.mlp.experts.11.down_proj", + "model.layers.47.mlp.experts.11.gate_proj", + "model.layers.47.mlp.experts.11.up_proj", + "model.layers.47.mlp.experts.110.down_proj", + "model.layers.47.mlp.experts.110.gate_proj", + "model.layers.47.mlp.experts.110.up_proj", + "model.layers.47.mlp.experts.111.down_proj", + "model.layers.47.mlp.experts.111.gate_proj", + "model.layers.47.mlp.experts.111.up_proj", + "model.layers.47.mlp.experts.112.down_proj", + "model.layers.47.mlp.experts.112.gate_proj", + "model.layers.47.mlp.experts.112.up_proj", + "model.layers.47.mlp.experts.113.down_proj", + "model.layers.47.mlp.experts.113.gate_proj", + "model.layers.47.mlp.experts.113.up_proj", + "model.layers.47.mlp.experts.114.down_proj", + "model.layers.47.mlp.experts.114.gate_proj", + "model.layers.47.mlp.experts.114.up_proj", + "model.layers.47.mlp.experts.115.down_proj", + "model.layers.47.mlp.experts.115.gate_proj", + "model.layers.47.mlp.experts.115.up_proj", + "model.layers.47.mlp.experts.116.down_proj", + "model.layers.47.mlp.experts.116.gate_proj", + "model.layers.47.mlp.experts.116.up_proj", + "model.layers.47.mlp.experts.117.down_proj", + "model.layers.47.mlp.experts.117.gate_proj", + "model.layers.47.mlp.experts.117.up_proj", + "model.layers.47.mlp.experts.118.down_proj", + "model.layers.47.mlp.experts.118.gate_proj", + "model.layers.47.mlp.experts.118.up_proj", + "model.layers.47.mlp.experts.119.down_proj", + "model.layers.47.mlp.experts.119.gate_proj", + "model.layers.47.mlp.experts.119.up_proj", + "model.layers.47.mlp.experts.12.down_proj", + "model.layers.47.mlp.experts.12.gate_proj", + "model.layers.47.mlp.experts.12.up_proj", + "model.layers.47.mlp.experts.120.down_proj", + "model.layers.47.mlp.experts.120.gate_proj", + "model.layers.47.mlp.experts.120.up_proj", + "model.layers.47.mlp.experts.121.down_proj", + "model.layers.47.mlp.experts.121.gate_proj", + "model.layers.47.mlp.experts.121.up_proj", + "model.layers.47.mlp.experts.122.down_proj", + "model.layers.47.mlp.experts.122.gate_proj", + "model.layers.47.mlp.experts.122.up_proj", + "model.layers.47.mlp.experts.123.down_proj", + "model.layers.47.mlp.experts.123.gate_proj", + "model.layers.47.mlp.experts.123.up_proj", + "model.layers.47.mlp.experts.124.down_proj", + "model.layers.47.mlp.experts.124.gate_proj", + "model.layers.47.mlp.experts.124.up_proj", + "model.layers.47.mlp.experts.125.down_proj", + "model.layers.47.mlp.experts.125.gate_proj", + "model.layers.47.mlp.experts.125.up_proj", + "model.layers.47.mlp.experts.126.down_proj", + "model.layers.47.mlp.experts.126.gate_proj", + "model.layers.47.mlp.experts.126.up_proj", + "model.layers.47.mlp.experts.127.down_proj", + "model.layers.47.mlp.experts.127.gate_proj", + "model.layers.47.mlp.experts.127.up_proj", + "model.layers.47.mlp.experts.128.down_proj", + "model.layers.47.mlp.experts.128.gate_proj", + "model.layers.47.mlp.experts.128.up_proj", + "model.layers.47.mlp.experts.129.down_proj", + "model.layers.47.mlp.experts.129.gate_proj", + "model.layers.47.mlp.experts.129.up_proj", + "model.layers.47.mlp.experts.13.down_proj", + "model.layers.47.mlp.experts.13.gate_proj", + "model.layers.47.mlp.experts.13.up_proj", + "model.layers.47.mlp.experts.130.down_proj", + "model.layers.47.mlp.experts.130.gate_proj", + "model.layers.47.mlp.experts.130.up_proj", + "model.layers.47.mlp.experts.131.down_proj", + "model.layers.47.mlp.experts.131.gate_proj", + "model.layers.47.mlp.experts.131.up_proj", + "model.layers.47.mlp.experts.132.down_proj", + "model.layers.47.mlp.experts.132.gate_proj", + "model.layers.47.mlp.experts.132.up_proj", + "model.layers.47.mlp.experts.133.down_proj", + "model.layers.47.mlp.experts.133.gate_proj", + "model.layers.47.mlp.experts.133.up_proj", + "model.layers.47.mlp.experts.134.down_proj", + "model.layers.47.mlp.experts.134.gate_proj", + "model.layers.47.mlp.experts.134.up_proj", + "model.layers.47.mlp.experts.135.down_proj", + "model.layers.47.mlp.experts.135.gate_proj", + "model.layers.47.mlp.experts.135.up_proj", + "model.layers.47.mlp.experts.136.down_proj", + "model.layers.47.mlp.experts.136.gate_proj", + "model.layers.47.mlp.experts.136.up_proj", + "model.layers.47.mlp.experts.137.down_proj", + "model.layers.47.mlp.experts.137.gate_proj", + "model.layers.47.mlp.experts.137.up_proj", + "model.layers.47.mlp.experts.138.down_proj", + "model.layers.47.mlp.experts.138.gate_proj", + "model.layers.47.mlp.experts.138.up_proj", + "model.layers.47.mlp.experts.139.down_proj", + "model.layers.47.mlp.experts.139.gate_proj", + "model.layers.47.mlp.experts.139.up_proj", + "model.layers.47.mlp.experts.14.down_proj", + "model.layers.47.mlp.experts.14.gate_proj", + "model.layers.47.mlp.experts.14.up_proj", + "model.layers.47.mlp.experts.140.down_proj", + "model.layers.47.mlp.experts.140.gate_proj", + "model.layers.47.mlp.experts.140.up_proj", + "model.layers.47.mlp.experts.141.down_proj", + "model.layers.47.mlp.experts.141.gate_proj", + "model.layers.47.mlp.experts.141.up_proj", + "model.layers.47.mlp.experts.142.down_proj", + "model.layers.47.mlp.experts.142.gate_proj", + "model.layers.47.mlp.experts.142.up_proj", + "model.layers.47.mlp.experts.143.down_proj", + "model.layers.47.mlp.experts.143.gate_proj", + "model.layers.47.mlp.experts.143.up_proj", + "model.layers.47.mlp.experts.144.down_proj", + "model.layers.47.mlp.experts.144.gate_proj", + "model.layers.47.mlp.experts.144.up_proj", + "model.layers.47.mlp.experts.145.down_proj", + "model.layers.47.mlp.experts.145.gate_proj", + "model.layers.47.mlp.experts.145.up_proj", + "model.layers.47.mlp.experts.146.down_proj", + "model.layers.47.mlp.experts.146.gate_proj", + "model.layers.47.mlp.experts.146.up_proj", + "model.layers.47.mlp.experts.147.down_proj", + "model.layers.47.mlp.experts.147.gate_proj", + "model.layers.47.mlp.experts.147.up_proj", + "model.layers.47.mlp.experts.148.down_proj", + "model.layers.47.mlp.experts.148.gate_proj", + "model.layers.47.mlp.experts.148.up_proj", + "model.layers.47.mlp.experts.149.down_proj", + "model.layers.47.mlp.experts.149.gate_proj", + "model.layers.47.mlp.experts.149.up_proj", + "model.layers.47.mlp.experts.15.down_proj", + "model.layers.47.mlp.experts.15.gate_proj", + "model.layers.47.mlp.experts.15.up_proj", + "model.layers.47.mlp.experts.150.down_proj", + "model.layers.47.mlp.experts.150.gate_proj", + "model.layers.47.mlp.experts.150.up_proj", + "model.layers.47.mlp.experts.151.down_proj", + "model.layers.47.mlp.experts.151.gate_proj", + "model.layers.47.mlp.experts.151.up_proj", + "model.layers.47.mlp.experts.152.down_proj", + "model.layers.47.mlp.experts.152.gate_proj", + "model.layers.47.mlp.experts.152.up_proj", + "model.layers.47.mlp.experts.153.down_proj", + "model.layers.47.mlp.experts.153.gate_proj", + "model.layers.47.mlp.experts.153.up_proj", + "model.layers.47.mlp.experts.154.down_proj", + "model.layers.47.mlp.experts.154.gate_proj", + "model.layers.47.mlp.experts.154.up_proj", + "model.layers.47.mlp.experts.155.down_proj", + "model.layers.47.mlp.experts.155.gate_proj", + "model.layers.47.mlp.experts.155.up_proj", + "model.layers.47.mlp.experts.156.down_proj", + "model.layers.47.mlp.experts.156.gate_proj", + "model.layers.47.mlp.experts.156.up_proj", + "model.layers.47.mlp.experts.157.down_proj", + "model.layers.47.mlp.experts.157.gate_proj", + "model.layers.47.mlp.experts.157.up_proj", + "model.layers.47.mlp.experts.158.down_proj", + "model.layers.47.mlp.experts.158.gate_proj", + "model.layers.47.mlp.experts.158.up_proj", + "model.layers.47.mlp.experts.159.down_proj", + "model.layers.47.mlp.experts.159.gate_proj", + "model.layers.47.mlp.experts.159.up_proj", + "model.layers.47.mlp.experts.16.down_proj", + "model.layers.47.mlp.experts.16.gate_proj", + "model.layers.47.mlp.experts.16.up_proj", + "model.layers.47.mlp.experts.160.down_proj", + "model.layers.47.mlp.experts.160.gate_proj", + "model.layers.47.mlp.experts.160.up_proj", + "model.layers.47.mlp.experts.161.down_proj", + "model.layers.47.mlp.experts.161.gate_proj", + "model.layers.47.mlp.experts.161.up_proj", + "model.layers.47.mlp.experts.162.down_proj", + "model.layers.47.mlp.experts.162.gate_proj", + "model.layers.47.mlp.experts.162.up_proj", + "model.layers.47.mlp.experts.163.down_proj", + "model.layers.47.mlp.experts.163.gate_proj", + "model.layers.47.mlp.experts.163.up_proj", + "model.layers.47.mlp.experts.164.down_proj", + "model.layers.47.mlp.experts.164.gate_proj", + "model.layers.47.mlp.experts.164.up_proj", + "model.layers.47.mlp.experts.165.down_proj", + "model.layers.47.mlp.experts.165.gate_proj", + "model.layers.47.mlp.experts.165.up_proj", + "model.layers.47.mlp.experts.166.down_proj", + "model.layers.47.mlp.experts.166.gate_proj", + "model.layers.47.mlp.experts.166.up_proj", + "model.layers.47.mlp.experts.167.down_proj", + "model.layers.47.mlp.experts.167.gate_proj", + "model.layers.47.mlp.experts.167.up_proj", + "model.layers.47.mlp.experts.168.down_proj", + "model.layers.47.mlp.experts.168.gate_proj", + "model.layers.47.mlp.experts.168.up_proj", + "model.layers.47.mlp.experts.169.down_proj", + "model.layers.47.mlp.experts.169.gate_proj", + "model.layers.47.mlp.experts.169.up_proj", + "model.layers.47.mlp.experts.17.down_proj", + "model.layers.47.mlp.experts.17.gate_proj", + "model.layers.47.mlp.experts.17.up_proj", + "model.layers.47.mlp.experts.170.down_proj", + "model.layers.47.mlp.experts.170.gate_proj", + "model.layers.47.mlp.experts.170.up_proj", + "model.layers.47.mlp.experts.171.down_proj", + "model.layers.47.mlp.experts.171.gate_proj", + "model.layers.47.mlp.experts.171.up_proj", + "model.layers.47.mlp.experts.172.down_proj", + "model.layers.47.mlp.experts.172.gate_proj", + "model.layers.47.mlp.experts.172.up_proj", + "model.layers.47.mlp.experts.173.down_proj", + "model.layers.47.mlp.experts.173.gate_proj", + "model.layers.47.mlp.experts.173.up_proj", + "model.layers.47.mlp.experts.174.down_proj", + "model.layers.47.mlp.experts.174.gate_proj", + "model.layers.47.mlp.experts.174.up_proj", + "model.layers.47.mlp.experts.175.down_proj", + "model.layers.47.mlp.experts.175.gate_proj", + "model.layers.47.mlp.experts.175.up_proj", + "model.layers.47.mlp.experts.176.down_proj", + "model.layers.47.mlp.experts.176.gate_proj", + "model.layers.47.mlp.experts.176.up_proj", + "model.layers.47.mlp.experts.177.down_proj", + "model.layers.47.mlp.experts.177.gate_proj", + "model.layers.47.mlp.experts.177.up_proj", + "model.layers.47.mlp.experts.178.down_proj", + "model.layers.47.mlp.experts.178.gate_proj", + "model.layers.47.mlp.experts.178.up_proj", + "model.layers.47.mlp.experts.179.down_proj", + "model.layers.47.mlp.experts.179.gate_proj", + "model.layers.47.mlp.experts.179.up_proj", + "model.layers.47.mlp.experts.18.down_proj", + "model.layers.47.mlp.experts.18.gate_proj", + "model.layers.47.mlp.experts.18.up_proj", + "model.layers.47.mlp.experts.180.down_proj", + "model.layers.47.mlp.experts.180.gate_proj", + "model.layers.47.mlp.experts.180.up_proj", + "model.layers.47.mlp.experts.181.down_proj", + "model.layers.47.mlp.experts.181.gate_proj", + "model.layers.47.mlp.experts.181.up_proj", + "model.layers.47.mlp.experts.182.down_proj", + "model.layers.47.mlp.experts.182.gate_proj", + "model.layers.47.mlp.experts.182.up_proj", + "model.layers.47.mlp.experts.183.down_proj", + "model.layers.47.mlp.experts.183.gate_proj", + "model.layers.47.mlp.experts.183.up_proj", + "model.layers.47.mlp.experts.184.down_proj", + "model.layers.47.mlp.experts.184.gate_proj", + "model.layers.47.mlp.experts.184.up_proj", + "model.layers.47.mlp.experts.185.down_proj", + "model.layers.47.mlp.experts.185.gate_proj", + "model.layers.47.mlp.experts.185.up_proj", + "model.layers.47.mlp.experts.186.down_proj", + "model.layers.47.mlp.experts.186.gate_proj", + "model.layers.47.mlp.experts.186.up_proj", + "model.layers.47.mlp.experts.187.down_proj", + "model.layers.47.mlp.experts.187.gate_proj", + "model.layers.47.mlp.experts.187.up_proj", + "model.layers.47.mlp.experts.188.down_proj", + "model.layers.47.mlp.experts.188.gate_proj", + "model.layers.47.mlp.experts.188.up_proj", + "model.layers.47.mlp.experts.189.down_proj", + "model.layers.47.mlp.experts.189.gate_proj", + "model.layers.47.mlp.experts.189.up_proj", + "model.layers.47.mlp.experts.19.down_proj", + "model.layers.47.mlp.experts.19.gate_proj", + "model.layers.47.mlp.experts.19.up_proj", + "model.layers.47.mlp.experts.190.down_proj", + "model.layers.47.mlp.experts.190.gate_proj", + "model.layers.47.mlp.experts.190.up_proj", + "model.layers.47.mlp.experts.191.down_proj", + "model.layers.47.mlp.experts.191.gate_proj", + "model.layers.47.mlp.experts.191.up_proj", + "model.layers.47.mlp.experts.192.down_proj", + "model.layers.47.mlp.experts.192.gate_proj", + "model.layers.47.mlp.experts.192.up_proj", + "model.layers.47.mlp.experts.193.down_proj", + "model.layers.47.mlp.experts.193.gate_proj", + "model.layers.47.mlp.experts.193.up_proj", + "model.layers.47.mlp.experts.194.down_proj", + "model.layers.47.mlp.experts.194.gate_proj", + "model.layers.47.mlp.experts.194.up_proj", + "model.layers.47.mlp.experts.195.down_proj", + "model.layers.47.mlp.experts.195.gate_proj", + "model.layers.47.mlp.experts.195.up_proj", + "model.layers.47.mlp.experts.196.down_proj", + "model.layers.47.mlp.experts.196.gate_proj", + "model.layers.47.mlp.experts.196.up_proj", + "model.layers.47.mlp.experts.197.down_proj", + "model.layers.47.mlp.experts.197.gate_proj", + "model.layers.47.mlp.experts.197.up_proj", + "model.layers.47.mlp.experts.198.down_proj", + "model.layers.47.mlp.experts.198.gate_proj", + "model.layers.47.mlp.experts.198.up_proj", + "model.layers.47.mlp.experts.199.down_proj", + "model.layers.47.mlp.experts.199.gate_proj", + "model.layers.47.mlp.experts.199.up_proj", + "model.layers.47.mlp.experts.2.down_proj", + "model.layers.47.mlp.experts.2.gate_proj", + "model.layers.47.mlp.experts.2.up_proj", + "model.layers.47.mlp.experts.20.down_proj", + "model.layers.47.mlp.experts.20.gate_proj", + "model.layers.47.mlp.experts.20.up_proj", + "model.layers.47.mlp.experts.200.down_proj", + "model.layers.47.mlp.experts.200.gate_proj", + "model.layers.47.mlp.experts.200.up_proj", + "model.layers.47.mlp.experts.201.down_proj", + "model.layers.47.mlp.experts.201.gate_proj", + "model.layers.47.mlp.experts.201.up_proj", + "model.layers.47.mlp.experts.202.down_proj", + "model.layers.47.mlp.experts.202.gate_proj", + "model.layers.47.mlp.experts.202.up_proj", + "model.layers.47.mlp.experts.203.down_proj", + "model.layers.47.mlp.experts.203.gate_proj", + "model.layers.47.mlp.experts.203.up_proj", + "model.layers.47.mlp.experts.204.down_proj", + "model.layers.47.mlp.experts.204.gate_proj", + "model.layers.47.mlp.experts.204.up_proj", + "model.layers.47.mlp.experts.205.down_proj", + "model.layers.47.mlp.experts.205.gate_proj", + "model.layers.47.mlp.experts.205.up_proj", + "model.layers.47.mlp.experts.206.down_proj", + "model.layers.47.mlp.experts.206.gate_proj", + "model.layers.47.mlp.experts.206.up_proj", + "model.layers.47.mlp.experts.207.down_proj", + "model.layers.47.mlp.experts.207.gate_proj", + "model.layers.47.mlp.experts.207.up_proj", + "model.layers.47.mlp.experts.208.down_proj", + "model.layers.47.mlp.experts.208.gate_proj", + "model.layers.47.mlp.experts.208.up_proj", + "model.layers.47.mlp.experts.209.down_proj", + "model.layers.47.mlp.experts.209.gate_proj", + "model.layers.47.mlp.experts.209.up_proj", + "model.layers.47.mlp.experts.21.down_proj", + "model.layers.47.mlp.experts.21.gate_proj", + "model.layers.47.mlp.experts.21.up_proj", + "model.layers.47.mlp.experts.210.down_proj", + "model.layers.47.mlp.experts.210.gate_proj", + "model.layers.47.mlp.experts.210.up_proj", + "model.layers.47.mlp.experts.211.down_proj", + "model.layers.47.mlp.experts.211.gate_proj", + "model.layers.47.mlp.experts.211.up_proj", + "model.layers.47.mlp.experts.212.down_proj", + "model.layers.47.mlp.experts.212.gate_proj", + "model.layers.47.mlp.experts.212.up_proj", + "model.layers.47.mlp.experts.213.down_proj", + "model.layers.47.mlp.experts.213.gate_proj", + "model.layers.47.mlp.experts.213.up_proj", + "model.layers.47.mlp.experts.214.down_proj", + "model.layers.47.mlp.experts.214.gate_proj", + "model.layers.47.mlp.experts.214.up_proj", + "model.layers.47.mlp.experts.215.down_proj", + "model.layers.47.mlp.experts.215.gate_proj", + "model.layers.47.mlp.experts.215.up_proj", + "model.layers.47.mlp.experts.216.down_proj", + "model.layers.47.mlp.experts.216.gate_proj", + "model.layers.47.mlp.experts.216.up_proj", + "model.layers.47.mlp.experts.217.down_proj", + "model.layers.47.mlp.experts.217.gate_proj", + "model.layers.47.mlp.experts.217.up_proj", + "model.layers.47.mlp.experts.218.down_proj", + "model.layers.47.mlp.experts.218.gate_proj", + "model.layers.47.mlp.experts.218.up_proj", + "model.layers.47.mlp.experts.219.down_proj", + "model.layers.47.mlp.experts.219.gate_proj", + "model.layers.47.mlp.experts.219.up_proj", + "model.layers.47.mlp.experts.22.down_proj", + "model.layers.47.mlp.experts.22.gate_proj", + "model.layers.47.mlp.experts.22.up_proj", + "model.layers.47.mlp.experts.220.down_proj", + "model.layers.47.mlp.experts.220.gate_proj", + "model.layers.47.mlp.experts.220.up_proj", + "model.layers.47.mlp.experts.221.down_proj", + "model.layers.47.mlp.experts.221.gate_proj", + "model.layers.47.mlp.experts.221.up_proj", + "model.layers.47.mlp.experts.222.down_proj", + "model.layers.47.mlp.experts.222.gate_proj", + "model.layers.47.mlp.experts.222.up_proj", + "model.layers.47.mlp.experts.223.down_proj", + "model.layers.47.mlp.experts.223.gate_proj", + "model.layers.47.mlp.experts.223.up_proj", + "model.layers.47.mlp.experts.224.down_proj", + "model.layers.47.mlp.experts.224.gate_proj", + "model.layers.47.mlp.experts.224.up_proj", + "model.layers.47.mlp.experts.225.down_proj", + "model.layers.47.mlp.experts.225.gate_proj", + "model.layers.47.mlp.experts.225.up_proj", + "model.layers.47.mlp.experts.226.down_proj", + "model.layers.47.mlp.experts.226.gate_proj", + "model.layers.47.mlp.experts.226.up_proj", + "model.layers.47.mlp.experts.227.down_proj", + "model.layers.47.mlp.experts.227.gate_proj", + "model.layers.47.mlp.experts.227.up_proj", + "model.layers.47.mlp.experts.228.down_proj", + "model.layers.47.mlp.experts.228.gate_proj", + "model.layers.47.mlp.experts.228.up_proj", + "model.layers.47.mlp.experts.229.down_proj", + "model.layers.47.mlp.experts.229.gate_proj", + "model.layers.47.mlp.experts.229.up_proj", + "model.layers.47.mlp.experts.23.down_proj", + "model.layers.47.mlp.experts.23.gate_proj", + "model.layers.47.mlp.experts.23.up_proj", + "model.layers.47.mlp.experts.230.down_proj", + "model.layers.47.mlp.experts.230.gate_proj", + "model.layers.47.mlp.experts.230.up_proj", + "model.layers.47.mlp.experts.231.down_proj", + "model.layers.47.mlp.experts.231.gate_proj", + "model.layers.47.mlp.experts.231.up_proj", + "model.layers.47.mlp.experts.232.down_proj", + "model.layers.47.mlp.experts.232.gate_proj", + "model.layers.47.mlp.experts.232.up_proj", + "model.layers.47.mlp.experts.233.down_proj", + "model.layers.47.mlp.experts.233.gate_proj", + "model.layers.47.mlp.experts.233.up_proj", + "model.layers.47.mlp.experts.234.down_proj", + "model.layers.47.mlp.experts.234.gate_proj", + "model.layers.47.mlp.experts.234.up_proj", + "model.layers.47.mlp.experts.235.down_proj", + "model.layers.47.mlp.experts.235.gate_proj", + "model.layers.47.mlp.experts.235.up_proj", + "model.layers.47.mlp.experts.236.down_proj", + "model.layers.47.mlp.experts.236.gate_proj", + "model.layers.47.mlp.experts.236.up_proj", + "model.layers.47.mlp.experts.237.down_proj", + "model.layers.47.mlp.experts.237.gate_proj", + "model.layers.47.mlp.experts.237.up_proj", + "model.layers.47.mlp.experts.238.down_proj", + "model.layers.47.mlp.experts.238.gate_proj", + "model.layers.47.mlp.experts.238.up_proj", + "model.layers.47.mlp.experts.239.down_proj", + "model.layers.47.mlp.experts.239.gate_proj", + "model.layers.47.mlp.experts.239.up_proj", + "model.layers.47.mlp.experts.24.down_proj", + "model.layers.47.mlp.experts.24.gate_proj", + "model.layers.47.mlp.experts.24.up_proj", + "model.layers.47.mlp.experts.240.down_proj", + "model.layers.47.mlp.experts.240.gate_proj", + "model.layers.47.mlp.experts.240.up_proj", + "model.layers.47.mlp.experts.241.down_proj", + "model.layers.47.mlp.experts.241.gate_proj", + "model.layers.47.mlp.experts.241.up_proj", + "model.layers.47.mlp.experts.242.down_proj", + "model.layers.47.mlp.experts.242.gate_proj", + "model.layers.47.mlp.experts.242.up_proj", + "model.layers.47.mlp.experts.243.down_proj", + "model.layers.47.mlp.experts.243.gate_proj", + "model.layers.47.mlp.experts.243.up_proj", + "model.layers.47.mlp.experts.244.down_proj", + "model.layers.47.mlp.experts.244.gate_proj", + "model.layers.47.mlp.experts.244.up_proj", + "model.layers.47.mlp.experts.245.down_proj", + "model.layers.47.mlp.experts.245.gate_proj", + "model.layers.47.mlp.experts.245.up_proj", + "model.layers.47.mlp.experts.246.down_proj", + "model.layers.47.mlp.experts.246.gate_proj", + "model.layers.47.mlp.experts.246.up_proj", + "model.layers.47.mlp.experts.247.down_proj", + "model.layers.47.mlp.experts.247.gate_proj", + "model.layers.47.mlp.experts.247.up_proj", + "model.layers.47.mlp.experts.248.down_proj", + "model.layers.47.mlp.experts.248.gate_proj", + "model.layers.47.mlp.experts.248.up_proj", + "model.layers.47.mlp.experts.249.down_proj", + "model.layers.47.mlp.experts.249.gate_proj", + "model.layers.47.mlp.experts.249.up_proj", + "model.layers.47.mlp.experts.25.down_proj", + "model.layers.47.mlp.experts.25.gate_proj", + "model.layers.47.mlp.experts.25.up_proj", + "model.layers.47.mlp.experts.250.down_proj", + "model.layers.47.mlp.experts.250.gate_proj", + "model.layers.47.mlp.experts.250.up_proj", + "model.layers.47.mlp.experts.251.down_proj", + "model.layers.47.mlp.experts.251.gate_proj", + "model.layers.47.mlp.experts.251.up_proj", + "model.layers.47.mlp.experts.252.down_proj", + "model.layers.47.mlp.experts.252.gate_proj", + "model.layers.47.mlp.experts.252.up_proj", + "model.layers.47.mlp.experts.253.down_proj", + "model.layers.47.mlp.experts.253.gate_proj", + "model.layers.47.mlp.experts.253.up_proj", + "model.layers.47.mlp.experts.254.down_proj", + "model.layers.47.mlp.experts.254.gate_proj", + "model.layers.47.mlp.experts.254.up_proj", + "model.layers.47.mlp.experts.255.down_proj", + "model.layers.47.mlp.experts.255.gate_proj", + "model.layers.47.mlp.experts.255.up_proj", + "model.layers.47.mlp.experts.26.down_proj", + "model.layers.47.mlp.experts.26.gate_proj", + "model.layers.47.mlp.experts.26.up_proj", + "model.layers.47.mlp.experts.27.down_proj", + "model.layers.47.mlp.experts.27.gate_proj", + "model.layers.47.mlp.experts.27.up_proj", + "model.layers.47.mlp.experts.28.down_proj", + "model.layers.47.mlp.experts.28.gate_proj", + "model.layers.47.mlp.experts.28.up_proj", + "model.layers.47.mlp.experts.29.down_proj", + "model.layers.47.mlp.experts.29.gate_proj", + "model.layers.47.mlp.experts.29.up_proj", + "model.layers.47.mlp.experts.3.down_proj", + "model.layers.47.mlp.experts.3.gate_proj", + "model.layers.47.mlp.experts.3.up_proj", + "model.layers.47.mlp.experts.30.down_proj", + "model.layers.47.mlp.experts.30.gate_proj", + "model.layers.47.mlp.experts.30.up_proj", + "model.layers.47.mlp.experts.31.down_proj", + "model.layers.47.mlp.experts.31.gate_proj", + "model.layers.47.mlp.experts.31.up_proj", + "model.layers.47.mlp.experts.32.down_proj", + "model.layers.47.mlp.experts.32.gate_proj", + "model.layers.47.mlp.experts.32.up_proj", + "model.layers.47.mlp.experts.33.down_proj", + "model.layers.47.mlp.experts.33.gate_proj", + "model.layers.47.mlp.experts.33.up_proj", + "model.layers.47.mlp.experts.34.down_proj", + "model.layers.47.mlp.experts.34.gate_proj", + "model.layers.47.mlp.experts.34.up_proj", + "model.layers.47.mlp.experts.35.down_proj", + "model.layers.47.mlp.experts.35.gate_proj", + "model.layers.47.mlp.experts.35.up_proj", + "model.layers.47.mlp.experts.36.down_proj", + "model.layers.47.mlp.experts.36.gate_proj", + "model.layers.47.mlp.experts.36.up_proj", + "model.layers.47.mlp.experts.37.down_proj", + "model.layers.47.mlp.experts.37.gate_proj", + "model.layers.47.mlp.experts.37.up_proj", + "model.layers.47.mlp.experts.38.down_proj", + "model.layers.47.mlp.experts.38.gate_proj", + "model.layers.47.mlp.experts.38.up_proj", + "model.layers.47.mlp.experts.39.down_proj", + "model.layers.47.mlp.experts.39.gate_proj", + "model.layers.47.mlp.experts.39.up_proj", + "model.layers.47.mlp.experts.4.down_proj", + "model.layers.47.mlp.experts.4.gate_proj", + "model.layers.47.mlp.experts.4.up_proj", + "model.layers.47.mlp.experts.40.down_proj", + "model.layers.47.mlp.experts.40.gate_proj", + "model.layers.47.mlp.experts.40.up_proj", + "model.layers.47.mlp.experts.41.down_proj", + "model.layers.47.mlp.experts.41.gate_proj", + "model.layers.47.mlp.experts.41.up_proj", + "model.layers.47.mlp.experts.42.down_proj", + "model.layers.47.mlp.experts.42.gate_proj", + "model.layers.47.mlp.experts.42.up_proj", + "model.layers.47.mlp.experts.43.down_proj", + "model.layers.47.mlp.experts.43.gate_proj", + "model.layers.47.mlp.experts.43.up_proj", + "model.layers.47.mlp.experts.44.down_proj", + "model.layers.47.mlp.experts.44.gate_proj", + "model.layers.47.mlp.experts.44.up_proj", + "model.layers.47.mlp.experts.45.down_proj", + "model.layers.47.mlp.experts.45.gate_proj", + "model.layers.47.mlp.experts.45.up_proj", + "model.layers.47.mlp.experts.46.down_proj", + "model.layers.47.mlp.experts.46.gate_proj", + "model.layers.47.mlp.experts.46.up_proj", + "model.layers.47.mlp.experts.47.down_proj", + "model.layers.47.mlp.experts.47.gate_proj", + "model.layers.47.mlp.experts.47.up_proj", + "model.layers.47.mlp.experts.48.down_proj", + "model.layers.47.mlp.experts.48.gate_proj", + "model.layers.47.mlp.experts.48.up_proj", + "model.layers.47.mlp.experts.49.down_proj", + "model.layers.47.mlp.experts.49.gate_proj", + "model.layers.47.mlp.experts.49.up_proj", + "model.layers.47.mlp.experts.5.down_proj", + "model.layers.47.mlp.experts.5.gate_proj", + "model.layers.47.mlp.experts.5.up_proj", + "model.layers.47.mlp.experts.50.down_proj", + "model.layers.47.mlp.experts.50.gate_proj", + "model.layers.47.mlp.experts.50.up_proj", + "model.layers.47.mlp.experts.51.down_proj", + "model.layers.47.mlp.experts.51.gate_proj", + "model.layers.47.mlp.experts.51.up_proj", + "model.layers.47.mlp.experts.52.down_proj", + "model.layers.47.mlp.experts.52.gate_proj", + "model.layers.47.mlp.experts.52.up_proj", + "model.layers.47.mlp.experts.53.down_proj", + "model.layers.47.mlp.experts.53.gate_proj", + "model.layers.47.mlp.experts.53.up_proj", + "model.layers.47.mlp.experts.54.down_proj", + "model.layers.47.mlp.experts.54.gate_proj", + "model.layers.47.mlp.experts.54.up_proj", + "model.layers.47.mlp.experts.55.down_proj", + "model.layers.47.mlp.experts.55.gate_proj", + "model.layers.47.mlp.experts.55.up_proj", + "model.layers.47.mlp.experts.56.down_proj", + "model.layers.47.mlp.experts.56.gate_proj", + "model.layers.47.mlp.experts.56.up_proj", + "model.layers.47.mlp.experts.57.down_proj", + "model.layers.47.mlp.experts.57.gate_proj", + "model.layers.47.mlp.experts.57.up_proj", + "model.layers.47.mlp.experts.58.down_proj", + "model.layers.47.mlp.experts.58.gate_proj", + "model.layers.47.mlp.experts.58.up_proj", + "model.layers.47.mlp.experts.59.down_proj", + "model.layers.47.mlp.experts.59.gate_proj", + "model.layers.47.mlp.experts.59.up_proj", + "model.layers.47.mlp.experts.6.down_proj", + "model.layers.47.mlp.experts.6.gate_proj", + "model.layers.47.mlp.experts.6.up_proj", + "model.layers.47.mlp.experts.60.down_proj", + "model.layers.47.mlp.experts.60.gate_proj", + "model.layers.47.mlp.experts.60.up_proj", + "model.layers.47.mlp.experts.61.down_proj", + "model.layers.47.mlp.experts.61.gate_proj", + "model.layers.47.mlp.experts.61.up_proj", + "model.layers.47.mlp.experts.62.down_proj", + "model.layers.47.mlp.experts.62.gate_proj", + "model.layers.47.mlp.experts.62.up_proj", + "model.layers.47.mlp.experts.63.down_proj", + "model.layers.47.mlp.experts.63.gate_proj", + "model.layers.47.mlp.experts.63.up_proj", + "model.layers.47.mlp.experts.64.down_proj", + "model.layers.47.mlp.experts.64.gate_proj", + "model.layers.47.mlp.experts.64.up_proj", + "model.layers.47.mlp.experts.65.down_proj", + "model.layers.47.mlp.experts.65.gate_proj", + "model.layers.47.mlp.experts.65.up_proj", + "model.layers.47.mlp.experts.66.down_proj", + "model.layers.47.mlp.experts.66.gate_proj", + "model.layers.47.mlp.experts.66.up_proj", + "model.layers.47.mlp.experts.67.down_proj", + "model.layers.47.mlp.experts.67.gate_proj", + "model.layers.47.mlp.experts.67.up_proj", + "model.layers.47.mlp.experts.68.down_proj", + "model.layers.47.mlp.experts.68.gate_proj", + "model.layers.47.mlp.experts.68.up_proj", + "model.layers.47.mlp.experts.69.down_proj", + "model.layers.47.mlp.experts.69.gate_proj", + "model.layers.47.mlp.experts.69.up_proj", + "model.layers.47.mlp.experts.7.down_proj", + "model.layers.47.mlp.experts.7.gate_proj", + "model.layers.47.mlp.experts.7.up_proj", + "model.layers.47.mlp.experts.70.down_proj", + "model.layers.47.mlp.experts.70.gate_proj", + "model.layers.47.mlp.experts.70.up_proj", + "model.layers.47.mlp.experts.71.down_proj", + "model.layers.47.mlp.experts.71.gate_proj", + "model.layers.47.mlp.experts.71.up_proj", + "model.layers.47.mlp.experts.72.down_proj", + "model.layers.47.mlp.experts.72.gate_proj", + "model.layers.47.mlp.experts.72.up_proj", + "model.layers.47.mlp.experts.73.down_proj", + "model.layers.47.mlp.experts.73.gate_proj", + "model.layers.47.mlp.experts.73.up_proj", + "model.layers.47.mlp.experts.74.down_proj", + "model.layers.47.mlp.experts.74.gate_proj", + "model.layers.47.mlp.experts.74.up_proj", + "model.layers.47.mlp.experts.75.down_proj", + "model.layers.47.mlp.experts.75.gate_proj", + "model.layers.47.mlp.experts.75.up_proj", + "model.layers.47.mlp.experts.76.down_proj", + "model.layers.47.mlp.experts.76.gate_proj", + "model.layers.47.mlp.experts.76.up_proj", + "model.layers.47.mlp.experts.77.down_proj", + "model.layers.47.mlp.experts.77.gate_proj", + "model.layers.47.mlp.experts.77.up_proj", + "model.layers.47.mlp.experts.78.down_proj", + "model.layers.47.mlp.experts.78.gate_proj", + "model.layers.47.mlp.experts.78.up_proj", + "model.layers.47.mlp.experts.79.down_proj", + "model.layers.47.mlp.experts.79.gate_proj", + "model.layers.47.mlp.experts.79.up_proj", + "model.layers.47.mlp.experts.8.down_proj", + "model.layers.47.mlp.experts.8.gate_proj", + "model.layers.47.mlp.experts.8.up_proj", + "model.layers.47.mlp.experts.80.down_proj", + "model.layers.47.mlp.experts.80.gate_proj", + "model.layers.47.mlp.experts.80.up_proj", + "model.layers.47.mlp.experts.81.down_proj", + "model.layers.47.mlp.experts.81.gate_proj", + "model.layers.47.mlp.experts.81.up_proj", + "model.layers.47.mlp.experts.82.down_proj", + "model.layers.47.mlp.experts.82.gate_proj", + "model.layers.47.mlp.experts.82.up_proj", + "model.layers.47.mlp.experts.83.down_proj", + "model.layers.47.mlp.experts.83.gate_proj", + "model.layers.47.mlp.experts.83.up_proj", + "model.layers.47.mlp.experts.84.down_proj", + "model.layers.47.mlp.experts.84.gate_proj", + "model.layers.47.mlp.experts.84.up_proj", + "model.layers.47.mlp.experts.85.down_proj", + "model.layers.47.mlp.experts.85.gate_proj", + "model.layers.47.mlp.experts.85.up_proj", + "model.layers.47.mlp.experts.86.down_proj", + "model.layers.47.mlp.experts.86.gate_proj", + "model.layers.47.mlp.experts.86.up_proj", + "model.layers.47.mlp.experts.87.down_proj", + "model.layers.47.mlp.experts.87.gate_proj", + "model.layers.47.mlp.experts.87.up_proj", + "model.layers.47.mlp.experts.88.down_proj", + "model.layers.47.mlp.experts.88.gate_proj", + "model.layers.47.mlp.experts.88.up_proj", + "model.layers.47.mlp.experts.89.down_proj", + "model.layers.47.mlp.experts.89.gate_proj", + "model.layers.47.mlp.experts.89.up_proj", + "model.layers.47.mlp.experts.9.down_proj", + "model.layers.47.mlp.experts.9.gate_proj", + "model.layers.47.mlp.experts.9.up_proj", + "model.layers.47.mlp.experts.90.down_proj", + "model.layers.47.mlp.experts.90.gate_proj", + "model.layers.47.mlp.experts.90.up_proj", + "model.layers.47.mlp.experts.91.down_proj", + "model.layers.47.mlp.experts.91.gate_proj", + "model.layers.47.mlp.experts.91.up_proj", + "model.layers.47.mlp.experts.92.down_proj", + "model.layers.47.mlp.experts.92.gate_proj", + "model.layers.47.mlp.experts.92.up_proj", + "model.layers.47.mlp.experts.93.down_proj", + "model.layers.47.mlp.experts.93.gate_proj", + "model.layers.47.mlp.experts.93.up_proj", + "model.layers.47.mlp.experts.94.down_proj", + "model.layers.47.mlp.experts.94.gate_proj", + "model.layers.47.mlp.experts.94.up_proj", + "model.layers.47.mlp.experts.95.down_proj", + "model.layers.47.mlp.experts.95.gate_proj", + "model.layers.47.mlp.experts.95.up_proj", + "model.layers.47.mlp.experts.96.down_proj", + "model.layers.47.mlp.experts.96.gate_proj", + "model.layers.47.mlp.experts.96.up_proj", + "model.layers.47.mlp.experts.97.down_proj", + "model.layers.47.mlp.experts.97.gate_proj", + "model.layers.47.mlp.experts.97.up_proj", + "model.layers.47.mlp.experts.98.down_proj", + "model.layers.47.mlp.experts.98.gate_proj", + "model.layers.47.mlp.experts.98.up_proj", + "model.layers.47.mlp.experts.99.down_proj", + "model.layers.47.mlp.experts.99.gate_proj", + "model.layers.47.mlp.experts.99.up_proj", + "model.layers.48.mlp.experts.0.down_proj", + "model.layers.48.mlp.experts.0.gate_proj", + "model.layers.48.mlp.experts.0.up_proj", + "model.layers.48.mlp.experts.1.down_proj", + "model.layers.48.mlp.experts.1.gate_proj", + "model.layers.48.mlp.experts.1.up_proj", + "model.layers.48.mlp.experts.10.down_proj", + "model.layers.48.mlp.experts.10.gate_proj", + "model.layers.48.mlp.experts.10.up_proj", + "model.layers.48.mlp.experts.100.down_proj", + "model.layers.48.mlp.experts.100.gate_proj", + "model.layers.48.mlp.experts.100.up_proj", + "model.layers.48.mlp.experts.101.down_proj", + "model.layers.48.mlp.experts.101.gate_proj", + "model.layers.48.mlp.experts.101.up_proj", + "model.layers.48.mlp.experts.102.down_proj", + "model.layers.48.mlp.experts.102.gate_proj", + "model.layers.48.mlp.experts.102.up_proj", + "model.layers.48.mlp.experts.103.down_proj", + "model.layers.48.mlp.experts.103.gate_proj", + "model.layers.48.mlp.experts.103.up_proj", + "model.layers.48.mlp.experts.104.down_proj", + "model.layers.48.mlp.experts.104.gate_proj", + "model.layers.48.mlp.experts.104.up_proj", + "model.layers.48.mlp.experts.105.down_proj", + "model.layers.48.mlp.experts.105.gate_proj", + "model.layers.48.mlp.experts.105.up_proj", + "model.layers.48.mlp.experts.106.down_proj", + "model.layers.48.mlp.experts.106.gate_proj", + "model.layers.48.mlp.experts.106.up_proj", + "model.layers.48.mlp.experts.107.down_proj", + "model.layers.48.mlp.experts.107.gate_proj", + "model.layers.48.mlp.experts.107.up_proj", + "model.layers.48.mlp.experts.108.down_proj", + "model.layers.48.mlp.experts.108.gate_proj", + "model.layers.48.mlp.experts.108.up_proj", + "model.layers.48.mlp.experts.109.down_proj", + "model.layers.48.mlp.experts.109.gate_proj", + "model.layers.48.mlp.experts.109.up_proj", + "model.layers.48.mlp.experts.11.down_proj", + "model.layers.48.mlp.experts.11.gate_proj", + "model.layers.48.mlp.experts.11.up_proj", + "model.layers.48.mlp.experts.110.down_proj", + "model.layers.48.mlp.experts.110.gate_proj", + "model.layers.48.mlp.experts.110.up_proj", + "model.layers.48.mlp.experts.111.down_proj", + "model.layers.48.mlp.experts.111.gate_proj", + "model.layers.48.mlp.experts.111.up_proj", + "model.layers.48.mlp.experts.112.down_proj", + "model.layers.48.mlp.experts.112.gate_proj", + "model.layers.48.mlp.experts.112.up_proj", + "model.layers.48.mlp.experts.113.down_proj", + "model.layers.48.mlp.experts.113.gate_proj", + "model.layers.48.mlp.experts.113.up_proj", + "model.layers.48.mlp.experts.114.down_proj", + "model.layers.48.mlp.experts.114.gate_proj", + "model.layers.48.mlp.experts.114.up_proj", + "model.layers.48.mlp.experts.115.down_proj", + "model.layers.48.mlp.experts.115.gate_proj", + "model.layers.48.mlp.experts.115.up_proj", + "model.layers.48.mlp.experts.116.down_proj", + "model.layers.48.mlp.experts.116.gate_proj", + "model.layers.48.mlp.experts.116.up_proj", + "model.layers.48.mlp.experts.117.down_proj", + "model.layers.48.mlp.experts.117.gate_proj", + "model.layers.48.mlp.experts.117.up_proj", + "model.layers.48.mlp.experts.118.down_proj", + "model.layers.48.mlp.experts.118.gate_proj", + "model.layers.48.mlp.experts.118.up_proj", + "model.layers.48.mlp.experts.119.down_proj", + "model.layers.48.mlp.experts.119.gate_proj", + "model.layers.48.mlp.experts.119.up_proj", + "model.layers.48.mlp.experts.12.down_proj", + "model.layers.48.mlp.experts.12.gate_proj", + "model.layers.48.mlp.experts.12.up_proj", + "model.layers.48.mlp.experts.120.down_proj", + "model.layers.48.mlp.experts.120.gate_proj", + "model.layers.48.mlp.experts.120.up_proj", + "model.layers.48.mlp.experts.121.down_proj", + "model.layers.48.mlp.experts.121.gate_proj", + "model.layers.48.mlp.experts.121.up_proj", + "model.layers.48.mlp.experts.122.down_proj", + "model.layers.48.mlp.experts.122.gate_proj", + "model.layers.48.mlp.experts.122.up_proj", + "model.layers.48.mlp.experts.123.down_proj", + "model.layers.48.mlp.experts.123.gate_proj", + "model.layers.48.mlp.experts.123.up_proj", + "model.layers.48.mlp.experts.124.down_proj", + "model.layers.48.mlp.experts.124.gate_proj", + "model.layers.48.mlp.experts.124.up_proj", + "model.layers.48.mlp.experts.125.down_proj", + "model.layers.48.mlp.experts.125.gate_proj", + "model.layers.48.mlp.experts.125.up_proj", + "model.layers.48.mlp.experts.126.down_proj", + "model.layers.48.mlp.experts.126.gate_proj", + "model.layers.48.mlp.experts.126.up_proj", + "model.layers.48.mlp.experts.127.down_proj", + "model.layers.48.mlp.experts.127.gate_proj", + "model.layers.48.mlp.experts.127.up_proj", + "model.layers.48.mlp.experts.128.down_proj", + "model.layers.48.mlp.experts.128.gate_proj", + "model.layers.48.mlp.experts.128.up_proj", + "model.layers.48.mlp.experts.129.down_proj", + "model.layers.48.mlp.experts.129.gate_proj", + "model.layers.48.mlp.experts.129.up_proj", + "model.layers.48.mlp.experts.13.down_proj", + "model.layers.48.mlp.experts.13.gate_proj", + "model.layers.48.mlp.experts.13.up_proj", + "model.layers.48.mlp.experts.130.down_proj", + "model.layers.48.mlp.experts.130.gate_proj", + "model.layers.48.mlp.experts.130.up_proj", + "model.layers.48.mlp.experts.131.down_proj", + "model.layers.48.mlp.experts.131.gate_proj", + "model.layers.48.mlp.experts.131.up_proj", + "model.layers.48.mlp.experts.132.down_proj", + "model.layers.48.mlp.experts.132.gate_proj", + "model.layers.48.mlp.experts.132.up_proj", + "model.layers.48.mlp.experts.133.down_proj", + "model.layers.48.mlp.experts.133.gate_proj", + "model.layers.48.mlp.experts.133.up_proj", + "model.layers.48.mlp.experts.134.down_proj", + "model.layers.48.mlp.experts.134.gate_proj", + "model.layers.48.mlp.experts.134.up_proj", + "model.layers.48.mlp.experts.135.down_proj", + "model.layers.48.mlp.experts.135.gate_proj", + "model.layers.48.mlp.experts.135.up_proj", + "model.layers.48.mlp.experts.136.down_proj", + "model.layers.48.mlp.experts.136.gate_proj", + "model.layers.48.mlp.experts.136.up_proj", + "model.layers.48.mlp.experts.137.down_proj", + "model.layers.48.mlp.experts.137.gate_proj", + "model.layers.48.mlp.experts.137.up_proj", + "model.layers.48.mlp.experts.138.down_proj", + "model.layers.48.mlp.experts.138.gate_proj", + "model.layers.48.mlp.experts.138.up_proj", + "model.layers.48.mlp.experts.139.down_proj", + "model.layers.48.mlp.experts.139.gate_proj", + "model.layers.48.mlp.experts.139.up_proj", + "model.layers.48.mlp.experts.14.down_proj", + "model.layers.48.mlp.experts.14.gate_proj", + "model.layers.48.mlp.experts.14.up_proj", + "model.layers.48.mlp.experts.140.down_proj", + "model.layers.48.mlp.experts.140.gate_proj", + "model.layers.48.mlp.experts.140.up_proj", + "model.layers.48.mlp.experts.141.down_proj", + "model.layers.48.mlp.experts.141.gate_proj", + "model.layers.48.mlp.experts.141.up_proj", + "model.layers.48.mlp.experts.142.down_proj", + "model.layers.48.mlp.experts.142.gate_proj", + "model.layers.48.mlp.experts.142.up_proj", + "model.layers.48.mlp.experts.143.down_proj", + "model.layers.48.mlp.experts.143.gate_proj", + "model.layers.48.mlp.experts.143.up_proj", + "model.layers.48.mlp.experts.144.down_proj", + "model.layers.48.mlp.experts.144.gate_proj", + "model.layers.48.mlp.experts.144.up_proj", + "model.layers.48.mlp.experts.145.down_proj", + "model.layers.48.mlp.experts.145.gate_proj", + "model.layers.48.mlp.experts.145.up_proj", + "model.layers.48.mlp.experts.146.down_proj", + "model.layers.48.mlp.experts.146.gate_proj", + "model.layers.48.mlp.experts.146.up_proj", + "model.layers.48.mlp.experts.147.down_proj", + "model.layers.48.mlp.experts.147.gate_proj", + "model.layers.48.mlp.experts.147.up_proj", + "model.layers.48.mlp.experts.148.down_proj", + "model.layers.48.mlp.experts.148.gate_proj", + "model.layers.48.mlp.experts.148.up_proj", + "model.layers.48.mlp.experts.149.down_proj", + "model.layers.48.mlp.experts.149.gate_proj", + "model.layers.48.mlp.experts.149.up_proj", + "model.layers.48.mlp.experts.15.down_proj", + "model.layers.48.mlp.experts.15.gate_proj", + "model.layers.48.mlp.experts.15.up_proj", + "model.layers.48.mlp.experts.150.down_proj", + "model.layers.48.mlp.experts.150.gate_proj", + "model.layers.48.mlp.experts.150.up_proj", + "model.layers.48.mlp.experts.151.down_proj", + "model.layers.48.mlp.experts.151.gate_proj", + "model.layers.48.mlp.experts.151.up_proj", + "model.layers.48.mlp.experts.152.down_proj", + "model.layers.48.mlp.experts.152.gate_proj", + "model.layers.48.mlp.experts.152.up_proj", + "model.layers.48.mlp.experts.153.down_proj", + "model.layers.48.mlp.experts.153.gate_proj", + "model.layers.48.mlp.experts.153.up_proj", + "model.layers.48.mlp.experts.154.down_proj", + "model.layers.48.mlp.experts.154.gate_proj", + "model.layers.48.mlp.experts.154.up_proj", + "model.layers.48.mlp.experts.155.down_proj", + "model.layers.48.mlp.experts.155.gate_proj", + "model.layers.48.mlp.experts.155.up_proj", + "model.layers.48.mlp.experts.156.down_proj", + "model.layers.48.mlp.experts.156.gate_proj", + "model.layers.48.mlp.experts.156.up_proj", + "model.layers.48.mlp.experts.157.down_proj", + "model.layers.48.mlp.experts.157.gate_proj", + "model.layers.48.mlp.experts.157.up_proj", + "model.layers.48.mlp.experts.158.down_proj", + "model.layers.48.mlp.experts.158.gate_proj", + "model.layers.48.mlp.experts.158.up_proj", + "model.layers.48.mlp.experts.159.down_proj", + "model.layers.48.mlp.experts.159.gate_proj", + "model.layers.48.mlp.experts.159.up_proj", + "model.layers.48.mlp.experts.16.down_proj", + "model.layers.48.mlp.experts.16.gate_proj", + "model.layers.48.mlp.experts.16.up_proj", + "model.layers.48.mlp.experts.160.down_proj", + "model.layers.48.mlp.experts.160.gate_proj", + "model.layers.48.mlp.experts.160.up_proj", + "model.layers.48.mlp.experts.161.down_proj", + "model.layers.48.mlp.experts.161.gate_proj", + "model.layers.48.mlp.experts.161.up_proj", + "model.layers.48.mlp.experts.162.down_proj", + "model.layers.48.mlp.experts.162.gate_proj", + "model.layers.48.mlp.experts.162.up_proj", + "model.layers.48.mlp.experts.163.down_proj", + "model.layers.48.mlp.experts.163.gate_proj", + "model.layers.48.mlp.experts.163.up_proj", + "model.layers.48.mlp.experts.164.down_proj", + "model.layers.48.mlp.experts.164.gate_proj", + "model.layers.48.mlp.experts.164.up_proj", + "model.layers.48.mlp.experts.165.down_proj", + "model.layers.48.mlp.experts.165.gate_proj", + "model.layers.48.mlp.experts.165.up_proj", + "model.layers.48.mlp.experts.166.down_proj", + "model.layers.48.mlp.experts.166.gate_proj", + "model.layers.48.mlp.experts.166.up_proj", + "model.layers.48.mlp.experts.167.down_proj", + "model.layers.48.mlp.experts.167.gate_proj", + "model.layers.48.mlp.experts.167.up_proj", + "model.layers.48.mlp.experts.168.down_proj", + "model.layers.48.mlp.experts.168.gate_proj", + "model.layers.48.mlp.experts.168.up_proj", + "model.layers.48.mlp.experts.169.down_proj", + "model.layers.48.mlp.experts.169.gate_proj", + "model.layers.48.mlp.experts.169.up_proj", + "model.layers.48.mlp.experts.17.down_proj", + "model.layers.48.mlp.experts.17.gate_proj", + "model.layers.48.mlp.experts.17.up_proj", + "model.layers.48.mlp.experts.170.down_proj", + "model.layers.48.mlp.experts.170.gate_proj", + "model.layers.48.mlp.experts.170.up_proj", + "model.layers.48.mlp.experts.171.down_proj", + "model.layers.48.mlp.experts.171.gate_proj", + "model.layers.48.mlp.experts.171.up_proj", + "model.layers.48.mlp.experts.172.down_proj", + "model.layers.48.mlp.experts.172.gate_proj", + "model.layers.48.mlp.experts.172.up_proj", + "model.layers.48.mlp.experts.173.down_proj", + "model.layers.48.mlp.experts.173.gate_proj", + "model.layers.48.mlp.experts.173.up_proj", + "model.layers.48.mlp.experts.174.down_proj", + "model.layers.48.mlp.experts.174.gate_proj", + "model.layers.48.mlp.experts.174.up_proj", + "model.layers.48.mlp.experts.175.down_proj", + "model.layers.48.mlp.experts.175.gate_proj", + "model.layers.48.mlp.experts.175.up_proj", + "model.layers.48.mlp.experts.176.down_proj", + "model.layers.48.mlp.experts.176.gate_proj", + "model.layers.48.mlp.experts.176.up_proj", + "model.layers.48.mlp.experts.177.down_proj", + "model.layers.48.mlp.experts.177.gate_proj", + "model.layers.48.mlp.experts.177.up_proj", + "model.layers.48.mlp.experts.178.down_proj", + "model.layers.48.mlp.experts.178.gate_proj", + "model.layers.48.mlp.experts.178.up_proj", + "model.layers.48.mlp.experts.179.down_proj", + "model.layers.48.mlp.experts.179.gate_proj", + "model.layers.48.mlp.experts.179.up_proj", + "model.layers.48.mlp.experts.18.down_proj", + "model.layers.48.mlp.experts.18.gate_proj", + "model.layers.48.mlp.experts.18.up_proj", + "model.layers.48.mlp.experts.180.down_proj", + "model.layers.48.mlp.experts.180.gate_proj", + "model.layers.48.mlp.experts.180.up_proj", + "model.layers.48.mlp.experts.181.down_proj", + "model.layers.48.mlp.experts.181.gate_proj", + "model.layers.48.mlp.experts.181.up_proj", + "model.layers.48.mlp.experts.182.down_proj", + "model.layers.48.mlp.experts.182.gate_proj", + "model.layers.48.mlp.experts.182.up_proj", + "model.layers.48.mlp.experts.183.down_proj", + "model.layers.48.mlp.experts.183.gate_proj", + "model.layers.48.mlp.experts.183.up_proj", + "model.layers.48.mlp.experts.184.down_proj", + "model.layers.48.mlp.experts.184.gate_proj", + "model.layers.48.mlp.experts.184.up_proj", + "model.layers.48.mlp.experts.185.down_proj", + "model.layers.48.mlp.experts.185.gate_proj", + "model.layers.48.mlp.experts.185.up_proj", + "model.layers.48.mlp.experts.186.down_proj", + "model.layers.48.mlp.experts.186.gate_proj", + "model.layers.48.mlp.experts.186.up_proj", + "model.layers.48.mlp.experts.187.down_proj", + "model.layers.48.mlp.experts.187.gate_proj", + "model.layers.48.mlp.experts.187.up_proj", + "model.layers.48.mlp.experts.188.down_proj", + "model.layers.48.mlp.experts.188.gate_proj", + "model.layers.48.mlp.experts.188.up_proj", + "model.layers.48.mlp.experts.189.down_proj", + "model.layers.48.mlp.experts.189.gate_proj", + "model.layers.48.mlp.experts.189.up_proj", + "model.layers.48.mlp.experts.19.down_proj", + "model.layers.48.mlp.experts.19.gate_proj", + "model.layers.48.mlp.experts.19.up_proj", + "model.layers.48.mlp.experts.190.down_proj", + "model.layers.48.mlp.experts.190.gate_proj", + "model.layers.48.mlp.experts.190.up_proj", + "model.layers.48.mlp.experts.191.down_proj", + "model.layers.48.mlp.experts.191.gate_proj", + "model.layers.48.mlp.experts.191.up_proj", + "model.layers.48.mlp.experts.192.down_proj", + "model.layers.48.mlp.experts.192.gate_proj", + "model.layers.48.mlp.experts.192.up_proj", + "model.layers.48.mlp.experts.193.down_proj", + "model.layers.48.mlp.experts.193.gate_proj", + "model.layers.48.mlp.experts.193.up_proj", + "model.layers.48.mlp.experts.194.down_proj", + "model.layers.48.mlp.experts.194.gate_proj", + "model.layers.48.mlp.experts.194.up_proj", + "model.layers.48.mlp.experts.195.down_proj", + "model.layers.48.mlp.experts.195.gate_proj", + "model.layers.48.mlp.experts.195.up_proj", + "model.layers.48.mlp.experts.196.down_proj", + "model.layers.48.mlp.experts.196.gate_proj", + "model.layers.48.mlp.experts.196.up_proj", + "model.layers.48.mlp.experts.197.down_proj", + "model.layers.48.mlp.experts.197.gate_proj", + "model.layers.48.mlp.experts.197.up_proj", + "model.layers.48.mlp.experts.198.down_proj", + "model.layers.48.mlp.experts.198.gate_proj", + "model.layers.48.mlp.experts.198.up_proj", + "model.layers.48.mlp.experts.199.down_proj", + "model.layers.48.mlp.experts.199.gate_proj", + "model.layers.48.mlp.experts.199.up_proj", + "model.layers.48.mlp.experts.2.down_proj", + "model.layers.48.mlp.experts.2.gate_proj", + "model.layers.48.mlp.experts.2.up_proj", + "model.layers.48.mlp.experts.20.down_proj", + "model.layers.48.mlp.experts.20.gate_proj", + "model.layers.48.mlp.experts.20.up_proj", + "model.layers.48.mlp.experts.200.down_proj", + "model.layers.48.mlp.experts.200.gate_proj", + "model.layers.48.mlp.experts.200.up_proj", + "model.layers.48.mlp.experts.201.down_proj", + "model.layers.48.mlp.experts.201.gate_proj", + "model.layers.48.mlp.experts.201.up_proj", + "model.layers.48.mlp.experts.202.down_proj", + "model.layers.48.mlp.experts.202.gate_proj", + "model.layers.48.mlp.experts.202.up_proj", + "model.layers.48.mlp.experts.203.down_proj", + "model.layers.48.mlp.experts.203.gate_proj", + "model.layers.48.mlp.experts.203.up_proj", + "model.layers.48.mlp.experts.204.down_proj", + "model.layers.48.mlp.experts.204.gate_proj", + "model.layers.48.mlp.experts.204.up_proj", + "model.layers.48.mlp.experts.205.down_proj", + "model.layers.48.mlp.experts.205.gate_proj", + "model.layers.48.mlp.experts.205.up_proj", + "model.layers.48.mlp.experts.206.down_proj", + "model.layers.48.mlp.experts.206.gate_proj", + "model.layers.48.mlp.experts.206.up_proj", + "model.layers.48.mlp.experts.207.down_proj", + "model.layers.48.mlp.experts.207.gate_proj", + "model.layers.48.mlp.experts.207.up_proj", + "model.layers.48.mlp.experts.208.down_proj", + "model.layers.48.mlp.experts.208.gate_proj", + "model.layers.48.mlp.experts.208.up_proj", + "model.layers.48.mlp.experts.209.down_proj", + "model.layers.48.mlp.experts.209.gate_proj", + "model.layers.48.mlp.experts.209.up_proj", + "model.layers.48.mlp.experts.21.down_proj", + "model.layers.48.mlp.experts.21.gate_proj", + "model.layers.48.mlp.experts.21.up_proj", + "model.layers.48.mlp.experts.210.down_proj", + "model.layers.48.mlp.experts.210.gate_proj", + "model.layers.48.mlp.experts.210.up_proj", + "model.layers.48.mlp.experts.211.down_proj", + "model.layers.48.mlp.experts.211.gate_proj", + "model.layers.48.mlp.experts.211.up_proj", + "model.layers.48.mlp.experts.212.down_proj", + "model.layers.48.mlp.experts.212.gate_proj", + "model.layers.48.mlp.experts.212.up_proj", + "model.layers.48.mlp.experts.213.down_proj", + "model.layers.48.mlp.experts.213.gate_proj", + "model.layers.48.mlp.experts.213.up_proj", + "model.layers.48.mlp.experts.214.down_proj", + "model.layers.48.mlp.experts.214.gate_proj", + "model.layers.48.mlp.experts.214.up_proj", + "model.layers.48.mlp.experts.215.down_proj", + "model.layers.48.mlp.experts.215.gate_proj", + "model.layers.48.mlp.experts.215.up_proj", + "model.layers.48.mlp.experts.216.down_proj", + "model.layers.48.mlp.experts.216.gate_proj", + "model.layers.48.mlp.experts.216.up_proj", + "model.layers.48.mlp.experts.217.down_proj", + "model.layers.48.mlp.experts.217.gate_proj", + "model.layers.48.mlp.experts.217.up_proj", + "model.layers.48.mlp.experts.218.down_proj", + "model.layers.48.mlp.experts.218.gate_proj", + "model.layers.48.mlp.experts.218.up_proj", + "model.layers.48.mlp.experts.219.down_proj", + "model.layers.48.mlp.experts.219.gate_proj", + "model.layers.48.mlp.experts.219.up_proj", + "model.layers.48.mlp.experts.22.down_proj", + "model.layers.48.mlp.experts.22.gate_proj", + "model.layers.48.mlp.experts.22.up_proj", + "model.layers.48.mlp.experts.220.down_proj", + "model.layers.48.mlp.experts.220.gate_proj", + "model.layers.48.mlp.experts.220.up_proj", + "model.layers.48.mlp.experts.221.down_proj", + "model.layers.48.mlp.experts.221.gate_proj", + "model.layers.48.mlp.experts.221.up_proj", + "model.layers.48.mlp.experts.222.down_proj", + "model.layers.48.mlp.experts.222.gate_proj", + "model.layers.48.mlp.experts.222.up_proj", + "model.layers.48.mlp.experts.223.down_proj", + "model.layers.48.mlp.experts.223.gate_proj", + "model.layers.48.mlp.experts.223.up_proj", + "model.layers.48.mlp.experts.224.down_proj", + "model.layers.48.mlp.experts.224.gate_proj", + "model.layers.48.mlp.experts.224.up_proj", + "model.layers.48.mlp.experts.225.down_proj", + "model.layers.48.mlp.experts.225.gate_proj", + "model.layers.48.mlp.experts.225.up_proj", + "model.layers.48.mlp.experts.226.down_proj", + "model.layers.48.mlp.experts.226.gate_proj", + "model.layers.48.mlp.experts.226.up_proj", + "model.layers.48.mlp.experts.227.down_proj", + "model.layers.48.mlp.experts.227.gate_proj", + "model.layers.48.mlp.experts.227.up_proj", + "model.layers.48.mlp.experts.228.down_proj", + "model.layers.48.mlp.experts.228.gate_proj", + "model.layers.48.mlp.experts.228.up_proj", + "model.layers.48.mlp.experts.229.down_proj", + "model.layers.48.mlp.experts.229.gate_proj", + "model.layers.48.mlp.experts.229.up_proj", + "model.layers.48.mlp.experts.23.down_proj", + "model.layers.48.mlp.experts.23.gate_proj", + "model.layers.48.mlp.experts.23.up_proj", + "model.layers.48.mlp.experts.230.down_proj", + "model.layers.48.mlp.experts.230.gate_proj", + "model.layers.48.mlp.experts.230.up_proj", + "model.layers.48.mlp.experts.231.down_proj", + "model.layers.48.mlp.experts.231.gate_proj", + "model.layers.48.mlp.experts.231.up_proj", + "model.layers.48.mlp.experts.232.down_proj", + "model.layers.48.mlp.experts.232.gate_proj", + "model.layers.48.mlp.experts.232.up_proj", + "model.layers.48.mlp.experts.233.down_proj", + "model.layers.48.mlp.experts.233.gate_proj", + "model.layers.48.mlp.experts.233.up_proj", + "model.layers.48.mlp.experts.234.down_proj", + "model.layers.48.mlp.experts.234.gate_proj", + "model.layers.48.mlp.experts.234.up_proj", + "model.layers.48.mlp.experts.235.down_proj", + "model.layers.48.mlp.experts.235.gate_proj", + "model.layers.48.mlp.experts.235.up_proj", + "model.layers.48.mlp.experts.236.down_proj", + "model.layers.48.mlp.experts.236.gate_proj", + "model.layers.48.mlp.experts.236.up_proj", + "model.layers.48.mlp.experts.237.down_proj", + "model.layers.48.mlp.experts.237.gate_proj", + "model.layers.48.mlp.experts.237.up_proj", + "model.layers.48.mlp.experts.238.down_proj", + "model.layers.48.mlp.experts.238.gate_proj", + "model.layers.48.mlp.experts.238.up_proj", + "model.layers.48.mlp.experts.239.down_proj", + "model.layers.48.mlp.experts.239.gate_proj", + "model.layers.48.mlp.experts.239.up_proj", + "model.layers.48.mlp.experts.24.down_proj", + "model.layers.48.mlp.experts.24.gate_proj", + "model.layers.48.mlp.experts.24.up_proj", + "model.layers.48.mlp.experts.240.down_proj", + "model.layers.48.mlp.experts.240.gate_proj", + "model.layers.48.mlp.experts.240.up_proj", + "model.layers.48.mlp.experts.241.down_proj", + "model.layers.48.mlp.experts.241.gate_proj", + "model.layers.48.mlp.experts.241.up_proj", + "model.layers.48.mlp.experts.242.down_proj", + "model.layers.48.mlp.experts.242.gate_proj", + "model.layers.48.mlp.experts.242.up_proj", + "model.layers.48.mlp.experts.243.down_proj", + "model.layers.48.mlp.experts.243.gate_proj", + "model.layers.48.mlp.experts.243.up_proj", + "model.layers.48.mlp.experts.244.down_proj", + "model.layers.48.mlp.experts.244.gate_proj", + "model.layers.48.mlp.experts.244.up_proj", + "model.layers.48.mlp.experts.245.down_proj", + "model.layers.48.mlp.experts.245.gate_proj", + "model.layers.48.mlp.experts.245.up_proj", + "model.layers.48.mlp.experts.246.down_proj", + "model.layers.48.mlp.experts.246.gate_proj", + "model.layers.48.mlp.experts.246.up_proj", + "model.layers.48.mlp.experts.247.down_proj", + "model.layers.48.mlp.experts.247.gate_proj", + "model.layers.48.mlp.experts.247.up_proj", + "model.layers.48.mlp.experts.248.down_proj", + "model.layers.48.mlp.experts.248.gate_proj", + "model.layers.48.mlp.experts.248.up_proj", + "model.layers.48.mlp.experts.249.down_proj", + "model.layers.48.mlp.experts.249.gate_proj", + "model.layers.48.mlp.experts.249.up_proj", + "model.layers.48.mlp.experts.25.down_proj", + "model.layers.48.mlp.experts.25.gate_proj", + "model.layers.48.mlp.experts.25.up_proj", + "model.layers.48.mlp.experts.250.down_proj", + "model.layers.48.mlp.experts.250.gate_proj", + "model.layers.48.mlp.experts.250.up_proj", + "model.layers.48.mlp.experts.251.down_proj", + "model.layers.48.mlp.experts.251.gate_proj", + "model.layers.48.mlp.experts.251.up_proj", + "model.layers.48.mlp.experts.252.down_proj", + "model.layers.48.mlp.experts.252.gate_proj", + "model.layers.48.mlp.experts.252.up_proj", + "model.layers.48.mlp.experts.253.down_proj", + "model.layers.48.mlp.experts.253.gate_proj", + "model.layers.48.mlp.experts.253.up_proj", + "model.layers.48.mlp.experts.254.down_proj", + "model.layers.48.mlp.experts.254.gate_proj", + "model.layers.48.mlp.experts.254.up_proj", + "model.layers.48.mlp.experts.255.down_proj", + "model.layers.48.mlp.experts.255.gate_proj", + "model.layers.48.mlp.experts.255.up_proj", + "model.layers.48.mlp.experts.26.down_proj", + "model.layers.48.mlp.experts.26.gate_proj", + "model.layers.48.mlp.experts.26.up_proj", + "model.layers.48.mlp.experts.27.down_proj", + "model.layers.48.mlp.experts.27.gate_proj", + "model.layers.48.mlp.experts.27.up_proj", + "model.layers.48.mlp.experts.28.down_proj", + "model.layers.48.mlp.experts.28.gate_proj", + "model.layers.48.mlp.experts.28.up_proj", + "model.layers.48.mlp.experts.29.down_proj", + "model.layers.48.mlp.experts.29.gate_proj", + "model.layers.48.mlp.experts.29.up_proj", + "model.layers.48.mlp.experts.3.down_proj", + "model.layers.48.mlp.experts.3.gate_proj", + "model.layers.48.mlp.experts.3.up_proj", + "model.layers.48.mlp.experts.30.down_proj", + "model.layers.48.mlp.experts.30.gate_proj", + "model.layers.48.mlp.experts.30.up_proj", + "model.layers.48.mlp.experts.31.down_proj", + "model.layers.48.mlp.experts.31.gate_proj", + "model.layers.48.mlp.experts.31.up_proj", + "model.layers.48.mlp.experts.32.down_proj", + "model.layers.48.mlp.experts.32.gate_proj", + "model.layers.48.mlp.experts.32.up_proj", + "model.layers.48.mlp.experts.33.down_proj", + "model.layers.48.mlp.experts.33.gate_proj", + "model.layers.48.mlp.experts.33.up_proj", + "model.layers.48.mlp.experts.34.down_proj", + "model.layers.48.mlp.experts.34.gate_proj", + "model.layers.48.mlp.experts.34.up_proj", + "model.layers.48.mlp.experts.35.down_proj", + "model.layers.48.mlp.experts.35.gate_proj", + "model.layers.48.mlp.experts.35.up_proj", + "model.layers.48.mlp.experts.36.down_proj", + "model.layers.48.mlp.experts.36.gate_proj", + "model.layers.48.mlp.experts.36.up_proj", + "model.layers.48.mlp.experts.37.down_proj", + "model.layers.48.mlp.experts.37.gate_proj", + "model.layers.48.mlp.experts.37.up_proj", + "model.layers.48.mlp.experts.38.down_proj", + "model.layers.48.mlp.experts.38.gate_proj", + "model.layers.48.mlp.experts.38.up_proj", + "model.layers.48.mlp.experts.39.down_proj", + "model.layers.48.mlp.experts.39.gate_proj", + "model.layers.48.mlp.experts.39.up_proj", + "model.layers.48.mlp.experts.4.down_proj", + "model.layers.48.mlp.experts.4.gate_proj", + "model.layers.48.mlp.experts.4.up_proj", + "model.layers.48.mlp.experts.40.down_proj", + "model.layers.48.mlp.experts.40.gate_proj", + "model.layers.48.mlp.experts.40.up_proj", + "model.layers.48.mlp.experts.41.down_proj", + "model.layers.48.mlp.experts.41.gate_proj", + "model.layers.48.mlp.experts.41.up_proj", + "model.layers.48.mlp.experts.42.down_proj", + "model.layers.48.mlp.experts.42.gate_proj", + "model.layers.48.mlp.experts.42.up_proj", + "model.layers.48.mlp.experts.43.down_proj", + "model.layers.48.mlp.experts.43.gate_proj", + "model.layers.48.mlp.experts.43.up_proj", + "model.layers.48.mlp.experts.44.down_proj", + "model.layers.48.mlp.experts.44.gate_proj", + "model.layers.48.mlp.experts.44.up_proj", + "model.layers.48.mlp.experts.45.down_proj", + "model.layers.48.mlp.experts.45.gate_proj", + "model.layers.48.mlp.experts.45.up_proj", + "model.layers.48.mlp.experts.46.down_proj", + "model.layers.48.mlp.experts.46.gate_proj", + "model.layers.48.mlp.experts.46.up_proj", + "model.layers.48.mlp.experts.47.down_proj", + "model.layers.48.mlp.experts.47.gate_proj", + "model.layers.48.mlp.experts.47.up_proj", + "model.layers.48.mlp.experts.48.down_proj", + "model.layers.48.mlp.experts.48.gate_proj", + "model.layers.48.mlp.experts.48.up_proj", + "model.layers.48.mlp.experts.49.down_proj", + "model.layers.48.mlp.experts.49.gate_proj", + "model.layers.48.mlp.experts.49.up_proj", + "model.layers.48.mlp.experts.5.down_proj", + "model.layers.48.mlp.experts.5.gate_proj", + "model.layers.48.mlp.experts.5.up_proj", + "model.layers.48.mlp.experts.50.down_proj", + "model.layers.48.mlp.experts.50.gate_proj", + "model.layers.48.mlp.experts.50.up_proj", + "model.layers.48.mlp.experts.51.down_proj", + "model.layers.48.mlp.experts.51.gate_proj", + "model.layers.48.mlp.experts.51.up_proj", + "model.layers.48.mlp.experts.52.down_proj", + "model.layers.48.mlp.experts.52.gate_proj", + "model.layers.48.mlp.experts.52.up_proj", + "model.layers.48.mlp.experts.53.down_proj", + "model.layers.48.mlp.experts.53.gate_proj", + "model.layers.48.mlp.experts.53.up_proj", + "model.layers.48.mlp.experts.54.down_proj", + "model.layers.48.mlp.experts.54.gate_proj", + "model.layers.48.mlp.experts.54.up_proj", + "model.layers.48.mlp.experts.55.down_proj", + "model.layers.48.mlp.experts.55.gate_proj", + "model.layers.48.mlp.experts.55.up_proj", + "model.layers.48.mlp.experts.56.down_proj", + "model.layers.48.mlp.experts.56.gate_proj", + "model.layers.48.mlp.experts.56.up_proj", + "model.layers.48.mlp.experts.57.down_proj", + "model.layers.48.mlp.experts.57.gate_proj", + "model.layers.48.mlp.experts.57.up_proj", + "model.layers.48.mlp.experts.58.down_proj", + "model.layers.48.mlp.experts.58.gate_proj", + "model.layers.48.mlp.experts.58.up_proj", + "model.layers.48.mlp.experts.59.down_proj", + "model.layers.48.mlp.experts.59.gate_proj", + "model.layers.48.mlp.experts.59.up_proj", + "model.layers.48.mlp.experts.6.down_proj", + "model.layers.48.mlp.experts.6.gate_proj", + "model.layers.48.mlp.experts.6.up_proj", + "model.layers.48.mlp.experts.60.down_proj", + "model.layers.48.mlp.experts.60.gate_proj", + "model.layers.48.mlp.experts.60.up_proj", + "model.layers.48.mlp.experts.61.down_proj", + "model.layers.48.mlp.experts.61.gate_proj", + "model.layers.48.mlp.experts.61.up_proj", + "model.layers.48.mlp.experts.62.down_proj", + "model.layers.48.mlp.experts.62.gate_proj", + "model.layers.48.mlp.experts.62.up_proj", + "model.layers.48.mlp.experts.63.down_proj", + "model.layers.48.mlp.experts.63.gate_proj", + "model.layers.48.mlp.experts.63.up_proj", + "model.layers.48.mlp.experts.64.down_proj", + "model.layers.48.mlp.experts.64.gate_proj", + "model.layers.48.mlp.experts.64.up_proj", + "model.layers.48.mlp.experts.65.down_proj", + "model.layers.48.mlp.experts.65.gate_proj", + "model.layers.48.mlp.experts.65.up_proj", + "model.layers.48.mlp.experts.66.down_proj", + "model.layers.48.mlp.experts.66.gate_proj", + "model.layers.48.mlp.experts.66.up_proj", + "model.layers.48.mlp.experts.67.down_proj", + "model.layers.48.mlp.experts.67.gate_proj", + "model.layers.48.mlp.experts.67.up_proj", + "model.layers.48.mlp.experts.68.down_proj", + "model.layers.48.mlp.experts.68.gate_proj", + "model.layers.48.mlp.experts.68.up_proj", + "model.layers.48.mlp.experts.69.down_proj", + "model.layers.48.mlp.experts.69.gate_proj", + "model.layers.48.mlp.experts.69.up_proj", + "model.layers.48.mlp.experts.7.down_proj", + "model.layers.48.mlp.experts.7.gate_proj", + "model.layers.48.mlp.experts.7.up_proj", + "model.layers.48.mlp.experts.70.down_proj", + "model.layers.48.mlp.experts.70.gate_proj", + "model.layers.48.mlp.experts.70.up_proj", + "model.layers.48.mlp.experts.71.down_proj", + "model.layers.48.mlp.experts.71.gate_proj", + "model.layers.48.mlp.experts.71.up_proj", + "model.layers.48.mlp.experts.72.down_proj", + "model.layers.48.mlp.experts.72.gate_proj", + "model.layers.48.mlp.experts.72.up_proj", + "model.layers.48.mlp.experts.73.down_proj", + "model.layers.48.mlp.experts.73.gate_proj", + "model.layers.48.mlp.experts.73.up_proj", + "model.layers.48.mlp.experts.74.down_proj", + "model.layers.48.mlp.experts.74.gate_proj", + "model.layers.48.mlp.experts.74.up_proj", + "model.layers.48.mlp.experts.75.down_proj", + "model.layers.48.mlp.experts.75.gate_proj", + "model.layers.48.mlp.experts.75.up_proj", + "model.layers.48.mlp.experts.76.down_proj", + "model.layers.48.mlp.experts.76.gate_proj", + "model.layers.48.mlp.experts.76.up_proj", + "model.layers.48.mlp.experts.77.down_proj", + "model.layers.48.mlp.experts.77.gate_proj", + "model.layers.48.mlp.experts.77.up_proj", + "model.layers.48.mlp.experts.78.down_proj", + "model.layers.48.mlp.experts.78.gate_proj", + "model.layers.48.mlp.experts.78.up_proj", + "model.layers.48.mlp.experts.79.down_proj", + "model.layers.48.mlp.experts.79.gate_proj", + "model.layers.48.mlp.experts.79.up_proj", + "model.layers.48.mlp.experts.8.down_proj", + "model.layers.48.mlp.experts.8.gate_proj", + "model.layers.48.mlp.experts.8.up_proj", + "model.layers.48.mlp.experts.80.down_proj", + "model.layers.48.mlp.experts.80.gate_proj", + "model.layers.48.mlp.experts.80.up_proj", + "model.layers.48.mlp.experts.81.down_proj", + "model.layers.48.mlp.experts.81.gate_proj", + "model.layers.48.mlp.experts.81.up_proj", + "model.layers.48.mlp.experts.82.down_proj", + "model.layers.48.mlp.experts.82.gate_proj", + "model.layers.48.mlp.experts.82.up_proj", + "model.layers.48.mlp.experts.83.down_proj", + "model.layers.48.mlp.experts.83.gate_proj", + "model.layers.48.mlp.experts.83.up_proj", + "model.layers.48.mlp.experts.84.down_proj", + "model.layers.48.mlp.experts.84.gate_proj", + "model.layers.48.mlp.experts.84.up_proj", + "model.layers.48.mlp.experts.85.down_proj", + "model.layers.48.mlp.experts.85.gate_proj", + "model.layers.48.mlp.experts.85.up_proj", + "model.layers.48.mlp.experts.86.down_proj", + "model.layers.48.mlp.experts.86.gate_proj", + "model.layers.48.mlp.experts.86.up_proj", + "model.layers.48.mlp.experts.87.down_proj", + "model.layers.48.mlp.experts.87.gate_proj", + "model.layers.48.mlp.experts.87.up_proj", + "model.layers.48.mlp.experts.88.down_proj", + "model.layers.48.mlp.experts.88.gate_proj", + "model.layers.48.mlp.experts.88.up_proj", + "model.layers.48.mlp.experts.89.down_proj", + "model.layers.48.mlp.experts.89.gate_proj", + "model.layers.48.mlp.experts.89.up_proj", + "model.layers.48.mlp.experts.9.down_proj", + "model.layers.48.mlp.experts.9.gate_proj", + "model.layers.48.mlp.experts.9.up_proj", + "model.layers.48.mlp.experts.90.down_proj", + "model.layers.48.mlp.experts.90.gate_proj", + "model.layers.48.mlp.experts.90.up_proj", + "model.layers.48.mlp.experts.91.down_proj", + "model.layers.48.mlp.experts.91.gate_proj", + "model.layers.48.mlp.experts.91.up_proj", + "model.layers.48.mlp.experts.92.down_proj", + "model.layers.48.mlp.experts.92.gate_proj", + "model.layers.48.mlp.experts.92.up_proj", + "model.layers.48.mlp.experts.93.down_proj", + "model.layers.48.mlp.experts.93.gate_proj", + "model.layers.48.mlp.experts.93.up_proj", + "model.layers.48.mlp.experts.94.down_proj", + "model.layers.48.mlp.experts.94.gate_proj", + "model.layers.48.mlp.experts.94.up_proj", + "model.layers.48.mlp.experts.95.down_proj", + "model.layers.48.mlp.experts.95.gate_proj", + "model.layers.48.mlp.experts.95.up_proj", + "model.layers.48.mlp.experts.96.down_proj", + "model.layers.48.mlp.experts.96.gate_proj", + "model.layers.48.mlp.experts.96.up_proj", + "model.layers.48.mlp.experts.97.down_proj", + "model.layers.48.mlp.experts.97.gate_proj", + "model.layers.48.mlp.experts.97.up_proj", + "model.layers.48.mlp.experts.98.down_proj", + "model.layers.48.mlp.experts.98.gate_proj", + "model.layers.48.mlp.experts.98.up_proj", + "model.layers.48.mlp.experts.99.down_proj", + "model.layers.48.mlp.experts.99.gate_proj", + "model.layers.48.mlp.experts.99.up_proj", + "model.layers.49.mlp.experts.0.down_proj", + "model.layers.49.mlp.experts.0.gate_proj", + "model.layers.49.mlp.experts.0.up_proj", + "model.layers.49.mlp.experts.1.down_proj", + "model.layers.49.mlp.experts.1.gate_proj", + "model.layers.49.mlp.experts.1.up_proj", + "model.layers.49.mlp.experts.10.down_proj", + "model.layers.49.mlp.experts.10.gate_proj", + "model.layers.49.mlp.experts.10.up_proj", + "model.layers.49.mlp.experts.100.down_proj", + "model.layers.49.mlp.experts.100.gate_proj", + "model.layers.49.mlp.experts.100.up_proj", + "model.layers.49.mlp.experts.101.down_proj", + "model.layers.49.mlp.experts.101.gate_proj", + "model.layers.49.mlp.experts.101.up_proj", + "model.layers.49.mlp.experts.102.down_proj", + "model.layers.49.mlp.experts.102.gate_proj", + "model.layers.49.mlp.experts.102.up_proj", + "model.layers.49.mlp.experts.103.down_proj", + "model.layers.49.mlp.experts.103.gate_proj", + "model.layers.49.mlp.experts.103.up_proj", + "model.layers.49.mlp.experts.104.down_proj", + "model.layers.49.mlp.experts.104.gate_proj", + "model.layers.49.mlp.experts.104.up_proj", + "model.layers.49.mlp.experts.105.down_proj", + "model.layers.49.mlp.experts.105.gate_proj", + "model.layers.49.mlp.experts.105.up_proj", + "model.layers.49.mlp.experts.106.down_proj", + "model.layers.49.mlp.experts.106.gate_proj", + "model.layers.49.mlp.experts.106.up_proj", + "model.layers.49.mlp.experts.107.down_proj", + "model.layers.49.mlp.experts.107.gate_proj", + "model.layers.49.mlp.experts.107.up_proj", + "model.layers.49.mlp.experts.108.down_proj", + "model.layers.49.mlp.experts.108.gate_proj", + "model.layers.49.mlp.experts.108.up_proj", + "model.layers.49.mlp.experts.109.down_proj", + "model.layers.49.mlp.experts.109.gate_proj", + "model.layers.49.mlp.experts.109.up_proj", + "model.layers.49.mlp.experts.11.down_proj", + "model.layers.49.mlp.experts.11.gate_proj", + "model.layers.49.mlp.experts.11.up_proj", + "model.layers.49.mlp.experts.110.down_proj", + "model.layers.49.mlp.experts.110.gate_proj", + "model.layers.49.mlp.experts.110.up_proj", + "model.layers.49.mlp.experts.111.down_proj", + "model.layers.49.mlp.experts.111.gate_proj", + "model.layers.49.mlp.experts.111.up_proj", + "model.layers.49.mlp.experts.112.down_proj", + "model.layers.49.mlp.experts.112.gate_proj", + "model.layers.49.mlp.experts.112.up_proj", + "model.layers.49.mlp.experts.113.down_proj", + "model.layers.49.mlp.experts.113.gate_proj", + "model.layers.49.mlp.experts.113.up_proj", + "model.layers.49.mlp.experts.114.down_proj", + "model.layers.49.mlp.experts.114.gate_proj", + "model.layers.49.mlp.experts.114.up_proj", + "model.layers.49.mlp.experts.115.down_proj", + "model.layers.49.mlp.experts.115.gate_proj", + "model.layers.49.mlp.experts.115.up_proj", + "model.layers.49.mlp.experts.116.down_proj", + "model.layers.49.mlp.experts.116.gate_proj", + "model.layers.49.mlp.experts.116.up_proj", + "model.layers.49.mlp.experts.117.down_proj", + "model.layers.49.mlp.experts.117.gate_proj", + "model.layers.49.mlp.experts.117.up_proj", + "model.layers.49.mlp.experts.118.down_proj", + "model.layers.49.mlp.experts.118.gate_proj", + "model.layers.49.mlp.experts.118.up_proj", + "model.layers.49.mlp.experts.119.down_proj", + "model.layers.49.mlp.experts.119.gate_proj", + "model.layers.49.mlp.experts.119.up_proj", + "model.layers.49.mlp.experts.12.down_proj", + "model.layers.49.mlp.experts.12.gate_proj", + "model.layers.49.mlp.experts.12.up_proj", + "model.layers.49.mlp.experts.120.down_proj", + "model.layers.49.mlp.experts.120.gate_proj", + "model.layers.49.mlp.experts.120.up_proj", + "model.layers.49.mlp.experts.121.down_proj", + "model.layers.49.mlp.experts.121.gate_proj", + "model.layers.49.mlp.experts.121.up_proj", + "model.layers.49.mlp.experts.122.down_proj", + "model.layers.49.mlp.experts.122.gate_proj", + "model.layers.49.mlp.experts.122.up_proj", + "model.layers.49.mlp.experts.123.down_proj", + "model.layers.49.mlp.experts.123.gate_proj", + "model.layers.49.mlp.experts.123.up_proj", + "model.layers.49.mlp.experts.124.down_proj", + "model.layers.49.mlp.experts.124.gate_proj", + "model.layers.49.mlp.experts.124.up_proj", + "model.layers.49.mlp.experts.125.down_proj", + "model.layers.49.mlp.experts.125.gate_proj", + "model.layers.49.mlp.experts.125.up_proj", + "model.layers.49.mlp.experts.126.down_proj", + "model.layers.49.mlp.experts.126.gate_proj", + "model.layers.49.mlp.experts.126.up_proj", + "model.layers.49.mlp.experts.127.down_proj", + "model.layers.49.mlp.experts.127.gate_proj", + "model.layers.49.mlp.experts.127.up_proj", + "model.layers.49.mlp.experts.128.down_proj", + "model.layers.49.mlp.experts.128.gate_proj", + "model.layers.49.mlp.experts.128.up_proj", + "model.layers.49.mlp.experts.129.down_proj", + "model.layers.49.mlp.experts.129.gate_proj", + "model.layers.49.mlp.experts.129.up_proj", + "model.layers.49.mlp.experts.13.down_proj", + "model.layers.49.mlp.experts.13.gate_proj", + "model.layers.49.mlp.experts.13.up_proj", + "model.layers.49.mlp.experts.130.down_proj", + "model.layers.49.mlp.experts.130.gate_proj", + "model.layers.49.mlp.experts.130.up_proj", + "model.layers.49.mlp.experts.131.down_proj", + "model.layers.49.mlp.experts.131.gate_proj", + "model.layers.49.mlp.experts.131.up_proj", + "model.layers.49.mlp.experts.132.down_proj", + "model.layers.49.mlp.experts.132.gate_proj", + "model.layers.49.mlp.experts.132.up_proj", + "model.layers.49.mlp.experts.133.down_proj", + "model.layers.49.mlp.experts.133.gate_proj", + "model.layers.49.mlp.experts.133.up_proj", + "model.layers.49.mlp.experts.134.down_proj", + "model.layers.49.mlp.experts.134.gate_proj", + "model.layers.49.mlp.experts.134.up_proj", + "model.layers.49.mlp.experts.135.down_proj", + "model.layers.49.mlp.experts.135.gate_proj", + "model.layers.49.mlp.experts.135.up_proj", + "model.layers.49.mlp.experts.136.down_proj", + "model.layers.49.mlp.experts.136.gate_proj", + "model.layers.49.mlp.experts.136.up_proj", + "model.layers.49.mlp.experts.137.down_proj", + "model.layers.49.mlp.experts.137.gate_proj", + "model.layers.49.mlp.experts.137.up_proj", + "model.layers.49.mlp.experts.138.down_proj", + "model.layers.49.mlp.experts.138.gate_proj", + "model.layers.49.mlp.experts.138.up_proj", + "model.layers.49.mlp.experts.139.down_proj", + "model.layers.49.mlp.experts.139.gate_proj", + "model.layers.49.mlp.experts.139.up_proj", + "model.layers.49.mlp.experts.14.down_proj", + "model.layers.49.mlp.experts.14.gate_proj", + "model.layers.49.mlp.experts.14.up_proj", + "model.layers.49.mlp.experts.140.down_proj", + "model.layers.49.mlp.experts.140.gate_proj", + "model.layers.49.mlp.experts.140.up_proj", + "model.layers.49.mlp.experts.141.down_proj", + "model.layers.49.mlp.experts.141.gate_proj", + "model.layers.49.mlp.experts.141.up_proj", + "model.layers.49.mlp.experts.142.down_proj", + "model.layers.49.mlp.experts.142.gate_proj", + "model.layers.49.mlp.experts.142.up_proj", + "model.layers.49.mlp.experts.143.down_proj", + "model.layers.49.mlp.experts.143.gate_proj", + "model.layers.49.mlp.experts.143.up_proj", + "model.layers.49.mlp.experts.144.down_proj", + "model.layers.49.mlp.experts.144.gate_proj", + "model.layers.49.mlp.experts.144.up_proj", + "model.layers.49.mlp.experts.145.down_proj", + "model.layers.49.mlp.experts.145.gate_proj", + "model.layers.49.mlp.experts.145.up_proj", + "model.layers.49.mlp.experts.146.down_proj", + "model.layers.49.mlp.experts.146.gate_proj", + "model.layers.49.mlp.experts.146.up_proj", + "model.layers.49.mlp.experts.147.down_proj", + "model.layers.49.mlp.experts.147.gate_proj", + "model.layers.49.mlp.experts.147.up_proj", + "model.layers.49.mlp.experts.148.down_proj", + "model.layers.49.mlp.experts.148.gate_proj", + "model.layers.49.mlp.experts.148.up_proj", + "model.layers.49.mlp.experts.149.down_proj", + "model.layers.49.mlp.experts.149.gate_proj", + "model.layers.49.mlp.experts.149.up_proj", + "model.layers.49.mlp.experts.15.down_proj", + "model.layers.49.mlp.experts.15.gate_proj", + "model.layers.49.mlp.experts.15.up_proj", + "model.layers.49.mlp.experts.150.down_proj", + "model.layers.49.mlp.experts.150.gate_proj", + "model.layers.49.mlp.experts.150.up_proj", + "model.layers.49.mlp.experts.151.down_proj", + "model.layers.49.mlp.experts.151.gate_proj", + "model.layers.49.mlp.experts.151.up_proj", + "model.layers.49.mlp.experts.152.down_proj", + "model.layers.49.mlp.experts.152.gate_proj", + "model.layers.49.mlp.experts.152.up_proj", + "model.layers.49.mlp.experts.153.down_proj", + "model.layers.49.mlp.experts.153.gate_proj", + "model.layers.49.mlp.experts.153.up_proj", + "model.layers.49.mlp.experts.154.down_proj", + "model.layers.49.mlp.experts.154.gate_proj", + "model.layers.49.mlp.experts.154.up_proj", + "model.layers.49.mlp.experts.155.down_proj", + "model.layers.49.mlp.experts.155.gate_proj", + "model.layers.49.mlp.experts.155.up_proj", + "model.layers.49.mlp.experts.156.down_proj", + "model.layers.49.mlp.experts.156.gate_proj", + "model.layers.49.mlp.experts.156.up_proj", + "model.layers.49.mlp.experts.157.down_proj", + "model.layers.49.mlp.experts.157.gate_proj", + "model.layers.49.mlp.experts.157.up_proj", + "model.layers.49.mlp.experts.158.down_proj", + "model.layers.49.mlp.experts.158.gate_proj", + "model.layers.49.mlp.experts.158.up_proj", + "model.layers.49.mlp.experts.159.down_proj", + "model.layers.49.mlp.experts.159.gate_proj", + "model.layers.49.mlp.experts.159.up_proj", + "model.layers.49.mlp.experts.16.down_proj", + "model.layers.49.mlp.experts.16.gate_proj", + "model.layers.49.mlp.experts.16.up_proj", + "model.layers.49.mlp.experts.160.down_proj", + "model.layers.49.mlp.experts.160.gate_proj", + "model.layers.49.mlp.experts.160.up_proj", + "model.layers.49.mlp.experts.161.down_proj", + "model.layers.49.mlp.experts.161.gate_proj", + "model.layers.49.mlp.experts.161.up_proj", + "model.layers.49.mlp.experts.162.down_proj", + "model.layers.49.mlp.experts.162.gate_proj", + "model.layers.49.mlp.experts.162.up_proj", + "model.layers.49.mlp.experts.163.down_proj", + "model.layers.49.mlp.experts.163.gate_proj", + "model.layers.49.mlp.experts.163.up_proj", + "model.layers.49.mlp.experts.164.down_proj", + "model.layers.49.mlp.experts.164.gate_proj", + "model.layers.49.mlp.experts.164.up_proj", + "model.layers.49.mlp.experts.165.down_proj", + "model.layers.49.mlp.experts.165.gate_proj", + "model.layers.49.mlp.experts.165.up_proj", + "model.layers.49.mlp.experts.166.down_proj", + "model.layers.49.mlp.experts.166.gate_proj", + "model.layers.49.mlp.experts.166.up_proj", + "model.layers.49.mlp.experts.167.down_proj", + "model.layers.49.mlp.experts.167.gate_proj", + "model.layers.49.mlp.experts.167.up_proj", + "model.layers.49.mlp.experts.168.down_proj", + "model.layers.49.mlp.experts.168.gate_proj", + "model.layers.49.mlp.experts.168.up_proj", + "model.layers.49.mlp.experts.169.down_proj", + "model.layers.49.mlp.experts.169.gate_proj", + "model.layers.49.mlp.experts.169.up_proj", + "model.layers.49.mlp.experts.17.down_proj", + "model.layers.49.mlp.experts.17.gate_proj", + "model.layers.49.mlp.experts.17.up_proj", + "model.layers.49.mlp.experts.170.down_proj", + "model.layers.49.mlp.experts.170.gate_proj", + "model.layers.49.mlp.experts.170.up_proj", + "model.layers.49.mlp.experts.171.down_proj", + "model.layers.49.mlp.experts.171.gate_proj", + "model.layers.49.mlp.experts.171.up_proj", + "model.layers.49.mlp.experts.172.down_proj", + "model.layers.49.mlp.experts.172.gate_proj", + "model.layers.49.mlp.experts.172.up_proj", + "model.layers.49.mlp.experts.173.down_proj", + "model.layers.49.mlp.experts.173.gate_proj", + "model.layers.49.mlp.experts.173.up_proj", + "model.layers.49.mlp.experts.174.down_proj", + "model.layers.49.mlp.experts.174.gate_proj", + "model.layers.49.mlp.experts.174.up_proj", + "model.layers.49.mlp.experts.175.down_proj", + "model.layers.49.mlp.experts.175.gate_proj", + "model.layers.49.mlp.experts.175.up_proj", + "model.layers.49.mlp.experts.176.down_proj", + "model.layers.49.mlp.experts.176.gate_proj", + "model.layers.49.mlp.experts.176.up_proj", + "model.layers.49.mlp.experts.177.down_proj", + "model.layers.49.mlp.experts.177.gate_proj", + "model.layers.49.mlp.experts.177.up_proj", + "model.layers.49.mlp.experts.178.down_proj", + "model.layers.49.mlp.experts.178.gate_proj", + "model.layers.49.mlp.experts.178.up_proj", + "model.layers.49.mlp.experts.179.down_proj", + "model.layers.49.mlp.experts.179.gate_proj", + "model.layers.49.mlp.experts.179.up_proj", + "model.layers.49.mlp.experts.18.down_proj", + "model.layers.49.mlp.experts.18.gate_proj", + "model.layers.49.mlp.experts.18.up_proj", + "model.layers.49.mlp.experts.180.down_proj", + "model.layers.49.mlp.experts.180.gate_proj", + "model.layers.49.mlp.experts.180.up_proj", + "model.layers.49.mlp.experts.181.down_proj", + "model.layers.49.mlp.experts.181.gate_proj", + "model.layers.49.mlp.experts.181.up_proj", + "model.layers.49.mlp.experts.182.down_proj", + "model.layers.49.mlp.experts.182.gate_proj", + "model.layers.49.mlp.experts.182.up_proj", + "model.layers.49.mlp.experts.183.down_proj", + "model.layers.49.mlp.experts.183.gate_proj", + "model.layers.49.mlp.experts.183.up_proj", + "model.layers.49.mlp.experts.184.down_proj", + "model.layers.49.mlp.experts.184.gate_proj", + "model.layers.49.mlp.experts.184.up_proj", + "model.layers.49.mlp.experts.185.down_proj", + "model.layers.49.mlp.experts.185.gate_proj", + "model.layers.49.mlp.experts.185.up_proj", + "model.layers.49.mlp.experts.186.down_proj", + "model.layers.49.mlp.experts.186.gate_proj", + "model.layers.49.mlp.experts.186.up_proj", + "model.layers.49.mlp.experts.187.down_proj", + "model.layers.49.mlp.experts.187.gate_proj", + "model.layers.49.mlp.experts.187.up_proj", + "model.layers.49.mlp.experts.188.down_proj", + "model.layers.49.mlp.experts.188.gate_proj", + "model.layers.49.mlp.experts.188.up_proj", + "model.layers.49.mlp.experts.189.down_proj", + "model.layers.49.mlp.experts.189.gate_proj", + "model.layers.49.mlp.experts.189.up_proj", + "model.layers.49.mlp.experts.19.down_proj", + "model.layers.49.mlp.experts.19.gate_proj", + "model.layers.49.mlp.experts.19.up_proj", + "model.layers.49.mlp.experts.190.down_proj", + "model.layers.49.mlp.experts.190.gate_proj", + "model.layers.49.mlp.experts.190.up_proj", + "model.layers.49.mlp.experts.191.down_proj", + "model.layers.49.mlp.experts.191.gate_proj", + "model.layers.49.mlp.experts.191.up_proj", + "model.layers.49.mlp.experts.192.down_proj", + "model.layers.49.mlp.experts.192.gate_proj", + "model.layers.49.mlp.experts.192.up_proj", + "model.layers.49.mlp.experts.193.down_proj", + "model.layers.49.mlp.experts.193.gate_proj", + "model.layers.49.mlp.experts.193.up_proj", + "model.layers.49.mlp.experts.194.down_proj", + "model.layers.49.mlp.experts.194.gate_proj", + "model.layers.49.mlp.experts.194.up_proj", + "model.layers.49.mlp.experts.195.down_proj", + "model.layers.49.mlp.experts.195.gate_proj", + "model.layers.49.mlp.experts.195.up_proj", + "model.layers.49.mlp.experts.196.down_proj", + "model.layers.49.mlp.experts.196.gate_proj", + "model.layers.49.mlp.experts.196.up_proj", + "model.layers.49.mlp.experts.197.down_proj", + "model.layers.49.mlp.experts.197.gate_proj", + "model.layers.49.mlp.experts.197.up_proj", + "model.layers.49.mlp.experts.198.down_proj", + "model.layers.49.mlp.experts.198.gate_proj", + "model.layers.49.mlp.experts.198.up_proj", + "model.layers.49.mlp.experts.199.down_proj", + "model.layers.49.mlp.experts.199.gate_proj", + "model.layers.49.mlp.experts.199.up_proj", + "model.layers.49.mlp.experts.2.down_proj", + "model.layers.49.mlp.experts.2.gate_proj", + "model.layers.49.mlp.experts.2.up_proj", + "model.layers.49.mlp.experts.20.down_proj", + "model.layers.49.mlp.experts.20.gate_proj", + "model.layers.49.mlp.experts.20.up_proj", + "model.layers.49.mlp.experts.200.down_proj", + "model.layers.49.mlp.experts.200.gate_proj", + "model.layers.49.mlp.experts.200.up_proj", + "model.layers.49.mlp.experts.201.down_proj", + "model.layers.49.mlp.experts.201.gate_proj", + "model.layers.49.mlp.experts.201.up_proj", + "model.layers.49.mlp.experts.202.down_proj", + "model.layers.49.mlp.experts.202.gate_proj", + "model.layers.49.mlp.experts.202.up_proj", + "model.layers.49.mlp.experts.203.down_proj", + "model.layers.49.mlp.experts.203.gate_proj", + "model.layers.49.mlp.experts.203.up_proj", + "model.layers.49.mlp.experts.204.down_proj", + "model.layers.49.mlp.experts.204.gate_proj", + "model.layers.49.mlp.experts.204.up_proj", + "model.layers.49.mlp.experts.205.down_proj", + "model.layers.49.mlp.experts.205.gate_proj", + "model.layers.49.mlp.experts.205.up_proj", + "model.layers.49.mlp.experts.206.down_proj", + "model.layers.49.mlp.experts.206.gate_proj", + "model.layers.49.mlp.experts.206.up_proj", + "model.layers.49.mlp.experts.207.down_proj", + "model.layers.49.mlp.experts.207.gate_proj", + "model.layers.49.mlp.experts.207.up_proj", + "model.layers.49.mlp.experts.208.down_proj", + "model.layers.49.mlp.experts.208.gate_proj", + "model.layers.49.mlp.experts.208.up_proj", + "model.layers.49.mlp.experts.209.down_proj", + "model.layers.49.mlp.experts.209.gate_proj", + "model.layers.49.mlp.experts.209.up_proj", + "model.layers.49.mlp.experts.21.down_proj", + "model.layers.49.mlp.experts.21.gate_proj", + "model.layers.49.mlp.experts.21.up_proj", + "model.layers.49.mlp.experts.210.down_proj", + "model.layers.49.mlp.experts.210.gate_proj", + "model.layers.49.mlp.experts.210.up_proj", + "model.layers.49.mlp.experts.211.down_proj", + "model.layers.49.mlp.experts.211.gate_proj", + "model.layers.49.mlp.experts.211.up_proj", + "model.layers.49.mlp.experts.212.down_proj", + "model.layers.49.mlp.experts.212.gate_proj", + "model.layers.49.mlp.experts.212.up_proj", + "model.layers.49.mlp.experts.213.down_proj", + "model.layers.49.mlp.experts.213.gate_proj", + "model.layers.49.mlp.experts.213.up_proj", + "model.layers.49.mlp.experts.214.down_proj", + "model.layers.49.mlp.experts.214.gate_proj", + "model.layers.49.mlp.experts.214.up_proj", + "model.layers.49.mlp.experts.215.down_proj", + "model.layers.49.mlp.experts.215.gate_proj", + "model.layers.49.mlp.experts.215.up_proj", + "model.layers.49.mlp.experts.216.down_proj", + "model.layers.49.mlp.experts.216.gate_proj", + "model.layers.49.mlp.experts.216.up_proj", + "model.layers.49.mlp.experts.217.down_proj", + "model.layers.49.mlp.experts.217.gate_proj", + "model.layers.49.mlp.experts.217.up_proj", + "model.layers.49.mlp.experts.218.down_proj", + "model.layers.49.mlp.experts.218.gate_proj", + "model.layers.49.mlp.experts.218.up_proj", + "model.layers.49.mlp.experts.219.down_proj", + "model.layers.49.mlp.experts.219.gate_proj", + "model.layers.49.mlp.experts.219.up_proj", + "model.layers.49.mlp.experts.22.down_proj", + "model.layers.49.mlp.experts.22.gate_proj", + "model.layers.49.mlp.experts.22.up_proj", + "model.layers.49.mlp.experts.220.down_proj", + "model.layers.49.mlp.experts.220.gate_proj", + "model.layers.49.mlp.experts.220.up_proj", + "model.layers.49.mlp.experts.221.down_proj", + "model.layers.49.mlp.experts.221.gate_proj", + "model.layers.49.mlp.experts.221.up_proj", + "model.layers.49.mlp.experts.222.down_proj", + "model.layers.49.mlp.experts.222.gate_proj", + "model.layers.49.mlp.experts.222.up_proj", + "model.layers.49.mlp.experts.223.down_proj", + "model.layers.49.mlp.experts.223.gate_proj", + "model.layers.49.mlp.experts.223.up_proj", + "model.layers.49.mlp.experts.224.down_proj", + "model.layers.49.mlp.experts.224.gate_proj", + "model.layers.49.mlp.experts.224.up_proj", + "model.layers.49.mlp.experts.225.down_proj", + "model.layers.49.mlp.experts.225.gate_proj", + "model.layers.49.mlp.experts.225.up_proj", + "model.layers.49.mlp.experts.226.down_proj", + "model.layers.49.mlp.experts.226.gate_proj", + "model.layers.49.mlp.experts.226.up_proj", + "model.layers.49.mlp.experts.227.down_proj", + "model.layers.49.mlp.experts.227.gate_proj", + "model.layers.49.mlp.experts.227.up_proj", + "model.layers.49.mlp.experts.228.down_proj", + "model.layers.49.mlp.experts.228.gate_proj", + "model.layers.49.mlp.experts.228.up_proj", + "model.layers.49.mlp.experts.229.down_proj", + "model.layers.49.mlp.experts.229.gate_proj", + "model.layers.49.mlp.experts.229.up_proj", + "model.layers.49.mlp.experts.23.down_proj", + "model.layers.49.mlp.experts.23.gate_proj", + "model.layers.49.mlp.experts.23.up_proj", + "model.layers.49.mlp.experts.230.down_proj", + "model.layers.49.mlp.experts.230.gate_proj", + "model.layers.49.mlp.experts.230.up_proj", + "model.layers.49.mlp.experts.231.down_proj", + "model.layers.49.mlp.experts.231.gate_proj", + "model.layers.49.mlp.experts.231.up_proj", + "model.layers.49.mlp.experts.232.down_proj", + "model.layers.49.mlp.experts.232.gate_proj", + "model.layers.49.mlp.experts.232.up_proj", + "model.layers.49.mlp.experts.233.down_proj", + "model.layers.49.mlp.experts.233.gate_proj", + "model.layers.49.mlp.experts.233.up_proj", + "model.layers.49.mlp.experts.234.down_proj", + "model.layers.49.mlp.experts.234.gate_proj", + "model.layers.49.mlp.experts.234.up_proj", + "model.layers.49.mlp.experts.235.down_proj", + "model.layers.49.mlp.experts.235.gate_proj", + "model.layers.49.mlp.experts.235.up_proj", + "model.layers.49.mlp.experts.236.down_proj", + "model.layers.49.mlp.experts.236.gate_proj", + "model.layers.49.mlp.experts.236.up_proj", + "model.layers.49.mlp.experts.237.down_proj", + "model.layers.49.mlp.experts.237.gate_proj", + "model.layers.49.mlp.experts.237.up_proj", + "model.layers.49.mlp.experts.238.down_proj", + "model.layers.49.mlp.experts.238.gate_proj", + "model.layers.49.mlp.experts.238.up_proj", + "model.layers.49.mlp.experts.239.down_proj", + "model.layers.49.mlp.experts.239.gate_proj", + "model.layers.49.mlp.experts.239.up_proj", + "model.layers.49.mlp.experts.24.down_proj", + "model.layers.49.mlp.experts.24.gate_proj", + "model.layers.49.mlp.experts.24.up_proj", + "model.layers.49.mlp.experts.240.down_proj", + "model.layers.49.mlp.experts.240.gate_proj", + "model.layers.49.mlp.experts.240.up_proj", + "model.layers.49.mlp.experts.241.down_proj", + "model.layers.49.mlp.experts.241.gate_proj", + "model.layers.49.mlp.experts.241.up_proj", + "model.layers.49.mlp.experts.242.down_proj", + "model.layers.49.mlp.experts.242.gate_proj", + "model.layers.49.mlp.experts.242.up_proj", + "model.layers.49.mlp.experts.243.down_proj", + "model.layers.49.mlp.experts.243.gate_proj", + "model.layers.49.mlp.experts.243.up_proj", + "model.layers.49.mlp.experts.244.down_proj", + "model.layers.49.mlp.experts.244.gate_proj", + "model.layers.49.mlp.experts.244.up_proj", + "model.layers.49.mlp.experts.245.down_proj", + "model.layers.49.mlp.experts.245.gate_proj", + "model.layers.49.mlp.experts.245.up_proj", + "model.layers.49.mlp.experts.246.down_proj", + "model.layers.49.mlp.experts.246.gate_proj", + "model.layers.49.mlp.experts.246.up_proj", + "model.layers.49.mlp.experts.247.down_proj", + "model.layers.49.mlp.experts.247.gate_proj", + "model.layers.49.mlp.experts.247.up_proj", + "model.layers.49.mlp.experts.248.down_proj", + "model.layers.49.mlp.experts.248.gate_proj", + "model.layers.49.mlp.experts.248.up_proj", + "model.layers.49.mlp.experts.249.down_proj", + "model.layers.49.mlp.experts.249.gate_proj", + "model.layers.49.mlp.experts.249.up_proj", + "model.layers.49.mlp.experts.25.down_proj", + "model.layers.49.mlp.experts.25.gate_proj", + "model.layers.49.mlp.experts.25.up_proj", + "model.layers.49.mlp.experts.250.down_proj", + "model.layers.49.mlp.experts.250.gate_proj", + "model.layers.49.mlp.experts.250.up_proj", + "model.layers.49.mlp.experts.251.down_proj", + "model.layers.49.mlp.experts.251.gate_proj", + "model.layers.49.mlp.experts.251.up_proj", + "model.layers.49.mlp.experts.252.down_proj", + "model.layers.49.mlp.experts.252.gate_proj", + "model.layers.49.mlp.experts.252.up_proj", + "model.layers.49.mlp.experts.253.down_proj", + "model.layers.49.mlp.experts.253.gate_proj", + "model.layers.49.mlp.experts.253.up_proj", + "model.layers.49.mlp.experts.254.down_proj", + "model.layers.49.mlp.experts.254.gate_proj", + "model.layers.49.mlp.experts.254.up_proj", + "model.layers.49.mlp.experts.255.down_proj", + "model.layers.49.mlp.experts.255.gate_proj", + "model.layers.49.mlp.experts.255.up_proj", + "model.layers.49.mlp.experts.26.down_proj", + "model.layers.49.mlp.experts.26.gate_proj", + "model.layers.49.mlp.experts.26.up_proj", + "model.layers.49.mlp.experts.27.down_proj", + "model.layers.49.mlp.experts.27.gate_proj", + "model.layers.49.mlp.experts.27.up_proj", + "model.layers.49.mlp.experts.28.down_proj", + "model.layers.49.mlp.experts.28.gate_proj", + "model.layers.49.mlp.experts.28.up_proj", + "model.layers.49.mlp.experts.29.down_proj", + "model.layers.49.mlp.experts.29.gate_proj", + "model.layers.49.mlp.experts.29.up_proj", + "model.layers.49.mlp.experts.3.down_proj", + "model.layers.49.mlp.experts.3.gate_proj", + "model.layers.49.mlp.experts.3.up_proj", + "model.layers.49.mlp.experts.30.down_proj", + "model.layers.49.mlp.experts.30.gate_proj", + "model.layers.49.mlp.experts.30.up_proj", + "model.layers.49.mlp.experts.31.down_proj", + "model.layers.49.mlp.experts.31.gate_proj", + "model.layers.49.mlp.experts.31.up_proj", + "model.layers.49.mlp.experts.32.down_proj", + "model.layers.49.mlp.experts.32.gate_proj", + "model.layers.49.mlp.experts.32.up_proj", + "model.layers.49.mlp.experts.33.down_proj", + "model.layers.49.mlp.experts.33.gate_proj", + "model.layers.49.mlp.experts.33.up_proj", + "model.layers.49.mlp.experts.34.down_proj", + "model.layers.49.mlp.experts.34.gate_proj", + "model.layers.49.mlp.experts.34.up_proj", + "model.layers.49.mlp.experts.35.down_proj", + "model.layers.49.mlp.experts.35.gate_proj", + "model.layers.49.mlp.experts.35.up_proj", + "model.layers.49.mlp.experts.36.down_proj", + "model.layers.49.mlp.experts.36.gate_proj", + "model.layers.49.mlp.experts.36.up_proj", + "model.layers.49.mlp.experts.37.down_proj", + "model.layers.49.mlp.experts.37.gate_proj", + "model.layers.49.mlp.experts.37.up_proj", + "model.layers.49.mlp.experts.38.down_proj", + "model.layers.49.mlp.experts.38.gate_proj", + "model.layers.49.mlp.experts.38.up_proj", + "model.layers.49.mlp.experts.39.down_proj", + "model.layers.49.mlp.experts.39.gate_proj", + "model.layers.49.mlp.experts.39.up_proj", + "model.layers.49.mlp.experts.4.down_proj", + "model.layers.49.mlp.experts.4.gate_proj", + "model.layers.49.mlp.experts.4.up_proj", + "model.layers.49.mlp.experts.40.down_proj", + "model.layers.49.mlp.experts.40.gate_proj", + "model.layers.49.mlp.experts.40.up_proj", + "model.layers.49.mlp.experts.41.down_proj", + "model.layers.49.mlp.experts.41.gate_proj", + "model.layers.49.mlp.experts.41.up_proj", + "model.layers.49.mlp.experts.42.down_proj", + "model.layers.49.mlp.experts.42.gate_proj", + "model.layers.49.mlp.experts.42.up_proj", + "model.layers.49.mlp.experts.43.down_proj", + "model.layers.49.mlp.experts.43.gate_proj", + "model.layers.49.mlp.experts.43.up_proj", + "model.layers.49.mlp.experts.44.down_proj", + "model.layers.49.mlp.experts.44.gate_proj", + "model.layers.49.mlp.experts.44.up_proj", + "model.layers.49.mlp.experts.45.down_proj", + "model.layers.49.mlp.experts.45.gate_proj", + "model.layers.49.mlp.experts.45.up_proj", + "model.layers.49.mlp.experts.46.down_proj", + "model.layers.49.mlp.experts.46.gate_proj", + "model.layers.49.mlp.experts.46.up_proj", + "model.layers.49.mlp.experts.47.down_proj", + "model.layers.49.mlp.experts.47.gate_proj", + "model.layers.49.mlp.experts.47.up_proj", + "model.layers.49.mlp.experts.48.down_proj", + "model.layers.49.mlp.experts.48.gate_proj", + "model.layers.49.mlp.experts.48.up_proj", + "model.layers.49.mlp.experts.49.down_proj", + "model.layers.49.mlp.experts.49.gate_proj", + "model.layers.49.mlp.experts.49.up_proj", + "model.layers.49.mlp.experts.5.down_proj", + "model.layers.49.mlp.experts.5.gate_proj", + "model.layers.49.mlp.experts.5.up_proj", + "model.layers.49.mlp.experts.50.down_proj", + "model.layers.49.mlp.experts.50.gate_proj", + "model.layers.49.mlp.experts.50.up_proj", + "model.layers.49.mlp.experts.51.down_proj", + "model.layers.49.mlp.experts.51.gate_proj", + "model.layers.49.mlp.experts.51.up_proj", + "model.layers.49.mlp.experts.52.down_proj", + "model.layers.49.mlp.experts.52.gate_proj", + "model.layers.49.mlp.experts.52.up_proj", + "model.layers.49.mlp.experts.53.down_proj", + "model.layers.49.mlp.experts.53.gate_proj", + "model.layers.49.mlp.experts.53.up_proj", + "model.layers.49.mlp.experts.54.down_proj", + "model.layers.49.mlp.experts.54.gate_proj", + "model.layers.49.mlp.experts.54.up_proj", + "model.layers.49.mlp.experts.55.down_proj", + "model.layers.49.mlp.experts.55.gate_proj", + "model.layers.49.mlp.experts.55.up_proj", + "model.layers.49.mlp.experts.56.down_proj", + "model.layers.49.mlp.experts.56.gate_proj", + "model.layers.49.mlp.experts.56.up_proj", + "model.layers.49.mlp.experts.57.down_proj", + "model.layers.49.mlp.experts.57.gate_proj", + "model.layers.49.mlp.experts.57.up_proj", + "model.layers.49.mlp.experts.58.down_proj", + "model.layers.49.mlp.experts.58.gate_proj", + "model.layers.49.mlp.experts.58.up_proj", + "model.layers.49.mlp.experts.59.down_proj", + "model.layers.49.mlp.experts.59.gate_proj", + "model.layers.49.mlp.experts.59.up_proj", + "model.layers.49.mlp.experts.6.down_proj", + "model.layers.49.mlp.experts.6.gate_proj", + "model.layers.49.mlp.experts.6.up_proj", + "model.layers.49.mlp.experts.60.down_proj", + "model.layers.49.mlp.experts.60.gate_proj", + "model.layers.49.mlp.experts.60.up_proj", + "model.layers.49.mlp.experts.61.down_proj", + "model.layers.49.mlp.experts.61.gate_proj", + "model.layers.49.mlp.experts.61.up_proj", + "model.layers.49.mlp.experts.62.down_proj", + "model.layers.49.mlp.experts.62.gate_proj", + "model.layers.49.mlp.experts.62.up_proj", + "model.layers.49.mlp.experts.63.down_proj", + "model.layers.49.mlp.experts.63.gate_proj", + "model.layers.49.mlp.experts.63.up_proj", + "model.layers.49.mlp.experts.64.down_proj", + "model.layers.49.mlp.experts.64.gate_proj", + "model.layers.49.mlp.experts.64.up_proj", + "model.layers.49.mlp.experts.65.down_proj", + "model.layers.49.mlp.experts.65.gate_proj", + "model.layers.49.mlp.experts.65.up_proj", + "model.layers.49.mlp.experts.66.down_proj", + "model.layers.49.mlp.experts.66.gate_proj", + "model.layers.49.mlp.experts.66.up_proj", + "model.layers.49.mlp.experts.67.down_proj", + "model.layers.49.mlp.experts.67.gate_proj", + "model.layers.49.mlp.experts.67.up_proj", + "model.layers.49.mlp.experts.68.down_proj", + "model.layers.49.mlp.experts.68.gate_proj", + "model.layers.49.mlp.experts.68.up_proj", + "model.layers.49.mlp.experts.69.down_proj", + "model.layers.49.mlp.experts.69.gate_proj", + "model.layers.49.mlp.experts.69.up_proj", + "model.layers.49.mlp.experts.7.down_proj", + "model.layers.49.mlp.experts.7.gate_proj", + "model.layers.49.mlp.experts.7.up_proj", + "model.layers.49.mlp.experts.70.down_proj", + "model.layers.49.mlp.experts.70.gate_proj", + "model.layers.49.mlp.experts.70.up_proj", + "model.layers.49.mlp.experts.71.down_proj", + "model.layers.49.mlp.experts.71.gate_proj", + "model.layers.49.mlp.experts.71.up_proj", + "model.layers.49.mlp.experts.72.down_proj", + "model.layers.49.mlp.experts.72.gate_proj", + "model.layers.49.mlp.experts.72.up_proj", + "model.layers.49.mlp.experts.73.down_proj", + "model.layers.49.mlp.experts.73.gate_proj", + "model.layers.49.mlp.experts.73.up_proj", + "model.layers.49.mlp.experts.74.down_proj", + "model.layers.49.mlp.experts.74.gate_proj", + "model.layers.49.mlp.experts.74.up_proj", + "model.layers.49.mlp.experts.75.down_proj", + "model.layers.49.mlp.experts.75.gate_proj", + "model.layers.49.mlp.experts.75.up_proj", + "model.layers.49.mlp.experts.76.down_proj", + "model.layers.49.mlp.experts.76.gate_proj", + "model.layers.49.mlp.experts.76.up_proj", + "model.layers.49.mlp.experts.77.down_proj", + "model.layers.49.mlp.experts.77.gate_proj", + "model.layers.49.mlp.experts.77.up_proj", + "model.layers.49.mlp.experts.78.down_proj", + "model.layers.49.mlp.experts.78.gate_proj", + "model.layers.49.mlp.experts.78.up_proj", + "model.layers.49.mlp.experts.79.down_proj", + "model.layers.49.mlp.experts.79.gate_proj", + "model.layers.49.mlp.experts.79.up_proj", + "model.layers.49.mlp.experts.8.down_proj", + "model.layers.49.mlp.experts.8.gate_proj", + "model.layers.49.mlp.experts.8.up_proj", + "model.layers.49.mlp.experts.80.down_proj", + "model.layers.49.mlp.experts.80.gate_proj", + "model.layers.49.mlp.experts.80.up_proj", + "model.layers.49.mlp.experts.81.down_proj", + "model.layers.49.mlp.experts.81.gate_proj", + "model.layers.49.mlp.experts.81.up_proj", + "model.layers.49.mlp.experts.82.down_proj", + "model.layers.49.mlp.experts.82.gate_proj", + "model.layers.49.mlp.experts.82.up_proj", + "model.layers.49.mlp.experts.83.down_proj", + "model.layers.49.mlp.experts.83.gate_proj", + "model.layers.49.mlp.experts.83.up_proj", + "model.layers.49.mlp.experts.84.down_proj", + "model.layers.49.mlp.experts.84.gate_proj", + "model.layers.49.mlp.experts.84.up_proj", + "model.layers.49.mlp.experts.85.down_proj", + "model.layers.49.mlp.experts.85.gate_proj", + "model.layers.49.mlp.experts.85.up_proj", + "model.layers.49.mlp.experts.86.down_proj", + "model.layers.49.mlp.experts.86.gate_proj", + "model.layers.49.mlp.experts.86.up_proj", + "model.layers.49.mlp.experts.87.down_proj", + "model.layers.49.mlp.experts.87.gate_proj", + "model.layers.49.mlp.experts.87.up_proj", + "model.layers.49.mlp.experts.88.down_proj", + "model.layers.49.mlp.experts.88.gate_proj", + "model.layers.49.mlp.experts.88.up_proj", + "model.layers.49.mlp.experts.89.down_proj", + "model.layers.49.mlp.experts.89.gate_proj", + "model.layers.49.mlp.experts.89.up_proj", + "model.layers.49.mlp.experts.9.down_proj", + "model.layers.49.mlp.experts.9.gate_proj", + "model.layers.49.mlp.experts.9.up_proj", + "model.layers.49.mlp.experts.90.down_proj", + "model.layers.49.mlp.experts.90.gate_proj", + "model.layers.49.mlp.experts.90.up_proj", + "model.layers.49.mlp.experts.91.down_proj", + "model.layers.49.mlp.experts.91.gate_proj", + "model.layers.49.mlp.experts.91.up_proj", + "model.layers.49.mlp.experts.92.down_proj", + "model.layers.49.mlp.experts.92.gate_proj", + "model.layers.49.mlp.experts.92.up_proj", + "model.layers.49.mlp.experts.93.down_proj", + "model.layers.49.mlp.experts.93.gate_proj", + "model.layers.49.mlp.experts.93.up_proj", + "model.layers.49.mlp.experts.94.down_proj", + "model.layers.49.mlp.experts.94.gate_proj", + "model.layers.49.mlp.experts.94.up_proj", + "model.layers.49.mlp.experts.95.down_proj", + "model.layers.49.mlp.experts.95.gate_proj", + "model.layers.49.mlp.experts.95.up_proj", + "model.layers.49.mlp.experts.96.down_proj", + "model.layers.49.mlp.experts.96.gate_proj", + "model.layers.49.mlp.experts.96.up_proj", + "model.layers.49.mlp.experts.97.down_proj", + "model.layers.49.mlp.experts.97.gate_proj", + "model.layers.49.mlp.experts.97.up_proj", + "model.layers.49.mlp.experts.98.down_proj", + "model.layers.49.mlp.experts.98.gate_proj", + "model.layers.49.mlp.experts.98.up_proj", + "model.layers.49.mlp.experts.99.down_proj", + "model.layers.49.mlp.experts.99.gate_proj", + "model.layers.49.mlp.experts.99.up_proj", + "model.layers.5.mlp.experts.0.down_proj", + "model.layers.5.mlp.experts.0.gate_proj", + "model.layers.5.mlp.experts.0.up_proj", + "model.layers.5.mlp.experts.1.down_proj", + "model.layers.5.mlp.experts.1.gate_proj", + "model.layers.5.mlp.experts.1.up_proj", + "model.layers.5.mlp.experts.10.down_proj", + "model.layers.5.mlp.experts.10.gate_proj", + "model.layers.5.mlp.experts.10.up_proj", + "model.layers.5.mlp.experts.100.down_proj", + "model.layers.5.mlp.experts.100.gate_proj", + "model.layers.5.mlp.experts.100.up_proj", + "model.layers.5.mlp.experts.101.down_proj", + "model.layers.5.mlp.experts.101.gate_proj", + "model.layers.5.mlp.experts.101.up_proj", + "model.layers.5.mlp.experts.102.down_proj", + "model.layers.5.mlp.experts.102.gate_proj", + "model.layers.5.mlp.experts.102.up_proj", + "model.layers.5.mlp.experts.103.down_proj", + "model.layers.5.mlp.experts.103.gate_proj", + "model.layers.5.mlp.experts.103.up_proj", + "model.layers.5.mlp.experts.104.down_proj", + "model.layers.5.mlp.experts.104.gate_proj", + "model.layers.5.mlp.experts.104.up_proj", + "model.layers.5.mlp.experts.105.down_proj", + "model.layers.5.mlp.experts.105.gate_proj", + "model.layers.5.mlp.experts.105.up_proj", + "model.layers.5.mlp.experts.106.down_proj", + "model.layers.5.mlp.experts.106.gate_proj", + "model.layers.5.mlp.experts.106.up_proj", + "model.layers.5.mlp.experts.107.down_proj", + "model.layers.5.mlp.experts.107.gate_proj", + "model.layers.5.mlp.experts.107.up_proj", + "model.layers.5.mlp.experts.108.down_proj", + "model.layers.5.mlp.experts.108.gate_proj", + "model.layers.5.mlp.experts.108.up_proj", + "model.layers.5.mlp.experts.109.down_proj", + "model.layers.5.mlp.experts.109.gate_proj", + "model.layers.5.mlp.experts.109.up_proj", + "model.layers.5.mlp.experts.11.down_proj", + "model.layers.5.mlp.experts.11.gate_proj", + "model.layers.5.mlp.experts.11.up_proj", + "model.layers.5.mlp.experts.110.down_proj", + "model.layers.5.mlp.experts.110.gate_proj", + "model.layers.5.mlp.experts.110.up_proj", + "model.layers.5.mlp.experts.111.down_proj", + "model.layers.5.mlp.experts.111.gate_proj", + "model.layers.5.mlp.experts.111.up_proj", + "model.layers.5.mlp.experts.112.down_proj", + "model.layers.5.mlp.experts.112.gate_proj", + "model.layers.5.mlp.experts.112.up_proj", + "model.layers.5.mlp.experts.113.down_proj", + "model.layers.5.mlp.experts.113.gate_proj", + "model.layers.5.mlp.experts.113.up_proj", + "model.layers.5.mlp.experts.114.down_proj", + "model.layers.5.mlp.experts.114.gate_proj", + "model.layers.5.mlp.experts.114.up_proj", + "model.layers.5.mlp.experts.115.down_proj", + "model.layers.5.mlp.experts.115.gate_proj", + "model.layers.5.mlp.experts.115.up_proj", + "model.layers.5.mlp.experts.116.down_proj", + "model.layers.5.mlp.experts.116.gate_proj", + "model.layers.5.mlp.experts.116.up_proj", + "model.layers.5.mlp.experts.117.down_proj", + "model.layers.5.mlp.experts.117.gate_proj", + "model.layers.5.mlp.experts.117.up_proj", + "model.layers.5.mlp.experts.118.down_proj", + "model.layers.5.mlp.experts.118.gate_proj", + "model.layers.5.mlp.experts.118.up_proj", + "model.layers.5.mlp.experts.119.down_proj", + "model.layers.5.mlp.experts.119.gate_proj", + "model.layers.5.mlp.experts.119.up_proj", + "model.layers.5.mlp.experts.12.down_proj", + "model.layers.5.mlp.experts.12.gate_proj", + "model.layers.5.mlp.experts.12.up_proj", + "model.layers.5.mlp.experts.120.down_proj", + "model.layers.5.mlp.experts.120.gate_proj", + "model.layers.5.mlp.experts.120.up_proj", + "model.layers.5.mlp.experts.121.down_proj", + "model.layers.5.mlp.experts.121.gate_proj", + "model.layers.5.mlp.experts.121.up_proj", + "model.layers.5.mlp.experts.122.down_proj", + "model.layers.5.mlp.experts.122.gate_proj", + "model.layers.5.mlp.experts.122.up_proj", + "model.layers.5.mlp.experts.123.down_proj", + "model.layers.5.mlp.experts.123.gate_proj", + "model.layers.5.mlp.experts.123.up_proj", + "model.layers.5.mlp.experts.124.down_proj", + "model.layers.5.mlp.experts.124.gate_proj", + "model.layers.5.mlp.experts.124.up_proj", + "model.layers.5.mlp.experts.125.down_proj", + "model.layers.5.mlp.experts.125.gate_proj", + "model.layers.5.mlp.experts.125.up_proj", + "model.layers.5.mlp.experts.126.down_proj", + "model.layers.5.mlp.experts.126.gate_proj", + "model.layers.5.mlp.experts.126.up_proj", + "model.layers.5.mlp.experts.127.down_proj", + "model.layers.5.mlp.experts.127.gate_proj", + "model.layers.5.mlp.experts.127.up_proj", + "model.layers.5.mlp.experts.128.down_proj", + "model.layers.5.mlp.experts.128.gate_proj", + "model.layers.5.mlp.experts.128.up_proj", + "model.layers.5.mlp.experts.129.down_proj", + "model.layers.5.mlp.experts.129.gate_proj", + "model.layers.5.mlp.experts.129.up_proj", + "model.layers.5.mlp.experts.13.down_proj", + "model.layers.5.mlp.experts.13.gate_proj", + "model.layers.5.mlp.experts.13.up_proj", + "model.layers.5.mlp.experts.130.down_proj", + "model.layers.5.mlp.experts.130.gate_proj", + "model.layers.5.mlp.experts.130.up_proj", + "model.layers.5.mlp.experts.131.down_proj", + "model.layers.5.mlp.experts.131.gate_proj", + "model.layers.5.mlp.experts.131.up_proj", + "model.layers.5.mlp.experts.132.down_proj", + "model.layers.5.mlp.experts.132.gate_proj", + "model.layers.5.mlp.experts.132.up_proj", + "model.layers.5.mlp.experts.133.down_proj", + "model.layers.5.mlp.experts.133.gate_proj", + "model.layers.5.mlp.experts.133.up_proj", + "model.layers.5.mlp.experts.134.down_proj", + "model.layers.5.mlp.experts.134.gate_proj", + "model.layers.5.mlp.experts.134.up_proj", + "model.layers.5.mlp.experts.135.down_proj", + "model.layers.5.mlp.experts.135.gate_proj", + "model.layers.5.mlp.experts.135.up_proj", + "model.layers.5.mlp.experts.136.down_proj", + "model.layers.5.mlp.experts.136.gate_proj", + "model.layers.5.mlp.experts.136.up_proj", + "model.layers.5.mlp.experts.137.down_proj", + "model.layers.5.mlp.experts.137.gate_proj", + "model.layers.5.mlp.experts.137.up_proj", + "model.layers.5.mlp.experts.138.down_proj", + "model.layers.5.mlp.experts.138.gate_proj", + "model.layers.5.mlp.experts.138.up_proj", + "model.layers.5.mlp.experts.139.down_proj", + "model.layers.5.mlp.experts.139.gate_proj", + "model.layers.5.mlp.experts.139.up_proj", + "model.layers.5.mlp.experts.14.down_proj", + "model.layers.5.mlp.experts.14.gate_proj", + "model.layers.5.mlp.experts.14.up_proj", + "model.layers.5.mlp.experts.140.down_proj", + "model.layers.5.mlp.experts.140.gate_proj", + "model.layers.5.mlp.experts.140.up_proj", + "model.layers.5.mlp.experts.141.down_proj", + "model.layers.5.mlp.experts.141.gate_proj", + "model.layers.5.mlp.experts.141.up_proj", + "model.layers.5.mlp.experts.142.down_proj", + "model.layers.5.mlp.experts.142.gate_proj", + "model.layers.5.mlp.experts.142.up_proj", + "model.layers.5.mlp.experts.143.down_proj", + "model.layers.5.mlp.experts.143.gate_proj", + "model.layers.5.mlp.experts.143.up_proj", + "model.layers.5.mlp.experts.144.down_proj", + "model.layers.5.mlp.experts.144.gate_proj", + "model.layers.5.mlp.experts.144.up_proj", + "model.layers.5.mlp.experts.145.down_proj", + "model.layers.5.mlp.experts.145.gate_proj", + "model.layers.5.mlp.experts.145.up_proj", + "model.layers.5.mlp.experts.146.down_proj", + "model.layers.5.mlp.experts.146.gate_proj", + "model.layers.5.mlp.experts.146.up_proj", + "model.layers.5.mlp.experts.147.down_proj", + "model.layers.5.mlp.experts.147.gate_proj", + "model.layers.5.mlp.experts.147.up_proj", + "model.layers.5.mlp.experts.148.down_proj", + "model.layers.5.mlp.experts.148.gate_proj", + "model.layers.5.mlp.experts.148.up_proj", + "model.layers.5.mlp.experts.149.down_proj", + "model.layers.5.mlp.experts.149.gate_proj", + "model.layers.5.mlp.experts.149.up_proj", + "model.layers.5.mlp.experts.15.down_proj", + "model.layers.5.mlp.experts.15.gate_proj", + "model.layers.5.mlp.experts.15.up_proj", + "model.layers.5.mlp.experts.150.down_proj", + "model.layers.5.mlp.experts.150.gate_proj", + "model.layers.5.mlp.experts.150.up_proj", + "model.layers.5.mlp.experts.151.down_proj", + "model.layers.5.mlp.experts.151.gate_proj", + "model.layers.5.mlp.experts.151.up_proj", + "model.layers.5.mlp.experts.152.down_proj", + "model.layers.5.mlp.experts.152.gate_proj", + "model.layers.5.mlp.experts.152.up_proj", + "model.layers.5.mlp.experts.153.down_proj", + "model.layers.5.mlp.experts.153.gate_proj", + "model.layers.5.mlp.experts.153.up_proj", + "model.layers.5.mlp.experts.154.down_proj", + "model.layers.5.mlp.experts.154.gate_proj", + "model.layers.5.mlp.experts.154.up_proj", + "model.layers.5.mlp.experts.155.down_proj", + "model.layers.5.mlp.experts.155.gate_proj", + "model.layers.5.mlp.experts.155.up_proj", + "model.layers.5.mlp.experts.156.down_proj", + "model.layers.5.mlp.experts.156.gate_proj", + "model.layers.5.mlp.experts.156.up_proj", + "model.layers.5.mlp.experts.157.down_proj", + "model.layers.5.mlp.experts.157.gate_proj", + "model.layers.5.mlp.experts.157.up_proj", + "model.layers.5.mlp.experts.158.down_proj", + "model.layers.5.mlp.experts.158.gate_proj", + "model.layers.5.mlp.experts.158.up_proj", + "model.layers.5.mlp.experts.159.down_proj", + "model.layers.5.mlp.experts.159.gate_proj", + "model.layers.5.mlp.experts.159.up_proj", + "model.layers.5.mlp.experts.16.down_proj", + "model.layers.5.mlp.experts.16.gate_proj", + "model.layers.5.mlp.experts.16.up_proj", + "model.layers.5.mlp.experts.160.down_proj", + "model.layers.5.mlp.experts.160.gate_proj", + "model.layers.5.mlp.experts.160.up_proj", + "model.layers.5.mlp.experts.161.down_proj", + "model.layers.5.mlp.experts.161.gate_proj", + "model.layers.5.mlp.experts.161.up_proj", + "model.layers.5.mlp.experts.162.down_proj", + "model.layers.5.mlp.experts.162.gate_proj", + "model.layers.5.mlp.experts.162.up_proj", + "model.layers.5.mlp.experts.163.down_proj", + "model.layers.5.mlp.experts.163.gate_proj", + "model.layers.5.mlp.experts.163.up_proj", + "model.layers.5.mlp.experts.164.down_proj", + "model.layers.5.mlp.experts.164.gate_proj", + "model.layers.5.mlp.experts.164.up_proj", + "model.layers.5.mlp.experts.165.down_proj", + "model.layers.5.mlp.experts.165.gate_proj", + "model.layers.5.mlp.experts.165.up_proj", + "model.layers.5.mlp.experts.166.down_proj", + "model.layers.5.mlp.experts.166.gate_proj", + "model.layers.5.mlp.experts.166.up_proj", + "model.layers.5.mlp.experts.167.down_proj", + "model.layers.5.mlp.experts.167.gate_proj", + "model.layers.5.mlp.experts.167.up_proj", + "model.layers.5.mlp.experts.168.down_proj", + "model.layers.5.mlp.experts.168.gate_proj", + "model.layers.5.mlp.experts.168.up_proj", + "model.layers.5.mlp.experts.169.down_proj", + "model.layers.5.mlp.experts.169.gate_proj", + "model.layers.5.mlp.experts.169.up_proj", + "model.layers.5.mlp.experts.17.down_proj", + "model.layers.5.mlp.experts.17.gate_proj", + "model.layers.5.mlp.experts.17.up_proj", + "model.layers.5.mlp.experts.170.down_proj", + "model.layers.5.mlp.experts.170.gate_proj", + "model.layers.5.mlp.experts.170.up_proj", + "model.layers.5.mlp.experts.171.down_proj", + "model.layers.5.mlp.experts.171.gate_proj", + "model.layers.5.mlp.experts.171.up_proj", + "model.layers.5.mlp.experts.172.down_proj", + "model.layers.5.mlp.experts.172.gate_proj", + "model.layers.5.mlp.experts.172.up_proj", + "model.layers.5.mlp.experts.173.down_proj", + "model.layers.5.mlp.experts.173.gate_proj", + "model.layers.5.mlp.experts.173.up_proj", + "model.layers.5.mlp.experts.174.down_proj", + "model.layers.5.mlp.experts.174.gate_proj", + "model.layers.5.mlp.experts.174.up_proj", + "model.layers.5.mlp.experts.175.down_proj", + "model.layers.5.mlp.experts.175.gate_proj", + "model.layers.5.mlp.experts.175.up_proj", + "model.layers.5.mlp.experts.176.down_proj", + "model.layers.5.mlp.experts.176.gate_proj", + "model.layers.5.mlp.experts.176.up_proj", + "model.layers.5.mlp.experts.177.down_proj", + "model.layers.5.mlp.experts.177.gate_proj", + "model.layers.5.mlp.experts.177.up_proj", + "model.layers.5.mlp.experts.178.down_proj", + "model.layers.5.mlp.experts.178.gate_proj", + "model.layers.5.mlp.experts.178.up_proj", + "model.layers.5.mlp.experts.179.down_proj", + "model.layers.5.mlp.experts.179.gate_proj", + "model.layers.5.mlp.experts.179.up_proj", + "model.layers.5.mlp.experts.18.down_proj", + "model.layers.5.mlp.experts.18.gate_proj", + "model.layers.5.mlp.experts.18.up_proj", + "model.layers.5.mlp.experts.180.down_proj", + "model.layers.5.mlp.experts.180.gate_proj", + "model.layers.5.mlp.experts.180.up_proj", + "model.layers.5.mlp.experts.181.down_proj", + "model.layers.5.mlp.experts.181.gate_proj", + "model.layers.5.mlp.experts.181.up_proj", + "model.layers.5.mlp.experts.182.down_proj", + "model.layers.5.mlp.experts.182.gate_proj", + "model.layers.5.mlp.experts.182.up_proj", + "model.layers.5.mlp.experts.183.down_proj", + "model.layers.5.mlp.experts.183.gate_proj", + "model.layers.5.mlp.experts.183.up_proj", + "model.layers.5.mlp.experts.184.down_proj", + "model.layers.5.mlp.experts.184.gate_proj", + "model.layers.5.mlp.experts.184.up_proj", + "model.layers.5.mlp.experts.185.down_proj", + "model.layers.5.mlp.experts.185.gate_proj", + "model.layers.5.mlp.experts.185.up_proj", + "model.layers.5.mlp.experts.186.down_proj", + "model.layers.5.mlp.experts.186.gate_proj", + "model.layers.5.mlp.experts.186.up_proj", + "model.layers.5.mlp.experts.187.down_proj", + "model.layers.5.mlp.experts.187.gate_proj", + "model.layers.5.mlp.experts.187.up_proj", + "model.layers.5.mlp.experts.188.down_proj", + "model.layers.5.mlp.experts.188.gate_proj", + "model.layers.5.mlp.experts.188.up_proj", + "model.layers.5.mlp.experts.189.down_proj", + "model.layers.5.mlp.experts.189.gate_proj", + "model.layers.5.mlp.experts.189.up_proj", + "model.layers.5.mlp.experts.19.down_proj", + "model.layers.5.mlp.experts.19.gate_proj", + "model.layers.5.mlp.experts.19.up_proj", + "model.layers.5.mlp.experts.190.down_proj", + "model.layers.5.mlp.experts.190.gate_proj", + "model.layers.5.mlp.experts.190.up_proj", + "model.layers.5.mlp.experts.191.down_proj", + "model.layers.5.mlp.experts.191.gate_proj", + "model.layers.5.mlp.experts.191.up_proj", + "model.layers.5.mlp.experts.192.down_proj", + "model.layers.5.mlp.experts.192.gate_proj", + "model.layers.5.mlp.experts.192.up_proj", + "model.layers.5.mlp.experts.193.down_proj", + "model.layers.5.mlp.experts.193.gate_proj", + "model.layers.5.mlp.experts.193.up_proj", + "model.layers.5.mlp.experts.194.down_proj", + "model.layers.5.mlp.experts.194.gate_proj", + "model.layers.5.mlp.experts.194.up_proj", + "model.layers.5.mlp.experts.195.down_proj", + "model.layers.5.mlp.experts.195.gate_proj", + "model.layers.5.mlp.experts.195.up_proj", + "model.layers.5.mlp.experts.196.down_proj", + "model.layers.5.mlp.experts.196.gate_proj", + "model.layers.5.mlp.experts.196.up_proj", + "model.layers.5.mlp.experts.197.down_proj", + "model.layers.5.mlp.experts.197.gate_proj", + "model.layers.5.mlp.experts.197.up_proj", + "model.layers.5.mlp.experts.198.down_proj", + "model.layers.5.mlp.experts.198.gate_proj", + "model.layers.5.mlp.experts.198.up_proj", + "model.layers.5.mlp.experts.199.down_proj", + "model.layers.5.mlp.experts.199.gate_proj", + "model.layers.5.mlp.experts.199.up_proj", + "model.layers.5.mlp.experts.2.down_proj", + "model.layers.5.mlp.experts.2.gate_proj", + "model.layers.5.mlp.experts.2.up_proj", + "model.layers.5.mlp.experts.20.down_proj", + "model.layers.5.mlp.experts.20.gate_proj", + "model.layers.5.mlp.experts.20.up_proj", + "model.layers.5.mlp.experts.200.down_proj", + "model.layers.5.mlp.experts.200.gate_proj", + "model.layers.5.mlp.experts.200.up_proj", + "model.layers.5.mlp.experts.201.down_proj", + "model.layers.5.mlp.experts.201.gate_proj", + "model.layers.5.mlp.experts.201.up_proj", + "model.layers.5.mlp.experts.202.down_proj", + "model.layers.5.mlp.experts.202.gate_proj", + "model.layers.5.mlp.experts.202.up_proj", + "model.layers.5.mlp.experts.203.down_proj", + "model.layers.5.mlp.experts.203.gate_proj", + "model.layers.5.mlp.experts.203.up_proj", + "model.layers.5.mlp.experts.204.down_proj", + "model.layers.5.mlp.experts.204.gate_proj", + "model.layers.5.mlp.experts.204.up_proj", + "model.layers.5.mlp.experts.205.down_proj", + "model.layers.5.mlp.experts.205.gate_proj", + "model.layers.5.mlp.experts.205.up_proj", + "model.layers.5.mlp.experts.206.down_proj", + "model.layers.5.mlp.experts.206.gate_proj", + "model.layers.5.mlp.experts.206.up_proj", + "model.layers.5.mlp.experts.207.down_proj", + "model.layers.5.mlp.experts.207.gate_proj", + "model.layers.5.mlp.experts.207.up_proj", + "model.layers.5.mlp.experts.208.down_proj", + "model.layers.5.mlp.experts.208.gate_proj", + "model.layers.5.mlp.experts.208.up_proj", + "model.layers.5.mlp.experts.209.down_proj", + "model.layers.5.mlp.experts.209.gate_proj", + "model.layers.5.mlp.experts.209.up_proj", + "model.layers.5.mlp.experts.21.down_proj", + "model.layers.5.mlp.experts.21.gate_proj", + "model.layers.5.mlp.experts.21.up_proj", + "model.layers.5.mlp.experts.210.down_proj", + "model.layers.5.mlp.experts.210.gate_proj", + "model.layers.5.mlp.experts.210.up_proj", + "model.layers.5.mlp.experts.211.down_proj", + "model.layers.5.mlp.experts.211.gate_proj", + "model.layers.5.mlp.experts.211.up_proj", + "model.layers.5.mlp.experts.212.down_proj", + "model.layers.5.mlp.experts.212.gate_proj", + "model.layers.5.mlp.experts.212.up_proj", + "model.layers.5.mlp.experts.213.down_proj", + "model.layers.5.mlp.experts.213.gate_proj", + "model.layers.5.mlp.experts.213.up_proj", + "model.layers.5.mlp.experts.214.down_proj", + "model.layers.5.mlp.experts.214.gate_proj", + "model.layers.5.mlp.experts.214.up_proj", + "model.layers.5.mlp.experts.215.down_proj", + "model.layers.5.mlp.experts.215.gate_proj", + "model.layers.5.mlp.experts.215.up_proj", + "model.layers.5.mlp.experts.216.down_proj", + "model.layers.5.mlp.experts.216.gate_proj", + "model.layers.5.mlp.experts.216.up_proj", + "model.layers.5.mlp.experts.217.down_proj", + "model.layers.5.mlp.experts.217.gate_proj", + "model.layers.5.mlp.experts.217.up_proj", + "model.layers.5.mlp.experts.218.down_proj", + "model.layers.5.mlp.experts.218.gate_proj", + "model.layers.5.mlp.experts.218.up_proj", + "model.layers.5.mlp.experts.219.down_proj", + "model.layers.5.mlp.experts.219.gate_proj", + "model.layers.5.mlp.experts.219.up_proj", + "model.layers.5.mlp.experts.22.down_proj", + "model.layers.5.mlp.experts.22.gate_proj", + "model.layers.5.mlp.experts.22.up_proj", + "model.layers.5.mlp.experts.220.down_proj", + "model.layers.5.mlp.experts.220.gate_proj", + "model.layers.5.mlp.experts.220.up_proj", + "model.layers.5.mlp.experts.221.down_proj", + "model.layers.5.mlp.experts.221.gate_proj", + "model.layers.5.mlp.experts.221.up_proj", + "model.layers.5.mlp.experts.222.down_proj", + "model.layers.5.mlp.experts.222.gate_proj", + "model.layers.5.mlp.experts.222.up_proj", + "model.layers.5.mlp.experts.223.down_proj", + "model.layers.5.mlp.experts.223.gate_proj", + "model.layers.5.mlp.experts.223.up_proj", + "model.layers.5.mlp.experts.224.down_proj", + "model.layers.5.mlp.experts.224.gate_proj", + "model.layers.5.mlp.experts.224.up_proj", + "model.layers.5.mlp.experts.225.down_proj", + "model.layers.5.mlp.experts.225.gate_proj", + "model.layers.5.mlp.experts.225.up_proj", + "model.layers.5.mlp.experts.226.down_proj", + "model.layers.5.mlp.experts.226.gate_proj", + "model.layers.5.mlp.experts.226.up_proj", + "model.layers.5.mlp.experts.227.down_proj", + "model.layers.5.mlp.experts.227.gate_proj", + "model.layers.5.mlp.experts.227.up_proj", + "model.layers.5.mlp.experts.228.down_proj", + "model.layers.5.mlp.experts.228.gate_proj", + "model.layers.5.mlp.experts.228.up_proj", + "model.layers.5.mlp.experts.229.down_proj", + "model.layers.5.mlp.experts.229.gate_proj", + "model.layers.5.mlp.experts.229.up_proj", + "model.layers.5.mlp.experts.23.down_proj", + "model.layers.5.mlp.experts.23.gate_proj", + "model.layers.5.mlp.experts.23.up_proj", + "model.layers.5.mlp.experts.230.down_proj", + "model.layers.5.mlp.experts.230.gate_proj", + "model.layers.5.mlp.experts.230.up_proj", + "model.layers.5.mlp.experts.231.down_proj", + "model.layers.5.mlp.experts.231.gate_proj", + "model.layers.5.mlp.experts.231.up_proj", + "model.layers.5.mlp.experts.232.down_proj", + "model.layers.5.mlp.experts.232.gate_proj", + "model.layers.5.mlp.experts.232.up_proj", + "model.layers.5.mlp.experts.233.down_proj", + "model.layers.5.mlp.experts.233.gate_proj", + "model.layers.5.mlp.experts.233.up_proj", + "model.layers.5.mlp.experts.234.down_proj", + "model.layers.5.mlp.experts.234.gate_proj", + "model.layers.5.mlp.experts.234.up_proj", + "model.layers.5.mlp.experts.235.down_proj", + "model.layers.5.mlp.experts.235.gate_proj", + "model.layers.5.mlp.experts.235.up_proj", + "model.layers.5.mlp.experts.236.down_proj", + "model.layers.5.mlp.experts.236.gate_proj", + "model.layers.5.mlp.experts.236.up_proj", + "model.layers.5.mlp.experts.237.down_proj", + "model.layers.5.mlp.experts.237.gate_proj", + "model.layers.5.mlp.experts.237.up_proj", + "model.layers.5.mlp.experts.238.down_proj", + "model.layers.5.mlp.experts.238.gate_proj", + "model.layers.5.mlp.experts.238.up_proj", + "model.layers.5.mlp.experts.239.down_proj", + "model.layers.5.mlp.experts.239.gate_proj", + "model.layers.5.mlp.experts.239.up_proj", + "model.layers.5.mlp.experts.24.down_proj", + "model.layers.5.mlp.experts.24.gate_proj", + "model.layers.5.mlp.experts.24.up_proj", + "model.layers.5.mlp.experts.240.down_proj", + "model.layers.5.mlp.experts.240.gate_proj", + "model.layers.5.mlp.experts.240.up_proj", + "model.layers.5.mlp.experts.241.down_proj", + "model.layers.5.mlp.experts.241.gate_proj", + "model.layers.5.mlp.experts.241.up_proj", + "model.layers.5.mlp.experts.242.down_proj", + "model.layers.5.mlp.experts.242.gate_proj", + "model.layers.5.mlp.experts.242.up_proj", + "model.layers.5.mlp.experts.243.down_proj", + "model.layers.5.mlp.experts.243.gate_proj", + "model.layers.5.mlp.experts.243.up_proj", + "model.layers.5.mlp.experts.244.down_proj", + "model.layers.5.mlp.experts.244.gate_proj", + "model.layers.5.mlp.experts.244.up_proj", + "model.layers.5.mlp.experts.245.down_proj", + "model.layers.5.mlp.experts.245.gate_proj", + "model.layers.5.mlp.experts.245.up_proj", + "model.layers.5.mlp.experts.246.down_proj", + "model.layers.5.mlp.experts.246.gate_proj", + "model.layers.5.mlp.experts.246.up_proj", + "model.layers.5.mlp.experts.247.down_proj", + "model.layers.5.mlp.experts.247.gate_proj", + "model.layers.5.mlp.experts.247.up_proj", + "model.layers.5.mlp.experts.248.down_proj", + "model.layers.5.mlp.experts.248.gate_proj", + "model.layers.5.mlp.experts.248.up_proj", + "model.layers.5.mlp.experts.249.down_proj", + "model.layers.5.mlp.experts.249.gate_proj", + "model.layers.5.mlp.experts.249.up_proj", + "model.layers.5.mlp.experts.25.down_proj", + "model.layers.5.mlp.experts.25.gate_proj", + "model.layers.5.mlp.experts.25.up_proj", + "model.layers.5.mlp.experts.250.down_proj", + "model.layers.5.mlp.experts.250.gate_proj", + "model.layers.5.mlp.experts.250.up_proj", + "model.layers.5.mlp.experts.251.down_proj", + "model.layers.5.mlp.experts.251.gate_proj", + "model.layers.5.mlp.experts.251.up_proj", + "model.layers.5.mlp.experts.252.down_proj", + "model.layers.5.mlp.experts.252.gate_proj", + "model.layers.5.mlp.experts.252.up_proj", + "model.layers.5.mlp.experts.253.down_proj", + "model.layers.5.mlp.experts.253.gate_proj", + "model.layers.5.mlp.experts.253.up_proj", + "model.layers.5.mlp.experts.254.down_proj", + "model.layers.5.mlp.experts.254.gate_proj", + "model.layers.5.mlp.experts.254.up_proj", + "model.layers.5.mlp.experts.255.down_proj", + "model.layers.5.mlp.experts.255.gate_proj", + "model.layers.5.mlp.experts.255.up_proj", + "model.layers.5.mlp.experts.26.down_proj", + "model.layers.5.mlp.experts.26.gate_proj", + "model.layers.5.mlp.experts.26.up_proj", + "model.layers.5.mlp.experts.27.down_proj", + "model.layers.5.mlp.experts.27.gate_proj", + "model.layers.5.mlp.experts.27.up_proj", + "model.layers.5.mlp.experts.28.down_proj", + "model.layers.5.mlp.experts.28.gate_proj", + "model.layers.5.mlp.experts.28.up_proj", + "model.layers.5.mlp.experts.29.down_proj", + "model.layers.5.mlp.experts.29.gate_proj", + "model.layers.5.mlp.experts.29.up_proj", + "model.layers.5.mlp.experts.3.down_proj", + "model.layers.5.mlp.experts.3.gate_proj", + "model.layers.5.mlp.experts.3.up_proj", + "model.layers.5.mlp.experts.30.down_proj", + "model.layers.5.mlp.experts.30.gate_proj", + "model.layers.5.mlp.experts.30.up_proj", + "model.layers.5.mlp.experts.31.down_proj", + "model.layers.5.mlp.experts.31.gate_proj", + "model.layers.5.mlp.experts.31.up_proj", + "model.layers.5.mlp.experts.32.down_proj", + "model.layers.5.mlp.experts.32.gate_proj", + "model.layers.5.mlp.experts.32.up_proj", + "model.layers.5.mlp.experts.33.down_proj", + "model.layers.5.mlp.experts.33.gate_proj", + "model.layers.5.mlp.experts.33.up_proj", + "model.layers.5.mlp.experts.34.down_proj", + "model.layers.5.mlp.experts.34.gate_proj", + "model.layers.5.mlp.experts.34.up_proj", + "model.layers.5.mlp.experts.35.down_proj", + "model.layers.5.mlp.experts.35.gate_proj", + "model.layers.5.mlp.experts.35.up_proj", + "model.layers.5.mlp.experts.36.down_proj", + "model.layers.5.mlp.experts.36.gate_proj", + "model.layers.5.mlp.experts.36.up_proj", + "model.layers.5.mlp.experts.37.down_proj", + "model.layers.5.mlp.experts.37.gate_proj", + "model.layers.5.mlp.experts.37.up_proj", + "model.layers.5.mlp.experts.38.down_proj", + "model.layers.5.mlp.experts.38.gate_proj", + "model.layers.5.mlp.experts.38.up_proj", + "model.layers.5.mlp.experts.39.down_proj", + "model.layers.5.mlp.experts.39.gate_proj", + "model.layers.5.mlp.experts.39.up_proj", + "model.layers.5.mlp.experts.4.down_proj", + "model.layers.5.mlp.experts.4.gate_proj", + "model.layers.5.mlp.experts.4.up_proj", + "model.layers.5.mlp.experts.40.down_proj", + "model.layers.5.mlp.experts.40.gate_proj", + "model.layers.5.mlp.experts.40.up_proj", + "model.layers.5.mlp.experts.41.down_proj", + "model.layers.5.mlp.experts.41.gate_proj", + "model.layers.5.mlp.experts.41.up_proj", + "model.layers.5.mlp.experts.42.down_proj", + "model.layers.5.mlp.experts.42.gate_proj", + "model.layers.5.mlp.experts.42.up_proj", + "model.layers.5.mlp.experts.43.down_proj", + "model.layers.5.mlp.experts.43.gate_proj", + "model.layers.5.mlp.experts.43.up_proj", + "model.layers.5.mlp.experts.44.down_proj", + "model.layers.5.mlp.experts.44.gate_proj", + "model.layers.5.mlp.experts.44.up_proj", + "model.layers.5.mlp.experts.45.down_proj", + "model.layers.5.mlp.experts.45.gate_proj", + "model.layers.5.mlp.experts.45.up_proj", + "model.layers.5.mlp.experts.46.down_proj", + "model.layers.5.mlp.experts.46.gate_proj", + "model.layers.5.mlp.experts.46.up_proj", + "model.layers.5.mlp.experts.47.down_proj", + "model.layers.5.mlp.experts.47.gate_proj", + "model.layers.5.mlp.experts.47.up_proj", + "model.layers.5.mlp.experts.48.down_proj", + "model.layers.5.mlp.experts.48.gate_proj", + "model.layers.5.mlp.experts.48.up_proj", + "model.layers.5.mlp.experts.49.down_proj", + "model.layers.5.mlp.experts.49.gate_proj", + "model.layers.5.mlp.experts.49.up_proj", + "model.layers.5.mlp.experts.5.down_proj", + "model.layers.5.mlp.experts.5.gate_proj", + "model.layers.5.mlp.experts.5.up_proj", + "model.layers.5.mlp.experts.50.down_proj", + "model.layers.5.mlp.experts.50.gate_proj", + "model.layers.5.mlp.experts.50.up_proj", + "model.layers.5.mlp.experts.51.down_proj", + "model.layers.5.mlp.experts.51.gate_proj", + "model.layers.5.mlp.experts.51.up_proj", + "model.layers.5.mlp.experts.52.down_proj", + "model.layers.5.mlp.experts.52.gate_proj", + "model.layers.5.mlp.experts.52.up_proj", + "model.layers.5.mlp.experts.53.down_proj", + "model.layers.5.mlp.experts.53.gate_proj", + "model.layers.5.mlp.experts.53.up_proj", + "model.layers.5.mlp.experts.54.down_proj", + "model.layers.5.mlp.experts.54.gate_proj", + "model.layers.5.mlp.experts.54.up_proj", + "model.layers.5.mlp.experts.55.down_proj", + "model.layers.5.mlp.experts.55.gate_proj", + "model.layers.5.mlp.experts.55.up_proj", + "model.layers.5.mlp.experts.56.down_proj", + "model.layers.5.mlp.experts.56.gate_proj", + "model.layers.5.mlp.experts.56.up_proj", + "model.layers.5.mlp.experts.57.down_proj", + "model.layers.5.mlp.experts.57.gate_proj", + "model.layers.5.mlp.experts.57.up_proj", + "model.layers.5.mlp.experts.58.down_proj", + "model.layers.5.mlp.experts.58.gate_proj", + "model.layers.5.mlp.experts.58.up_proj", + "model.layers.5.mlp.experts.59.down_proj", + "model.layers.5.mlp.experts.59.gate_proj", + "model.layers.5.mlp.experts.59.up_proj", + "model.layers.5.mlp.experts.6.down_proj", + "model.layers.5.mlp.experts.6.gate_proj", + "model.layers.5.mlp.experts.6.up_proj", + "model.layers.5.mlp.experts.60.down_proj", + "model.layers.5.mlp.experts.60.gate_proj", + "model.layers.5.mlp.experts.60.up_proj", + "model.layers.5.mlp.experts.61.down_proj", + "model.layers.5.mlp.experts.61.gate_proj", + "model.layers.5.mlp.experts.61.up_proj", + "model.layers.5.mlp.experts.62.down_proj", + "model.layers.5.mlp.experts.62.gate_proj", + "model.layers.5.mlp.experts.62.up_proj", + "model.layers.5.mlp.experts.63.down_proj", + "model.layers.5.mlp.experts.63.gate_proj", + "model.layers.5.mlp.experts.63.up_proj", + "model.layers.5.mlp.experts.64.down_proj", + "model.layers.5.mlp.experts.64.gate_proj", + "model.layers.5.mlp.experts.64.up_proj", + "model.layers.5.mlp.experts.65.down_proj", + "model.layers.5.mlp.experts.65.gate_proj", + "model.layers.5.mlp.experts.65.up_proj", + "model.layers.5.mlp.experts.66.down_proj", + "model.layers.5.mlp.experts.66.gate_proj", + "model.layers.5.mlp.experts.66.up_proj", + "model.layers.5.mlp.experts.67.down_proj", + "model.layers.5.mlp.experts.67.gate_proj", + "model.layers.5.mlp.experts.67.up_proj", + "model.layers.5.mlp.experts.68.down_proj", + "model.layers.5.mlp.experts.68.gate_proj", + "model.layers.5.mlp.experts.68.up_proj", + "model.layers.5.mlp.experts.69.down_proj", + "model.layers.5.mlp.experts.69.gate_proj", + "model.layers.5.mlp.experts.69.up_proj", + "model.layers.5.mlp.experts.7.down_proj", + "model.layers.5.mlp.experts.7.gate_proj", + "model.layers.5.mlp.experts.7.up_proj", + "model.layers.5.mlp.experts.70.down_proj", + "model.layers.5.mlp.experts.70.gate_proj", + "model.layers.5.mlp.experts.70.up_proj", + "model.layers.5.mlp.experts.71.down_proj", + "model.layers.5.mlp.experts.71.gate_proj", + "model.layers.5.mlp.experts.71.up_proj", + "model.layers.5.mlp.experts.72.down_proj", + "model.layers.5.mlp.experts.72.gate_proj", + "model.layers.5.mlp.experts.72.up_proj", + "model.layers.5.mlp.experts.73.down_proj", + "model.layers.5.mlp.experts.73.gate_proj", + "model.layers.5.mlp.experts.73.up_proj", + "model.layers.5.mlp.experts.74.down_proj", + "model.layers.5.mlp.experts.74.gate_proj", + "model.layers.5.mlp.experts.74.up_proj", + "model.layers.5.mlp.experts.75.down_proj", + "model.layers.5.mlp.experts.75.gate_proj", + "model.layers.5.mlp.experts.75.up_proj", + "model.layers.5.mlp.experts.76.down_proj", + "model.layers.5.mlp.experts.76.gate_proj", + "model.layers.5.mlp.experts.76.up_proj", + "model.layers.5.mlp.experts.77.down_proj", + "model.layers.5.mlp.experts.77.gate_proj", + "model.layers.5.mlp.experts.77.up_proj", + "model.layers.5.mlp.experts.78.down_proj", + "model.layers.5.mlp.experts.78.gate_proj", + "model.layers.5.mlp.experts.78.up_proj", + "model.layers.5.mlp.experts.79.down_proj", + "model.layers.5.mlp.experts.79.gate_proj", + "model.layers.5.mlp.experts.79.up_proj", + "model.layers.5.mlp.experts.8.down_proj", + "model.layers.5.mlp.experts.8.gate_proj", + "model.layers.5.mlp.experts.8.up_proj", + "model.layers.5.mlp.experts.80.down_proj", + "model.layers.5.mlp.experts.80.gate_proj", + "model.layers.5.mlp.experts.80.up_proj", + "model.layers.5.mlp.experts.81.down_proj", + "model.layers.5.mlp.experts.81.gate_proj", + "model.layers.5.mlp.experts.81.up_proj", + "model.layers.5.mlp.experts.82.down_proj", + "model.layers.5.mlp.experts.82.gate_proj", + "model.layers.5.mlp.experts.82.up_proj", + "model.layers.5.mlp.experts.83.down_proj", + "model.layers.5.mlp.experts.83.gate_proj", + "model.layers.5.mlp.experts.83.up_proj", + "model.layers.5.mlp.experts.84.down_proj", + "model.layers.5.mlp.experts.84.gate_proj", + "model.layers.5.mlp.experts.84.up_proj", + "model.layers.5.mlp.experts.85.down_proj", + "model.layers.5.mlp.experts.85.gate_proj", + "model.layers.5.mlp.experts.85.up_proj", + "model.layers.5.mlp.experts.86.down_proj", + "model.layers.5.mlp.experts.86.gate_proj", + "model.layers.5.mlp.experts.86.up_proj", + "model.layers.5.mlp.experts.87.down_proj", + "model.layers.5.mlp.experts.87.gate_proj", + "model.layers.5.mlp.experts.87.up_proj", + "model.layers.5.mlp.experts.88.down_proj", + "model.layers.5.mlp.experts.88.gate_proj", + "model.layers.5.mlp.experts.88.up_proj", + "model.layers.5.mlp.experts.89.down_proj", + "model.layers.5.mlp.experts.89.gate_proj", + "model.layers.5.mlp.experts.89.up_proj", + "model.layers.5.mlp.experts.9.down_proj", + "model.layers.5.mlp.experts.9.gate_proj", + "model.layers.5.mlp.experts.9.up_proj", + "model.layers.5.mlp.experts.90.down_proj", + "model.layers.5.mlp.experts.90.gate_proj", + "model.layers.5.mlp.experts.90.up_proj", + "model.layers.5.mlp.experts.91.down_proj", + "model.layers.5.mlp.experts.91.gate_proj", + "model.layers.5.mlp.experts.91.up_proj", + "model.layers.5.mlp.experts.92.down_proj", + "model.layers.5.mlp.experts.92.gate_proj", + "model.layers.5.mlp.experts.92.up_proj", + "model.layers.5.mlp.experts.93.down_proj", + "model.layers.5.mlp.experts.93.gate_proj", + "model.layers.5.mlp.experts.93.up_proj", + "model.layers.5.mlp.experts.94.down_proj", + "model.layers.5.mlp.experts.94.gate_proj", + "model.layers.5.mlp.experts.94.up_proj", + "model.layers.5.mlp.experts.95.down_proj", + "model.layers.5.mlp.experts.95.gate_proj", + "model.layers.5.mlp.experts.95.up_proj", + "model.layers.5.mlp.experts.96.down_proj", + "model.layers.5.mlp.experts.96.gate_proj", + "model.layers.5.mlp.experts.96.up_proj", + "model.layers.5.mlp.experts.97.down_proj", + "model.layers.5.mlp.experts.97.gate_proj", + "model.layers.5.mlp.experts.97.up_proj", + "model.layers.5.mlp.experts.98.down_proj", + "model.layers.5.mlp.experts.98.gate_proj", + "model.layers.5.mlp.experts.98.up_proj", + "model.layers.5.mlp.experts.99.down_proj", + "model.layers.5.mlp.experts.99.gate_proj", + "model.layers.5.mlp.experts.99.up_proj", + "model.layers.50.mlp.experts.0.down_proj", + "model.layers.50.mlp.experts.0.gate_proj", + "model.layers.50.mlp.experts.0.up_proj", + "model.layers.50.mlp.experts.1.down_proj", + "model.layers.50.mlp.experts.1.gate_proj", + "model.layers.50.mlp.experts.1.up_proj", + "model.layers.50.mlp.experts.10.down_proj", + "model.layers.50.mlp.experts.10.gate_proj", + "model.layers.50.mlp.experts.10.up_proj", + "model.layers.50.mlp.experts.100.down_proj", + "model.layers.50.mlp.experts.100.gate_proj", + "model.layers.50.mlp.experts.100.up_proj", + "model.layers.50.mlp.experts.101.down_proj", + "model.layers.50.mlp.experts.101.gate_proj", + "model.layers.50.mlp.experts.101.up_proj", + "model.layers.50.mlp.experts.102.down_proj", + "model.layers.50.mlp.experts.102.gate_proj", + "model.layers.50.mlp.experts.102.up_proj", + "model.layers.50.mlp.experts.103.down_proj", + "model.layers.50.mlp.experts.103.gate_proj", + "model.layers.50.mlp.experts.103.up_proj", + "model.layers.50.mlp.experts.104.down_proj", + "model.layers.50.mlp.experts.104.gate_proj", + "model.layers.50.mlp.experts.104.up_proj", + "model.layers.50.mlp.experts.105.down_proj", + "model.layers.50.mlp.experts.105.gate_proj", + "model.layers.50.mlp.experts.105.up_proj", + "model.layers.50.mlp.experts.106.down_proj", + "model.layers.50.mlp.experts.106.gate_proj", + "model.layers.50.mlp.experts.106.up_proj", + "model.layers.50.mlp.experts.107.down_proj", + "model.layers.50.mlp.experts.107.gate_proj", + "model.layers.50.mlp.experts.107.up_proj", + "model.layers.50.mlp.experts.108.down_proj", + "model.layers.50.mlp.experts.108.gate_proj", + "model.layers.50.mlp.experts.108.up_proj", + "model.layers.50.mlp.experts.109.down_proj", + "model.layers.50.mlp.experts.109.gate_proj", + "model.layers.50.mlp.experts.109.up_proj", + "model.layers.50.mlp.experts.11.down_proj", + "model.layers.50.mlp.experts.11.gate_proj", + "model.layers.50.mlp.experts.11.up_proj", + "model.layers.50.mlp.experts.110.down_proj", + "model.layers.50.mlp.experts.110.gate_proj", + "model.layers.50.mlp.experts.110.up_proj", + "model.layers.50.mlp.experts.111.down_proj", + "model.layers.50.mlp.experts.111.gate_proj", + "model.layers.50.mlp.experts.111.up_proj", + "model.layers.50.mlp.experts.112.down_proj", + "model.layers.50.mlp.experts.112.gate_proj", + "model.layers.50.mlp.experts.112.up_proj", + "model.layers.50.mlp.experts.113.down_proj", + "model.layers.50.mlp.experts.113.gate_proj", + "model.layers.50.mlp.experts.113.up_proj", + "model.layers.50.mlp.experts.114.down_proj", + "model.layers.50.mlp.experts.114.gate_proj", + "model.layers.50.mlp.experts.114.up_proj", + "model.layers.50.mlp.experts.115.down_proj", + "model.layers.50.mlp.experts.115.gate_proj", + "model.layers.50.mlp.experts.115.up_proj", + "model.layers.50.mlp.experts.116.down_proj", + "model.layers.50.mlp.experts.116.gate_proj", + "model.layers.50.mlp.experts.116.up_proj", + "model.layers.50.mlp.experts.117.down_proj", + "model.layers.50.mlp.experts.117.gate_proj", + "model.layers.50.mlp.experts.117.up_proj", + "model.layers.50.mlp.experts.118.down_proj", + "model.layers.50.mlp.experts.118.gate_proj", + "model.layers.50.mlp.experts.118.up_proj", + "model.layers.50.mlp.experts.119.down_proj", + "model.layers.50.mlp.experts.119.gate_proj", + "model.layers.50.mlp.experts.119.up_proj", + "model.layers.50.mlp.experts.12.down_proj", + "model.layers.50.mlp.experts.12.gate_proj", + "model.layers.50.mlp.experts.12.up_proj", + "model.layers.50.mlp.experts.120.down_proj", + "model.layers.50.mlp.experts.120.gate_proj", + "model.layers.50.mlp.experts.120.up_proj", + "model.layers.50.mlp.experts.121.down_proj", + "model.layers.50.mlp.experts.121.gate_proj", + "model.layers.50.mlp.experts.121.up_proj", + "model.layers.50.mlp.experts.122.down_proj", + "model.layers.50.mlp.experts.122.gate_proj", + "model.layers.50.mlp.experts.122.up_proj", + "model.layers.50.mlp.experts.123.down_proj", + "model.layers.50.mlp.experts.123.gate_proj", + "model.layers.50.mlp.experts.123.up_proj", + "model.layers.50.mlp.experts.124.down_proj", + "model.layers.50.mlp.experts.124.gate_proj", + "model.layers.50.mlp.experts.124.up_proj", + "model.layers.50.mlp.experts.125.down_proj", + "model.layers.50.mlp.experts.125.gate_proj", + "model.layers.50.mlp.experts.125.up_proj", + "model.layers.50.mlp.experts.126.down_proj", + "model.layers.50.mlp.experts.126.gate_proj", + "model.layers.50.mlp.experts.126.up_proj", + "model.layers.50.mlp.experts.127.down_proj", + "model.layers.50.mlp.experts.127.gate_proj", + "model.layers.50.mlp.experts.127.up_proj", + "model.layers.50.mlp.experts.128.down_proj", + "model.layers.50.mlp.experts.128.gate_proj", + "model.layers.50.mlp.experts.128.up_proj", + "model.layers.50.mlp.experts.129.down_proj", + "model.layers.50.mlp.experts.129.gate_proj", + "model.layers.50.mlp.experts.129.up_proj", + "model.layers.50.mlp.experts.13.down_proj", + "model.layers.50.mlp.experts.13.gate_proj", + "model.layers.50.mlp.experts.13.up_proj", + "model.layers.50.mlp.experts.130.down_proj", + "model.layers.50.mlp.experts.130.gate_proj", + "model.layers.50.mlp.experts.130.up_proj", + "model.layers.50.mlp.experts.131.down_proj", + "model.layers.50.mlp.experts.131.gate_proj", + "model.layers.50.mlp.experts.131.up_proj", + "model.layers.50.mlp.experts.132.down_proj", + "model.layers.50.mlp.experts.132.gate_proj", + "model.layers.50.mlp.experts.132.up_proj", + "model.layers.50.mlp.experts.133.down_proj", + "model.layers.50.mlp.experts.133.gate_proj", + "model.layers.50.mlp.experts.133.up_proj", + "model.layers.50.mlp.experts.134.down_proj", + "model.layers.50.mlp.experts.134.gate_proj", + "model.layers.50.mlp.experts.134.up_proj", + "model.layers.50.mlp.experts.135.down_proj", + "model.layers.50.mlp.experts.135.gate_proj", + "model.layers.50.mlp.experts.135.up_proj", + "model.layers.50.mlp.experts.136.down_proj", + "model.layers.50.mlp.experts.136.gate_proj", + "model.layers.50.mlp.experts.136.up_proj", + "model.layers.50.mlp.experts.137.down_proj", + "model.layers.50.mlp.experts.137.gate_proj", + "model.layers.50.mlp.experts.137.up_proj", + "model.layers.50.mlp.experts.138.down_proj", + "model.layers.50.mlp.experts.138.gate_proj", + "model.layers.50.mlp.experts.138.up_proj", + "model.layers.50.mlp.experts.139.down_proj", + "model.layers.50.mlp.experts.139.gate_proj", + "model.layers.50.mlp.experts.139.up_proj", + "model.layers.50.mlp.experts.14.down_proj", + "model.layers.50.mlp.experts.14.gate_proj", + "model.layers.50.mlp.experts.14.up_proj", + "model.layers.50.mlp.experts.140.down_proj", + "model.layers.50.mlp.experts.140.gate_proj", + "model.layers.50.mlp.experts.140.up_proj", + "model.layers.50.mlp.experts.141.down_proj", + "model.layers.50.mlp.experts.141.gate_proj", + "model.layers.50.mlp.experts.141.up_proj", + "model.layers.50.mlp.experts.142.down_proj", + "model.layers.50.mlp.experts.142.gate_proj", + "model.layers.50.mlp.experts.142.up_proj", + "model.layers.50.mlp.experts.143.down_proj", + "model.layers.50.mlp.experts.143.gate_proj", + "model.layers.50.mlp.experts.143.up_proj", + "model.layers.50.mlp.experts.144.down_proj", + "model.layers.50.mlp.experts.144.gate_proj", + "model.layers.50.mlp.experts.144.up_proj", + "model.layers.50.mlp.experts.145.down_proj", + "model.layers.50.mlp.experts.145.gate_proj", + "model.layers.50.mlp.experts.145.up_proj", + "model.layers.50.mlp.experts.146.down_proj", + "model.layers.50.mlp.experts.146.gate_proj", + "model.layers.50.mlp.experts.146.up_proj", + "model.layers.50.mlp.experts.147.down_proj", + "model.layers.50.mlp.experts.147.gate_proj", + "model.layers.50.mlp.experts.147.up_proj", + "model.layers.50.mlp.experts.148.down_proj", + "model.layers.50.mlp.experts.148.gate_proj", + "model.layers.50.mlp.experts.148.up_proj", + "model.layers.50.mlp.experts.149.down_proj", + "model.layers.50.mlp.experts.149.gate_proj", + "model.layers.50.mlp.experts.149.up_proj", + "model.layers.50.mlp.experts.15.down_proj", + "model.layers.50.mlp.experts.15.gate_proj", + "model.layers.50.mlp.experts.15.up_proj", + "model.layers.50.mlp.experts.150.down_proj", + "model.layers.50.mlp.experts.150.gate_proj", + "model.layers.50.mlp.experts.150.up_proj", + "model.layers.50.mlp.experts.151.down_proj", + "model.layers.50.mlp.experts.151.gate_proj", + "model.layers.50.mlp.experts.151.up_proj", + "model.layers.50.mlp.experts.152.down_proj", + "model.layers.50.mlp.experts.152.gate_proj", + "model.layers.50.mlp.experts.152.up_proj", + "model.layers.50.mlp.experts.153.down_proj", + "model.layers.50.mlp.experts.153.gate_proj", + "model.layers.50.mlp.experts.153.up_proj", + "model.layers.50.mlp.experts.154.down_proj", + "model.layers.50.mlp.experts.154.gate_proj", + "model.layers.50.mlp.experts.154.up_proj", + "model.layers.50.mlp.experts.155.down_proj", + "model.layers.50.mlp.experts.155.gate_proj", + "model.layers.50.mlp.experts.155.up_proj", + "model.layers.50.mlp.experts.156.down_proj", + "model.layers.50.mlp.experts.156.gate_proj", + "model.layers.50.mlp.experts.156.up_proj", + "model.layers.50.mlp.experts.157.down_proj", + "model.layers.50.mlp.experts.157.gate_proj", + "model.layers.50.mlp.experts.157.up_proj", + "model.layers.50.mlp.experts.158.down_proj", + "model.layers.50.mlp.experts.158.gate_proj", + "model.layers.50.mlp.experts.158.up_proj", + "model.layers.50.mlp.experts.159.down_proj", + "model.layers.50.mlp.experts.159.gate_proj", + "model.layers.50.mlp.experts.159.up_proj", + "model.layers.50.mlp.experts.16.down_proj", + "model.layers.50.mlp.experts.16.gate_proj", + "model.layers.50.mlp.experts.16.up_proj", + "model.layers.50.mlp.experts.160.down_proj", + "model.layers.50.mlp.experts.160.gate_proj", + "model.layers.50.mlp.experts.160.up_proj", + "model.layers.50.mlp.experts.161.down_proj", + "model.layers.50.mlp.experts.161.gate_proj", + "model.layers.50.mlp.experts.161.up_proj", + "model.layers.50.mlp.experts.162.down_proj", + "model.layers.50.mlp.experts.162.gate_proj", + "model.layers.50.mlp.experts.162.up_proj", + "model.layers.50.mlp.experts.163.down_proj", + "model.layers.50.mlp.experts.163.gate_proj", + "model.layers.50.mlp.experts.163.up_proj", + "model.layers.50.mlp.experts.164.down_proj", + "model.layers.50.mlp.experts.164.gate_proj", + "model.layers.50.mlp.experts.164.up_proj", + "model.layers.50.mlp.experts.165.down_proj", + "model.layers.50.mlp.experts.165.gate_proj", + "model.layers.50.mlp.experts.165.up_proj", + "model.layers.50.mlp.experts.166.down_proj", + "model.layers.50.mlp.experts.166.gate_proj", + "model.layers.50.mlp.experts.166.up_proj", + "model.layers.50.mlp.experts.167.down_proj", + "model.layers.50.mlp.experts.167.gate_proj", + "model.layers.50.mlp.experts.167.up_proj", + "model.layers.50.mlp.experts.168.down_proj", + "model.layers.50.mlp.experts.168.gate_proj", + "model.layers.50.mlp.experts.168.up_proj", + "model.layers.50.mlp.experts.169.down_proj", + "model.layers.50.mlp.experts.169.gate_proj", + "model.layers.50.mlp.experts.169.up_proj", + "model.layers.50.mlp.experts.17.down_proj", + "model.layers.50.mlp.experts.17.gate_proj", + "model.layers.50.mlp.experts.17.up_proj", + "model.layers.50.mlp.experts.170.down_proj", + "model.layers.50.mlp.experts.170.gate_proj", + "model.layers.50.mlp.experts.170.up_proj", + "model.layers.50.mlp.experts.171.down_proj", + "model.layers.50.mlp.experts.171.gate_proj", + "model.layers.50.mlp.experts.171.up_proj", + "model.layers.50.mlp.experts.172.down_proj", + "model.layers.50.mlp.experts.172.gate_proj", + "model.layers.50.mlp.experts.172.up_proj", + "model.layers.50.mlp.experts.173.down_proj", + "model.layers.50.mlp.experts.173.gate_proj", + "model.layers.50.mlp.experts.173.up_proj", + "model.layers.50.mlp.experts.174.down_proj", + "model.layers.50.mlp.experts.174.gate_proj", + "model.layers.50.mlp.experts.174.up_proj", + "model.layers.50.mlp.experts.175.down_proj", + "model.layers.50.mlp.experts.175.gate_proj", + "model.layers.50.mlp.experts.175.up_proj", + "model.layers.50.mlp.experts.176.down_proj", + "model.layers.50.mlp.experts.176.gate_proj", + "model.layers.50.mlp.experts.176.up_proj", + "model.layers.50.mlp.experts.177.down_proj", + "model.layers.50.mlp.experts.177.gate_proj", + "model.layers.50.mlp.experts.177.up_proj", + "model.layers.50.mlp.experts.178.down_proj", + "model.layers.50.mlp.experts.178.gate_proj", + "model.layers.50.mlp.experts.178.up_proj", + "model.layers.50.mlp.experts.179.down_proj", + "model.layers.50.mlp.experts.179.gate_proj", + "model.layers.50.mlp.experts.179.up_proj", + "model.layers.50.mlp.experts.18.down_proj", + "model.layers.50.mlp.experts.18.gate_proj", + "model.layers.50.mlp.experts.18.up_proj", + "model.layers.50.mlp.experts.180.down_proj", + "model.layers.50.mlp.experts.180.gate_proj", + "model.layers.50.mlp.experts.180.up_proj", + "model.layers.50.mlp.experts.181.down_proj", + "model.layers.50.mlp.experts.181.gate_proj", + "model.layers.50.mlp.experts.181.up_proj", + "model.layers.50.mlp.experts.182.down_proj", + "model.layers.50.mlp.experts.182.gate_proj", + "model.layers.50.mlp.experts.182.up_proj", + "model.layers.50.mlp.experts.183.down_proj", + "model.layers.50.mlp.experts.183.gate_proj", + "model.layers.50.mlp.experts.183.up_proj", + "model.layers.50.mlp.experts.184.down_proj", + "model.layers.50.mlp.experts.184.gate_proj", + "model.layers.50.mlp.experts.184.up_proj", + "model.layers.50.mlp.experts.185.down_proj", + "model.layers.50.mlp.experts.185.gate_proj", + "model.layers.50.mlp.experts.185.up_proj", + "model.layers.50.mlp.experts.186.down_proj", + "model.layers.50.mlp.experts.186.gate_proj", + "model.layers.50.mlp.experts.186.up_proj", + "model.layers.50.mlp.experts.187.down_proj", + "model.layers.50.mlp.experts.187.gate_proj", + "model.layers.50.mlp.experts.187.up_proj", + "model.layers.50.mlp.experts.188.down_proj", + "model.layers.50.mlp.experts.188.gate_proj", + "model.layers.50.mlp.experts.188.up_proj", + "model.layers.50.mlp.experts.189.down_proj", + "model.layers.50.mlp.experts.189.gate_proj", + "model.layers.50.mlp.experts.189.up_proj", + "model.layers.50.mlp.experts.19.down_proj", + "model.layers.50.mlp.experts.19.gate_proj", + "model.layers.50.mlp.experts.19.up_proj", + "model.layers.50.mlp.experts.190.down_proj", + "model.layers.50.mlp.experts.190.gate_proj", + "model.layers.50.mlp.experts.190.up_proj", + "model.layers.50.mlp.experts.191.down_proj", + "model.layers.50.mlp.experts.191.gate_proj", + "model.layers.50.mlp.experts.191.up_proj", + "model.layers.50.mlp.experts.192.down_proj", + "model.layers.50.mlp.experts.192.gate_proj", + "model.layers.50.mlp.experts.192.up_proj", + "model.layers.50.mlp.experts.193.down_proj", + "model.layers.50.mlp.experts.193.gate_proj", + "model.layers.50.mlp.experts.193.up_proj", + "model.layers.50.mlp.experts.194.down_proj", + "model.layers.50.mlp.experts.194.gate_proj", + "model.layers.50.mlp.experts.194.up_proj", + "model.layers.50.mlp.experts.195.down_proj", + "model.layers.50.mlp.experts.195.gate_proj", + "model.layers.50.mlp.experts.195.up_proj", + "model.layers.50.mlp.experts.196.down_proj", + "model.layers.50.mlp.experts.196.gate_proj", + "model.layers.50.mlp.experts.196.up_proj", + "model.layers.50.mlp.experts.197.down_proj", + "model.layers.50.mlp.experts.197.gate_proj", + "model.layers.50.mlp.experts.197.up_proj", + "model.layers.50.mlp.experts.198.down_proj", + "model.layers.50.mlp.experts.198.gate_proj", + "model.layers.50.mlp.experts.198.up_proj", + "model.layers.50.mlp.experts.199.down_proj", + "model.layers.50.mlp.experts.199.gate_proj", + "model.layers.50.mlp.experts.199.up_proj", + "model.layers.50.mlp.experts.2.down_proj", + "model.layers.50.mlp.experts.2.gate_proj", + "model.layers.50.mlp.experts.2.up_proj", + "model.layers.50.mlp.experts.20.down_proj", + "model.layers.50.mlp.experts.20.gate_proj", + "model.layers.50.mlp.experts.20.up_proj", + "model.layers.50.mlp.experts.200.down_proj", + "model.layers.50.mlp.experts.200.gate_proj", + "model.layers.50.mlp.experts.200.up_proj", + "model.layers.50.mlp.experts.201.down_proj", + "model.layers.50.mlp.experts.201.gate_proj", + "model.layers.50.mlp.experts.201.up_proj", + "model.layers.50.mlp.experts.202.down_proj", + "model.layers.50.mlp.experts.202.gate_proj", + "model.layers.50.mlp.experts.202.up_proj", + "model.layers.50.mlp.experts.203.down_proj", + "model.layers.50.mlp.experts.203.gate_proj", + "model.layers.50.mlp.experts.203.up_proj", + "model.layers.50.mlp.experts.204.down_proj", + "model.layers.50.mlp.experts.204.gate_proj", + "model.layers.50.mlp.experts.204.up_proj", + "model.layers.50.mlp.experts.205.down_proj", + "model.layers.50.mlp.experts.205.gate_proj", + "model.layers.50.mlp.experts.205.up_proj", + "model.layers.50.mlp.experts.206.down_proj", + "model.layers.50.mlp.experts.206.gate_proj", + "model.layers.50.mlp.experts.206.up_proj", + "model.layers.50.mlp.experts.207.down_proj", + "model.layers.50.mlp.experts.207.gate_proj", + "model.layers.50.mlp.experts.207.up_proj", + "model.layers.50.mlp.experts.208.down_proj", + "model.layers.50.mlp.experts.208.gate_proj", + "model.layers.50.mlp.experts.208.up_proj", + "model.layers.50.mlp.experts.209.down_proj", + "model.layers.50.mlp.experts.209.gate_proj", + "model.layers.50.mlp.experts.209.up_proj", + "model.layers.50.mlp.experts.21.down_proj", + "model.layers.50.mlp.experts.21.gate_proj", + "model.layers.50.mlp.experts.21.up_proj", + "model.layers.50.mlp.experts.210.down_proj", + "model.layers.50.mlp.experts.210.gate_proj", + "model.layers.50.mlp.experts.210.up_proj", + "model.layers.50.mlp.experts.211.down_proj", + "model.layers.50.mlp.experts.211.gate_proj", + "model.layers.50.mlp.experts.211.up_proj", + "model.layers.50.mlp.experts.212.down_proj", + "model.layers.50.mlp.experts.212.gate_proj", + "model.layers.50.mlp.experts.212.up_proj", + "model.layers.50.mlp.experts.213.down_proj", + "model.layers.50.mlp.experts.213.gate_proj", + "model.layers.50.mlp.experts.213.up_proj", + "model.layers.50.mlp.experts.214.down_proj", + "model.layers.50.mlp.experts.214.gate_proj", + "model.layers.50.mlp.experts.214.up_proj", + "model.layers.50.mlp.experts.215.down_proj", + "model.layers.50.mlp.experts.215.gate_proj", + "model.layers.50.mlp.experts.215.up_proj", + "model.layers.50.mlp.experts.216.down_proj", + "model.layers.50.mlp.experts.216.gate_proj", + "model.layers.50.mlp.experts.216.up_proj", + "model.layers.50.mlp.experts.217.down_proj", + "model.layers.50.mlp.experts.217.gate_proj", + "model.layers.50.mlp.experts.217.up_proj", + "model.layers.50.mlp.experts.218.down_proj", + "model.layers.50.mlp.experts.218.gate_proj", + "model.layers.50.mlp.experts.218.up_proj", + "model.layers.50.mlp.experts.219.down_proj", + "model.layers.50.mlp.experts.219.gate_proj", + "model.layers.50.mlp.experts.219.up_proj", + "model.layers.50.mlp.experts.22.down_proj", + "model.layers.50.mlp.experts.22.gate_proj", + "model.layers.50.mlp.experts.22.up_proj", + "model.layers.50.mlp.experts.220.down_proj", + "model.layers.50.mlp.experts.220.gate_proj", + "model.layers.50.mlp.experts.220.up_proj", + "model.layers.50.mlp.experts.221.down_proj", + "model.layers.50.mlp.experts.221.gate_proj", + "model.layers.50.mlp.experts.221.up_proj", + "model.layers.50.mlp.experts.222.down_proj", + "model.layers.50.mlp.experts.222.gate_proj", + "model.layers.50.mlp.experts.222.up_proj", + "model.layers.50.mlp.experts.223.down_proj", + "model.layers.50.mlp.experts.223.gate_proj", + "model.layers.50.mlp.experts.223.up_proj", + "model.layers.50.mlp.experts.224.down_proj", + "model.layers.50.mlp.experts.224.gate_proj", + "model.layers.50.mlp.experts.224.up_proj", + "model.layers.50.mlp.experts.225.down_proj", + "model.layers.50.mlp.experts.225.gate_proj", + "model.layers.50.mlp.experts.225.up_proj", + "model.layers.50.mlp.experts.226.down_proj", + "model.layers.50.mlp.experts.226.gate_proj", + "model.layers.50.mlp.experts.226.up_proj", + "model.layers.50.mlp.experts.227.down_proj", + "model.layers.50.mlp.experts.227.gate_proj", + "model.layers.50.mlp.experts.227.up_proj", + "model.layers.50.mlp.experts.228.down_proj", + "model.layers.50.mlp.experts.228.gate_proj", + "model.layers.50.mlp.experts.228.up_proj", + "model.layers.50.mlp.experts.229.down_proj", + "model.layers.50.mlp.experts.229.gate_proj", + "model.layers.50.mlp.experts.229.up_proj", + "model.layers.50.mlp.experts.23.down_proj", + "model.layers.50.mlp.experts.23.gate_proj", + "model.layers.50.mlp.experts.23.up_proj", + "model.layers.50.mlp.experts.230.down_proj", + "model.layers.50.mlp.experts.230.gate_proj", + "model.layers.50.mlp.experts.230.up_proj", + "model.layers.50.mlp.experts.231.down_proj", + "model.layers.50.mlp.experts.231.gate_proj", + "model.layers.50.mlp.experts.231.up_proj", + "model.layers.50.mlp.experts.232.down_proj", + "model.layers.50.mlp.experts.232.gate_proj", + "model.layers.50.mlp.experts.232.up_proj", + "model.layers.50.mlp.experts.233.down_proj", + "model.layers.50.mlp.experts.233.gate_proj", + "model.layers.50.mlp.experts.233.up_proj", + "model.layers.50.mlp.experts.234.down_proj", + "model.layers.50.mlp.experts.234.gate_proj", + "model.layers.50.mlp.experts.234.up_proj", + "model.layers.50.mlp.experts.235.down_proj", + "model.layers.50.mlp.experts.235.gate_proj", + "model.layers.50.mlp.experts.235.up_proj", + "model.layers.50.mlp.experts.236.down_proj", + "model.layers.50.mlp.experts.236.gate_proj", + "model.layers.50.mlp.experts.236.up_proj", + "model.layers.50.mlp.experts.237.down_proj", + "model.layers.50.mlp.experts.237.gate_proj", + "model.layers.50.mlp.experts.237.up_proj", + "model.layers.50.mlp.experts.238.down_proj", + "model.layers.50.mlp.experts.238.gate_proj", + "model.layers.50.mlp.experts.238.up_proj", + "model.layers.50.mlp.experts.239.down_proj", + "model.layers.50.mlp.experts.239.gate_proj", + "model.layers.50.mlp.experts.239.up_proj", + "model.layers.50.mlp.experts.24.down_proj", + "model.layers.50.mlp.experts.24.gate_proj", + "model.layers.50.mlp.experts.24.up_proj", + "model.layers.50.mlp.experts.240.down_proj", + "model.layers.50.mlp.experts.240.gate_proj", + "model.layers.50.mlp.experts.240.up_proj", + "model.layers.50.mlp.experts.241.down_proj", + "model.layers.50.mlp.experts.241.gate_proj", + "model.layers.50.mlp.experts.241.up_proj", + "model.layers.50.mlp.experts.242.down_proj", + "model.layers.50.mlp.experts.242.gate_proj", + "model.layers.50.mlp.experts.242.up_proj", + "model.layers.50.mlp.experts.243.down_proj", + "model.layers.50.mlp.experts.243.gate_proj", + "model.layers.50.mlp.experts.243.up_proj", + "model.layers.50.mlp.experts.244.down_proj", + "model.layers.50.mlp.experts.244.gate_proj", + "model.layers.50.mlp.experts.244.up_proj", + "model.layers.50.mlp.experts.245.down_proj", + "model.layers.50.mlp.experts.245.gate_proj", + "model.layers.50.mlp.experts.245.up_proj", + "model.layers.50.mlp.experts.246.down_proj", + "model.layers.50.mlp.experts.246.gate_proj", + "model.layers.50.mlp.experts.246.up_proj", + "model.layers.50.mlp.experts.247.down_proj", + "model.layers.50.mlp.experts.247.gate_proj", + "model.layers.50.mlp.experts.247.up_proj", + "model.layers.50.mlp.experts.248.down_proj", + "model.layers.50.mlp.experts.248.gate_proj", + "model.layers.50.mlp.experts.248.up_proj", + "model.layers.50.mlp.experts.249.down_proj", + "model.layers.50.mlp.experts.249.gate_proj", + "model.layers.50.mlp.experts.249.up_proj", + "model.layers.50.mlp.experts.25.down_proj", + "model.layers.50.mlp.experts.25.gate_proj", + "model.layers.50.mlp.experts.25.up_proj", + "model.layers.50.mlp.experts.250.down_proj", + "model.layers.50.mlp.experts.250.gate_proj", + "model.layers.50.mlp.experts.250.up_proj", + "model.layers.50.mlp.experts.251.down_proj", + "model.layers.50.mlp.experts.251.gate_proj", + "model.layers.50.mlp.experts.251.up_proj", + "model.layers.50.mlp.experts.252.down_proj", + "model.layers.50.mlp.experts.252.gate_proj", + "model.layers.50.mlp.experts.252.up_proj", + "model.layers.50.mlp.experts.253.down_proj", + "model.layers.50.mlp.experts.253.gate_proj", + "model.layers.50.mlp.experts.253.up_proj", + "model.layers.50.mlp.experts.254.down_proj", + "model.layers.50.mlp.experts.254.gate_proj", + "model.layers.50.mlp.experts.254.up_proj", + "model.layers.50.mlp.experts.255.down_proj", + "model.layers.50.mlp.experts.255.gate_proj", + "model.layers.50.mlp.experts.255.up_proj", + "model.layers.50.mlp.experts.26.down_proj", + "model.layers.50.mlp.experts.26.gate_proj", + "model.layers.50.mlp.experts.26.up_proj", + "model.layers.50.mlp.experts.27.down_proj", + "model.layers.50.mlp.experts.27.gate_proj", + "model.layers.50.mlp.experts.27.up_proj", + "model.layers.50.mlp.experts.28.down_proj", + "model.layers.50.mlp.experts.28.gate_proj", + "model.layers.50.mlp.experts.28.up_proj", + "model.layers.50.mlp.experts.29.down_proj", + "model.layers.50.mlp.experts.29.gate_proj", + "model.layers.50.mlp.experts.29.up_proj", + "model.layers.50.mlp.experts.3.down_proj", + "model.layers.50.mlp.experts.3.gate_proj", + "model.layers.50.mlp.experts.3.up_proj", + "model.layers.50.mlp.experts.30.down_proj", + "model.layers.50.mlp.experts.30.gate_proj", + "model.layers.50.mlp.experts.30.up_proj", + "model.layers.50.mlp.experts.31.down_proj", + "model.layers.50.mlp.experts.31.gate_proj", + "model.layers.50.mlp.experts.31.up_proj", + "model.layers.50.mlp.experts.32.down_proj", + "model.layers.50.mlp.experts.32.gate_proj", + "model.layers.50.mlp.experts.32.up_proj", + "model.layers.50.mlp.experts.33.down_proj", + "model.layers.50.mlp.experts.33.gate_proj", + "model.layers.50.mlp.experts.33.up_proj", + "model.layers.50.mlp.experts.34.down_proj", + "model.layers.50.mlp.experts.34.gate_proj", + "model.layers.50.mlp.experts.34.up_proj", + "model.layers.50.mlp.experts.35.down_proj", + "model.layers.50.mlp.experts.35.gate_proj", + "model.layers.50.mlp.experts.35.up_proj", + "model.layers.50.mlp.experts.36.down_proj", + "model.layers.50.mlp.experts.36.gate_proj", + "model.layers.50.mlp.experts.36.up_proj", + "model.layers.50.mlp.experts.37.down_proj", + "model.layers.50.mlp.experts.37.gate_proj", + "model.layers.50.mlp.experts.37.up_proj", + "model.layers.50.mlp.experts.38.down_proj", + "model.layers.50.mlp.experts.38.gate_proj", + "model.layers.50.mlp.experts.38.up_proj", + "model.layers.50.mlp.experts.39.down_proj", + "model.layers.50.mlp.experts.39.gate_proj", + "model.layers.50.mlp.experts.39.up_proj", + "model.layers.50.mlp.experts.4.down_proj", + "model.layers.50.mlp.experts.4.gate_proj", + "model.layers.50.mlp.experts.4.up_proj", + "model.layers.50.mlp.experts.40.down_proj", + "model.layers.50.mlp.experts.40.gate_proj", + "model.layers.50.mlp.experts.40.up_proj", + "model.layers.50.mlp.experts.41.down_proj", + "model.layers.50.mlp.experts.41.gate_proj", + "model.layers.50.mlp.experts.41.up_proj", + "model.layers.50.mlp.experts.42.down_proj", + "model.layers.50.mlp.experts.42.gate_proj", + "model.layers.50.mlp.experts.42.up_proj", + "model.layers.50.mlp.experts.43.down_proj", + "model.layers.50.mlp.experts.43.gate_proj", + "model.layers.50.mlp.experts.43.up_proj", + "model.layers.50.mlp.experts.44.down_proj", + "model.layers.50.mlp.experts.44.gate_proj", + "model.layers.50.mlp.experts.44.up_proj", + "model.layers.50.mlp.experts.45.down_proj", + "model.layers.50.mlp.experts.45.gate_proj", + "model.layers.50.mlp.experts.45.up_proj", + "model.layers.50.mlp.experts.46.down_proj", + "model.layers.50.mlp.experts.46.gate_proj", + "model.layers.50.mlp.experts.46.up_proj", + "model.layers.50.mlp.experts.47.down_proj", + "model.layers.50.mlp.experts.47.gate_proj", + "model.layers.50.mlp.experts.47.up_proj", + "model.layers.50.mlp.experts.48.down_proj", + "model.layers.50.mlp.experts.48.gate_proj", + "model.layers.50.mlp.experts.48.up_proj", + "model.layers.50.mlp.experts.49.down_proj", + "model.layers.50.mlp.experts.49.gate_proj", + "model.layers.50.mlp.experts.49.up_proj", + "model.layers.50.mlp.experts.5.down_proj", + "model.layers.50.mlp.experts.5.gate_proj", + "model.layers.50.mlp.experts.5.up_proj", + "model.layers.50.mlp.experts.50.down_proj", + "model.layers.50.mlp.experts.50.gate_proj", + "model.layers.50.mlp.experts.50.up_proj", + "model.layers.50.mlp.experts.51.down_proj", + "model.layers.50.mlp.experts.51.gate_proj", + "model.layers.50.mlp.experts.51.up_proj", + "model.layers.50.mlp.experts.52.down_proj", + "model.layers.50.mlp.experts.52.gate_proj", + "model.layers.50.mlp.experts.52.up_proj", + "model.layers.50.mlp.experts.53.down_proj", + "model.layers.50.mlp.experts.53.gate_proj", + "model.layers.50.mlp.experts.53.up_proj", + "model.layers.50.mlp.experts.54.down_proj", + "model.layers.50.mlp.experts.54.gate_proj", + "model.layers.50.mlp.experts.54.up_proj", + "model.layers.50.mlp.experts.55.down_proj", + "model.layers.50.mlp.experts.55.gate_proj", + "model.layers.50.mlp.experts.55.up_proj", + "model.layers.50.mlp.experts.56.down_proj", + "model.layers.50.mlp.experts.56.gate_proj", + "model.layers.50.mlp.experts.56.up_proj", + "model.layers.50.mlp.experts.57.down_proj", + "model.layers.50.mlp.experts.57.gate_proj", + "model.layers.50.mlp.experts.57.up_proj", + "model.layers.50.mlp.experts.58.down_proj", + "model.layers.50.mlp.experts.58.gate_proj", + "model.layers.50.mlp.experts.58.up_proj", + "model.layers.50.mlp.experts.59.down_proj", + "model.layers.50.mlp.experts.59.gate_proj", + "model.layers.50.mlp.experts.59.up_proj", + "model.layers.50.mlp.experts.6.down_proj", + "model.layers.50.mlp.experts.6.gate_proj", + "model.layers.50.mlp.experts.6.up_proj", + "model.layers.50.mlp.experts.60.down_proj", + "model.layers.50.mlp.experts.60.gate_proj", + "model.layers.50.mlp.experts.60.up_proj", + "model.layers.50.mlp.experts.61.down_proj", + "model.layers.50.mlp.experts.61.gate_proj", + "model.layers.50.mlp.experts.61.up_proj", + "model.layers.50.mlp.experts.62.down_proj", + "model.layers.50.mlp.experts.62.gate_proj", + "model.layers.50.mlp.experts.62.up_proj", + "model.layers.50.mlp.experts.63.down_proj", + "model.layers.50.mlp.experts.63.gate_proj", + "model.layers.50.mlp.experts.63.up_proj", + "model.layers.50.mlp.experts.64.down_proj", + "model.layers.50.mlp.experts.64.gate_proj", + "model.layers.50.mlp.experts.64.up_proj", + "model.layers.50.mlp.experts.65.down_proj", + "model.layers.50.mlp.experts.65.gate_proj", + "model.layers.50.mlp.experts.65.up_proj", + "model.layers.50.mlp.experts.66.down_proj", + "model.layers.50.mlp.experts.66.gate_proj", + "model.layers.50.mlp.experts.66.up_proj", + "model.layers.50.mlp.experts.67.down_proj", + "model.layers.50.mlp.experts.67.gate_proj", + "model.layers.50.mlp.experts.67.up_proj", + "model.layers.50.mlp.experts.68.down_proj", + "model.layers.50.mlp.experts.68.gate_proj", + "model.layers.50.mlp.experts.68.up_proj", + "model.layers.50.mlp.experts.69.down_proj", + "model.layers.50.mlp.experts.69.gate_proj", + "model.layers.50.mlp.experts.69.up_proj", + "model.layers.50.mlp.experts.7.down_proj", + "model.layers.50.mlp.experts.7.gate_proj", + "model.layers.50.mlp.experts.7.up_proj", + "model.layers.50.mlp.experts.70.down_proj", + "model.layers.50.mlp.experts.70.gate_proj", + "model.layers.50.mlp.experts.70.up_proj", + "model.layers.50.mlp.experts.71.down_proj", + "model.layers.50.mlp.experts.71.gate_proj", + "model.layers.50.mlp.experts.71.up_proj", + "model.layers.50.mlp.experts.72.down_proj", + "model.layers.50.mlp.experts.72.gate_proj", + "model.layers.50.mlp.experts.72.up_proj", + "model.layers.50.mlp.experts.73.down_proj", + "model.layers.50.mlp.experts.73.gate_proj", + "model.layers.50.mlp.experts.73.up_proj", + "model.layers.50.mlp.experts.74.down_proj", + "model.layers.50.mlp.experts.74.gate_proj", + "model.layers.50.mlp.experts.74.up_proj", + "model.layers.50.mlp.experts.75.down_proj", + "model.layers.50.mlp.experts.75.gate_proj", + "model.layers.50.mlp.experts.75.up_proj", + "model.layers.50.mlp.experts.76.down_proj", + "model.layers.50.mlp.experts.76.gate_proj", + "model.layers.50.mlp.experts.76.up_proj", + "model.layers.50.mlp.experts.77.down_proj", + "model.layers.50.mlp.experts.77.gate_proj", + "model.layers.50.mlp.experts.77.up_proj", + "model.layers.50.mlp.experts.78.down_proj", + "model.layers.50.mlp.experts.78.gate_proj", + "model.layers.50.mlp.experts.78.up_proj", + "model.layers.50.mlp.experts.79.down_proj", + "model.layers.50.mlp.experts.79.gate_proj", + "model.layers.50.mlp.experts.79.up_proj", + "model.layers.50.mlp.experts.8.down_proj", + "model.layers.50.mlp.experts.8.gate_proj", + "model.layers.50.mlp.experts.8.up_proj", + "model.layers.50.mlp.experts.80.down_proj", + "model.layers.50.mlp.experts.80.gate_proj", + "model.layers.50.mlp.experts.80.up_proj", + "model.layers.50.mlp.experts.81.down_proj", + "model.layers.50.mlp.experts.81.gate_proj", + "model.layers.50.mlp.experts.81.up_proj", + "model.layers.50.mlp.experts.82.down_proj", + "model.layers.50.mlp.experts.82.gate_proj", + "model.layers.50.mlp.experts.82.up_proj", + "model.layers.50.mlp.experts.83.down_proj", + "model.layers.50.mlp.experts.83.gate_proj", + "model.layers.50.mlp.experts.83.up_proj", + "model.layers.50.mlp.experts.84.down_proj", + "model.layers.50.mlp.experts.84.gate_proj", + "model.layers.50.mlp.experts.84.up_proj", + "model.layers.50.mlp.experts.85.down_proj", + "model.layers.50.mlp.experts.85.gate_proj", + "model.layers.50.mlp.experts.85.up_proj", + "model.layers.50.mlp.experts.86.down_proj", + "model.layers.50.mlp.experts.86.gate_proj", + "model.layers.50.mlp.experts.86.up_proj", + "model.layers.50.mlp.experts.87.down_proj", + "model.layers.50.mlp.experts.87.gate_proj", + "model.layers.50.mlp.experts.87.up_proj", + "model.layers.50.mlp.experts.88.down_proj", + "model.layers.50.mlp.experts.88.gate_proj", + "model.layers.50.mlp.experts.88.up_proj", + "model.layers.50.mlp.experts.89.down_proj", + "model.layers.50.mlp.experts.89.gate_proj", + "model.layers.50.mlp.experts.89.up_proj", + "model.layers.50.mlp.experts.9.down_proj", + "model.layers.50.mlp.experts.9.gate_proj", + "model.layers.50.mlp.experts.9.up_proj", + "model.layers.50.mlp.experts.90.down_proj", + "model.layers.50.mlp.experts.90.gate_proj", + "model.layers.50.mlp.experts.90.up_proj", + "model.layers.50.mlp.experts.91.down_proj", + "model.layers.50.mlp.experts.91.gate_proj", + "model.layers.50.mlp.experts.91.up_proj", + "model.layers.50.mlp.experts.92.down_proj", + "model.layers.50.mlp.experts.92.gate_proj", + "model.layers.50.mlp.experts.92.up_proj", + "model.layers.50.mlp.experts.93.down_proj", + "model.layers.50.mlp.experts.93.gate_proj", + "model.layers.50.mlp.experts.93.up_proj", + "model.layers.50.mlp.experts.94.down_proj", + "model.layers.50.mlp.experts.94.gate_proj", + "model.layers.50.mlp.experts.94.up_proj", + "model.layers.50.mlp.experts.95.down_proj", + "model.layers.50.mlp.experts.95.gate_proj", + "model.layers.50.mlp.experts.95.up_proj", + "model.layers.50.mlp.experts.96.down_proj", + "model.layers.50.mlp.experts.96.gate_proj", + "model.layers.50.mlp.experts.96.up_proj", + "model.layers.50.mlp.experts.97.down_proj", + "model.layers.50.mlp.experts.97.gate_proj", + "model.layers.50.mlp.experts.97.up_proj", + "model.layers.50.mlp.experts.98.down_proj", + "model.layers.50.mlp.experts.98.gate_proj", + "model.layers.50.mlp.experts.98.up_proj", + "model.layers.50.mlp.experts.99.down_proj", + "model.layers.50.mlp.experts.99.gate_proj", + "model.layers.50.mlp.experts.99.up_proj", + "model.layers.51.mlp.experts.0.down_proj", + "model.layers.51.mlp.experts.0.gate_proj", + "model.layers.51.mlp.experts.0.up_proj", + "model.layers.51.mlp.experts.1.down_proj", + "model.layers.51.mlp.experts.1.gate_proj", + "model.layers.51.mlp.experts.1.up_proj", + "model.layers.51.mlp.experts.10.down_proj", + "model.layers.51.mlp.experts.10.gate_proj", + "model.layers.51.mlp.experts.10.up_proj", + "model.layers.51.mlp.experts.100.down_proj", + "model.layers.51.mlp.experts.100.gate_proj", + "model.layers.51.mlp.experts.100.up_proj", + "model.layers.51.mlp.experts.101.down_proj", + "model.layers.51.mlp.experts.101.gate_proj", + "model.layers.51.mlp.experts.101.up_proj", + "model.layers.51.mlp.experts.102.down_proj", + "model.layers.51.mlp.experts.102.gate_proj", + "model.layers.51.mlp.experts.102.up_proj", + "model.layers.51.mlp.experts.103.down_proj", + "model.layers.51.mlp.experts.103.gate_proj", + "model.layers.51.mlp.experts.103.up_proj", + "model.layers.51.mlp.experts.104.down_proj", + "model.layers.51.mlp.experts.104.gate_proj", + "model.layers.51.mlp.experts.104.up_proj", + "model.layers.51.mlp.experts.105.down_proj", + "model.layers.51.mlp.experts.105.gate_proj", + "model.layers.51.mlp.experts.105.up_proj", + "model.layers.51.mlp.experts.106.down_proj", + "model.layers.51.mlp.experts.106.gate_proj", + "model.layers.51.mlp.experts.106.up_proj", + "model.layers.51.mlp.experts.107.down_proj", + "model.layers.51.mlp.experts.107.gate_proj", + "model.layers.51.mlp.experts.107.up_proj", + "model.layers.51.mlp.experts.108.down_proj", + "model.layers.51.mlp.experts.108.gate_proj", + "model.layers.51.mlp.experts.108.up_proj", + "model.layers.51.mlp.experts.109.down_proj", + "model.layers.51.mlp.experts.109.gate_proj", + "model.layers.51.mlp.experts.109.up_proj", + "model.layers.51.mlp.experts.11.down_proj", + "model.layers.51.mlp.experts.11.gate_proj", + "model.layers.51.mlp.experts.11.up_proj", + "model.layers.51.mlp.experts.110.down_proj", + "model.layers.51.mlp.experts.110.gate_proj", + "model.layers.51.mlp.experts.110.up_proj", + "model.layers.51.mlp.experts.111.down_proj", + "model.layers.51.mlp.experts.111.gate_proj", + "model.layers.51.mlp.experts.111.up_proj", + "model.layers.51.mlp.experts.112.down_proj", + "model.layers.51.mlp.experts.112.gate_proj", + "model.layers.51.mlp.experts.112.up_proj", + "model.layers.51.mlp.experts.113.down_proj", + "model.layers.51.mlp.experts.113.gate_proj", + "model.layers.51.mlp.experts.113.up_proj", + "model.layers.51.mlp.experts.114.down_proj", + "model.layers.51.mlp.experts.114.gate_proj", + "model.layers.51.mlp.experts.114.up_proj", + "model.layers.51.mlp.experts.115.down_proj", + "model.layers.51.mlp.experts.115.gate_proj", + "model.layers.51.mlp.experts.115.up_proj", + "model.layers.51.mlp.experts.116.down_proj", + "model.layers.51.mlp.experts.116.gate_proj", + "model.layers.51.mlp.experts.116.up_proj", + "model.layers.51.mlp.experts.117.down_proj", + "model.layers.51.mlp.experts.117.gate_proj", + "model.layers.51.mlp.experts.117.up_proj", + "model.layers.51.mlp.experts.118.down_proj", + "model.layers.51.mlp.experts.118.gate_proj", + "model.layers.51.mlp.experts.118.up_proj", + "model.layers.51.mlp.experts.119.down_proj", + "model.layers.51.mlp.experts.119.gate_proj", + "model.layers.51.mlp.experts.119.up_proj", + "model.layers.51.mlp.experts.12.down_proj", + "model.layers.51.mlp.experts.12.gate_proj", + "model.layers.51.mlp.experts.12.up_proj", + "model.layers.51.mlp.experts.120.down_proj", + "model.layers.51.mlp.experts.120.gate_proj", + "model.layers.51.mlp.experts.120.up_proj", + "model.layers.51.mlp.experts.121.down_proj", + "model.layers.51.mlp.experts.121.gate_proj", + "model.layers.51.mlp.experts.121.up_proj", + "model.layers.51.mlp.experts.122.down_proj", + "model.layers.51.mlp.experts.122.gate_proj", + "model.layers.51.mlp.experts.122.up_proj", + "model.layers.51.mlp.experts.123.down_proj", + "model.layers.51.mlp.experts.123.gate_proj", + "model.layers.51.mlp.experts.123.up_proj", + "model.layers.51.mlp.experts.124.down_proj", + "model.layers.51.mlp.experts.124.gate_proj", + "model.layers.51.mlp.experts.124.up_proj", + "model.layers.51.mlp.experts.125.down_proj", + "model.layers.51.mlp.experts.125.gate_proj", + "model.layers.51.mlp.experts.125.up_proj", + "model.layers.51.mlp.experts.126.down_proj", + "model.layers.51.mlp.experts.126.gate_proj", + "model.layers.51.mlp.experts.126.up_proj", + "model.layers.51.mlp.experts.127.down_proj", + "model.layers.51.mlp.experts.127.gate_proj", + "model.layers.51.mlp.experts.127.up_proj", + "model.layers.51.mlp.experts.128.down_proj", + "model.layers.51.mlp.experts.128.gate_proj", + "model.layers.51.mlp.experts.128.up_proj", + "model.layers.51.mlp.experts.129.down_proj", + "model.layers.51.mlp.experts.129.gate_proj", + "model.layers.51.mlp.experts.129.up_proj", + "model.layers.51.mlp.experts.13.down_proj", + "model.layers.51.mlp.experts.13.gate_proj", + "model.layers.51.mlp.experts.13.up_proj", + "model.layers.51.mlp.experts.130.down_proj", + "model.layers.51.mlp.experts.130.gate_proj", + "model.layers.51.mlp.experts.130.up_proj", + "model.layers.51.mlp.experts.131.down_proj", + "model.layers.51.mlp.experts.131.gate_proj", + "model.layers.51.mlp.experts.131.up_proj", + "model.layers.51.mlp.experts.132.down_proj", + "model.layers.51.mlp.experts.132.gate_proj", + "model.layers.51.mlp.experts.132.up_proj", + "model.layers.51.mlp.experts.133.down_proj", + "model.layers.51.mlp.experts.133.gate_proj", + "model.layers.51.mlp.experts.133.up_proj", + "model.layers.51.mlp.experts.134.down_proj", + "model.layers.51.mlp.experts.134.gate_proj", + "model.layers.51.mlp.experts.134.up_proj", + "model.layers.51.mlp.experts.135.down_proj", + "model.layers.51.mlp.experts.135.gate_proj", + "model.layers.51.mlp.experts.135.up_proj", + "model.layers.51.mlp.experts.136.down_proj", + "model.layers.51.mlp.experts.136.gate_proj", + "model.layers.51.mlp.experts.136.up_proj", + "model.layers.51.mlp.experts.137.down_proj", + "model.layers.51.mlp.experts.137.gate_proj", + "model.layers.51.mlp.experts.137.up_proj", + "model.layers.51.mlp.experts.138.down_proj", + "model.layers.51.mlp.experts.138.gate_proj", + "model.layers.51.mlp.experts.138.up_proj", + "model.layers.51.mlp.experts.139.down_proj", + "model.layers.51.mlp.experts.139.gate_proj", + "model.layers.51.mlp.experts.139.up_proj", + "model.layers.51.mlp.experts.14.down_proj", + "model.layers.51.mlp.experts.14.gate_proj", + "model.layers.51.mlp.experts.14.up_proj", + "model.layers.51.mlp.experts.140.down_proj", + "model.layers.51.mlp.experts.140.gate_proj", + "model.layers.51.mlp.experts.140.up_proj", + "model.layers.51.mlp.experts.141.down_proj", + "model.layers.51.mlp.experts.141.gate_proj", + "model.layers.51.mlp.experts.141.up_proj", + "model.layers.51.mlp.experts.142.down_proj", + "model.layers.51.mlp.experts.142.gate_proj", + "model.layers.51.mlp.experts.142.up_proj", + "model.layers.51.mlp.experts.143.down_proj", + "model.layers.51.mlp.experts.143.gate_proj", + "model.layers.51.mlp.experts.143.up_proj", + "model.layers.51.mlp.experts.144.down_proj", + "model.layers.51.mlp.experts.144.gate_proj", + "model.layers.51.mlp.experts.144.up_proj", + "model.layers.51.mlp.experts.145.down_proj", + "model.layers.51.mlp.experts.145.gate_proj", + "model.layers.51.mlp.experts.145.up_proj", + "model.layers.51.mlp.experts.146.down_proj", + "model.layers.51.mlp.experts.146.gate_proj", + "model.layers.51.mlp.experts.146.up_proj", + "model.layers.51.mlp.experts.147.down_proj", + "model.layers.51.mlp.experts.147.gate_proj", + "model.layers.51.mlp.experts.147.up_proj", + "model.layers.51.mlp.experts.148.down_proj", + "model.layers.51.mlp.experts.148.gate_proj", + "model.layers.51.mlp.experts.148.up_proj", + "model.layers.51.mlp.experts.149.down_proj", + "model.layers.51.mlp.experts.149.gate_proj", + "model.layers.51.mlp.experts.149.up_proj", + "model.layers.51.mlp.experts.15.down_proj", + "model.layers.51.mlp.experts.15.gate_proj", + "model.layers.51.mlp.experts.15.up_proj", + "model.layers.51.mlp.experts.150.down_proj", + "model.layers.51.mlp.experts.150.gate_proj", + "model.layers.51.mlp.experts.150.up_proj", + "model.layers.51.mlp.experts.151.down_proj", + "model.layers.51.mlp.experts.151.gate_proj", + "model.layers.51.mlp.experts.151.up_proj", + "model.layers.51.mlp.experts.152.down_proj", + "model.layers.51.mlp.experts.152.gate_proj", + "model.layers.51.mlp.experts.152.up_proj", + "model.layers.51.mlp.experts.153.down_proj", + "model.layers.51.mlp.experts.153.gate_proj", + "model.layers.51.mlp.experts.153.up_proj", + "model.layers.51.mlp.experts.154.down_proj", + "model.layers.51.mlp.experts.154.gate_proj", + "model.layers.51.mlp.experts.154.up_proj", + "model.layers.51.mlp.experts.155.down_proj", + "model.layers.51.mlp.experts.155.gate_proj", + "model.layers.51.mlp.experts.155.up_proj", + "model.layers.51.mlp.experts.156.down_proj", + "model.layers.51.mlp.experts.156.gate_proj", + "model.layers.51.mlp.experts.156.up_proj", + "model.layers.51.mlp.experts.157.down_proj", + "model.layers.51.mlp.experts.157.gate_proj", + "model.layers.51.mlp.experts.157.up_proj", + "model.layers.51.mlp.experts.158.down_proj", + "model.layers.51.mlp.experts.158.gate_proj", + "model.layers.51.mlp.experts.158.up_proj", + "model.layers.51.mlp.experts.159.down_proj", + "model.layers.51.mlp.experts.159.gate_proj", + "model.layers.51.mlp.experts.159.up_proj", + "model.layers.51.mlp.experts.16.down_proj", + "model.layers.51.mlp.experts.16.gate_proj", + "model.layers.51.mlp.experts.16.up_proj", + "model.layers.51.mlp.experts.160.down_proj", + "model.layers.51.mlp.experts.160.gate_proj", + "model.layers.51.mlp.experts.160.up_proj", + "model.layers.51.mlp.experts.161.down_proj", + "model.layers.51.mlp.experts.161.gate_proj", + "model.layers.51.mlp.experts.161.up_proj", + "model.layers.51.mlp.experts.162.down_proj", + "model.layers.51.mlp.experts.162.gate_proj", + "model.layers.51.mlp.experts.162.up_proj", + "model.layers.51.mlp.experts.163.down_proj", + "model.layers.51.mlp.experts.163.gate_proj", + "model.layers.51.mlp.experts.163.up_proj", + "model.layers.51.mlp.experts.164.down_proj", + "model.layers.51.mlp.experts.164.gate_proj", + "model.layers.51.mlp.experts.164.up_proj", + "model.layers.51.mlp.experts.165.down_proj", + "model.layers.51.mlp.experts.165.gate_proj", + "model.layers.51.mlp.experts.165.up_proj", + "model.layers.51.mlp.experts.166.down_proj", + "model.layers.51.mlp.experts.166.gate_proj", + "model.layers.51.mlp.experts.166.up_proj", + "model.layers.51.mlp.experts.167.down_proj", + "model.layers.51.mlp.experts.167.gate_proj", + "model.layers.51.mlp.experts.167.up_proj", + "model.layers.51.mlp.experts.168.down_proj", + "model.layers.51.mlp.experts.168.gate_proj", + "model.layers.51.mlp.experts.168.up_proj", + "model.layers.51.mlp.experts.169.down_proj", + "model.layers.51.mlp.experts.169.gate_proj", + "model.layers.51.mlp.experts.169.up_proj", + "model.layers.51.mlp.experts.17.down_proj", + "model.layers.51.mlp.experts.17.gate_proj", + "model.layers.51.mlp.experts.17.up_proj", + "model.layers.51.mlp.experts.170.down_proj", + "model.layers.51.mlp.experts.170.gate_proj", + "model.layers.51.mlp.experts.170.up_proj", + "model.layers.51.mlp.experts.171.down_proj", + "model.layers.51.mlp.experts.171.gate_proj", + "model.layers.51.mlp.experts.171.up_proj", + "model.layers.51.mlp.experts.172.down_proj", + "model.layers.51.mlp.experts.172.gate_proj", + "model.layers.51.mlp.experts.172.up_proj", + "model.layers.51.mlp.experts.173.down_proj", + "model.layers.51.mlp.experts.173.gate_proj", + "model.layers.51.mlp.experts.173.up_proj", + "model.layers.51.mlp.experts.174.down_proj", + "model.layers.51.mlp.experts.174.gate_proj", + "model.layers.51.mlp.experts.174.up_proj", + "model.layers.51.mlp.experts.175.down_proj", + "model.layers.51.mlp.experts.175.gate_proj", + "model.layers.51.mlp.experts.175.up_proj", + "model.layers.51.mlp.experts.176.down_proj", + "model.layers.51.mlp.experts.176.gate_proj", + "model.layers.51.mlp.experts.176.up_proj", + "model.layers.51.mlp.experts.177.down_proj", + "model.layers.51.mlp.experts.177.gate_proj", + "model.layers.51.mlp.experts.177.up_proj", + "model.layers.51.mlp.experts.178.down_proj", + "model.layers.51.mlp.experts.178.gate_proj", + "model.layers.51.mlp.experts.178.up_proj", + "model.layers.51.mlp.experts.179.down_proj", + "model.layers.51.mlp.experts.179.gate_proj", + "model.layers.51.mlp.experts.179.up_proj", + "model.layers.51.mlp.experts.18.down_proj", + "model.layers.51.mlp.experts.18.gate_proj", + "model.layers.51.mlp.experts.18.up_proj", + "model.layers.51.mlp.experts.180.down_proj", + "model.layers.51.mlp.experts.180.gate_proj", + "model.layers.51.mlp.experts.180.up_proj", + "model.layers.51.mlp.experts.181.down_proj", + "model.layers.51.mlp.experts.181.gate_proj", + "model.layers.51.mlp.experts.181.up_proj", + "model.layers.51.mlp.experts.182.down_proj", + "model.layers.51.mlp.experts.182.gate_proj", + "model.layers.51.mlp.experts.182.up_proj", + "model.layers.51.mlp.experts.183.down_proj", + "model.layers.51.mlp.experts.183.gate_proj", + "model.layers.51.mlp.experts.183.up_proj", + "model.layers.51.mlp.experts.184.down_proj", + "model.layers.51.mlp.experts.184.gate_proj", + "model.layers.51.mlp.experts.184.up_proj", + "model.layers.51.mlp.experts.185.down_proj", + "model.layers.51.mlp.experts.185.gate_proj", + "model.layers.51.mlp.experts.185.up_proj", + "model.layers.51.mlp.experts.186.down_proj", + "model.layers.51.mlp.experts.186.gate_proj", + "model.layers.51.mlp.experts.186.up_proj", + "model.layers.51.mlp.experts.187.down_proj", + "model.layers.51.mlp.experts.187.gate_proj", + "model.layers.51.mlp.experts.187.up_proj", + "model.layers.51.mlp.experts.188.down_proj", + "model.layers.51.mlp.experts.188.gate_proj", + "model.layers.51.mlp.experts.188.up_proj", + "model.layers.51.mlp.experts.189.down_proj", + "model.layers.51.mlp.experts.189.gate_proj", + "model.layers.51.mlp.experts.189.up_proj", + "model.layers.51.mlp.experts.19.down_proj", + "model.layers.51.mlp.experts.19.gate_proj", + "model.layers.51.mlp.experts.19.up_proj", + "model.layers.51.mlp.experts.190.down_proj", + "model.layers.51.mlp.experts.190.gate_proj", + "model.layers.51.mlp.experts.190.up_proj", + "model.layers.51.mlp.experts.191.down_proj", + "model.layers.51.mlp.experts.191.gate_proj", + "model.layers.51.mlp.experts.191.up_proj", + "model.layers.51.mlp.experts.192.down_proj", + "model.layers.51.mlp.experts.192.gate_proj", + "model.layers.51.mlp.experts.192.up_proj", + "model.layers.51.mlp.experts.193.down_proj", + "model.layers.51.mlp.experts.193.gate_proj", + "model.layers.51.mlp.experts.193.up_proj", + "model.layers.51.mlp.experts.194.down_proj", + "model.layers.51.mlp.experts.194.gate_proj", + "model.layers.51.mlp.experts.194.up_proj", + "model.layers.51.mlp.experts.195.down_proj", + "model.layers.51.mlp.experts.195.gate_proj", + "model.layers.51.mlp.experts.195.up_proj", + "model.layers.51.mlp.experts.196.down_proj", + "model.layers.51.mlp.experts.196.gate_proj", + "model.layers.51.mlp.experts.196.up_proj", + "model.layers.51.mlp.experts.197.down_proj", + "model.layers.51.mlp.experts.197.gate_proj", + "model.layers.51.mlp.experts.197.up_proj", + "model.layers.51.mlp.experts.198.down_proj", + "model.layers.51.mlp.experts.198.gate_proj", + "model.layers.51.mlp.experts.198.up_proj", + "model.layers.51.mlp.experts.199.down_proj", + "model.layers.51.mlp.experts.199.gate_proj", + "model.layers.51.mlp.experts.199.up_proj", + "model.layers.51.mlp.experts.2.down_proj", + "model.layers.51.mlp.experts.2.gate_proj", + "model.layers.51.mlp.experts.2.up_proj", + "model.layers.51.mlp.experts.20.down_proj", + "model.layers.51.mlp.experts.20.gate_proj", + "model.layers.51.mlp.experts.20.up_proj", + "model.layers.51.mlp.experts.200.down_proj", + "model.layers.51.mlp.experts.200.gate_proj", + "model.layers.51.mlp.experts.200.up_proj", + "model.layers.51.mlp.experts.201.down_proj", + "model.layers.51.mlp.experts.201.gate_proj", + "model.layers.51.mlp.experts.201.up_proj", + "model.layers.51.mlp.experts.202.down_proj", + "model.layers.51.mlp.experts.202.gate_proj", + "model.layers.51.mlp.experts.202.up_proj", + "model.layers.51.mlp.experts.203.down_proj", + "model.layers.51.mlp.experts.203.gate_proj", + "model.layers.51.mlp.experts.203.up_proj", + "model.layers.51.mlp.experts.204.down_proj", + "model.layers.51.mlp.experts.204.gate_proj", + "model.layers.51.mlp.experts.204.up_proj", + "model.layers.51.mlp.experts.205.down_proj", + "model.layers.51.mlp.experts.205.gate_proj", + "model.layers.51.mlp.experts.205.up_proj", + "model.layers.51.mlp.experts.206.down_proj", + "model.layers.51.mlp.experts.206.gate_proj", + "model.layers.51.mlp.experts.206.up_proj", + "model.layers.51.mlp.experts.207.down_proj", + "model.layers.51.mlp.experts.207.gate_proj", + "model.layers.51.mlp.experts.207.up_proj", + "model.layers.51.mlp.experts.208.down_proj", + "model.layers.51.mlp.experts.208.gate_proj", + "model.layers.51.mlp.experts.208.up_proj", + "model.layers.51.mlp.experts.209.down_proj", + "model.layers.51.mlp.experts.209.gate_proj", + "model.layers.51.mlp.experts.209.up_proj", + "model.layers.51.mlp.experts.21.down_proj", + "model.layers.51.mlp.experts.21.gate_proj", + "model.layers.51.mlp.experts.21.up_proj", + "model.layers.51.mlp.experts.210.down_proj", + "model.layers.51.mlp.experts.210.gate_proj", + "model.layers.51.mlp.experts.210.up_proj", + "model.layers.51.mlp.experts.211.down_proj", + "model.layers.51.mlp.experts.211.gate_proj", + "model.layers.51.mlp.experts.211.up_proj", + "model.layers.51.mlp.experts.212.down_proj", + "model.layers.51.mlp.experts.212.gate_proj", + "model.layers.51.mlp.experts.212.up_proj", + "model.layers.51.mlp.experts.213.down_proj", + "model.layers.51.mlp.experts.213.gate_proj", + "model.layers.51.mlp.experts.213.up_proj", + "model.layers.51.mlp.experts.214.down_proj", + "model.layers.51.mlp.experts.214.gate_proj", + "model.layers.51.mlp.experts.214.up_proj", + "model.layers.51.mlp.experts.215.down_proj", + "model.layers.51.mlp.experts.215.gate_proj", + "model.layers.51.mlp.experts.215.up_proj", + "model.layers.51.mlp.experts.216.down_proj", + "model.layers.51.mlp.experts.216.gate_proj", + "model.layers.51.mlp.experts.216.up_proj", + "model.layers.51.mlp.experts.217.down_proj", + "model.layers.51.mlp.experts.217.gate_proj", + "model.layers.51.mlp.experts.217.up_proj", + "model.layers.51.mlp.experts.218.down_proj", + "model.layers.51.mlp.experts.218.gate_proj", + "model.layers.51.mlp.experts.218.up_proj", + "model.layers.51.mlp.experts.219.down_proj", + "model.layers.51.mlp.experts.219.gate_proj", + "model.layers.51.mlp.experts.219.up_proj", + "model.layers.51.mlp.experts.22.down_proj", + "model.layers.51.mlp.experts.22.gate_proj", + "model.layers.51.mlp.experts.22.up_proj", + "model.layers.51.mlp.experts.220.down_proj", + "model.layers.51.mlp.experts.220.gate_proj", + "model.layers.51.mlp.experts.220.up_proj", + "model.layers.51.mlp.experts.221.down_proj", + "model.layers.51.mlp.experts.221.gate_proj", + "model.layers.51.mlp.experts.221.up_proj", + "model.layers.51.mlp.experts.222.down_proj", + "model.layers.51.mlp.experts.222.gate_proj", + "model.layers.51.mlp.experts.222.up_proj", + "model.layers.51.mlp.experts.223.down_proj", + "model.layers.51.mlp.experts.223.gate_proj", + "model.layers.51.mlp.experts.223.up_proj", + "model.layers.51.mlp.experts.224.down_proj", + "model.layers.51.mlp.experts.224.gate_proj", + "model.layers.51.mlp.experts.224.up_proj", + "model.layers.51.mlp.experts.225.down_proj", + "model.layers.51.mlp.experts.225.gate_proj", + "model.layers.51.mlp.experts.225.up_proj", + "model.layers.51.mlp.experts.226.down_proj", + "model.layers.51.mlp.experts.226.gate_proj", + "model.layers.51.mlp.experts.226.up_proj", + "model.layers.51.mlp.experts.227.down_proj", + "model.layers.51.mlp.experts.227.gate_proj", + "model.layers.51.mlp.experts.227.up_proj", + "model.layers.51.mlp.experts.228.down_proj", + "model.layers.51.mlp.experts.228.gate_proj", + "model.layers.51.mlp.experts.228.up_proj", + "model.layers.51.mlp.experts.229.down_proj", + "model.layers.51.mlp.experts.229.gate_proj", + "model.layers.51.mlp.experts.229.up_proj", + "model.layers.51.mlp.experts.23.down_proj", + "model.layers.51.mlp.experts.23.gate_proj", + "model.layers.51.mlp.experts.23.up_proj", + "model.layers.51.mlp.experts.230.down_proj", + "model.layers.51.mlp.experts.230.gate_proj", + "model.layers.51.mlp.experts.230.up_proj", + "model.layers.51.mlp.experts.231.down_proj", + "model.layers.51.mlp.experts.231.gate_proj", + "model.layers.51.mlp.experts.231.up_proj", + "model.layers.51.mlp.experts.232.down_proj", + "model.layers.51.mlp.experts.232.gate_proj", + "model.layers.51.mlp.experts.232.up_proj", + "model.layers.51.mlp.experts.233.down_proj", + "model.layers.51.mlp.experts.233.gate_proj", + "model.layers.51.mlp.experts.233.up_proj", + "model.layers.51.mlp.experts.234.down_proj", + "model.layers.51.mlp.experts.234.gate_proj", + "model.layers.51.mlp.experts.234.up_proj", + "model.layers.51.mlp.experts.235.down_proj", + "model.layers.51.mlp.experts.235.gate_proj", + "model.layers.51.mlp.experts.235.up_proj", + "model.layers.51.mlp.experts.236.down_proj", + "model.layers.51.mlp.experts.236.gate_proj", + "model.layers.51.mlp.experts.236.up_proj", + "model.layers.51.mlp.experts.237.down_proj", + "model.layers.51.mlp.experts.237.gate_proj", + "model.layers.51.mlp.experts.237.up_proj", + "model.layers.51.mlp.experts.238.down_proj", + "model.layers.51.mlp.experts.238.gate_proj", + "model.layers.51.mlp.experts.238.up_proj", + "model.layers.51.mlp.experts.239.down_proj", + "model.layers.51.mlp.experts.239.gate_proj", + "model.layers.51.mlp.experts.239.up_proj", + "model.layers.51.mlp.experts.24.down_proj", + "model.layers.51.mlp.experts.24.gate_proj", + "model.layers.51.mlp.experts.24.up_proj", + "model.layers.51.mlp.experts.240.down_proj", + "model.layers.51.mlp.experts.240.gate_proj", + "model.layers.51.mlp.experts.240.up_proj", + "model.layers.51.mlp.experts.241.down_proj", + "model.layers.51.mlp.experts.241.gate_proj", + "model.layers.51.mlp.experts.241.up_proj", + "model.layers.51.mlp.experts.242.down_proj", + "model.layers.51.mlp.experts.242.gate_proj", + "model.layers.51.mlp.experts.242.up_proj", + "model.layers.51.mlp.experts.243.down_proj", + "model.layers.51.mlp.experts.243.gate_proj", + "model.layers.51.mlp.experts.243.up_proj", + "model.layers.51.mlp.experts.244.down_proj", + "model.layers.51.mlp.experts.244.gate_proj", + "model.layers.51.mlp.experts.244.up_proj", + "model.layers.51.mlp.experts.245.down_proj", + "model.layers.51.mlp.experts.245.gate_proj", + "model.layers.51.mlp.experts.245.up_proj", + "model.layers.51.mlp.experts.246.down_proj", + "model.layers.51.mlp.experts.246.gate_proj", + "model.layers.51.mlp.experts.246.up_proj", + "model.layers.51.mlp.experts.247.down_proj", + "model.layers.51.mlp.experts.247.gate_proj", + "model.layers.51.mlp.experts.247.up_proj", + "model.layers.51.mlp.experts.248.down_proj", + "model.layers.51.mlp.experts.248.gate_proj", + "model.layers.51.mlp.experts.248.up_proj", + "model.layers.51.mlp.experts.249.down_proj", + "model.layers.51.mlp.experts.249.gate_proj", + "model.layers.51.mlp.experts.249.up_proj", + "model.layers.51.mlp.experts.25.down_proj", + "model.layers.51.mlp.experts.25.gate_proj", + "model.layers.51.mlp.experts.25.up_proj", + "model.layers.51.mlp.experts.250.down_proj", + "model.layers.51.mlp.experts.250.gate_proj", + "model.layers.51.mlp.experts.250.up_proj", + "model.layers.51.mlp.experts.251.down_proj", + "model.layers.51.mlp.experts.251.gate_proj", + "model.layers.51.mlp.experts.251.up_proj", + "model.layers.51.mlp.experts.252.down_proj", + "model.layers.51.mlp.experts.252.gate_proj", + "model.layers.51.mlp.experts.252.up_proj", + "model.layers.51.mlp.experts.253.down_proj", + "model.layers.51.mlp.experts.253.gate_proj", + "model.layers.51.mlp.experts.253.up_proj", + "model.layers.51.mlp.experts.254.down_proj", + "model.layers.51.mlp.experts.254.gate_proj", + "model.layers.51.mlp.experts.254.up_proj", + "model.layers.51.mlp.experts.255.down_proj", + "model.layers.51.mlp.experts.255.gate_proj", + "model.layers.51.mlp.experts.255.up_proj", + "model.layers.51.mlp.experts.26.down_proj", + "model.layers.51.mlp.experts.26.gate_proj", + "model.layers.51.mlp.experts.26.up_proj", + "model.layers.51.mlp.experts.27.down_proj", + "model.layers.51.mlp.experts.27.gate_proj", + "model.layers.51.mlp.experts.27.up_proj", + "model.layers.51.mlp.experts.28.down_proj", + "model.layers.51.mlp.experts.28.gate_proj", + "model.layers.51.mlp.experts.28.up_proj", + "model.layers.51.mlp.experts.29.down_proj", + "model.layers.51.mlp.experts.29.gate_proj", + "model.layers.51.mlp.experts.29.up_proj", + "model.layers.51.mlp.experts.3.down_proj", + "model.layers.51.mlp.experts.3.gate_proj", + "model.layers.51.mlp.experts.3.up_proj", + "model.layers.51.mlp.experts.30.down_proj", + "model.layers.51.mlp.experts.30.gate_proj", + "model.layers.51.mlp.experts.30.up_proj", + "model.layers.51.mlp.experts.31.down_proj", + "model.layers.51.mlp.experts.31.gate_proj", + "model.layers.51.mlp.experts.31.up_proj", + "model.layers.51.mlp.experts.32.down_proj", + "model.layers.51.mlp.experts.32.gate_proj", + "model.layers.51.mlp.experts.32.up_proj", + "model.layers.51.mlp.experts.33.down_proj", + "model.layers.51.mlp.experts.33.gate_proj", + "model.layers.51.mlp.experts.33.up_proj", + "model.layers.51.mlp.experts.34.down_proj", + "model.layers.51.mlp.experts.34.gate_proj", + "model.layers.51.mlp.experts.34.up_proj", + "model.layers.51.mlp.experts.35.down_proj", + "model.layers.51.mlp.experts.35.gate_proj", + "model.layers.51.mlp.experts.35.up_proj", + "model.layers.51.mlp.experts.36.down_proj", + "model.layers.51.mlp.experts.36.gate_proj", + "model.layers.51.mlp.experts.36.up_proj", + "model.layers.51.mlp.experts.37.down_proj", + "model.layers.51.mlp.experts.37.gate_proj", + "model.layers.51.mlp.experts.37.up_proj", + "model.layers.51.mlp.experts.38.down_proj", + "model.layers.51.mlp.experts.38.gate_proj", + "model.layers.51.mlp.experts.38.up_proj", + "model.layers.51.mlp.experts.39.down_proj", + "model.layers.51.mlp.experts.39.gate_proj", + "model.layers.51.mlp.experts.39.up_proj", + "model.layers.51.mlp.experts.4.down_proj", + "model.layers.51.mlp.experts.4.gate_proj", + "model.layers.51.mlp.experts.4.up_proj", + "model.layers.51.mlp.experts.40.down_proj", + "model.layers.51.mlp.experts.40.gate_proj", + "model.layers.51.mlp.experts.40.up_proj", + "model.layers.51.mlp.experts.41.down_proj", + "model.layers.51.mlp.experts.41.gate_proj", + "model.layers.51.mlp.experts.41.up_proj", + "model.layers.51.mlp.experts.42.down_proj", + "model.layers.51.mlp.experts.42.gate_proj", + "model.layers.51.mlp.experts.42.up_proj", + "model.layers.51.mlp.experts.43.down_proj", + "model.layers.51.mlp.experts.43.gate_proj", + "model.layers.51.mlp.experts.43.up_proj", + "model.layers.51.mlp.experts.44.down_proj", + "model.layers.51.mlp.experts.44.gate_proj", + "model.layers.51.mlp.experts.44.up_proj", + "model.layers.51.mlp.experts.45.down_proj", + "model.layers.51.mlp.experts.45.gate_proj", + "model.layers.51.mlp.experts.45.up_proj", + "model.layers.51.mlp.experts.46.down_proj", + "model.layers.51.mlp.experts.46.gate_proj", + "model.layers.51.mlp.experts.46.up_proj", + "model.layers.51.mlp.experts.47.down_proj", + "model.layers.51.mlp.experts.47.gate_proj", + "model.layers.51.mlp.experts.47.up_proj", + "model.layers.51.mlp.experts.48.down_proj", + "model.layers.51.mlp.experts.48.gate_proj", + "model.layers.51.mlp.experts.48.up_proj", + "model.layers.51.mlp.experts.49.down_proj", + "model.layers.51.mlp.experts.49.gate_proj", + "model.layers.51.mlp.experts.49.up_proj", + "model.layers.51.mlp.experts.5.down_proj", + "model.layers.51.mlp.experts.5.gate_proj", + "model.layers.51.mlp.experts.5.up_proj", + "model.layers.51.mlp.experts.50.down_proj", + "model.layers.51.mlp.experts.50.gate_proj", + "model.layers.51.mlp.experts.50.up_proj", + "model.layers.51.mlp.experts.51.down_proj", + "model.layers.51.mlp.experts.51.gate_proj", + "model.layers.51.mlp.experts.51.up_proj", + "model.layers.51.mlp.experts.52.down_proj", + "model.layers.51.mlp.experts.52.gate_proj", + "model.layers.51.mlp.experts.52.up_proj", + "model.layers.51.mlp.experts.53.down_proj", + "model.layers.51.mlp.experts.53.gate_proj", + "model.layers.51.mlp.experts.53.up_proj", + "model.layers.51.mlp.experts.54.down_proj", + "model.layers.51.mlp.experts.54.gate_proj", + "model.layers.51.mlp.experts.54.up_proj", + "model.layers.51.mlp.experts.55.down_proj", + "model.layers.51.mlp.experts.55.gate_proj", + "model.layers.51.mlp.experts.55.up_proj", + "model.layers.51.mlp.experts.56.down_proj", + "model.layers.51.mlp.experts.56.gate_proj", + "model.layers.51.mlp.experts.56.up_proj", + "model.layers.51.mlp.experts.57.down_proj", + "model.layers.51.mlp.experts.57.gate_proj", + "model.layers.51.mlp.experts.57.up_proj", + "model.layers.51.mlp.experts.58.down_proj", + "model.layers.51.mlp.experts.58.gate_proj", + "model.layers.51.mlp.experts.58.up_proj", + "model.layers.51.mlp.experts.59.down_proj", + "model.layers.51.mlp.experts.59.gate_proj", + "model.layers.51.mlp.experts.59.up_proj", + "model.layers.51.mlp.experts.6.down_proj", + "model.layers.51.mlp.experts.6.gate_proj", + "model.layers.51.mlp.experts.6.up_proj", + "model.layers.51.mlp.experts.60.down_proj", + "model.layers.51.mlp.experts.60.gate_proj", + "model.layers.51.mlp.experts.60.up_proj", + "model.layers.51.mlp.experts.61.down_proj", + "model.layers.51.mlp.experts.61.gate_proj", + "model.layers.51.mlp.experts.61.up_proj", + "model.layers.51.mlp.experts.62.down_proj", + "model.layers.51.mlp.experts.62.gate_proj", + "model.layers.51.mlp.experts.62.up_proj", + "model.layers.51.mlp.experts.63.down_proj", + "model.layers.51.mlp.experts.63.gate_proj", + "model.layers.51.mlp.experts.63.up_proj", + "model.layers.51.mlp.experts.64.down_proj", + "model.layers.51.mlp.experts.64.gate_proj", + "model.layers.51.mlp.experts.64.up_proj", + "model.layers.51.mlp.experts.65.down_proj", + "model.layers.51.mlp.experts.65.gate_proj", + "model.layers.51.mlp.experts.65.up_proj", + "model.layers.51.mlp.experts.66.down_proj", + "model.layers.51.mlp.experts.66.gate_proj", + "model.layers.51.mlp.experts.66.up_proj", + "model.layers.51.mlp.experts.67.down_proj", + "model.layers.51.mlp.experts.67.gate_proj", + "model.layers.51.mlp.experts.67.up_proj", + "model.layers.51.mlp.experts.68.down_proj", + "model.layers.51.mlp.experts.68.gate_proj", + "model.layers.51.mlp.experts.68.up_proj", + "model.layers.51.mlp.experts.69.down_proj", + "model.layers.51.mlp.experts.69.gate_proj", + "model.layers.51.mlp.experts.69.up_proj", + "model.layers.51.mlp.experts.7.down_proj", + "model.layers.51.mlp.experts.7.gate_proj", + "model.layers.51.mlp.experts.7.up_proj", + "model.layers.51.mlp.experts.70.down_proj", + "model.layers.51.mlp.experts.70.gate_proj", + "model.layers.51.mlp.experts.70.up_proj", + "model.layers.51.mlp.experts.71.down_proj", + "model.layers.51.mlp.experts.71.gate_proj", + "model.layers.51.mlp.experts.71.up_proj", + "model.layers.51.mlp.experts.72.down_proj", + "model.layers.51.mlp.experts.72.gate_proj", + "model.layers.51.mlp.experts.72.up_proj", + "model.layers.51.mlp.experts.73.down_proj", + "model.layers.51.mlp.experts.73.gate_proj", + "model.layers.51.mlp.experts.73.up_proj", + "model.layers.51.mlp.experts.74.down_proj", + "model.layers.51.mlp.experts.74.gate_proj", + "model.layers.51.mlp.experts.74.up_proj", + "model.layers.51.mlp.experts.75.down_proj", + "model.layers.51.mlp.experts.75.gate_proj", + "model.layers.51.mlp.experts.75.up_proj", + "model.layers.51.mlp.experts.76.down_proj", + "model.layers.51.mlp.experts.76.gate_proj", + "model.layers.51.mlp.experts.76.up_proj", + "model.layers.51.mlp.experts.77.down_proj", + "model.layers.51.mlp.experts.77.gate_proj", + "model.layers.51.mlp.experts.77.up_proj", + "model.layers.51.mlp.experts.78.down_proj", + "model.layers.51.mlp.experts.78.gate_proj", + "model.layers.51.mlp.experts.78.up_proj", + "model.layers.51.mlp.experts.79.down_proj", + "model.layers.51.mlp.experts.79.gate_proj", + "model.layers.51.mlp.experts.79.up_proj", + "model.layers.51.mlp.experts.8.down_proj", + "model.layers.51.mlp.experts.8.gate_proj", + "model.layers.51.mlp.experts.8.up_proj", + "model.layers.51.mlp.experts.80.down_proj", + "model.layers.51.mlp.experts.80.gate_proj", + "model.layers.51.mlp.experts.80.up_proj", + "model.layers.51.mlp.experts.81.down_proj", + "model.layers.51.mlp.experts.81.gate_proj", + "model.layers.51.mlp.experts.81.up_proj", + "model.layers.51.mlp.experts.82.down_proj", + "model.layers.51.mlp.experts.82.gate_proj", + "model.layers.51.mlp.experts.82.up_proj", + "model.layers.51.mlp.experts.83.down_proj", + "model.layers.51.mlp.experts.83.gate_proj", + "model.layers.51.mlp.experts.83.up_proj", + "model.layers.51.mlp.experts.84.down_proj", + "model.layers.51.mlp.experts.84.gate_proj", + "model.layers.51.mlp.experts.84.up_proj", + "model.layers.51.mlp.experts.85.down_proj", + "model.layers.51.mlp.experts.85.gate_proj", + "model.layers.51.mlp.experts.85.up_proj", + "model.layers.51.mlp.experts.86.down_proj", + "model.layers.51.mlp.experts.86.gate_proj", + "model.layers.51.mlp.experts.86.up_proj", + "model.layers.51.mlp.experts.87.down_proj", + "model.layers.51.mlp.experts.87.gate_proj", + "model.layers.51.mlp.experts.87.up_proj", + "model.layers.51.mlp.experts.88.down_proj", + "model.layers.51.mlp.experts.88.gate_proj", + "model.layers.51.mlp.experts.88.up_proj", + "model.layers.51.mlp.experts.89.down_proj", + "model.layers.51.mlp.experts.89.gate_proj", + "model.layers.51.mlp.experts.89.up_proj", + "model.layers.51.mlp.experts.9.down_proj", + "model.layers.51.mlp.experts.9.gate_proj", + "model.layers.51.mlp.experts.9.up_proj", + "model.layers.51.mlp.experts.90.down_proj", + "model.layers.51.mlp.experts.90.gate_proj", + "model.layers.51.mlp.experts.90.up_proj", + "model.layers.51.mlp.experts.91.down_proj", + "model.layers.51.mlp.experts.91.gate_proj", + "model.layers.51.mlp.experts.91.up_proj", + "model.layers.51.mlp.experts.92.down_proj", + "model.layers.51.mlp.experts.92.gate_proj", + "model.layers.51.mlp.experts.92.up_proj", + "model.layers.51.mlp.experts.93.down_proj", + "model.layers.51.mlp.experts.93.gate_proj", + "model.layers.51.mlp.experts.93.up_proj", + "model.layers.51.mlp.experts.94.down_proj", + "model.layers.51.mlp.experts.94.gate_proj", + "model.layers.51.mlp.experts.94.up_proj", + "model.layers.51.mlp.experts.95.down_proj", + "model.layers.51.mlp.experts.95.gate_proj", + "model.layers.51.mlp.experts.95.up_proj", + "model.layers.51.mlp.experts.96.down_proj", + "model.layers.51.mlp.experts.96.gate_proj", + "model.layers.51.mlp.experts.96.up_proj", + "model.layers.51.mlp.experts.97.down_proj", + "model.layers.51.mlp.experts.97.gate_proj", + "model.layers.51.mlp.experts.97.up_proj", + "model.layers.51.mlp.experts.98.down_proj", + "model.layers.51.mlp.experts.98.gate_proj", + "model.layers.51.mlp.experts.98.up_proj", + "model.layers.51.mlp.experts.99.down_proj", + "model.layers.51.mlp.experts.99.gate_proj", + "model.layers.51.mlp.experts.99.up_proj", + "model.layers.52.mlp.experts.0.down_proj", + "model.layers.52.mlp.experts.0.gate_proj", + "model.layers.52.mlp.experts.0.up_proj", + "model.layers.52.mlp.experts.1.down_proj", + "model.layers.52.mlp.experts.1.gate_proj", + "model.layers.52.mlp.experts.1.up_proj", + "model.layers.52.mlp.experts.10.down_proj", + "model.layers.52.mlp.experts.10.gate_proj", + "model.layers.52.mlp.experts.10.up_proj", + "model.layers.52.mlp.experts.100.down_proj", + "model.layers.52.mlp.experts.100.gate_proj", + "model.layers.52.mlp.experts.100.up_proj", + "model.layers.52.mlp.experts.101.down_proj", + "model.layers.52.mlp.experts.101.gate_proj", + "model.layers.52.mlp.experts.101.up_proj", + "model.layers.52.mlp.experts.102.down_proj", + "model.layers.52.mlp.experts.102.gate_proj", + "model.layers.52.mlp.experts.102.up_proj", + "model.layers.52.mlp.experts.103.down_proj", + "model.layers.52.mlp.experts.103.gate_proj", + "model.layers.52.mlp.experts.103.up_proj", + "model.layers.52.mlp.experts.104.down_proj", + "model.layers.52.mlp.experts.104.gate_proj", + "model.layers.52.mlp.experts.104.up_proj", + "model.layers.52.mlp.experts.105.down_proj", + "model.layers.52.mlp.experts.105.gate_proj", + "model.layers.52.mlp.experts.105.up_proj", + "model.layers.52.mlp.experts.106.down_proj", + "model.layers.52.mlp.experts.106.gate_proj", + "model.layers.52.mlp.experts.106.up_proj", + "model.layers.52.mlp.experts.107.down_proj", + "model.layers.52.mlp.experts.107.gate_proj", + "model.layers.52.mlp.experts.107.up_proj", + "model.layers.52.mlp.experts.108.down_proj", + "model.layers.52.mlp.experts.108.gate_proj", + "model.layers.52.mlp.experts.108.up_proj", + "model.layers.52.mlp.experts.109.down_proj", + "model.layers.52.mlp.experts.109.gate_proj", + "model.layers.52.mlp.experts.109.up_proj", + "model.layers.52.mlp.experts.11.down_proj", + "model.layers.52.mlp.experts.11.gate_proj", + "model.layers.52.mlp.experts.11.up_proj", + "model.layers.52.mlp.experts.110.down_proj", + "model.layers.52.mlp.experts.110.gate_proj", + "model.layers.52.mlp.experts.110.up_proj", + "model.layers.52.mlp.experts.111.down_proj", + "model.layers.52.mlp.experts.111.gate_proj", + "model.layers.52.mlp.experts.111.up_proj", + "model.layers.52.mlp.experts.112.down_proj", + "model.layers.52.mlp.experts.112.gate_proj", + "model.layers.52.mlp.experts.112.up_proj", + "model.layers.52.mlp.experts.113.down_proj", + "model.layers.52.mlp.experts.113.gate_proj", + "model.layers.52.mlp.experts.113.up_proj", + "model.layers.52.mlp.experts.114.down_proj", + "model.layers.52.mlp.experts.114.gate_proj", + "model.layers.52.mlp.experts.114.up_proj", + "model.layers.52.mlp.experts.115.down_proj", + "model.layers.52.mlp.experts.115.gate_proj", + "model.layers.52.mlp.experts.115.up_proj", + "model.layers.52.mlp.experts.116.down_proj", + "model.layers.52.mlp.experts.116.gate_proj", + "model.layers.52.mlp.experts.116.up_proj", + "model.layers.52.mlp.experts.117.down_proj", + "model.layers.52.mlp.experts.117.gate_proj", + "model.layers.52.mlp.experts.117.up_proj", + "model.layers.52.mlp.experts.118.down_proj", + "model.layers.52.mlp.experts.118.gate_proj", + "model.layers.52.mlp.experts.118.up_proj", + "model.layers.52.mlp.experts.119.down_proj", + "model.layers.52.mlp.experts.119.gate_proj", + "model.layers.52.mlp.experts.119.up_proj", + "model.layers.52.mlp.experts.12.down_proj", + "model.layers.52.mlp.experts.12.gate_proj", + "model.layers.52.mlp.experts.12.up_proj", + "model.layers.52.mlp.experts.120.down_proj", + "model.layers.52.mlp.experts.120.gate_proj", + "model.layers.52.mlp.experts.120.up_proj", + "model.layers.52.mlp.experts.121.down_proj", + "model.layers.52.mlp.experts.121.gate_proj", + "model.layers.52.mlp.experts.121.up_proj", + "model.layers.52.mlp.experts.122.down_proj", + "model.layers.52.mlp.experts.122.gate_proj", + "model.layers.52.mlp.experts.122.up_proj", + "model.layers.52.mlp.experts.123.down_proj", + "model.layers.52.mlp.experts.123.gate_proj", + "model.layers.52.mlp.experts.123.up_proj", + "model.layers.52.mlp.experts.124.down_proj", + "model.layers.52.mlp.experts.124.gate_proj", + "model.layers.52.mlp.experts.124.up_proj", + "model.layers.52.mlp.experts.125.down_proj", + "model.layers.52.mlp.experts.125.gate_proj", + "model.layers.52.mlp.experts.125.up_proj", + "model.layers.52.mlp.experts.126.down_proj", + "model.layers.52.mlp.experts.126.gate_proj", + "model.layers.52.mlp.experts.126.up_proj", + "model.layers.52.mlp.experts.127.down_proj", + "model.layers.52.mlp.experts.127.gate_proj", + "model.layers.52.mlp.experts.127.up_proj", + "model.layers.52.mlp.experts.128.down_proj", + "model.layers.52.mlp.experts.128.gate_proj", + "model.layers.52.mlp.experts.128.up_proj", + "model.layers.52.mlp.experts.129.down_proj", + "model.layers.52.mlp.experts.129.gate_proj", + "model.layers.52.mlp.experts.129.up_proj", + "model.layers.52.mlp.experts.13.down_proj", + "model.layers.52.mlp.experts.13.gate_proj", + "model.layers.52.mlp.experts.13.up_proj", + "model.layers.52.mlp.experts.130.down_proj", + "model.layers.52.mlp.experts.130.gate_proj", + "model.layers.52.mlp.experts.130.up_proj", + "model.layers.52.mlp.experts.131.down_proj", + "model.layers.52.mlp.experts.131.gate_proj", + "model.layers.52.mlp.experts.131.up_proj", + "model.layers.52.mlp.experts.132.down_proj", + "model.layers.52.mlp.experts.132.gate_proj", + "model.layers.52.mlp.experts.132.up_proj", + "model.layers.52.mlp.experts.133.down_proj", + "model.layers.52.mlp.experts.133.gate_proj", + "model.layers.52.mlp.experts.133.up_proj", + "model.layers.52.mlp.experts.134.down_proj", + "model.layers.52.mlp.experts.134.gate_proj", + "model.layers.52.mlp.experts.134.up_proj", + "model.layers.52.mlp.experts.135.down_proj", + "model.layers.52.mlp.experts.135.gate_proj", + "model.layers.52.mlp.experts.135.up_proj", + "model.layers.52.mlp.experts.136.down_proj", + "model.layers.52.mlp.experts.136.gate_proj", + "model.layers.52.mlp.experts.136.up_proj", + "model.layers.52.mlp.experts.137.down_proj", + "model.layers.52.mlp.experts.137.gate_proj", + "model.layers.52.mlp.experts.137.up_proj", + "model.layers.52.mlp.experts.138.down_proj", + "model.layers.52.mlp.experts.138.gate_proj", + "model.layers.52.mlp.experts.138.up_proj", + "model.layers.52.mlp.experts.139.down_proj", + "model.layers.52.mlp.experts.139.gate_proj", + "model.layers.52.mlp.experts.139.up_proj", + "model.layers.52.mlp.experts.14.down_proj", + "model.layers.52.mlp.experts.14.gate_proj", + "model.layers.52.mlp.experts.14.up_proj", + "model.layers.52.mlp.experts.140.down_proj", + "model.layers.52.mlp.experts.140.gate_proj", + "model.layers.52.mlp.experts.140.up_proj", + "model.layers.52.mlp.experts.141.down_proj", + "model.layers.52.mlp.experts.141.gate_proj", + "model.layers.52.mlp.experts.141.up_proj", + "model.layers.52.mlp.experts.142.down_proj", + "model.layers.52.mlp.experts.142.gate_proj", + "model.layers.52.mlp.experts.142.up_proj", + "model.layers.52.mlp.experts.143.down_proj", + "model.layers.52.mlp.experts.143.gate_proj", + "model.layers.52.mlp.experts.143.up_proj", + "model.layers.52.mlp.experts.144.down_proj", + "model.layers.52.mlp.experts.144.gate_proj", + "model.layers.52.mlp.experts.144.up_proj", + "model.layers.52.mlp.experts.145.down_proj", + "model.layers.52.mlp.experts.145.gate_proj", + "model.layers.52.mlp.experts.145.up_proj", + "model.layers.52.mlp.experts.146.down_proj", + "model.layers.52.mlp.experts.146.gate_proj", + "model.layers.52.mlp.experts.146.up_proj", + "model.layers.52.mlp.experts.147.down_proj", + "model.layers.52.mlp.experts.147.gate_proj", + "model.layers.52.mlp.experts.147.up_proj", + "model.layers.52.mlp.experts.148.down_proj", + "model.layers.52.mlp.experts.148.gate_proj", + "model.layers.52.mlp.experts.148.up_proj", + "model.layers.52.mlp.experts.149.down_proj", + "model.layers.52.mlp.experts.149.gate_proj", + "model.layers.52.mlp.experts.149.up_proj", + "model.layers.52.mlp.experts.15.down_proj", + "model.layers.52.mlp.experts.15.gate_proj", + "model.layers.52.mlp.experts.15.up_proj", + "model.layers.52.mlp.experts.150.down_proj", + "model.layers.52.mlp.experts.150.gate_proj", + "model.layers.52.mlp.experts.150.up_proj", + "model.layers.52.mlp.experts.151.down_proj", + "model.layers.52.mlp.experts.151.gate_proj", + "model.layers.52.mlp.experts.151.up_proj", + "model.layers.52.mlp.experts.152.down_proj", + "model.layers.52.mlp.experts.152.gate_proj", + "model.layers.52.mlp.experts.152.up_proj", + "model.layers.52.mlp.experts.153.down_proj", + "model.layers.52.mlp.experts.153.gate_proj", + "model.layers.52.mlp.experts.153.up_proj", + "model.layers.52.mlp.experts.154.down_proj", + "model.layers.52.mlp.experts.154.gate_proj", + "model.layers.52.mlp.experts.154.up_proj", + "model.layers.52.mlp.experts.155.down_proj", + "model.layers.52.mlp.experts.155.gate_proj", + "model.layers.52.mlp.experts.155.up_proj", + "model.layers.52.mlp.experts.156.down_proj", + "model.layers.52.mlp.experts.156.gate_proj", + "model.layers.52.mlp.experts.156.up_proj", + "model.layers.52.mlp.experts.157.down_proj", + "model.layers.52.mlp.experts.157.gate_proj", + "model.layers.52.mlp.experts.157.up_proj", + "model.layers.52.mlp.experts.158.down_proj", + "model.layers.52.mlp.experts.158.gate_proj", + "model.layers.52.mlp.experts.158.up_proj", + "model.layers.52.mlp.experts.159.down_proj", + "model.layers.52.mlp.experts.159.gate_proj", + "model.layers.52.mlp.experts.159.up_proj", + "model.layers.52.mlp.experts.16.down_proj", + "model.layers.52.mlp.experts.16.gate_proj", + "model.layers.52.mlp.experts.16.up_proj", + "model.layers.52.mlp.experts.160.down_proj", + "model.layers.52.mlp.experts.160.gate_proj", + "model.layers.52.mlp.experts.160.up_proj", + "model.layers.52.mlp.experts.161.down_proj", + "model.layers.52.mlp.experts.161.gate_proj", + "model.layers.52.mlp.experts.161.up_proj", + "model.layers.52.mlp.experts.162.down_proj", + "model.layers.52.mlp.experts.162.gate_proj", + "model.layers.52.mlp.experts.162.up_proj", + "model.layers.52.mlp.experts.163.down_proj", + "model.layers.52.mlp.experts.163.gate_proj", + "model.layers.52.mlp.experts.163.up_proj", + "model.layers.52.mlp.experts.164.down_proj", + "model.layers.52.mlp.experts.164.gate_proj", + "model.layers.52.mlp.experts.164.up_proj", + "model.layers.52.mlp.experts.165.down_proj", + "model.layers.52.mlp.experts.165.gate_proj", + "model.layers.52.mlp.experts.165.up_proj", + "model.layers.52.mlp.experts.166.down_proj", + "model.layers.52.mlp.experts.166.gate_proj", + "model.layers.52.mlp.experts.166.up_proj", + "model.layers.52.mlp.experts.167.down_proj", + "model.layers.52.mlp.experts.167.gate_proj", + "model.layers.52.mlp.experts.167.up_proj", + "model.layers.52.mlp.experts.168.down_proj", + "model.layers.52.mlp.experts.168.gate_proj", + "model.layers.52.mlp.experts.168.up_proj", + "model.layers.52.mlp.experts.169.down_proj", + "model.layers.52.mlp.experts.169.gate_proj", + "model.layers.52.mlp.experts.169.up_proj", + "model.layers.52.mlp.experts.17.down_proj", + "model.layers.52.mlp.experts.17.gate_proj", + "model.layers.52.mlp.experts.17.up_proj", + "model.layers.52.mlp.experts.170.down_proj", + "model.layers.52.mlp.experts.170.gate_proj", + "model.layers.52.mlp.experts.170.up_proj", + "model.layers.52.mlp.experts.171.down_proj", + "model.layers.52.mlp.experts.171.gate_proj", + "model.layers.52.mlp.experts.171.up_proj", + "model.layers.52.mlp.experts.172.down_proj", + "model.layers.52.mlp.experts.172.gate_proj", + "model.layers.52.mlp.experts.172.up_proj", + "model.layers.52.mlp.experts.173.down_proj", + "model.layers.52.mlp.experts.173.gate_proj", + "model.layers.52.mlp.experts.173.up_proj", + "model.layers.52.mlp.experts.174.down_proj", + "model.layers.52.mlp.experts.174.gate_proj", + "model.layers.52.mlp.experts.174.up_proj", + "model.layers.52.mlp.experts.175.down_proj", + "model.layers.52.mlp.experts.175.gate_proj", + "model.layers.52.mlp.experts.175.up_proj", + "model.layers.52.mlp.experts.176.down_proj", + "model.layers.52.mlp.experts.176.gate_proj", + "model.layers.52.mlp.experts.176.up_proj", + "model.layers.52.mlp.experts.177.down_proj", + "model.layers.52.mlp.experts.177.gate_proj", + "model.layers.52.mlp.experts.177.up_proj", + "model.layers.52.mlp.experts.178.down_proj", + "model.layers.52.mlp.experts.178.gate_proj", + "model.layers.52.mlp.experts.178.up_proj", + "model.layers.52.mlp.experts.179.down_proj", + "model.layers.52.mlp.experts.179.gate_proj", + "model.layers.52.mlp.experts.179.up_proj", + "model.layers.52.mlp.experts.18.down_proj", + "model.layers.52.mlp.experts.18.gate_proj", + "model.layers.52.mlp.experts.18.up_proj", + "model.layers.52.mlp.experts.180.down_proj", + "model.layers.52.mlp.experts.180.gate_proj", + "model.layers.52.mlp.experts.180.up_proj", + "model.layers.52.mlp.experts.181.down_proj", + "model.layers.52.mlp.experts.181.gate_proj", + "model.layers.52.mlp.experts.181.up_proj", + "model.layers.52.mlp.experts.182.down_proj", + "model.layers.52.mlp.experts.182.gate_proj", + "model.layers.52.mlp.experts.182.up_proj", + "model.layers.52.mlp.experts.183.down_proj", + "model.layers.52.mlp.experts.183.gate_proj", + "model.layers.52.mlp.experts.183.up_proj", + "model.layers.52.mlp.experts.184.down_proj", + "model.layers.52.mlp.experts.184.gate_proj", + "model.layers.52.mlp.experts.184.up_proj", + "model.layers.52.mlp.experts.185.down_proj", + "model.layers.52.mlp.experts.185.gate_proj", + "model.layers.52.mlp.experts.185.up_proj", + "model.layers.52.mlp.experts.186.down_proj", + "model.layers.52.mlp.experts.186.gate_proj", + "model.layers.52.mlp.experts.186.up_proj", + "model.layers.52.mlp.experts.187.down_proj", + "model.layers.52.mlp.experts.187.gate_proj", + "model.layers.52.mlp.experts.187.up_proj", + "model.layers.52.mlp.experts.188.down_proj", + "model.layers.52.mlp.experts.188.gate_proj", + "model.layers.52.mlp.experts.188.up_proj", + "model.layers.52.mlp.experts.189.down_proj", + "model.layers.52.mlp.experts.189.gate_proj", + "model.layers.52.mlp.experts.189.up_proj", + "model.layers.52.mlp.experts.19.down_proj", + "model.layers.52.mlp.experts.19.gate_proj", + "model.layers.52.mlp.experts.19.up_proj", + "model.layers.52.mlp.experts.190.down_proj", + "model.layers.52.mlp.experts.190.gate_proj", + "model.layers.52.mlp.experts.190.up_proj", + "model.layers.52.mlp.experts.191.down_proj", + "model.layers.52.mlp.experts.191.gate_proj", + "model.layers.52.mlp.experts.191.up_proj", + "model.layers.52.mlp.experts.192.down_proj", + "model.layers.52.mlp.experts.192.gate_proj", + "model.layers.52.mlp.experts.192.up_proj", + "model.layers.52.mlp.experts.193.down_proj", + "model.layers.52.mlp.experts.193.gate_proj", + "model.layers.52.mlp.experts.193.up_proj", + "model.layers.52.mlp.experts.194.down_proj", + "model.layers.52.mlp.experts.194.gate_proj", + "model.layers.52.mlp.experts.194.up_proj", + "model.layers.52.mlp.experts.195.down_proj", + "model.layers.52.mlp.experts.195.gate_proj", + "model.layers.52.mlp.experts.195.up_proj", + "model.layers.52.mlp.experts.196.down_proj", + "model.layers.52.mlp.experts.196.gate_proj", + "model.layers.52.mlp.experts.196.up_proj", + "model.layers.52.mlp.experts.197.down_proj", + "model.layers.52.mlp.experts.197.gate_proj", + "model.layers.52.mlp.experts.197.up_proj", + "model.layers.52.mlp.experts.198.down_proj", + "model.layers.52.mlp.experts.198.gate_proj", + "model.layers.52.mlp.experts.198.up_proj", + "model.layers.52.mlp.experts.199.down_proj", + "model.layers.52.mlp.experts.199.gate_proj", + "model.layers.52.mlp.experts.199.up_proj", + "model.layers.52.mlp.experts.2.down_proj", + "model.layers.52.mlp.experts.2.gate_proj", + "model.layers.52.mlp.experts.2.up_proj", + "model.layers.52.mlp.experts.20.down_proj", + "model.layers.52.mlp.experts.20.gate_proj", + "model.layers.52.mlp.experts.20.up_proj", + "model.layers.52.mlp.experts.200.down_proj", + "model.layers.52.mlp.experts.200.gate_proj", + "model.layers.52.mlp.experts.200.up_proj", + "model.layers.52.mlp.experts.201.down_proj", + "model.layers.52.mlp.experts.201.gate_proj", + "model.layers.52.mlp.experts.201.up_proj", + "model.layers.52.mlp.experts.202.down_proj", + "model.layers.52.mlp.experts.202.gate_proj", + "model.layers.52.mlp.experts.202.up_proj", + "model.layers.52.mlp.experts.203.down_proj", + "model.layers.52.mlp.experts.203.gate_proj", + "model.layers.52.mlp.experts.203.up_proj", + "model.layers.52.mlp.experts.204.down_proj", + "model.layers.52.mlp.experts.204.gate_proj", + "model.layers.52.mlp.experts.204.up_proj", + "model.layers.52.mlp.experts.205.down_proj", + "model.layers.52.mlp.experts.205.gate_proj", + "model.layers.52.mlp.experts.205.up_proj", + "model.layers.52.mlp.experts.206.down_proj", + "model.layers.52.mlp.experts.206.gate_proj", + "model.layers.52.mlp.experts.206.up_proj", + "model.layers.52.mlp.experts.207.down_proj", + "model.layers.52.mlp.experts.207.gate_proj", + "model.layers.52.mlp.experts.207.up_proj", + "model.layers.52.mlp.experts.208.down_proj", + "model.layers.52.mlp.experts.208.gate_proj", + "model.layers.52.mlp.experts.208.up_proj", + "model.layers.52.mlp.experts.209.down_proj", + "model.layers.52.mlp.experts.209.gate_proj", + "model.layers.52.mlp.experts.209.up_proj", + "model.layers.52.mlp.experts.21.down_proj", + "model.layers.52.mlp.experts.21.gate_proj", + "model.layers.52.mlp.experts.21.up_proj", + "model.layers.52.mlp.experts.210.down_proj", + "model.layers.52.mlp.experts.210.gate_proj", + "model.layers.52.mlp.experts.210.up_proj", + "model.layers.52.mlp.experts.211.down_proj", + "model.layers.52.mlp.experts.211.gate_proj", + "model.layers.52.mlp.experts.211.up_proj", + "model.layers.52.mlp.experts.212.down_proj", + "model.layers.52.mlp.experts.212.gate_proj", + "model.layers.52.mlp.experts.212.up_proj", + "model.layers.52.mlp.experts.213.down_proj", + "model.layers.52.mlp.experts.213.gate_proj", + "model.layers.52.mlp.experts.213.up_proj", + "model.layers.52.mlp.experts.214.down_proj", + "model.layers.52.mlp.experts.214.gate_proj", + "model.layers.52.mlp.experts.214.up_proj", + "model.layers.52.mlp.experts.215.down_proj", + "model.layers.52.mlp.experts.215.gate_proj", + "model.layers.52.mlp.experts.215.up_proj", + "model.layers.52.mlp.experts.216.down_proj", + "model.layers.52.mlp.experts.216.gate_proj", + "model.layers.52.mlp.experts.216.up_proj", + "model.layers.52.mlp.experts.217.down_proj", + "model.layers.52.mlp.experts.217.gate_proj", + "model.layers.52.mlp.experts.217.up_proj", + "model.layers.52.mlp.experts.218.down_proj", + "model.layers.52.mlp.experts.218.gate_proj", + "model.layers.52.mlp.experts.218.up_proj", + "model.layers.52.mlp.experts.219.down_proj", + "model.layers.52.mlp.experts.219.gate_proj", + "model.layers.52.mlp.experts.219.up_proj", + "model.layers.52.mlp.experts.22.down_proj", + "model.layers.52.mlp.experts.22.gate_proj", + "model.layers.52.mlp.experts.22.up_proj", + "model.layers.52.mlp.experts.220.down_proj", + "model.layers.52.mlp.experts.220.gate_proj", + "model.layers.52.mlp.experts.220.up_proj", + "model.layers.52.mlp.experts.221.down_proj", + "model.layers.52.mlp.experts.221.gate_proj", + "model.layers.52.mlp.experts.221.up_proj", + "model.layers.52.mlp.experts.222.down_proj", + "model.layers.52.mlp.experts.222.gate_proj", + "model.layers.52.mlp.experts.222.up_proj", + "model.layers.52.mlp.experts.223.down_proj", + "model.layers.52.mlp.experts.223.gate_proj", + "model.layers.52.mlp.experts.223.up_proj", + "model.layers.52.mlp.experts.224.down_proj", + "model.layers.52.mlp.experts.224.gate_proj", + "model.layers.52.mlp.experts.224.up_proj", + "model.layers.52.mlp.experts.225.down_proj", + "model.layers.52.mlp.experts.225.gate_proj", + "model.layers.52.mlp.experts.225.up_proj", + "model.layers.52.mlp.experts.226.down_proj", + "model.layers.52.mlp.experts.226.gate_proj", + "model.layers.52.mlp.experts.226.up_proj", + "model.layers.52.mlp.experts.227.down_proj", + "model.layers.52.mlp.experts.227.gate_proj", + "model.layers.52.mlp.experts.227.up_proj", + "model.layers.52.mlp.experts.228.down_proj", + "model.layers.52.mlp.experts.228.gate_proj", + "model.layers.52.mlp.experts.228.up_proj", + "model.layers.52.mlp.experts.229.down_proj", + "model.layers.52.mlp.experts.229.gate_proj", + "model.layers.52.mlp.experts.229.up_proj", + "model.layers.52.mlp.experts.23.down_proj", + "model.layers.52.mlp.experts.23.gate_proj", + "model.layers.52.mlp.experts.23.up_proj", + "model.layers.52.mlp.experts.230.down_proj", + "model.layers.52.mlp.experts.230.gate_proj", + "model.layers.52.mlp.experts.230.up_proj", + "model.layers.52.mlp.experts.231.down_proj", + "model.layers.52.mlp.experts.231.gate_proj", + "model.layers.52.mlp.experts.231.up_proj", + "model.layers.52.mlp.experts.232.down_proj", + "model.layers.52.mlp.experts.232.gate_proj", + "model.layers.52.mlp.experts.232.up_proj", + "model.layers.52.mlp.experts.233.down_proj", + "model.layers.52.mlp.experts.233.gate_proj", + "model.layers.52.mlp.experts.233.up_proj", + "model.layers.52.mlp.experts.234.down_proj", + "model.layers.52.mlp.experts.234.gate_proj", + "model.layers.52.mlp.experts.234.up_proj", + "model.layers.52.mlp.experts.235.down_proj", + "model.layers.52.mlp.experts.235.gate_proj", + "model.layers.52.mlp.experts.235.up_proj", + "model.layers.52.mlp.experts.236.down_proj", + "model.layers.52.mlp.experts.236.gate_proj", + "model.layers.52.mlp.experts.236.up_proj", + "model.layers.52.mlp.experts.237.down_proj", + "model.layers.52.mlp.experts.237.gate_proj", + "model.layers.52.mlp.experts.237.up_proj", + "model.layers.52.mlp.experts.238.down_proj", + "model.layers.52.mlp.experts.238.gate_proj", + "model.layers.52.mlp.experts.238.up_proj", + "model.layers.52.mlp.experts.239.down_proj", + "model.layers.52.mlp.experts.239.gate_proj", + "model.layers.52.mlp.experts.239.up_proj", + "model.layers.52.mlp.experts.24.down_proj", + "model.layers.52.mlp.experts.24.gate_proj", + "model.layers.52.mlp.experts.24.up_proj", + "model.layers.52.mlp.experts.240.down_proj", + "model.layers.52.mlp.experts.240.gate_proj", + "model.layers.52.mlp.experts.240.up_proj", + "model.layers.52.mlp.experts.241.down_proj", + "model.layers.52.mlp.experts.241.gate_proj", + "model.layers.52.mlp.experts.241.up_proj", + "model.layers.52.mlp.experts.242.down_proj", + "model.layers.52.mlp.experts.242.gate_proj", + "model.layers.52.mlp.experts.242.up_proj", + "model.layers.52.mlp.experts.243.down_proj", + "model.layers.52.mlp.experts.243.gate_proj", + "model.layers.52.mlp.experts.243.up_proj", + "model.layers.52.mlp.experts.244.down_proj", + "model.layers.52.mlp.experts.244.gate_proj", + "model.layers.52.mlp.experts.244.up_proj", + "model.layers.52.mlp.experts.245.down_proj", + "model.layers.52.mlp.experts.245.gate_proj", + "model.layers.52.mlp.experts.245.up_proj", + "model.layers.52.mlp.experts.246.down_proj", + "model.layers.52.mlp.experts.246.gate_proj", + "model.layers.52.mlp.experts.246.up_proj", + "model.layers.52.mlp.experts.247.down_proj", + "model.layers.52.mlp.experts.247.gate_proj", + "model.layers.52.mlp.experts.247.up_proj", + "model.layers.52.mlp.experts.248.down_proj", + "model.layers.52.mlp.experts.248.gate_proj", + "model.layers.52.mlp.experts.248.up_proj", + "model.layers.52.mlp.experts.249.down_proj", + "model.layers.52.mlp.experts.249.gate_proj", + "model.layers.52.mlp.experts.249.up_proj", + "model.layers.52.mlp.experts.25.down_proj", + "model.layers.52.mlp.experts.25.gate_proj", + "model.layers.52.mlp.experts.25.up_proj", + "model.layers.52.mlp.experts.250.down_proj", + "model.layers.52.mlp.experts.250.gate_proj", + "model.layers.52.mlp.experts.250.up_proj", + "model.layers.52.mlp.experts.251.down_proj", + "model.layers.52.mlp.experts.251.gate_proj", + "model.layers.52.mlp.experts.251.up_proj", + "model.layers.52.mlp.experts.252.down_proj", + "model.layers.52.mlp.experts.252.gate_proj", + "model.layers.52.mlp.experts.252.up_proj", + "model.layers.52.mlp.experts.253.down_proj", + "model.layers.52.mlp.experts.253.gate_proj", + "model.layers.52.mlp.experts.253.up_proj", + "model.layers.52.mlp.experts.254.down_proj", + "model.layers.52.mlp.experts.254.gate_proj", + "model.layers.52.mlp.experts.254.up_proj", + "model.layers.52.mlp.experts.255.down_proj", + "model.layers.52.mlp.experts.255.gate_proj", + "model.layers.52.mlp.experts.255.up_proj", + "model.layers.52.mlp.experts.26.down_proj", + "model.layers.52.mlp.experts.26.gate_proj", + "model.layers.52.mlp.experts.26.up_proj", + "model.layers.52.mlp.experts.27.down_proj", + "model.layers.52.mlp.experts.27.gate_proj", + "model.layers.52.mlp.experts.27.up_proj", + "model.layers.52.mlp.experts.28.down_proj", + "model.layers.52.mlp.experts.28.gate_proj", + "model.layers.52.mlp.experts.28.up_proj", + "model.layers.52.mlp.experts.29.down_proj", + "model.layers.52.mlp.experts.29.gate_proj", + "model.layers.52.mlp.experts.29.up_proj", + "model.layers.52.mlp.experts.3.down_proj", + "model.layers.52.mlp.experts.3.gate_proj", + "model.layers.52.mlp.experts.3.up_proj", + "model.layers.52.mlp.experts.30.down_proj", + "model.layers.52.mlp.experts.30.gate_proj", + "model.layers.52.mlp.experts.30.up_proj", + "model.layers.52.mlp.experts.31.down_proj", + "model.layers.52.mlp.experts.31.gate_proj", + "model.layers.52.mlp.experts.31.up_proj", + "model.layers.52.mlp.experts.32.down_proj", + "model.layers.52.mlp.experts.32.gate_proj", + "model.layers.52.mlp.experts.32.up_proj", + "model.layers.52.mlp.experts.33.down_proj", + "model.layers.52.mlp.experts.33.gate_proj", + "model.layers.52.mlp.experts.33.up_proj", + "model.layers.52.mlp.experts.34.down_proj", + "model.layers.52.mlp.experts.34.gate_proj", + "model.layers.52.mlp.experts.34.up_proj", + "model.layers.52.mlp.experts.35.down_proj", + "model.layers.52.mlp.experts.35.gate_proj", + "model.layers.52.mlp.experts.35.up_proj", + "model.layers.52.mlp.experts.36.down_proj", + "model.layers.52.mlp.experts.36.gate_proj", + "model.layers.52.mlp.experts.36.up_proj", + "model.layers.52.mlp.experts.37.down_proj", + "model.layers.52.mlp.experts.37.gate_proj", + "model.layers.52.mlp.experts.37.up_proj", + "model.layers.52.mlp.experts.38.down_proj", + "model.layers.52.mlp.experts.38.gate_proj", + "model.layers.52.mlp.experts.38.up_proj", + "model.layers.52.mlp.experts.39.down_proj", + "model.layers.52.mlp.experts.39.gate_proj", + "model.layers.52.mlp.experts.39.up_proj", + "model.layers.52.mlp.experts.4.down_proj", + "model.layers.52.mlp.experts.4.gate_proj", + "model.layers.52.mlp.experts.4.up_proj", + "model.layers.52.mlp.experts.40.down_proj", + "model.layers.52.mlp.experts.40.gate_proj", + "model.layers.52.mlp.experts.40.up_proj", + "model.layers.52.mlp.experts.41.down_proj", + "model.layers.52.mlp.experts.41.gate_proj", + "model.layers.52.mlp.experts.41.up_proj", + "model.layers.52.mlp.experts.42.down_proj", + "model.layers.52.mlp.experts.42.gate_proj", + "model.layers.52.mlp.experts.42.up_proj", + "model.layers.52.mlp.experts.43.down_proj", + "model.layers.52.mlp.experts.43.gate_proj", + "model.layers.52.mlp.experts.43.up_proj", + "model.layers.52.mlp.experts.44.down_proj", + "model.layers.52.mlp.experts.44.gate_proj", + "model.layers.52.mlp.experts.44.up_proj", + "model.layers.52.mlp.experts.45.down_proj", + "model.layers.52.mlp.experts.45.gate_proj", + "model.layers.52.mlp.experts.45.up_proj", + "model.layers.52.mlp.experts.46.down_proj", + "model.layers.52.mlp.experts.46.gate_proj", + "model.layers.52.mlp.experts.46.up_proj", + "model.layers.52.mlp.experts.47.down_proj", + "model.layers.52.mlp.experts.47.gate_proj", + "model.layers.52.mlp.experts.47.up_proj", + "model.layers.52.mlp.experts.48.down_proj", + "model.layers.52.mlp.experts.48.gate_proj", + "model.layers.52.mlp.experts.48.up_proj", + "model.layers.52.mlp.experts.49.down_proj", + "model.layers.52.mlp.experts.49.gate_proj", + "model.layers.52.mlp.experts.49.up_proj", + "model.layers.52.mlp.experts.5.down_proj", + "model.layers.52.mlp.experts.5.gate_proj", + "model.layers.52.mlp.experts.5.up_proj", + "model.layers.52.mlp.experts.50.down_proj", + "model.layers.52.mlp.experts.50.gate_proj", + "model.layers.52.mlp.experts.50.up_proj", + "model.layers.52.mlp.experts.51.down_proj", + "model.layers.52.mlp.experts.51.gate_proj", + "model.layers.52.mlp.experts.51.up_proj", + "model.layers.52.mlp.experts.52.down_proj", + "model.layers.52.mlp.experts.52.gate_proj", + "model.layers.52.mlp.experts.52.up_proj", + "model.layers.52.mlp.experts.53.down_proj", + "model.layers.52.mlp.experts.53.gate_proj", + "model.layers.52.mlp.experts.53.up_proj", + "model.layers.52.mlp.experts.54.down_proj", + "model.layers.52.mlp.experts.54.gate_proj", + "model.layers.52.mlp.experts.54.up_proj", + "model.layers.52.mlp.experts.55.down_proj", + "model.layers.52.mlp.experts.55.gate_proj", + "model.layers.52.mlp.experts.55.up_proj", + "model.layers.52.mlp.experts.56.down_proj", + "model.layers.52.mlp.experts.56.gate_proj", + "model.layers.52.mlp.experts.56.up_proj", + "model.layers.52.mlp.experts.57.down_proj", + "model.layers.52.mlp.experts.57.gate_proj", + "model.layers.52.mlp.experts.57.up_proj", + "model.layers.52.mlp.experts.58.down_proj", + "model.layers.52.mlp.experts.58.gate_proj", + "model.layers.52.mlp.experts.58.up_proj", + "model.layers.52.mlp.experts.59.down_proj", + "model.layers.52.mlp.experts.59.gate_proj", + "model.layers.52.mlp.experts.59.up_proj", + "model.layers.52.mlp.experts.6.down_proj", + "model.layers.52.mlp.experts.6.gate_proj", + "model.layers.52.mlp.experts.6.up_proj", + "model.layers.52.mlp.experts.60.down_proj", + "model.layers.52.mlp.experts.60.gate_proj", + "model.layers.52.mlp.experts.60.up_proj", + "model.layers.52.mlp.experts.61.down_proj", + "model.layers.52.mlp.experts.61.gate_proj", + "model.layers.52.mlp.experts.61.up_proj", + "model.layers.52.mlp.experts.62.down_proj", + "model.layers.52.mlp.experts.62.gate_proj", + "model.layers.52.mlp.experts.62.up_proj", + "model.layers.52.mlp.experts.63.down_proj", + "model.layers.52.mlp.experts.63.gate_proj", + "model.layers.52.mlp.experts.63.up_proj", + "model.layers.52.mlp.experts.64.down_proj", + "model.layers.52.mlp.experts.64.gate_proj", + "model.layers.52.mlp.experts.64.up_proj", + "model.layers.52.mlp.experts.65.down_proj", + "model.layers.52.mlp.experts.65.gate_proj", + "model.layers.52.mlp.experts.65.up_proj", + "model.layers.52.mlp.experts.66.down_proj", + "model.layers.52.mlp.experts.66.gate_proj", + "model.layers.52.mlp.experts.66.up_proj", + "model.layers.52.mlp.experts.67.down_proj", + "model.layers.52.mlp.experts.67.gate_proj", + "model.layers.52.mlp.experts.67.up_proj", + "model.layers.52.mlp.experts.68.down_proj", + "model.layers.52.mlp.experts.68.gate_proj", + "model.layers.52.mlp.experts.68.up_proj", + "model.layers.52.mlp.experts.69.down_proj", + "model.layers.52.mlp.experts.69.gate_proj", + "model.layers.52.mlp.experts.69.up_proj", + "model.layers.52.mlp.experts.7.down_proj", + "model.layers.52.mlp.experts.7.gate_proj", + "model.layers.52.mlp.experts.7.up_proj", + "model.layers.52.mlp.experts.70.down_proj", + "model.layers.52.mlp.experts.70.gate_proj", + "model.layers.52.mlp.experts.70.up_proj", + "model.layers.52.mlp.experts.71.down_proj", + "model.layers.52.mlp.experts.71.gate_proj", + "model.layers.52.mlp.experts.71.up_proj", + "model.layers.52.mlp.experts.72.down_proj", + "model.layers.52.mlp.experts.72.gate_proj", + "model.layers.52.mlp.experts.72.up_proj", + "model.layers.52.mlp.experts.73.down_proj", + "model.layers.52.mlp.experts.73.gate_proj", + "model.layers.52.mlp.experts.73.up_proj", + "model.layers.52.mlp.experts.74.down_proj", + "model.layers.52.mlp.experts.74.gate_proj", + "model.layers.52.mlp.experts.74.up_proj", + "model.layers.52.mlp.experts.75.down_proj", + "model.layers.52.mlp.experts.75.gate_proj", + "model.layers.52.mlp.experts.75.up_proj", + "model.layers.52.mlp.experts.76.down_proj", + "model.layers.52.mlp.experts.76.gate_proj", + "model.layers.52.mlp.experts.76.up_proj", + "model.layers.52.mlp.experts.77.down_proj", + "model.layers.52.mlp.experts.77.gate_proj", + "model.layers.52.mlp.experts.77.up_proj", + "model.layers.52.mlp.experts.78.down_proj", + "model.layers.52.mlp.experts.78.gate_proj", + "model.layers.52.mlp.experts.78.up_proj", + "model.layers.52.mlp.experts.79.down_proj", + "model.layers.52.mlp.experts.79.gate_proj", + "model.layers.52.mlp.experts.79.up_proj", + "model.layers.52.mlp.experts.8.down_proj", + "model.layers.52.mlp.experts.8.gate_proj", + "model.layers.52.mlp.experts.8.up_proj", + "model.layers.52.mlp.experts.80.down_proj", + "model.layers.52.mlp.experts.80.gate_proj", + "model.layers.52.mlp.experts.80.up_proj", + "model.layers.52.mlp.experts.81.down_proj", + "model.layers.52.mlp.experts.81.gate_proj", + "model.layers.52.mlp.experts.81.up_proj", + "model.layers.52.mlp.experts.82.down_proj", + "model.layers.52.mlp.experts.82.gate_proj", + "model.layers.52.mlp.experts.82.up_proj", + "model.layers.52.mlp.experts.83.down_proj", + "model.layers.52.mlp.experts.83.gate_proj", + "model.layers.52.mlp.experts.83.up_proj", + "model.layers.52.mlp.experts.84.down_proj", + "model.layers.52.mlp.experts.84.gate_proj", + "model.layers.52.mlp.experts.84.up_proj", + "model.layers.52.mlp.experts.85.down_proj", + "model.layers.52.mlp.experts.85.gate_proj", + "model.layers.52.mlp.experts.85.up_proj", + "model.layers.52.mlp.experts.86.down_proj", + "model.layers.52.mlp.experts.86.gate_proj", + "model.layers.52.mlp.experts.86.up_proj", + "model.layers.52.mlp.experts.87.down_proj", + "model.layers.52.mlp.experts.87.gate_proj", + "model.layers.52.mlp.experts.87.up_proj", + "model.layers.52.mlp.experts.88.down_proj", + "model.layers.52.mlp.experts.88.gate_proj", + "model.layers.52.mlp.experts.88.up_proj", + "model.layers.52.mlp.experts.89.down_proj", + "model.layers.52.mlp.experts.89.gate_proj", + "model.layers.52.mlp.experts.89.up_proj", + "model.layers.52.mlp.experts.9.down_proj", + "model.layers.52.mlp.experts.9.gate_proj", + "model.layers.52.mlp.experts.9.up_proj", + "model.layers.52.mlp.experts.90.down_proj", + "model.layers.52.mlp.experts.90.gate_proj", + "model.layers.52.mlp.experts.90.up_proj", + "model.layers.52.mlp.experts.91.down_proj", + "model.layers.52.mlp.experts.91.gate_proj", + "model.layers.52.mlp.experts.91.up_proj", + "model.layers.52.mlp.experts.92.down_proj", + "model.layers.52.mlp.experts.92.gate_proj", + "model.layers.52.mlp.experts.92.up_proj", + "model.layers.52.mlp.experts.93.down_proj", + "model.layers.52.mlp.experts.93.gate_proj", + "model.layers.52.mlp.experts.93.up_proj", + "model.layers.52.mlp.experts.94.down_proj", + "model.layers.52.mlp.experts.94.gate_proj", + "model.layers.52.mlp.experts.94.up_proj", + "model.layers.52.mlp.experts.95.down_proj", + "model.layers.52.mlp.experts.95.gate_proj", + "model.layers.52.mlp.experts.95.up_proj", + "model.layers.52.mlp.experts.96.down_proj", + "model.layers.52.mlp.experts.96.gate_proj", + "model.layers.52.mlp.experts.96.up_proj", + "model.layers.52.mlp.experts.97.down_proj", + "model.layers.52.mlp.experts.97.gate_proj", + "model.layers.52.mlp.experts.97.up_proj", + "model.layers.52.mlp.experts.98.down_proj", + "model.layers.52.mlp.experts.98.gate_proj", + "model.layers.52.mlp.experts.98.up_proj", + "model.layers.52.mlp.experts.99.down_proj", + "model.layers.52.mlp.experts.99.gate_proj", + "model.layers.52.mlp.experts.99.up_proj", + "model.layers.53.mlp.experts.0.down_proj", + "model.layers.53.mlp.experts.0.gate_proj", + "model.layers.53.mlp.experts.0.up_proj", + "model.layers.53.mlp.experts.1.down_proj", + "model.layers.53.mlp.experts.1.gate_proj", + "model.layers.53.mlp.experts.1.up_proj", + "model.layers.53.mlp.experts.10.down_proj", + "model.layers.53.mlp.experts.10.gate_proj", + "model.layers.53.mlp.experts.10.up_proj", + "model.layers.53.mlp.experts.100.down_proj", + "model.layers.53.mlp.experts.100.gate_proj", + "model.layers.53.mlp.experts.100.up_proj", + "model.layers.53.mlp.experts.101.down_proj", + "model.layers.53.mlp.experts.101.gate_proj", + "model.layers.53.mlp.experts.101.up_proj", + "model.layers.53.mlp.experts.102.down_proj", + "model.layers.53.mlp.experts.102.gate_proj", + "model.layers.53.mlp.experts.102.up_proj", + "model.layers.53.mlp.experts.103.down_proj", + "model.layers.53.mlp.experts.103.gate_proj", + "model.layers.53.mlp.experts.103.up_proj", + "model.layers.53.mlp.experts.104.down_proj", + "model.layers.53.mlp.experts.104.gate_proj", + "model.layers.53.mlp.experts.104.up_proj", + "model.layers.53.mlp.experts.105.down_proj", + "model.layers.53.mlp.experts.105.gate_proj", + "model.layers.53.mlp.experts.105.up_proj", + "model.layers.53.mlp.experts.106.down_proj", + "model.layers.53.mlp.experts.106.gate_proj", + "model.layers.53.mlp.experts.106.up_proj", + "model.layers.53.mlp.experts.107.down_proj", + "model.layers.53.mlp.experts.107.gate_proj", + "model.layers.53.mlp.experts.107.up_proj", + "model.layers.53.mlp.experts.108.down_proj", + "model.layers.53.mlp.experts.108.gate_proj", + "model.layers.53.mlp.experts.108.up_proj", + "model.layers.53.mlp.experts.109.down_proj", + "model.layers.53.mlp.experts.109.gate_proj", + "model.layers.53.mlp.experts.109.up_proj", + "model.layers.53.mlp.experts.11.down_proj", + "model.layers.53.mlp.experts.11.gate_proj", + "model.layers.53.mlp.experts.11.up_proj", + "model.layers.53.mlp.experts.110.down_proj", + "model.layers.53.mlp.experts.110.gate_proj", + "model.layers.53.mlp.experts.110.up_proj", + "model.layers.53.mlp.experts.111.down_proj", + "model.layers.53.mlp.experts.111.gate_proj", + "model.layers.53.mlp.experts.111.up_proj", + "model.layers.53.mlp.experts.112.down_proj", + "model.layers.53.mlp.experts.112.gate_proj", + "model.layers.53.mlp.experts.112.up_proj", + "model.layers.53.mlp.experts.113.down_proj", + "model.layers.53.mlp.experts.113.gate_proj", + "model.layers.53.mlp.experts.113.up_proj", + "model.layers.53.mlp.experts.114.down_proj", + "model.layers.53.mlp.experts.114.gate_proj", + "model.layers.53.mlp.experts.114.up_proj", + "model.layers.53.mlp.experts.115.down_proj", + "model.layers.53.mlp.experts.115.gate_proj", + "model.layers.53.mlp.experts.115.up_proj", + "model.layers.53.mlp.experts.116.down_proj", + "model.layers.53.mlp.experts.116.gate_proj", + "model.layers.53.mlp.experts.116.up_proj", + "model.layers.53.mlp.experts.117.down_proj", + "model.layers.53.mlp.experts.117.gate_proj", + "model.layers.53.mlp.experts.117.up_proj", + "model.layers.53.mlp.experts.118.down_proj", + "model.layers.53.mlp.experts.118.gate_proj", + "model.layers.53.mlp.experts.118.up_proj", + "model.layers.53.mlp.experts.119.down_proj", + "model.layers.53.mlp.experts.119.gate_proj", + "model.layers.53.mlp.experts.119.up_proj", + "model.layers.53.mlp.experts.12.down_proj", + "model.layers.53.mlp.experts.12.gate_proj", + "model.layers.53.mlp.experts.12.up_proj", + "model.layers.53.mlp.experts.120.down_proj", + "model.layers.53.mlp.experts.120.gate_proj", + "model.layers.53.mlp.experts.120.up_proj", + "model.layers.53.mlp.experts.121.down_proj", + "model.layers.53.mlp.experts.121.gate_proj", + "model.layers.53.mlp.experts.121.up_proj", + "model.layers.53.mlp.experts.122.down_proj", + "model.layers.53.mlp.experts.122.gate_proj", + "model.layers.53.mlp.experts.122.up_proj", + "model.layers.53.mlp.experts.123.down_proj", + "model.layers.53.mlp.experts.123.gate_proj", + "model.layers.53.mlp.experts.123.up_proj", + "model.layers.53.mlp.experts.124.down_proj", + "model.layers.53.mlp.experts.124.gate_proj", + "model.layers.53.mlp.experts.124.up_proj", + "model.layers.53.mlp.experts.125.down_proj", + "model.layers.53.mlp.experts.125.gate_proj", + "model.layers.53.mlp.experts.125.up_proj", + "model.layers.53.mlp.experts.126.down_proj", + "model.layers.53.mlp.experts.126.gate_proj", + "model.layers.53.mlp.experts.126.up_proj", + "model.layers.53.mlp.experts.127.down_proj", + "model.layers.53.mlp.experts.127.gate_proj", + "model.layers.53.mlp.experts.127.up_proj", + "model.layers.53.mlp.experts.128.down_proj", + "model.layers.53.mlp.experts.128.gate_proj", + "model.layers.53.mlp.experts.128.up_proj", + "model.layers.53.mlp.experts.129.down_proj", + "model.layers.53.mlp.experts.129.gate_proj", + "model.layers.53.mlp.experts.129.up_proj", + "model.layers.53.mlp.experts.13.down_proj", + "model.layers.53.mlp.experts.13.gate_proj", + "model.layers.53.mlp.experts.13.up_proj", + "model.layers.53.mlp.experts.130.down_proj", + "model.layers.53.mlp.experts.130.gate_proj", + "model.layers.53.mlp.experts.130.up_proj", + "model.layers.53.mlp.experts.131.down_proj", + "model.layers.53.mlp.experts.131.gate_proj", + "model.layers.53.mlp.experts.131.up_proj", + "model.layers.53.mlp.experts.132.down_proj", + "model.layers.53.mlp.experts.132.gate_proj", + "model.layers.53.mlp.experts.132.up_proj", + "model.layers.53.mlp.experts.133.down_proj", + "model.layers.53.mlp.experts.133.gate_proj", + "model.layers.53.mlp.experts.133.up_proj", + "model.layers.53.mlp.experts.134.down_proj", + "model.layers.53.mlp.experts.134.gate_proj", + "model.layers.53.mlp.experts.134.up_proj", + "model.layers.53.mlp.experts.135.down_proj", + "model.layers.53.mlp.experts.135.gate_proj", + "model.layers.53.mlp.experts.135.up_proj", + "model.layers.53.mlp.experts.136.down_proj", + "model.layers.53.mlp.experts.136.gate_proj", + "model.layers.53.mlp.experts.136.up_proj", + "model.layers.53.mlp.experts.137.down_proj", + "model.layers.53.mlp.experts.137.gate_proj", + "model.layers.53.mlp.experts.137.up_proj", + "model.layers.53.mlp.experts.138.down_proj", + "model.layers.53.mlp.experts.138.gate_proj", + "model.layers.53.mlp.experts.138.up_proj", + "model.layers.53.mlp.experts.139.down_proj", + "model.layers.53.mlp.experts.139.gate_proj", + "model.layers.53.mlp.experts.139.up_proj", + "model.layers.53.mlp.experts.14.down_proj", + "model.layers.53.mlp.experts.14.gate_proj", + "model.layers.53.mlp.experts.14.up_proj", + "model.layers.53.mlp.experts.140.down_proj", + "model.layers.53.mlp.experts.140.gate_proj", + "model.layers.53.mlp.experts.140.up_proj", + "model.layers.53.mlp.experts.141.down_proj", + "model.layers.53.mlp.experts.141.gate_proj", + "model.layers.53.mlp.experts.141.up_proj", + "model.layers.53.mlp.experts.142.down_proj", + "model.layers.53.mlp.experts.142.gate_proj", + "model.layers.53.mlp.experts.142.up_proj", + "model.layers.53.mlp.experts.143.down_proj", + "model.layers.53.mlp.experts.143.gate_proj", + "model.layers.53.mlp.experts.143.up_proj", + "model.layers.53.mlp.experts.144.down_proj", + "model.layers.53.mlp.experts.144.gate_proj", + "model.layers.53.mlp.experts.144.up_proj", + "model.layers.53.mlp.experts.145.down_proj", + "model.layers.53.mlp.experts.145.gate_proj", + "model.layers.53.mlp.experts.145.up_proj", + "model.layers.53.mlp.experts.146.down_proj", + "model.layers.53.mlp.experts.146.gate_proj", + "model.layers.53.mlp.experts.146.up_proj", + "model.layers.53.mlp.experts.147.down_proj", + "model.layers.53.mlp.experts.147.gate_proj", + "model.layers.53.mlp.experts.147.up_proj", + "model.layers.53.mlp.experts.148.down_proj", + "model.layers.53.mlp.experts.148.gate_proj", + "model.layers.53.mlp.experts.148.up_proj", + "model.layers.53.mlp.experts.149.down_proj", + "model.layers.53.mlp.experts.149.gate_proj", + "model.layers.53.mlp.experts.149.up_proj", + "model.layers.53.mlp.experts.15.down_proj", + "model.layers.53.mlp.experts.15.gate_proj", + "model.layers.53.mlp.experts.15.up_proj", + "model.layers.53.mlp.experts.150.down_proj", + "model.layers.53.mlp.experts.150.gate_proj", + "model.layers.53.mlp.experts.150.up_proj", + "model.layers.53.mlp.experts.151.down_proj", + "model.layers.53.mlp.experts.151.gate_proj", + "model.layers.53.mlp.experts.151.up_proj", + "model.layers.53.mlp.experts.152.down_proj", + "model.layers.53.mlp.experts.152.gate_proj", + "model.layers.53.mlp.experts.152.up_proj", + "model.layers.53.mlp.experts.153.down_proj", + "model.layers.53.mlp.experts.153.gate_proj", + "model.layers.53.mlp.experts.153.up_proj", + "model.layers.53.mlp.experts.154.down_proj", + "model.layers.53.mlp.experts.154.gate_proj", + "model.layers.53.mlp.experts.154.up_proj", + "model.layers.53.mlp.experts.155.down_proj", + "model.layers.53.mlp.experts.155.gate_proj", + "model.layers.53.mlp.experts.155.up_proj", + "model.layers.53.mlp.experts.156.down_proj", + "model.layers.53.mlp.experts.156.gate_proj", + "model.layers.53.mlp.experts.156.up_proj", + "model.layers.53.mlp.experts.157.down_proj", + "model.layers.53.mlp.experts.157.gate_proj", + "model.layers.53.mlp.experts.157.up_proj", + "model.layers.53.mlp.experts.158.down_proj", + "model.layers.53.mlp.experts.158.gate_proj", + "model.layers.53.mlp.experts.158.up_proj", + "model.layers.53.mlp.experts.159.down_proj", + "model.layers.53.mlp.experts.159.gate_proj", + "model.layers.53.mlp.experts.159.up_proj", + "model.layers.53.mlp.experts.16.down_proj", + "model.layers.53.mlp.experts.16.gate_proj", + "model.layers.53.mlp.experts.16.up_proj", + "model.layers.53.mlp.experts.160.down_proj", + "model.layers.53.mlp.experts.160.gate_proj", + "model.layers.53.mlp.experts.160.up_proj", + "model.layers.53.mlp.experts.161.down_proj", + "model.layers.53.mlp.experts.161.gate_proj", + "model.layers.53.mlp.experts.161.up_proj", + "model.layers.53.mlp.experts.162.down_proj", + "model.layers.53.mlp.experts.162.gate_proj", + "model.layers.53.mlp.experts.162.up_proj", + "model.layers.53.mlp.experts.163.down_proj", + "model.layers.53.mlp.experts.163.gate_proj", + "model.layers.53.mlp.experts.163.up_proj", + "model.layers.53.mlp.experts.164.down_proj", + "model.layers.53.mlp.experts.164.gate_proj", + "model.layers.53.mlp.experts.164.up_proj", + "model.layers.53.mlp.experts.165.down_proj", + "model.layers.53.mlp.experts.165.gate_proj", + "model.layers.53.mlp.experts.165.up_proj", + "model.layers.53.mlp.experts.166.down_proj", + "model.layers.53.mlp.experts.166.gate_proj", + "model.layers.53.mlp.experts.166.up_proj", + "model.layers.53.mlp.experts.167.down_proj", + "model.layers.53.mlp.experts.167.gate_proj", + "model.layers.53.mlp.experts.167.up_proj", + "model.layers.53.mlp.experts.168.down_proj", + "model.layers.53.mlp.experts.168.gate_proj", + "model.layers.53.mlp.experts.168.up_proj", + "model.layers.53.mlp.experts.169.down_proj", + "model.layers.53.mlp.experts.169.gate_proj", + "model.layers.53.mlp.experts.169.up_proj", + "model.layers.53.mlp.experts.17.down_proj", + "model.layers.53.mlp.experts.17.gate_proj", + "model.layers.53.mlp.experts.17.up_proj", + "model.layers.53.mlp.experts.170.down_proj", + "model.layers.53.mlp.experts.170.gate_proj", + "model.layers.53.mlp.experts.170.up_proj", + "model.layers.53.mlp.experts.171.down_proj", + "model.layers.53.mlp.experts.171.gate_proj", + "model.layers.53.mlp.experts.171.up_proj", + "model.layers.53.mlp.experts.172.down_proj", + "model.layers.53.mlp.experts.172.gate_proj", + "model.layers.53.mlp.experts.172.up_proj", + "model.layers.53.mlp.experts.173.down_proj", + "model.layers.53.mlp.experts.173.gate_proj", + "model.layers.53.mlp.experts.173.up_proj", + "model.layers.53.mlp.experts.174.down_proj", + "model.layers.53.mlp.experts.174.gate_proj", + "model.layers.53.mlp.experts.174.up_proj", + "model.layers.53.mlp.experts.175.down_proj", + "model.layers.53.mlp.experts.175.gate_proj", + "model.layers.53.mlp.experts.175.up_proj", + "model.layers.53.mlp.experts.176.down_proj", + "model.layers.53.mlp.experts.176.gate_proj", + "model.layers.53.mlp.experts.176.up_proj", + "model.layers.53.mlp.experts.177.down_proj", + "model.layers.53.mlp.experts.177.gate_proj", + "model.layers.53.mlp.experts.177.up_proj", + "model.layers.53.mlp.experts.178.down_proj", + "model.layers.53.mlp.experts.178.gate_proj", + "model.layers.53.mlp.experts.178.up_proj", + "model.layers.53.mlp.experts.179.down_proj", + "model.layers.53.mlp.experts.179.gate_proj", + "model.layers.53.mlp.experts.179.up_proj", + "model.layers.53.mlp.experts.18.down_proj", + "model.layers.53.mlp.experts.18.gate_proj", + "model.layers.53.mlp.experts.18.up_proj", + "model.layers.53.mlp.experts.180.down_proj", + "model.layers.53.mlp.experts.180.gate_proj", + "model.layers.53.mlp.experts.180.up_proj", + "model.layers.53.mlp.experts.181.down_proj", + "model.layers.53.mlp.experts.181.gate_proj", + "model.layers.53.mlp.experts.181.up_proj", + "model.layers.53.mlp.experts.182.down_proj", + "model.layers.53.mlp.experts.182.gate_proj", + "model.layers.53.mlp.experts.182.up_proj", + "model.layers.53.mlp.experts.183.down_proj", + "model.layers.53.mlp.experts.183.gate_proj", + "model.layers.53.mlp.experts.183.up_proj", + "model.layers.53.mlp.experts.184.down_proj", + "model.layers.53.mlp.experts.184.gate_proj", + "model.layers.53.mlp.experts.184.up_proj", + "model.layers.53.mlp.experts.185.down_proj", + "model.layers.53.mlp.experts.185.gate_proj", + "model.layers.53.mlp.experts.185.up_proj", + "model.layers.53.mlp.experts.186.down_proj", + "model.layers.53.mlp.experts.186.gate_proj", + "model.layers.53.mlp.experts.186.up_proj", + "model.layers.53.mlp.experts.187.down_proj", + "model.layers.53.mlp.experts.187.gate_proj", + "model.layers.53.mlp.experts.187.up_proj", + "model.layers.53.mlp.experts.188.down_proj", + "model.layers.53.mlp.experts.188.gate_proj", + "model.layers.53.mlp.experts.188.up_proj", + "model.layers.53.mlp.experts.189.down_proj", + "model.layers.53.mlp.experts.189.gate_proj", + "model.layers.53.mlp.experts.189.up_proj", + "model.layers.53.mlp.experts.19.down_proj", + "model.layers.53.mlp.experts.19.gate_proj", + "model.layers.53.mlp.experts.19.up_proj", + "model.layers.53.mlp.experts.190.down_proj", + "model.layers.53.mlp.experts.190.gate_proj", + "model.layers.53.mlp.experts.190.up_proj", + "model.layers.53.mlp.experts.191.down_proj", + "model.layers.53.mlp.experts.191.gate_proj", + "model.layers.53.mlp.experts.191.up_proj", + "model.layers.53.mlp.experts.192.down_proj", + "model.layers.53.mlp.experts.192.gate_proj", + "model.layers.53.mlp.experts.192.up_proj", + "model.layers.53.mlp.experts.193.down_proj", + "model.layers.53.mlp.experts.193.gate_proj", + "model.layers.53.mlp.experts.193.up_proj", + "model.layers.53.mlp.experts.194.down_proj", + "model.layers.53.mlp.experts.194.gate_proj", + "model.layers.53.mlp.experts.194.up_proj", + "model.layers.53.mlp.experts.195.down_proj", + "model.layers.53.mlp.experts.195.gate_proj", + "model.layers.53.mlp.experts.195.up_proj", + "model.layers.53.mlp.experts.196.down_proj", + "model.layers.53.mlp.experts.196.gate_proj", + "model.layers.53.mlp.experts.196.up_proj", + "model.layers.53.mlp.experts.197.down_proj", + "model.layers.53.mlp.experts.197.gate_proj", + "model.layers.53.mlp.experts.197.up_proj", + "model.layers.53.mlp.experts.198.down_proj", + "model.layers.53.mlp.experts.198.gate_proj", + "model.layers.53.mlp.experts.198.up_proj", + "model.layers.53.mlp.experts.199.down_proj", + "model.layers.53.mlp.experts.199.gate_proj", + "model.layers.53.mlp.experts.199.up_proj", + "model.layers.53.mlp.experts.2.down_proj", + "model.layers.53.mlp.experts.2.gate_proj", + "model.layers.53.mlp.experts.2.up_proj", + "model.layers.53.mlp.experts.20.down_proj", + "model.layers.53.mlp.experts.20.gate_proj", + "model.layers.53.mlp.experts.20.up_proj", + "model.layers.53.mlp.experts.200.down_proj", + "model.layers.53.mlp.experts.200.gate_proj", + "model.layers.53.mlp.experts.200.up_proj", + "model.layers.53.mlp.experts.201.down_proj", + "model.layers.53.mlp.experts.201.gate_proj", + "model.layers.53.mlp.experts.201.up_proj", + "model.layers.53.mlp.experts.202.down_proj", + "model.layers.53.mlp.experts.202.gate_proj", + "model.layers.53.mlp.experts.202.up_proj", + "model.layers.53.mlp.experts.203.down_proj", + "model.layers.53.mlp.experts.203.gate_proj", + "model.layers.53.mlp.experts.203.up_proj", + "model.layers.53.mlp.experts.204.down_proj", + "model.layers.53.mlp.experts.204.gate_proj", + "model.layers.53.mlp.experts.204.up_proj", + "model.layers.53.mlp.experts.205.down_proj", + "model.layers.53.mlp.experts.205.gate_proj", + "model.layers.53.mlp.experts.205.up_proj", + "model.layers.53.mlp.experts.206.down_proj", + "model.layers.53.mlp.experts.206.gate_proj", + "model.layers.53.mlp.experts.206.up_proj", + "model.layers.53.mlp.experts.207.down_proj", + "model.layers.53.mlp.experts.207.gate_proj", + "model.layers.53.mlp.experts.207.up_proj", + "model.layers.53.mlp.experts.208.down_proj", + "model.layers.53.mlp.experts.208.gate_proj", + "model.layers.53.mlp.experts.208.up_proj", + "model.layers.53.mlp.experts.209.down_proj", + "model.layers.53.mlp.experts.209.gate_proj", + "model.layers.53.mlp.experts.209.up_proj", + "model.layers.53.mlp.experts.21.down_proj", + "model.layers.53.mlp.experts.21.gate_proj", + "model.layers.53.mlp.experts.21.up_proj", + "model.layers.53.mlp.experts.210.down_proj", + "model.layers.53.mlp.experts.210.gate_proj", + "model.layers.53.mlp.experts.210.up_proj", + "model.layers.53.mlp.experts.211.down_proj", + "model.layers.53.mlp.experts.211.gate_proj", + "model.layers.53.mlp.experts.211.up_proj", + "model.layers.53.mlp.experts.212.down_proj", + "model.layers.53.mlp.experts.212.gate_proj", + "model.layers.53.mlp.experts.212.up_proj", + "model.layers.53.mlp.experts.213.down_proj", + "model.layers.53.mlp.experts.213.gate_proj", + "model.layers.53.mlp.experts.213.up_proj", + "model.layers.53.mlp.experts.214.down_proj", + "model.layers.53.mlp.experts.214.gate_proj", + "model.layers.53.mlp.experts.214.up_proj", + "model.layers.53.mlp.experts.215.down_proj", + "model.layers.53.mlp.experts.215.gate_proj", + "model.layers.53.mlp.experts.215.up_proj", + "model.layers.53.mlp.experts.216.down_proj", + "model.layers.53.mlp.experts.216.gate_proj", + "model.layers.53.mlp.experts.216.up_proj", + "model.layers.53.mlp.experts.217.down_proj", + "model.layers.53.mlp.experts.217.gate_proj", + "model.layers.53.mlp.experts.217.up_proj", + "model.layers.53.mlp.experts.218.down_proj", + "model.layers.53.mlp.experts.218.gate_proj", + "model.layers.53.mlp.experts.218.up_proj", + "model.layers.53.mlp.experts.219.down_proj", + "model.layers.53.mlp.experts.219.gate_proj", + "model.layers.53.mlp.experts.219.up_proj", + "model.layers.53.mlp.experts.22.down_proj", + "model.layers.53.mlp.experts.22.gate_proj", + "model.layers.53.mlp.experts.22.up_proj", + "model.layers.53.mlp.experts.220.down_proj", + "model.layers.53.mlp.experts.220.gate_proj", + "model.layers.53.mlp.experts.220.up_proj", + "model.layers.53.mlp.experts.221.down_proj", + "model.layers.53.mlp.experts.221.gate_proj", + "model.layers.53.mlp.experts.221.up_proj", + "model.layers.53.mlp.experts.222.down_proj", + "model.layers.53.mlp.experts.222.gate_proj", + "model.layers.53.mlp.experts.222.up_proj", + "model.layers.53.mlp.experts.223.down_proj", + "model.layers.53.mlp.experts.223.gate_proj", + "model.layers.53.mlp.experts.223.up_proj", + "model.layers.53.mlp.experts.224.down_proj", + "model.layers.53.mlp.experts.224.gate_proj", + "model.layers.53.mlp.experts.224.up_proj", + "model.layers.53.mlp.experts.225.down_proj", + "model.layers.53.mlp.experts.225.gate_proj", + "model.layers.53.mlp.experts.225.up_proj", + "model.layers.53.mlp.experts.226.down_proj", + "model.layers.53.mlp.experts.226.gate_proj", + "model.layers.53.mlp.experts.226.up_proj", + "model.layers.53.mlp.experts.227.down_proj", + "model.layers.53.mlp.experts.227.gate_proj", + "model.layers.53.mlp.experts.227.up_proj", + "model.layers.53.mlp.experts.228.down_proj", + "model.layers.53.mlp.experts.228.gate_proj", + "model.layers.53.mlp.experts.228.up_proj", + "model.layers.53.mlp.experts.229.down_proj", + "model.layers.53.mlp.experts.229.gate_proj", + "model.layers.53.mlp.experts.229.up_proj", + "model.layers.53.mlp.experts.23.down_proj", + "model.layers.53.mlp.experts.23.gate_proj", + "model.layers.53.mlp.experts.23.up_proj", + "model.layers.53.mlp.experts.230.down_proj", + "model.layers.53.mlp.experts.230.gate_proj", + "model.layers.53.mlp.experts.230.up_proj", + "model.layers.53.mlp.experts.231.down_proj", + "model.layers.53.mlp.experts.231.gate_proj", + "model.layers.53.mlp.experts.231.up_proj", + "model.layers.53.mlp.experts.232.down_proj", + "model.layers.53.mlp.experts.232.gate_proj", + "model.layers.53.mlp.experts.232.up_proj", + "model.layers.53.mlp.experts.233.down_proj", + "model.layers.53.mlp.experts.233.gate_proj", + "model.layers.53.mlp.experts.233.up_proj", + "model.layers.53.mlp.experts.234.down_proj", + "model.layers.53.mlp.experts.234.gate_proj", + "model.layers.53.mlp.experts.234.up_proj", + "model.layers.53.mlp.experts.235.down_proj", + "model.layers.53.mlp.experts.235.gate_proj", + "model.layers.53.mlp.experts.235.up_proj", + "model.layers.53.mlp.experts.236.down_proj", + "model.layers.53.mlp.experts.236.gate_proj", + "model.layers.53.mlp.experts.236.up_proj", + "model.layers.53.mlp.experts.237.down_proj", + "model.layers.53.mlp.experts.237.gate_proj", + "model.layers.53.mlp.experts.237.up_proj", + "model.layers.53.mlp.experts.238.down_proj", + "model.layers.53.mlp.experts.238.gate_proj", + "model.layers.53.mlp.experts.238.up_proj", + "model.layers.53.mlp.experts.239.down_proj", + "model.layers.53.mlp.experts.239.gate_proj", + "model.layers.53.mlp.experts.239.up_proj", + "model.layers.53.mlp.experts.24.down_proj", + "model.layers.53.mlp.experts.24.gate_proj", + "model.layers.53.mlp.experts.24.up_proj", + "model.layers.53.mlp.experts.240.down_proj", + "model.layers.53.mlp.experts.240.gate_proj", + "model.layers.53.mlp.experts.240.up_proj", + "model.layers.53.mlp.experts.241.down_proj", + "model.layers.53.mlp.experts.241.gate_proj", + "model.layers.53.mlp.experts.241.up_proj", + "model.layers.53.mlp.experts.242.down_proj", + "model.layers.53.mlp.experts.242.gate_proj", + "model.layers.53.mlp.experts.242.up_proj", + "model.layers.53.mlp.experts.243.down_proj", + "model.layers.53.mlp.experts.243.gate_proj", + "model.layers.53.mlp.experts.243.up_proj", + "model.layers.53.mlp.experts.244.down_proj", + "model.layers.53.mlp.experts.244.gate_proj", + "model.layers.53.mlp.experts.244.up_proj", + "model.layers.53.mlp.experts.245.down_proj", + "model.layers.53.mlp.experts.245.gate_proj", + "model.layers.53.mlp.experts.245.up_proj", + "model.layers.53.mlp.experts.246.down_proj", + "model.layers.53.mlp.experts.246.gate_proj", + "model.layers.53.mlp.experts.246.up_proj", + "model.layers.53.mlp.experts.247.down_proj", + "model.layers.53.mlp.experts.247.gate_proj", + "model.layers.53.mlp.experts.247.up_proj", + "model.layers.53.mlp.experts.248.down_proj", + "model.layers.53.mlp.experts.248.gate_proj", + "model.layers.53.mlp.experts.248.up_proj", + "model.layers.53.mlp.experts.249.down_proj", + "model.layers.53.mlp.experts.249.gate_proj", + "model.layers.53.mlp.experts.249.up_proj", + "model.layers.53.mlp.experts.25.down_proj", + "model.layers.53.mlp.experts.25.gate_proj", + "model.layers.53.mlp.experts.25.up_proj", + "model.layers.53.mlp.experts.250.down_proj", + "model.layers.53.mlp.experts.250.gate_proj", + "model.layers.53.mlp.experts.250.up_proj", + "model.layers.53.mlp.experts.251.down_proj", + "model.layers.53.mlp.experts.251.gate_proj", + "model.layers.53.mlp.experts.251.up_proj", + "model.layers.53.mlp.experts.252.down_proj", + "model.layers.53.mlp.experts.252.gate_proj", + "model.layers.53.mlp.experts.252.up_proj", + "model.layers.53.mlp.experts.253.down_proj", + "model.layers.53.mlp.experts.253.gate_proj", + "model.layers.53.mlp.experts.253.up_proj", + "model.layers.53.mlp.experts.254.down_proj", + "model.layers.53.mlp.experts.254.gate_proj", + "model.layers.53.mlp.experts.254.up_proj", + "model.layers.53.mlp.experts.255.down_proj", + "model.layers.53.mlp.experts.255.gate_proj", + "model.layers.53.mlp.experts.255.up_proj", + "model.layers.53.mlp.experts.26.down_proj", + "model.layers.53.mlp.experts.26.gate_proj", + "model.layers.53.mlp.experts.26.up_proj", + "model.layers.53.mlp.experts.27.down_proj", + "model.layers.53.mlp.experts.27.gate_proj", + "model.layers.53.mlp.experts.27.up_proj", + "model.layers.53.mlp.experts.28.down_proj", + "model.layers.53.mlp.experts.28.gate_proj", + "model.layers.53.mlp.experts.28.up_proj", + "model.layers.53.mlp.experts.29.down_proj", + "model.layers.53.mlp.experts.29.gate_proj", + "model.layers.53.mlp.experts.29.up_proj", + "model.layers.53.mlp.experts.3.down_proj", + "model.layers.53.mlp.experts.3.gate_proj", + "model.layers.53.mlp.experts.3.up_proj", + "model.layers.53.mlp.experts.30.down_proj", + "model.layers.53.mlp.experts.30.gate_proj", + "model.layers.53.mlp.experts.30.up_proj", + "model.layers.53.mlp.experts.31.down_proj", + "model.layers.53.mlp.experts.31.gate_proj", + "model.layers.53.mlp.experts.31.up_proj", + "model.layers.53.mlp.experts.32.down_proj", + "model.layers.53.mlp.experts.32.gate_proj", + "model.layers.53.mlp.experts.32.up_proj", + "model.layers.53.mlp.experts.33.down_proj", + "model.layers.53.mlp.experts.33.gate_proj", + "model.layers.53.mlp.experts.33.up_proj", + "model.layers.53.mlp.experts.34.down_proj", + "model.layers.53.mlp.experts.34.gate_proj", + "model.layers.53.mlp.experts.34.up_proj", + "model.layers.53.mlp.experts.35.down_proj", + "model.layers.53.mlp.experts.35.gate_proj", + "model.layers.53.mlp.experts.35.up_proj", + "model.layers.53.mlp.experts.36.down_proj", + "model.layers.53.mlp.experts.36.gate_proj", + "model.layers.53.mlp.experts.36.up_proj", + "model.layers.53.mlp.experts.37.down_proj", + "model.layers.53.mlp.experts.37.gate_proj", + "model.layers.53.mlp.experts.37.up_proj", + "model.layers.53.mlp.experts.38.down_proj", + "model.layers.53.mlp.experts.38.gate_proj", + "model.layers.53.mlp.experts.38.up_proj", + "model.layers.53.mlp.experts.39.down_proj", + "model.layers.53.mlp.experts.39.gate_proj", + "model.layers.53.mlp.experts.39.up_proj", + "model.layers.53.mlp.experts.4.down_proj", + "model.layers.53.mlp.experts.4.gate_proj", + "model.layers.53.mlp.experts.4.up_proj", + "model.layers.53.mlp.experts.40.down_proj", + "model.layers.53.mlp.experts.40.gate_proj", + "model.layers.53.mlp.experts.40.up_proj", + "model.layers.53.mlp.experts.41.down_proj", + "model.layers.53.mlp.experts.41.gate_proj", + "model.layers.53.mlp.experts.41.up_proj", + "model.layers.53.mlp.experts.42.down_proj", + "model.layers.53.mlp.experts.42.gate_proj", + "model.layers.53.mlp.experts.42.up_proj", + "model.layers.53.mlp.experts.43.down_proj", + "model.layers.53.mlp.experts.43.gate_proj", + "model.layers.53.mlp.experts.43.up_proj", + "model.layers.53.mlp.experts.44.down_proj", + "model.layers.53.mlp.experts.44.gate_proj", + "model.layers.53.mlp.experts.44.up_proj", + "model.layers.53.mlp.experts.45.down_proj", + "model.layers.53.mlp.experts.45.gate_proj", + "model.layers.53.mlp.experts.45.up_proj", + "model.layers.53.mlp.experts.46.down_proj", + "model.layers.53.mlp.experts.46.gate_proj", + "model.layers.53.mlp.experts.46.up_proj", + "model.layers.53.mlp.experts.47.down_proj", + "model.layers.53.mlp.experts.47.gate_proj", + "model.layers.53.mlp.experts.47.up_proj", + "model.layers.53.mlp.experts.48.down_proj", + "model.layers.53.mlp.experts.48.gate_proj", + "model.layers.53.mlp.experts.48.up_proj", + "model.layers.53.mlp.experts.49.down_proj", + "model.layers.53.mlp.experts.49.gate_proj", + "model.layers.53.mlp.experts.49.up_proj", + "model.layers.53.mlp.experts.5.down_proj", + "model.layers.53.mlp.experts.5.gate_proj", + "model.layers.53.mlp.experts.5.up_proj", + "model.layers.53.mlp.experts.50.down_proj", + "model.layers.53.mlp.experts.50.gate_proj", + "model.layers.53.mlp.experts.50.up_proj", + "model.layers.53.mlp.experts.51.down_proj", + "model.layers.53.mlp.experts.51.gate_proj", + "model.layers.53.mlp.experts.51.up_proj", + "model.layers.53.mlp.experts.52.down_proj", + "model.layers.53.mlp.experts.52.gate_proj", + "model.layers.53.mlp.experts.52.up_proj", + "model.layers.53.mlp.experts.53.down_proj", + "model.layers.53.mlp.experts.53.gate_proj", + "model.layers.53.mlp.experts.53.up_proj", + "model.layers.53.mlp.experts.54.down_proj", + "model.layers.53.mlp.experts.54.gate_proj", + "model.layers.53.mlp.experts.54.up_proj", + "model.layers.53.mlp.experts.55.down_proj", + "model.layers.53.mlp.experts.55.gate_proj", + "model.layers.53.mlp.experts.55.up_proj", + "model.layers.53.mlp.experts.56.down_proj", + "model.layers.53.mlp.experts.56.gate_proj", + "model.layers.53.mlp.experts.56.up_proj", + "model.layers.53.mlp.experts.57.down_proj", + "model.layers.53.mlp.experts.57.gate_proj", + "model.layers.53.mlp.experts.57.up_proj", + "model.layers.53.mlp.experts.58.down_proj", + "model.layers.53.mlp.experts.58.gate_proj", + "model.layers.53.mlp.experts.58.up_proj", + "model.layers.53.mlp.experts.59.down_proj", + "model.layers.53.mlp.experts.59.gate_proj", + "model.layers.53.mlp.experts.59.up_proj", + "model.layers.53.mlp.experts.6.down_proj", + "model.layers.53.mlp.experts.6.gate_proj", + "model.layers.53.mlp.experts.6.up_proj", + "model.layers.53.mlp.experts.60.down_proj", + "model.layers.53.mlp.experts.60.gate_proj", + "model.layers.53.mlp.experts.60.up_proj", + "model.layers.53.mlp.experts.61.down_proj", + "model.layers.53.mlp.experts.61.gate_proj", + "model.layers.53.mlp.experts.61.up_proj", + "model.layers.53.mlp.experts.62.down_proj", + "model.layers.53.mlp.experts.62.gate_proj", + "model.layers.53.mlp.experts.62.up_proj", + "model.layers.53.mlp.experts.63.down_proj", + "model.layers.53.mlp.experts.63.gate_proj", + "model.layers.53.mlp.experts.63.up_proj", + "model.layers.53.mlp.experts.64.down_proj", + "model.layers.53.mlp.experts.64.gate_proj", + "model.layers.53.mlp.experts.64.up_proj", + "model.layers.53.mlp.experts.65.down_proj", + "model.layers.53.mlp.experts.65.gate_proj", + "model.layers.53.mlp.experts.65.up_proj", + "model.layers.53.mlp.experts.66.down_proj", + "model.layers.53.mlp.experts.66.gate_proj", + "model.layers.53.mlp.experts.66.up_proj", + "model.layers.53.mlp.experts.67.down_proj", + "model.layers.53.mlp.experts.67.gate_proj", + "model.layers.53.mlp.experts.67.up_proj", + "model.layers.53.mlp.experts.68.down_proj", + "model.layers.53.mlp.experts.68.gate_proj", + "model.layers.53.mlp.experts.68.up_proj", + "model.layers.53.mlp.experts.69.down_proj", + "model.layers.53.mlp.experts.69.gate_proj", + "model.layers.53.mlp.experts.69.up_proj", + "model.layers.53.mlp.experts.7.down_proj", + "model.layers.53.mlp.experts.7.gate_proj", + "model.layers.53.mlp.experts.7.up_proj", + "model.layers.53.mlp.experts.70.down_proj", + "model.layers.53.mlp.experts.70.gate_proj", + "model.layers.53.mlp.experts.70.up_proj", + "model.layers.53.mlp.experts.71.down_proj", + "model.layers.53.mlp.experts.71.gate_proj", + "model.layers.53.mlp.experts.71.up_proj", + "model.layers.53.mlp.experts.72.down_proj", + "model.layers.53.mlp.experts.72.gate_proj", + "model.layers.53.mlp.experts.72.up_proj", + "model.layers.53.mlp.experts.73.down_proj", + "model.layers.53.mlp.experts.73.gate_proj", + "model.layers.53.mlp.experts.73.up_proj", + "model.layers.53.mlp.experts.74.down_proj", + "model.layers.53.mlp.experts.74.gate_proj", + "model.layers.53.mlp.experts.74.up_proj", + "model.layers.53.mlp.experts.75.down_proj", + "model.layers.53.mlp.experts.75.gate_proj", + "model.layers.53.mlp.experts.75.up_proj", + "model.layers.53.mlp.experts.76.down_proj", + "model.layers.53.mlp.experts.76.gate_proj", + "model.layers.53.mlp.experts.76.up_proj", + "model.layers.53.mlp.experts.77.down_proj", + "model.layers.53.mlp.experts.77.gate_proj", + "model.layers.53.mlp.experts.77.up_proj", + "model.layers.53.mlp.experts.78.down_proj", + "model.layers.53.mlp.experts.78.gate_proj", + "model.layers.53.mlp.experts.78.up_proj", + "model.layers.53.mlp.experts.79.down_proj", + "model.layers.53.mlp.experts.79.gate_proj", + "model.layers.53.mlp.experts.79.up_proj", + "model.layers.53.mlp.experts.8.down_proj", + "model.layers.53.mlp.experts.8.gate_proj", + "model.layers.53.mlp.experts.8.up_proj", + "model.layers.53.mlp.experts.80.down_proj", + "model.layers.53.mlp.experts.80.gate_proj", + "model.layers.53.mlp.experts.80.up_proj", + "model.layers.53.mlp.experts.81.down_proj", + "model.layers.53.mlp.experts.81.gate_proj", + "model.layers.53.mlp.experts.81.up_proj", + "model.layers.53.mlp.experts.82.down_proj", + "model.layers.53.mlp.experts.82.gate_proj", + "model.layers.53.mlp.experts.82.up_proj", + "model.layers.53.mlp.experts.83.down_proj", + "model.layers.53.mlp.experts.83.gate_proj", + "model.layers.53.mlp.experts.83.up_proj", + "model.layers.53.mlp.experts.84.down_proj", + "model.layers.53.mlp.experts.84.gate_proj", + "model.layers.53.mlp.experts.84.up_proj", + "model.layers.53.mlp.experts.85.down_proj", + "model.layers.53.mlp.experts.85.gate_proj", + "model.layers.53.mlp.experts.85.up_proj", + "model.layers.53.mlp.experts.86.down_proj", + "model.layers.53.mlp.experts.86.gate_proj", + "model.layers.53.mlp.experts.86.up_proj", + "model.layers.53.mlp.experts.87.down_proj", + "model.layers.53.mlp.experts.87.gate_proj", + "model.layers.53.mlp.experts.87.up_proj", + "model.layers.53.mlp.experts.88.down_proj", + "model.layers.53.mlp.experts.88.gate_proj", + "model.layers.53.mlp.experts.88.up_proj", + "model.layers.53.mlp.experts.89.down_proj", + "model.layers.53.mlp.experts.89.gate_proj", + "model.layers.53.mlp.experts.89.up_proj", + "model.layers.53.mlp.experts.9.down_proj", + "model.layers.53.mlp.experts.9.gate_proj", + "model.layers.53.mlp.experts.9.up_proj", + "model.layers.53.mlp.experts.90.down_proj", + "model.layers.53.mlp.experts.90.gate_proj", + "model.layers.53.mlp.experts.90.up_proj", + "model.layers.53.mlp.experts.91.down_proj", + "model.layers.53.mlp.experts.91.gate_proj", + "model.layers.53.mlp.experts.91.up_proj", + "model.layers.53.mlp.experts.92.down_proj", + "model.layers.53.mlp.experts.92.gate_proj", + "model.layers.53.mlp.experts.92.up_proj", + "model.layers.53.mlp.experts.93.down_proj", + "model.layers.53.mlp.experts.93.gate_proj", + "model.layers.53.mlp.experts.93.up_proj", + "model.layers.53.mlp.experts.94.down_proj", + "model.layers.53.mlp.experts.94.gate_proj", + "model.layers.53.mlp.experts.94.up_proj", + "model.layers.53.mlp.experts.95.down_proj", + "model.layers.53.mlp.experts.95.gate_proj", + "model.layers.53.mlp.experts.95.up_proj", + "model.layers.53.mlp.experts.96.down_proj", + "model.layers.53.mlp.experts.96.gate_proj", + "model.layers.53.mlp.experts.96.up_proj", + "model.layers.53.mlp.experts.97.down_proj", + "model.layers.53.mlp.experts.97.gate_proj", + "model.layers.53.mlp.experts.97.up_proj", + "model.layers.53.mlp.experts.98.down_proj", + "model.layers.53.mlp.experts.98.gate_proj", + "model.layers.53.mlp.experts.98.up_proj", + "model.layers.53.mlp.experts.99.down_proj", + "model.layers.53.mlp.experts.99.gate_proj", + "model.layers.53.mlp.experts.99.up_proj", + "model.layers.54.mlp.experts.0.down_proj", + "model.layers.54.mlp.experts.0.gate_proj", + "model.layers.54.mlp.experts.0.up_proj", + "model.layers.54.mlp.experts.1.down_proj", + "model.layers.54.mlp.experts.1.gate_proj", + "model.layers.54.mlp.experts.1.up_proj", + "model.layers.54.mlp.experts.10.down_proj", + "model.layers.54.mlp.experts.10.gate_proj", + "model.layers.54.mlp.experts.10.up_proj", + "model.layers.54.mlp.experts.100.down_proj", + "model.layers.54.mlp.experts.100.gate_proj", + "model.layers.54.mlp.experts.100.up_proj", + "model.layers.54.mlp.experts.101.down_proj", + "model.layers.54.mlp.experts.101.gate_proj", + "model.layers.54.mlp.experts.101.up_proj", + "model.layers.54.mlp.experts.102.down_proj", + "model.layers.54.mlp.experts.102.gate_proj", + "model.layers.54.mlp.experts.102.up_proj", + "model.layers.54.mlp.experts.103.down_proj", + "model.layers.54.mlp.experts.103.gate_proj", + "model.layers.54.mlp.experts.103.up_proj", + "model.layers.54.mlp.experts.104.down_proj", + "model.layers.54.mlp.experts.104.gate_proj", + "model.layers.54.mlp.experts.104.up_proj", + "model.layers.54.mlp.experts.105.down_proj", + "model.layers.54.mlp.experts.105.gate_proj", + "model.layers.54.mlp.experts.105.up_proj", + "model.layers.54.mlp.experts.106.down_proj", + "model.layers.54.mlp.experts.106.gate_proj", + "model.layers.54.mlp.experts.106.up_proj", + "model.layers.54.mlp.experts.107.down_proj", + "model.layers.54.mlp.experts.107.gate_proj", + "model.layers.54.mlp.experts.107.up_proj", + "model.layers.54.mlp.experts.108.down_proj", + "model.layers.54.mlp.experts.108.gate_proj", + "model.layers.54.mlp.experts.108.up_proj", + "model.layers.54.mlp.experts.109.down_proj", + "model.layers.54.mlp.experts.109.gate_proj", + "model.layers.54.mlp.experts.109.up_proj", + "model.layers.54.mlp.experts.11.down_proj", + "model.layers.54.mlp.experts.11.gate_proj", + "model.layers.54.mlp.experts.11.up_proj", + "model.layers.54.mlp.experts.110.down_proj", + "model.layers.54.mlp.experts.110.gate_proj", + "model.layers.54.mlp.experts.110.up_proj", + "model.layers.54.mlp.experts.111.down_proj", + "model.layers.54.mlp.experts.111.gate_proj", + "model.layers.54.mlp.experts.111.up_proj", + "model.layers.54.mlp.experts.112.down_proj", + "model.layers.54.mlp.experts.112.gate_proj", + "model.layers.54.mlp.experts.112.up_proj", + "model.layers.54.mlp.experts.113.down_proj", + "model.layers.54.mlp.experts.113.gate_proj", + "model.layers.54.mlp.experts.113.up_proj", + "model.layers.54.mlp.experts.114.down_proj", + "model.layers.54.mlp.experts.114.gate_proj", + "model.layers.54.mlp.experts.114.up_proj", + "model.layers.54.mlp.experts.115.down_proj", + "model.layers.54.mlp.experts.115.gate_proj", + "model.layers.54.mlp.experts.115.up_proj", + "model.layers.54.mlp.experts.116.down_proj", + "model.layers.54.mlp.experts.116.gate_proj", + "model.layers.54.mlp.experts.116.up_proj", + "model.layers.54.mlp.experts.117.down_proj", + "model.layers.54.mlp.experts.117.gate_proj", + "model.layers.54.mlp.experts.117.up_proj", + "model.layers.54.mlp.experts.118.down_proj", + "model.layers.54.mlp.experts.118.gate_proj", + "model.layers.54.mlp.experts.118.up_proj", + "model.layers.54.mlp.experts.119.down_proj", + "model.layers.54.mlp.experts.119.gate_proj", + "model.layers.54.mlp.experts.119.up_proj", + "model.layers.54.mlp.experts.12.down_proj", + "model.layers.54.mlp.experts.12.gate_proj", + "model.layers.54.mlp.experts.12.up_proj", + "model.layers.54.mlp.experts.120.down_proj", + "model.layers.54.mlp.experts.120.gate_proj", + "model.layers.54.mlp.experts.120.up_proj", + "model.layers.54.mlp.experts.121.down_proj", + "model.layers.54.mlp.experts.121.gate_proj", + "model.layers.54.mlp.experts.121.up_proj", + "model.layers.54.mlp.experts.122.down_proj", + "model.layers.54.mlp.experts.122.gate_proj", + "model.layers.54.mlp.experts.122.up_proj", + "model.layers.54.mlp.experts.123.down_proj", + "model.layers.54.mlp.experts.123.gate_proj", + "model.layers.54.mlp.experts.123.up_proj", + "model.layers.54.mlp.experts.124.down_proj", + "model.layers.54.mlp.experts.124.gate_proj", + "model.layers.54.mlp.experts.124.up_proj", + "model.layers.54.mlp.experts.125.down_proj", + "model.layers.54.mlp.experts.125.gate_proj", + "model.layers.54.mlp.experts.125.up_proj", + "model.layers.54.mlp.experts.126.down_proj", + "model.layers.54.mlp.experts.126.gate_proj", + "model.layers.54.mlp.experts.126.up_proj", + "model.layers.54.mlp.experts.127.down_proj", + "model.layers.54.mlp.experts.127.gate_proj", + "model.layers.54.mlp.experts.127.up_proj", + "model.layers.54.mlp.experts.128.down_proj", + "model.layers.54.mlp.experts.128.gate_proj", + "model.layers.54.mlp.experts.128.up_proj", + "model.layers.54.mlp.experts.129.down_proj", + "model.layers.54.mlp.experts.129.gate_proj", + "model.layers.54.mlp.experts.129.up_proj", + "model.layers.54.mlp.experts.13.down_proj", + "model.layers.54.mlp.experts.13.gate_proj", + "model.layers.54.mlp.experts.13.up_proj", + "model.layers.54.mlp.experts.130.down_proj", + "model.layers.54.mlp.experts.130.gate_proj", + "model.layers.54.mlp.experts.130.up_proj", + "model.layers.54.mlp.experts.131.down_proj", + "model.layers.54.mlp.experts.131.gate_proj", + "model.layers.54.mlp.experts.131.up_proj", + "model.layers.54.mlp.experts.132.down_proj", + "model.layers.54.mlp.experts.132.gate_proj", + "model.layers.54.mlp.experts.132.up_proj", + "model.layers.54.mlp.experts.133.down_proj", + "model.layers.54.mlp.experts.133.gate_proj", + "model.layers.54.mlp.experts.133.up_proj", + "model.layers.54.mlp.experts.134.down_proj", + "model.layers.54.mlp.experts.134.gate_proj", + "model.layers.54.mlp.experts.134.up_proj", + "model.layers.54.mlp.experts.135.down_proj", + "model.layers.54.mlp.experts.135.gate_proj", + "model.layers.54.mlp.experts.135.up_proj", + "model.layers.54.mlp.experts.136.down_proj", + "model.layers.54.mlp.experts.136.gate_proj", + "model.layers.54.mlp.experts.136.up_proj", + "model.layers.54.mlp.experts.137.down_proj", + "model.layers.54.mlp.experts.137.gate_proj", + "model.layers.54.mlp.experts.137.up_proj", + "model.layers.54.mlp.experts.138.down_proj", + "model.layers.54.mlp.experts.138.gate_proj", + "model.layers.54.mlp.experts.138.up_proj", + "model.layers.54.mlp.experts.139.down_proj", + "model.layers.54.mlp.experts.139.gate_proj", + "model.layers.54.mlp.experts.139.up_proj", + "model.layers.54.mlp.experts.14.down_proj", + "model.layers.54.mlp.experts.14.gate_proj", + "model.layers.54.mlp.experts.14.up_proj", + "model.layers.54.mlp.experts.140.down_proj", + "model.layers.54.mlp.experts.140.gate_proj", + "model.layers.54.mlp.experts.140.up_proj", + "model.layers.54.mlp.experts.141.down_proj", + "model.layers.54.mlp.experts.141.gate_proj", + "model.layers.54.mlp.experts.141.up_proj", + "model.layers.54.mlp.experts.142.down_proj", + "model.layers.54.mlp.experts.142.gate_proj", + "model.layers.54.mlp.experts.142.up_proj", + "model.layers.54.mlp.experts.143.down_proj", + "model.layers.54.mlp.experts.143.gate_proj", + "model.layers.54.mlp.experts.143.up_proj", + "model.layers.54.mlp.experts.144.down_proj", + "model.layers.54.mlp.experts.144.gate_proj", + "model.layers.54.mlp.experts.144.up_proj", + "model.layers.54.mlp.experts.145.down_proj", + "model.layers.54.mlp.experts.145.gate_proj", + "model.layers.54.mlp.experts.145.up_proj", + "model.layers.54.mlp.experts.146.down_proj", + "model.layers.54.mlp.experts.146.gate_proj", + "model.layers.54.mlp.experts.146.up_proj", + "model.layers.54.mlp.experts.147.down_proj", + "model.layers.54.mlp.experts.147.gate_proj", + "model.layers.54.mlp.experts.147.up_proj", + "model.layers.54.mlp.experts.148.down_proj", + "model.layers.54.mlp.experts.148.gate_proj", + "model.layers.54.mlp.experts.148.up_proj", + "model.layers.54.mlp.experts.149.down_proj", + "model.layers.54.mlp.experts.149.gate_proj", + "model.layers.54.mlp.experts.149.up_proj", + "model.layers.54.mlp.experts.15.down_proj", + "model.layers.54.mlp.experts.15.gate_proj", + "model.layers.54.mlp.experts.15.up_proj", + "model.layers.54.mlp.experts.150.down_proj", + "model.layers.54.mlp.experts.150.gate_proj", + "model.layers.54.mlp.experts.150.up_proj", + "model.layers.54.mlp.experts.151.down_proj", + "model.layers.54.mlp.experts.151.gate_proj", + "model.layers.54.mlp.experts.151.up_proj", + "model.layers.54.mlp.experts.152.down_proj", + "model.layers.54.mlp.experts.152.gate_proj", + "model.layers.54.mlp.experts.152.up_proj", + "model.layers.54.mlp.experts.153.down_proj", + "model.layers.54.mlp.experts.153.gate_proj", + "model.layers.54.mlp.experts.153.up_proj", + "model.layers.54.mlp.experts.154.down_proj", + "model.layers.54.mlp.experts.154.gate_proj", + "model.layers.54.mlp.experts.154.up_proj", + "model.layers.54.mlp.experts.155.down_proj", + "model.layers.54.mlp.experts.155.gate_proj", + "model.layers.54.mlp.experts.155.up_proj", + "model.layers.54.mlp.experts.156.down_proj", + "model.layers.54.mlp.experts.156.gate_proj", + "model.layers.54.mlp.experts.156.up_proj", + "model.layers.54.mlp.experts.157.down_proj", + "model.layers.54.mlp.experts.157.gate_proj", + "model.layers.54.mlp.experts.157.up_proj", + "model.layers.54.mlp.experts.158.down_proj", + "model.layers.54.mlp.experts.158.gate_proj", + "model.layers.54.mlp.experts.158.up_proj", + "model.layers.54.mlp.experts.159.down_proj", + "model.layers.54.mlp.experts.159.gate_proj", + "model.layers.54.mlp.experts.159.up_proj", + "model.layers.54.mlp.experts.16.down_proj", + "model.layers.54.mlp.experts.16.gate_proj", + "model.layers.54.mlp.experts.16.up_proj", + "model.layers.54.mlp.experts.160.down_proj", + "model.layers.54.mlp.experts.160.gate_proj", + "model.layers.54.mlp.experts.160.up_proj", + "model.layers.54.mlp.experts.161.down_proj", + "model.layers.54.mlp.experts.161.gate_proj", + "model.layers.54.mlp.experts.161.up_proj", + "model.layers.54.mlp.experts.162.down_proj", + "model.layers.54.mlp.experts.162.gate_proj", + "model.layers.54.mlp.experts.162.up_proj", + "model.layers.54.mlp.experts.163.down_proj", + "model.layers.54.mlp.experts.163.gate_proj", + "model.layers.54.mlp.experts.163.up_proj", + "model.layers.54.mlp.experts.164.down_proj", + "model.layers.54.mlp.experts.164.gate_proj", + "model.layers.54.mlp.experts.164.up_proj", + "model.layers.54.mlp.experts.165.down_proj", + "model.layers.54.mlp.experts.165.gate_proj", + "model.layers.54.mlp.experts.165.up_proj", + "model.layers.54.mlp.experts.166.down_proj", + "model.layers.54.mlp.experts.166.gate_proj", + "model.layers.54.mlp.experts.166.up_proj", + "model.layers.54.mlp.experts.167.down_proj", + "model.layers.54.mlp.experts.167.gate_proj", + "model.layers.54.mlp.experts.167.up_proj", + "model.layers.54.mlp.experts.168.down_proj", + "model.layers.54.mlp.experts.168.gate_proj", + "model.layers.54.mlp.experts.168.up_proj", + "model.layers.54.mlp.experts.169.down_proj", + "model.layers.54.mlp.experts.169.gate_proj", + "model.layers.54.mlp.experts.169.up_proj", + "model.layers.54.mlp.experts.17.down_proj", + "model.layers.54.mlp.experts.17.gate_proj", + "model.layers.54.mlp.experts.17.up_proj", + "model.layers.54.mlp.experts.170.down_proj", + "model.layers.54.mlp.experts.170.gate_proj", + "model.layers.54.mlp.experts.170.up_proj", + "model.layers.54.mlp.experts.171.down_proj", + "model.layers.54.mlp.experts.171.gate_proj", + "model.layers.54.mlp.experts.171.up_proj", + "model.layers.54.mlp.experts.172.down_proj", + "model.layers.54.mlp.experts.172.gate_proj", + "model.layers.54.mlp.experts.172.up_proj", + "model.layers.54.mlp.experts.173.down_proj", + "model.layers.54.mlp.experts.173.gate_proj", + "model.layers.54.mlp.experts.173.up_proj", + "model.layers.54.mlp.experts.174.down_proj", + "model.layers.54.mlp.experts.174.gate_proj", + "model.layers.54.mlp.experts.174.up_proj", + "model.layers.54.mlp.experts.175.down_proj", + "model.layers.54.mlp.experts.175.gate_proj", + "model.layers.54.mlp.experts.175.up_proj", + "model.layers.54.mlp.experts.176.down_proj", + "model.layers.54.mlp.experts.176.gate_proj", + "model.layers.54.mlp.experts.176.up_proj", + "model.layers.54.mlp.experts.177.down_proj", + "model.layers.54.mlp.experts.177.gate_proj", + "model.layers.54.mlp.experts.177.up_proj", + "model.layers.54.mlp.experts.178.down_proj", + "model.layers.54.mlp.experts.178.gate_proj", + "model.layers.54.mlp.experts.178.up_proj", + "model.layers.54.mlp.experts.179.down_proj", + "model.layers.54.mlp.experts.179.gate_proj", + "model.layers.54.mlp.experts.179.up_proj", + "model.layers.54.mlp.experts.18.down_proj", + "model.layers.54.mlp.experts.18.gate_proj", + "model.layers.54.mlp.experts.18.up_proj", + "model.layers.54.mlp.experts.180.down_proj", + "model.layers.54.mlp.experts.180.gate_proj", + "model.layers.54.mlp.experts.180.up_proj", + "model.layers.54.mlp.experts.181.down_proj", + "model.layers.54.mlp.experts.181.gate_proj", + "model.layers.54.mlp.experts.181.up_proj", + "model.layers.54.mlp.experts.182.down_proj", + "model.layers.54.mlp.experts.182.gate_proj", + "model.layers.54.mlp.experts.182.up_proj", + "model.layers.54.mlp.experts.183.down_proj", + "model.layers.54.mlp.experts.183.gate_proj", + "model.layers.54.mlp.experts.183.up_proj", + "model.layers.54.mlp.experts.184.down_proj", + "model.layers.54.mlp.experts.184.gate_proj", + "model.layers.54.mlp.experts.184.up_proj", + "model.layers.54.mlp.experts.185.down_proj", + "model.layers.54.mlp.experts.185.gate_proj", + "model.layers.54.mlp.experts.185.up_proj", + "model.layers.54.mlp.experts.186.down_proj", + "model.layers.54.mlp.experts.186.gate_proj", + "model.layers.54.mlp.experts.186.up_proj", + "model.layers.54.mlp.experts.187.down_proj", + "model.layers.54.mlp.experts.187.gate_proj", + "model.layers.54.mlp.experts.187.up_proj", + "model.layers.54.mlp.experts.188.down_proj", + "model.layers.54.mlp.experts.188.gate_proj", + "model.layers.54.mlp.experts.188.up_proj", + "model.layers.54.mlp.experts.189.down_proj", + "model.layers.54.mlp.experts.189.gate_proj", + "model.layers.54.mlp.experts.189.up_proj", + "model.layers.54.mlp.experts.19.down_proj", + "model.layers.54.mlp.experts.19.gate_proj", + "model.layers.54.mlp.experts.19.up_proj", + "model.layers.54.mlp.experts.190.down_proj", + "model.layers.54.mlp.experts.190.gate_proj", + "model.layers.54.mlp.experts.190.up_proj", + "model.layers.54.mlp.experts.191.down_proj", + "model.layers.54.mlp.experts.191.gate_proj", + "model.layers.54.mlp.experts.191.up_proj", + "model.layers.54.mlp.experts.192.down_proj", + "model.layers.54.mlp.experts.192.gate_proj", + "model.layers.54.mlp.experts.192.up_proj", + "model.layers.54.mlp.experts.193.down_proj", + "model.layers.54.mlp.experts.193.gate_proj", + "model.layers.54.mlp.experts.193.up_proj", + "model.layers.54.mlp.experts.194.down_proj", + "model.layers.54.mlp.experts.194.gate_proj", + "model.layers.54.mlp.experts.194.up_proj", + "model.layers.54.mlp.experts.195.down_proj", + "model.layers.54.mlp.experts.195.gate_proj", + "model.layers.54.mlp.experts.195.up_proj", + "model.layers.54.mlp.experts.196.down_proj", + "model.layers.54.mlp.experts.196.gate_proj", + "model.layers.54.mlp.experts.196.up_proj", + "model.layers.54.mlp.experts.197.down_proj", + "model.layers.54.mlp.experts.197.gate_proj", + "model.layers.54.mlp.experts.197.up_proj", + "model.layers.54.mlp.experts.198.down_proj", + "model.layers.54.mlp.experts.198.gate_proj", + "model.layers.54.mlp.experts.198.up_proj", + "model.layers.54.mlp.experts.199.down_proj", + "model.layers.54.mlp.experts.199.gate_proj", + "model.layers.54.mlp.experts.199.up_proj", + "model.layers.54.mlp.experts.2.down_proj", + "model.layers.54.mlp.experts.2.gate_proj", + "model.layers.54.mlp.experts.2.up_proj", + "model.layers.54.mlp.experts.20.down_proj", + "model.layers.54.mlp.experts.20.gate_proj", + "model.layers.54.mlp.experts.20.up_proj", + "model.layers.54.mlp.experts.200.down_proj", + "model.layers.54.mlp.experts.200.gate_proj", + "model.layers.54.mlp.experts.200.up_proj", + "model.layers.54.mlp.experts.201.down_proj", + "model.layers.54.mlp.experts.201.gate_proj", + "model.layers.54.mlp.experts.201.up_proj", + "model.layers.54.mlp.experts.202.down_proj", + "model.layers.54.mlp.experts.202.gate_proj", + "model.layers.54.mlp.experts.202.up_proj", + "model.layers.54.mlp.experts.203.down_proj", + "model.layers.54.mlp.experts.203.gate_proj", + "model.layers.54.mlp.experts.203.up_proj", + "model.layers.54.mlp.experts.204.down_proj", + "model.layers.54.mlp.experts.204.gate_proj", + "model.layers.54.mlp.experts.204.up_proj", + "model.layers.54.mlp.experts.205.down_proj", + "model.layers.54.mlp.experts.205.gate_proj", + "model.layers.54.mlp.experts.205.up_proj", + "model.layers.54.mlp.experts.206.down_proj", + "model.layers.54.mlp.experts.206.gate_proj", + "model.layers.54.mlp.experts.206.up_proj", + "model.layers.54.mlp.experts.207.down_proj", + "model.layers.54.mlp.experts.207.gate_proj", + "model.layers.54.mlp.experts.207.up_proj", + "model.layers.54.mlp.experts.208.down_proj", + "model.layers.54.mlp.experts.208.gate_proj", + "model.layers.54.mlp.experts.208.up_proj", + "model.layers.54.mlp.experts.209.down_proj", + "model.layers.54.mlp.experts.209.gate_proj", + "model.layers.54.mlp.experts.209.up_proj", + "model.layers.54.mlp.experts.21.down_proj", + "model.layers.54.mlp.experts.21.gate_proj", + "model.layers.54.mlp.experts.21.up_proj", + "model.layers.54.mlp.experts.210.down_proj", + "model.layers.54.mlp.experts.210.gate_proj", + "model.layers.54.mlp.experts.210.up_proj", + "model.layers.54.mlp.experts.211.down_proj", + "model.layers.54.mlp.experts.211.gate_proj", + "model.layers.54.mlp.experts.211.up_proj", + "model.layers.54.mlp.experts.212.down_proj", + "model.layers.54.mlp.experts.212.gate_proj", + "model.layers.54.mlp.experts.212.up_proj", + "model.layers.54.mlp.experts.213.down_proj", + "model.layers.54.mlp.experts.213.gate_proj", + "model.layers.54.mlp.experts.213.up_proj", + "model.layers.54.mlp.experts.214.down_proj", + "model.layers.54.mlp.experts.214.gate_proj", + "model.layers.54.mlp.experts.214.up_proj", + "model.layers.54.mlp.experts.215.down_proj", + "model.layers.54.mlp.experts.215.gate_proj", + "model.layers.54.mlp.experts.215.up_proj", + "model.layers.54.mlp.experts.216.down_proj", + "model.layers.54.mlp.experts.216.gate_proj", + "model.layers.54.mlp.experts.216.up_proj", + "model.layers.54.mlp.experts.217.down_proj", + "model.layers.54.mlp.experts.217.gate_proj", + "model.layers.54.mlp.experts.217.up_proj", + "model.layers.54.mlp.experts.218.down_proj", + "model.layers.54.mlp.experts.218.gate_proj", + "model.layers.54.mlp.experts.218.up_proj", + "model.layers.54.mlp.experts.219.down_proj", + "model.layers.54.mlp.experts.219.gate_proj", + "model.layers.54.mlp.experts.219.up_proj", + "model.layers.54.mlp.experts.22.down_proj", + "model.layers.54.mlp.experts.22.gate_proj", + "model.layers.54.mlp.experts.22.up_proj", + "model.layers.54.mlp.experts.220.down_proj", + "model.layers.54.mlp.experts.220.gate_proj", + "model.layers.54.mlp.experts.220.up_proj", + "model.layers.54.mlp.experts.221.down_proj", + "model.layers.54.mlp.experts.221.gate_proj", + "model.layers.54.mlp.experts.221.up_proj", + "model.layers.54.mlp.experts.222.down_proj", + "model.layers.54.mlp.experts.222.gate_proj", + "model.layers.54.mlp.experts.222.up_proj", + "model.layers.54.mlp.experts.223.down_proj", + "model.layers.54.mlp.experts.223.gate_proj", + "model.layers.54.mlp.experts.223.up_proj", + "model.layers.54.mlp.experts.224.down_proj", + "model.layers.54.mlp.experts.224.gate_proj", + "model.layers.54.mlp.experts.224.up_proj", + "model.layers.54.mlp.experts.225.down_proj", + "model.layers.54.mlp.experts.225.gate_proj", + "model.layers.54.mlp.experts.225.up_proj", + "model.layers.54.mlp.experts.226.down_proj", + "model.layers.54.mlp.experts.226.gate_proj", + "model.layers.54.mlp.experts.226.up_proj", + "model.layers.54.mlp.experts.227.down_proj", + "model.layers.54.mlp.experts.227.gate_proj", + "model.layers.54.mlp.experts.227.up_proj", + "model.layers.54.mlp.experts.228.down_proj", + "model.layers.54.mlp.experts.228.gate_proj", + "model.layers.54.mlp.experts.228.up_proj", + "model.layers.54.mlp.experts.229.down_proj", + "model.layers.54.mlp.experts.229.gate_proj", + "model.layers.54.mlp.experts.229.up_proj", + "model.layers.54.mlp.experts.23.down_proj", + "model.layers.54.mlp.experts.23.gate_proj", + "model.layers.54.mlp.experts.23.up_proj", + "model.layers.54.mlp.experts.230.down_proj", + "model.layers.54.mlp.experts.230.gate_proj", + "model.layers.54.mlp.experts.230.up_proj", + "model.layers.54.mlp.experts.231.down_proj", + "model.layers.54.mlp.experts.231.gate_proj", + "model.layers.54.mlp.experts.231.up_proj", + "model.layers.54.mlp.experts.232.down_proj", + "model.layers.54.mlp.experts.232.gate_proj", + "model.layers.54.mlp.experts.232.up_proj", + "model.layers.54.mlp.experts.233.down_proj", + "model.layers.54.mlp.experts.233.gate_proj", + "model.layers.54.mlp.experts.233.up_proj", + "model.layers.54.mlp.experts.234.down_proj", + "model.layers.54.mlp.experts.234.gate_proj", + "model.layers.54.mlp.experts.234.up_proj", + "model.layers.54.mlp.experts.235.down_proj", + "model.layers.54.mlp.experts.235.gate_proj", + "model.layers.54.mlp.experts.235.up_proj", + "model.layers.54.mlp.experts.236.down_proj", + "model.layers.54.mlp.experts.236.gate_proj", + "model.layers.54.mlp.experts.236.up_proj", + "model.layers.54.mlp.experts.237.down_proj", + "model.layers.54.mlp.experts.237.gate_proj", + "model.layers.54.mlp.experts.237.up_proj", + "model.layers.54.mlp.experts.238.down_proj", + "model.layers.54.mlp.experts.238.gate_proj", + "model.layers.54.mlp.experts.238.up_proj", + "model.layers.54.mlp.experts.239.down_proj", + "model.layers.54.mlp.experts.239.gate_proj", + "model.layers.54.mlp.experts.239.up_proj", + "model.layers.54.mlp.experts.24.down_proj", + "model.layers.54.mlp.experts.24.gate_proj", + "model.layers.54.mlp.experts.24.up_proj", + "model.layers.54.mlp.experts.240.down_proj", + "model.layers.54.mlp.experts.240.gate_proj", + "model.layers.54.mlp.experts.240.up_proj", + "model.layers.54.mlp.experts.241.down_proj", + "model.layers.54.mlp.experts.241.gate_proj", + "model.layers.54.mlp.experts.241.up_proj", + "model.layers.54.mlp.experts.242.down_proj", + "model.layers.54.mlp.experts.242.gate_proj", + "model.layers.54.mlp.experts.242.up_proj", + "model.layers.54.mlp.experts.243.down_proj", + "model.layers.54.mlp.experts.243.gate_proj", + "model.layers.54.mlp.experts.243.up_proj", + "model.layers.54.mlp.experts.244.down_proj", + "model.layers.54.mlp.experts.244.gate_proj", + "model.layers.54.mlp.experts.244.up_proj", + "model.layers.54.mlp.experts.245.down_proj", + "model.layers.54.mlp.experts.245.gate_proj", + "model.layers.54.mlp.experts.245.up_proj", + "model.layers.54.mlp.experts.246.down_proj", + "model.layers.54.mlp.experts.246.gate_proj", + "model.layers.54.mlp.experts.246.up_proj", + "model.layers.54.mlp.experts.247.down_proj", + "model.layers.54.mlp.experts.247.gate_proj", + "model.layers.54.mlp.experts.247.up_proj", + "model.layers.54.mlp.experts.248.down_proj", + "model.layers.54.mlp.experts.248.gate_proj", + "model.layers.54.mlp.experts.248.up_proj", + "model.layers.54.mlp.experts.249.down_proj", + "model.layers.54.mlp.experts.249.gate_proj", + "model.layers.54.mlp.experts.249.up_proj", + "model.layers.54.mlp.experts.25.down_proj", + "model.layers.54.mlp.experts.25.gate_proj", + "model.layers.54.mlp.experts.25.up_proj", + "model.layers.54.mlp.experts.250.down_proj", + "model.layers.54.mlp.experts.250.gate_proj", + "model.layers.54.mlp.experts.250.up_proj", + "model.layers.54.mlp.experts.251.down_proj", + "model.layers.54.mlp.experts.251.gate_proj", + "model.layers.54.mlp.experts.251.up_proj", + "model.layers.54.mlp.experts.252.down_proj", + "model.layers.54.mlp.experts.252.gate_proj", + "model.layers.54.mlp.experts.252.up_proj", + "model.layers.54.mlp.experts.253.down_proj", + "model.layers.54.mlp.experts.253.gate_proj", + "model.layers.54.mlp.experts.253.up_proj", + "model.layers.54.mlp.experts.254.down_proj", + "model.layers.54.mlp.experts.254.gate_proj", + "model.layers.54.mlp.experts.254.up_proj", + "model.layers.54.mlp.experts.255.down_proj", + "model.layers.54.mlp.experts.255.gate_proj", + "model.layers.54.mlp.experts.255.up_proj", + "model.layers.54.mlp.experts.26.down_proj", + "model.layers.54.mlp.experts.26.gate_proj", + "model.layers.54.mlp.experts.26.up_proj", + "model.layers.54.mlp.experts.27.down_proj", + "model.layers.54.mlp.experts.27.gate_proj", + "model.layers.54.mlp.experts.27.up_proj", + "model.layers.54.mlp.experts.28.down_proj", + "model.layers.54.mlp.experts.28.gate_proj", + "model.layers.54.mlp.experts.28.up_proj", + "model.layers.54.mlp.experts.29.down_proj", + "model.layers.54.mlp.experts.29.gate_proj", + "model.layers.54.mlp.experts.29.up_proj", + "model.layers.54.mlp.experts.3.down_proj", + "model.layers.54.mlp.experts.3.gate_proj", + "model.layers.54.mlp.experts.3.up_proj", + "model.layers.54.mlp.experts.30.down_proj", + "model.layers.54.mlp.experts.30.gate_proj", + "model.layers.54.mlp.experts.30.up_proj", + "model.layers.54.mlp.experts.31.down_proj", + "model.layers.54.mlp.experts.31.gate_proj", + "model.layers.54.mlp.experts.31.up_proj", + "model.layers.54.mlp.experts.32.down_proj", + "model.layers.54.mlp.experts.32.gate_proj", + "model.layers.54.mlp.experts.32.up_proj", + "model.layers.54.mlp.experts.33.down_proj", + "model.layers.54.mlp.experts.33.gate_proj", + "model.layers.54.mlp.experts.33.up_proj", + "model.layers.54.mlp.experts.34.down_proj", + "model.layers.54.mlp.experts.34.gate_proj", + "model.layers.54.mlp.experts.34.up_proj", + "model.layers.54.mlp.experts.35.down_proj", + "model.layers.54.mlp.experts.35.gate_proj", + "model.layers.54.mlp.experts.35.up_proj", + "model.layers.54.mlp.experts.36.down_proj", + "model.layers.54.mlp.experts.36.gate_proj", + "model.layers.54.mlp.experts.36.up_proj", + "model.layers.54.mlp.experts.37.down_proj", + "model.layers.54.mlp.experts.37.gate_proj", + "model.layers.54.mlp.experts.37.up_proj", + "model.layers.54.mlp.experts.38.down_proj", + "model.layers.54.mlp.experts.38.gate_proj", + "model.layers.54.mlp.experts.38.up_proj", + "model.layers.54.mlp.experts.39.down_proj", + "model.layers.54.mlp.experts.39.gate_proj", + "model.layers.54.mlp.experts.39.up_proj", + "model.layers.54.mlp.experts.4.down_proj", + "model.layers.54.mlp.experts.4.gate_proj", + "model.layers.54.mlp.experts.4.up_proj", + "model.layers.54.mlp.experts.40.down_proj", + "model.layers.54.mlp.experts.40.gate_proj", + "model.layers.54.mlp.experts.40.up_proj", + "model.layers.54.mlp.experts.41.down_proj", + "model.layers.54.mlp.experts.41.gate_proj", + "model.layers.54.mlp.experts.41.up_proj", + "model.layers.54.mlp.experts.42.down_proj", + "model.layers.54.mlp.experts.42.gate_proj", + "model.layers.54.mlp.experts.42.up_proj", + "model.layers.54.mlp.experts.43.down_proj", + "model.layers.54.mlp.experts.43.gate_proj", + "model.layers.54.mlp.experts.43.up_proj", + "model.layers.54.mlp.experts.44.down_proj", + "model.layers.54.mlp.experts.44.gate_proj", + "model.layers.54.mlp.experts.44.up_proj", + "model.layers.54.mlp.experts.45.down_proj", + "model.layers.54.mlp.experts.45.gate_proj", + "model.layers.54.mlp.experts.45.up_proj", + "model.layers.54.mlp.experts.46.down_proj", + "model.layers.54.mlp.experts.46.gate_proj", + "model.layers.54.mlp.experts.46.up_proj", + "model.layers.54.mlp.experts.47.down_proj", + "model.layers.54.mlp.experts.47.gate_proj", + "model.layers.54.mlp.experts.47.up_proj", + "model.layers.54.mlp.experts.48.down_proj", + "model.layers.54.mlp.experts.48.gate_proj", + "model.layers.54.mlp.experts.48.up_proj", + "model.layers.54.mlp.experts.49.down_proj", + "model.layers.54.mlp.experts.49.gate_proj", + "model.layers.54.mlp.experts.49.up_proj", + "model.layers.54.mlp.experts.5.down_proj", + "model.layers.54.mlp.experts.5.gate_proj", + "model.layers.54.mlp.experts.5.up_proj", + "model.layers.54.mlp.experts.50.down_proj", + "model.layers.54.mlp.experts.50.gate_proj", + "model.layers.54.mlp.experts.50.up_proj", + "model.layers.54.mlp.experts.51.down_proj", + "model.layers.54.mlp.experts.51.gate_proj", + "model.layers.54.mlp.experts.51.up_proj", + "model.layers.54.mlp.experts.52.down_proj", + "model.layers.54.mlp.experts.52.gate_proj", + "model.layers.54.mlp.experts.52.up_proj", + "model.layers.54.mlp.experts.53.down_proj", + "model.layers.54.mlp.experts.53.gate_proj", + "model.layers.54.mlp.experts.53.up_proj", + "model.layers.54.mlp.experts.54.down_proj", + "model.layers.54.mlp.experts.54.gate_proj", + "model.layers.54.mlp.experts.54.up_proj", + "model.layers.54.mlp.experts.55.down_proj", + "model.layers.54.mlp.experts.55.gate_proj", + "model.layers.54.mlp.experts.55.up_proj", + "model.layers.54.mlp.experts.56.down_proj", + "model.layers.54.mlp.experts.56.gate_proj", + "model.layers.54.mlp.experts.56.up_proj", + "model.layers.54.mlp.experts.57.down_proj", + "model.layers.54.mlp.experts.57.gate_proj", + "model.layers.54.mlp.experts.57.up_proj", + "model.layers.54.mlp.experts.58.down_proj", + "model.layers.54.mlp.experts.58.gate_proj", + "model.layers.54.mlp.experts.58.up_proj", + "model.layers.54.mlp.experts.59.down_proj", + "model.layers.54.mlp.experts.59.gate_proj", + "model.layers.54.mlp.experts.59.up_proj", + "model.layers.54.mlp.experts.6.down_proj", + "model.layers.54.mlp.experts.6.gate_proj", + "model.layers.54.mlp.experts.6.up_proj", + "model.layers.54.mlp.experts.60.down_proj", + "model.layers.54.mlp.experts.60.gate_proj", + "model.layers.54.mlp.experts.60.up_proj", + "model.layers.54.mlp.experts.61.down_proj", + "model.layers.54.mlp.experts.61.gate_proj", + "model.layers.54.mlp.experts.61.up_proj", + "model.layers.54.mlp.experts.62.down_proj", + "model.layers.54.mlp.experts.62.gate_proj", + "model.layers.54.mlp.experts.62.up_proj", + "model.layers.54.mlp.experts.63.down_proj", + "model.layers.54.mlp.experts.63.gate_proj", + "model.layers.54.mlp.experts.63.up_proj", + "model.layers.54.mlp.experts.64.down_proj", + "model.layers.54.mlp.experts.64.gate_proj", + "model.layers.54.mlp.experts.64.up_proj", + "model.layers.54.mlp.experts.65.down_proj", + "model.layers.54.mlp.experts.65.gate_proj", + "model.layers.54.mlp.experts.65.up_proj", + "model.layers.54.mlp.experts.66.down_proj", + "model.layers.54.mlp.experts.66.gate_proj", + "model.layers.54.mlp.experts.66.up_proj", + "model.layers.54.mlp.experts.67.down_proj", + "model.layers.54.mlp.experts.67.gate_proj", + "model.layers.54.mlp.experts.67.up_proj", + "model.layers.54.mlp.experts.68.down_proj", + "model.layers.54.mlp.experts.68.gate_proj", + "model.layers.54.mlp.experts.68.up_proj", + "model.layers.54.mlp.experts.69.down_proj", + "model.layers.54.mlp.experts.69.gate_proj", + "model.layers.54.mlp.experts.69.up_proj", + "model.layers.54.mlp.experts.7.down_proj", + "model.layers.54.mlp.experts.7.gate_proj", + "model.layers.54.mlp.experts.7.up_proj", + "model.layers.54.mlp.experts.70.down_proj", + "model.layers.54.mlp.experts.70.gate_proj", + "model.layers.54.mlp.experts.70.up_proj", + "model.layers.54.mlp.experts.71.down_proj", + "model.layers.54.mlp.experts.71.gate_proj", + "model.layers.54.mlp.experts.71.up_proj", + "model.layers.54.mlp.experts.72.down_proj", + "model.layers.54.mlp.experts.72.gate_proj", + "model.layers.54.mlp.experts.72.up_proj", + "model.layers.54.mlp.experts.73.down_proj", + "model.layers.54.mlp.experts.73.gate_proj", + "model.layers.54.mlp.experts.73.up_proj", + "model.layers.54.mlp.experts.74.down_proj", + "model.layers.54.mlp.experts.74.gate_proj", + "model.layers.54.mlp.experts.74.up_proj", + "model.layers.54.mlp.experts.75.down_proj", + "model.layers.54.mlp.experts.75.gate_proj", + "model.layers.54.mlp.experts.75.up_proj", + "model.layers.54.mlp.experts.76.down_proj", + "model.layers.54.mlp.experts.76.gate_proj", + "model.layers.54.mlp.experts.76.up_proj", + "model.layers.54.mlp.experts.77.down_proj", + "model.layers.54.mlp.experts.77.gate_proj", + "model.layers.54.mlp.experts.77.up_proj", + "model.layers.54.mlp.experts.78.down_proj", + "model.layers.54.mlp.experts.78.gate_proj", + "model.layers.54.mlp.experts.78.up_proj", + "model.layers.54.mlp.experts.79.down_proj", + "model.layers.54.mlp.experts.79.gate_proj", + "model.layers.54.mlp.experts.79.up_proj", + "model.layers.54.mlp.experts.8.down_proj", + "model.layers.54.mlp.experts.8.gate_proj", + "model.layers.54.mlp.experts.8.up_proj", + "model.layers.54.mlp.experts.80.down_proj", + "model.layers.54.mlp.experts.80.gate_proj", + "model.layers.54.mlp.experts.80.up_proj", + "model.layers.54.mlp.experts.81.down_proj", + "model.layers.54.mlp.experts.81.gate_proj", + "model.layers.54.mlp.experts.81.up_proj", + "model.layers.54.mlp.experts.82.down_proj", + "model.layers.54.mlp.experts.82.gate_proj", + "model.layers.54.mlp.experts.82.up_proj", + "model.layers.54.mlp.experts.83.down_proj", + "model.layers.54.mlp.experts.83.gate_proj", + "model.layers.54.mlp.experts.83.up_proj", + "model.layers.54.mlp.experts.84.down_proj", + "model.layers.54.mlp.experts.84.gate_proj", + "model.layers.54.mlp.experts.84.up_proj", + "model.layers.54.mlp.experts.85.down_proj", + "model.layers.54.mlp.experts.85.gate_proj", + "model.layers.54.mlp.experts.85.up_proj", + "model.layers.54.mlp.experts.86.down_proj", + "model.layers.54.mlp.experts.86.gate_proj", + "model.layers.54.mlp.experts.86.up_proj", + "model.layers.54.mlp.experts.87.down_proj", + "model.layers.54.mlp.experts.87.gate_proj", + "model.layers.54.mlp.experts.87.up_proj", + "model.layers.54.mlp.experts.88.down_proj", + "model.layers.54.mlp.experts.88.gate_proj", + "model.layers.54.mlp.experts.88.up_proj", + "model.layers.54.mlp.experts.89.down_proj", + "model.layers.54.mlp.experts.89.gate_proj", + "model.layers.54.mlp.experts.89.up_proj", + "model.layers.54.mlp.experts.9.down_proj", + "model.layers.54.mlp.experts.9.gate_proj", + "model.layers.54.mlp.experts.9.up_proj", + "model.layers.54.mlp.experts.90.down_proj", + "model.layers.54.mlp.experts.90.gate_proj", + "model.layers.54.mlp.experts.90.up_proj", + "model.layers.54.mlp.experts.91.down_proj", + "model.layers.54.mlp.experts.91.gate_proj", + "model.layers.54.mlp.experts.91.up_proj", + "model.layers.54.mlp.experts.92.down_proj", + "model.layers.54.mlp.experts.92.gate_proj", + "model.layers.54.mlp.experts.92.up_proj", + "model.layers.54.mlp.experts.93.down_proj", + "model.layers.54.mlp.experts.93.gate_proj", + "model.layers.54.mlp.experts.93.up_proj", + "model.layers.54.mlp.experts.94.down_proj", + "model.layers.54.mlp.experts.94.gate_proj", + "model.layers.54.mlp.experts.94.up_proj", + "model.layers.54.mlp.experts.95.down_proj", + "model.layers.54.mlp.experts.95.gate_proj", + "model.layers.54.mlp.experts.95.up_proj", + "model.layers.54.mlp.experts.96.down_proj", + "model.layers.54.mlp.experts.96.gate_proj", + "model.layers.54.mlp.experts.96.up_proj", + "model.layers.54.mlp.experts.97.down_proj", + "model.layers.54.mlp.experts.97.gate_proj", + "model.layers.54.mlp.experts.97.up_proj", + "model.layers.54.mlp.experts.98.down_proj", + "model.layers.54.mlp.experts.98.gate_proj", + "model.layers.54.mlp.experts.98.up_proj", + "model.layers.54.mlp.experts.99.down_proj", + "model.layers.54.mlp.experts.99.gate_proj", + "model.layers.54.mlp.experts.99.up_proj", + "model.layers.55.mlp.experts.0.down_proj", + "model.layers.55.mlp.experts.0.gate_proj", + "model.layers.55.mlp.experts.0.up_proj", + "model.layers.55.mlp.experts.1.down_proj", + "model.layers.55.mlp.experts.1.gate_proj", + "model.layers.55.mlp.experts.1.up_proj", + "model.layers.55.mlp.experts.10.down_proj", + "model.layers.55.mlp.experts.10.gate_proj", + "model.layers.55.mlp.experts.10.up_proj", + "model.layers.55.mlp.experts.100.down_proj", + "model.layers.55.mlp.experts.100.gate_proj", + "model.layers.55.mlp.experts.100.up_proj", + "model.layers.55.mlp.experts.101.down_proj", + "model.layers.55.mlp.experts.101.gate_proj", + "model.layers.55.mlp.experts.101.up_proj", + "model.layers.55.mlp.experts.102.down_proj", + "model.layers.55.mlp.experts.102.gate_proj", + "model.layers.55.mlp.experts.102.up_proj", + "model.layers.55.mlp.experts.103.down_proj", + "model.layers.55.mlp.experts.103.gate_proj", + "model.layers.55.mlp.experts.103.up_proj", + "model.layers.55.mlp.experts.104.down_proj", + "model.layers.55.mlp.experts.104.gate_proj", + "model.layers.55.mlp.experts.104.up_proj", + "model.layers.55.mlp.experts.105.down_proj", + "model.layers.55.mlp.experts.105.gate_proj", + "model.layers.55.mlp.experts.105.up_proj", + "model.layers.55.mlp.experts.106.down_proj", + "model.layers.55.mlp.experts.106.gate_proj", + "model.layers.55.mlp.experts.106.up_proj", + "model.layers.55.mlp.experts.107.down_proj", + "model.layers.55.mlp.experts.107.gate_proj", + "model.layers.55.mlp.experts.107.up_proj", + "model.layers.55.mlp.experts.108.down_proj", + "model.layers.55.mlp.experts.108.gate_proj", + "model.layers.55.mlp.experts.108.up_proj", + "model.layers.55.mlp.experts.109.down_proj", + "model.layers.55.mlp.experts.109.gate_proj", + "model.layers.55.mlp.experts.109.up_proj", + "model.layers.55.mlp.experts.11.down_proj", + "model.layers.55.mlp.experts.11.gate_proj", + "model.layers.55.mlp.experts.11.up_proj", + "model.layers.55.mlp.experts.110.down_proj", + "model.layers.55.mlp.experts.110.gate_proj", + "model.layers.55.mlp.experts.110.up_proj", + "model.layers.55.mlp.experts.111.down_proj", + "model.layers.55.mlp.experts.111.gate_proj", + "model.layers.55.mlp.experts.111.up_proj", + "model.layers.55.mlp.experts.112.down_proj", + "model.layers.55.mlp.experts.112.gate_proj", + "model.layers.55.mlp.experts.112.up_proj", + "model.layers.55.mlp.experts.113.down_proj", + "model.layers.55.mlp.experts.113.gate_proj", + "model.layers.55.mlp.experts.113.up_proj", + "model.layers.55.mlp.experts.114.down_proj", + "model.layers.55.mlp.experts.114.gate_proj", + "model.layers.55.mlp.experts.114.up_proj", + "model.layers.55.mlp.experts.115.down_proj", + "model.layers.55.mlp.experts.115.gate_proj", + "model.layers.55.mlp.experts.115.up_proj", + "model.layers.55.mlp.experts.116.down_proj", + "model.layers.55.mlp.experts.116.gate_proj", + "model.layers.55.mlp.experts.116.up_proj", + "model.layers.55.mlp.experts.117.down_proj", + "model.layers.55.mlp.experts.117.gate_proj", + "model.layers.55.mlp.experts.117.up_proj", + "model.layers.55.mlp.experts.118.down_proj", + "model.layers.55.mlp.experts.118.gate_proj", + "model.layers.55.mlp.experts.118.up_proj", + "model.layers.55.mlp.experts.119.down_proj", + "model.layers.55.mlp.experts.119.gate_proj", + "model.layers.55.mlp.experts.119.up_proj", + "model.layers.55.mlp.experts.12.down_proj", + "model.layers.55.mlp.experts.12.gate_proj", + "model.layers.55.mlp.experts.12.up_proj", + "model.layers.55.mlp.experts.120.down_proj", + "model.layers.55.mlp.experts.120.gate_proj", + "model.layers.55.mlp.experts.120.up_proj", + "model.layers.55.mlp.experts.121.down_proj", + "model.layers.55.mlp.experts.121.gate_proj", + "model.layers.55.mlp.experts.121.up_proj", + "model.layers.55.mlp.experts.122.down_proj", + "model.layers.55.mlp.experts.122.gate_proj", + "model.layers.55.mlp.experts.122.up_proj", + "model.layers.55.mlp.experts.123.down_proj", + "model.layers.55.mlp.experts.123.gate_proj", + "model.layers.55.mlp.experts.123.up_proj", + "model.layers.55.mlp.experts.124.down_proj", + "model.layers.55.mlp.experts.124.gate_proj", + "model.layers.55.mlp.experts.124.up_proj", + "model.layers.55.mlp.experts.125.down_proj", + "model.layers.55.mlp.experts.125.gate_proj", + "model.layers.55.mlp.experts.125.up_proj", + "model.layers.55.mlp.experts.126.down_proj", + "model.layers.55.mlp.experts.126.gate_proj", + "model.layers.55.mlp.experts.126.up_proj", + "model.layers.55.mlp.experts.127.down_proj", + "model.layers.55.mlp.experts.127.gate_proj", + "model.layers.55.mlp.experts.127.up_proj", + "model.layers.55.mlp.experts.128.down_proj", + "model.layers.55.mlp.experts.128.gate_proj", + "model.layers.55.mlp.experts.128.up_proj", + "model.layers.55.mlp.experts.129.down_proj", + "model.layers.55.mlp.experts.129.gate_proj", + "model.layers.55.mlp.experts.129.up_proj", + "model.layers.55.mlp.experts.13.down_proj", + "model.layers.55.mlp.experts.13.gate_proj", + "model.layers.55.mlp.experts.13.up_proj", + "model.layers.55.mlp.experts.130.down_proj", + "model.layers.55.mlp.experts.130.gate_proj", + "model.layers.55.mlp.experts.130.up_proj", + "model.layers.55.mlp.experts.131.down_proj", + "model.layers.55.mlp.experts.131.gate_proj", + "model.layers.55.mlp.experts.131.up_proj", + "model.layers.55.mlp.experts.132.down_proj", + "model.layers.55.mlp.experts.132.gate_proj", + "model.layers.55.mlp.experts.132.up_proj", + "model.layers.55.mlp.experts.133.down_proj", + "model.layers.55.mlp.experts.133.gate_proj", + "model.layers.55.mlp.experts.133.up_proj", + "model.layers.55.mlp.experts.134.down_proj", + "model.layers.55.mlp.experts.134.gate_proj", + "model.layers.55.mlp.experts.134.up_proj", + "model.layers.55.mlp.experts.135.down_proj", + "model.layers.55.mlp.experts.135.gate_proj", + "model.layers.55.mlp.experts.135.up_proj", + "model.layers.55.mlp.experts.136.down_proj", + "model.layers.55.mlp.experts.136.gate_proj", + "model.layers.55.mlp.experts.136.up_proj", + "model.layers.55.mlp.experts.137.down_proj", + "model.layers.55.mlp.experts.137.gate_proj", + "model.layers.55.mlp.experts.137.up_proj", + "model.layers.55.mlp.experts.138.down_proj", + "model.layers.55.mlp.experts.138.gate_proj", + "model.layers.55.mlp.experts.138.up_proj", + "model.layers.55.mlp.experts.139.down_proj", + "model.layers.55.mlp.experts.139.gate_proj", + "model.layers.55.mlp.experts.139.up_proj", + "model.layers.55.mlp.experts.14.down_proj", + "model.layers.55.mlp.experts.14.gate_proj", + "model.layers.55.mlp.experts.14.up_proj", + "model.layers.55.mlp.experts.140.down_proj", + "model.layers.55.mlp.experts.140.gate_proj", + "model.layers.55.mlp.experts.140.up_proj", + "model.layers.55.mlp.experts.141.down_proj", + "model.layers.55.mlp.experts.141.gate_proj", + "model.layers.55.mlp.experts.141.up_proj", + "model.layers.55.mlp.experts.142.down_proj", + "model.layers.55.mlp.experts.142.gate_proj", + "model.layers.55.mlp.experts.142.up_proj", + "model.layers.55.mlp.experts.143.down_proj", + "model.layers.55.mlp.experts.143.gate_proj", + "model.layers.55.mlp.experts.143.up_proj", + "model.layers.55.mlp.experts.144.down_proj", + "model.layers.55.mlp.experts.144.gate_proj", + "model.layers.55.mlp.experts.144.up_proj", + "model.layers.55.mlp.experts.145.down_proj", + "model.layers.55.mlp.experts.145.gate_proj", + "model.layers.55.mlp.experts.145.up_proj", + "model.layers.55.mlp.experts.146.down_proj", + "model.layers.55.mlp.experts.146.gate_proj", + "model.layers.55.mlp.experts.146.up_proj", + "model.layers.55.mlp.experts.147.down_proj", + "model.layers.55.mlp.experts.147.gate_proj", + "model.layers.55.mlp.experts.147.up_proj", + "model.layers.55.mlp.experts.148.down_proj", + "model.layers.55.mlp.experts.148.gate_proj", + "model.layers.55.mlp.experts.148.up_proj", + "model.layers.55.mlp.experts.149.down_proj", + "model.layers.55.mlp.experts.149.gate_proj", + "model.layers.55.mlp.experts.149.up_proj", + "model.layers.55.mlp.experts.15.down_proj", + "model.layers.55.mlp.experts.15.gate_proj", + "model.layers.55.mlp.experts.15.up_proj", + "model.layers.55.mlp.experts.150.down_proj", + "model.layers.55.mlp.experts.150.gate_proj", + "model.layers.55.mlp.experts.150.up_proj", + "model.layers.55.mlp.experts.151.down_proj", + "model.layers.55.mlp.experts.151.gate_proj", + "model.layers.55.mlp.experts.151.up_proj", + "model.layers.55.mlp.experts.152.down_proj", + "model.layers.55.mlp.experts.152.gate_proj", + "model.layers.55.mlp.experts.152.up_proj", + "model.layers.55.mlp.experts.153.down_proj", + "model.layers.55.mlp.experts.153.gate_proj", + "model.layers.55.mlp.experts.153.up_proj", + "model.layers.55.mlp.experts.154.down_proj", + "model.layers.55.mlp.experts.154.gate_proj", + "model.layers.55.mlp.experts.154.up_proj", + "model.layers.55.mlp.experts.155.down_proj", + "model.layers.55.mlp.experts.155.gate_proj", + "model.layers.55.mlp.experts.155.up_proj", + "model.layers.55.mlp.experts.156.down_proj", + "model.layers.55.mlp.experts.156.gate_proj", + "model.layers.55.mlp.experts.156.up_proj", + "model.layers.55.mlp.experts.157.down_proj", + "model.layers.55.mlp.experts.157.gate_proj", + "model.layers.55.mlp.experts.157.up_proj", + "model.layers.55.mlp.experts.158.down_proj", + "model.layers.55.mlp.experts.158.gate_proj", + "model.layers.55.mlp.experts.158.up_proj", + "model.layers.55.mlp.experts.159.down_proj", + "model.layers.55.mlp.experts.159.gate_proj", + "model.layers.55.mlp.experts.159.up_proj", + "model.layers.55.mlp.experts.16.down_proj", + "model.layers.55.mlp.experts.16.gate_proj", + "model.layers.55.mlp.experts.16.up_proj", + "model.layers.55.mlp.experts.160.down_proj", + "model.layers.55.mlp.experts.160.gate_proj", + "model.layers.55.mlp.experts.160.up_proj", + "model.layers.55.mlp.experts.161.down_proj", + "model.layers.55.mlp.experts.161.gate_proj", + "model.layers.55.mlp.experts.161.up_proj", + "model.layers.55.mlp.experts.162.down_proj", + "model.layers.55.mlp.experts.162.gate_proj", + "model.layers.55.mlp.experts.162.up_proj", + "model.layers.55.mlp.experts.163.down_proj", + "model.layers.55.mlp.experts.163.gate_proj", + "model.layers.55.mlp.experts.163.up_proj", + "model.layers.55.mlp.experts.164.down_proj", + "model.layers.55.mlp.experts.164.gate_proj", + "model.layers.55.mlp.experts.164.up_proj", + "model.layers.55.mlp.experts.165.down_proj", + "model.layers.55.mlp.experts.165.gate_proj", + "model.layers.55.mlp.experts.165.up_proj", + "model.layers.55.mlp.experts.166.down_proj", + "model.layers.55.mlp.experts.166.gate_proj", + "model.layers.55.mlp.experts.166.up_proj", + "model.layers.55.mlp.experts.167.down_proj", + "model.layers.55.mlp.experts.167.gate_proj", + "model.layers.55.mlp.experts.167.up_proj", + "model.layers.55.mlp.experts.168.down_proj", + "model.layers.55.mlp.experts.168.gate_proj", + "model.layers.55.mlp.experts.168.up_proj", + "model.layers.55.mlp.experts.169.down_proj", + "model.layers.55.mlp.experts.169.gate_proj", + "model.layers.55.mlp.experts.169.up_proj", + "model.layers.55.mlp.experts.17.down_proj", + "model.layers.55.mlp.experts.17.gate_proj", + "model.layers.55.mlp.experts.17.up_proj", + "model.layers.55.mlp.experts.170.down_proj", + "model.layers.55.mlp.experts.170.gate_proj", + "model.layers.55.mlp.experts.170.up_proj", + "model.layers.55.mlp.experts.171.down_proj", + "model.layers.55.mlp.experts.171.gate_proj", + "model.layers.55.mlp.experts.171.up_proj", + "model.layers.55.mlp.experts.172.down_proj", + "model.layers.55.mlp.experts.172.gate_proj", + "model.layers.55.mlp.experts.172.up_proj", + "model.layers.55.mlp.experts.173.down_proj", + "model.layers.55.mlp.experts.173.gate_proj", + "model.layers.55.mlp.experts.173.up_proj", + "model.layers.55.mlp.experts.174.down_proj", + "model.layers.55.mlp.experts.174.gate_proj", + "model.layers.55.mlp.experts.174.up_proj", + "model.layers.55.mlp.experts.175.down_proj", + "model.layers.55.mlp.experts.175.gate_proj", + "model.layers.55.mlp.experts.175.up_proj", + "model.layers.55.mlp.experts.176.down_proj", + "model.layers.55.mlp.experts.176.gate_proj", + "model.layers.55.mlp.experts.176.up_proj", + "model.layers.55.mlp.experts.177.down_proj", + "model.layers.55.mlp.experts.177.gate_proj", + "model.layers.55.mlp.experts.177.up_proj", + "model.layers.55.mlp.experts.178.down_proj", + "model.layers.55.mlp.experts.178.gate_proj", + "model.layers.55.mlp.experts.178.up_proj", + "model.layers.55.mlp.experts.179.down_proj", + "model.layers.55.mlp.experts.179.gate_proj", + "model.layers.55.mlp.experts.179.up_proj", + "model.layers.55.mlp.experts.18.down_proj", + "model.layers.55.mlp.experts.18.gate_proj", + "model.layers.55.mlp.experts.18.up_proj", + "model.layers.55.mlp.experts.180.down_proj", + "model.layers.55.mlp.experts.180.gate_proj", + "model.layers.55.mlp.experts.180.up_proj", + "model.layers.55.mlp.experts.181.down_proj", + "model.layers.55.mlp.experts.181.gate_proj", + "model.layers.55.mlp.experts.181.up_proj", + "model.layers.55.mlp.experts.182.down_proj", + "model.layers.55.mlp.experts.182.gate_proj", + "model.layers.55.mlp.experts.182.up_proj", + "model.layers.55.mlp.experts.183.down_proj", + "model.layers.55.mlp.experts.183.gate_proj", + "model.layers.55.mlp.experts.183.up_proj", + "model.layers.55.mlp.experts.184.down_proj", + "model.layers.55.mlp.experts.184.gate_proj", + "model.layers.55.mlp.experts.184.up_proj", + "model.layers.55.mlp.experts.185.down_proj", + "model.layers.55.mlp.experts.185.gate_proj", + "model.layers.55.mlp.experts.185.up_proj", + "model.layers.55.mlp.experts.186.down_proj", + "model.layers.55.mlp.experts.186.gate_proj", + "model.layers.55.mlp.experts.186.up_proj", + "model.layers.55.mlp.experts.187.down_proj", + "model.layers.55.mlp.experts.187.gate_proj", + "model.layers.55.mlp.experts.187.up_proj", + "model.layers.55.mlp.experts.188.down_proj", + "model.layers.55.mlp.experts.188.gate_proj", + "model.layers.55.mlp.experts.188.up_proj", + "model.layers.55.mlp.experts.189.down_proj", + "model.layers.55.mlp.experts.189.gate_proj", + "model.layers.55.mlp.experts.189.up_proj", + "model.layers.55.mlp.experts.19.down_proj", + "model.layers.55.mlp.experts.19.gate_proj", + "model.layers.55.mlp.experts.19.up_proj", + "model.layers.55.mlp.experts.190.down_proj", + "model.layers.55.mlp.experts.190.gate_proj", + "model.layers.55.mlp.experts.190.up_proj", + "model.layers.55.mlp.experts.191.down_proj", + "model.layers.55.mlp.experts.191.gate_proj", + "model.layers.55.mlp.experts.191.up_proj", + "model.layers.55.mlp.experts.192.down_proj", + "model.layers.55.mlp.experts.192.gate_proj", + "model.layers.55.mlp.experts.192.up_proj", + "model.layers.55.mlp.experts.193.down_proj", + "model.layers.55.mlp.experts.193.gate_proj", + "model.layers.55.mlp.experts.193.up_proj", + "model.layers.55.mlp.experts.194.down_proj", + "model.layers.55.mlp.experts.194.gate_proj", + "model.layers.55.mlp.experts.194.up_proj", + "model.layers.55.mlp.experts.195.down_proj", + "model.layers.55.mlp.experts.195.gate_proj", + "model.layers.55.mlp.experts.195.up_proj", + "model.layers.55.mlp.experts.196.down_proj", + "model.layers.55.mlp.experts.196.gate_proj", + "model.layers.55.mlp.experts.196.up_proj", + "model.layers.55.mlp.experts.197.down_proj", + "model.layers.55.mlp.experts.197.gate_proj", + "model.layers.55.mlp.experts.197.up_proj", + "model.layers.55.mlp.experts.198.down_proj", + "model.layers.55.mlp.experts.198.gate_proj", + "model.layers.55.mlp.experts.198.up_proj", + "model.layers.55.mlp.experts.199.down_proj", + "model.layers.55.mlp.experts.199.gate_proj", + "model.layers.55.mlp.experts.199.up_proj", + "model.layers.55.mlp.experts.2.down_proj", + "model.layers.55.mlp.experts.2.gate_proj", + "model.layers.55.mlp.experts.2.up_proj", + "model.layers.55.mlp.experts.20.down_proj", + "model.layers.55.mlp.experts.20.gate_proj", + "model.layers.55.mlp.experts.20.up_proj", + "model.layers.55.mlp.experts.200.down_proj", + "model.layers.55.mlp.experts.200.gate_proj", + "model.layers.55.mlp.experts.200.up_proj", + "model.layers.55.mlp.experts.201.down_proj", + "model.layers.55.mlp.experts.201.gate_proj", + "model.layers.55.mlp.experts.201.up_proj", + "model.layers.55.mlp.experts.202.down_proj", + "model.layers.55.mlp.experts.202.gate_proj", + "model.layers.55.mlp.experts.202.up_proj", + "model.layers.55.mlp.experts.203.down_proj", + "model.layers.55.mlp.experts.203.gate_proj", + "model.layers.55.mlp.experts.203.up_proj", + "model.layers.55.mlp.experts.204.down_proj", + "model.layers.55.mlp.experts.204.gate_proj", + "model.layers.55.mlp.experts.204.up_proj", + "model.layers.55.mlp.experts.205.down_proj", + "model.layers.55.mlp.experts.205.gate_proj", + "model.layers.55.mlp.experts.205.up_proj", + "model.layers.55.mlp.experts.206.down_proj", + "model.layers.55.mlp.experts.206.gate_proj", + "model.layers.55.mlp.experts.206.up_proj", + "model.layers.55.mlp.experts.207.down_proj", + "model.layers.55.mlp.experts.207.gate_proj", + "model.layers.55.mlp.experts.207.up_proj", + "model.layers.55.mlp.experts.208.down_proj", + "model.layers.55.mlp.experts.208.gate_proj", + "model.layers.55.mlp.experts.208.up_proj", + "model.layers.55.mlp.experts.209.down_proj", + "model.layers.55.mlp.experts.209.gate_proj", + "model.layers.55.mlp.experts.209.up_proj", + "model.layers.55.mlp.experts.21.down_proj", + "model.layers.55.mlp.experts.21.gate_proj", + "model.layers.55.mlp.experts.21.up_proj", + "model.layers.55.mlp.experts.210.down_proj", + "model.layers.55.mlp.experts.210.gate_proj", + "model.layers.55.mlp.experts.210.up_proj", + "model.layers.55.mlp.experts.211.down_proj", + "model.layers.55.mlp.experts.211.gate_proj", + "model.layers.55.mlp.experts.211.up_proj", + "model.layers.55.mlp.experts.212.down_proj", + "model.layers.55.mlp.experts.212.gate_proj", + "model.layers.55.mlp.experts.212.up_proj", + "model.layers.55.mlp.experts.213.down_proj", + "model.layers.55.mlp.experts.213.gate_proj", + "model.layers.55.mlp.experts.213.up_proj", + "model.layers.55.mlp.experts.214.down_proj", + "model.layers.55.mlp.experts.214.gate_proj", + "model.layers.55.mlp.experts.214.up_proj", + "model.layers.55.mlp.experts.215.down_proj", + "model.layers.55.mlp.experts.215.gate_proj", + "model.layers.55.mlp.experts.215.up_proj", + "model.layers.55.mlp.experts.216.down_proj", + "model.layers.55.mlp.experts.216.gate_proj", + "model.layers.55.mlp.experts.216.up_proj", + "model.layers.55.mlp.experts.217.down_proj", + "model.layers.55.mlp.experts.217.gate_proj", + "model.layers.55.mlp.experts.217.up_proj", + "model.layers.55.mlp.experts.218.down_proj", + "model.layers.55.mlp.experts.218.gate_proj", + "model.layers.55.mlp.experts.218.up_proj", + "model.layers.55.mlp.experts.219.down_proj", + "model.layers.55.mlp.experts.219.gate_proj", + "model.layers.55.mlp.experts.219.up_proj", + "model.layers.55.mlp.experts.22.down_proj", + "model.layers.55.mlp.experts.22.gate_proj", + "model.layers.55.mlp.experts.22.up_proj", + "model.layers.55.mlp.experts.220.down_proj", + "model.layers.55.mlp.experts.220.gate_proj", + "model.layers.55.mlp.experts.220.up_proj", + "model.layers.55.mlp.experts.221.down_proj", + "model.layers.55.mlp.experts.221.gate_proj", + "model.layers.55.mlp.experts.221.up_proj", + "model.layers.55.mlp.experts.222.down_proj", + "model.layers.55.mlp.experts.222.gate_proj", + "model.layers.55.mlp.experts.222.up_proj", + "model.layers.55.mlp.experts.223.down_proj", + "model.layers.55.mlp.experts.223.gate_proj", + "model.layers.55.mlp.experts.223.up_proj", + "model.layers.55.mlp.experts.224.down_proj", + "model.layers.55.mlp.experts.224.gate_proj", + "model.layers.55.mlp.experts.224.up_proj", + "model.layers.55.mlp.experts.225.down_proj", + "model.layers.55.mlp.experts.225.gate_proj", + "model.layers.55.mlp.experts.225.up_proj", + "model.layers.55.mlp.experts.226.down_proj", + "model.layers.55.mlp.experts.226.gate_proj", + "model.layers.55.mlp.experts.226.up_proj", + "model.layers.55.mlp.experts.227.down_proj", + "model.layers.55.mlp.experts.227.gate_proj", + "model.layers.55.mlp.experts.227.up_proj", + "model.layers.55.mlp.experts.228.down_proj", + "model.layers.55.mlp.experts.228.gate_proj", + "model.layers.55.mlp.experts.228.up_proj", + "model.layers.55.mlp.experts.229.down_proj", + "model.layers.55.mlp.experts.229.gate_proj", + "model.layers.55.mlp.experts.229.up_proj", + "model.layers.55.mlp.experts.23.down_proj", + "model.layers.55.mlp.experts.23.gate_proj", + "model.layers.55.mlp.experts.23.up_proj", + "model.layers.55.mlp.experts.230.down_proj", + "model.layers.55.mlp.experts.230.gate_proj", + "model.layers.55.mlp.experts.230.up_proj", + "model.layers.55.mlp.experts.231.down_proj", + "model.layers.55.mlp.experts.231.gate_proj", + "model.layers.55.mlp.experts.231.up_proj", + "model.layers.55.mlp.experts.232.down_proj", + "model.layers.55.mlp.experts.232.gate_proj", + "model.layers.55.mlp.experts.232.up_proj", + "model.layers.55.mlp.experts.233.down_proj", + "model.layers.55.mlp.experts.233.gate_proj", + "model.layers.55.mlp.experts.233.up_proj", + "model.layers.55.mlp.experts.234.down_proj", + "model.layers.55.mlp.experts.234.gate_proj", + "model.layers.55.mlp.experts.234.up_proj", + "model.layers.55.mlp.experts.235.down_proj", + "model.layers.55.mlp.experts.235.gate_proj", + "model.layers.55.mlp.experts.235.up_proj", + "model.layers.55.mlp.experts.236.down_proj", + "model.layers.55.mlp.experts.236.gate_proj", + "model.layers.55.mlp.experts.236.up_proj", + "model.layers.55.mlp.experts.237.down_proj", + "model.layers.55.mlp.experts.237.gate_proj", + "model.layers.55.mlp.experts.237.up_proj", + "model.layers.55.mlp.experts.238.down_proj", + "model.layers.55.mlp.experts.238.gate_proj", + "model.layers.55.mlp.experts.238.up_proj", + "model.layers.55.mlp.experts.239.down_proj", + "model.layers.55.mlp.experts.239.gate_proj", + "model.layers.55.mlp.experts.239.up_proj", + "model.layers.55.mlp.experts.24.down_proj", + "model.layers.55.mlp.experts.24.gate_proj", + "model.layers.55.mlp.experts.24.up_proj", + "model.layers.55.mlp.experts.240.down_proj", + "model.layers.55.mlp.experts.240.gate_proj", + "model.layers.55.mlp.experts.240.up_proj", + "model.layers.55.mlp.experts.241.down_proj", + "model.layers.55.mlp.experts.241.gate_proj", + "model.layers.55.mlp.experts.241.up_proj", + "model.layers.55.mlp.experts.242.down_proj", + "model.layers.55.mlp.experts.242.gate_proj", + "model.layers.55.mlp.experts.242.up_proj", + "model.layers.55.mlp.experts.243.down_proj", + "model.layers.55.mlp.experts.243.gate_proj", + "model.layers.55.mlp.experts.243.up_proj", + "model.layers.55.mlp.experts.244.down_proj", + "model.layers.55.mlp.experts.244.gate_proj", + "model.layers.55.mlp.experts.244.up_proj", + "model.layers.55.mlp.experts.245.down_proj", + "model.layers.55.mlp.experts.245.gate_proj", + "model.layers.55.mlp.experts.245.up_proj", + "model.layers.55.mlp.experts.246.down_proj", + "model.layers.55.mlp.experts.246.gate_proj", + "model.layers.55.mlp.experts.246.up_proj", + "model.layers.55.mlp.experts.247.down_proj", + "model.layers.55.mlp.experts.247.gate_proj", + "model.layers.55.mlp.experts.247.up_proj", + "model.layers.55.mlp.experts.248.down_proj", + "model.layers.55.mlp.experts.248.gate_proj", + "model.layers.55.mlp.experts.248.up_proj", + "model.layers.55.mlp.experts.249.down_proj", + "model.layers.55.mlp.experts.249.gate_proj", + "model.layers.55.mlp.experts.249.up_proj", + "model.layers.55.mlp.experts.25.down_proj", + "model.layers.55.mlp.experts.25.gate_proj", + "model.layers.55.mlp.experts.25.up_proj", + "model.layers.55.mlp.experts.250.down_proj", + "model.layers.55.mlp.experts.250.gate_proj", + "model.layers.55.mlp.experts.250.up_proj", + "model.layers.55.mlp.experts.251.down_proj", + "model.layers.55.mlp.experts.251.gate_proj", + "model.layers.55.mlp.experts.251.up_proj", + "model.layers.55.mlp.experts.252.down_proj", + "model.layers.55.mlp.experts.252.gate_proj", + "model.layers.55.mlp.experts.252.up_proj", + "model.layers.55.mlp.experts.253.down_proj", + "model.layers.55.mlp.experts.253.gate_proj", + "model.layers.55.mlp.experts.253.up_proj", + "model.layers.55.mlp.experts.254.down_proj", + "model.layers.55.mlp.experts.254.gate_proj", + "model.layers.55.mlp.experts.254.up_proj", + "model.layers.55.mlp.experts.255.down_proj", + "model.layers.55.mlp.experts.255.gate_proj", + "model.layers.55.mlp.experts.255.up_proj", + "model.layers.55.mlp.experts.26.down_proj", + "model.layers.55.mlp.experts.26.gate_proj", + "model.layers.55.mlp.experts.26.up_proj", + "model.layers.55.mlp.experts.27.down_proj", + "model.layers.55.mlp.experts.27.gate_proj", + "model.layers.55.mlp.experts.27.up_proj", + "model.layers.55.mlp.experts.28.down_proj", + "model.layers.55.mlp.experts.28.gate_proj", + "model.layers.55.mlp.experts.28.up_proj", + "model.layers.55.mlp.experts.29.down_proj", + "model.layers.55.mlp.experts.29.gate_proj", + "model.layers.55.mlp.experts.29.up_proj", + "model.layers.55.mlp.experts.3.down_proj", + "model.layers.55.mlp.experts.3.gate_proj", + "model.layers.55.mlp.experts.3.up_proj", + "model.layers.55.mlp.experts.30.down_proj", + "model.layers.55.mlp.experts.30.gate_proj", + "model.layers.55.mlp.experts.30.up_proj", + "model.layers.55.mlp.experts.31.down_proj", + "model.layers.55.mlp.experts.31.gate_proj", + "model.layers.55.mlp.experts.31.up_proj", + "model.layers.55.mlp.experts.32.down_proj", + "model.layers.55.mlp.experts.32.gate_proj", + "model.layers.55.mlp.experts.32.up_proj", + "model.layers.55.mlp.experts.33.down_proj", + "model.layers.55.mlp.experts.33.gate_proj", + "model.layers.55.mlp.experts.33.up_proj", + "model.layers.55.mlp.experts.34.down_proj", + "model.layers.55.mlp.experts.34.gate_proj", + "model.layers.55.mlp.experts.34.up_proj", + "model.layers.55.mlp.experts.35.down_proj", + "model.layers.55.mlp.experts.35.gate_proj", + "model.layers.55.mlp.experts.35.up_proj", + "model.layers.55.mlp.experts.36.down_proj", + "model.layers.55.mlp.experts.36.gate_proj", + "model.layers.55.mlp.experts.36.up_proj", + "model.layers.55.mlp.experts.37.down_proj", + "model.layers.55.mlp.experts.37.gate_proj", + "model.layers.55.mlp.experts.37.up_proj", + "model.layers.55.mlp.experts.38.down_proj", + "model.layers.55.mlp.experts.38.gate_proj", + "model.layers.55.mlp.experts.38.up_proj", + "model.layers.55.mlp.experts.39.down_proj", + "model.layers.55.mlp.experts.39.gate_proj", + "model.layers.55.mlp.experts.39.up_proj", + "model.layers.55.mlp.experts.4.down_proj", + "model.layers.55.mlp.experts.4.gate_proj", + "model.layers.55.mlp.experts.4.up_proj", + "model.layers.55.mlp.experts.40.down_proj", + "model.layers.55.mlp.experts.40.gate_proj", + "model.layers.55.mlp.experts.40.up_proj", + "model.layers.55.mlp.experts.41.down_proj", + "model.layers.55.mlp.experts.41.gate_proj", + "model.layers.55.mlp.experts.41.up_proj", + "model.layers.55.mlp.experts.42.down_proj", + "model.layers.55.mlp.experts.42.gate_proj", + "model.layers.55.mlp.experts.42.up_proj", + "model.layers.55.mlp.experts.43.down_proj", + "model.layers.55.mlp.experts.43.gate_proj", + "model.layers.55.mlp.experts.43.up_proj", + "model.layers.55.mlp.experts.44.down_proj", + "model.layers.55.mlp.experts.44.gate_proj", + "model.layers.55.mlp.experts.44.up_proj", + "model.layers.55.mlp.experts.45.down_proj", + "model.layers.55.mlp.experts.45.gate_proj", + "model.layers.55.mlp.experts.45.up_proj", + "model.layers.55.mlp.experts.46.down_proj", + "model.layers.55.mlp.experts.46.gate_proj", + "model.layers.55.mlp.experts.46.up_proj", + "model.layers.55.mlp.experts.47.down_proj", + "model.layers.55.mlp.experts.47.gate_proj", + "model.layers.55.mlp.experts.47.up_proj", + "model.layers.55.mlp.experts.48.down_proj", + "model.layers.55.mlp.experts.48.gate_proj", + "model.layers.55.mlp.experts.48.up_proj", + "model.layers.55.mlp.experts.49.down_proj", + "model.layers.55.mlp.experts.49.gate_proj", + "model.layers.55.mlp.experts.49.up_proj", + "model.layers.55.mlp.experts.5.down_proj", + "model.layers.55.mlp.experts.5.gate_proj", + "model.layers.55.mlp.experts.5.up_proj", + "model.layers.55.mlp.experts.50.down_proj", + "model.layers.55.mlp.experts.50.gate_proj", + "model.layers.55.mlp.experts.50.up_proj", + "model.layers.55.mlp.experts.51.down_proj", + "model.layers.55.mlp.experts.51.gate_proj", + "model.layers.55.mlp.experts.51.up_proj", + "model.layers.55.mlp.experts.52.down_proj", + "model.layers.55.mlp.experts.52.gate_proj", + "model.layers.55.mlp.experts.52.up_proj", + "model.layers.55.mlp.experts.53.down_proj", + "model.layers.55.mlp.experts.53.gate_proj", + "model.layers.55.mlp.experts.53.up_proj", + "model.layers.55.mlp.experts.54.down_proj", + "model.layers.55.mlp.experts.54.gate_proj", + "model.layers.55.mlp.experts.54.up_proj", + "model.layers.55.mlp.experts.55.down_proj", + "model.layers.55.mlp.experts.55.gate_proj", + "model.layers.55.mlp.experts.55.up_proj", + "model.layers.55.mlp.experts.56.down_proj", + "model.layers.55.mlp.experts.56.gate_proj", + "model.layers.55.mlp.experts.56.up_proj", + "model.layers.55.mlp.experts.57.down_proj", + "model.layers.55.mlp.experts.57.gate_proj", + "model.layers.55.mlp.experts.57.up_proj", + "model.layers.55.mlp.experts.58.down_proj", + "model.layers.55.mlp.experts.58.gate_proj", + "model.layers.55.mlp.experts.58.up_proj", + "model.layers.55.mlp.experts.59.down_proj", + "model.layers.55.mlp.experts.59.gate_proj", + "model.layers.55.mlp.experts.59.up_proj", + "model.layers.55.mlp.experts.6.down_proj", + "model.layers.55.mlp.experts.6.gate_proj", + "model.layers.55.mlp.experts.6.up_proj", + "model.layers.55.mlp.experts.60.down_proj", + "model.layers.55.mlp.experts.60.gate_proj", + "model.layers.55.mlp.experts.60.up_proj", + "model.layers.55.mlp.experts.61.down_proj", + "model.layers.55.mlp.experts.61.gate_proj", + "model.layers.55.mlp.experts.61.up_proj", + "model.layers.55.mlp.experts.62.down_proj", + "model.layers.55.mlp.experts.62.gate_proj", + "model.layers.55.mlp.experts.62.up_proj", + "model.layers.55.mlp.experts.63.down_proj", + "model.layers.55.mlp.experts.63.gate_proj", + "model.layers.55.mlp.experts.63.up_proj", + "model.layers.55.mlp.experts.64.down_proj", + "model.layers.55.mlp.experts.64.gate_proj", + "model.layers.55.mlp.experts.64.up_proj", + "model.layers.55.mlp.experts.65.down_proj", + "model.layers.55.mlp.experts.65.gate_proj", + "model.layers.55.mlp.experts.65.up_proj", + "model.layers.55.mlp.experts.66.down_proj", + "model.layers.55.mlp.experts.66.gate_proj", + "model.layers.55.mlp.experts.66.up_proj", + "model.layers.55.mlp.experts.67.down_proj", + "model.layers.55.mlp.experts.67.gate_proj", + "model.layers.55.mlp.experts.67.up_proj", + "model.layers.55.mlp.experts.68.down_proj", + "model.layers.55.mlp.experts.68.gate_proj", + "model.layers.55.mlp.experts.68.up_proj", + "model.layers.55.mlp.experts.69.down_proj", + "model.layers.55.mlp.experts.69.gate_proj", + "model.layers.55.mlp.experts.69.up_proj", + "model.layers.55.mlp.experts.7.down_proj", + "model.layers.55.mlp.experts.7.gate_proj", + "model.layers.55.mlp.experts.7.up_proj", + "model.layers.55.mlp.experts.70.down_proj", + "model.layers.55.mlp.experts.70.gate_proj", + "model.layers.55.mlp.experts.70.up_proj", + "model.layers.55.mlp.experts.71.down_proj", + "model.layers.55.mlp.experts.71.gate_proj", + "model.layers.55.mlp.experts.71.up_proj", + "model.layers.55.mlp.experts.72.down_proj", + "model.layers.55.mlp.experts.72.gate_proj", + "model.layers.55.mlp.experts.72.up_proj", + "model.layers.55.mlp.experts.73.down_proj", + "model.layers.55.mlp.experts.73.gate_proj", + "model.layers.55.mlp.experts.73.up_proj", + "model.layers.55.mlp.experts.74.down_proj", + "model.layers.55.mlp.experts.74.gate_proj", + "model.layers.55.mlp.experts.74.up_proj", + "model.layers.55.mlp.experts.75.down_proj", + "model.layers.55.mlp.experts.75.gate_proj", + "model.layers.55.mlp.experts.75.up_proj", + "model.layers.55.mlp.experts.76.down_proj", + "model.layers.55.mlp.experts.76.gate_proj", + "model.layers.55.mlp.experts.76.up_proj", + "model.layers.55.mlp.experts.77.down_proj", + "model.layers.55.mlp.experts.77.gate_proj", + "model.layers.55.mlp.experts.77.up_proj", + "model.layers.55.mlp.experts.78.down_proj", + "model.layers.55.mlp.experts.78.gate_proj", + "model.layers.55.mlp.experts.78.up_proj", + "model.layers.55.mlp.experts.79.down_proj", + "model.layers.55.mlp.experts.79.gate_proj", + "model.layers.55.mlp.experts.79.up_proj", + "model.layers.55.mlp.experts.8.down_proj", + "model.layers.55.mlp.experts.8.gate_proj", + "model.layers.55.mlp.experts.8.up_proj", + "model.layers.55.mlp.experts.80.down_proj", + "model.layers.55.mlp.experts.80.gate_proj", + "model.layers.55.mlp.experts.80.up_proj", + "model.layers.55.mlp.experts.81.down_proj", + "model.layers.55.mlp.experts.81.gate_proj", + "model.layers.55.mlp.experts.81.up_proj", + "model.layers.55.mlp.experts.82.down_proj", + "model.layers.55.mlp.experts.82.gate_proj", + "model.layers.55.mlp.experts.82.up_proj", + "model.layers.55.mlp.experts.83.down_proj", + "model.layers.55.mlp.experts.83.gate_proj", + "model.layers.55.mlp.experts.83.up_proj", + "model.layers.55.mlp.experts.84.down_proj", + "model.layers.55.mlp.experts.84.gate_proj", + "model.layers.55.mlp.experts.84.up_proj", + "model.layers.55.mlp.experts.85.down_proj", + "model.layers.55.mlp.experts.85.gate_proj", + "model.layers.55.mlp.experts.85.up_proj", + "model.layers.55.mlp.experts.86.down_proj", + "model.layers.55.mlp.experts.86.gate_proj", + "model.layers.55.mlp.experts.86.up_proj", + "model.layers.55.mlp.experts.87.down_proj", + "model.layers.55.mlp.experts.87.gate_proj", + "model.layers.55.mlp.experts.87.up_proj", + "model.layers.55.mlp.experts.88.down_proj", + "model.layers.55.mlp.experts.88.gate_proj", + "model.layers.55.mlp.experts.88.up_proj", + "model.layers.55.mlp.experts.89.down_proj", + "model.layers.55.mlp.experts.89.gate_proj", + "model.layers.55.mlp.experts.89.up_proj", + "model.layers.55.mlp.experts.9.down_proj", + "model.layers.55.mlp.experts.9.gate_proj", + "model.layers.55.mlp.experts.9.up_proj", + "model.layers.55.mlp.experts.90.down_proj", + "model.layers.55.mlp.experts.90.gate_proj", + "model.layers.55.mlp.experts.90.up_proj", + "model.layers.55.mlp.experts.91.down_proj", + "model.layers.55.mlp.experts.91.gate_proj", + "model.layers.55.mlp.experts.91.up_proj", + "model.layers.55.mlp.experts.92.down_proj", + "model.layers.55.mlp.experts.92.gate_proj", + "model.layers.55.mlp.experts.92.up_proj", + "model.layers.55.mlp.experts.93.down_proj", + "model.layers.55.mlp.experts.93.gate_proj", + "model.layers.55.mlp.experts.93.up_proj", + "model.layers.55.mlp.experts.94.down_proj", + "model.layers.55.mlp.experts.94.gate_proj", + "model.layers.55.mlp.experts.94.up_proj", + "model.layers.55.mlp.experts.95.down_proj", + "model.layers.55.mlp.experts.95.gate_proj", + "model.layers.55.mlp.experts.95.up_proj", + "model.layers.55.mlp.experts.96.down_proj", + "model.layers.55.mlp.experts.96.gate_proj", + "model.layers.55.mlp.experts.96.up_proj", + "model.layers.55.mlp.experts.97.down_proj", + "model.layers.55.mlp.experts.97.gate_proj", + "model.layers.55.mlp.experts.97.up_proj", + "model.layers.55.mlp.experts.98.down_proj", + "model.layers.55.mlp.experts.98.gate_proj", + "model.layers.55.mlp.experts.98.up_proj", + "model.layers.55.mlp.experts.99.down_proj", + "model.layers.55.mlp.experts.99.gate_proj", + "model.layers.55.mlp.experts.99.up_proj", + "model.layers.56.mlp.experts.0.down_proj", + "model.layers.56.mlp.experts.0.gate_proj", + "model.layers.56.mlp.experts.0.up_proj", + "model.layers.56.mlp.experts.1.down_proj", + "model.layers.56.mlp.experts.1.gate_proj", + "model.layers.56.mlp.experts.1.up_proj", + "model.layers.56.mlp.experts.10.down_proj", + "model.layers.56.mlp.experts.10.gate_proj", + "model.layers.56.mlp.experts.10.up_proj", + "model.layers.56.mlp.experts.100.down_proj", + "model.layers.56.mlp.experts.100.gate_proj", + "model.layers.56.mlp.experts.100.up_proj", + "model.layers.56.mlp.experts.101.down_proj", + "model.layers.56.mlp.experts.101.gate_proj", + "model.layers.56.mlp.experts.101.up_proj", + "model.layers.56.mlp.experts.102.down_proj", + "model.layers.56.mlp.experts.102.gate_proj", + "model.layers.56.mlp.experts.102.up_proj", + "model.layers.56.mlp.experts.103.down_proj", + "model.layers.56.mlp.experts.103.gate_proj", + "model.layers.56.mlp.experts.103.up_proj", + "model.layers.56.mlp.experts.104.down_proj", + "model.layers.56.mlp.experts.104.gate_proj", + "model.layers.56.mlp.experts.104.up_proj", + "model.layers.56.mlp.experts.105.down_proj", + "model.layers.56.mlp.experts.105.gate_proj", + "model.layers.56.mlp.experts.105.up_proj", + "model.layers.56.mlp.experts.106.down_proj", + "model.layers.56.mlp.experts.106.gate_proj", + "model.layers.56.mlp.experts.106.up_proj", + "model.layers.56.mlp.experts.107.down_proj", + "model.layers.56.mlp.experts.107.gate_proj", + "model.layers.56.mlp.experts.107.up_proj", + "model.layers.56.mlp.experts.108.down_proj", + "model.layers.56.mlp.experts.108.gate_proj", + "model.layers.56.mlp.experts.108.up_proj", + "model.layers.56.mlp.experts.109.down_proj", + "model.layers.56.mlp.experts.109.gate_proj", + "model.layers.56.mlp.experts.109.up_proj", + "model.layers.56.mlp.experts.11.down_proj", + "model.layers.56.mlp.experts.11.gate_proj", + "model.layers.56.mlp.experts.11.up_proj", + "model.layers.56.mlp.experts.110.down_proj", + "model.layers.56.mlp.experts.110.gate_proj", + "model.layers.56.mlp.experts.110.up_proj", + "model.layers.56.mlp.experts.111.down_proj", + "model.layers.56.mlp.experts.111.gate_proj", + "model.layers.56.mlp.experts.111.up_proj", + "model.layers.56.mlp.experts.112.down_proj", + "model.layers.56.mlp.experts.112.gate_proj", + "model.layers.56.mlp.experts.112.up_proj", + "model.layers.56.mlp.experts.113.down_proj", + "model.layers.56.mlp.experts.113.gate_proj", + "model.layers.56.mlp.experts.113.up_proj", + "model.layers.56.mlp.experts.114.down_proj", + "model.layers.56.mlp.experts.114.gate_proj", + "model.layers.56.mlp.experts.114.up_proj", + "model.layers.56.mlp.experts.115.down_proj", + "model.layers.56.mlp.experts.115.gate_proj", + "model.layers.56.mlp.experts.115.up_proj", + "model.layers.56.mlp.experts.116.down_proj", + "model.layers.56.mlp.experts.116.gate_proj", + "model.layers.56.mlp.experts.116.up_proj", + "model.layers.56.mlp.experts.117.down_proj", + "model.layers.56.mlp.experts.117.gate_proj", + "model.layers.56.mlp.experts.117.up_proj", + "model.layers.56.mlp.experts.118.down_proj", + "model.layers.56.mlp.experts.118.gate_proj", + "model.layers.56.mlp.experts.118.up_proj", + "model.layers.56.mlp.experts.119.down_proj", + "model.layers.56.mlp.experts.119.gate_proj", + "model.layers.56.mlp.experts.119.up_proj", + "model.layers.56.mlp.experts.12.down_proj", + "model.layers.56.mlp.experts.12.gate_proj", + "model.layers.56.mlp.experts.12.up_proj", + "model.layers.56.mlp.experts.120.down_proj", + "model.layers.56.mlp.experts.120.gate_proj", + "model.layers.56.mlp.experts.120.up_proj", + "model.layers.56.mlp.experts.121.down_proj", + "model.layers.56.mlp.experts.121.gate_proj", + "model.layers.56.mlp.experts.121.up_proj", + "model.layers.56.mlp.experts.122.down_proj", + "model.layers.56.mlp.experts.122.gate_proj", + "model.layers.56.mlp.experts.122.up_proj", + "model.layers.56.mlp.experts.123.down_proj", + "model.layers.56.mlp.experts.123.gate_proj", + "model.layers.56.mlp.experts.123.up_proj", + "model.layers.56.mlp.experts.124.down_proj", + "model.layers.56.mlp.experts.124.gate_proj", + "model.layers.56.mlp.experts.124.up_proj", + "model.layers.56.mlp.experts.125.down_proj", + "model.layers.56.mlp.experts.125.gate_proj", + "model.layers.56.mlp.experts.125.up_proj", + "model.layers.56.mlp.experts.126.down_proj", + "model.layers.56.mlp.experts.126.gate_proj", + "model.layers.56.mlp.experts.126.up_proj", + "model.layers.56.mlp.experts.127.down_proj", + "model.layers.56.mlp.experts.127.gate_proj", + "model.layers.56.mlp.experts.127.up_proj", + "model.layers.56.mlp.experts.128.down_proj", + "model.layers.56.mlp.experts.128.gate_proj", + "model.layers.56.mlp.experts.128.up_proj", + "model.layers.56.mlp.experts.129.down_proj", + "model.layers.56.mlp.experts.129.gate_proj", + "model.layers.56.mlp.experts.129.up_proj", + "model.layers.56.mlp.experts.13.down_proj", + "model.layers.56.mlp.experts.13.gate_proj", + "model.layers.56.mlp.experts.13.up_proj", + "model.layers.56.mlp.experts.130.down_proj", + "model.layers.56.mlp.experts.130.gate_proj", + "model.layers.56.mlp.experts.130.up_proj", + "model.layers.56.mlp.experts.131.down_proj", + "model.layers.56.mlp.experts.131.gate_proj", + "model.layers.56.mlp.experts.131.up_proj", + "model.layers.56.mlp.experts.132.down_proj", + "model.layers.56.mlp.experts.132.gate_proj", + "model.layers.56.mlp.experts.132.up_proj", + "model.layers.56.mlp.experts.133.down_proj", + "model.layers.56.mlp.experts.133.gate_proj", + "model.layers.56.mlp.experts.133.up_proj", + "model.layers.56.mlp.experts.134.down_proj", + "model.layers.56.mlp.experts.134.gate_proj", + "model.layers.56.mlp.experts.134.up_proj", + "model.layers.56.mlp.experts.135.down_proj", + "model.layers.56.mlp.experts.135.gate_proj", + "model.layers.56.mlp.experts.135.up_proj", + "model.layers.56.mlp.experts.136.down_proj", + "model.layers.56.mlp.experts.136.gate_proj", + "model.layers.56.mlp.experts.136.up_proj", + "model.layers.56.mlp.experts.137.down_proj", + "model.layers.56.mlp.experts.137.gate_proj", + "model.layers.56.mlp.experts.137.up_proj", + "model.layers.56.mlp.experts.138.down_proj", + "model.layers.56.mlp.experts.138.gate_proj", + "model.layers.56.mlp.experts.138.up_proj", + "model.layers.56.mlp.experts.139.down_proj", + "model.layers.56.mlp.experts.139.gate_proj", + "model.layers.56.mlp.experts.139.up_proj", + "model.layers.56.mlp.experts.14.down_proj", + "model.layers.56.mlp.experts.14.gate_proj", + "model.layers.56.mlp.experts.14.up_proj", + "model.layers.56.mlp.experts.140.down_proj", + "model.layers.56.mlp.experts.140.gate_proj", + "model.layers.56.mlp.experts.140.up_proj", + "model.layers.56.mlp.experts.141.down_proj", + "model.layers.56.mlp.experts.141.gate_proj", + "model.layers.56.mlp.experts.141.up_proj", + "model.layers.56.mlp.experts.142.down_proj", + "model.layers.56.mlp.experts.142.gate_proj", + "model.layers.56.mlp.experts.142.up_proj", + "model.layers.56.mlp.experts.143.down_proj", + "model.layers.56.mlp.experts.143.gate_proj", + "model.layers.56.mlp.experts.143.up_proj", + "model.layers.56.mlp.experts.144.down_proj", + "model.layers.56.mlp.experts.144.gate_proj", + "model.layers.56.mlp.experts.144.up_proj", + "model.layers.56.mlp.experts.145.down_proj", + "model.layers.56.mlp.experts.145.gate_proj", + "model.layers.56.mlp.experts.145.up_proj", + "model.layers.56.mlp.experts.146.down_proj", + "model.layers.56.mlp.experts.146.gate_proj", + "model.layers.56.mlp.experts.146.up_proj", + "model.layers.56.mlp.experts.147.down_proj", + "model.layers.56.mlp.experts.147.gate_proj", + "model.layers.56.mlp.experts.147.up_proj", + "model.layers.56.mlp.experts.148.down_proj", + "model.layers.56.mlp.experts.148.gate_proj", + "model.layers.56.mlp.experts.148.up_proj", + "model.layers.56.mlp.experts.149.down_proj", + "model.layers.56.mlp.experts.149.gate_proj", + "model.layers.56.mlp.experts.149.up_proj", + "model.layers.56.mlp.experts.15.down_proj", + "model.layers.56.mlp.experts.15.gate_proj", + "model.layers.56.mlp.experts.15.up_proj", + "model.layers.56.mlp.experts.150.down_proj", + "model.layers.56.mlp.experts.150.gate_proj", + "model.layers.56.mlp.experts.150.up_proj", + "model.layers.56.mlp.experts.151.down_proj", + "model.layers.56.mlp.experts.151.gate_proj", + "model.layers.56.mlp.experts.151.up_proj", + "model.layers.56.mlp.experts.152.down_proj", + "model.layers.56.mlp.experts.152.gate_proj", + "model.layers.56.mlp.experts.152.up_proj", + "model.layers.56.mlp.experts.153.down_proj", + "model.layers.56.mlp.experts.153.gate_proj", + "model.layers.56.mlp.experts.153.up_proj", + "model.layers.56.mlp.experts.154.down_proj", + "model.layers.56.mlp.experts.154.gate_proj", + "model.layers.56.mlp.experts.154.up_proj", + "model.layers.56.mlp.experts.155.down_proj", + "model.layers.56.mlp.experts.155.gate_proj", + "model.layers.56.mlp.experts.155.up_proj", + "model.layers.56.mlp.experts.156.down_proj", + "model.layers.56.mlp.experts.156.gate_proj", + "model.layers.56.mlp.experts.156.up_proj", + "model.layers.56.mlp.experts.157.down_proj", + "model.layers.56.mlp.experts.157.gate_proj", + "model.layers.56.mlp.experts.157.up_proj", + "model.layers.56.mlp.experts.158.down_proj", + "model.layers.56.mlp.experts.158.gate_proj", + "model.layers.56.mlp.experts.158.up_proj", + "model.layers.56.mlp.experts.159.down_proj", + "model.layers.56.mlp.experts.159.gate_proj", + "model.layers.56.mlp.experts.159.up_proj", + "model.layers.56.mlp.experts.16.down_proj", + "model.layers.56.mlp.experts.16.gate_proj", + "model.layers.56.mlp.experts.16.up_proj", + "model.layers.56.mlp.experts.160.down_proj", + "model.layers.56.mlp.experts.160.gate_proj", + "model.layers.56.mlp.experts.160.up_proj", + "model.layers.56.mlp.experts.161.down_proj", + "model.layers.56.mlp.experts.161.gate_proj", + "model.layers.56.mlp.experts.161.up_proj", + "model.layers.56.mlp.experts.162.down_proj", + "model.layers.56.mlp.experts.162.gate_proj", + "model.layers.56.mlp.experts.162.up_proj", + "model.layers.56.mlp.experts.163.down_proj", + "model.layers.56.mlp.experts.163.gate_proj", + "model.layers.56.mlp.experts.163.up_proj", + "model.layers.56.mlp.experts.164.down_proj", + "model.layers.56.mlp.experts.164.gate_proj", + "model.layers.56.mlp.experts.164.up_proj", + "model.layers.56.mlp.experts.165.down_proj", + "model.layers.56.mlp.experts.165.gate_proj", + "model.layers.56.mlp.experts.165.up_proj", + "model.layers.56.mlp.experts.166.down_proj", + "model.layers.56.mlp.experts.166.gate_proj", + "model.layers.56.mlp.experts.166.up_proj", + "model.layers.56.mlp.experts.167.down_proj", + "model.layers.56.mlp.experts.167.gate_proj", + "model.layers.56.mlp.experts.167.up_proj", + "model.layers.56.mlp.experts.168.down_proj", + "model.layers.56.mlp.experts.168.gate_proj", + "model.layers.56.mlp.experts.168.up_proj", + "model.layers.56.mlp.experts.169.down_proj", + "model.layers.56.mlp.experts.169.gate_proj", + "model.layers.56.mlp.experts.169.up_proj", + "model.layers.56.mlp.experts.17.down_proj", + "model.layers.56.mlp.experts.17.gate_proj", + "model.layers.56.mlp.experts.17.up_proj", + "model.layers.56.mlp.experts.170.down_proj", + "model.layers.56.mlp.experts.170.gate_proj", + "model.layers.56.mlp.experts.170.up_proj", + "model.layers.56.mlp.experts.171.down_proj", + "model.layers.56.mlp.experts.171.gate_proj", + "model.layers.56.mlp.experts.171.up_proj", + "model.layers.56.mlp.experts.172.down_proj", + "model.layers.56.mlp.experts.172.gate_proj", + "model.layers.56.mlp.experts.172.up_proj", + "model.layers.56.mlp.experts.173.down_proj", + "model.layers.56.mlp.experts.173.gate_proj", + "model.layers.56.mlp.experts.173.up_proj", + "model.layers.56.mlp.experts.174.down_proj", + "model.layers.56.mlp.experts.174.gate_proj", + "model.layers.56.mlp.experts.174.up_proj", + "model.layers.56.mlp.experts.175.down_proj", + "model.layers.56.mlp.experts.175.gate_proj", + "model.layers.56.mlp.experts.175.up_proj", + "model.layers.56.mlp.experts.176.down_proj", + "model.layers.56.mlp.experts.176.gate_proj", + "model.layers.56.mlp.experts.176.up_proj", + "model.layers.56.mlp.experts.177.down_proj", + "model.layers.56.mlp.experts.177.gate_proj", + "model.layers.56.mlp.experts.177.up_proj", + "model.layers.56.mlp.experts.178.down_proj", + "model.layers.56.mlp.experts.178.gate_proj", + "model.layers.56.mlp.experts.178.up_proj", + "model.layers.56.mlp.experts.179.down_proj", + "model.layers.56.mlp.experts.179.gate_proj", + "model.layers.56.mlp.experts.179.up_proj", + "model.layers.56.mlp.experts.18.down_proj", + "model.layers.56.mlp.experts.18.gate_proj", + "model.layers.56.mlp.experts.18.up_proj", + "model.layers.56.mlp.experts.180.down_proj", + "model.layers.56.mlp.experts.180.gate_proj", + "model.layers.56.mlp.experts.180.up_proj", + "model.layers.56.mlp.experts.181.down_proj", + "model.layers.56.mlp.experts.181.gate_proj", + "model.layers.56.mlp.experts.181.up_proj", + "model.layers.56.mlp.experts.182.down_proj", + "model.layers.56.mlp.experts.182.gate_proj", + "model.layers.56.mlp.experts.182.up_proj", + "model.layers.56.mlp.experts.183.down_proj", + "model.layers.56.mlp.experts.183.gate_proj", + "model.layers.56.mlp.experts.183.up_proj", + "model.layers.56.mlp.experts.184.down_proj", + "model.layers.56.mlp.experts.184.gate_proj", + "model.layers.56.mlp.experts.184.up_proj", + "model.layers.56.mlp.experts.185.down_proj", + "model.layers.56.mlp.experts.185.gate_proj", + "model.layers.56.mlp.experts.185.up_proj", + "model.layers.56.mlp.experts.186.down_proj", + "model.layers.56.mlp.experts.186.gate_proj", + "model.layers.56.mlp.experts.186.up_proj", + "model.layers.56.mlp.experts.187.down_proj", + "model.layers.56.mlp.experts.187.gate_proj", + "model.layers.56.mlp.experts.187.up_proj", + "model.layers.56.mlp.experts.188.down_proj", + "model.layers.56.mlp.experts.188.gate_proj", + "model.layers.56.mlp.experts.188.up_proj", + "model.layers.56.mlp.experts.189.down_proj", + "model.layers.56.mlp.experts.189.gate_proj", + "model.layers.56.mlp.experts.189.up_proj", + "model.layers.56.mlp.experts.19.down_proj", + "model.layers.56.mlp.experts.19.gate_proj", + "model.layers.56.mlp.experts.19.up_proj", + "model.layers.56.mlp.experts.190.down_proj", + "model.layers.56.mlp.experts.190.gate_proj", + "model.layers.56.mlp.experts.190.up_proj", + "model.layers.56.mlp.experts.191.down_proj", + "model.layers.56.mlp.experts.191.gate_proj", + "model.layers.56.mlp.experts.191.up_proj", + "model.layers.56.mlp.experts.192.down_proj", + "model.layers.56.mlp.experts.192.gate_proj", + "model.layers.56.mlp.experts.192.up_proj", + "model.layers.56.mlp.experts.193.down_proj", + "model.layers.56.mlp.experts.193.gate_proj", + "model.layers.56.mlp.experts.193.up_proj", + "model.layers.56.mlp.experts.194.down_proj", + "model.layers.56.mlp.experts.194.gate_proj", + "model.layers.56.mlp.experts.194.up_proj", + "model.layers.56.mlp.experts.195.down_proj", + "model.layers.56.mlp.experts.195.gate_proj", + "model.layers.56.mlp.experts.195.up_proj", + "model.layers.56.mlp.experts.196.down_proj", + "model.layers.56.mlp.experts.196.gate_proj", + "model.layers.56.mlp.experts.196.up_proj", + "model.layers.56.mlp.experts.197.down_proj", + "model.layers.56.mlp.experts.197.gate_proj", + "model.layers.56.mlp.experts.197.up_proj", + "model.layers.56.mlp.experts.198.down_proj", + "model.layers.56.mlp.experts.198.gate_proj", + "model.layers.56.mlp.experts.198.up_proj", + "model.layers.56.mlp.experts.199.down_proj", + "model.layers.56.mlp.experts.199.gate_proj", + "model.layers.56.mlp.experts.199.up_proj", + "model.layers.56.mlp.experts.2.down_proj", + "model.layers.56.mlp.experts.2.gate_proj", + "model.layers.56.mlp.experts.2.up_proj", + "model.layers.56.mlp.experts.20.down_proj", + "model.layers.56.mlp.experts.20.gate_proj", + "model.layers.56.mlp.experts.20.up_proj", + "model.layers.56.mlp.experts.200.down_proj", + "model.layers.56.mlp.experts.200.gate_proj", + "model.layers.56.mlp.experts.200.up_proj", + "model.layers.56.mlp.experts.201.down_proj", + "model.layers.56.mlp.experts.201.gate_proj", + "model.layers.56.mlp.experts.201.up_proj", + "model.layers.56.mlp.experts.202.down_proj", + "model.layers.56.mlp.experts.202.gate_proj", + "model.layers.56.mlp.experts.202.up_proj", + "model.layers.56.mlp.experts.203.down_proj", + "model.layers.56.mlp.experts.203.gate_proj", + "model.layers.56.mlp.experts.203.up_proj", + "model.layers.56.mlp.experts.204.down_proj", + "model.layers.56.mlp.experts.204.gate_proj", + "model.layers.56.mlp.experts.204.up_proj", + "model.layers.56.mlp.experts.205.down_proj", + "model.layers.56.mlp.experts.205.gate_proj", + "model.layers.56.mlp.experts.205.up_proj", + "model.layers.56.mlp.experts.206.down_proj", + "model.layers.56.mlp.experts.206.gate_proj", + "model.layers.56.mlp.experts.206.up_proj", + "model.layers.56.mlp.experts.207.down_proj", + "model.layers.56.mlp.experts.207.gate_proj", + "model.layers.56.mlp.experts.207.up_proj", + "model.layers.56.mlp.experts.208.down_proj", + "model.layers.56.mlp.experts.208.gate_proj", + "model.layers.56.mlp.experts.208.up_proj", + "model.layers.56.mlp.experts.209.down_proj", + "model.layers.56.mlp.experts.209.gate_proj", + "model.layers.56.mlp.experts.209.up_proj", + "model.layers.56.mlp.experts.21.down_proj", + "model.layers.56.mlp.experts.21.gate_proj", + "model.layers.56.mlp.experts.21.up_proj", + "model.layers.56.mlp.experts.210.down_proj", + "model.layers.56.mlp.experts.210.gate_proj", + "model.layers.56.mlp.experts.210.up_proj", + "model.layers.56.mlp.experts.211.down_proj", + "model.layers.56.mlp.experts.211.gate_proj", + "model.layers.56.mlp.experts.211.up_proj", + "model.layers.56.mlp.experts.212.down_proj", + "model.layers.56.mlp.experts.212.gate_proj", + "model.layers.56.mlp.experts.212.up_proj", + "model.layers.56.mlp.experts.213.down_proj", + "model.layers.56.mlp.experts.213.gate_proj", + "model.layers.56.mlp.experts.213.up_proj", + "model.layers.56.mlp.experts.214.down_proj", + "model.layers.56.mlp.experts.214.gate_proj", + "model.layers.56.mlp.experts.214.up_proj", + "model.layers.56.mlp.experts.215.down_proj", + "model.layers.56.mlp.experts.215.gate_proj", + "model.layers.56.mlp.experts.215.up_proj", + "model.layers.56.mlp.experts.216.down_proj", + "model.layers.56.mlp.experts.216.gate_proj", + "model.layers.56.mlp.experts.216.up_proj", + "model.layers.56.mlp.experts.217.down_proj", + "model.layers.56.mlp.experts.217.gate_proj", + "model.layers.56.mlp.experts.217.up_proj", + "model.layers.56.mlp.experts.218.down_proj", + "model.layers.56.mlp.experts.218.gate_proj", + "model.layers.56.mlp.experts.218.up_proj", + "model.layers.56.mlp.experts.219.down_proj", + "model.layers.56.mlp.experts.219.gate_proj", + "model.layers.56.mlp.experts.219.up_proj", + "model.layers.56.mlp.experts.22.down_proj", + "model.layers.56.mlp.experts.22.gate_proj", + "model.layers.56.mlp.experts.22.up_proj", + "model.layers.56.mlp.experts.220.down_proj", + "model.layers.56.mlp.experts.220.gate_proj", + "model.layers.56.mlp.experts.220.up_proj", + "model.layers.56.mlp.experts.221.down_proj", + "model.layers.56.mlp.experts.221.gate_proj", + "model.layers.56.mlp.experts.221.up_proj", + "model.layers.56.mlp.experts.222.down_proj", + "model.layers.56.mlp.experts.222.gate_proj", + "model.layers.56.mlp.experts.222.up_proj", + "model.layers.56.mlp.experts.223.down_proj", + "model.layers.56.mlp.experts.223.gate_proj", + "model.layers.56.mlp.experts.223.up_proj", + "model.layers.56.mlp.experts.224.down_proj", + "model.layers.56.mlp.experts.224.gate_proj", + "model.layers.56.mlp.experts.224.up_proj", + "model.layers.56.mlp.experts.225.down_proj", + "model.layers.56.mlp.experts.225.gate_proj", + "model.layers.56.mlp.experts.225.up_proj", + "model.layers.56.mlp.experts.226.down_proj", + "model.layers.56.mlp.experts.226.gate_proj", + "model.layers.56.mlp.experts.226.up_proj", + "model.layers.56.mlp.experts.227.down_proj", + "model.layers.56.mlp.experts.227.gate_proj", + "model.layers.56.mlp.experts.227.up_proj", + "model.layers.56.mlp.experts.228.down_proj", + "model.layers.56.mlp.experts.228.gate_proj", + "model.layers.56.mlp.experts.228.up_proj", + "model.layers.56.mlp.experts.229.down_proj", + "model.layers.56.mlp.experts.229.gate_proj", + "model.layers.56.mlp.experts.229.up_proj", + "model.layers.56.mlp.experts.23.down_proj", + "model.layers.56.mlp.experts.23.gate_proj", + "model.layers.56.mlp.experts.23.up_proj", + "model.layers.56.mlp.experts.230.down_proj", + "model.layers.56.mlp.experts.230.gate_proj", + "model.layers.56.mlp.experts.230.up_proj", + "model.layers.56.mlp.experts.231.down_proj", + "model.layers.56.mlp.experts.231.gate_proj", + "model.layers.56.mlp.experts.231.up_proj", + "model.layers.56.mlp.experts.232.down_proj", + "model.layers.56.mlp.experts.232.gate_proj", + "model.layers.56.mlp.experts.232.up_proj", + "model.layers.56.mlp.experts.233.down_proj", + "model.layers.56.mlp.experts.233.gate_proj", + "model.layers.56.mlp.experts.233.up_proj", + "model.layers.56.mlp.experts.234.down_proj", + "model.layers.56.mlp.experts.234.gate_proj", + "model.layers.56.mlp.experts.234.up_proj", + "model.layers.56.mlp.experts.235.down_proj", + "model.layers.56.mlp.experts.235.gate_proj", + "model.layers.56.mlp.experts.235.up_proj", + "model.layers.56.mlp.experts.236.down_proj", + "model.layers.56.mlp.experts.236.gate_proj", + "model.layers.56.mlp.experts.236.up_proj", + "model.layers.56.mlp.experts.237.down_proj", + "model.layers.56.mlp.experts.237.gate_proj", + "model.layers.56.mlp.experts.237.up_proj", + "model.layers.56.mlp.experts.238.down_proj", + "model.layers.56.mlp.experts.238.gate_proj", + "model.layers.56.mlp.experts.238.up_proj", + "model.layers.56.mlp.experts.239.down_proj", + "model.layers.56.mlp.experts.239.gate_proj", + "model.layers.56.mlp.experts.239.up_proj", + "model.layers.56.mlp.experts.24.down_proj", + "model.layers.56.mlp.experts.24.gate_proj", + "model.layers.56.mlp.experts.24.up_proj", + "model.layers.56.mlp.experts.240.down_proj", + "model.layers.56.mlp.experts.240.gate_proj", + "model.layers.56.mlp.experts.240.up_proj", + "model.layers.56.mlp.experts.241.down_proj", + "model.layers.56.mlp.experts.241.gate_proj", + "model.layers.56.mlp.experts.241.up_proj", + "model.layers.56.mlp.experts.242.down_proj", + "model.layers.56.mlp.experts.242.gate_proj", + "model.layers.56.mlp.experts.242.up_proj", + "model.layers.56.mlp.experts.243.down_proj", + "model.layers.56.mlp.experts.243.gate_proj", + "model.layers.56.mlp.experts.243.up_proj", + "model.layers.56.mlp.experts.244.down_proj", + "model.layers.56.mlp.experts.244.gate_proj", + "model.layers.56.mlp.experts.244.up_proj", + "model.layers.56.mlp.experts.245.down_proj", + "model.layers.56.mlp.experts.245.gate_proj", + "model.layers.56.mlp.experts.245.up_proj", + "model.layers.56.mlp.experts.246.down_proj", + "model.layers.56.mlp.experts.246.gate_proj", + "model.layers.56.mlp.experts.246.up_proj", + "model.layers.56.mlp.experts.247.down_proj", + "model.layers.56.mlp.experts.247.gate_proj", + "model.layers.56.mlp.experts.247.up_proj", + "model.layers.56.mlp.experts.248.down_proj", + "model.layers.56.mlp.experts.248.gate_proj", + "model.layers.56.mlp.experts.248.up_proj", + "model.layers.56.mlp.experts.249.down_proj", + "model.layers.56.mlp.experts.249.gate_proj", + "model.layers.56.mlp.experts.249.up_proj", + "model.layers.56.mlp.experts.25.down_proj", + "model.layers.56.mlp.experts.25.gate_proj", + "model.layers.56.mlp.experts.25.up_proj", + "model.layers.56.mlp.experts.250.down_proj", + "model.layers.56.mlp.experts.250.gate_proj", + "model.layers.56.mlp.experts.250.up_proj", + "model.layers.56.mlp.experts.251.down_proj", + "model.layers.56.mlp.experts.251.gate_proj", + "model.layers.56.mlp.experts.251.up_proj", + "model.layers.56.mlp.experts.252.down_proj", + "model.layers.56.mlp.experts.252.gate_proj", + "model.layers.56.mlp.experts.252.up_proj", + "model.layers.56.mlp.experts.253.down_proj", + "model.layers.56.mlp.experts.253.gate_proj", + "model.layers.56.mlp.experts.253.up_proj", + "model.layers.56.mlp.experts.254.down_proj", + "model.layers.56.mlp.experts.254.gate_proj", + "model.layers.56.mlp.experts.254.up_proj", + "model.layers.56.mlp.experts.255.down_proj", + "model.layers.56.mlp.experts.255.gate_proj", + "model.layers.56.mlp.experts.255.up_proj", + "model.layers.56.mlp.experts.26.down_proj", + "model.layers.56.mlp.experts.26.gate_proj", + "model.layers.56.mlp.experts.26.up_proj", + "model.layers.56.mlp.experts.27.down_proj", + "model.layers.56.mlp.experts.27.gate_proj", + "model.layers.56.mlp.experts.27.up_proj", + "model.layers.56.mlp.experts.28.down_proj", + "model.layers.56.mlp.experts.28.gate_proj", + "model.layers.56.mlp.experts.28.up_proj", + "model.layers.56.mlp.experts.29.down_proj", + "model.layers.56.mlp.experts.29.gate_proj", + "model.layers.56.mlp.experts.29.up_proj", + "model.layers.56.mlp.experts.3.down_proj", + "model.layers.56.mlp.experts.3.gate_proj", + "model.layers.56.mlp.experts.3.up_proj", + "model.layers.56.mlp.experts.30.down_proj", + "model.layers.56.mlp.experts.30.gate_proj", + "model.layers.56.mlp.experts.30.up_proj", + "model.layers.56.mlp.experts.31.down_proj", + "model.layers.56.mlp.experts.31.gate_proj", + "model.layers.56.mlp.experts.31.up_proj", + "model.layers.56.mlp.experts.32.down_proj", + "model.layers.56.mlp.experts.32.gate_proj", + "model.layers.56.mlp.experts.32.up_proj", + "model.layers.56.mlp.experts.33.down_proj", + "model.layers.56.mlp.experts.33.gate_proj", + "model.layers.56.mlp.experts.33.up_proj", + "model.layers.56.mlp.experts.34.down_proj", + "model.layers.56.mlp.experts.34.gate_proj", + "model.layers.56.mlp.experts.34.up_proj", + "model.layers.56.mlp.experts.35.down_proj", + "model.layers.56.mlp.experts.35.gate_proj", + "model.layers.56.mlp.experts.35.up_proj", + "model.layers.56.mlp.experts.36.down_proj", + "model.layers.56.mlp.experts.36.gate_proj", + "model.layers.56.mlp.experts.36.up_proj", + "model.layers.56.mlp.experts.37.down_proj", + "model.layers.56.mlp.experts.37.gate_proj", + "model.layers.56.mlp.experts.37.up_proj", + "model.layers.56.mlp.experts.38.down_proj", + "model.layers.56.mlp.experts.38.gate_proj", + "model.layers.56.mlp.experts.38.up_proj", + "model.layers.56.mlp.experts.39.down_proj", + "model.layers.56.mlp.experts.39.gate_proj", + "model.layers.56.mlp.experts.39.up_proj", + "model.layers.56.mlp.experts.4.down_proj", + "model.layers.56.mlp.experts.4.gate_proj", + "model.layers.56.mlp.experts.4.up_proj", + "model.layers.56.mlp.experts.40.down_proj", + "model.layers.56.mlp.experts.40.gate_proj", + "model.layers.56.mlp.experts.40.up_proj", + "model.layers.56.mlp.experts.41.down_proj", + "model.layers.56.mlp.experts.41.gate_proj", + "model.layers.56.mlp.experts.41.up_proj", + "model.layers.56.mlp.experts.42.down_proj", + "model.layers.56.mlp.experts.42.gate_proj", + "model.layers.56.mlp.experts.42.up_proj", + "model.layers.56.mlp.experts.43.down_proj", + "model.layers.56.mlp.experts.43.gate_proj", + "model.layers.56.mlp.experts.43.up_proj", + "model.layers.56.mlp.experts.44.down_proj", + "model.layers.56.mlp.experts.44.gate_proj", + "model.layers.56.mlp.experts.44.up_proj", + "model.layers.56.mlp.experts.45.down_proj", + "model.layers.56.mlp.experts.45.gate_proj", + "model.layers.56.mlp.experts.45.up_proj", + "model.layers.56.mlp.experts.46.down_proj", + "model.layers.56.mlp.experts.46.gate_proj", + "model.layers.56.mlp.experts.46.up_proj", + "model.layers.56.mlp.experts.47.down_proj", + "model.layers.56.mlp.experts.47.gate_proj", + "model.layers.56.mlp.experts.47.up_proj", + "model.layers.56.mlp.experts.48.down_proj", + "model.layers.56.mlp.experts.48.gate_proj", + "model.layers.56.mlp.experts.48.up_proj", + "model.layers.56.mlp.experts.49.down_proj", + "model.layers.56.mlp.experts.49.gate_proj", + "model.layers.56.mlp.experts.49.up_proj", + "model.layers.56.mlp.experts.5.down_proj", + "model.layers.56.mlp.experts.5.gate_proj", + "model.layers.56.mlp.experts.5.up_proj", + "model.layers.56.mlp.experts.50.down_proj", + "model.layers.56.mlp.experts.50.gate_proj", + "model.layers.56.mlp.experts.50.up_proj", + "model.layers.56.mlp.experts.51.down_proj", + "model.layers.56.mlp.experts.51.gate_proj", + "model.layers.56.mlp.experts.51.up_proj", + "model.layers.56.mlp.experts.52.down_proj", + "model.layers.56.mlp.experts.52.gate_proj", + "model.layers.56.mlp.experts.52.up_proj", + "model.layers.56.mlp.experts.53.down_proj", + "model.layers.56.mlp.experts.53.gate_proj", + "model.layers.56.mlp.experts.53.up_proj", + "model.layers.56.mlp.experts.54.down_proj", + "model.layers.56.mlp.experts.54.gate_proj", + "model.layers.56.mlp.experts.54.up_proj", + "model.layers.56.mlp.experts.55.down_proj", + "model.layers.56.mlp.experts.55.gate_proj", + "model.layers.56.mlp.experts.55.up_proj", + "model.layers.56.mlp.experts.56.down_proj", + "model.layers.56.mlp.experts.56.gate_proj", + "model.layers.56.mlp.experts.56.up_proj", + "model.layers.56.mlp.experts.57.down_proj", + "model.layers.56.mlp.experts.57.gate_proj", + "model.layers.56.mlp.experts.57.up_proj", + "model.layers.56.mlp.experts.58.down_proj", + "model.layers.56.mlp.experts.58.gate_proj", + "model.layers.56.mlp.experts.58.up_proj", + "model.layers.56.mlp.experts.59.down_proj", + "model.layers.56.mlp.experts.59.gate_proj", + "model.layers.56.mlp.experts.59.up_proj", + "model.layers.56.mlp.experts.6.down_proj", + "model.layers.56.mlp.experts.6.gate_proj", + "model.layers.56.mlp.experts.6.up_proj", + "model.layers.56.mlp.experts.60.down_proj", + "model.layers.56.mlp.experts.60.gate_proj", + "model.layers.56.mlp.experts.60.up_proj", + "model.layers.56.mlp.experts.61.down_proj", + "model.layers.56.mlp.experts.61.gate_proj", + "model.layers.56.mlp.experts.61.up_proj", + "model.layers.56.mlp.experts.62.down_proj", + "model.layers.56.mlp.experts.62.gate_proj", + "model.layers.56.mlp.experts.62.up_proj", + "model.layers.56.mlp.experts.63.down_proj", + "model.layers.56.mlp.experts.63.gate_proj", + "model.layers.56.mlp.experts.63.up_proj", + "model.layers.56.mlp.experts.64.down_proj", + "model.layers.56.mlp.experts.64.gate_proj", + "model.layers.56.mlp.experts.64.up_proj", + "model.layers.56.mlp.experts.65.down_proj", + "model.layers.56.mlp.experts.65.gate_proj", + "model.layers.56.mlp.experts.65.up_proj", + "model.layers.56.mlp.experts.66.down_proj", + "model.layers.56.mlp.experts.66.gate_proj", + "model.layers.56.mlp.experts.66.up_proj", + "model.layers.56.mlp.experts.67.down_proj", + "model.layers.56.mlp.experts.67.gate_proj", + "model.layers.56.mlp.experts.67.up_proj", + "model.layers.56.mlp.experts.68.down_proj", + "model.layers.56.mlp.experts.68.gate_proj", + "model.layers.56.mlp.experts.68.up_proj", + "model.layers.56.mlp.experts.69.down_proj", + "model.layers.56.mlp.experts.69.gate_proj", + "model.layers.56.mlp.experts.69.up_proj", + "model.layers.56.mlp.experts.7.down_proj", + "model.layers.56.mlp.experts.7.gate_proj", + "model.layers.56.mlp.experts.7.up_proj", + "model.layers.56.mlp.experts.70.down_proj", + "model.layers.56.mlp.experts.70.gate_proj", + "model.layers.56.mlp.experts.70.up_proj", + "model.layers.56.mlp.experts.71.down_proj", + "model.layers.56.mlp.experts.71.gate_proj", + "model.layers.56.mlp.experts.71.up_proj", + "model.layers.56.mlp.experts.72.down_proj", + "model.layers.56.mlp.experts.72.gate_proj", + "model.layers.56.mlp.experts.72.up_proj", + "model.layers.56.mlp.experts.73.down_proj", + "model.layers.56.mlp.experts.73.gate_proj", + "model.layers.56.mlp.experts.73.up_proj", + "model.layers.56.mlp.experts.74.down_proj", + "model.layers.56.mlp.experts.74.gate_proj", + "model.layers.56.mlp.experts.74.up_proj", + "model.layers.56.mlp.experts.75.down_proj", + "model.layers.56.mlp.experts.75.gate_proj", + "model.layers.56.mlp.experts.75.up_proj", + "model.layers.56.mlp.experts.76.down_proj", + "model.layers.56.mlp.experts.76.gate_proj", + "model.layers.56.mlp.experts.76.up_proj", + "model.layers.56.mlp.experts.77.down_proj", + "model.layers.56.mlp.experts.77.gate_proj", + "model.layers.56.mlp.experts.77.up_proj", + "model.layers.56.mlp.experts.78.down_proj", + "model.layers.56.mlp.experts.78.gate_proj", + "model.layers.56.mlp.experts.78.up_proj", + "model.layers.56.mlp.experts.79.down_proj", + "model.layers.56.mlp.experts.79.gate_proj", + "model.layers.56.mlp.experts.79.up_proj", + "model.layers.56.mlp.experts.8.down_proj", + "model.layers.56.mlp.experts.8.gate_proj", + "model.layers.56.mlp.experts.8.up_proj", + "model.layers.56.mlp.experts.80.down_proj", + "model.layers.56.mlp.experts.80.gate_proj", + "model.layers.56.mlp.experts.80.up_proj", + "model.layers.56.mlp.experts.81.down_proj", + "model.layers.56.mlp.experts.81.gate_proj", + "model.layers.56.mlp.experts.81.up_proj", + "model.layers.56.mlp.experts.82.down_proj", + "model.layers.56.mlp.experts.82.gate_proj", + "model.layers.56.mlp.experts.82.up_proj", + "model.layers.56.mlp.experts.83.down_proj", + "model.layers.56.mlp.experts.83.gate_proj", + "model.layers.56.mlp.experts.83.up_proj", + "model.layers.56.mlp.experts.84.down_proj", + "model.layers.56.mlp.experts.84.gate_proj", + "model.layers.56.mlp.experts.84.up_proj", + "model.layers.56.mlp.experts.85.down_proj", + "model.layers.56.mlp.experts.85.gate_proj", + "model.layers.56.mlp.experts.85.up_proj", + "model.layers.56.mlp.experts.86.down_proj", + "model.layers.56.mlp.experts.86.gate_proj", + "model.layers.56.mlp.experts.86.up_proj", + "model.layers.56.mlp.experts.87.down_proj", + "model.layers.56.mlp.experts.87.gate_proj", + "model.layers.56.mlp.experts.87.up_proj", + "model.layers.56.mlp.experts.88.down_proj", + "model.layers.56.mlp.experts.88.gate_proj", + "model.layers.56.mlp.experts.88.up_proj", + "model.layers.56.mlp.experts.89.down_proj", + "model.layers.56.mlp.experts.89.gate_proj", + "model.layers.56.mlp.experts.89.up_proj", + "model.layers.56.mlp.experts.9.down_proj", + "model.layers.56.mlp.experts.9.gate_proj", + "model.layers.56.mlp.experts.9.up_proj", + "model.layers.56.mlp.experts.90.down_proj", + "model.layers.56.mlp.experts.90.gate_proj", + "model.layers.56.mlp.experts.90.up_proj", + "model.layers.56.mlp.experts.91.down_proj", + "model.layers.56.mlp.experts.91.gate_proj", + "model.layers.56.mlp.experts.91.up_proj", + "model.layers.56.mlp.experts.92.down_proj", + "model.layers.56.mlp.experts.92.gate_proj", + "model.layers.56.mlp.experts.92.up_proj", + "model.layers.56.mlp.experts.93.down_proj", + "model.layers.56.mlp.experts.93.gate_proj", + "model.layers.56.mlp.experts.93.up_proj", + "model.layers.56.mlp.experts.94.down_proj", + "model.layers.56.mlp.experts.94.gate_proj", + "model.layers.56.mlp.experts.94.up_proj", + "model.layers.56.mlp.experts.95.down_proj", + "model.layers.56.mlp.experts.95.gate_proj", + "model.layers.56.mlp.experts.95.up_proj", + "model.layers.56.mlp.experts.96.down_proj", + "model.layers.56.mlp.experts.96.gate_proj", + "model.layers.56.mlp.experts.96.up_proj", + "model.layers.56.mlp.experts.97.down_proj", + "model.layers.56.mlp.experts.97.gate_proj", + "model.layers.56.mlp.experts.97.up_proj", + "model.layers.56.mlp.experts.98.down_proj", + "model.layers.56.mlp.experts.98.gate_proj", + "model.layers.56.mlp.experts.98.up_proj", + "model.layers.56.mlp.experts.99.down_proj", + "model.layers.56.mlp.experts.99.gate_proj", + "model.layers.56.mlp.experts.99.up_proj", + "model.layers.57.mlp.experts.0.down_proj", + "model.layers.57.mlp.experts.0.gate_proj", + "model.layers.57.mlp.experts.0.up_proj", + "model.layers.57.mlp.experts.1.down_proj", + "model.layers.57.mlp.experts.1.gate_proj", + "model.layers.57.mlp.experts.1.up_proj", + "model.layers.57.mlp.experts.10.down_proj", + "model.layers.57.mlp.experts.10.gate_proj", + "model.layers.57.mlp.experts.10.up_proj", + "model.layers.57.mlp.experts.100.down_proj", + "model.layers.57.mlp.experts.100.gate_proj", + "model.layers.57.mlp.experts.100.up_proj", + "model.layers.57.mlp.experts.101.down_proj", + "model.layers.57.mlp.experts.101.gate_proj", + "model.layers.57.mlp.experts.101.up_proj", + "model.layers.57.mlp.experts.102.down_proj", + "model.layers.57.mlp.experts.102.gate_proj", + "model.layers.57.mlp.experts.102.up_proj", + "model.layers.57.mlp.experts.103.down_proj", + "model.layers.57.mlp.experts.103.gate_proj", + "model.layers.57.mlp.experts.103.up_proj", + "model.layers.57.mlp.experts.104.down_proj", + "model.layers.57.mlp.experts.104.gate_proj", + "model.layers.57.mlp.experts.104.up_proj", + "model.layers.57.mlp.experts.105.down_proj", + "model.layers.57.mlp.experts.105.gate_proj", + "model.layers.57.mlp.experts.105.up_proj", + "model.layers.57.mlp.experts.106.down_proj", + "model.layers.57.mlp.experts.106.gate_proj", + "model.layers.57.mlp.experts.106.up_proj", + "model.layers.57.mlp.experts.107.down_proj", + "model.layers.57.mlp.experts.107.gate_proj", + "model.layers.57.mlp.experts.107.up_proj", + "model.layers.57.mlp.experts.108.down_proj", + "model.layers.57.mlp.experts.108.gate_proj", + "model.layers.57.mlp.experts.108.up_proj", + "model.layers.57.mlp.experts.109.down_proj", + "model.layers.57.mlp.experts.109.gate_proj", + "model.layers.57.mlp.experts.109.up_proj", + "model.layers.57.mlp.experts.11.down_proj", + "model.layers.57.mlp.experts.11.gate_proj", + "model.layers.57.mlp.experts.11.up_proj", + "model.layers.57.mlp.experts.110.down_proj", + "model.layers.57.mlp.experts.110.gate_proj", + "model.layers.57.mlp.experts.110.up_proj", + "model.layers.57.mlp.experts.111.down_proj", + "model.layers.57.mlp.experts.111.gate_proj", + "model.layers.57.mlp.experts.111.up_proj", + "model.layers.57.mlp.experts.112.down_proj", + "model.layers.57.mlp.experts.112.gate_proj", + "model.layers.57.mlp.experts.112.up_proj", + "model.layers.57.mlp.experts.113.down_proj", + "model.layers.57.mlp.experts.113.gate_proj", + "model.layers.57.mlp.experts.113.up_proj", + "model.layers.57.mlp.experts.114.down_proj", + "model.layers.57.mlp.experts.114.gate_proj", + "model.layers.57.mlp.experts.114.up_proj", + "model.layers.57.mlp.experts.115.down_proj", + "model.layers.57.mlp.experts.115.gate_proj", + "model.layers.57.mlp.experts.115.up_proj", + "model.layers.57.mlp.experts.116.down_proj", + "model.layers.57.mlp.experts.116.gate_proj", + "model.layers.57.mlp.experts.116.up_proj", + "model.layers.57.mlp.experts.117.down_proj", + "model.layers.57.mlp.experts.117.gate_proj", + "model.layers.57.mlp.experts.117.up_proj", + "model.layers.57.mlp.experts.118.down_proj", + "model.layers.57.mlp.experts.118.gate_proj", + "model.layers.57.mlp.experts.118.up_proj", + "model.layers.57.mlp.experts.119.down_proj", + "model.layers.57.mlp.experts.119.gate_proj", + "model.layers.57.mlp.experts.119.up_proj", + "model.layers.57.mlp.experts.12.down_proj", + "model.layers.57.mlp.experts.12.gate_proj", + "model.layers.57.mlp.experts.12.up_proj", + "model.layers.57.mlp.experts.120.down_proj", + "model.layers.57.mlp.experts.120.gate_proj", + "model.layers.57.mlp.experts.120.up_proj", + "model.layers.57.mlp.experts.121.down_proj", + "model.layers.57.mlp.experts.121.gate_proj", + "model.layers.57.mlp.experts.121.up_proj", + "model.layers.57.mlp.experts.122.down_proj", + "model.layers.57.mlp.experts.122.gate_proj", + "model.layers.57.mlp.experts.122.up_proj", + "model.layers.57.mlp.experts.123.down_proj", + "model.layers.57.mlp.experts.123.gate_proj", + "model.layers.57.mlp.experts.123.up_proj", + "model.layers.57.mlp.experts.124.down_proj", + "model.layers.57.mlp.experts.124.gate_proj", + "model.layers.57.mlp.experts.124.up_proj", + "model.layers.57.mlp.experts.125.down_proj", + "model.layers.57.mlp.experts.125.gate_proj", + "model.layers.57.mlp.experts.125.up_proj", + "model.layers.57.mlp.experts.126.down_proj", + "model.layers.57.mlp.experts.126.gate_proj", + "model.layers.57.mlp.experts.126.up_proj", + "model.layers.57.mlp.experts.127.down_proj", + "model.layers.57.mlp.experts.127.gate_proj", + "model.layers.57.mlp.experts.127.up_proj", + "model.layers.57.mlp.experts.128.down_proj", + "model.layers.57.mlp.experts.128.gate_proj", + "model.layers.57.mlp.experts.128.up_proj", + "model.layers.57.mlp.experts.129.down_proj", + "model.layers.57.mlp.experts.129.gate_proj", + "model.layers.57.mlp.experts.129.up_proj", + "model.layers.57.mlp.experts.13.down_proj", + "model.layers.57.mlp.experts.13.gate_proj", + "model.layers.57.mlp.experts.13.up_proj", + "model.layers.57.mlp.experts.130.down_proj", + "model.layers.57.mlp.experts.130.gate_proj", + "model.layers.57.mlp.experts.130.up_proj", + "model.layers.57.mlp.experts.131.down_proj", + "model.layers.57.mlp.experts.131.gate_proj", + "model.layers.57.mlp.experts.131.up_proj", + "model.layers.57.mlp.experts.132.down_proj", + "model.layers.57.mlp.experts.132.gate_proj", + "model.layers.57.mlp.experts.132.up_proj", + "model.layers.57.mlp.experts.133.down_proj", + "model.layers.57.mlp.experts.133.gate_proj", + "model.layers.57.mlp.experts.133.up_proj", + "model.layers.57.mlp.experts.134.down_proj", + "model.layers.57.mlp.experts.134.gate_proj", + "model.layers.57.mlp.experts.134.up_proj", + "model.layers.57.mlp.experts.135.down_proj", + "model.layers.57.mlp.experts.135.gate_proj", + "model.layers.57.mlp.experts.135.up_proj", + "model.layers.57.mlp.experts.136.down_proj", + "model.layers.57.mlp.experts.136.gate_proj", + "model.layers.57.mlp.experts.136.up_proj", + "model.layers.57.mlp.experts.137.down_proj", + "model.layers.57.mlp.experts.137.gate_proj", + "model.layers.57.mlp.experts.137.up_proj", + "model.layers.57.mlp.experts.138.down_proj", + "model.layers.57.mlp.experts.138.gate_proj", + "model.layers.57.mlp.experts.138.up_proj", + "model.layers.57.mlp.experts.139.down_proj", + "model.layers.57.mlp.experts.139.gate_proj", + "model.layers.57.mlp.experts.139.up_proj", + "model.layers.57.mlp.experts.14.down_proj", + "model.layers.57.mlp.experts.14.gate_proj", + "model.layers.57.mlp.experts.14.up_proj", + "model.layers.57.mlp.experts.140.down_proj", + "model.layers.57.mlp.experts.140.gate_proj", + "model.layers.57.mlp.experts.140.up_proj", + "model.layers.57.mlp.experts.141.down_proj", + "model.layers.57.mlp.experts.141.gate_proj", + "model.layers.57.mlp.experts.141.up_proj", + "model.layers.57.mlp.experts.142.down_proj", + "model.layers.57.mlp.experts.142.gate_proj", + "model.layers.57.mlp.experts.142.up_proj", + "model.layers.57.mlp.experts.143.down_proj", + "model.layers.57.mlp.experts.143.gate_proj", + "model.layers.57.mlp.experts.143.up_proj", + "model.layers.57.mlp.experts.144.down_proj", + "model.layers.57.mlp.experts.144.gate_proj", + "model.layers.57.mlp.experts.144.up_proj", + "model.layers.57.mlp.experts.145.down_proj", + "model.layers.57.mlp.experts.145.gate_proj", + "model.layers.57.mlp.experts.145.up_proj", + "model.layers.57.mlp.experts.146.down_proj", + "model.layers.57.mlp.experts.146.gate_proj", + "model.layers.57.mlp.experts.146.up_proj", + "model.layers.57.mlp.experts.147.down_proj", + "model.layers.57.mlp.experts.147.gate_proj", + "model.layers.57.mlp.experts.147.up_proj", + "model.layers.57.mlp.experts.148.down_proj", + "model.layers.57.mlp.experts.148.gate_proj", + "model.layers.57.mlp.experts.148.up_proj", + "model.layers.57.mlp.experts.149.down_proj", + "model.layers.57.mlp.experts.149.gate_proj", + "model.layers.57.mlp.experts.149.up_proj", + "model.layers.57.mlp.experts.15.down_proj", + "model.layers.57.mlp.experts.15.gate_proj", + "model.layers.57.mlp.experts.15.up_proj", + "model.layers.57.mlp.experts.150.down_proj", + "model.layers.57.mlp.experts.150.gate_proj", + "model.layers.57.mlp.experts.150.up_proj", + "model.layers.57.mlp.experts.151.down_proj", + "model.layers.57.mlp.experts.151.gate_proj", + "model.layers.57.mlp.experts.151.up_proj", + "model.layers.57.mlp.experts.152.down_proj", + "model.layers.57.mlp.experts.152.gate_proj", + "model.layers.57.mlp.experts.152.up_proj", + "model.layers.57.mlp.experts.153.down_proj", + "model.layers.57.mlp.experts.153.gate_proj", + "model.layers.57.mlp.experts.153.up_proj", + "model.layers.57.mlp.experts.154.down_proj", + "model.layers.57.mlp.experts.154.gate_proj", + "model.layers.57.mlp.experts.154.up_proj", + "model.layers.57.mlp.experts.155.down_proj", + "model.layers.57.mlp.experts.155.gate_proj", + "model.layers.57.mlp.experts.155.up_proj", + "model.layers.57.mlp.experts.156.down_proj", + "model.layers.57.mlp.experts.156.gate_proj", + "model.layers.57.mlp.experts.156.up_proj", + "model.layers.57.mlp.experts.157.down_proj", + "model.layers.57.mlp.experts.157.gate_proj", + "model.layers.57.mlp.experts.157.up_proj", + "model.layers.57.mlp.experts.158.down_proj", + "model.layers.57.mlp.experts.158.gate_proj", + "model.layers.57.mlp.experts.158.up_proj", + "model.layers.57.mlp.experts.159.down_proj", + "model.layers.57.mlp.experts.159.gate_proj", + "model.layers.57.mlp.experts.159.up_proj", + "model.layers.57.mlp.experts.16.down_proj", + "model.layers.57.mlp.experts.16.gate_proj", + "model.layers.57.mlp.experts.16.up_proj", + "model.layers.57.mlp.experts.160.down_proj", + "model.layers.57.mlp.experts.160.gate_proj", + "model.layers.57.mlp.experts.160.up_proj", + "model.layers.57.mlp.experts.161.down_proj", + "model.layers.57.mlp.experts.161.gate_proj", + "model.layers.57.mlp.experts.161.up_proj", + "model.layers.57.mlp.experts.162.down_proj", + "model.layers.57.mlp.experts.162.gate_proj", + "model.layers.57.mlp.experts.162.up_proj", + "model.layers.57.mlp.experts.163.down_proj", + "model.layers.57.mlp.experts.163.gate_proj", + "model.layers.57.mlp.experts.163.up_proj", + "model.layers.57.mlp.experts.164.down_proj", + "model.layers.57.mlp.experts.164.gate_proj", + "model.layers.57.mlp.experts.164.up_proj", + "model.layers.57.mlp.experts.165.down_proj", + "model.layers.57.mlp.experts.165.gate_proj", + "model.layers.57.mlp.experts.165.up_proj", + "model.layers.57.mlp.experts.166.down_proj", + "model.layers.57.mlp.experts.166.gate_proj", + "model.layers.57.mlp.experts.166.up_proj", + "model.layers.57.mlp.experts.167.down_proj", + "model.layers.57.mlp.experts.167.gate_proj", + "model.layers.57.mlp.experts.167.up_proj", + "model.layers.57.mlp.experts.168.down_proj", + "model.layers.57.mlp.experts.168.gate_proj", + "model.layers.57.mlp.experts.168.up_proj", + "model.layers.57.mlp.experts.169.down_proj", + "model.layers.57.mlp.experts.169.gate_proj", + "model.layers.57.mlp.experts.169.up_proj", + "model.layers.57.mlp.experts.17.down_proj", + "model.layers.57.mlp.experts.17.gate_proj", + "model.layers.57.mlp.experts.17.up_proj", + "model.layers.57.mlp.experts.170.down_proj", + "model.layers.57.mlp.experts.170.gate_proj", + "model.layers.57.mlp.experts.170.up_proj", + "model.layers.57.mlp.experts.171.down_proj", + "model.layers.57.mlp.experts.171.gate_proj", + "model.layers.57.mlp.experts.171.up_proj", + "model.layers.57.mlp.experts.172.down_proj", + "model.layers.57.mlp.experts.172.gate_proj", + "model.layers.57.mlp.experts.172.up_proj", + "model.layers.57.mlp.experts.173.down_proj", + "model.layers.57.mlp.experts.173.gate_proj", + "model.layers.57.mlp.experts.173.up_proj", + "model.layers.57.mlp.experts.174.down_proj", + "model.layers.57.mlp.experts.174.gate_proj", + "model.layers.57.mlp.experts.174.up_proj", + "model.layers.57.mlp.experts.175.down_proj", + "model.layers.57.mlp.experts.175.gate_proj", + "model.layers.57.mlp.experts.175.up_proj", + "model.layers.57.mlp.experts.176.down_proj", + "model.layers.57.mlp.experts.176.gate_proj", + "model.layers.57.mlp.experts.176.up_proj", + "model.layers.57.mlp.experts.177.down_proj", + "model.layers.57.mlp.experts.177.gate_proj", + "model.layers.57.mlp.experts.177.up_proj", + "model.layers.57.mlp.experts.178.down_proj", + "model.layers.57.mlp.experts.178.gate_proj", + "model.layers.57.mlp.experts.178.up_proj", + "model.layers.57.mlp.experts.179.down_proj", + "model.layers.57.mlp.experts.179.gate_proj", + "model.layers.57.mlp.experts.179.up_proj", + "model.layers.57.mlp.experts.18.down_proj", + "model.layers.57.mlp.experts.18.gate_proj", + "model.layers.57.mlp.experts.18.up_proj", + "model.layers.57.mlp.experts.180.down_proj", + "model.layers.57.mlp.experts.180.gate_proj", + "model.layers.57.mlp.experts.180.up_proj", + "model.layers.57.mlp.experts.181.down_proj", + "model.layers.57.mlp.experts.181.gate_proj", + "model.layers.57.mlp.experts.181.up_proj", + "model.layers.57.mlp.experts.182.down_proj", + "model.layers.57.mlp.experts.182.gate_proj", + "model.layers.57.mlp.experts.182.up_proj", + "model.layers.57.mlp.experts.183.down_proj", + "model.layers.57.mlp.experts.183.gate_proj", + "model.layers.57.mlp.experts.183.up_proj", + "model.layers.57.mlp.experts.184.down_proj", + "model.layers.57.mlp.experts.184.gate_proj", + "model.layers.57.mlp.experts.184.up_proj", + "model.layers.57.mlp.experts.185.down_proj", + "model.layers.57.mlp.experts.185.gate_proj", + "model.layers.57.mlp.experts.185.up_proj", + "model.layers.57.mlp.experts.186.down_proj", + "model.layers.57.mlp.experts.186.gate_proj", + "model.layers.57.mlp.experts.186.up_proj", + "model.layers.57.mlp.experts.187.down_proj", + "model.layers.57.mlp.experts.187.gate_proj", + "model.layers.57.mlp.experts.187.up_proj", + "model.layers.57.mlp.experts.188.down_proj", + "model.layers.57.mlp.experts.188.gate_proj", + "model.layers.57.mlp.experts.188.up_proj", + "model.layers.57.mlp.experts.189.down_proj", + "model.layers.57.mlp.experts.189.gate_proj", + "model.layers.57.mlp.experts.189.up_proj", + "model.layers.57.mlp.experts.19.down_proj", + "model.layers.57.mlp.experts.19.gate_proj", + "model.layers.57.mlp.experts.19.up_proj", + "model.layers.57.mlp.experts.190.down_proj", + "model.layers.57.mlp.experts.190.gate_proj", + "model.layers.57.mlp.experts.190.up_proj", + "model.layers.57.mlp.experts.191.down_proj", + "model.layers.57.mlp.experts.191.gate_proj", + "model.layers.57.mlp.experts.191.up_proj", + "model.layers.57.mlp.experts.192.down_proj", + "model.layers.57.mlp.experts.192.gate_proj", + "model.layers.57.mlp.experts.192.up_proj", + "model.layers.57.mlp.experts.193.down_proj", + "model.layers.57.mlp.experts.193.gate_proj", + "model.layers.57.mlp.experts.193.up_proj", + "model.layers.57.mlp.experts.194.down_proj", + "model.layers.57.mlp.experts.194.gate_proj", + "model.layers.57.mlp.experts.194.up_proj", + "model.layers.57.mlp.experts.195.down_proj", + "model.layers.57.mlp.experts.195.gate_proj", + "model.layers.57.mlp.experts.195.up_proj", + "model.layers.57.mlp.experts.196.down_proj", + "model.layers.57.mlp.experts.196.gate_proj", + "model.layers.57.mlp.experts.196.up_proj", + "model.layers.57.mlp.experts.197.down_proj", + "model.layers.57.mlp.experts.197.gate_proj", + "model.layers.57.mlp.experts.197.up_proj", + "model.layers.57.mlp.experts.198.down_proj", + "model.layers.57.mlp.experts.198.gate_proj", + "model.layers.57.mlp.experts.198.up_proj", + "model.layers.57.mlp.experts.199.down_proj", + "model.layers.57.mlp.experts.199.gate_proj", + "model.layers.57.mlp.experts.199.up_proj", + "model.layers.57.mlp.experts.2.down_proj", + "model.layers.57.mlp.experts.2.gate_proj", + "model.layers.57.mlp.experts.2.up_proj", + "model.layers.57.mlp.experts.20.down_proj", + "model.layers.57.mlp.experts.20.gate_proj", + "model.layers.57.mlp.experts.20.up_proj", + "model.layers.57.mlp.experts.200.down_proj", + "model.layers.57.mlp.experts.200.gate_proj", + "model.layers.57.mlp.experts.200.up_proj", + "model.layers.57.mlp.experts.201.down_proj", + "model.layers.57.mlp.experts.201.gate_proj", + "model.layers.57.mlp.experts.201.up_proj", + "model.layers.57.mlp.experts.202.down_proj", + "model.layers.57.mlp.experts.202.gate_proj", + "model.layers.57.mlp.experts.202.up_proj", + "model.layers.57.mlp.experts.203.down_proj", + "model.layers.57.mlp.experts.203.gate_proj", + "model.layers.57.mlp.experts.203.up_proj", + "model.layers.57.mlp.experts.204.down_proj", + "model.layers.57.mlp.experts.204.gate_proj", + "model.layers.57.mlp.experts.204.up_proj", + "model.layers.57.mlp.experts.205.down_proj", + "model.layers.57.mlp.experts.205.gate_proj", + "model.layers.57.mlp.experts.205.up_proj", + "model.layers.57.mlp.experts.206.down_proj", + "model.layers.57.mlp.experts.206.gate_proj", + "model.layers.57.mlp.experts.206.up_proj", + "model.layers.57.mlp.experts.207.down_proj", + "model.layers.57.mlp.experts.207.gate_proj", + "model.layers.57.mlp.experts.207.up_proj", + "model.layers.57.mlp.experts.208.down_proj", + "model.layers.57.mlp.experts.208.gate_proj", + "model.layers.57.mlp.experts.208.up_proj", + "model.layers.57.mlp.experts.209.down_proj", + "model.layers.57.mlp.experts.209.gate_proj", + "model.layers.57.mlp.experts.209.up_proj", + "model.layers.57.mlp.experts.21.down_proj", + "model.layers.57.mlp.experts.21.gate_proj", + "model.layers.57.mlp.experts.21.up_proj", + "model.layers.57.mlp.experts.210.down_proj", + "model.layers.57.mlp.experts.210.gate_proj", + "model.layers.57.mlp.experts.210.up_proj", + "model.layers.57.mlp.experts.211.down_proj", + "model.layers.57.mlp.experts.211.gate_proj", + "model.layers.57.mlp.experts.211.up_proj", + "model.layers.57.mlp.experts.212.down_proj", + "model.layers.57.mlp.experts.212.gate_proj", + "model.layers.57.mlp.experts.212.up_proj", + "model.layers.57.mlp.experts.213.down_proj", + "model.layers.57.mlp.experts.213.gate_proj", + "model.layers.57.mlp.experts.213.up_proj", + "model.layers.57.mlp.experts.214.down_proj", + "model.layers.57.mlp.experts.214.gate_proj", + "model.layers.57.mlp.experts.214.up_proj", + "model.layers.57.mlp.experts.215.down_proj", + "model.layers.57.mlp.experts.215.gate_proj", + "model.layers.57.mlp.experts.215.up_proj", + "model.layers.57.mlp.experts.216.down_proj", + "model.layers.57.mlp.experts.216.gate_proj", + "model.layers.57.mlp.experts.216.up_proj", + "model.layers.57.mlp.experts.217.down_proj", + "model.layers.57.mlp.experts.217.gate_proj", + "model.layers.57.mlp.experts.217.up_proj", + "model.layers.57.mlp.experts.218.down_proj", + "model.layers.57.mlp.experts.218.gate_proj", + "model.layers.57.mlp.experts.218.up_proj", + "model.layers.57.mlp.experts.219.down_proj", + "model.layers.57.mlp.experts.219.gate_proj", + "model.layers.57.mlp.experts.219.up_proj", + "model.layers.57.mlp.experts.22.down_proj", + "model.layers.57.mlp.experts.22.gate_proj", + "model.layers.57.mlp.experts.22.up_proj", + "model.layers.57.mlp.experts.220.down_proj", + "model.layers.57.mlp.experts.220.gate_proj", + "model.layers.57.mlp.experts.220.up_proj", + "model.layers.57.mlp.experts.221.down_proj", + "model.layers.57.mlp.experts.221.gate_proj", + "model.layers.57.mlp.experts.221.up_proj", + "model.layers.57.mlp.experts.222.down_proj", + "model.layers.57.mlp.experts.222.gate_proj", + "model.layers.57.mlp.experts.222.up_proj", + "model.layers.57.mlp.experts.223.down_proj", + "model.layers.57.mlp.experts.223.gate_proj", + "model.layers.57.mlp.experts.223.up_proj", + "model.layers.57.mlp.experts.224.down_proj", + "model.layers.57.mlp.experts.224.gate_proj", + "model.layers.57.mlp.experts.224.up_proj", + "model.layers.57.mlp.experts.225.down_proj", + "model.layers.57.mlp.experts.225.gate_proj", + "model.layers.57.mlp.experts.225.up_proj", + "model.layers.57.mlp.experts.226.down_proj", + "model.layers.57.mlp.experts.226.gate_proj", + "model.layers.57.mlp.experts.226.up_proj", + "model.layers.57.mlp.experts.227.down_proj", + "model.layers.57.mlp.experts.227.gate_proj", + "model.layers.57.mlp.experts.227.up_proj", + "model.layers.57.mlp.experts.228.down_proj", + "model.layers.57.mlp.experts.228.gate_proj", + "model.layers.57.mlp.experts.228.up_proj", + "model.layers.57.mlp.experts.229.down_proj", + "model.layers.57.mlp.experts.229.gate_proj", + "model.layers.57.mlp.experts.229.up_proj", + "model.layers.57.mlp.experts.23.down_proj", + "model.layers.57.mlp.experts.23.gate_proj", + "model.layers.57.mlp.experts.23.up_proj", + "model.layers.57.mlp.experts.230.down_proj", + "model.layers.57.mlp.experts.230.gate_proj", + "model.layers.57.mlp.experts.230.up_proj", + "model.layers.57.mlp.experts.231.down_proj", + "model.layers.57.mlp.experts.231.gate_proj", + "model.layers.57.mlp.experts.231.up_proj", + "model.layers.57.mlp.experts.232.down_proj", + "model.layers.57.mlp.experts.232.gate_proj", + "model.layers.57.mlp.experts.232.up_proj", + "model.layers.57.mlp.experts.233.down_proj", + "model.layers.57.mlp.experts.233.gate_proj", + "model.layers.57.mlp.experts.233.up_proj", + "model.layers.57.mlp.experts.234.down_proj", + "model.layers.57.mlp.experts.234.gate_proj", + "model.layers.57.mlp.experts.234.up_proj", + "model.layers.57.mlp.experts.235.down_proj", + "model.layers.57.mlp.experts.235.gate_proj", + "model.layers.57.mlp.experts.235.up_proj", + "model.layers.57.mlp.experts.236.down_proj", + "model.layers.57.mlp.experts.236.gate_proj", + "model.layers.57.mlp.experts.236.up_proj", + "model.layers.57.mlp.experts.237.down_proj", + "model.layers.57.mlp.experts.237.gate_proj", + "model.layers.57.mlp.experts.237.up_proj", + "model.layers.57.mlp.experts.238.down_proj", + "model.layers.57.mlp.experts.238.gate_proj", + "model.layers.57.mlp.experts.238.up_proj", + "model.layers.57.mlp.experts.239.down_proj", + "model.layers.57.mlp.experts.239.gate_proj", + "model.layers.57.mlp.experts.239.up_proj", + "model.layers.57.mlp.experts.24.down_proj", + "model.layers.57.mlp.experts.24.gate_proj", + "model.layers.57.mlp.experts.24.up_proj", + "model.layers.57.mlp.experts.240.down_proj", + "model.layers.57.mlp.experts.240.gate_proj", + "model.layers.57.mlp.experts.240.up_proj", + "model.layers.57.mlp.experts.241.down_proj", + "model.layers.57.mlp.experts.241.gate_proj", + "model.layers.57.mlp.experts.241.up_proj", + "model.layers.57.mlp.experts.242.down_proj", + "model.layers.57.mlp.experts.242.gate_proj", + "model.layers.57.mlp.experts.242.up_proj", + "model.layers.57.mlp.experts.243.down_proj", + "model.layers.57.mlp.experts.243.gate_proj", + "model.layers.57.mlp.experts.243.up_proj", + "model.layers.57.mlp.experts.244.down_proj", + "model.layers.57.mlp.experts.244.gate_proj", + "model.layers.57.mlp.experts.244.up_proj", + "model.layers.57.mlp.experts.245.down_proj", + "model.layers.57.mlp.experts.245.gate_proj", + "model.layers.57.mlp.experts.245.up_proj", + "model.layers.57.mlp.experts.246.down_proj", + "model.layers.57.mlp.experts.246.gate_proj", + "model.layers.57.mlp.experts.246.up_proj", + "model.layers.57.mlp.experts.247.down_proj", + "model.layers.57.mlp.experts.247.gate_proj", + "model.layers.57.mlp.experts.247.up_proj", + "model.layers.57.mlp.experts.248.down_proj", + "model.layers.57.mlp.experts.248.gate_proj", + "model.layers.57.mlp.experts.248.up_proj", + "model.layers.57.mlp.experts.249.down_proj", + "model.layers.57.mlp.experts.249.gate_proj", + "model.layers.57.mlp.experts.249.up_proj", + "model.layers.57.mlp.experts.25.down_proj", + "model.layers.57.mlp.experts.25.gate_proj", + "model.layers.57.mlp.experts.25.up_proj", + "model.layers.57.mlp.experts.250.down_proj", + "model.layers.57.mlp.experts.250.gate_proj", + "model.layers.57.mlp.experts.250.up_proj", + "model.layers.57.mlp.experts.251.down_proj", + "model.layers.57.mlp.experts.251.gate_proj", + "model.layers.57.mlp.experts.251.up_proj", + "model.layers.57.mlp.experts.252.down_proj", + "model.layers.57.mlp.experts.252.gate_proj", + "model.layers.57.mlp.experts.252.up_proj", + "model.layers.57.mlp.experts.253.down_proj", + "model.layers.57.mlp.experts.253.gate_proj", + "model.layers.57.mlp.experts.253.up_proj", + "model.layers.57.mlp.experts.254.down_proj", + "model.layers.57.mlp.experts.254.gate_proj", + "model.layers.57.mlp.experts.254.up_proj", + "model.layers.57.mlp.experts.255.down_proj", + "model.layers.57.mlp.experts.255.gate_proj", + "model.layers.57.mlp.experts.255.up_proj", + "model.layers.57.mlp.experts.26.down_proj", + "model.layers.57.mlp.experts.26.gate_proj", + "model.layers.57.mlp.experts.26.up_proj", + "model.layers.57.mlp.experts.27.down_proj", + "model.layers.57.mlp.experts.27.gate_proj", + "model.layers.57.mlp.experts.27.up_proj", + "model.layers.57.mlp.experts.28.down_proj", + "model.layers.57.mlp.experts.28.gate_proj", + "model.layers.57.mlp.experts.28.up_proj", + "model.layers.57.mlp.experts.29.down_proj", + "model.layers.57.mlp.experts.29.gate_proj", + "model.layers.57.mlp.experts.29.up_proj", + "model.layers.57.mlp.experts.3.down_proj", + "model.layers.57.mlp.experts.3.gate_proj", + "model.layers.57.mlp.experts.3.up_proj", + "model.layers.57.mlp.experts.30.down_proj", + "model.layers.57.mlp.experts.30.gate_proj", + "model.layers.57.mlp.experts.30.up_proj", + "model.layers.57.mlp.experts.31.down_proj", + "model.layers.57.mlp.experts.31.gate_proj", + "model.layers.57.mlp.experts.31.up_proj", + "model.layers.57.mlp.experts.32.down_proj", + "model.layers.57.mlp.experts.32.gate_proj", + "model.layers.57.mlp.experts.32.up_proj", + "model.layers.57.mlp.experts.33.down_proj", + "model.layers.57.mlp.experts.33.gate_proj", + "model.layers.57.mlp.experts.33.up_proj", + "model.layers.57.mlp.experts.34.down_proj", + "model.layers.57.mlp.experts.34.gate_proj", + "model.layers.57.mlp.experts.34.up_proj", + "model.layers.57.mlp.experts.35.down_proj", + "model.layers.57.mlp.experts.35.gate_proj", + "model.layers.57.mlp.experts.35.up_proj", + "model.layers.57.mlp.experts.36.down_proj", + "model.layers.57.mlp.experts.36.gate_proj", + "model.layers.57.mlp.experts.36.up_proj", + "model.layers.57.mlp.experts.37.down_proj", + "model.layers.57.mlp.experts.37.gate_proj", + "model.layers.57.mlp.experts.37.up_proj", + "model.layers.57.mlp.experts.38.down_proj", + "model.layers.57.mlp.experts.38.gate_proj", + "model.layers.57.mlp.experts.38.up_proj", + "model.layers.57.mlp.experts.39.down_proj", + "model.layers.57.mlp.experts.39.gate_proj", + "model.layers.57.mlp.experts.39.up_proj", + "model.layers.57.mlp.experts.4.down_proj", + "model.layers.57.mlp.experts.4.gate_proj", + "model.layers.57.mlp.experts.4.up_proj", + "model.layers.57.mlp.experts.40.down_proj", + "model.layers.57.mlp.experts.40.gate_proj", + "model.layers.57.mlp.experts.40.up_proj", + "model.layers.57.mlp.experts.41.down_proj", + "model.layers.57.mlp.experts.41.gate_proj", + "model.layers.57.mlp.experts.41.up_proj", + "model.layers.57.mlp.experts.42.down_proj", + "model.layers.57.mlp.experts.42.gate_proj", + "model.layers.57.mlp.experts.42.up_proj", + "model.layers.57.mlp.experts.43.down_proj", + "model.layers.57.mlp.experts.43.gate_proj", + "model.layers.57.mlp.experts.43.up_proj", + "model.layers.57.mlp.experts.44.down_proj", + "model.layers.57.mlp.experts.44.gate_proj", + "model.layers.57.mlp.experts.44.up_proj", + "model.layers.57.mlp.experts.45.down_proj", + "model.layers.57.mlp.experts.45.gate_proj", + "model.layers.57.mlp.experts.45.up_proj", + "model.layers.57.mlp.experts.46.down_proj", + "model.layers.57.mlp.experts.46.gate_proj", + "model.layers.57.mlp.experts.46.up_proj", + "model.layers.57.mlp.experts.47.down_proj", + "model.layers.57.mlp.experts.47.gate_proj", + "model.layers.57.mlp.experts.47.up_proj", + "model.layers.57.mlp.experts.48.down_proj", + "model.layers.57.mlp.experts.48.gate_proj", + "model.layers.57.mlp.experts.48.up_proj", + "model.layers.57.mlp.experts.49.down_proj", + "model.layers.57.mlp.experts.49.gate_proj", + "model.layers.57.mlp.experts.49.up_proj", + "model.layers.57.mlp.experts.5.down_proj", + "model.layers.57.mlp.experts.5.gate_proj", + "model.layers.57.mlp.experts.5.up_proj", + "model.layers.57.mlp.experts.50.down_proj", + "model.layers.57.mlp.experts.50.gate_proj", + "model.layers.57.mlp.experts.50.up_proj", + "model.layers.57.mlp.experts.51.down_proj", + "model.layers.57.mlp.experts.51.gate_proj", + "model.layers.57.mlp.experts.51.up_proj", + "model.layers.57.mlp.experts.52.down_proj", + "model.layers.57.mlp.experts.52.gate_proj", + "model.layers.57.mlp.experts.52.up_proj", + "model.layers.57.mlp.experts.53.down_proj", + "model.layers.57.mlp.experts.53.gate_proj", + "model.layers.57.mlp.experts.53.up_proj", + "model.layers.57.mlp.experts.54.down_proj", + "model.layers.57.mlp.experts.54.gate_proj", + "model.layers.57.mlp.experts.54.up_proj", + "model.layers.57.mlp.experts.55.down_proj", + "model.layers.57.mlp.experts.55.gate_proj", + "model.layers.57.mlp.experts.55.up_proj", + "model.layers.57.mlp.experts.56.down_proj", + "model.layers.57.mlp.experts.56.gate_proj", + "model.layers.57.mlp.experts.56.up_proj", + "model.layers.57.mlp.experts.57.down_proj", + "model.layers.57.mlp.experts.57.gate_proj", + "model.layers.57.mlp.experts.57.up_proj", + "model.layers.57.mlp.experts.58.down_proj", + "model.layers.57.mlp.experts.58.gate_proj", + "model.layers.57.mlp.experts.58.up_proj", + "model.layers.57.mlp.experts.59.down_proj", + "model.layers.57.mlp.experts.59.gate_proj", + "model.layers.57.mlp.experts.59.up_proj", + "model.layers.57.mlp.experts.6.down_proj", + "model.layers.57.mlp.experts.6.gate_proj", + "model.layers.57.mlp.experts.6.up_proj", + "model.layers.57.mlp.experts.60.down_proj", + "model.layers.57.mlp.experts.60.gate_proj", + "model.layers.57.mlp.experts.60.up_proj", + "model.layers.57.mlp.experts.61.down_proj", + "model.layers.57.mlp.experts.61.gate_proj", + "model.layers.57.mlp.experts.61.up_proj", + "model.layers.57.mlp.experts.62.down_proj", + "model.layers.57.mlp.experts.62.gate_proj", + "model.layers.57.mlp.experts.62.up_proj", + "model.layers.57.mlp.experts.63.down_proj", + "model.layers.57.mlp.experts.63.gate_proj", + "model.layers.57.mlp.experts.63.up_proj", + "model.layers.57.mlp.experts.64.down_proj", + "model.layers.57.mlp.experts.64.gate_proj", + "model.layers.57.mlp.experts.64.up_proj", + "model.layers.57.mlp.experts.65.down_proj", + "model.layers.57.mlp.experts.65.gate_proj", + "model.layers.57.mlp.experts.65.up_proj", + "model.layers.57.mlp.experts.66.down_proj", + "model.layers.57.mlp.experts.66.gate_proj", + "model.layers.57.mlp.experts.66.up_proj", + "model.layers.57.mlp.experts.67.down_proj", + "model.layers.57.mlp.experts.67.gate_proj", + "model.layers.57.mlp.experts.67.up_proj", + "model.layers.57.mlp.experts.68.down_proj", + "model.layers.57.mlp.experts.68.gate_proj", + "model.layers.57.mlp.experts.68.up_proj", + "model.layers.57.mlp.experts.69.down_proj", + "model.layers.57.mlp.experts.69.gate_proj", + "model.layers.57.mlp.experts.69.up_proj", + "model.layers.57.mlp.experts.7.down_proj", + "model.layers.57.mlp.experts.7.gate_proj", + "model.layers.57.mlp.experts.7.up_proj", + "model.layers.57.mlp.experts.70.down_proj", + "model.layers.57.mlp.experts.70.gate_proj", + "model.layers.57.mlp.experts.70.up_proj", + "model.layers.57.mlp.experts.71.down_proj", + "model.layers.57.mlp.experts.71.gate_proj", + "model.layers.57.mlp.experts.71.up_proj", + "model.layers.57.mlp.experts.72.down_proj", + "model.layers.57.mlp.experts.72.gate_proj", + "model.layers.57.mlp.experts.72.up_proj", + "model.layers.57.mlp.experts.73.down_proj", + "model.layers.57.mlp.experts.73.gate_proj", + "model.layers.57.mlp.experts.73.up_proj", + "model.layers.57.mlp.experts.74.down_proj", + "model.layers.57.mlp.experts.74.gate_proj", + "model.layers.57.mlp.experts.74.up_proj", + "model.layers.57.mlp.experts.75.down_proj", + "model.layers.57.mlp.experts.75.gate_proj", + "model.layers.57.mlp.experts.75.up_proj", + "model.layers.57.mlp.experts.76.down_proj", + "model.layers.57.mlp.experts.76.gate_proj", + "model.layers.57.mlp.experts.76.up_proj", + "model.layers.57.mlp.experts.77.down_proj", + "model.layers.57.mlp.experts.77.gate_proj", + "model.layers.57.mlp.experts.77.up_proj", + "model.layers.57.mlp.experts.78.down_proj", + "model.layers.57.mlp.experts.78.gate_proj", + "model.layers.57.mlp.experts.78.up_proj", + "model.layers.57.mlp.experts.79.down_proj", + "model.layers.57.mlp.experts.79.gate_proj", + "model.layers.57.mlp.experts.79.up_proj", + "model.layers.57.mlp.experts.8.down_proj", + "model.layers.57.mlp.experts.8.gate_proj", + "model.layers.57.mlp.experts.8.up_proj", + "model.layers.57.mlp.experts.80.down_proj", + "model.layers.57.mlp.experts.80.gate_proj", + "model.layers.57.mlp.experts.80.up_proj", + "model.layers.57.mlp.experts.81.down_proj", + "model.layers.57.mlp.experts.81.gate_proj", + "model.layers.57.mlp.experts.81.up_proj", + "model.layers.57.mlp.experts.82.down_proj", + "model.layers.57.mlp.experts.82.gate_proj", + "model.layers.57.mlp.experts.82.up_proj", + "model.layers.57.mlp.experts.83.down_proj", + "model.layers.57.mlp.experts.83.gate_proj", + "model.layers.57.mlp.experts.83.up_proj", + "model.layers.57.mlp.experts.84.down_proj", + "model.layers.57.mlp.experts.84.gate_proj", + "model.layers.57.mlp.experts.84.up_proj", + "model.layers.57.mlp.experts.85.down_proj", + "model.layers.57.mlp.experts.85.gate_proj", + "model.layers.57.mlp.experts.85.up_proj", + "model.layers.57.mlp.experts.86.down_proj", + "model.layers.57.mlp.experts.86.gate_proj", + "model.layers.57.mlp.experts.86.up_proj", + "model.layers.57.mlp.experts.87.down_proj", + "model.layers.57.mlp.experts.87.gate_proj", + "model.layers.57.mlp.experts.87.up_proj", + "model.layers.57.mlp.experts.88.down_proj", + "model.layers.57.mlp.experts.88.gate_proj", + "model.layers.57.mlp.experts.88.up_proj", + "model.layers.57.mlp.experts.89.down_proj", + "model.layers.57.mlp.experts.89.gate_proj", + "model.layers.57.mlp.experts.89.up_proj", + "model.layers.57.mlp.experts.9.down_proj", + "model.layers.57.mlp.experts.9.gate_proj", + "model.layers.57.mlp.experts.9.up_proj", + "model.layers.57.mlp.experts.90.down_proj", + "model.layers.57.mlp.experts.90.gate_proj", + "model.layers.57.mlp.experts.90.up_proj", + "model.layers.57.mlp.experts.91.down_proj", + "model.layers.57.mlp.experts.91.gate_proj", + "model.layers.57.mlp.experts.91.up_proj", + "model.layers.57.mlp.experts.92.down_proj", + "model.layers.57.mlp.experts.92.gate_proj", + "model.layers.57.mlp.experts.92.up_proj", + "model.layers.57.mlp.experts.93.down_proj", + "model.layers.57.mlp.experts.93.gate_proj", + "model.layers.57.mlp.experts.93.up_proj", + "model.layers.57.mlp.experts.94.down_proj", + "model.layers.57.mlp.experts.94.gate_proj", + "model.layers.57.mlp.experts.94.up_proj", + "model.layers.57.mlp.experts.95.down_proj", + "model.layers.57.mlp.experts.95.gate_proj", + "model.layers.57.mlp.experts.95.up_proj", + "model.layers.57.mlp.experts.96.down_proj", + "model.layers.57.mlp.experts.96.gate_proj", + "model.layers.57.mlp.experts.96.up_proj", + "model.layers.57.mlp.experts.97.down_proj", + "model.layers.57.mlp.experts.97.gate_proj", + "model.layers.57.mlp.experts.97.up_proj", + "model.layers.57.mlp.experts.98.down_proj", + "model.layers.57.mlp.experts.98.gate_proj", + "model.layers.57.mlp.experts.98.up_proj", + "model.layers.57.mlp.experts.99.down_proj", + "model.layers.57.mlp.experts.99.gate_proj", + "model.layers.57.mlp.experts.99.up_proj", + "model.layers.58.mlp.experts.0.down_proj", + "model.layers.58.mlp.experts.0.gate_proj", + "model.layers.58.mlp.experts.0.up_proj", + "model.layers.58.mlp.experts.1.down_proj", + "model.layers.58.mlp.experts.1.gate_proj", + "model.layers.58.mlp.experts.1.up_proj", + "model.layers.58.mlp.experts.10.down_proj", + "model.layers.58.mlp.experts.10.gate_proj", + "model.layers.58.mlp.experts.10.up_proj", + "model.layers.58.mlp.experts.100.down_proj", + "model.layers.58.mlp.experts.100.gate_proj", + "model.layers.58.mlp.experts.100.up_proj", + "model.layers.58.mlp.experts.101.down_proj", + "model.layers.58.mlp.experts.101.gate_proj", + "model.layers.58.mlp.experts.101.up_proj", + "model.layers.58.mlp.experts.102.down_proj", + "model.layers.58.mlp.experts.102.gate_proj", + "model.layers.58.mlp.experts.102.up_proj", + "model.layers.58.mlp.experts.103.down_proj", + "model.layers.58.mlp.experts.103.gate_proj", + "model.layers.58.mlp.experts.103.up_proj", + "model.layers.58.mlp.experts.104.down_proj", + "model.layers.58.mlp.experts.104.gate_proj", + "model.layers.58.mlp.experts.104.up_proj", + "model.layers.58.mlp.experts.105.down_proj", + "model.layers.58.mlp.experts.105.gate_proj", + "model.layers.58.mlp.experts.105.up_proj", + "model.layers.58.mlp.experts.106.down_proj", + "model.layers.58.mlp.experts.106.gate_proj", + "model.layers.58.mlp.experts.106.up_proj", + "model.layers.58.mlp.experts.107.down_proj", + "model.layers.58.mlp.experts.107.gate_proj", + "model.layers.58.mlp.experts.107.up_proj", + "model.layers.58.mlp.experts.108.down_proj", + "model.layers.58.mlp.experts.108.gate_proj", + "model.layers.58.mlp.experts.108.up_proj", + "model.layers.58.mlp.experts.109.down_proj", + "model.layers.58.mlp.experts.109.gate_proj", + "model.layers.58.mlp.experts.109.up_proj", + "model.layers.58.mlp.experts.11.down_proj", + "model.layers.58.mlp.experts.11.gate_proj", + "model.layers.58.mlp.experts.11.up_proj", + "model.layers.58.mlp.experts.110.down_proj", + "model.layers.58.mlp.experts.110.gate_proj", + "model.layers.58.mlp.experts.110.up_proj", + "model.layers.58.mlp.experts.111.down_proj", + "model.layers.58.mlp.experts.111.gate_proj", + "model.layers.58.mlp.experts.111.up_proj", + "model.layers.58.mlp.experts.112.down_proj", + "model.layers.58.mlp.experts.112.gate_proj", + "model.layers.58.mlp.experts.112.up_proj", + "model.layers.58.mlp.experts.113.down_proj", + "model.layers.58.mlp.experts.113.gate_proj", + "model.layers.58.mlp.experts.113.up_proj", + "model.layers.58.mlp.experts.114.down_proj", + "model.layers.58.mlp.experts.114.gate_proj", + "model.layers.58.mlp.experts.114.up_proj", + "model.layers.58.mlp.experts.115.down_proj", + "model.layers.58.mlp.experts.115.gate_proj", + "model.layers.58.mlp.experts.115.up_proj", + "model.layers.58.mlp.experts.116.down_proj", + "model.layers.58.mlp.experts.116.gate_proj", + "model.layers.58.mlp.experts.116.up_proj", + "model.layers.58.mlp.experts.117.down_proj", + "model.layers.58.mlp.experts.117.gate_proj", + "model.layers.58.mlp.experts.117.up_proj", + "model.layers.58.mlp.experts.118.down_proj", + "model.layers.58.mlp.experts.118.gate_proj", + "model.layers.58.mlp.experts.118.up_proj", + "model.layers.58.mlp.experts.119.down_proj", + "model.layers.58.mlp.experts.119.gate_proj", + "model.layers.58.mlp.experts.119.up_proj", + "model.layers.58.mlp.experts.12.down_proj", + "model.layers.58.mlp.experts.12.gate_proj", + "model.layers.58.mlp.experts.12.up_proj", + "model.layers.58.mlp.experts.120.down_proj", + "model.layers.58.mlp.experts.120.gate_proj", + "model.layers.58.mlp.experts.120.up_proj", + "model.layers.58.mlp.experts.121.down_proj", + "model.layers.58.mlp.experts.121.gate_proj", + "model.layers.58.mlp.experts.121.up_proj", + "model.layers.58.mlp.experts.122.down_proj", + "model.layers.58.mlp.experts.122.gate_proj", + "model.layers.58.mlp.experts.122.up_proj", + "model.layers.58.mlp.experts.123.down_proj", + "model.layers.58.mlp.experts.123.gate_proj", + "model.layers.58.mlp.experts.123.up_proj", + "model.layers.58.mlp.experts.124.down_proj", + "model.layers.58.mlp.experts.124.gate_proj", + "model.layers.58.mlp.experts.124.up_proj", + "model.layers.58.mlp.experts.125.down_proj", + "model.layers.58.mlp.experts.125.gate_proj", + "model.layers.58.mlp.experts.125.up_proj", + "model.layers.58.mlp.experts.126.down_proj", + "model.layers.58.mlp.experts.126.gate_proj", + "model.layers.58.mlp.experts.126.up_proj", + "model.layers.58.mlp.experts.127.down_proj", + "model.layers.58.mlp.experts.127.gate_proj", + "model.layers.58.mlp.experts.127.up_proj", + "model.layers.58.mlp.experts.128.down_proj", + "model.layers.58.mlp.experts.128.gate_proj", + "model.layers.58.mlp.experts.128.up_proj", + "model.layers.58.mlp.experts.129.down_proj", + "model.layers.58.mlp.experts.129.gate_proj", + "model.layers.58.mlp.experts.129.up_proj", + "model.layers.58.mlp.experts.13.down_proj", + "model.layers.58.mlp.experts.13.gate_proj", + "model.layers.58.mlp.experts.13.up_proj", + "model.layers.58.mlp.experts.130.down_proj", + "model.layers.58.mlp.experts.130.gate_proj", + "model.layers.58.mlp.experts.130.up_proj", + "model.layers.58.mlp.experts.131.down_proj", + "model.layers.58.mlp.experts.131.gate_proj", + "model.layers.58.mlp.experts.131.up_proj", + "model.layers.58.mlp.experts.132.down_proj", + "model.layers.58.mlp.experts.132.gate_proj", + "model.layers.58.mlp.experts.132.up_proj", + "model.layers.58.mlp.experts.133.down_proj", + "model.layers.58.mlp.experts.133.gate_proj", + "model.layers.58.mlp.experts.133.up_proj", + "model.layers.58.mlp.experts.134.down_proj", + "model.layers.58.mlp.experts.134.gate_proj", + "model.layers.58.mlp.experts.134.up_proj", + "model.layers.58.mlp.experts.135.down_proj", + "model.layers.58.mlp.experts.135.gate_proj", + "model.layers.58.mlp.experts.135.up_proj", + "model.layers.58.mlp.experts.136.down_proj", + "model.layers.58.mlp.experts.136.gate_proj", + "model.layers.58.mlp.experts.136.up_proj", + "model.layers.58.mlp.experts.137.down_proj", + "model.layers.58.mlp.experts.137.gate_proj", + "model.layers.58.mlp.experts.137.up_proj", + "model.layers.58.mlp.experts.138.down_proj", + "model.layers.58.mlp.experts.138.gate_proj", + "model.layers.58.mlp.experts.138.up_proj", + "model.layers.58.mlp.experts.139.down_proj", + "model.layers.58.mlp.experts.139.gate_proj", + "model.layers.58.mlp.experts.139.up_proj", + "model.layers.58.mlp.experts.14.down_proj", + "model.layers.58.mlp.experts.14.gate_proj", + "model.layers.58.mlp.experts.14.up_proj", + "model.layers.58.mlp.experts.140.down_proj", + "model.layers.58.mlp.experts.140.gate_proj", + "model.layers.58.mlp.experts.140.up_proj", + "model.layers.58.mlp.experts.141.down_proj", + "model.layers.58.mlp.experts.141.gate_proj", + "model.layers.58.mlp.experts.141.up_proj", + "model.layers.58.mlp.experts.142.down_proj", + "model.layers.58.mlp.experts.142.gate_proj", + "model.layers.58.mlp.experts.142.up_proj", + "model.layers.58.mlp.experts.143.down_proj", + "model.layers.58.mlp.experts.143.gate_proj", + "model.layers.58.mlp.experts.143.up_proj", + "model.layers.58.mlp.experts.144.down_proj", + "model.layers.58.mlp.experts.144.gate_proj", + "model.layers.58.mlp.experts.144.up_proj", + "model.layers.58.mlp.experts.145.down_proj", + "model.layers.58.mlp.experts.145.gate_proj", + "model.layers.58.mlp.experts.145.up_proj", + "model.layers.58.mlp.experts.146.down_proj", + "model.layers.58.mlp.experts.146.gate_proj", + "model.layers.58.mlp.experts.146.up_proj", + "model.layers.58.mlp.experts.147.down_proj", + "model.layers.58.mlp.experts.147.gate_proj", + "model.layers.58.mlp.experts.147.up_proj", + "model.layers.58.mlp.experts.148.down_proj", + "model.layers.58.mlp.experts.148.gate_proj", + "model.layers.58.mlp.experts.148.up_proj", + "model.layers.58.mlp.experts.149.down_proj", + "model.layers.58.mlp.experts.149.gate_proj", + "model.layers.58.mlp.experts.149.up_proj", + "model.layers.58.mlp.experts.15.down_proj", + "model.layers.58.mlp.experts.15.gate_proj", + "model.layers.58.mlp.experts.15.up_proj", + "model.layers.58.mlp.experts.150.down_proj", + "model.layers.58.mlp.experts.150.gate_proj", + "model.layers.58.mlp.experts.150.up_proj", + "model.layers.58.mlp.experts.151.down_proj", + "model.layers.58.mlp.experts.151.gate_proj", + "model.layers.58.mlp.experts.151.up_proj", + "model.layers.58.mlp.experts.152.down_proj", + "model.layers.58.mlp.experts.152.gate_proj", + "model.layers.58.mlp.experts.152.up_proj", + "model.layers.58.mlp.experts.153.down_proj", + "model.layers.58.mlp.experts.153.gate_proj", + "model.layers.58.mlp.experts.153.up_proj", + "model.layers.58.mlp.experts.154.down_proj", + "model.layers.58.mlp.experts.154.gate_proj", + "model.layers.58.mlp.experts.154.up_proj", + "model.layers.58.mlp.experts.155.down_proj", + "model.layers.58.mlp.experts.155.gate_proj", + "model.layers.58.mlp.experts.155.up_proj", + "model.layers.58.mlp.experts.156.down_proj", + "model.layers.58.mlp.experts.156.gate_proj", + "model.layers.58.mlp.experts.156.up_proj", + "model.layers.58.mlp.experts.157.down_proj", + "model.layers.58.mlp.experts.157.gate_proj", + "model.layers.58.mlp.experts.157.up_proj", + "model.layers.58.mlp.experts.158.down_proj", + "model.layers.58.mlp.experts.158.gate_proj", + "model.layers.58.mlp.experts.158.up_proj", + "model.layers.58.mlp.experts.159.down_proj", + "model.layers.58.mlp.experts.159.gate_proj", + "model.layers.58.mlp.experts.159.up_proj", + "model.layers.58.mlp.experts.16.down_proj", + "model.layers.58.mlp.experts.16.gate_proj", + "model.layers.58.mlp.experts.16.up_proj", + "model.layers.58.mlp.experts.160.down_proj", + "model.layers.58.mlp.experts.160.gate_proj", + "model.layers.58.mlp.experts.160.up_proj", + "model.layers.58.mlp.experts.161.down_proj", + "model.layers.58.mlp.experts.161.gate_proj", + "model.layers.58.mlp.experts.161.up_proj", + "model.layers.58.mlp.experts.162.down_proj", + "model.layers.58.mlp.experts.162.gate_proj", + "model.layers.58.mlp.experts.162.up_proj", + "model.layers.58.mlp.experts.163.down_proj", + "model.layers.58.mlp.experts.163.gate_proj", + "model.layers.58.mlp.experts.163.up_proj", + "model.layers.58.mlp.experts.164.down_proj", + "model.layers.58.mlp.experts.164.gate_proj", + "model.layers.58.mlp.experts.164.up_proj", + "model.layers.58.mlp.experts.165.down_proj", + "model.layers.58.mlp.experts.165.gate_proj", + "model.layers.58.mlp.experts.165.up_proj", + "model.layers.58.mlp.experts.166.down_proj", + "model.layers.58.mlp.experts.166.gate_proj", + "model.layers.58.mlp.experts.166.up_proj", + "model.layers.58.mlp.experts.167.down_proj", + "model.layers.58.mlp.experts.167.gate_proj", + "model.layers.58.mlp.experts.167.up_proj", + "model.layers.58.mlp.experts.168.down_proj", + "model.layers.58.mlp.experts.168.gate_proj", + "model.layers.58.mlp.experts.168.up_proj", + "model.layers.58.mlp.experts.169.down_proj", + "model.layers.58.mlp.experts.169.gate_proj", + "model.layers.58.mlp.experts.169.up_proj", + "model.layers.58.mlp.experts.17.down_proj", + "model.layers.58.mlp.experts.17.gate_proj", + "model.layers.58.mlp.experts.17.up_proj", + "model.layers.58.mlp.experts.170.down_proj", + "model.layers.58.mlp.experts.170.gate_proj", + "model.layers.58.mlp.experts.170.up_proj", + "model.layers.58.mlp.experts.171.down_proj", + "model.layers.58.mlp.experts.171.gate_proj", + "model.layers.58.mlp.experts.171.up_proj", + "model.layers.58.mlp.experts.172.down_proj", + "model.layers.58.mlp.experts.172.gate_proj", + "model.layers.58.mlp.experts.172.up_proj", + "model.layers.58.mlp.experts.173.down_proj", + "model.layers.58.mlp.experts.173.gate_proj", + "model.layers.58.mlp.experts.173.up_proj", + "model.layers.58.mlp.experts.174.down_proj", + "model.layers.58.mlp.experts.174.gate_proj", + "model.layers.58.mlp.experts.174.up_proj", + "model.layers.58.mlp.experts.175.down_proj", + "model.layers.58.mlp.experts.175.gate_proj", + "model.layers.58.mlp.experts.175.up_proj", + "model.layers.58.mlp.experts.176.down_proj", + "model.layers.58.mlp.experts.176.gate_proj", + "model.layers.58.mlp.experts.176.up_proj", + "model.layers.58.mlp.experts.177.down_proj", + "model.layers.58.mlp.experts.177.gate_proj", + "model.layers.58.mlp.experts.177.up_proj", + "model.layers.58.mlp.experts.178.down_proj", + "model.layers.58.mlp.experts.178.gate_proj", + "model.layers.58.mlp.experts.178.up_proj", + "model.layers.58.mlp.experts.179.down_proj", + "model.layers.58.mlp.experts.179.gate_proj", + "model.layers.58.mlp.experts.179.up_proj", + "model.layers.58.mlp.experts.18.down_proj", + "model.layers.58.mlp.experts.18.gate_proj", + "model.layers.58.mlp.experts.18.up_proj", + "model.layers.58.mlp.experts.180.down_proj", + "model.layers.58.mlp.experts.180.gate_proj", + "model.layers.58.mlp.experts.180.up_proj", + "model.layers.58.mlp.experts.181.down_proj", + "model.layers.58.mlp.experts.181.gate_proj", + "model.layers.58.mlp.experts.181.up_proj", + "model.layers.58.mlp.experts.182.down_proj", + "model.layers.58.mlp.experts.182.gate_proj", + "model.layers.58.mlp.experts.182.up_proj", + "model.layers.58.mlp.experts.183.down_proj", + "model.layers.58.mlp.experts.183.gate_proj", + "model.layers.58.mlp.experts.183.up_proj", + "model.layers.58.mlp.experts.184.down_proj", + "model.layers.58.mlp.experts.184.gate_proj", + "model.layers.58.mlp.experts.184.up_proj", + "model.layers.58.mlp.experts.185.down_proj", + "model.layers.58.mlp.experts.185.gate_proj", + "model.layers.58.mlp.experts.185.up_proj", + "model.layers.58.mlp.experts.186.down_proj", + "model.layers.58.mlp.experts.186.gate_proj", + "model.layers.58.mlp.experts.186.up_proj", + "model.layers.58.mlp.experts.187.down_proj", + "model.layers.58.mlp.experts.187.gate_proj", + "model.layers.58.mlp.experts.187.up_proj", + "model.layers.58.mlp.experts.188.down_proj", + "model.layers.58.mlp.experts.188.gate_proj", + "model.layers.58.mlp.experts.188.up_proj", + "model.layers.58.mlp.experts.189.down_proj", + "model.layers.58.mlp.experts.189.gate_proj", + "model.layers.58.mlp.experts.189.up_proj", + "model.layers.58.mlp.experts.19.down_proj", + "model.layers.58.mlp.experts.19.gate_proj", + "model.layers.58.mlp.experts.19.up_proj", + "model.layers.58.mlp.experts.190.down_proj", + "model.layers.58.mlp.experts.190.gate_proj", + "model.layers.58.mlp.experts.190.up_proj", + "model.layers.58.mlp.experts.191.down_proj", + "model.layers.58.mlp.experts.191.gate_proj", + "model.layers.58.mlp.experts.191.up_proj", + "model.layers.58.mlp.experts.192.down_proj", + "model.layers.58.mlp.experts.192.gate_proj", + "model.layers.58.mlp.experts.192.up_proj", + "model.layers.58.mlp.experts.193.down_proj", + "model.layers.58.mlp.experts.193.gate_proj", + "model.layers.58.mlp.experts.193.up_proj", + "model.layers.58.mlp.experts.194.down_proj", + "model.layers.58.mlp.experts.194.gate_proj", + "model.layers.58.mlp.experts.194.up_proj", + "model.layers.58.mlp.experts.195.down_proj", + "model.layers.58.mlp.experts.195.gate_proj", + "model.layers.58.mlp.experts.195.up_proj", + "model.layers.58.mlp.experts.196.down_proj", + "model.layers.58.mlp.experts.196.gate_proj", + "model.layers.58.mlp.experts.196.up_proj", + "model.layers.58.mlp.experts.197.down_proj", + "model.layers.58.mlp.experts.197.gate_proj", + "model.layers.58.mlp.experts.197.up_proj", + "model.layers.58.mlp.experts.198.down_proj", + "model.layers.58.mlp.experts.198.gate_proj", + "model.layers.58.mlp.experts.198.up_proj", + "model.layers.58.mlp.experts.199.down_proj", + "model.layers.58.mlp.experts.199.gate_proj", + "model.layers.58.mlp.experts.199.up_proj", + "model.layers.58.mlp.experts.2.down_proj", + "model.layers.58.mlp.experts.2.gate_proj", + "model.layers.58.mlp.experts.2.up_proj", + "model.layers.58.mlp.experts.20.down_proj", + "model.layers.58.mlp.experts.20.gate_proj", + "model.layers.58.mlp.experts.20.up_proj", + "model.layers.58.mlp.experts.200.down_proj", + "model.layers.58.mlp.experts.200.gate_proj", + "model.layers.58.mlp.experts.200.up_proj", + "model.layers.58.mlp.experts.201.down_proj", + "model.layers.58.mlp.experts.201.gate_proj", + "model.layers.58.mlp.experts.201.up_proj", + "model.layers.58.mlp.experts.202.down_proj", + "model.layers.58.mlp.experts.202.gate_proj", + "model.layers.58.mlp.experts.202.up_proj", + "model.layers.58.mlp.experts.203.down_proj", + "model.layers.58.mlp.experts.203.gate_proj", + "model.layers.58.mlp.experts.203.up_proj", + "model.layers.58.mlp.experts.204.down_proj", + "model.layers.58.mlp.experts.204.gate_proj", + "model.layers.58.mlp.experts.204.up_proj", + "model.layers.58.mlp.experts.205.down_proj", + "model.layers.58.mlp.experts.205.gate_proj", + "model.layers.58.mlp.experts.205.up_proj", + "model.layers.58.mlp.experts.206.down_proj", + "model.layers.58.mlp.experts.206.gate_proj", + "model.layers.58.mlp.experts.206.up_proj", + "model.layers.58.mlp.experts.207.down_proj", + "model.layers.58.mlp.experts.207.gate_proj", + "model.layers.58.mlp.experts.207.up_proj", + "model.layers.58.mlp.experts.208.down_proj", + "model.layers.58.mlp.experts.208.gate_proj", + "model.layers.58.mlp.experts.208.up_proj", + "model.layers.58.mlp.experts.209.down_proj", + "model.layers.58.mlp.experts.209.gate_proj", + "model.layers.58.mlp.experts.209.up_proj", + "model.layers.58.mlp.experts.21.down_proj", + "model.layers.58.mlp.experts.21.gate_proj", + "model.layers.58.mlp.experts.21.up_proj", + "model.layers.58.mlp.experts.210.down_proj", + "model.layers.58.mlp.experts.210.gate_proj", + "model.layers.58.mlp.experts.210.up_proj", + "model.layers.58.mlp.experts.211.down_proj", + "model.layers.58.mlp.experts.211.gate_proj", + "model.layers.58.mlp.experts.211.up_proj", + "model.layers.58.mlp.experts.212.down_proj", + "model.layers.58.mlp.experts.212.gate_proj", + "model.layers.58.mlp.experts.212.up_proj", + "model.layers.58.mlp.experts.213.down_proj", + "model.layers.58.mlp.experts.213.gate_proj", + "model.layers.58.mlp.experts.213.up_proj", + "model.layers.58.mlp.experts.214.down_proj", + "model.layers.58.mlp.experts.214.gate_proj", + "model.layers.58.mlp.experts.214.up_proj", + "model.layers.58.mlp.experts.215.down_proj", + "model.layers.58.mlp.experts.215.gate_proj", + "model.layers.58.mlp.experts.215.up_proj", + "model.layers.58.mlp.experts.216.down_proj", + "model.layers.58.mlp.experts.216.gate_proj", + "model.layers.58.mlp.experts.216.up_proj", + "model.layers.58.mlp.experts.217.down_proj", + "model.layers.58.mlp.experts.217.gate_proj", + "model.layers.58.mlp.experts.217.up_proj", + "model.layers.58.mlp.experts.218.down_proj", + "model.layers.58.mlp.experts.218.gate_proj", + "model.layers.58.mlp.experts.218.up_proj", + "model.layers.58.mlp.experts.219.down_proj", + "model.layers.58.mlp.experts.219.gate_proj", + "model.layers.58.mlp.experts.219.up_proj", + "model.layers.58.mlp.experts.22.down_proj", + "model.layers.58.mlp.experts.22.gate_proj", + "model.layers.58.mlp.experts.22.up_proj", + "model.layers.58.mlp.experts.220.down_proj", + "model.layers.58.mlp.experts.220.gate_proj", + "model.layers.58.mlp.experts.220.up_proj", + "model.layers.58.mlp.experts.221.down_proj", + "model.layers.58.mlp.experts.221.gate_proj", + "model.layers.58.mlp.experts.221.up_proj", + "model.layers.58.mlp.experts.222.down_proj", + "model.layers.58.mlp.experts.222.gate_proj", + "model.layers.58.mlp.experts.222.up_proj", + "model.layers.58.mlp.experts.223.down_proj", + "model.layers.58.mlp.experts.223.gate_proj", + "model.layers.58.mlp.experts.223.up_proj", + "model.layers.58.mlp.experts.224.down_proj", + "model.layers.58.mlp.experts.224.gate_proj", + "model.layers.58.mlp.experts.224.up_proj", + "model.layers.58.mlp.experts.225.down_proj", + "model.layers.58.mlp.experts.225.gate_proj", + "model.layers.58.mlp.experts.225.up_proj", + "model.layers.58.mlp.experts.226.down_proj", + "model.layers.58.mlp.experts.226.gate_proj", + "model.layers.58.mlp.experts.226.up_proj", + "model.layers.58.mlp.experts.227.down_proj", + "model.layers.58.mlp.experts.227.gate_proj", + "model.layers.58.mlp.experts.227.up_proj", + "model.layers.58.mlp.experts.228.down_proj", + "model.layers.58.mlp.experts.228.gate_proj", + "model.layers.58.mlp.experts.228.up_proj", + "model.layers.58.mlp.experts.229.down_proj", + "model.layers.58.mlp.experts.229.gate_proj", + "model.layers.58.mlp.experts.229.up_proj", + "model.layers.58.mlp.experts.23.down_proj", + "model.layers.58.mlp.experts.23.gate_proj", + "model.layers.58.mlp.experts.23.up_proj", + "model.layers.58.mlp.experts.230.down_proj", + "model.layers.58.mlp.experts.230.gate_proj", + "model.layers.58.mlp.experts.230.up_proj", + "model.layers.58.mlp.experts.231.down_proj", + "model.layers.58.mlp.experts.231.gate_proj", + "model.layers.58.mlp.experts.231.up_proj", + "model.layers.58.mlp.experts.232.down_proj", + "model.layers.58.mlp.experts.232.gate_proj", + "model.layers.58.mlp.experts.232.up_proj", + "model.layers.58.mlp.experts.233.down_proj", + "model.layers.58.mlp.experts.233.gate_proj", + "model.layers.58.mlp.experts.233.up_proj", + "model.layers.58.mlp.experts.234.down_proj", + "model.layers.58.mlp.experts.234.gate_proj", + "model.layers.58.mlp.experts.234.up_proj", + "model.layers.58.mlp.experts.235.down_proj", + "model.layers.58.mlp.experts.235.gate_proj", + "model.layers.58.mlp.experts.235.up_proj", + "model.layers.58.mlp.experts.236.down_proj", + "model.layers.58.mlp.experts.236.gate_proj", + "model.layers.58.mlp.experts.236.up_proj", + "model.layers.58.mlp.experts.237.down_proj", + "model.layers.58.mlp.experts.237.gate_proj", + "model.layers.58.mlp.experts.237.up_proj", + "model.layers.58.mlp.experts.238.down_proj", + "model.layers.58.mlp.experts.238.gate_proj", + "model.layers.58.mlp.experts.238.up_proj", + "model.layers.58.mlp.experts.239.down_proj", + "model.layers.58.mlp.experts.239.gate_proj", + "model.layers.58.mlp.experts.239.up_proj", + "model.layers.58.mlp.experts.24.down_proj", + "model.layers.58.mlp.experts.24.gate_proj", + "model.layers.58.mlp.experts.24.up_proj", + "model.layers.58.mlp.experts.240.down_proj", + "model.layers.58.mlp.experts.240.gate_proj", + "model.layers.58.mlp.experts.240.up_proj", + "model.layers.58.mlp.experts.241.down_proj", + "model.layers.58.mlp.experts.241.gate_proj", + "model.layers.58.mlp.experts.241.up_proj", + "model.layers.58.mlp.experts.242.down_proj", + "model.layers.58.mlp.experts.242.gate_proj", + "model.layers.58.mlp.experts.242.up_proj", + "model.layers.58.mlp.experts.243.down_proj", + "model.layers.58.mlp.experts.243.gate_proj", + "model.layers.58.mlp.experts.243.up_proj", + "model.layers.58.mlp.experts.244.down_proj", + "model.layers.58.mlp.experts.244.gate_proj", + "model.layers.58.mlp.experts.244.up_proj", + "model.layers.58.mlp.experts.245.down_proj", + "model.layers.58.mlp.experts.245.gate_proj", + "model.layers.58.mlp.experts.245.up_proj", + "model.layers.58.mlp.experts.246.down_proj", + "model.layers.58.mlp.experts.246.gate_proj", + "model.layers.58.mlp.experts.246.up_proj", + "model.layers.58.mlp.experts.247.down_proj", + "model.layers.58.mlp.experts.247.gate_proj", + "model.layers.58.mlp.experts.247.up_proj", + "model.layers.58.mlp.experts.248.down_proj", + "model.layers.58.mlp.experts.248.gate_proj", + "model.layers.58.mlp.experts.248.up_proj", + "model.layers.58.mlp.experts.249.down_proj", + "model.layers.58.mlp.experts.249.gate_proj", + "model.layers.58.mlp.experts.249.up_proj", + "model.layers.58.mlp.experts.25.down_proj", + "model.layers.58.mlp.experts.25.gate_proj", + "model.layers.58.mlp.experts.25.up_proj", + "model.layers.58.mlp.experts.250.down_proj", + "model.layers.58.mlp.experts.250.gate_proj", + "model.layers.58.mlp.experts.250.up_proj", + "model.layers.58.mlp.experts.251.down_proj", + "model.layers.58.mlp.experts.251.gate_proj", + "model.layers.58.mlp.experts.251.up_proj", + "model.layers.58.mlp.experts.252.down_proj", + "model.layers.58.mlp.experts.252.gate_proj", + "model.layers.58.mlp.experts.252.up_proj", + "model.layers.58.mlp.experts.253.down_proj", + "model.layers.58.mlp.experts.253.gate_proj", + "model.layers.58.mlp.experts.253.up_proj", + "model.layers.58.mlp.experts.254.down_proj", + "model.layers.58.mlp.experts.254.gate_proj", + "model.layers.58.mlp.experts.254.up_proj", + "model.layers.58.mlp.experts.255.down_proj", + "model.layers.58.mlp.experts.255.gate_proj", + "model.layers.58.mlp.experts.255.up_proj", + "model.layers.58.mlp.experts.26.down_proj", + "model.layers.58.mlp.experts.26.gate_proj", + "model.layers.58.mlp.experts.26.up_proj", + "model.layers.58.mlp.experts.27.down_proj", + "model.layers.58.mlp.experts.27.gate_proj", + "model.layers.58.mlp.experts.27.up_proj", + "model.layers.58.mlp.experts.28.down_proj", + "model.layers.58.mlp.experts.28.gate_proj", + "model.layers.58.mlp.experts.28.up_proj", + "model.layers.58.mlp.experts.29.down_proj", + "model.layers.58.mlp.experts.29.gate_proj", + "model.layers.58.mlp.experts.29.up_proj", + "model.layers.58.mlp.experts.3.down_proj", + "model.layers.58.mlp.experts.3.gate_proj", + "model.layers.58.mlp.experts.3.up_proj", + "model.layers.58.mlp.experts.30.down_proj", + "model.layers.58.mlp.experts.30.gate_proj", + "model.layers.58.mlp.experts.30.up_proj", + "model.layers.58.mlp.experts.31.down_proj", + "model.layers.58.mlp.experts.31.gate_proj", + "model.layers.58.mlp.experts.31.up_proj", + "model.layers.58.mlp.experts.32.down_proj", + "model.layers.58.mlp.experts.32.gate_proj", + "model.layers.58.mlp.experts.32.up_proj", + "model.layers.58.mlp.experts.33.down_proj", + "model.layers.58.mlp.experts.33.gate_proj", + "model.layers.58.mlp.experts.33.up_proj", + "model.layers.58.mlp.experts.34.down_proj", + "model.layers.58.mlp.experts.34.gate_proj", + "model.layers.58.mlp.experts.34.up_proj", + "model.layers.58.mlp.experts.35.down_proj", + "model.layers.58.mlp.experts.35.gate_proj", + "model.layers.58.mlp.experts.35.up_proj", + "model.layers.58.mlp.experts.36.down_proj", + "model.layers.58.mlp.experts.36.gate_proj", + "model.layers.58.mlp.experts.36.up_proj", + "model.layers.58.mlp.experts.37.down_proj", + "model.layers.58.mlp.experts.37.gate_proj", + "model.layers.58.mlp.experts.37.up_proj", + "model.layers.58.mlp.experts.38.down_proj", + "model.layers.58.mlp.experts.38.gate_proj", + "model.layers.58.mlp.experts.38.up_proj", + "model.layers.58.mlp.experts.39.down_proj", + "model.layers.58.mlp.experts.39.gate_proj", + "model.layers.58.mlp.experts.39.up_proj", + "model.layers.58.mlp.experts.4.down_proj", + "model.layers.58.mlp.experts.4.gate_proj", + "model.layers.58.mlp.experts.4.up_proj", + "model.layers.58.mlp.experts.40.down_proj", + "model.layers.58.mlp.experts.40.gate_proj", + "model.layers.58.mlp.experts.40.up_proj", + "model.layers.58.mlp.experts.41.down_proj", + "model.layers.58.mlp.experts.41.gate_proj", + "model.layers.58.mlp.experts.41.up_proj", + "model.layers.58.mlp.experts.42.down_proj", + "model.layers.58.mlp.experts.42.gate_proj", + "model.layers.58.mlp.experts.42.up_proj", + "model.layers.58.mlp.experts.43.down_proj", + "model.layers.58.mlp.experts.43.gate_proj", + "model.layers.58.mlp.experts.43.up_proj", + "model.layers.58.mlp.experts.44.down_proj", + "model.layers.58.mlp.experts.44.gate_proj", + "model.layers.58.mlp.experts.44.up_proj", + "model.layers.58.mlp.experts.45.down_proj", + "model.layers.58.mlp.experts.45.gate_proj", + "model.layers.58.mlp.experts.45.up_proj", + "model.layers.58.mlp.experts.46.down_proj", + "model.layers.58.mlp.experts.46.gate_proj", + "model.layers.58.mlp.experts.46.up_proj", + "model.layers.58.mlp.experts.47.down_proj", + "model.layers.58.mlp.experts.47.gate_proj", + "model.layers.58.mlp.experts.47.up_proj", + "model.layers.58.mlp.experts.48.down_proj", + "model.layers.58.mlp.experts.48.gate_proj", + "model.layers.58.mlp.experts.48.up_proj", + "model.layers.58.mlp.experts.49.down_proj", + "model.layers.58.mlp.experts.49.gate_proj", + "model.layers.58.mlp.experts.49.up_proj", + "model.layers.58.mlp.experts.5.down_proj", + "model.layers.58.mlp.experts.5.gate_proj", + "model.layers.58.mlp.experts.5.up_proj", + "model.layers.58.mlp.experts.50.down_proj", + "model.layers.58.mlp.experts.50.gate_proj", + "model.layers.58.mlp.experts.50.up_proj", + "model.layers.58.mlp.experts.51.down_proj", + "model.layers.58.mlp.experts.51.gate_proj", + "model.layers.58.mlp.experts.51.up_proj", + "model.layers.58.mlp.experts.52.down_proj", + "model.layers.58.mlp.experts.52.gate_proj", + "model.layers.58.mlp.experts.52.up_proj", + "model.layers.58.mlp.experts.53.down_proj", + "model.layers.58.mlp.experts.53.gate_proj", + "model.layers.58.mlp.experts.53.up_proj", + "model.layers.58.mlp.experts.54.down_proj", + "model.layers.58.mlp.experts.54.gate_proj", + "model.layers.58.mlp.experts.54.up_proj", + "model.layers.58.mlp.experts.55.down_proj", + "model.layers.58.mlp.experts.55.gate_proj", + "model.layers.58.mlp.experts.55.up_proj", + "model.layers.58.mlp.experts.56.down_proj", + "model.layers.58.mlp.experts.56.gate_proj", + "model.layers.58.mlp.experts.56.up_proj", + "model.layers.58.mlp.experts.57.down_proj", + "model.layers.58.mlp.experts.57.gate_proj", + "model.layers.58.mlp.experts.57.up_proj", + "model.layers.58.mlp.experts.58.down_proj", + "model.layers.58.mlp.experts.58.gate_proj", + "model.layers.58.mlp.experts.58.up_proj", + "model.layers.58.mlp.experts.59.down_proj", + "model.layers.58.mlp.experts.59.gate_proj", + "model.layers.58.mlp.experts.59.up_proj", + "model.layers.58.mlp.experts.6.down_proj", + "model.layers.58.mlp.experts.6.gate_proj", + "model.layers.58.mlp.experts.6.up_proj", + "model.layers.58.mlp.experts.60.down_proj", + "model.layers.58.mlp.experts.60.gate_proj", + "model.layers.58.mlp.experts.60.up_proj", + "model.layers.58.mlp.experts.61.down_proj", + "model.layers.58.mlp.experts.61.gate_proj", + "model.layers.58.mlp.experts.61.up_proj", + "model.layers.58.mlp.experts.62.down_proj", + "model.layers.58.mlp.experts.62.gate_proj", + "model.layers.58.mlp.experts.62.up_proj", + "model.layers.58.mlp.experts.63.down_proj", + "model.layers.58.mlp.experts.63.gate_proj", + "model.layers.58.mlp.experts.63.up_proj", + "model.layers.58.mlp.experts.64.down_proj", + "model.layers.58.mlp.experts.64.gate_proj", + "model.layers.58.mlp.experts.64.up_proj", + "model.layers.58.mlp.experts.65.down_proj", + "model.layers.58.mlp.experts.65.gate_proj", + "model.layers.58.mlp.experts.65.up_proj", + "model.layers.58.mlp.experts.66.down_proj", + "model.layers.58.mlp.experts.66.gate_proj", + "model.layers.58.mlp.experts.66.up_proj", + "model.layers.58.mlp.experts.67.down_proj", + "model.layers.58.mlp.experts.67.gate_proj", + "model.layers.58.mlp.experts.67.up_proj", + "model.layers.58.mlp.experts.68.down_proj", + "model.layers.58.mlp.experts.68.gate_proj", + "model.layers.58.mlp.experts.68.up_proj", + "model.layers.58.mlp.experts.69.down_proj", + "model.layers.58.mlp.experts.69.gate_proj", + "model.layers.58.mlp.experts.69.up_proj", + "model.layers.58.mlp.experts.7.down_proj", + "model.layers.58.mlp.experts.7.gate_proj", + "model.layers.58.mlp.experts.7.up_proj", + "model.layers.58.mlp.experts.70.down_proj", + "model.layers.58.mlp.experts.70.gate_proj", + "model.layers.58.mlp.experts.70.up_proj", + "model.layers.58.mlp.experts.71.down_proj", + "model.layers.58.mlp.experts.71.gate_proj", + "model.layers.58.mlp.experts.71.up_proj", + "model.layers.58.mlp.experts.72.down_proj", + "model.layers.58.mlp.experts.72.gate_proj", + "model.layers.58.mlp.experts.72.up_proj", + "model.layers.58.mlp.experts.73.down_proj", + "model.layers.58.mlp.experts.73.gate_proj", + "model.layers.58.mlp.experts.73.up_proj", + "model.layers.58.mlp.experts.74.down_proj", + "model.layers.58.mlp.experts.74.gate_proj", + "model.layers.58.mlp.experts.74.up_proj", + "model.layers.58.mlp.experts.75.down_proj", + "model.layers.58.mlp.experts.75.gate_proj", + "model.layers.58.mlp.experts.75.up_proj", + "model.layers.58.mlp.experts.76.down_proj", + "model.layers.58.mlp.experts.76.gate_proj", + "model.layers.58.mlp.experts.76.up_proj", + "model.layers.58.mlp.experts.77.down_proj", + "model.layers.58.mlp.experts.77.gate_proj", + "model.layers.58.mlp.experts.77.up_proj", + "model.layers.58.mlp.experts.78.down_proj", + "model.layers.58.mlp.experts.78.gate_proj", + "model.layers.58.mlp.experts.78.up_proj", + "model.layers.58.mlp.experts.79.down_proj", + "model.layers.58.mlp.experts.79.gate_proj", + "model.layers.58.mlp.experts.79.up_proj", + "model.layers.58.mlp.experts.8.down_proj", + "model.layers.58.mlp.experts.8.gate_proj", + "model.layers.58.mlp.experts.8.up_proj", + "model.layers.58.mlp.experts.80.down_proj", + "model.layers.58.mlp.experts.80.gate_proj", + "model.layers.58.mlp.experts.80.up_proj", + "model.layers.58.mlp.experts.81.down_proj", + "model.layers.58.mlp.experts.81.gate_proj", + "model.layers.58.mlp.experts.81.up_proj", + "model.layers.58.mlp.experts.82.down_proj", + "model.layers.58.mlp.experts.82.gate_proj", + "model.layers.58.mlp.experts.82.up_proj", + "model.layers.58.mlp.experts.83.down_proj", + "model.layers.58.mlp.experts.83.gate_proj", + "model.layers.58.mlp.experts.83.up_proj", + "model.layers.58.mlp.experts.84.down_proj", + "model.layers.58.mlp.experts.84.gate_proj", + "model.layers.58.mlp.experts.84.up_proj", + "model.layers.58.mlp.experts.85.down_proj", + "model.layers.58.mlp.experts.85.gate_proj", + "model.layers.58.mlp.experts.85.up_proj", + "model.layers.58.mlp.experts.86.down_proj", + "model.layers.58.mlp.experts.86.gate_proj", + "model.layers.58.mlp.experts.86.up_proj", + "model.layers.58.mlp.experts.87.down_proj", + "model.layers.58.mlp.experts.87.gate_proj", + "model.layers.58.mlp.experts.87.up_proj", + "model.layers.58.mlp.experts.88.down_proj", + "model.layers.58.mlp.experts.88.gate_proj", + "model.layers.58.mlp.experts.88.up_proj", + "model.layers.58.mlp.experts.89.down_proj", + "model.layers.58.mlp.experts.89.gate_proj", + "model.layers.58.mlp.experts.89.up_proj", + "model.layers.58.mlp.experts.9.down_proj", + "model.layers.58.mlp.experts.9.gate_proj", + "model.layers.58.mlp.experts.9.up_proj", + "model.layers.58.mlp.experts.90.down_proj", + "model.layers.58.mlp.experts.90.gate_proj", + "model.layers.58.mlp.experts.90.up_proj", + "model.layers.58.mlp.experts.91.down_proj", + "model.layers.58.mlp.experts.91.gate_proj", + "model.layers.58.mlp.experts.91.up_proj", + "model.layers.58.mlp.experts.92.down_proj", + "model.layers.58.mlp.experts.92.gate_proj", + "model.layers.58.mlp.experts.92.up_proj", + "model.layers.58.mlp.experts.93.down_proj", + "model.layers.58.mlp.experts.93.gate_proj", + "model.layers.58.mlp.experts.93.up_proj", + "model.layers.58.mlp.experts.94.down_proj", + "model.layers.58.mlp.experts.94.gate_proj", + "model.layers.58.mlp.experts.94.up_proj", + "model.layers.58.mlp.experts.95.down_proj", + "model.layers.58.mlp.experts.95.gate_proj", + "model.layers.58.mlp.experts.95.up_proj", + "model.layers.58.mlp.experts.96.down_proj", + "model.layers.58.mlp.experts.96.gate_proj", + "model.layers.58.mlp.experts.96.up_proj", + "model.layers.58.mlp.experts.97.down_proj", + "model.layers.58.mlp.experts.97.gate_proj", + "model.layers.58.mlp.experts.97.up_proj", + "model.layers.58.mlp.experts.98.down_proj", + "model.layers.58.mlp.experts.98.gate_proj", + "model.layers.58.mlp.experts.98.up_proj", + "model.layers.58.mlp.experts.99.down_proj", + "model.layers.58.mlp.experts.99.gate_proj", + "model.layers.58.mlp.experts.99.up_proj", + "model.layers.59.mlp.experts.0.down_proj", + "model.layers.59.mlp.experts.0.gate_proj", + "model.layers.59.mlp.experts.0.up_proj", + "model.layers.59.mlp.experts.1.down_proj", + "model.layers.59.mlp.experts.1.gate_proj", + "model.layers.59.mlp.experts.1.up_proj", + "model.layers.59.mlp.experts.10.down_proj", + "model.layers.59.mlp.experts.10.gate_proj", + "model.layers.59.mlp.experts.10.up_proj", + "model.layers.59.mlp.experts.100.down_proj", + "model.layers.59.mlp.experts.100.gate_proj", + "model.layers.59.mlp.experts.100.up_proj", + "model.layers.59.mlp.experts.101.down_proj", + "model.layers.59.mlp.experts.101.gate_proj", + "model.layers.59.mlp.experts.101.up_proj", + "model.layers.59.mlp.experts.102.down_proj", + "model.layers.59.mlp.experts.102.gate_proj", + "model.layers.59.mlp.experts.102.up_proj", + "model.layers.59.mlp.experts.103.down_proj", + "model.layers.59.mlp.experts.103.gate_proj", + "model.layers.59.mlp.experts.103.up_proj", + "model.layers.59.mlp.experts.104.down_proj", + "model.layers.59.mlp.experts.104.gate_proj", + "model.layers.59.mlp.experts.104.up_proj", + "model.layers.59.mlp.experts.105.down_proj", + "model.layers.59.mlp.experts.105.gate_proj", + "model.layers.59.mlp.experts.105.up_proj", + "model.layers.59.mlp.experts.106.down_proj", + "model.layers.59.mlp.experts.106.gate_proj", + "model.layers.59.mlp.experts.106.up_proj", + "model.layers.59.mlp.experts.107.down_proj", + "model.layers.59.mlp.experts.107.gate_proj", + "model.layers.59.mlp.experts.107.up_proj", + "model.layers.59.mlp.experts.108.down_proj", + "model.layers.59.mlp.experts.108.gate_proj", + "model.layers.59.mlp.experts.108.up_proj", + "model.layers.59.mlp.experts.109.down_proj", + "model.layers.59.mlp.experts.109.gate_proj", + "model.layers.59.mlp.experts.109.up_proj", + "model.layers.59.mlp.experts.11.down_proj", + "model.layers.59.mlp.experts.11.gate_proj", + "model.layers.59.mlp.experts.11.up_proj", + "model.layers.59.mlp.experts.110.down_proj", + "model.layers.59.mlp.experts.110.gate_proj", + "model.layers.59.mlp.experts.110.up_proj", + "model.layers.59.mlp.experts.111.down_proj", + "model.layers.59.mlp.experts.111.gate_proj", + "model.layers.59.mlp.experts.111.up_proj", + "model.layers.59.mlp.experts.112.down_proj", + "model.layers.59.mlp.experts.112.gate_proj", + "model.layers.59.mlp.experts.112.up_proj", + "model.layers.59.mlp.experts.113.down_proj", + "model.layers.59.mlp.experts.113.gate_proj", + "model.layers.59.mlp.experts.113.up_proj", + "model.layers.59.mlp.experts.114.down_proj", + "model.layers.59.mlp.experts.114.gate_proj", + "model.layers.59.mlp.experts.114.up_proj", + "model.layers.59.mlp.experts.115.down_proj", + "model.layers.59.mlp.experts.115.gate_proj", + "model.layers.59.mlp.experts.115.up_proj", + "model.layers.59.mlp.experts.116.down_proj", + "model.layers.59.mlp.experts.116.gate_proj", + "model.layers.59.mlp.experts.116.up_proj", + "model.layers.59.mlp.experts.117.down_proj", + "model.layers.59.mlp.experts.117.gate_proj", + "model.layers.59.mlp.experts.117.up_proj", + "model.layers.59.mlp.experts.118.down_proj", + "model.layers.59.mlp.experts.118.gate_proj", + "model.layers.59.mlp.experts.118.up_proj", + "model.layers.59.mlp.experts.119.down_proj", + "model.layers.59.mlp.experts.119.gate_proj", + "model.layers.59.mlp.experts.119.up_proj", + "model.layers.59.mlp.experts.12.down_proj", + "model.layers.59.mlp.experts.12.gate_proj", + "model.layers.59.mlp.experts.12.up_proj", + "model.layers.59.mlp.experts.120.down_proj", + "model.layers.59.mlp.experts.120.gate_proj", + "model.layers.59.mlp.experts.120.up_proj", + "model.layers.59.mlp.experts.121.down_proj", + "model.layers.59.mlp.experts.121.gate_proj", + "model.layers.59.mlp.experts.121.up_proj", + "model.layers.59.mlp.experts.122.down_proj", + "model.layers.59.mlp.experts.122.gate_proj", + "model.layers.59.mlp.experts.122.up_proj", + "model.layers.59.mlp.experts.123.down_proj", + "model.layers.59.mlp.experts.123.gate_proj", + "model.layers.59.mlp.experts.123.up_proj", + "model.layers.59.mlp.experts.124.down_proj", + "model.layers.59.mlp.experts.124.gate_proj", + "model.layers.59.mlp.experts.124.up_proj", + "model.layers.59.mlp.experts.125.down_proj", + "model.layers.59.mlp.experts.125.gate_proj", + "model.layers.59.mlp.experts.125.up_proj", + "model.layers.59.mlp.experts.126.down_proj", + "model.layers.59.mlp.experts.126.gate_proj", + "model.layers.59.mlp.experts.126.up_proj", + "model.layers.59.mlp.experts.127.down_proj", + "model.layers.59.mlp.experts.127.gate_proj", + "model.layers.59.mlp.experts.127.up_proj", + "model.layers.59.mlp.experts.128.down_proj", + "model.layers.59.mlp.experts.128.gate_proj", + "model.layers.59.mlp.experts.128.up_proj", + "model.layers.59.mlp.experts.129.down_proj", + "model.layers.59.mlp.experts.129.gate_proj", + "model.layers.59.mlp.experts.129.up_proj", + "model.layers.59.mlp.experts.13.down_proj", + "model.layers.59.mlp.experts.13.gate_proj", + "model.layers.59.mlp.experts.13.up_proj", + "model.layers.59.mlp.experts.130.down_proj", + "model.layers.59.mlp.experts.130.gate_proj", + "model.layers.59.mlp.experts.130.up_proj", + "model.layers.59.mlp.experts.131.down_proj", + "model.layers.59.mlp.experts.131.gate_proj", + "model.layers.59.mlp.experts.131.up_proj", + "model.layers.59.mlp.experts.132.down_proj", + "model.layers.59.mlp.experts.132.gate_proj", + "model.layers.59.mlp.experts.132.up_proj", + "model.layers.59.mlp.experts.133.down_proj", + "model.layers.59.mlp.experts.133.gate_proj", + "model.layers.59.mlp.experts.133.up_proj", + "model.layers.59.mlp.experts.134.down_proj", + "model.layers.59.mlp.experts.134.gate_proj", + "model.layers.59.mlp.experts.134.up_proj", + "model.layers.59.mlp.experts.135.down_proj", + "model.layers.59.mlp.experts.135.gate_proj", + "model.layers.59.mlp.experts.135.up_proj", + "model.layers.59.mlp.experts.136.down_proj", + "model.layers.59.mlp.experts.136.gate_proj", + "model.layers.59.mlp.experts.136.up_proj", + "model.layers.59.mlp.experts.137.down_proj", + "model.layers.59.mlp.experts.137.gate_proj", + "model.layers.59.mlp.experts.137.up_proj", + "model.layers.59.mlp.experts.138.down_proj", + "model.layers.59.mlp.experts.138.gate_proj", + "model.layers.59.mlp.experts.138.up_proj", + "model.layers.59.mlp.experts.139.down_proj", + "model.layers.59.mlp.experts.139.gate_proj", + "model.layers.59.mlp.experts.139.up_proj", + "model.layers.59.mlp.experts.14.down_proj", + "model.layers.59.mlp.experts.14.gate_proj", + "model.layers.59.mlp.experts.14.up_proj", + "model.layers.59.mlp.experts.140.down_proj", + "model.layers.59.mlp.experts.140.gate_proj", + "model.layers.59.mlp.experts.140.up_proj", + "model.layers.59.mlp.experts.141.down_proj", + "model.layers.59.mlp.experts.141.gate_proj", + "model.layers.59.mlp.experts.141.up_proj", + "model.layers.59.mlp.experts.142.down_proj", + "model.layers.59.mlp.experts.142.gate_proj", + "model.layers.59.mlp.experts.142.up_proj", + "model.layers.59.mlp.experts.143.down_proj", + "model.layers.59.mlp.experts.143.gate_proj", + "model.layers.59.mlp.experts.143.up_proj", + "model.layers.59.mlp.experts.144.down_proj", + "model.layers.59.mlp.experts.144.gate_proj", + "model.layers.59.mlp.experts.144.up_proj", + "model.layers.59.mlp.experts.145.down_proj", + "model.layers.59.mlp.experts.145.gate_proj", + "model.layers.59.mlp.experts.145.up_proj", + "model.layers.59.mlp.experts.146.down_proj", + "model.layers.59.mlp.experts.146.gate_proj", + "model.layers.59.mlp.experts.146.up_proj", + "model.layers.59.mlp.experts.147.down_proj", + "model.layers.59.mlp.experts.147.gate_proj", + "model.layers.59.mlp.experts.147.up_proj", + "model.layers.59.mlp.experts.148.down_proj", + "model.layers.59.mlp.experts.148.gate_proj", + "model.layers.59.mlp.experts.148.up_proj", + "model.layers.59.mlp.experts.149.down_proj", + "model.layers.59.mlp.experts.149.gate_proj", + "model.layers.59.mlp.experts.149.up_proj", + "model.layers.59.mlp.experts.15.down_proj", + "model.layers.59.mlp.experts.15.gate_proj", + "model.layers.59.mlp.experts.15.up_proj", + "model.layers.59.mlp.experts.150.down_proj", + "model.layers.59.mlp.experts.150.gate_proj", + "model.layers.59.mlp.experts.150.up_proj", + "model.layers.59.mlp.experts.151.down_proj", + "model.layers.59.mlp.experts.151.gate_proj", + "model.layers.59.mlp.experts.151.up_proj", + "model.layers.59.mlp.experts.152.down_proj", + "model.layers.59.mlp.experts.152.gate_proj", + "model.layers.59.mlp.experts.152.up_proj", + "model.layers.59.mlp.experts.153.down_proj", + "model.layers.59.mlp.experts.153.gate_proj", + "model.layers.59.mlp.experts.153.up_proj", + "model.layers.59.mlp.experts.154.down_proj", + "model.layers.59.mlp.experts.154.gate_proj", + "model.layers.59.mlp.experts.154.up_proj", + "model.layers.59.mlp.experts.155.down_proj", + "model.layers.59.mlp.experts.155.gate_proj", + "model.layers.59.mlp.experts.155.up_proj", + "model.layers.59.mlp.experts.156.down_proj", + "model.layers.59.mlp.experts.156.gate_proj", + "model.layers.59.mlp.experts.156.up_proj", + "model.layers.59.mlp.experts.157.down_proj", + "model.layers.59.mlp.experts.157.gate_proj", + "model.layers.59.mlp.experts.157.up_proj", + "model.layers.59.mlp.experts.158.down_proj", + "model.layers.59.mlp.experts.158.gate_proj", + "model.layers.59.mlp.experts.158.up_proj", + "model.layers.59.mlp.experts.159.down_proj", + "model.layers.59.mlp.experts.159.gate_proj", + "model.layers.59.mlp.experts.159.up_proj", + "model.layers.59.mlp.experts.16.down_proj", + "model.layers.59.mlp.experts.16.gate_proj", + "model.layers.59.mlp.experts.16.up_proj", + "model.layers.59.mlp.experts.160.down_proj", + "model.layers.59.mlp.experts.160.gate_proj", + "model.layers.59.mlp.experts.160.up_proj", + "model.layers.59.mlp.experts.161.down_proj", + "model.layers.59.mlp.experts.161.gate_proj", + "model.layers.59.mlp.experts.161.up_proj", + "model.layers.59.mlp.experts.162.down_proj", + "model.layers.59.mlp.experts.162.gate_proj", + "model.layers.59.mlp.experts.162.up_proj", + "model.layers.59.mlp.experts.163.down_proj", + "model.layers.59.mlp.experts.163.gate_proj", + "model.layers.59.mlp.experts.163.up_proj", + "model.layers.59.mlp.experts.164.down_proj", + "model.layers.59.mlp.experts.164.gate_proj", + "model.layers.59.mlp.experts.164.up_proj", + "model.layers.59.mlp.experts.165.down_proj", + "model.layers.59.mlp.experts.165.gate_proj", + "model.layers.59.mlp.experts.165.up_proj", + "model.layers.59.mlp.experts.166.down_proj", + "model.layers.59.mlp.experts.166.gate_proj", + "model.layers.59.mlp.experts.166.up_proj", + "model.layers.59.mlp.experts.167.down_proj", + "model.layers.59.mlp.experts.167.gate_proj", + "model.layers.59.mlp.experts.167.up_proj", + "model.layers.59.mlp.experts.168.down_proj", + "model.layers.59.mlp.experts.168.gate_proj", + "model.layers.59.mlp.experts.168.up_proj", + "model.layers.59.mlp.experts.169.down_proj", + "model.layers.59.mlp.experts.169.gate_proj", + "model.layers.59.mlp.experts.169.up_proj", + "model.layers.59.mlp.experts.17.down_proj", + "model.layers.59.mlp.experts.17.gate_proj", + "model.layers.59.mlp.experts.17.up_proj", + "model.layers.59.mlp.experts.170.down_proj", + "model.layers.59.mlp.experts.170.gate_proj", + "model.layers.59.mlp.experts.170.up_proj", + "model.layers.59.mlp.experts.171.down_proj", + "model.layers.59.mlp.experts.171.gate_proj", + "model.layers.59.mlp.experts.171.up_proj", + "model.layers.59.mlp.experts.172.down_proj", + "model.layers.59.mlp.experts.172.gate_proj", + "model.layers.59.mlp.experts.172.up_proj", + "model.layers.59.mlp.experts.173.down_proj", + "model.layers.59.mlp.experts.173.gate_proj", + "model.layers.59.mlp.experts.173.up_proj", + "model.layers.59.mlp.experts.174.down_proj", + "model.layers.59.mlp.experts.174.gate_proj", + "model.layers.59.mlp.experts.174.up_proj", + "model.layers.59.mlp.experts.175.down_proj", + "model.layers.59.mlp.experts.175.gate_proj", + "model.layers.59.mlp.experts.175.up_proj", + "model.layers.59.mlp.experts.176.down_proj", + "model.layers.59.mlp.experts.176.gate_proj", + "model.layers.59.mlp.experts.176.up_proj", + "model.layers.59.mlp.experts.177.down_proj", + "model.layers.59.mlp.experts.177.gate_proj", + "model.layers.59.mlp.experts.177.up_proj", + "model.layers.59.mlp.experts.178.down_proj", + "model.layers.59.mlp.experts.178.gate_proj", + "model.layers.59.mlp.experts.178.up_proj", + "model.layers.59.mlp.experts.179.down_proj", + "model.layers.59.mlp.experts.179.gate_proj", + "model.layers.59.mlp.experts.179.up_proj", + "model.layers.59.mlp.experts.18.down_proj", + "model.layers.59.mlp.experts.18.gate_proj", + "model.layers.59.mlp.experts.18.up_proj", + "model.layers.59.mlp.experts.180.down_proj", + "model.layers.59.mlp.experts.180.gate_proj", + "model.layers.59.mlp.experts.180.up_proj", + "model.layers.59.mlp.experts.181.down_proj", + "model.layers.59.mlp.experts.181.gate_proj", + "model.layers.59.mlp.experts.181.up_proj", + "model.layers.59.mlp.experts.182.down_proj", + "model.layers.59.mlp.experts.182.gate_proj", + "model.layers.59.mlp.experts.182.up_proj", + "model.layers.59.mlp.experts.183.down_proj", + "model.layers.59.mlp.experts.183.gate_proj", + "model.layers.59.mlp.experts.183.up_proj", + "model.layers.59.mlp.experts.184.down_proj", + "model.layers.59.mlp.experts.184.gate_proj", + "model.layers.59.mlp.experts.184.up_proj", + "model.layers.59.mlp.experts.185.down_proj", + "model.layers.59.mlp.experts.185.gate_proj", + "model.layers.59.mlp.experts.185.up_proj", + "model.layers.59.mlp.experts.186.down_proj", + "model.layers.59.mlp.experts.186.gate_proj", + "model.layers.59.mlp.experts.186.up_proj", + "model.layers.59.mlp.experts.187.down_proj", + "model.layers.59.mlp.experts.187.gate_proj", + "model.layers.59.mlp.experts.187.up_proj", + "model.layers.59.mlp.experts.188.down_proj", + "model.layers.59.mlp.experts.188.gate_proj", + "model.layers.59.mlp.experts.188.up_proj", + "model.layers.59.mlp.experts.189.down_proj", + "model.layers.59.mlp.experts.189.gate_proj", + "model.layers.59.mlp.experts.189.up_proj", + "model.layers.59.mlp.experts.19.down_proj", + "model.layers.59.mlp.experts.19.gate_proj", + "model.layers.59.mlp.experts.19.up_proj", + "model.layers.59.mlp.experts.190.down_proj", + "model.layers.59.mlp.experts.190.gate_proj", + "model.layers.59.mlp.experts.190.up_proj", + "model.layers.59.mlp.experts.191.down_proj", + "model.layers.59.mlp.experts.191.gate_proj", + "model.layers.59.mlp.experts.191.up_proj", + "model.layers.59.mlp.experts.192.down_proj", + "model.layers.59.mlp.experts.192.gate_proj", + "model.layers.59.mlp.experts.192.up_proj", + "model.layers.59.mlp.experts.193.down_proj", + "model.layers.59.mlp.experts.193.gate_proj", + "model.layers.59.mlp.experts.193.up_proj", + "model.layers.59.mlp.experts.194.down_proj", + "model.layers.59.mlp.experts.194.gate_proj", + "model.layers.59.mlp.experts.194.up_proj", + "model.layers.59.mlp.experts.195.down_proj", + "model.layers.59.mlp.experts.195.gate_proj", + "model.layers.59.mlp.experts.195.up_proj", + "model.layers.59.mlp.experts.196.down_proj", + "model.layers.59.mlp.experts.196.gate_proj", + "model.layers.59.mlp.experts.196.up_proj", + "model.layers.59.mlp.experts.197.down_proj", + "model.layers.59.mlp.experts.197.gate_proj", + "model.layers.59.mlp.experts.197.up_proj", + "model.layers.59.mlp.experts.198.down_proj", + "model.layers.59.mlp.experts.198.gate_proj", + "model.layers.59.mlp.experts.198.up_proj", + "model.layers.59.mlp.experts.199.down_proj", + "model.layers.59.mlp.experts.199.gate_proj", + "model.layers.59.mlp.experts.199.up_proj", + "model.layers.59.mlp.experts.2.down_proj", + "model.layers.59.mlp.experts.2.gate_proj", + "model.layers.59.mlp.experts.2.up_proj", + "model.layers.59.mlp.experts.20.down_proj", + "model.layers.59.mlp.experts.20.gate_proj", + "model.layers.59.mlp.experts.20.up_proj", + "model.layers.59.mlp.experts.200.down_proj", + "model.layers.59.mlp.experts.200.gate_proj", + "model.layers.59.mlp.experts.200.up_proj", + "model.layers.59.mlp.experts.201.down_proj", + "model.layers.59.mlp.experts.201.gate_proj", + "model.layers.59.mlp.experts.201.up_proj", + "model.layers.59.mlp.experts.202.down_proj", + "model.layers.59.mlp.experts.202.gate_proj", + "model.layers.59.mlp.experts.202.up_proj", + "model.layers.59.mlp.experts.203.down_proj", + "model.layers.59.mlp.experts.203.gate_proj", + "model.layers.59.mlp.experts.203.up_proj", + "model.layers.59.mlp.experts.204.down_proj", + "model.layers.59.mlp.experts.204.gate_proj", + "model.layers.59.mlp.experts.204.up_proj", + "model.layers.59.mlp.experts.205.down_proj", + "model.layers.59.mlp.experts.205.gate_proj", + "model.layers.59.mlp.experts.205.up_proj", + "model.layers.59.mlp.experts.206.down_proj", + "model.layers.59.mlp.experts.206.gate_proj", + "model.layers.59.mlp.experts.206.up_proj", + "model.layers.59.mlp.experts.207.down_proj", + "model.layers.59.mlp.experts.207.gate_proj", + "model.layers.59.mlp.experts.207.up_proj", + "model.layers.59.mlp.experts.208.down_proj", + "model.layers.59.mlp.experts.208.gate_proj", + "model.layers.59.mlp.experts.208.up_proj", + "model.layers.59.mlp.experts.209.down_proj", + "model.layers.59.mlp.experts.209.gate_proj", + "model.layers.59.mlp.experts.209.up_proj", + "model.layers.59.mlp.experts.21.down_proj", + "model.layers.59.mlp.experts.21.gate_proj", + "model.layers.59.mlp.experts.21.up_proj", + "model.layers.59.mlp.experts.210.down_proj", + "model.layers.59.mlp.experts.210.gate_proj", + "model.layers.59.mlp.experts.210.up_proj", + "model.layers.59.mlp.experts.211.down_proj", + "model.layers.59.mlp.experts.211.gate_proj", + "model.layers.59.mlp.experts.211.up_proj", + "model.layers.59.mlp.experts.212.down_proj", + "model.layers.59.mlp.experts.212.gate_proj", + "model.layers.59.mlp.experts.212.up_proj", + "model.layers.59.mlp.experts.213.down_proj", + "model.layers.59.mlp.experts.213.gate_proj", + "model.layers.59.mlp.experts.213.up_proj", + "model.layers.59.mlp.experts.214.down_proj", + "model.layers.59.mlp.experts.214.gate_proj", + "model.layers.59.mlp.experts.214.up_proj", + "model.layers.59.mlp.experts.215.down_proj", + "model.layers.59.mlp.experts.215.gate_proj", + "model.layers.59.mlp.experts.215.up_proj", + "model.layers.59.mlp.experts.216.down_proj", + "model.layers.59.mlp.experts.216.gate_proj", + "model.layers.59.mlp.experts.216.up_proj", + "model.layers.59.mlp.experts.217.down_proj", + "model.layers.59.mlp.experts.217.gate_proj", + "model.layers.59.mlp.experts.217.up_proj", + "model.layers.59.mlp.experts.218.down_proj", + "model.layers.59.mlp.experts.218.gate_proj", + "model.layers.59.mlp.experts.218.up_proj", + "model.layers.59.mlp.experts.219.down_proj", + "model.layers.59.mlp.experts.219.gate_proj", + "model.layers.59.mlp.experts.219.up_proj", + "model.layers.59.mlp.experts.22.down_proj", + "model.layers.59.mlp.experts.22.gate_proj", + "model.layers.59.mlp.experts.22.up_proj", + "model.layers.59.mlp.experts.220.down_proj", + "model.layers.59.mlp.experts.220.gate_proj", + "model.layers.59.mlp.experts.220.up_proj", + "model.layers.59.mlp.experts.221.down_proj", + "model.layers.59.mlp.experts.221.gate_proj", + "model.layers.59.mlp.experts.221.up_proj", + "model.layers.59.mlp.experts.222.down_proj", + "model.layers.59.mlp.experts.222.gate_proj", + "model.layers.59.mlp.experts.222.up_proj", + "model.layers.59.mlp.experts.223.down_proj", + "model.layers.59.mlp.experts.223.gate_proj", + "model.layers.59.mlp.experts.223.up_proj", + "model.layers.59.mlp.experts.224.down_proj", + "model.layers.59.mlp.experts.224.gate_proj", + "model.layers.59.mlp.experts.224.up_proj", + "model.layers.59.mlp.experts.225.down_proj", + "model.layers.59.mlp.experts.225.gate_proj", + "model.layers.59.mlp.experts.225.up_proj", + "model.layers.59.mlp.experts.226.down_proj", + "model.layers.59.mlp.experts.226.gate_proj", + "model.layers.59.mlp.experts.226.up_proj", + "model.layers.59.mlp.experts.227.down_proj", + "model.layers.59.mlp.experts.227.gate_proj", + "model.layers.59.mlp.experts.227.up_proj", + "model.layers.59.mlp.experts.228.down_proj", + "model.layers.59.mlp.experts.228.gate_proj", + "model.layers.59.mlp.experts.228.up_proj", + "model.layers.59.mlp.experts.229.down_proj", + "model.layers.59.mlp.experts.229.gate_proj", + "model.layers.59.mlp.experts.229.up_proj", + "model.layers.59.mlp.experts.23.down_proj", + "model.layers.59.mlp.experts.23.gate_proj", + "model.layers.59.mlp.experts.23.up_proj", + "model.layers.59.mlp.experts.230.down_proj", + "model.layers.59.mlp.experts.230.gate_proj", + "model.layers.59.mlp.experts.230.up_proj", + "model.layers.59.mlp.experts.231.down_proj", + "model.layers.59.mlp.experts.231.gate_proj", + "model.layers.59.mlp.experts.231.up_proj", + "model.layers.59.mlp.experts.232.down_proj", + "model.layers.59.mlp.experts.232.gate_proj", + "model.layers.59.mlp.experts.232.up_proj", + "model.layers.59.mlp.experts.233.down_proj", + "model.layers.59.mlp.experts.233.gate_proj", + "model.layers.59.mlp.experts.233.up_proj", + "model.layers.59.mlp.experts.234.down_proj", + "model.layers.59.mlp.experts.234.gate_proj", + "model.layers.59.mlp.experts.234.up_proj", + "model.layers.59.mlp.experts.235.down_proj", + "model.layers.59.mlp.experts.235.gate_proj", + "model.layers.59.mlp.experts.235.up_proj", + "model.layers.59.mlp.experts.236.down_proj", + "model.layers.59.mlp.experts.236.gate_proj", + "model.layers.59.mlp.experts.236.up_proj", + "model.layers.59.mlp.experts.237.down_proj", + "model.layers.59.mlp.experts.237.gate_proj", + "model.layers.59.mlp.experts.237.up_proj", + "model.layers.59.mlp.experts.238.down_proj", + "model.layers.59.mlp.experts.238.gate_proj", + "model.layers.59.mlp.experts.238.up_proj", + "model.layers.59.mlp.experts.239.down_proj", + "model.layers.59.mlp.experts.239.gate_proj", + "model.layers.59.mlp.experts.239.up_proj", + "model.layers.59.mlp.experts.24.down_proj", + "model.layers.59.mlp.experts.24.gate_proj", + "model.layers.59.mlp.experts.24.up_proj", + "model.layers.59.mlp.experts.240.down_proj", + "model.layers.59.mlp.experts.240.gate_proj", + "model.layers.59.mlp.experts.240.up_proj", + "model.layers.59.mlp.experts.241.down_proj", + "model.layers.59.mlp.experts.241.gate_proj", + "model.layers.59.mlp.experts.241.up_proj", + "model.layers.59.mlp.experts.242.down_proj", + "model.layers.59.mlp.experts.242.gate_proj", + "model.layers.59.mlp.experts.242.up_proj", + "model.layers.59.mlp.experts.243.down_proj", + "model.layers.59.mlp.experts.243.gate_proj", + "model.layers.59.mlp.experts.243.up_proj", + "model.layers.59.mlp.experts.244.down_proj", + "model.layers.59.mlp.experts.244.gate_proj", + "model.layers.59.mlp.experts.244.up_proj", + "model.layers.59.mlp.experts.245.down_proj", + "model.layers.59.mlp.experts.245.gate_proj", + "model.layers.59.mlp.experts.245.up_proj", + "model.layers.59.mlp.experts.246.down_proj", + "model.layers.59.mlp.experts.246.gate_proj", + "model.layers.59.mlp.experts.246.up_proj", + "model.layers.59.mlp.experts.247.down_proj", + "model.layers.59.mlp.experts.247.gate_proj", + "model.layers.59.mlp.experts.247.up_proj", + "model.layers.59.mlp.experts.248.down_proj", + "model.layers.59.mlp.experts.248.gate_proj", + "model.layers.59.mlp.experts.248.up_proj", + "model.layers.59.mlp.experts.249.down_proj", + "model.layers.59.mlp.experts.249.gate_proj", + "model.layers.59.mlp.experts.249.up_proj", + "model.layers.59.mlp.experts.25.down_proj", + "model.layers.59.mlp.experts.25.gate_proj", + "model.layers.59.mlp.experts.25.up_proj", + "model.layers.59.mlp.experts.250.down_proj", + "model.layers.59.mlp.experts.250.gate_proj", + "model.layers.59.mlp.experts.250.up_proj", + "model.layers.59.mlp.experts.251.down_proj", + "model.layers.59.mlp.experts.251.gate_proj", + "model.layers.59.mlp.experts.251.up_proj", + "model.layers.59.mlp.experts.252.down_proj", + "model.layers.59.mlp.experts.252.gate_proj", + "model.layers.59.mlp.experts.252.up_proj", + "model.layers.59.mlp.experts.253.down_proj", + "model.layers.59.mlp.experts.253.gate_proj", + "model.layers.59.mlp.experts.253.up_proj", + "model.layers.59.mlp.experts.254.down_proj", + "model.layers.59.mlp.experts.254.gate_proj", + "model.layers.59.mlp.experts.254.up_proj", + "model.layers.59.mlp.experts.255.down_proj", + "model.layers.59.mlp.experts.255.gate_proj", + "model.layers.59.mlp.experts.255.up_proj", + "model.layers.59.mlp.experts.26.down_proj", + "model.layers.59.mlp.experts.26.gate_proj", + "model.layers.59.mlp.experts.26.up_proj", + "model.layers.59.mlp.experts.27.down_proj", + "model.layers.59.mlp.experts.27.gate_proj", + "model.layers.59.mlp.experts.27.up_proj", + "model.layers.59.mlp.experts.28.down_proj", + "model.layers.59.mlp.experts.28.gate_proj", + "model.layers.59.mlp.experts.28.up_proj", + "model.layers.59.mlp.experts.29.down_proj", + "model.layers.59.mlp.experts.29.gate_proj", + "model.layers.59.mlp.experts.29.up_proj", + "model.layers.59.mlp.experts.3.down_proj", + "model.layers.59.mlp.experts.3.gate_proj", + "model.layers.59.mlp.experts.3.up_proj", + "model.layers.59.mlp.experts.30.down_proj", + "model.layers.59.mlp.experts.30.gate_proj", + "model.layers.59.mlp.experts.30.up_proj", + "model.layers.59.mlp.experts.31.down_proj", + "model.layers.59.mlp.experts.31.gate_proj", + "model.layers.59.mlp.experts.31.up_proj", + "model.layers.59.mlp.experts.32.down_proj", + "model.layers.59.mlp.experts.32.gate_proj", + "model.layers.59.mlp.experts.32.up_proj", + "model.layers.59.mlp.experts.33.down_proj", + "model.layers.59.mlp.experts.33.gate_proj", + "model.layers.59.mlp.experts.33.up_proj", + "model.layers.59.mlp.experts.34.down_proj", + "model.layers.59.mlp.experts.34.gate_proj", + "model.layers.59.mlp.experts.34.up_proj", + "model.layers.59.mlp.experts.35.down_proj", + "model.layers.59.mlp.experts.35.gate_proj", + "model.layers.59.mlp.experts.35.up_proj", + "model.layers.59.mlp.experts.36.down_proj", + "model.layers.59.mlp.experts.36.gate_proj", + "model.layers.59.mlp.experts.36.up_proj", + "model.layers.59.mlp.experts.37.down_proj", + "model.layers.59.mlp.experts.37.gate_proj", + "model.layers.59.mlp.experts.37.up_proj", + "model.layers.59.mlp.experts.38.down_proj", + "model.layers.59.mlp.experts.38.gate_proj", + "model.layers.59.mlp.experts.38.up_proj", + "model.layers.59.mlp.experts.39.down_proj", + "model.layers.59.mlp.experts.39.gate_proj", + "model.layers.59.mlp.experts.39.up_proj", + "model.layers.59.mlp.experts.4.down_proj", + "model.layers.59.mlp.experts.4.gate_proj", + "model.layers.59.mlp.experts.4.up_proj", + "model.layers.59.mlp.experts.40.down_proj", + "model.layers.59.mlp.experts.40.gate_proj", + "model.layers.59.mlp.experts.40.up_proj", + "model.layers.59.mlp.experts.41.down_proj", + "model.layers.59.mlp.experts.41.gate_proj", + "model.layers.59.mlp.experts.41.up_proj", + "model.layers.59.mlp.experts.42.down_proj", + "model.layers.59.mlp.experts.42.gate_proj", + "model.layers.59.mlp.experts.42.up_proj", + "model.layers.59.mlp.experts.43.down_proj", + "model.layers.59.mlp.experts.43.gate_proj", + "model.layers.59.mlp.experts.43.up_proj", + "model.layers.59.mlp.experts.44.down_proj", + "model.layers.59.mlp.experts.44.gate_proj", + "model.layers.59.mlp.experts.44.up_proj", + "model.layers.59.mlp.experts.45.down_proj", + "model.layers.59.mlp.experts.45.gate_proj", + "model.layers.59.mlp.experts.45.up_proj", + "model.layers.59.mlp.experts.46.down_proj", + "model.layers.59.mlp.experts.46.gate_proj", + "model.layers.59.mlp.experts.46.up_proj", + "model.layers.59.mlp.experts.47.down_proj", + "model.layers.59.mlp.experts.47.gate_proj", + "model.layers.59.mlp.experts.47.up_proj", + "model.layers.59.mlp.experts.48.down_proj", + "model.layers.59.mlp.experts.48.gate_proj", + "model.layers.59.mlp.experts.48.up_proj", + "model.layers.59.mlp.experts.49.down_proj", + "model.layers.59.mlp.experts.49.gate_proj", + "model.layers.59.mlp.experts.49.up_proj", + "model.layers.59.mlp.experts.5.down_proj", + "model.layers.59.mlp.experts.5.gate_proj", + "model.layers.59.mlp.experts.5.up_proj", + "model.layers.59.mlp.experts.50.down_proj", + "model.layers.59.mlp.experts.50.gate_proj", + "model.layers.59.mlp.experts.50.up_proj", + "model.layers.59.mlp.experts.51.down_proj", + "model.layers.59.mlp.experts.51.gate_proj", + "model.layers.59.mlp.experts.51.up_proj", + "model.layers.59.mlp.experts.52.down_proj", + "model.layers.59.mlp.experts.52.gate_proj", + "model.layers.59.mlp.experts.52.up_proj", + "model.layers.59.mlp.experts.53.down_proj", + "model.layers.59.mlp.experts.53.gate_proj", + "model.layers.59.mlp.experts.53.up_proj", + "model.layers.59.mlp.experts.54.down_proj", + "model.layers.59.mlp.experts.54.gate_proj", + "model.layers.59.mlp.experts.54.up_proj", + "model.layers.59.mlp.experts.55.down_proj", + "model.layers.59.mlp.experts.55.gate_proj", + "model.layers.59.mlp.experts.55.up_proj", + "model.layers.59.mlp.experts.56.down_proj", + "model.layers.59.mlp.experts.56.gate_proj", + "model.layers.59.mlp.experts.56.up_proj", + "model.layers.59.mlp.experts.57.down_proj", + "model.layers.59.mlp.experts.57.gate_proj", + "model.layers.59.mlp.experts.57.up_proj", + "model.layers.59.mlp.experts.58.down_proj", + "model.layers.59.mlp.experts.58.gate_proj", + "model.layers.59.mlp.experts.58.up_proj", + "model.layers.59.mlp.experts.59.down_proj", + "model.layers.59.mlp.experts.59.gate_proj", + "model.layers.59.mlp.experts.59.up_proj", + "model.layers.59.mlp.experts.6.down_proj", + "model.layers.59.mlp.experts.6.gate_proj", + "model.layers.59.mlp.experts.6.up_proj", + "model.layers.59.mlp.experts.60.down_proj", + "model.layers.59.mlp.experts.60.gate_proj", + "model.layers.59.mlp.experts.60.up_proj", + "model.layers.59.mlp.experts.61.down_proj", + "model.layers.59.mlp.experts.61.gate_proj", + "model.layers.59.mlp.experts.61.up_proj", + "model.layers.59.mlp.experts.62.down_proj", + "model.layers.59.mlp.experts.62.gate_proj", + "model.layers.59.mlp.experts.62.up_proj", + "model.layers.59.mlp.experts.63.down_proj", + "model.layers.59.mlp.experts.63.gate_proj", + "model.layers.59.mlp.experts.63.up_proj", + "model.layers.59.mlp.experts.64.down_proj", + "model.layers.59.mlp.experts.64.gate_proj", + "model.layers.59.mlp.experts.64.up_proj", + "model.layers.59.mlp.experts.65.down_proj", + "model.layers.59.mlp.experts.65.gate_proj", + "model.layers.59.mlp.experts.65.up_proj", + "model.layers.59.mlp.experts.66.down_proj", + "model.layers.59.mlp.experts.66.gate_proj", + "model.layers.59.mlp.experts.66.up_proj", + "model.layers.59.mlp.experts.67.down_proj", + "model.layers.59.mlp.experts.67.gate_proj", + "model.layers.59.mlp.experts.67.up_proj", + "model.layers.59.mlp.experts.68.down_proj", + "model.layers.59.mlp.experts.68.gate_proj", + "model.layers.59.mlp.experts.68.up_proj", + "model.layers.59.mlp.experts.69.down_proj", + "model.layers.59.mlp.experts.69.gate_proj", + "model.layers.59.mlp.experts.69.up_proj", + "model.layers.59.mlp.experts.7.down_proj", + "model.layers.59.mlp.experts.7.gate_proj", + "model.layers.59.mlp.experts.7.up_proj", + "model.layers.59.mlp.experts.70.down_proj", + "model.layers.59.mlp.experts.70.gate_proj", + "model.layers.59.mlp.experts.70.up_proj", + "model.layers.59.mlp.experts.71.down_proj", + "model.layers.59.mlp.experts.71.gate_proj", + "model.layers.59.mlp.experts.71.up_proj", + "model.layers.59.mlp.experts.72.down_proj", + "model.layers.59.mlp.experts.72.gate_proj", + "model.layers.59.mlp.experts.72.up_proj", + "model.layers.59.mlp.experts.73.down_proj", + "model.layers.59.mlp.experts.73.gate_proj", + "model.layers.59.mlp.experts.73.up_proj", + "model.layers.59.mlp.experts.74.down_proj", + "model.layers.59.mlp.experts.74.gate_proj", + "model.layers.59.mlp.experts.74.up_proj", + "model.layers.59.mlp.experts.75.down_proj", + "model.layers.59.mlp.experts.75.gate_proj", + "model.layers.59.mlp.experts.75.up_proj", + "model.layers.59.mlp.experts.76.down_proj", + "model.layers.59.mlp.experts.76.gate_proj", + "model.layers.59.mlp.experts.76.up_proj", + "model.layers.59.mlp.experts.77.down_proj", + "model.layers.59.mlp.experts.77.gate_proj", + "model.layers.59.mlp.experts.77.up_proj", + "model.layers.59.mlp.experts.78.down_proj", + "model.layers.59.mlp.experts.78.gate_proj", + "model.layers.59.mlp.experts.78.up_proj", + "model.layers.59.mlp.experts.79.down_proj", + "model.layers.59.mlp.experts.79.gate_proj", + "model.layers.59.mlp.experts.79.up_proj", + "model.layers.59.mlp.experts.8.down_proj", + "model.layers.59.mlp.experts.8.gate_proj", + "model.layers.59.mlp.experts.8.up_proj", + "model.layers.59.mlp.experts.80.down_proj", + "model.layers.59.mlp.experts.80.gate_proj", + "model.layers.59.mlp.experts.80.up_proj", + "model.layers.59.mlp.experts.81.down_proj", + "model.layers.59.mlp.experts.81.gate_proj", + "model.layers.59.mlp.experts.81.up_proj", + "model.layers.59.mlp.experts.82.down_proj", + "model.layers.59.mlp.experts.82.gate_proj", + "model.layers.59.mlp.experts.82.up_proj", + "model.layers.59.mlp.experts.83.down_proj", + "model.layers.59.mlp.experts.83.gate_proj", + "model.layers.59.mlp.experts.83.up_proj", + "model.layers.59.mlp.experts.84.down_proj", + "model.layers.59.mlp.experts.84.gate_proj", + "model.layers.59.mlp.experts.84.up_proj", + "model.layers.59.mlp.experts.85.down_proj", + "model.layers.59.mlp.experts.85.gate_proj", + "model.layers.59.mlp.experts.85.up_proj", + "model.layers.59.mlp.experts.86.down_proj", + "model.layers.59.mlp.experts.86.gate_proj", + "model.layers.59.mlp.experts.86.up_proj", + "model.layers.59.mlp.experts.87.down_proj", + "model.layers.59.mlp.experts.87.gate_proj", + "model.layers.59.mlp.experts.87.up_proj", + "model.layers.59.mlp.experts.88.down_proj", + "model.layers.59.mlp.experts.88.gate_proj", + "model.layers.59.mlp.experts.88.up_proj", + "model.layers.59.mlp.experts.89.down_proj", + "model.layers.59.mlp.experts.89.gate_proj", + "model.layers.59.mlp.experts.89.up_proj", + "model.layers.59.mlp.experts.9.down_proj", + "model.layers.59.mlp.experts.9.gate_proj", + "model.layers.59.mlp.experts.9.up_proj", + "model.layers.59.mlp.experts.90.down_proj", + "model.layers.59.mlp.experts.90.gate_proj", + "model.layers.59.mlp.experts.90.up_proj", + "model.layers.59.mlp.experts.91.down_proj", + "model.layers.59.mlp.experts.91.gate_proj", + "model.layers.59.mlp.experts.91.up_proj", + "model.layers.59.mlp.experts.92.down_proj", + "model.layers.59.mlp.experts.92.gate_proj", + "model.layers.59.mlp.experts.92.up_proj", + "model.layers.59.mlp.experts.93.down_proj", + "model.layers.59.mlp.experts.93.gate_proj", + "model.layers.59.mlp.experts.93.up_proj", + "model.layers.59.mlp.experts.94.down_proj", + "model.layers.59.mlp.experts.94.gate_proj", + "model.layers.59.mlp.experts.94.up_proj", + "model.layers.59.mlp.experts.95.down_proj", + "model.layers.59.mlp.experts.95.gate_proj", + "model.layers.59.mlp.experts.95.up_proj", + "model.layers.59.mlp.experts.96.down_proj", + "model.layers.59.mlp.experts.96.gate_proj", + "model.layers.59.mlp.experts.96.up_proj", + "model.layers.59.mlp.experts.97.down_proj", + "model.layers.59.mlp.experts.97.gate_proj", + "model.layers.59.mlp.experts.97.up_proj", + "model.layers.59.mlp.experts.98.down_proj", + "model.layers.59.mlp.experts.98.gate_proj", + "model.layers.59.mlp.experts.98.up_proj", + "model.layers.59.mlp.experts.99.down_proj", + "model.layers.59.mlp.experts.99.gate_proj", + "model.layers.59.mlp.experts.99.up_proj", + "model.layers.6.mlp.experts.0.down_proj", + "model.layers.6.mlp.experts.0.gate_proj", + "model.layers.6.mlp.experts.0.up_proj", + "model.layers.6.mlp.experts.1.down_proj", + "model.layers.6.mlp.experts.1.gate_proj", + "model.layers.6.mlp.experts.1.up_proj", + "model.layers.6.mlp.experts.10.down_proj", + "model.layers.6.mlp.experts.10.gate_proj", + "model.layers.6.mlp.experts.10.up_proj", + "model.layers.6.mlp.experts.100.down_proj", + "model.layers.6.mlp.experts.100.gate_proj", + "model.layers.6.mlp.experts.100.up_proj", + "model.layers.6.mlp.experts.101.down_proj", + "model.layers.6.mlp.experts.101.gate_proj", + "model.layers.6.mlp.experts.101.up_proj", + "model.layers.6.mlp.experts.102.down_proj", + "model.layers.6.mlp.experts.102.gate_proj", + "model.layers.6.mlp.experts.102.up_proj", + "model.layers.6.mlp.experts.103.down_proj", + "model.layers.6.mlp.experts.103.gate_proj", + "model.layers.6.mlp.experts.103.up_proj", + "model.layers.6.mlp.experts.104.down_proj", + "model.layers.6.mlp.experts.104.gate_proj", + "model.layers.6.mlp.experts.104.up_proj", + "model.layers.6.mlp.experts.105.down_proj", + "model.layers.6.mlp.experts.105.gate_proj", + "model.layers.6.mlp.experts.105.up_proj", + "model.layers.6.mlp.experts.106.down_proj", + "model.layers.6.mlp.experts.106.gate_proj", + "model.layers.6.mlp.experts.106.up_proj", + "model.layers.6.mlp.experts.107.down_proj", + "model.layers.6.mlp.experts.107.gate_proj", + "model.layers.6.mlp.experts.107.up_proj", + "model.layers.6.mlp.experts.108.down_proj", + "model.layers.6.mlp.experts.108.gate_proj", + "model.layers.6.mlp.experts.108.up_proj", + "model.layers.6.mlp.experts.109.down_proj", + "model.layers.6.mlp.experts.109.gate_proj", + "model.layers.6.mlp.experts.109.up_proj", + "model.layers.6.mlp.experts.11.down_proj", + "model.layers.6.mlp.experts.11.gate_proj", + "model.layers.6.mlp.experts.11.up_proj", + "model.layers.6.mlp.experts.110.down_proj", + "model.layers.6.mlp.experts.110.gate_proj", + "model.layers.6.mlp.experts.110.up_proj", + "model.layers.6.mlp.experts.111.down_proj", + "model.layers.6.mlp.experts.111.gate_proj", + "model.layers.6.mlp.experts.111.up_proj", + "model.layers.6.mlp.experts.112.down_proj", + "model.layers.6.mlp.experts.112.gate_proj", + "model.layers.6.mlp.experts.112.up_proj", + "model.layers.6.mlp.experts.113.down_proj", + "model.layers.6.mlp.experts.113.gate_proj", + "model.layers.6.mlp.experts.113.up_proj", + "model.layers.6.mlp.experts.114.down_proj", + "model.layers.6.mlp.experts.114.gate_proj", + "model.layers.6.mlp.experts.114.up_proj", + "model.layers.6.mlp.experts.115.down_proj", + "model.layers.6.mlp.experts.115.gate_proj", + "model.layers.6.mlp.experts.115.up_proj", + "model.layers.6.mlp.experts.116.down_proj", + "model.layers.6.mlp.experts.116.gate_proj", + "model.layers.6.mlp.experts.116.up_proj", + "model.layers.6.mlp.experts.117.down_proj", + "model.layers.6.mlp.experts.117.gate_proj", + "model.layers.6.mlp.experts.117.up_proj", + "model.layers.6.mlp.experts.118.down_proj", + "model.layers.6.mlp.experts.118.gate_proj", + "model.layers.6.mlp.experts.118.up_proj", + "model.layers.6.mlp.experts.119.down_proj", + "model.layers.6.mlp.experts.119.gate_proj", + "model.layers.6.mlp.experts.119.up_proj", + "model.layers.6.mlp.experts.12.down_proj", + "model.layers.6.mlp.experts.12.gate_proj", + "model.layers.6.mlp.experts.12.up_proj", + "model.layers.6.mlp.experts.120.down_proj", + "model.layers.6.mlp.experts.120.gate_proj", + "model.layers.6.mlp.experts.120.up_proj", + "model.layers.6.mlp.experts.121.down_proj", + "model.layers.6.mlp.experts.121.gate_proj", + "model.layers.6.mlp.experts.121.up_proj", + "model.layers.6.mlp.experts.122.down_proj", + "model.layers.6.mlp.experts.122.gate_proj", + "model.layers.6.mlp.experts.122.up_proj", + "model.layers.6.mlp.experts.123.down_proj", + "model.layers.6.mlp.experts.123.gate_proj", + "model.layers.6.mlp.experts.123.up_proj", + "model.layers.6.mlp.experts.124.down_proj", + "model.layers.6.mlp.experts.124.gate_proj", + "model.layers.6.mlp.experts.124.up_proj", + "model.layers.6.mlp.experts.125.down_proj", + "model.layers.6.mlp.experts.125.gate_proj", + "model.layers.6.mlp.experts.125.up_proj", + "model.layers.6.mlp.experts.126.down_proj", + "model.layers.6.mlp.experts.126.gate_proj", + "model.layers.6.mlp.experts.126.up_proj", + "model.layers.6.mlp.experts.127.down_proj", + "model.layers.6.mlp.experts.127.gate_proj", + "model.layers.6.mlp.experts.127.up_proj", + "model.layers.6.mlp.experts.128.down_proj", + "model.layers.6.mlp.experts.128.gate_proj", + "model.layers.6.mlp.experts.128.up_proj", + "model.layers.6.mlp.experts.129.down_proj", + "model.layers.6.mlp.experts.129.gate_proj", + "model.layers.6.mlp.experts.129.up_proj", + "model.layers.6.mlp.experts.13.down_proj", + "model.layers.6.mlp.experts.13.gate_proj", + "model.layers.6.mlp.experts.13.up_proj", + "model.layers.6.mlp.experts.130.down_proj", + "model.layers.6.mlp.experts.130.gate_proj", + "model.layers.6.mlp.experts.130.up_proj", + "model.layers.6.mlp.experts.131.down_proj", + "model.layers.6.mlp.experts.131.gate_proj", + "model.layers.6.mlp.experts.131.up_proj", + "model.layers.6.mlp.experts.132.down_proj", + "model.layers.6.mlp.experts.132.gate_proj", + "model.layers.6.mlp.experts.132.up_proj", + "model.layers.6.mlp.experts.133.down_proj", + "model.layers.6.mlp.experts.133.gate_proj", + "model.layers.6.mlp.experts.133.up_proj", + "model.layers.6.mlp.experts.134.down_proj", + "model.layers.6.mlp.experts.134.gate_proj", + "model.layers.6.mlp.experts.134.up_proj", + "model.layers.6.mlp.experts.135.down_proj", + "model.layers.6.mlp.experts.135.gate_proj", + "model.layers.6.mlp.experts.135.up_proj", + "model.layers.6.mlp.experts.136.down_proj", + "model.layers.6.mlp.experts.136.gate_proj", + "model.layers.6.mlp.experts.136.up_proj", + "model.layers.6.mlp.experts.137.down_proj", + "model.layers.6.mlp.experts.137.gate_proj", + "model.layers.6.mlp.experts.137.up_proj", + "model.layers.6.mlp.experts.138.down_proj", + "model.layers.6.mlp.experts.138.gate_proj", + "model.layers.6.mlp.experts.138.up_proj", + "model.layers.6.mlp.experts.139.down_proj", + "model.layers.6.mlp.experts.139.gate_proj", + "model.layers.6.mlp.experts.139.up_proj", + "model.layers.6.mlp.experts.14.down_proj", + "model.layers.6.mlp.experts.14.gate_proj", + "model.layers.6.mlp.experts.14.up_proj", + "model.layers.6.mlp.experts.140.down_proj", + "model.layers.6.mlp.experts.140.gate_proj", + "model.layers.6.mlp.experts.140.up_proj", + "model.layers.6.mlp.experts.141.down_proj", + "model.layers.6.mlp.experts.141.gate_proj", + "model.layers.6.mlp.experts.141.up_proj", + "model.layers.6.mlp.experts.142.down_proj", + "model.layers.6.mlp.experts.142.gate_proj", + "model.layers.6.mlp.experts.142.up_proj", + "model.layers.6.mlp.experts.143.down_proj", + "model.layers.6.mlp.experts.143.gate_proj", + "model.layers.6.mlp.experts.143.up_proj", + "model.layers.6.mlp.experts.144.down_proj", + "model.layers.6.mlp.experts.144.gate_proj", + "model.layers.6.mlp.experts.144.up_proj", + "model.layers.6.mlp.experts.145.down_proj", + "model.layers.6.mlp.experts.145.gate_proj", + "model.layers.6.mlp.experts.145.up_proj", + "model.layers.6.mlp.experts.146.down_proj", + "model.layers.6.mlp.experts.146.gate_proj", + "model.layers.6.mlp.experts.146.up_proj", + "model.layers.6.mlp.experts.147.down_proj", + "model.layers.6.mlp.experts.147.gate_proj", + "model.layers.6.mlp.experts.147.up_proj", + "model.layers.6.mlp.experts.148.down_proj", + "model.layers.6.mlp.experts.148.gate_proj", + "model.layers.6.mlp.experts.148.up_proj", + "model.layers.6.mlp.experts.149.down_proj", + "model.layers.6.mlp.experts.149.gate_proj", + "model.layers.6.mlp.experts.149.up_proj", + "model.layers.6.mlp.experts.15.down_proj", + "model.layers.6.mlp.experts.15.gate_proj", + "model.layers.6.mlp.experts.15.up_proj", + "model.layers.6.mlp.experts.150.down_proj", + "model.layers.6.mlp.experts.150.gate_proj", + "model.layers.6.mlp.experts.150.up_proj", + "model.layers.6.mlp.experts.151.down_proj", + "model.layers.6.mlp.experts.151.gate_proj", + "model.layers.6.mlp.experts.151.up_proj", + "model.layers.6.mlp.experts.152.down_proj", + "model.layers.6.mlp.experts.152.gate_proj", + "model.layers.6.mlp.experts.152.up_proj", + "model.layers.6.mlp.experts.153.down_proj", + "model.layers.6.mlp.experts.153.gate_proj", + "model.layers.6.mlp.experts.153.up_proj", + "model.layers.6.mlp.experts.154.down_proj", + "model.layers.6.mlp.experts.154.gate_proj", + "model.layers.6.mlp.experts.154.up_proj", + "model.layers.6.mlp.experts.155.down_proj", + "model.layers.6.mlp.experts.155.gate_proj", + "model.layers.6.mlp.experts.155.up_proj", + "model.layers.6.mlp.experts.156.down_proj", + "model.layers.6.mlp.experts.156.gate_proj", + "model.layers.6.mlp.experts.156.up_proj", + "model.layers.6.mlp.experts.157.down_proj", + "model.layers.6.mlp.experts.157.gate_proj", + "model.layers.6.mlp.experts.157.up_proj", + "model.layers.6.mlp.experts.158.down_proj", + "model.layers.6.mlp.experts.158.gate_proj", + "model.layers.6.mlp.experts.158.up_proj", + "model.layers.6.mlp.experts.159.down_proj", + "model.layers.6.mlp.experts.159.gate_proj", + "model.layers.6.mlp.experts.159.up_proj", + "model.layers.6.mlp.experts.16.down_proj", + "model.layers.6.mlp.experts.16.gate_proj", + "model.layers.6.mlp.experts.16.up_proj", + "model.layers.6.mlp.experts.160.down_proj", + "model.layers.6.mlp.experts.160.gate_proj", + "model.layers.6.mlp.experts.160.up_proj", + "model.layers.6.mlp.experts.161.down_proj", + "model.layers.6.mlp.experts.161.gate_proj", + "model.layers.6.mlp.experts.161.up_proj", + "model.layers.6.mlp.experts.162.down_proj", + "model.layers.6.mlp.experts.162.gate_proj", + "model.layers.6.mlp.experts.162.up_proj", + "model.layers.6.mlp.experts.163.down_proj", + "model.layers.6.mlp.experts.163.gate_proj", + "model.layers.6.mlp.experts.163.up_proj", + "model.layers.6.mlp.experts.164.down_proj", + "model.layers.6.mlp.experts.164.gate_proj", + "model.layers.6.mlp.experts.164.up_proj", + "model.layers.6.mlp.experts.165.down_proj", + "model.layers.6.mlp.experts.165.gate_proj", + "model.layers.6.mlp.experts.165.up_proj", + "model.layers.6.mlp.experts.166.down_proj", + "model.layers.6.mlp.experts.166.gate_proj", + "model.layers.6.mlp.experts.166.up_proj", + "model.layers.6.mlp.experts.167.down_proj", + "model.layers.6.mlp.experts.167.gate_proj", + "model.layers.6.mlp.experts.167.up_proj", + "model.layers.6.mlp.experts.168.down_proj", + "model.layers.6.mlp.experts.168.gate_proj", + "model.layers.6.mlp.experts.168.up_proj", + "model.layers.6.mlp.experts.169.down_proj", + "model.layers.6.mlp.experts.169.gate_proj", + "model.layers.6.mlp.experts.169.up_proj", + "model.layers.6.mlp.experts.17.down_proj", + "model.layers.6.mlp.experts.17.gate_proj", + "model.layers.6.mlp.experts.17.up_proj", + "model.layers.6.mlp.experts.170.down_proj", + "model.layers.6.mlp.experts.170.gate_proj", + "model.layers.6.mlp.experts.170.up_proj", + "model.layers.6.mlp.experts.171.down_proj", + "model.layers.6.mlp.experts.171.gate_proj", + "model.layers.6.mlp.experts.171.up_proj", + "model.layers.6.mlp.experts.172.down_proj", + "model.layers.6.mlp.experts.172.gate_proj", + "model.layers.6.mlp.experts.172.up_proj", + "model.layers.6.mlp.experts.173.down_proj", + "model.layers.6.mlp.experts.173.gate_proj", + "model.layers.6.mlp.experts.173.up_proj", + "model.layers.6.mlp.experts.174.down_proj", + "model.layers.6.mlp.experts.174.gate_proj", + "model.layers.6.mlp.experts.174.up_proj", + "model.layers.6.mlp.experts.175.down_proj", + "model.layers.6.mlp.experts.175.gate_proj", + "model.layers.6.mlp.experts.175.up_proj", + "model.layers.6.mlp.experts.176.down_proj", + "model.layers.6.mlp.experts.176.gate_proj", + "model.layers.6.mlp.experts.176.up_proj", + "model.layers.6.mlp.experts.177.down_proj", + "model.layers.6.mlp.experts.177.gate_proj", + "model.layers.6.mlp.experts.177.up_proj", + "model.layers.6.mlp.experts.178.down_proj", + "model.layers.6.mlp.experts.178.gate_proj", + "model.layers.6.mlp.experts.178.up_proj", + "model.layers.6.mlp.experts.179.down_proj", + "model.layers.6.mlp.experts.179.gate_proj", + "model.layers.6.mlp.experts.179.up_proj", + "model.layers.6.mlp.experts.18.down_proj", + "model.layers.6.mlp.experts.18.gate_proj", + "model.layers.6.mlp.experts.18.up_proj", + "model.layers.6.mlp.experts.180.down_proj", + "model.layers.6.mlp.experts.180.gate_proj", + "model.layers.6.mlp.experts.180.up_proj", + "model.layers.6.mlp.experts.181.down_proj", + "model.layers.6.mlp.experts.181.gate_proj", + "model.layers.6.mlp.experts.181.up_proj", + "model.layers.6.mlp.experts.182.down_proj", + "model.layers.6.mlp.experts.182.gate_proj", + "model.layers.6.mlp.experts.182.up_proj", + "model.layers.6.mlp.experts.183.down_proj", + "model.layers.6.mlp.experts.183.gate_proj", + "model.layers.6.mlp.experts.183.up_proj", + "model.layers.6.mlp.experts.184.down_proj", + "model.layers.6.mlp.experts.184.gate_proj", + "model.layers.6.mlp.experts.184.up_proj", + "model.layers.6.mlp.experts.185.down_proj", + "model.layers.6.mlp.experts.185.gate_proj", + "model.layers.6.mlp.experts.185.up_proj", + "model.layers.6.mlp.experts.186.down_proj", + "model.layers.6.mlp.experts.186.gate_proj", + "model.layers.6.mlp.experts.186.up_proj", + "model.layers.6.mlp.experts.187.down_proj", + "model.layers.6.mlp.experts.187.gate_proj", + "model.layers.6.mlp.experts.187.up_proj", + "model.layers.6.mlp.experts.188.down_proj", + "model.layers.6.mlp.experts.188.gate_proj", + "model.layers.6.mlp.experts.188.up_proj", + "model.layers.6.mlp.experts.189.down_proj", + "model.layers.6.mlp.experts.189.gate_proj", + "model.layers.6.mlp.experts.189.up_proj", + "model.layers.6.mlp.experts.19.down_proj", + "model.layers.6.mlp.experts.19.gate_proj", + "model.layers.6.mlp.experts.19.up_proj", + "model.layers.6.mlp.experts.190.down_proj", + "model.layers.6.mlp.experts.190.gate_proj", + "model.layers.6.mlp.experts.190.up_proj", + "model.layers.6.mlp.experts.191.down_proj", + "model.layers.6.mlp.experts.191.gate_proj", + "model.layers.6.mlp.experts.191.up_proj", + "model.layers.6.mlp.experts.192.down_proj", + "model.layers.6.mlp.experts.192.gate_proj", + "model.layers.6.mlp.experts.192.up_proj", + "model.layers.6.mlp.experts.193.down_proj", + "model.layers.6.mlp.experts.193.gate_proj", + "model.layers.6.mlp.experts.193.up_proj", + "model.layers.6.mlp.experts.194.down_proj", + "model.layers.6.mlp.experts.194.gate_proj", + "model.layers.6.mlp.experts.194.up_proj", + "model.layers.6.mlp.experts.195.down_proj", + "model.layers.6.mlp.experts.195.gate_proj", + "model.layers.6.mlp.experts.195.up_proj", + "model.layers.6.mlp.experts.196.down_proj", + "model.layers.6.mlp.experts.196.gate_proj", + "model.layers.6.mlp.experts.196.up_proj", + "model.layers.6.mlp.experts.197.down_proj", + "model.layers.6.mlp.experts.197.gate_proj", + "model.layers.6.mlp.experts.197.up_proj", + "model.layers.6.mlp.experts.198.down_proj", + "model.layers.6.mlp.experts.198.gate_proj", + "model.layers.6.mlp.experts.198.up_proj", + "model.layers.6.mlp.experts.199.down_proj", + "model.layers.6.mlp.experts.199.gate_proj", + "model.layers.6.mlp.experts.199.up_proj", + "model.layers.6.mlp.experts.2.down_proj", + "model.layers.6.mlp.experts.2.gate_proj", + "model.layers.6.mlp.experts.2.up_proj", + "model.layers.6.mlp.experts.20.down_proj", + "model.layers.6.mlp.experts.20.gate_proj", + "model.layers.6.mlp.experts.20.up_proj", + "model.layers.6.mlp.experts.200.down_proj", + "model.layers.6.mlp.experts.200.gate_proj", + "model.layers.6.mlp.experts.200.up_proj", + "model.layers.6.mlp.experts.201.down_proj", + "model.layers.6.mlp.experts.201.gate_proj", + "model.layers.6.mlp.experts.201.up_proj", + "model.layers.6.mlp.experts.202.down_proj", + "model.layers.6.mlp.experts.202.gate_proj", + "model.layers.6.mlp.experts.202.up_proj", + "model.layers.6.mlp.experts.203.down_proj", + "model.layers.6.mlp.experts.203.gate_proj", + "model.layers.6.mlp.experts.203.up_proj", + "model.layers.6.mlp.experts.204.down_proj", + "model.layers.6.mlp.experts.204.gate_proj", + "model.layers.6.mlp.experts.204.up_proj", + "model.layers.6.mlp.experts.205.down_proj", + "model.layers.6.mlp.experts.205.gate_proj", + "model.layers.6.mlp.experts.205.up_proj", + "model.layers.6.mlp.experts.206.down_proj", + "model.layers.6.mlp.experts.206.gate_proj", + "model.layers.6.mlp.experts.206.up_proj", + "model.layers.6.mlp.experts.207.down_proj", + "model.layers.6.mlp.experts.207.gate_proj", + "model.layers.6.mlp.experts.207.up_proj", + "model.layers.6.mlp.experts.208.down_proj", + "model.layers.6.mlp.experts.208.gate_proj", + "model.layers.6.mlp.experts.208.up_proj", + "model.layers.6.mlp.experts.209.down_proj", + "model.layers.6.mlp.experts.209.gate_proj", + "model.layers.6.mlp.experts.209.up_proj", + "model.layers.6.mlp.experts.21.down_proj", + "model.layers.6.mlp.experts.21.gate_proj", + "model.layers.6.mlp.experts.21.up_proj", + "model.layers.6.mlp.experts.210.down_proj", + "model.layers.6.mlp.experts.210.gate_proj", + "model.layers.6.mlp.experts.210.up_proj", + "model.layers.6.mlp.experts.211.down_proj", + "model.layers.6.mlp.experts.211.gate_proj", + "model.layers.6.mlp.experts.211.up_proj", + "model.layers.6.mlp.experts.212.down_proj", + "model.layers.6.mlp.experts.212.gate_proj", + "model.layers.6.mlp.experts.212.up_proj", + "model.layers.6.mlp.experts.213.down_proj", + "model.layers.6.mlp.experts.213.gate_proj", + "model.layers.6.mlp.experts.213.up_proj", + "model.layers.6.mlp.experts.214.down_proj", + "model.layers.6.mlp.experts.214.gate_proj", + "model.layers.6.mlp.experts.214.up_proj", + "model.layers.6.mlp.experts.215.down_proj", + "model.layers.6.mlp.experts.215.gate_proj", + "model.layers.6.mlp.experts.215.up_proj", + "model.layers.6.mlp.experts.216.down_proj", + "model.layers.6.mlp.experts.216.gate_proj", + "model.layers.6.mlp.experts.216.up_proj", + "model.layers.6.mlp.experts.217.down_proj", + "model.layers.6.mlp.experts.217.gate_proj", + "model.layers.6.mlp.experts.217.up_proj", + "model.layers.6.mlp.experts.218.down_proj", + "model.layers.6.mlp.experts.218.gate_proj", + "model.layers.6.mlp.experts.218.up_proj", + "model.layers.6.mlp.experts.219.down_proj", + "model.layers.6.mlp.experts.219.gate_proj", + "model.layers.6.mlp.experts.219.up_proj", + "model.layers.6.mlp.experts.22.down_proj", + "model.layers.6.mlp.experts.22.gate_proj", + "model.layers.6.mlp.experts.22.up_proj", + "model.layers.6.mlp.experts.220.down_proj", + "model.layers.6.mlp.experts.220.gate_proj", + "model.layers.6.mlp.experts.220.up_proj", + "model.layers.6.mlp.experts.221.down_proj", + "model.layers.6.mlp.experts.221.gate_proj", + "model.layers.6.mlp.experts.221.up_proj", + "model.layers.6.mlp.experts.222.down_proj", + "model.layers.6.mlp.experts.222.gate_proj", + "model.layers.6.mlp.experts.222.up_proj", + "model.layers.6.mlp.experts.223.down_proj", + "model.layers.6.mlp.experts.223.gate_proj", + "model.layers.6.mlp.experts.223.up_proj", + "model.layers.6.mlp.experts.224.down_proj", + "model.layers.6.mlp.experts.224.gate_proj", + "model.layers.6.mlp.experts.224.up_proj", + "model.layers.6.mlp.experts.225.down_proj", + "model.layers.6.mlp.experts.225.gate_proj", + "model.layers.6.mlp.experts.225.up_proj", + "model.layers.6.mlp.experts.226.down_proj", + "model.layers.6.mlp.experts.226.gate_proj", + "model.layers.6.mlp.experts.226.up_proj", + "model.layers.6.mlp.experts.227.down_proj", + "model.layers.6.mlp.experts.227.gate_proj", + "model.layers.6.mlp.experts.227.up_proj", + "model.layers.6.mlp.experts.228.down_proj", + "model.layers.6.mlp.experts.228.gate_proj", + "model.layers.6.mlp.experts.228.up_proj", + "model.layers.6.mlp.experts.229.down_proj", + "model.layers.6.mlp.experts.229.gate_proj", + "model.layers.6.mlp.experts.229.up_proj", + "model.layers.6.mlp.experts.23.down_proj", + "model.layers.6.mlp.experts.23.gate_proj", + "model.layers.6.mlp.experts.23.up_proj", + "model.layers.6.mlp.experts.230.down_proj", + "model.layers.6.mlp.experts.230.gate_proj", + "model.layers.6.mlp.experts.230.up_proj", + "model.layers.6.mlp.experts.231.down_proj", + "model.layers.6.mlp.experts.231.gate_proj", + "model.layers.6.mlp.experts.231.up_proj", + "model.layers.6.mlp.experts.232.down_proj", + "model.layers.6.mlp.experts.232.gate_proj", + "model.layers.6.mlp.experts.232.up_proj", + "model.layers.6.mlp.experts.233.down_proj", + "model.layers.6.mlp.experts.233.gate_proj", + "model.layers.6.mlp.experts.233.up_proj", + "model.layers.6.mlp.experts.234.down_proj", + "model.layers.6.mlp.experts.234.gate_proj", + "model.layers.6.mlp.experts.234.up_proj", + "model.layers.6.mlp.experts.235.down_proj", + "model.layers.6.mlp.experts.235.gate_proj", + "model.layers.6.mlp.experts.235.up_proj", + "model.layers.6.mlp.experts.236.down_proj", + "model.layers.6.mlp.experts.236.gate_proj", + "model.layers.6.mlp.experts.236.up_proj", + "model.layers.6.mlp.experts.237.down_proj", + "model.layers.6.mlp.experts.237.gate_proj", + "model.layers.6.mlp.experts.237.up_proj", + "model.layers.6.mlp.experts.238.down_proj", + "model.layers.6.mlp.experts.238.gate_proj", + "model.layers.6.mlp.experts.238.up_proj", + "model.layers.6.mlp.experts.239.down_proj", + "model.layers.6.mlp.experts.239.gate_proj", + "model.layers.6.mlp.experts.239.up_proj", + "model.layers.6.mlp.experts.24.down_proj", + "model.layers.6.mlp.experts.24.gate_proj", + "model.layers.6.mlp.experts.24.up_proj", + "model.layers.6.mlp.experts.240.down_proj", + "model.layers.6.mlp.experts.240.gate_proj", + "model.layers.6.mlp.experts.240.up_proj", + "model.layers.6.mlp.experts.241.down_proj", + "model.layers.6.mlp.experts.241.gate_proj", + "model.layers.6.mlp.experts.241.up_proj", + "model.layers.6.mlp.experts.242.down_proj", + "model.layers.6.mlp.experts.242.gate_proj", + "model.layers.6.mlp.experts.242.up_proj", + "model.layers.6.mlp.experts.243.down_proj", + "model.layers.6.mlp.experts.243.gate_proj", + "model.layers.6.mlp.experts.243.up_proj", + "model.layers.6.mlp.experts.244.down_proj", + "model.layers.6.mlp.experts.244.gate_proj", + "model.layers.6.mlp.experts.244.up_proj", + "model.layers.6.mlp.experts.245.down_proj", + "model.layers.6.mlp.experts.245.gate_proj", + "model.layers.6.mlp.experts.245.up_proj", + "model.layers.6.mlp.experts.246.down_proj", + "model.layers.6.mlp.experts.246.gate_proj", + "model.layers.6.mlp.experts.246.up_proj", + "model.layers.6.mlp.experts.247.down_proj", + "model.layers.6.mlp.experts.247.gate_proj", + "model.layers.6.mlp.experts.247.up_proj", + "model.layers.6.mlp.experts.248.down_proj", + "model.layers.6.mlp.experts.248.gate_proj", + "model.layers.6.mlp.experts.248.up_proj", + "model.layers.6.mlp.experts.249.down_proj", + "model.layers.6.mlp.experts.249.gate_proj", + "model.layers.6.mlp.experts.249.up_proj", + "model.layers.6.mlp.experts.25.down_proj", + "model.layers.6.mlp.experts.25.gate_proj", + "model.layers.6.mlp.experts.25.up_proj", + "model.layers.6.mlp.experts.250.down_proj", + "model.layers.6.mlp.experts.250.gate_proj", + "model.layers.6.mlp.experts.250.up_proj", + "model.layers.6.mlp.experts.251.down_proj", + "model.layers.6.mlp.experts.251.gate_proj", + "model.layers.6.mlp.experts.251.up_proj", + "model.layers.6.mlp.experts.252.down_proj", + "model.layers.6.mlp.experts.252.gate_proj", + "model.layers.6.mlp.experts.252.up_proj", + "model.layers.6.mlp.experts.253.down_proj", + "model.layers.6.mlp.experts.253.gate_proj", + "model.layers.6.mlp.experts.253.up_proj", + "model.layers.6.mlp.experts.254.down_proj", + "model.layers.6.mlp.experts.254.gate_proj", + "model.layers.6.mlp.experts.254.up_proj", + "model.layers.6.mlp.experts.255.down_proj", + "model.layers.6.mlp.experts.255.gate_proj", + "model.layers.6.mlp.experts.255.up_proj", + "model.layers.6.mlp.experts.26.down_proj", + "model.layers.6.mlp.experts.26.gate_proj", + "model.layers.6.mlp.experts.26.up_proj", + "model.layers.6.mlp.experts.27.down_proj", + "model.layers.6.mlp.experts.27.gate_proj", + "model.layers.6.mlp.experts.27.up_proj", + "model.layers.6.mlp.experts.28.down_proj", + "model.layers.6.mlp.experts.28.gate_proj", + "model.layers.6.mlp.experts.28.up_proj", + "model.layers.6.mlp.experts.29.down_proj", + "model.layers.6.mlp.experts.29.gate_proj", + "model.layers.6.mlp.experts.29.up_proj", + "model.layers.6.mlp.experts.3.down_proj", + "model.layers.6.mlp.experts.3.gate_proj", + "model.layers.6.mlp.experts.3.up_proj", + "model.layers.6.mlp.experts.30.down_proj", + "model.layers.6.mlp.experts.30.gate_proj", + "model.layers.6.mlp.experts.30.up_proj", + "model.layers.6.mlp.experts.31.down_proj", + "model.layers.6.mlp.experts.31.gate_proj", + "model.layers.6.mlp.experts.31.up_proj", + "model.layers.6.mlp.experts.32.down_proj", + "model.layers.6.mlp.experts.32.gate_proj", + "model.layers.6.mlp.experts.32.up_proj", + "model.layers.6.mlp.experts.33.down_proj", + "model.layers.6.mlp.experts.33.gate_proj", + "model.layers.6.mlp.experts.33.up_proj", + "model.layers.6.mlp.experts.34.down_proj", + "model.layers.6.mlp.experts.34.gate_proj", + "model.layers.6.mlp.experts.34.up_proj", + "model.layers.6.mlp.experts.35.down_proj", + "model.layers.6.mlp.experts.35.gate_proj", + "model.layers.6.mlp.experts.35.up_proj", + "model.layers.6.mlp.experts.36.down_proj", + "model.layers.6.mlp.experts.36.gate_proj", + "model.layers.6.mlp.experts.36.up_proj", + "model.layers.6.mlp.experts.37.down_proj", + "model.layers.6.mlp.experts.37.gate_proj", + "model.layers.6.mlp.experts.37.up_proj", + "model.layers.6.mlp.experts.38.down_proj", + "model.layers.6.mlp.experts.38.gate_proj", + "model.layers.6.mlp.experts.38.up_proj", + "model.layers.6.mlp.experts.39.down_proj", + "model.layers.6.mlp.experts.39.gate_proj", + "model.layers.6.mlp.experts.39.up_proj", + "model.layers.6.mlp.experts.4.down_proj", + "model.layers.6.mlp.experts.4.gate_proj", + "model.layers.6.mlp.experts.4.up_proj", + "model.layers.6.mlp.experts.40.down_proj", + "model.layers.6.mlp.experts.40.gate_proj", + "model.layers.6.mlp.experts.40.up_proj", + "model.layers.6.mlp.experts.41.down_proj", + "model.layers.6.mlp.experts.41.gate_proj", + "model.layers.6.mlp.experts.41.up_proj", + "model.layers.6.mlp.experts.42.down_proj", + "model.layers.6.mlp.experts.42.gate_proj", + "model.layers.6.mlp.experts.42.up_proj", + "model.layers.6.mlp.experts.43.down_proj", + "model.layers.6.mlp.experts.43.gate_proj", + "model.layers.6.mlp.experts.43.up_proj", + "model.layers.6.mlp.experts.44.down_proj", + "model.layers.6.mlp.experts.44.gate_proj", + "model.layers.6.mlp.experts.44.up_proj", + "model.layers.6.mlp.experts.45.down_proj", + "model.layers.6.mlp.experts.45.gate_proj", + "model.layers.6.mlp.experts.45.up_proj", + "model.layers.6.mlp.experts.46.down_proj", + "model.layers.6.mlp.experts.46.gate_proj", + "model.layers.6.mlp.experts.46.up_proj", + "model.layers.6.mlp.experts.47.down_proj", + "model.layers.6.mlp.experts.47.gate_proj", + "model.layers.6.mlp.experts.47.up_proj", + "model.layers.6.mlp.experts.48.down_proj", + "model.layers.6.mlp.experts.48.gate_proj", + "model.layers.6.mlp.experts.48.up_proj", + "model.layers.6.mlp.experts.49.down_proj", + "model.layers.6.mlp.experts.49.gate_proj", + "model.layers.6.mlp.experts.49.up_proj", + "model.layers.6.mlp.experts.5.down_proj", + "model.layers.6.mlp.experts.5.gate_proj", + "model.layers.6.mlp.experts.5.up_proj", + "model.layers.6.mlp.experts.50.down_proj", + "model.layers.6.mlp.experts.50.gate_proj", + "model.layers.6.mlp.experts.50.up_proj", + "model.layers.6.mlp.experts.51.down_proj", + "model.layers.6.mlp.experts.51.gate_proj", + "model.layers.6.mlp.experts.51.up_proj", + "model.layers.6.mlp.experts.52.down_proj", + "model.layers.6.mlp.experts.52.gate_proj", + "model.layers.6.mlp.experts.52.up_proj", + "model.layers.6.mlp.experts.53.down_proj", + "model.layers.6.mlp.experts.53.gate_proj", + "model.layers.6.mlp.experts.53.up_proj", + "model.layers.6.mlp.experts.54.down_proj", + "model.layers.6.mlp.experts.54.gate_proj", + "model.layers.6.mlp.experts.54.up_proj", + "model.layers.6.mlp.experts.55.down_proj", + "model.layers.6.mlp.experts.55.gate_proj", + "model.layers.6.mlp.experts.55.up_proj", + "model.layers.6.mlp.experts.56.down_proj", + "model.layers.6.mlp.experts.56.gate_proj", + "model.layers.6.mlp.experts.56.up_proj", + "model.layers.6.mlp.experts.57.down_proj", + "model.layers.6.mlp.experts.57.gate_proj", + "model.layers.6.mlp.experts.57.up_proj", + "model.layers.6.mlp.experts.58.down_proj", + "model.layers.6.mlp.experts.58.gate_proj", + "model.layers.6.mlp.experts.58.up_proj", + "model.layers.6.mlp.experts.59.down_proj", + "model.layers.6.mlp.experts.59.gate_proj", + "model.layers.6.mlp.experts.59.up_proj", + "model.layers.6.mlp.experts.6.down_proj", + "model.layers.6.mlp.experts.6.gate_proj", + "model.layers.6.mlp.experts.6.up_proj", + "model.layers.6.mlp.experts.60.down_proj", + "model.layers.6.mlp.experts.60.gate_proj", + "model.layers.6.mlp.experts.60.up_proj", + "model.layers.6.mlp.experts.61.down_proj", + "model.layers.6.mlp.experts.61.gate_proj", + "model.layers.6.mlp.experts.61.up_proj", + "model.layers.6.mlp.experts.62.down_proj", + "model.layers.6.mlp.experts.62.gate_proj", + "model.layers.6.mlp.experts.62.up_proj", + "model.layers.6.mlp.experts.63.down_proj", + "model.layers.6.mlp.experts.63.gate_proj", + "model.layers.6.mlp.experts.63.up_proj", + "model.layers.6.mlp.experts.64.down_proj", + "model.layers.6.mlp.experts.64.gate_proj", + "model.layers.6.mlp.experts.64.up_proj", + "model.layers.6.mlp.experts.65.down_proj", + "model.layers.6.mlp.experts.65.gate_proj", + "model.layers.6.mlp.experts.65.up_proj", + "model.layers.6.mlp.experts.66.down_proj", + "model.layers.6.mlp.experts.66.gate_proj", + "model.layers.6.mlp.experts.66.up_proj", + "model.layers.6.mlp.experts.67.down_proj", + "model.layers.6.mlp.experts.67.gate_proj", + "model.layers.6.mlp.experts.67.up_proj", + "model.layers.6.mlp.experts.68.down_proj", + "model.layers.6.mlp.experts.68.gate_proj", + "model.layers.6.mlp.experts.68.up_proj", + "model.layers.6.mlp.experts.69.down_proj", + "model.layers.6.mlp.experts.69.gate_proj", + "model.layers.6.mlp.experts.69.up_proj", + "model.layers.6.mlp.experts.7.down_proj", + "model.layers.6.mlp.experts.7.gate_proj", + "model.layers.6.mlp.experts.7.up_proj", + "model.layers.6.mlp.experts.70.down_proj", + "model.layers.6.mlp.experts.70.gate_proj", + "model.layers.6.mlp.experts.70.up_proj", + "model.layers.6.mlp.experts.71.down_proj", + "model.layers.6.mlp.experts.71.gate_proj", + "model.layers.6.mlp.experts.71.up_proj", + "model.layers.6.mlp.experts.72.down_proj", + "model.layers.6.mlp.experts.72.gate_proj", + "model.layers.6.mlp.experts.72.up_proj", + "model.layers.6.mlp.experts.73.down_proj", + "model.layers.6.mlp.experts.73.gate_proj", + "model.layers.6.mlp.experts.73.up_proj", + "model.layers.6.mlp.experts.74.down_proj", + "model.layers.6.mlp.experts.74.gate_proj", + "model.layers.6.mlp.experts.74.up_proj", + "model.layers.6.mlp.experts.75.down_proj", + "model.layers.6.mlp.experts.75.gate_proj", + "model.layers.6.mlp.experts.75.up_proj", + "model.layers.6.mlp.experts.76.down_proj", + "model.layers.6.mlp.experts.76.gate_proj", + "model.layers.6.mlp.experts.76.up_proj", + "model.layers.6.mlp.experts.77.down_proj", + "model.layers.6.mlp.experts.77.gate_proj", + "model.layers.6.mlp.experts.77.up_proj", + "model.layers.6.mlp.experts.78.down_proj", + "model.layers.6.mlp.experts.78.gate_proj", + "model.layers.6.mlp.experts.78.up_proj", + "model.layers.6.mlp.experts.79.down_proj", + "model.layers.6.mlp.experts.79.gate_proj", + "model.layers.6.mlp.experts.79.up_proj", + "model.layers.6.mlp.experts.8.down_proj", + "model.layers.6.mlp.experts.8.gate_proj", + "model.layers.6.mlp.experts.8.up_proj", + "model.layers.6.mlp.experts.80.down_proj", + "model.layers.6.mlp.experts.80.gate_proj", + "model.layers.6.mlp.experts.80.up_proj", + "model.layers.6.mlp.experts.81.down_proj", + "model.layers.6.mlp.experts.81.gate_proj", + "model.layers.6.mlp.experts.81.up_proj", + "model.layers.6.mlp.experts.82.down_proj", + "model.layers.6.mlp.experts.82.gate_proj", + "model.layers.6.mlp.experts.82.up_proj", + "model.layers.6.mlp.experts.83.down_proj", + "model.layers.6.mlp.experts.83.gate_proj", + "model.layers.6.mlp.experts.83.up_proj", + "model.layers.6.mlp.experts.84.down_proj", + "model.layers.6.mlp.experts.84.gate_proj", + "model.layers.6.mlp.experts.84.up_proj", + "model.layers.6.mlp.experts.85.down_proj", + "model.layers.6.mlp.experts.85.gate_proj", + "model.layers.6.mlp.experts.85.up_proj", + "model.layers.6.mlp.experts.86.down_proj", + "model.layers.6.mlp.experts.86.gate_proj", + "model.layers.6.mlp.experts.86.up_proj", + "model.layers.6.mlp.experts.87.down_proj", + "model.layers.6.mlp.experts.87.gate_proj", + "model.layers.6.mlp.experts.87.up_proj", + "model.layers.6.mlp.experts.88.down_proj", + "model.layers.6.mlp.experts.88.gate_proj", + "model.layers.6.mlp.experts.88.up_proj", + "model.layers.6.mlp.experts.89.down_proj", + "model.layers.6.mlp.experts.89.gate_proj", + "model.layers.6.mlp.experts.89.up_proj", + "model.layers.6.mlp.experts.9.down_proj", + "model.layers.6.mlp.experts.9.gate_proj", + "model.layers.6.mlp.experts.9.up_proj", + "model.layers.6.mlp.experts.90.down_proj", + "model.layers.6.mlp.experts.90.gate_proj", + "model.layers.6.mlp.experts.90.up_proj", + "model.layers.6.mlp.experts.91.down_proj", + "model.layers.6.mlp.experts.91.gate_proj", + "model.layers.6.mlp.experts.91.up_proj", + "model.layers.6.mlp.experts.92.down_proj", + "model.layers.6.mlp.experts.92.gate_proj", + "model.layers.6.mlp.experts.92.up_proj", + "model.layers.6.mlp.experts.93.down_proj", + "model.layers.6.mlp.experts.93.gate_proj", + "model.layers.6.mlp.experts.93.up_proj", + "model.layers.6.mlp.experts.94.down_proj", + "model.layers.6.mlp.experts.94.gate_proj", + "model.layers.6.mlp.experts.94.up_proj", + "model.layers.6.mlp.experts.95.down_proj", + "model.layers.6.mlp.experts.95.gate_proj", + "model.layers.6.mlp.experts.95.up_proj", + "model.layers.6.mlp.experts.96.down_proj", + "model.layers.6.mlp.experts.96.gate_proj", + "model.layers.6.mlp.experts.96.up_proj", + "model.layers.6.mlp.experts.97.down_proj", + "model.layers.6.mlp.experts.97.gate_proj", + "model.layers.6.mlp.experts.97.up_proj", + "model.layers.6.mlp.experts.98.down_proj", + "model.layers.6.mlp.experts.98.gate_proj", + "model.layers.6.mlp.experts.98.up_proj", + "model.layers.6.mlp.experts.99.down_proj", + "model.layers.6.mlp.experts.99.gate_proj", + "model.layers.6.mlp.experts.99.up_proj", + "model.layers.60.mlp.experts.0.down_proj", + "model.layers.60.mlp.experts.0.gate_proj", + "model.layers.60.mlp.experts.0.up_proj", + "model.layers.60.mlp.experts.1.down_proj", + "model.layers.60.mlp.experts.1.gate_proj", + "model.layers.60.mlp.experts.1.up_proj", + "model.layers.60.mlp.experts.10.down_proj", + "model.layers.60.mlp.experts.10.gate_proj", + "model.layers.60.mlp.experts.10.up_proj", + "model.layers.60.mlp.experts.100.down_proj", + "model.layers.60.mlp.experts.100.gate_proj", + "model.layers.60.mlp.experts.100.up_proj", + "model.layers.60.mlp.experts.101.down_proj", + "model.layers.60.mlp.experts.101.gate_proj", + "model.layers.60.mlp.experts.101.up_proj", + "model.layers.60.mlp.experts.102.down_proj", + "model.layers.60.mlp.experts.102.gate_proj", + "model.layers.60.mlp.experts.102.up_proj", + "model.layers.60.mlp.experts.103.down_proj", + "model.layers.60.mlp.experts.103.gate_proj", + "model.layers.60.mlp.experts.103.up_proj", + "model.layers.60.mlp.experts.104.down_proj", + "model.layers.60.mlp.experts.104.gate_proj", + "model.layers.60.mlp.experts.104.up_proj", + "model.layers.60.mlp.experts.105.down_proj", + "model.layers.60.mlp.experts.105.gate_proj", + "model.layers.60.mlp.experts.105.up_proj", + "model.layers.60.mlp.experts.106.down_proj", + "model.layers.60.mlp.experts.106.gate_proj", + "model.layers.60.mlp.experts.106.up_proj", + "model.layers.60.mlp.experts.107.down_proj", + "model.layers.60.mlp.experts.107.gate_proj", + "model.layers.60.mlp.experts.107.up_proj", + "model.layers.60.mlp.experts.108.down_proj", + "model.layers.60.mlp.experts.108.gate_proj", + "model.layers.60.mlp.experts.108.up_proj", + "model.layers.60.mlp.experts.109.down_proj", + "model.layers.60.mlp.experts.109.gate_proj", + "model.layers.60.mlp.experts.109.up_proj", + "model.layers.60.mlp.experts.11.down_proj", + "model.layers.60.mlp.experts.11.gate_proj", + "model.layers.60.mlp.experts.11.up_proj", + "model.layers.60.mlp.experts.110.down_proj", + "model.layers.60.mlp.experts.110.gate_proj", + "model.layers.60.mlp.experts.110.up_proj", + "model.layers.60.mlp.experts.111.down_proj", + "model.layers.60.mlp.experts.111.gate_proj", + "model.layers.60.mlp.experts.111.up_proj", + "model.layers.60.mlp.experts.112.down_proj", + "model.layers.60.mlp.experts.112.gate_proj", + "model.layers.60.mlp.experts.112.up_proj", + "model.layers.60.mlp.experts.113.down_proj", + "model.layers.60.mlp.experts.113.gate_proj", + "model.layers.60.mlp.experts.113.up_proj", + "model.layers.60.mlp.experts.114.down_proj", + "model.layers.60.mlp.experts.114.gate_proj", + "model.layers.60.mlp.experts.114.up_proj", + "model.layers.60.mlp.experts.115.down_proj", + "model.layers.60.mlp.experts.115.gate_proj", + "model.layers.60.mlp.experts.115.up_proj", + "model.layers.60.mlp.experts.116.down_proj", + "model.layers.60.mlp.experts.116.gate_proj", + "model.layers.60.mlp.experts.116.up_proj", + "model.layers.60.mlp.experts.117.down_proj", + "model.layers.60.mlp.experts.117.gate_proj", + "model.layers.60.mlp.experts.117.up_proj", + "model.layers.60.mlp.experts.118.down_proj", + "model.layers.60.mlp.experts.118.gate_proj", + "model.layers.60.mlp.experts.118.up_proj", + "model.layers.60.mlp.experts.119.down_proj", + "model.layers.60.mlp.experts.119.gate_proj", + "model.layers.60.mlp.experts.119.up_proj", + "model.layers.60.mlp.experts.12.down_proj", + "model.layers.60.mlp.experts.12.gate_proj", + "model.layers.60.mlp.experts.12.up_proj", + "model.layers.60.mlp.experts.120.down_proj", + "model.layers.60.mlp.experts.120.gate_proj", + "model.layers.60.mlp.experts.120.up_proj", + "model.layers.60.mlp.experts.121.down_proj", + "model.layers.60.mlp.experts.121.gate_proj", + "model.layers.60.mlp.experts.121.up_proj", + "model.layers.60.mlp.experts.122.down_proj", + "model.layers.60.mlp.experts.122.gate_proj", + "model.layers.60.mlp.experts.122.up_proj", + "model.layers.60.mlp.experts.123.down_proj", + "model.layers.60.mlp.experts.123.gate_proj", + "model.layers.60.mlp.experts.123.up_proj", + "model.layers.60.mlp.experts.124.down_proj", + "model.layers.60.mlp.experts.124.gate_proj", + "model.layers.60.mlp.experts.124.up_proj", + "model.layers.60.mlp.experts.125.down_proj", + "model.layers.60.mlp.experts.125.gate_proj", + "model.layers.60.mlp.experts.125.up_proj", + "model.layers.60.mlp.experts.126.down_proj", + "model.layers.60.mlp.experts.126.gate_proj", + "model.layers.60.mlp.experts.126.up_proj", + "model.layers.60.mlp.experts.127.down_proj", + "model.layers.60.mlp.experts.127.gate_proj", + "model.layers.60.mlp.experts.127.up_proj", + "model.layers.60.mlp.experts.128.down_proj", + "model.layers.60.mlp.experts.128.gate_proj", + "model.layers.60.mlp.experts.128.up_proj", + "model.layers.60.mlp.experts.129.down_proj", + "model.layers.60.mlp.experts.129.gate_proj", + "model.layers.60.mlp.experts.129.up_proj", + "model.layers.60.mlp.experts.13.down_proj", + "model.layers.60.mlp.experts.13.gate_proj", + "model.layers.60.mlp.experts.13.up_proj", + "model.layers.60.mlp.experts.130.down_proj", + "model.layers.60.mlp.experts.130.gate_proj", + "model.layers.60.mlp.experts.130.up_proj", + "model.layers.60.mlp.experts.131.down_proj", + "model.layers.60.mlp.experts.131.gate_proj", + "model.layers.60.mlp.experts.131.up_proj", + "model.layers.60.mlp.experts.132.down_proj", + "model.layers.60.mlp.experts.132.gate_proj", + "model.layers.60.mlp.experts.132.up_proj", + "model.layers.60.mlp.experts.133.down_proj", + "model.layers.60.mlp.experts.133.gate_proj", + "model.layers.60.mlp.experts.133.up_proj", + "model.layers.60.mlp.experts.134.down_proj", + "model.layers.60.mlp.experts.134.gate_proj", + "model.layers.60.mlp.experts.134.up_proj", + "model.layers.60.mlp.experts.135.down_proj", + "model.layers.60.mlp.experts.135.gate_proj", + "model.layers.60.mlp.experts.135.up_proj", + "model.layers.60.mlp.experts.136.down_proj", + "model.layers.60.mlp.experts.136.gate_proj", + "model.layers.60.mlp.experts.136.up_proj", + "model.layers.60.mlp.experts.137.down_proj", + "model.layers.60.mlp.experts.137.gate_proj", + "model.layers.60.mlp.experts.137.up_proj", + "model.layers.60.mlp.experts.138.down_proj", + "model.layers.60.mlp.experts.138.gate_proj", + "model.layers.60.mlp.experts.138.up_proj", + "model.layers.60.mlp.experts.139.down_proj", + "model.layers.60.mlp.experts.139.gate_proj", + "model.layers.60.mlp.experts.139.up_proj", + "model.layers.60.mlp.experts.14.down_proj", + "model.layers.60.mlp.experts.14.gate_proj", + "model.layers.60.mlp.experts.14.up_proj", + "model.layers.60.mlp.experts.140.down_proj", + "model.layers.60.mlp.experts.140.gate_proj", + "model.layers.60.mlp.experts.140.up_proj", + "model.layers.60.mlp.experts.141.down_proj", + "model.layers.60.mlp.experts.141.gate_proj", + "model.layers.60.mlp.experts.141.up_proj", + "model.layers.60.mlp.experts.142.down_proj", + "model.layers.60.mlp.experts.142.gate_proj", + "model.layers.60.mlp.experts.142.up_proj", + "model.layers.60.mlp.experts.143.down_proj", + "model.layers.60.mlp.experts.143.gate_proj", + "model.layers.60.mlp.experts.143.up_proj", + "model.layers.60.mlp.experts.144.down_proj", + "model.layers.60.mlp.experts.144.gate_proj", + "model.layers.60.mlp.experts.144.up_proj", + "model.layers.60.mlp.experts.145.down_proj", + "model.layers.60.mlp.experts.145.gate_proj", + "model.layers.60.mlp.experts.145.up_proj", + "model.layers.60.mlp.experts.146.down_proj", + "model.layers.60.mlp.experts.146.gate_proj", + "model.layers.60.mlp.experts.146.up_proj", + "model.layers.60.mlp.experts.147.down_proj", + "model.layers.60.mlp.experts.147.gate_proj", + "model.layers.60.mlp.experts.147.up_proj", + "model.layers.60.mlp.experts.148.down_proj", + "model.layers.60.mlp.experts.148.gate_proj", + "model.layers.60.mlp.experts.148.up_proj", + "model.layers.60.mlp.experts.149.down_proj", + "model.layers.60.mlp.experts.149.gate_proj", + "model.layers.60.mlp.experts.149.up_proj", + "model.layers.60.mlp.experts.15.down_proj", + "model.layers.60.mlp.experts.15.gate_proj", + "model.layers.60.mlp.experts.15.up_proj", + "model.layers.60.mlp.experts.150.down_proj", + "model.layers.60.mlp.experts.150.gate_proj", + "model.layers.60.mlp.experts.150.up_proj", + "model.layers.60.mlp.experts.151.down_proj", + "model.layers.60.mlp.experts.151.gate_proj", + "model.layers.60.mlp.experts.151.up_proj", + "model.layers.60.mlp.experts.152.down_proj", + "model.layers.60.mlp.experts.152.gate_proj", + "model.layers.60.mlp.experts.152.up_proj", + "model.layers.60.mlp.experts.153.down_proj", + "model.layers.60.mlp.experts.153.gate_proj", + "model.layers.60.mlp.experts.153.up_proj", + "model.layers.60.mlp.experts.154.down_proj", + "model.layers.60.mlp.experts.154.gate_proj", + "model.layers.60.mlp.experts.154.up_proj", + "model.layers.60.mlp.experts.155.down_proj", + "model.layers.60.mlp.experts.155.gate_proj", + "model.layers.60.mlp.experts.155.up_proj", + "model.layers.60.mlp.experts.156.down_proj", + "model.layers.60.mlp.experts.156.gate_proj", + "model.layers.60.mlp.experts.156.up_proj", + "model.layers.60.mlp.experts.157.down_proj", + "model.layers.60.mlp.experts.157.gate_proj", + "model.layers.60.mlp.experts.157.up_proj", + "model.layers.60.mlp.experts.158.down_proj", + "model.layers.60.mlp.experts.158.gate_proj", + "model.layers.60.mlp.experts.158.up_proj", + "model.layers.60.mlp.experts.159.down_proj", + "model.layers.60.mlp.experts.159.gate_proj", + "model.layers.60.mlp.experts.159.up_proj", + "model.layers.60.mlp.experts.16.down_proj", + "model.layers.60.mlp.experts.16.gate_proj", + "model.layers.60.mlp.experts.16.up_proj", + "model.layers.60.mlp.experts.160.down_proj", + "model.layers.60.mlp.experts.160.gate_proj", + "model.layers.60.mlp.experts.160.up_proj", + "model.layers.60.mlp.experts.161.down_proj", + "model.layers.60.mlp.experts.161.gate_proj", + "model.layers.60.mlp.experts.161.up_proj", + "model.layers.60.mlp.experts.162.down_proj", + "model.layers.60.mlp.experts.162.gate_proj", + "model.layers.60.mlp.experts.162.up_proj", + "model.layers.60.mlp.experts.163.down_proj", + "model.layers.60.mlp.experts.163.gate_proj", + "model.layers.60.mlp.experts.163.up_proj", + "model.layers.60.mlp.experts.164.down_proj", + "model.layers.60.mlp.experts.164.gate_proj", + "model.layers.60.mlp.experts.164.up_proj", + "model.layers.60.mlp.experts.165.down_proj", + "model.layers.60.mlp.experts.165.gate_proj", + "model.layers.60.mlp.experts.165.up_proj", + "model.layers.60.mlp.experts.166.down_proj", + "model.layers.60.mlp.experts.166.gate_proj", + "model.layers.60.mlp.experts.166.up_proj", + "model.layers.60.mlp.experts.167.down_proj", + "model.layers.60.mlp.experts.167.gate_proj", + "model.layers.60.mlp.experts.167.up_proj", + "model.layers.60.mlp.experts.168.down_proj", + "model.layers.60.mlp.experts.168.gate_proj", + "model.layers.60.mlp.experts.168.up_proj", + "model.layers.60.mlp.experts.169.down_proj", + "model.layers.60.mlp.experts.169.gate_proj", + "model.layers.60.mlp.experts.169.up_proj", + "model.layers.60.mlp.experts.17.down_proj", + "model.layers.60.mlp.experts.17.gate_proj", + "model.layers.60.mlp.experts.17.up_proj", + "model.layers.60.mlp.experts.170.down_proj", + "model.layers.60.mlp.experts.170.gate_proj", + "model.layers.60.mlp.experts.170.up_proj", + "model.layers.60.mlp.experts.171.down_proj", + "model.layers.60.mlp.experts.171.gate_proj", + "model.layers.60.mlp.experts.171.up_proj", + "model.layers.60.mlp.experts.172.down_proj", + "model.layers.60.mlp.experts.172.gate_proj", + "model.layers.60.mlp.experts.172.up_proj", + "model.layers.60.mlp.experts.173.down_proj", + "model.layers.60.mlp.experts.173.gate_proj", + "model.layers.60.mlp.experts.173.up_proj", + "model.layers.60.mlp.experts.174.down_proj", + "model.layers.60.mlp.experts.174.gate_proj", + "model.layers.60.mlp.experts.174.up_proj", + "model.layers.60.mlp.experts.175.down_proj", + "model.layers.60.mlp.experts.175.gate_proj", + "model.layers.60.mlp.experts.175.up_proj", + "model.layers.60.mlp.experts.176.down_proj", + "model.layers.60.mlp.experts.176.gate_proj", + "model.layers.60.mlp.experts.176.up_proj", + "model.layers.60.mlp.experts.177.down_proj", + "model.layers.60.mlp.experts.177.gate_proj", + "model.layers.60.mlp.experts.177.up_proj", + "model.layers.60.mlp.experts.178.down_proj", + "model.layers.60.mlp.experts.178.gate_proj", + "model.layers.60.mlp.experts.178.up_proj", + "model.layers.60.mlp.experts.179.down_proj", + "model.layers.60.mlp.experts.179.gate_proj", + "model.layers.60.mlp.experts.179.up_proj", + "model.layers.60.mlp.experts.18.down_proj", + "model.layers.60.mlp.experts.18.gate_proj", + "model.layers.60.mlp.experts.18.up_proj", + "model.layers.60.mlp.experts.180.down_proj", + "model.layers.60.mlp.experts.180.gate_proj", + "model.layers.60.mlp.experts.180.up_proj", + "model.layers.60.mlp.experts.181.down_proj", + "model.layers.60.mlp.experts.181.gate_proj", + "model.layers.60.mlp.experts.181.up_proj", + "model.layers.60.mlp.experts.182.down_proj", + "model.layers.60.mlp.experts.182.gate_proj", + "model.layers.60.mlp.experts.182.up_proj", + "model.layers.60.mlp.experts.183.down_proj", + "model.layers.60.mlp.experts.183.gate_proj", + "model.layers.60.mlp.experts.183.up_proj", + "model.layers.60.mlp.experts.184.down_proj", + "model.layers.60.mlp.experts.184.gate_proj", + "model.layers.60.mlp.experts.184.up_proj", + "model.layers.60.mlp.experts.185.down_proj", + "model.layers.60.mlp.experts.185.gate_proj", + "model.layers.60.mlp.experts.185.up_proj", + "model.layers.60.mlp.experts.186.down_proj", + "model.layers.60.mlp.experts.186.gate_proj", + "model.layers.60.mlp.experts.186.up_proj", + "model.layers.60.mlp.experts.187.down_proj", + "model.layers.60.mlp.experts.187.gate_proj", + "model.layers.60.mlp.experts.187.up_proj", + "model.layers.60.mlp.experts.188.down_proj", + "model.layers.60.mlp.experts.188.gate_proj", + "model.layers.60.mlp.experts.188.up_proj", + "model.layers.60.mlp.experts.189.down_proj", + "model.layers.60.mlp.experts.189.gate_proj", + "model.layers.60.mlp.experts.189.up_proj", + "model.layers.60.mlp.experts.19.down_proj", + "model.layers.60.mlp.experts.19.gate_proj", + "model.layers.60.mlp.experts.19.up_proj", + "model.layers.60.mlp.experts.190.down_proj", + "model.layers.60.mlp.experts.190.gate_proj", + "model.layers.60.mlp.experts.190.up_proj", + "model.layers.60.mlp.experts.191.down_proj", + "model.layers.60.mlp.experts.191.gate_proj", + "model.layers.60.mlp.experts.191.up_proj", + "model.layers.60.mlp.experts.192.down_proj", + "model.layers.60.mlp.experts.192.gate_proj", + "model.layers.60.mlp.experts.192.up_proj", + "model.layers.60.mlp.experts.193.down_proj", + "model.layers.60.mlp.experts.193.gate_proj", + "model.layers.60.mlp.experts.193.up_proj", + "model.layers.60.mlp.experts.194.down_proj", + "model.layers.60.mlp.experts.194.gate_proj", + "model.layers.60.mlp.experts.194.up_proj", + "model.layers.60.mlp.experts.195.down_proj", + "model.layers.60.mlp.experts.195.gate_proj", + "model.layers.60.mlp.experts.195.up_proj", + "model.layers.60.mlp.experts.196.down_proj", + "model.layers.60.mlp.experts.196.gate_proj", + "model.layers.60.mlp.experts.196.up_proj", + "model.layers.60.mlp.experts.197.down_proj", + "model.layers.60.mlp.experts.197.gate_proj", + "model.layers.60.mlp.experts.197.up_proj", + "model.layers.60.mlp.experts.198.down_proj", + "model.layers.60.mlp.experts.198.gate_proj", + "model.layers.60.mlp.experts.198.up_proj", + "model.layers.60.mlp.experts.199.down_proj", + "model.layers.60.mlp.experts.199.gate_proj", + "model.layers.60.mlp.experts.199.up_proj", + "model.layers.60.mlp.experts.2.down_proj", + "model.layers.60.mlp.experts.2.gate_proj", + "model.layers.60.mlp.experts.2.up_proj", + "model.layers.60.mlp.experts.20.down_proj", + "model.layers.60.mlp.experts.20.gate_proj", + "model.layers.60.mlp.experts.20.up_proj", + "model.layers.60.mlp.experts.200.down_proj", + "model.layers.60.mlp.experts.200.gate_proj", + "model.layers.60.mlp.experts.200.up_proj", + "model.layers.60.mlp.experts.201.down_proj", + "model.layers.60.mlp.experts.201.gate_proj", + "model.layers.60.mlp.experts.201.up_proj", + "model.layers.60.mlp.experts.202.down_proj", + "model.layers.60.mlp.experts.202.gate_proj", + "model.layers.60.mlp.experts.202.up_proj", + "model.layers.60.mlp.experts.203.down_proj", + "model.layers.60.mlp.experts.203.gate_proj", + "model.layers.60.mlp.experts.203.up_proj", + "model.layers.60.mlp.experts.204.down_proj", + "model.layers.60.mlp.experts.204.gate_proj", + "model.layers.60.mlp.experts.204.up_proj", + "model.layers.60.mlp.experts.205.down_proj", + "model.layers.60.mlp.experts.205.gate_proj", + "model.layers.60.mlp.experts.205.up_proj", + "model.layers.60.mlp.experts.206.down_proj", + "model.layers.60.mlp.experts.206.gate_proj", + "model.layers.60.mlp.experts.206.up_proj", + "model.layers.60.mlp.experts.207.down_proj", + "model.layers.60.mlp.experts.207.gate_proj", + "model.layers.60.mlp.experts.207.up_proj", + "model.layers.60.mlp.experts.208.down_proj", + "model.layers.60.mlp.experts.208.gate_proj", + "model.layers.60.mlp.experts.208.up_proj", + "model.layers.60.mlp.experts.209.down_proj", + "model.layers.60.mlp.experts.209.gate_proj", + "model.layers.60.mlp.experts.209.up_proj", + "model.layers.60.mlp.experts.21.down_proj", + "model.layers.60.mlp.experts.21.gate_proj", + "model.layers.60.mlp.experts.21.up_proj", + "model.layers.60.mlp.experts.210.down_proj", + "model.layers.60.mlp.experts.210.gate_proj", + "model.layers.60.mlp.experts.210.up_proj", + "model.layers.60.mlp.experts.211.down_proj", + "model.layers.60.mlp.experts.211.gate_proj", + "model.layers.60.mlp.experts.211.up_proj", + "model.layers.60.mlp.experts.212.down_proj", + "model.layers.60.mlp.experts.212.gate_proj", + "model.layers.60.mlp.experts.212.up_proj", + "model.layers.60.mlp.experts.213.down_proj", + "model.layers.60.mlp.experts.213.gate_proj", + "model.layers.60.mlp.experts.213.up_proj", + "model.layers.60.mlp.experts.214.down_proj", + "model.layers.60.mlp.experts.214.gate_proj", + "model.layers.60.mlp.experts.214.up_proj", + "model.layers.60.mlp.experts.215.down_proj", + "model.layers.60.mlp.experts.215.gate_proj", + "model.layers.60.mlp.experts.215.up_proj", + "model.layers.60.mlp.experts.216.down_proj", + "model.layers.60.mlp.experts.216.gate_proj", + "model.layers.60.mlp.experts.216.up_proj", + "model.layers.60.mlp.experts.217.down_proj", + "model.layers.60.mlp.experts.217.gate_proj", + "model.layers.60.mlp.experts.217.up_proj", + "model.layers.60.mlp.experts.218.down_proj", + "model.layers.60.mlp.experts.218.gate_proj", + "model.layers.60.mlp.experts.218.up_proj", + "model.layers.60.mlp.experts.219.down_proj", + "model.layers.60.mlp.experts.219.gate_proj", + "model.layers.60.mlp.experts.219.up_proj", + "model.layers.60.mlp.experts.22.down_proj", + "model.layers.60.mlp.experts.22.gate_proj", + "model.layers.60.mlp.experts.22.up_proj", + "model.layers.60.mlp.experts.220.down_proj", + "model.layers.60.mlp.experts.220.gate_proj", + "model.layers.60.mlp.experts.220.up_proj", + "model.layers.60.mlp.experts.221.down_proj", + "model.layers.60.mlp.experts.221.gate_proj", + "model.layers.60.mlp.experts.221.up_proj", + "model.layers.60.mlp.experts.222.down_proj", + "model.layers.60.mlp.experts.222.gate_proj", + "model.layers.60.mlp.experts.222.up_proj", + "model.layers.60.mlp.experts.223.down_proj", + "model.layers.60.mlp.experts.223.gate_proj", + "model.layers.60.mlp.experts.223.up_proj", + "model.layers.60.mlp.experts.224.down_proj", + "model.layers.60.mlp.experts.224.gate_proj", + "model.layers.60.mlp.experts.224.up_proj", + "model.layers.60.mlp.experts.225.down_proj", + "model.layers.60.mlp.experts.225.gate_proj", + "model.layers.60.mlp.experts.225.up_proj", + "model.layers.60.mlp.experts.226.down_proj", + "model.layers.60.mlp.experts.226.gate_proj", + "model.layers.60.mlp.experts.226.up_proj", + "model.layers.60.mlp.experts.227.down_proj", + "model.layers.60.mlp.experts.227.gate_proj", + "model.layers.60.mlp.experts.227.up_proj", + "model.layers.60.mlp.experts.228.down_proj", + "model.layers.60.mlp.experts.228.gate_proj", + "model.layers.60.mlp.experts.228.up_proj", + "model.layers.60.mlp.experts.229.down_proj", + "model.layers.60.mlp.experts.229.gate_proj", + "model.layers.60.mlp.experts.229.up_proj", + "model.layers.60.mlp.experts.23.down_proj", + "model.layers.60.mlp.experts.23.gate_proj", + "model.layers.60.mlp.experts.23.up_proj", + "model.layers.60.mlp.experts.230.down_proj", + "model.layers.60.mlp.experts.230.gate_proj", + "model.layers.60.mlp.experts.230.up_proj", + "model.layers.60.mlp.experts.231.down_proj", + "model.layers.60.mlp.experts.231.gate_proj", + "model.layers.60.mlp.experts.231.up_proj", + "model.layers.60.mlp.experts.232.down_proj", + "model.layers.60.mlp.experts.232.gate_proj", + "model.layers.60.mlp.experts.232.up_proj", + "model.layers.60.mlp.experts.233.down_proj", + "model.layers.60.mlp.experts.233.gate_proj", + "model.layers.60.mlp.experts.233.up_proj", + "model.layers.60.mlp.experts.234.down_proj", + "model.layers.60.mlp.experts.234.gate_proj", + "model.layers.60.mlp.experts.234.up_proj", + "model.layers.60.mlp.experts.235.down_proj", + "model.layers.60.mlp.experts.235.gate_proj", + "model.layers.60.mlp.experts.235.up_proj", + "model.layers.60.mlp.experts.236.down_proj", + "model.layers.60.mlp.experts.236.gate_proj", + "model.layers.60.mlp.experts.236.up_proj", + "model.layers.60.mlp.experts.237.down_proj", + "model.layers.60.mlp.experts.237.gate_proj", + "model.layers.60.mlp.experts.237.up_proj", + "model.layers.60.mlp.experts.238.down_proj", + "model.layers.60.mlp.experts.238.gate_proj", + "model.layers.60.mlp.experts.238.up_proj", + "model.layers.60.mlp.experts.239.down_proj", + "model.layers.60.mlp.experts.239.gate_proj", + "model.layers.60.mlp.experts.239.up_proj", + "model.layers.60.mlp.experts.24.down_proj", + "model.layers.60.mlp.experts.24.gate_proj", + "model.layers.60.mlp.experts.24.up_proj", + "model.layers.60.mlp.experts.240.down_proj", + "model.layers.60.mlp.experts.240.gate_proj", + "model.layers.60.mlp.experts.240.up_proj", + "model.layers.60.mlp.experts.241.down_proj", + "model.layers.60.mlp.experts.241.gate_proj", + "model.layers.60.mlp.experts.241.up_proj", + "model.layers.60.mlp.experts.242.down_proj", + "model.layers.60.mlp.experts.242.gate_proj", + "model.layers.60.mlp.experts.242.up_proj", + "model.layers.60.mlp.experts.243.down_proj", + "model.layers.60.mlp.experts.243.gate_proj", + "model.layers.60.mlp.experts.243.up_proj", + "model.layers.60.mlp.experts.244.down_proj", + "model.layers.60.mlp.experts.244.gate_proj", + "model.layers.60.mlp.experts.244.up_proj", + "model.layers.60.mlp.experts.245.down_proj", + "model.layers.60.mlp.experts.245.gate_proj", + "model.layers.60.mlp.experts.245.up_proj", + "model.layers.60.mlp.experts.246.down_proj", + "model.layers.60.mlp.experts.246.gate_proj", + "model.layers.60.mlp.experts.246.up_proj", + "model.layers.60.mlp.experts.247.down_proj", + "model.layers.60.mlp.experts.247.gate_proj", + "model.layers.60.mlp.experts.247.up_proj", + "model.layers.60.mlp.experts.248.down_proj", + "model.layers.60.mlp.experts.248.gate_proj", + "model.layers.60.mlp.experts.248.up_proj", + "model.layers.60.mlp.experts.249.down_proj", + "model.layers.60.mlp.experts.249.gate_proj", + "model.layers.60.mlp.experts.249.up_proj", + "model.layers.60.mlp.experts.25.down_proj", + "model.layers.60.mlp.experts.25.gate_proj", + "model.layers.60.mlp.experts.25.up_proj", + "model.layers.60.mlp.experts.250.down_proj", + "model.layers.60.mlp.experts.250.gate_proj", + "model.layers.60.mlp.experts.250.up_proj", + "model.layers.60.mlp.experts.251.down_proj", + "model.layers.60.mlp.experts.251.gate_proj", + "model.layers.60.mlp.experts.251.up_proj", + "model.layers.60.mlp.experts.252.down_proj", + "model.layers.60.mlp.experts.252.gate_proj", + "model.layers.60.mlp.experts.252.up_proj", + "model.layers.60.mlp.experts.253.down_proj", + "model.layers.60.mlp.experts.253.gate_proj", + "model.layers.60.mlp.experts.253.up_proj", + "model.layers.60.mlp.experts.254.down_proj", + "model.layers.60.mlp.experts.254.gate_proj", + "model.layers.60.mlp.experts.254.up_proj", + "model.layers.60.mlp.experts.255.down_proj", + "model.layers.60.mlp.experts.255.gate_proj", + "model.layers.60.mlp.experts.255.up_proj", + "model.layers.60.mlp.experts.26.down_proj", + "model.layers.60.mlp.experts.26.gate_proj", + "model.layers.60.mlp.experts.26.up_proj", + "model.layers.60.mlp.experts.27.down_proj", + "model.layers.60.mlp.experts.27.gate_proj", + "model.layers.60.mlp.experts.27.up_proj", + "model.layers.60.mlp.experts.28.down_proj", + "model.layers.60.mlp.experts.28.gate_proj", + "model.layers.60.mlp.experts.28.up_proj", + "model.layers.60.mlp.experts.29.down_proj", + "model.layers.60.mlp.experts.29.gate_proj", + "model.layers.60.mlp.experts.29.up_proj", + "model.layers.60.mlp.experts.3.down_proj", + "model.layers.60.mlp.experts.3.gate_proj", + "model.layers.60.mlp.experts.3.up_proj", + "model.layers.60.mlp.experts.30.down_proj", + "model.layers.60.mlp.experts.30.gate_proj", + "model.layers.60.mlp.experts.30.up_proj", + "model.layers.60.mlp.experts.31.down_proj", + "model.layers.60.mlp.experts.31.gate_proj", + "model.layers.60.mlp.experts.31.up_proj", + "model.layers.60.mlp.experts.32.down_proj", + "model.layers.60.mlp.experts.32.gate_proj", + "model.layers.60.mlp.experts.32.up_proj", + "model.layers.60.mlp.experts.33.down_proj", + "model.layers.60.mlp.experts.33.gate_proj", + "model.layers.60.mlp.experts.33.up_proj", + "model.layers.60.mlp.experts.34.down_proj", + "model.layers.60.mlp.experts.34.gate_proj", + "model.layers.60.mlp.experts.34.up_proj", + "model.layers.60.mlp.experts.35.down_proj", + "model.layers.60.mlp.experts.35.gate_proj", + "model.layers.60.mlp.experts.35.up_proj", + "model.layers.60.mlp.experts.36.down_proj", + "model.layers.60.mlp.experts.36.gate_proj", + "model.layers.60.mlp.experts.36.up_proj", + "model.layers.60.mlp.experts.37.down_proj", + "model.layers.60.mlp.experts.37.gate_proj", + "model.layers.60.mlp.experts.37.up_proj", + "model.layers.60.mlp.experts.38.down_proj", + "model.layers.60.mlp.experts.38.gate_proj", + "model.layers.60.mlp.experts.38.up_proj", + "model.layers.60.mlp.experts.39.down_proj", + "model.layers.60.mlp.experts.39.gate_proj", + "model.layers.60.mlp.experts.39.up_proj", + "model.layers.60.mlp.experts.4.down_proj", + "model.layers.60.mlp.experts.4.gate_proj", + "model.layers.60.mlp.experts.4.up_proj", + "model.layers.60.mlp.experts.40.down_proj", + "model.layers.60.mlp.experts.40.gate_proj", + "model.layers.60.mlp.experts.40.up_proj", + "model.layers.60.mlp.experts.41.down_proj", + "model.layers.60.mlp.experts.41.gate_proj", + "model.layers.60.mlp.experts.41.up_proj", + "model.layers.60.mlp.experts.42.down_proj", + "model.layers.60.mlp.experts.42.gate_proj", + "model.layers.60.mlp.experts.42.up_proj", + "model.layers.60.mlp.experts.43.down_proj", + "model.layers.60.mlp.experts.43.gate_proj", + "model.layers.60.mlp.experts.43.up_proj", + "model.layers.60.mlp.experts.44.down_proj", + "model.layers.60.mlp.experts.44.gate_proj", + "model.layers.60.mlp.experts.44.up_proj", + "model.layers.60.mlp.experts.45.down_proj", + "model.layers.60.mlp.experts.45.gate_proj", + "model.layers.60.mlp.experts.45.up_proj", + "model.layers.60.mlp.experts.46.down_proj", + "model.layers.60.mlp.experts.46.gate_proj", + "model.layers.60.mlp.experts.46.up_proj", + "model.layers.60.mlp.experts.47.down_proj", + "model.layers.60.mlp.experts.47.gate_proj", + "model.layers.60.mlp.experts.47.up_proj", + "model.layers.60.mlp.experts.48.down_proj", + "model.layers.60.mlp.experts.48.gate_proj", + "model.layers.60.mlp.experts.48.up_proj", + "model.layers.60.mlp.experts.49.down_proj", + "model.layers.60.mlp.experts.49.gate_proj", + "model.layers.60.mlp.experts.49.up_proj", + "model.layers.60.mlp.experts.5.down_proj", + "model.layers.60.mlp.experts.5.gate_proj", + "model.layers.60.mlp.experts.5.up_proj", + "model.layers.60.mlp.experts.50.down_proj", + "model.layers.60.mlp.experts.50.gate_proj", + "model.layers.60.mlp.experts.50.up_proj", + "model.layers.60.mlp.experts.51.down_proj", + "model.layers.60.mlp.experts.51.gate_proj", + "model.layers.60.mlp.experts.51.up_proj", + "model.layers.60.mlp.experts.52.down_proj", + "model.layers.60.mlp.experts.52.gate_proj", + "model.layers.60.mlp.experts.52.up_proj", + "model.layers.60.mlp.experts.53.down_proj", + "model.layers.60.mlp.experts.53.gate_proj", + "model.layers.60.mlp.experts.53.up_proj", + "model.layers.60.mlp.experts.54.down_proj", + "model.layers.60.mlp.experts.54.gate_proj", + "model.layers.60.mlp.experts.54.up_proj", + "model.layers.60.mlp.experts.55.down_proj", + "model.layers.60.mlp.experts.55.gate_proj", + "model.layers.60.mlp.experts.55.up_proj", + "model.layers.60.mlp.experts.56.down_proj", + "model.layers.60.mlp.experts.56.gate_proj", + "model.layers.60.mlp.experts.56.up_proj", + "model.layers.60.mlp.experts.57.down_proj", + "model.layers.60.mlp.experts.57.gate_proj", + "model.layers.60.mlp.experts.57.up_proj", + "model.layers.60.mlp.experts.58.down_proj", + "model.layers.60.mlp.experts.58.gate_proj", + "model.layers.60.mlp.experts.58.up_proj", + "model.layers.60.mlp.experts.59.down_proj", + "model.layers.60.mlp.experts.59.gate_proj", + "model.layers.60.mlp.experts.59.up_proj", + "model.layers.60.mlp.experts.6.down_proj", + "model.layers.60.mlp.experts.6.gate_proj", + "model.layers.60.mlp.experts.6.up_proj", + "model.layers.60.mlp.experts.60.down_proj", + "model.layers.60.mlp.experts.60.gate_proj", + "model.layers.60.mlp.experts.60.up_proj", + "model.layers.60.mlp.experts.61.down_proj", + "model.layers.60.mlp.experts.61.gate_proj", + "model.layers.60.mlp.experts.61.up_proj", + "model.layers.60.mlp.experts.62.down_proj", + "model.layers.60.mlp.experts.62.gate_proj", + "model.layers.60.mlp.experts.62.up_proj", + "model.layers.60.mlp.experts.63.down_proj", + "model.layers.60.mlp.experts.63.gate_proj", + "model.layers.60.mlp.experts.63.up_proj", + "model.layers.60.mlp.experts.64.down_proj", + "model.layers.60.mlp.experts.64.gate_proj", + "model.layers.60.mlp.experts.64.up_proj", + "model.layers.60.mlp.experts.65.down_proj", + "model.layers.60.mlp.experts.65.gate_proj", + "model.layers.60.mlp.experts.65.up_proj", + "model.layers.60.mlp.experts.66.down_proj", + "model.layers.60.mlp.experts.66.gate_proj", + "model.layers.60.mlp.experts.66.up_proj", + "model.layers.60.mlp.experts.67.down_proj", + "model.layers.60.mlp.experts.67.gate_proj", + "model.layers.60.mlp.experts.67.up_proj", + "model.layers.60.mlp.experts.68.down_proj", + "model.layers.60.mlp.experts.68.gate_proj", + "model.layers.60.mlp.experts.68.up_proj", + "model.layers.60.mlp.experts.69.down_proj", + "model.layers.60.mlp.experts.69.gate_proj", + "model.layers.60.mlp.experts.69.up_proj", + "model.layers.60.mlp.experts.7.down_proj", + "model.layers.60.mlp.experts.7.gate_proj", + "model.layers.60.mlp.experts.7.up_proj", + "model.layers.60.mlp.experts.70.down_proj", + "model.layers.60.mlp.experts.70.gate_proj", + "model.layers.60.mlp.experts.70.up_proj", + "model.layers.60.mlp.experts.71.down_proj", + "model.layers.60.mlp.experts.71.gate_proj", + "model.layers.60.mlp.experts.71.up_proj", + "model.layers.60.mlp.experts.72.down_proj", + "model.layers.60.mlp.experts.72.gate_proj", + "model.layers.60.mlp.experts.72.up_proj", + "model.layers.60.mlp.experts.73.down_proj", + "model.layers.60.mlp.experts.73.gate_proj", + "model.layers.60.mlp.experts.73.up_proj", + "model.layers.60.mlp.experts.74.down_proj", + "model.layers.60.mlp.experts.74.gate_proj", + "model.layers.60.mlp.experts.74.up_proj", + "model.layers.60.mlp.experts.75.down_proj", + "model.layers.60.mlp.experts.75.gate_proj", + "model.layers.60.mlp.experts.75.up_proj", + "model.layers.60.mlp.experts.76.down_proj", + "model.layers.60.mlp.experts.76.gate_proj", + "model.layers.60.mlp.experts.76.up_proj", + "model.layers.60.mlp.experts.77.down_proj", + "model.layers.60.mlp.experts.77.gate_proj", + "model.layers.60.mlp.experts.77.up_proj", + "model.layers.60.mlp.experts.78.down_proj", + "model.layers.60.mlp.experts.78.gate_proj", + "model.layers.60.mlp.experts.78.up_proj", + "model.layers.60.mlp.experts.79.down_proj", + "model.layers.60.mlp.experts.79.gate_proj", + "model.layers.60.mlp.experts.79.up_proj", + "model.layers.60.mlp.experts.8.down_proj", + "model.layers.60.mlp.experts.8.gate_proj", + "model.layers.60.mlp.experts.8.up_proj", + "model.layers.60.mlp.experts.80.down_proj", + "model.layers.60.mlp.experts.80.gate_proj", + "model.layers.60.mlp.experts.80.up_proj", + "model.layers.60.mlp.experts.81.down_proj", + "model.layers.60.mlp.experts.81.gate_proj", + "model.layers.60.mlp.experts.81.up_proj", + "model.layers.60.mlp.experts.82.down_proj", + "model.layers.60.mlp.experts.82.gate_proj", + "model.layers.60.mlp.experts.82.up_proj", + "model.layers.60.mlp.experts.83.down_proj", + "model.layers.60.mlp.experts.83.gate_proj", + "model.layers.60.mlp.experts.83.up_proj", + "model.layers.60.mlp.experts.84.down_proj", + "model.layers.60.mlp.experts.84.gate_proj", + "model.layers.60.mlp.experts.84.up_proj", + "model.layers.60.mlp.experts.85.down_proj", + "model.layers.60.mlp.experts.85.gate_proj", + "model.layers.60.mlp.experts.85.up_proj", + "model.layers.60.mlp.experts.86.down_proj", + "model.layers.60.mlp.experts.86.gate_proj", + "model.layers.60.mlp.experts.86.up_proj", + "model.layers.60.mlp.experts.87.down_proj", + "model.layers.60.mlp.experts.87.gate_proj", + "model.layers.60.mlp.experts.87.up_proj", + "model.layers.60.mlp.experts.88.down_proj", + "model.layers.60.mlp.experts.88.gate_proj", + "model.layers.60.mlp.experts.88.up_proj", + "model.layers.60.mlp.experts.89.down_proj", + "model.layers.60.mlp.experts.89.gate_proj", + "model.layers.60.mlp.experts.89.up_proj", + "model.layers.60.mlp.experts.9.down_proj", + "model.layers.60.mlp.experts.9.gate_proj", + "model.layers.60.mlp.experts.9.up_proj", + "model.layers.60.mlp.experts.90.down_proj", + "model.layers.60.mlp.experts.90.gate_proj", + "model.layers.60.mlp.experts.90.up_proj", + "model.layers.60.mlp.experts.91.down_proj", + "model.layers.60.mlp.experts.91.gate_proj", + "model.layers.60.mlp.experts.91.up_proj", + "model.layers.60.mlp.experts.92.down_proj", + "model.layers.60.mlp.experts.92.gate_proj", + "model.layers.60.mlp.experts.92.up_proj", + "model.layers.60.mlp.experts.93.down_proj", + "model.layers.60.mlp.experts.93.gate_proj", + "model.layers.60.mlp.experts.93.up_proj", + "model.layers.60.mlp.experts.94.down_proj", + "model.layers.60.mlp.experts.94.gate_proj", + "model.layers.60.mlp.experts.94.up_proj", + "model.layers.60.mlp.experts.95.down_proj", + "model.layers.60.mlp.experts.95.gate_proj", + "model.layers.60.mlp.experts.95.up_proj", + "model.layers.60.mlp.experts.96.down_proj", + "model.layers.60.mlp.experts.96.gate_proj", + "model.layers.60.mlp.experts.96.up_proj", + "model.layers.60.mlp.experts.97.down_proj", + "model.layers.60.mlp.experts.97.gate_proj", + "model.layers.60.mlp.experts.97.up_proj", + "model.layers.60.mlp.experts.98.down_proj", + "model.layers.60.mlp.experts.98.gate_proj", + "model.layers.60.mlp.experts.98.up_proj", + "model.layers.60.mlp.experts.99.down_proj", + "model.layers.60.mlp.experts.99.gate_proj", + "model.layers.60.mlp.experts.99.up_proj", + "model.layers.61.mlp.experts.0.down_proj", + "model.layers.61.mlp.experts.0.gate_proj", + "model.layers.61.mlp.experts.0.up_proj", + "model.layers.61.mlp.experts.1.down_proj", + "model.layers.61.mlp.experts.1.gate_proj", + "model.layers.61.mlp.experts.1.up_proj", + "model.layers.61.mlp.experts.10.down_proj", + "model.layers.61.mlp.experts.10.gate_proj", + "model.layers.61.mlp.experts.10.up_proj", + "model.layers.61.mlp.experts.100.down_proj", + "model.layers.61.mlp.experts.100.gate_proj", + "model.layers.61.mlp.experts.100.up_proj", + "model.layers.61.mlp.experts.101.down_proj", + "model.layers.61.mlp.experts.101.gate_proj", + "model.layers.61.mlp.experts.101.up_proj", + "model.layers.61.mlp.experts.102.down_proj", + "model.layers.61.mlp.experts.102.gate_proj", + "model.layers.61.mlp.experts.102.up_proj", + "model.layers.61.mlp.experts.103.down_proj", + "model.layers.61.mlp.experts.103.gate_proj", + "model.layers.61.mlp.experts.103.up_proj", + "model.layers.61.mlp.experts.104.down_proj", + "model.layers.61.mlp.experts.104.gate_proj", + "model.layers.61.mlp.experts.104.up_proj", + "model.layers.61.mlp.experts.105.down_proj", + "model.layers.61.mlp.experts.105.gate_proj", + "model.layers.61.mlp.experts.105.up_proj", + "model.layers.61.mlp.experts.106.down_proj", + "model.layers.61.mlp.experts.106.gate_proj", + "model.layers.61.mlp.experts.106.up_proj", + "model.layers.61.mlp.experts.107.down_proj", + "model.layers.61.mlp.experts.107.gate_proj", + "model.layers.61.mlp.experts.107.up_proj", + "model.layers.61.mlp.experts.108.down_proj", + "model.layers.61.mlp.experts.108.gate_proj", + "model.layers.61.mlp.experts.108.up_proj", + "model.layers.61.mlp.experts.109.down_proj", + "model.layers.61.mlp.experts.109.gate_proj", + "model.layers.61.mlp.experts.109.up_proj", + "model.layers.61.mlp.experts.11.down_proj", + "model.layers.61.mlp.experts.11.gate_proj", + "model.layers.61.mlp.experts.11.up_proj", + "model.layers.61.mlp.experts.110.down_proj", + "model.layers.61.mlp.experts.110.gate_proj", + "model.layers.61.mlp.experts.110.up_proj", + "model.layers.61.mlp.experts.111.down_proj", + "model.layers.61.mlp.experts.111.gate_proj", + "model.layers.61.mlp.experts.111.up_proj", + "model.layers.61.mlp.experts.112.down_proj", + "model.layers.61.mlp.experts.112.gate_proj", + "model.layers.61.mlp.experts.112.up_proj", + "model.layers.61.mlp.experts.113.down_proj", + "model.layers.61.mlp.experts.113.gate_proj", + "model.layers.61.mlp.experts.113.up_proj", + "model.layers.61.mlp.experts.114.down_proj", + "model.layers.61.mlp.experts.114.gate_proj", + "model.layers.61.mlp.experts.114.up_proj", + "model.layers.61.mlp.experts.115.down_proj", + "model.layers.61.mlp.experts.115.gate_proj", + "model.layers.61.mlp.experts.115.up_proj", + "model.layers.61.mlp.experts.116.down_proj", + "model.layers.61.mlp.experts.116.gate_proj", + "model.layers.61.mlp.experts.116.up_proj", + "model.layers.61.mlp.experts.117.down_proj", + "model.layers.61.mlp.experts.117.gate_proj", + "model.layers.61.mlp.experts.117.up_proj", + "model.layers.61.mlp.experts.118.down_proj", + "model.layers.61.mlp.experts.118.gate_proj", + "model.layers.61.mlp.experts.118.up_proj", + "model.layers.61.mlp.experts.119.down_proj", + "model.layers.61.mlp.experts.119.gate_proj", + "model.layers.61.mlp.experts.119.up_proj", + "model.layers.61.mlp.experts.12.down_proj", + "model.layers.61.mlp.experts.12.gate_proj", + "model.layers.61.mlp.experts.12.up_proj", + "model.layers.61.mlp.experts.120.down_proj", + "model.layers.61.mlp.experts.120.gate_proj", + "model.layers.61.mlp.experts.120.up_proj", + "model.layers.61.mlp.experts.121.down_proj", + "model.layers.61.mlp.experts.121.gate_proj", + "model.layers.61.mlp.experts.121.up_proj", + "model.layers.61.mlp.experts.122.down_proj", + "model.layers.61.mlp.experts.122.gate_proj", + "model.layers.61.mlp.experts.122.up_proj", + "model.layers.61.mlp.experts.123.down_proj", + "model.layers.61.mlp.experts.123.gate_proj", + "model.layers.61.mlp.experts.123.up_proj", + "model.layers.61.mlp.experts.124.down_proj", + "model.layers.61.mlp.experts.124.gate_proj", + "model.layers.61.mlp.experts.124.up_proj", + "model.layers.61.mlp.experts.125.down_proj", + "model.layers.61.mlp.experts.125.gate_proj", + "model.layers.61.mlp.experts.125.up_proj", + "model.layers.61.mlp.experts.126.down_proj", + "model.layers.61.mlp.experts.126.gate_proj", + "model.layers.61.mlp.experts.126.up_proj", + "model.layers.61.mlp.experts.127.down_proj", + "model.layers.61.mlp.experts.127.gate_proj", + "model.layers.61.mlp.experts.127.up_proj", + "model.layers.61.mlp.experts.128.down_proj", + "model.layers.61.mlp.experts.128.gate_proj", + "model.layers.61.mlp.experts.128.up_proj", + "model.layers.61.mlp.experts.129.down_proj", + "model.layers.61.mlp.experts.129.gate_proj", + "model.layers.61.mlp.experts.129.up_proj", + "model.layers.61.mlp.experts.13.down_proj", + "model.layers.61.mlp.experts.13.gate_proj", + "model.layers.61.mlp.experts.13.up_proj", + "model.layers.61.mlp.experts.130.down_proj", + "model.layers.61.mlp.experts.130.gate_proj", + "model.layers.61.mlp.experts.130.up_proj", + "model.layers.61.mlp.experts.131.down_proj", + "model.layers.61.mlp.experts.131.gate_proj", + "model.layers.61.mlp.experts.131.up_proj", + "model.layers.61.mlp.experts.132.down_proj", + "model.layers.61.mlp.experts.132.gate_proj", + "model.layers.61.mlp.experts.132.up_proj", + "model.layers.61.mlp.experts.133.down_proj", + "model.layers.61.mlp.experts.133.gate_proj", + "model.layers.61.mlp.experts.133.up_proj", + "model.layers.61.mlp.experts.134.down_proj", + "model.layers.61.mlp.experts.134.gate_proj", + "model.layers.61.mlp.experts.134.up_proj", + "model.layers.61.mlp.experts.135.down_proj", + "model.layers.61.mlp.experts.135.gate_proj", + "model.layers.61.mlp.experts.135.up_proj", + "model.layers.61.mlp.experts.136.down_proj", + "model.layers.61.mlp.experts.136.gate_proj", + "model.layers.61.mlp.experts.136.up_proj", + "model.layers.61.mlp.experts.137.down_proj", + "model.layers.61.mlp.experts.137.gate_proj", + "model.layers.61.mlp.experts.137.up_proj", + "model.layers.61.mlp.experts.138.down_proj", + "model.layers.61.mlp.experts.138.gate_proj", + "model.layers.61.mlp.experts.138.up_proj", + "model.layers.61.mlp.experts.139.down_proj", + "model.layers.61.mlp.experts.139.gate_proj", + "model.layers.61.mlp.experts.139.up_proj", + "model.layers.61.mlp.experts.14.down_proj", + "model.layers.61.mlp.experts.14.gate_proj", + "model.layers.61.mlp.experts.14.up_proj", + "model.layers.61.mlp.experts.140.down_proj", + "model.layers.61.mlp.experts.140.gate_proj", + "model.layers.61.mlp.experts.140.up_proj", + "model.layers.61.mlp.experts.141.down_proj", + "model.layers.61.mlp.experts.141.gate_proj", + "model.layers.61.mlp.experts.141.up_proj", + "model.layers.61.mlp.experts.142.down_proj", + "model.layers.61.mlp.experts.142.gate_proj", + "model.layers.61.mlp.experts.142.up_proj", + "model.layers.61.mlp.experts.143.down_proj", + "model.layers.61.mlp.experts.143.gate_proj", + "model.layers.61.mlp.experts.143.up_proj", + "model.layers.61.mlp.experts.144.down_proj", + "model.layers.61.mlp.experts.144.gate_proj", + "model.layers.61.mlp.experts.144.up_proj", + "model.layers.61.mlp.experts.145.down_proj", + "model.layers.61.mlp.experts.145.gate_proj", + "model.layers.61.mlp.experts.145.up_proj", + "model.layers.61.mlp.experts.146.down_proj", + "model.layers.61.mlp.experts.146.gate_proj", + "model.layers.61.mlp.experts.146.up_proj", + "model.layers.61.mlp.experts.147.down_proj", + "model.layers.61.mlp.experts.147.gate_proj", + "model.layers.61.mlp.experts.147.up_proj", + "model.layers.61.mlp.experts.148.down_proj", + "model.layers.61.mlp.experts.148.gate_proj", + "model.layers.61.mlp.experts.148.up_proj", + "model.layers.61.mlp.experts.149.down_proj", + "model.layers.61.mlp.experts.149.gate_proj", + "model.layers.61.mlp.experts.149.up_proj", + "model.layers.61.mlp.experts.15.down_proj", + "model.layers.61.mlp.experts.15.gate_proj", + "model.layers.61.mlp.experts.15.up_proj", + "model.layers.61.mlp.experts.150.down_proj", + "model.layers.61.mlp.experts.150.gate_proj", + "model.layers.61.mlp.experts.150.up_proj", + "model.layers.61.mlp.experts.151.down_proj", + "model.layers.61.mlp.experts.151.gate_proj", + "model.layers.61.mlp.experts.151.up_proj", + "model.layers.61.mlp.experts.152.down_proj", + "model.layers.61.mlp.experts.152.gate_proj", + "model.layers.61.mlp.experts.152.up_proj", + "model.layers.61.mlp.experts.153.down_proj", + "model.layers.61.mlp.experts.153.gate_proj", + "model.layers.61.mlp.experts.153.up_proj", + "model.layers.61.mlp.experts.154.down_proj", + "model.layers.61.mlp.experts.154.gate_proj", + "model.layers.61.mlp.experts.154.up_proj", + "model.layers.61.mlp.experts.155.down_proj", + "model.layers.61.mlp.experts.155.gate_proj", + "model.layers.61.mlp.experts.155.up_proj", + "model.layers.61.mlp.experts.156.down_proj", + "model.layers.61.mlp.experts.156.gate_proj", + "model.layers.61.mlp.experts.156.up_proj", + "model.layers.61.mlp.experts.157.down_proj", + "model.layers.61.mlp.experts.157.gate_proj", + "model.layers.61.mlp.experts.157.up_proj", + "model.layers.61.mlp.experts.158.down_proj", + "model.layers.61.mlp.experts.158.gate_proj", + "model.layers.61.mlp.experts.158.up_proj", + "model.layers.61.mlp.experts.159.down_proj", + "model.layers.61.mlp.experts.159.gate_proj", + "model.layers.61.mlp.experts.159.up_proj", + "model.layers.61.mlp.experts.16.down_proj", + "model.layers.61.mlp.experts.16.gate_proj", + "model.layers.61.mlp.experts.16.up_proj", + "model.layers.61.mlp.experts.160.down_proj", + "model.layers.61.mlp.experts.160.gate_proj", + "model.layers.61.mlp.experts.160.up_proj", + "model.layers.61.mlp.experts.161.down_proj", + "model.layers.61.mlp.experts.161.gate_proj", + "model.layers.61.mlp.experts.161.up_proj", + "model.layers.61.mlp.experts.162.down_proj", + "model.layers.61.mlp.experts.162.gate_proj", + "model.layers.61.mlp.experts.162.up_proj", + "model.layers.61.mlp.experts.163.down_proj", + "model.layers.61.mlp.experts.163.gate_proj", + "model.layers.61.mlp.experts.163.up_proj", + "model.layers.61.mlp.experts.164.down_proj", + "model.layers.61.mlp.experts.164.gate_proj", + "model.layers.61.mlp.experts.164.up_proj", + "model.layers.61.mlp.experts.165.down_proj", + "model.layers.61.mlp.experts.165.gate_proj", + "model.layers.61.mlp.experts.165.up_proj", + "model.layers.61.mlp.experts.166.down_proj", + "model.layers.61.mlp.experts.166.gate_proj", + "model.layers.61.mlp.experts.166.up_proj", + "model.layers.61.mlp.experts.167.down_proj", + "model.layers.61.mlp.experts.167.gate_proj", + "model.layers.61.mlp.experts.167.up_proj", + "model.layers.61.mlp.experts.168.down_proj", + "model.layers.61.mlp.experts.168.gate_proj", + "model.layers.61.mlp.experts.168.up_proj", + "model.layers.61.mlp.experts.169.down_proj", + "model.layers.61.mlp.experts.169.gate_proj", + "model.layers.61.mlp.experts.169.up_proj", + "model.layers.61.mlp.experts.17.down_proj", + "model.layers.61.mlp.experts.17.gate_proj", + "model.layers.61.mlp.experts.17.up_proj", + "model.layers.61.mlp.experts.170.down_proj", + "model.layers.61.mlp.experts.170.gate_proj", + "model.layers.61.mlp.experts.170.up_proj", + "model.layers.61.mlp.experts.171.down_proj", + "model.layers.61.mlp.experts.171.gate_proj", + "model.layers.61.mlp.experts.171.up_proj", + "model.layers.61.mlp.experts.172.down_proj", + "model.layers.61.mlp.experts.172.gate_proj", + "model.layers.61.mlp.experts.172.up_proj", + "model.layers.61.mlp.experts.173.down_proj", + "model.layers.61.mlp.experts.173.gate_proj", + "model.layers.61.mlp.experts.173.up_proj", + "model.layers.61.mlp.experts.174.down_proj", + "model.layers.61.mlp.experts.174.gate_proj", + "model.layers.61.mlp.experts.174.up_proj", + "model.layers.61.mlp.experts.175.down_proj", + "model.layers.61.mlp.experts.175.gate_proj", + "model.layers.61.mlp.experts.175.up_proj", + "model.layers.61.mlp.experts.176.down_proj", + "model.layers.61.mlp.experts.176.gate_proj", + "model.layers.61.mlp.experts.176.up_proj", + "model.layers.61.mlp.experts.177.down_proj", + "model.layers.61.mlp.experts.177.gate_proj", + "model.layers.61.mlp.experts.177.up_proj", + "model.layers.61.mlp.experts.178.down_proj", + "model.layers.61.mlp.experts.178.gate_proj", + "model.layers.61.mlp.experts.178.up_proj", + "model.layers.61.mlp.experts.179.down_proj", + "model.layers.61.mlp.experts.179.gate_proj", + "model.layers.61.mlp.experts.179.up_proj", + "model.layers.61.mlp.experts.18.down_proj", + "model.layers.61.mlp.experts.18.gate_proj", + "model.layers.61.mlp.experts.18.up_proj", + "model.layers.61.mlp.experts.180.down_proj", + "model.layers.61.mlp.experts.180.gate_proj", + "model.layers.61.mlp.experts.180.up_proj", + "model.layers.61.mlp.experts.181.down_proj", + "model.layers.61.mlp.experts.181.gate_proj", + "model.layers.61.mlp.experts.181.up_proj", + "model.layers.61.mlp.experts.182.down_proj", + "model.layers.61.mlp.experts.182.gate_proj", + "model.layers.61.mlp.experts.182.up_proj", + "model.layers.61.mlp.experts.183.down_proj", + "model.layers.61.mlp.experts.183.gate_proj", + "model.layers.61.mlp.experts.183.up_proj", + "model.layers.61.mlp.experts.184.down_proj", + "model.layers.61.mlp.experts.184.gate_proj", + "model.layers.61.mlp.experts.184.up_proj", + "model.layers.61.mlp.experts.185.down_proj", + "model.layers.61.mlp.experts.185.gate_proj", + "model.layers.61.mlp.experts.185.up_proj", + "model.layers.61.mlp.experts.186.down_proj", + "model.layers.61.mlp.experts.186.gate_proj", + "model.layers.61.mlp.experts.186.up_proj", + "model.layers.61.mlp.experts.187.down_proj", + "model.layers.61.mlp.experts.187.gate_proj", + "model.layers.61.mlp.experts.187.up_proj", + "model.layers.61.mlp.experts.188.down_proj", + "model.layers.61.mlp.experts.188.gate_proj", + "model.layers.61.mlp.experts.188.up_proj", + "model.layers.61.mlp.experts.189.down_proj", + "model.layers.61.mlp.experts.189.gate_proj", + "model.layers.61.mlp.experts.189.up_proj", + "model.layers.61.mlp.experts.19.down_proj", + "model.layers.61.mlp.experts.19.gate_proj", + "model.layers.61.mlp.experts.19.up_proj", + "model.layers.61.mlp.experts.190.down_proj", + "model.layers.61.mlp.experts.190.gate_proj", + "model.layers.61.mlp.experts.190.up_proj", + "model.layers.61.mlp.experts.191.down_proj", + "model.layers.61.mlp.experts.191.gate_proj", + "model.layers.61.mlp.experts.191.up_proj", + "model.layers.61.mlp.experts.192.down_proj", + "model.layers.61.mlp.experts.192.gate_proj", + "model.layers.61.mlp.experts.192.up_proj", + "model.layers.61.mlp.experts.193.down_proj", + "model.layers.61.mlp.experts.193.gate_proj", + "model.layers.61.mlp.experts.193.up_proj", + "model.layers.61.mlp.experts.194.down_proj", + "model.layers.61.mlp.experts.194.gate_proj", + "model.layers.61.mlp.experts.194.up_proj", + "model.layers.61.mlp.experts.195.down_proj", + "model.layers.61.mlp.experts.195.gate_proj", + "model.layers.61.mlp.experts.195.up_proj", + "model.layers.61.mlp.experts.196.down_proj", + "model.layers.61.mlp.experts.196.gate_proj", + "model.layers.61.mlp.experts.196.up_proj", + "model.layers.61.mlp.experts.197.down_proj", + "model.layers.61.mlp.experts.197.gate_proj", + "model.layers.61.mlp.experts.197.up_proj", + "model.layers.61.mlp.experts.198.down_proj", + "model.layers.61.mlp.experts.198.gate_proj", + "model.layers.61.mlp.experts.198.up_proj", + "model.layers.61.mlp.experts.199.down_proj", + "model.layers.61.mlp.experts.199.gate_proj", + "model.layers.61.mlp.experts.199.up_proj", + "model.layers.61.mlp.experts.2.down_proj", + "model.layers.61.mlp.experts.2.gate_proj", + "model.layers.61.mlp.experts.2.up_proj", + "model.layers.61.mlp.experts.20.down_proj", + "model.layers.61.mlp.experts.20.gate_proj", + "model.layers.61.mlp.experts.20.up_proj", + "model.layers.61.mlp.experts.200.down_proj", + "model.layers.61.mlp.experts.200.gate_proj", + "model.layers.61.mlp.experts.200.up_proj", + "model.layers.61.mlp.experts.201.down_proj", + "model.layers.61.mlp.experts.201.gate_proj", + "model.layers.61.mlp.experts.201.up_proj", + "model.layers.61.mlp.experts.202.down_proj", + "model.layers.61.mlp.experts.202.gate_proj", + "model.layers.61.mlp.experts.202.up_proj", + "model.layers.61.mlp.experts.203.down_proj", + "model.layers.61.mlp.experts.203.gate_proj", + "model.layers.61.mlp.experts.203.up_proj", + "model.layers.61.mlp.experts.204.down_proj", + "model.layers.61.mlp.experts.204.gate_proj", + "model.layers.61.mlp.experts.204.up_proj", + "model.layers.61.mlp.experts.205.down_proj", + "model.layers.61.mlp.experts.205.gate_proj", + "model.layers.61.mlp.experts.205.up_proj", + "model.layers.61.mlp.experts.206.down_proj", + "model.layers.61.mlp.experts.206.gate_proj", + "model.layers.61.mlp.experts.206.up_proj", + "model.layers.61.mlp.experts.207.down_proj", + "model.layers.61.mlp.experts.207.gate_proj", + "model.layers.61.mlp.experts.207.up_proj", + "model.layers.61.mlp.experts.208.down_proj", + "model.layers.61.mlp.experts.208.gate_proj", + "model.layers.61.mlp.experts.208.up_proj", + "model.layers.61.mlp.experts.209.down_proj", + "model.layers.61.mlp.experts.209.gate_proj", + "model.layers.61.mlp.experts.209.up_proj", + "model.layers.61.mlp.experts.21.down_proj", + "model.layers.61.mlp.experts.21.gate_proj", + "model.layers.61.mlp.experts.21.up_proj", + "model.layers.61.mlp.experts.210.down_proj", + "model.layers.61.mlp.experts.210.gate_proj", + "model.layers.61.mlp.experts.210.up_proj", + "model.layers.61.mlp.experts.211.down_proj", + "model.layers.61.mlp.experts.211.gate_proj", + "model.layers.61.mlp.experts.211.up_proj", + "model.layers.61.mlp.experts.212.down_proj", + "model.layers.61.mlp.experts.212.gate_proj", + "model.layers.61.mlp.experts.212.up_proj", + "model.layers.61.mlp.experts.213.down_proj", + "model.layers.61.mlp.experts.213.gate_proj", + "model.layers.61.mlp.experts.213.up_proj", + "model.layers.61.mlp.experts.214.down_proj", + "model.layers.61.mlp.experts.214.gate_proj", + "model.layers.61.mlp.experts.214.up_proj", + "model.layers.61.mlp.experts.215.down_proj", + "model.layers.61.mlp.experts.215.gate_proj", + "model.layers.61.mlp.experts.215.up_proj", + "model.layers.61.mlp.experts.216.down_proj", + "model.layers.61.mlp.experts.216.gate_proj", + "model.layers.61.mlp.experts.216.up_proj", + "model.layers.61.mlp.experts.217.down_proj", + "model.layers.61.mlp.experts.217.gate_proj", + "model.layers.61.mlp.experts.217.up_proj", + "model.layers.61.mlp.experts.218.down_proj", + "model.layers.61.mlp.experts.218.gate_proj", + "model.layers.61.mlp.experts.218.up_proj", + "model.layers.61.mlp.experts.219.down_proj", + "model.layers.61.mlp.experts.219.gate_proj", + "model.layers.61.mlp.experts.219.up_proj", + "model.layers.61.mlp.experts.22.down_proj", + "model.layers.61.mlp.experts.22.gate_proj", + "model.layers.61.mlp.experts.22.up_proj", + "model.layers.61.mlp.experts.220.down_proj", + "model.layers.61.mlp.experts.220.gate_proj", + "model.layers.61.mlp.experts.220.up_proj", + "model.layers.61.mlp.experts.221.down_proj", + "model.layers.61.mlp.experts.221.gate_proj", + "model.layers.61.mlp.experts.221.up_proj", + "model.layers.61.mlp.experts.222.down_proj", + "model.layers.61.mlp.experts.222.gate_proj", + "model.layers.61.mlp.experts.222.up_proj", + "model.layers.61.mlp.experts.223.down_proj", + "model.layers.61.mlp.experts.223.gate_proj", + "model.layers.61.mlp.experts.223.up_proj", + "model.layers.61.mlp.experts.224.down_proj", + "model.layers.61.mlp.experts.224.gate_proj", + "model.layers.61.mlp.experts.224.up_proj", + "model.layers.61.mlp.experts.225.down_proj", + "model.layers.61.mlp.experts.225.gate_proj", + "model.layers.61.mlp.experts.225.up_proj", + "model.layers.61.mlp.experts.226.down_proj", + "model.layers.61.mlp.experts.226.gate_proj", + "model.layers.61.mlp.experts.226.up_proj", + "model.layers.61.mlp.experts.227.down_proj", + "model.layers.61.mlp.experts.227.gate_proj", + "model.layers.61.mlp.experts.227.up_proj", + "model.layers.61.mlp.experts.228.down_proj", + "model.layers.61.mlp.experts.228.gate_proj", + "model.layers.61.mlp.experts.228.up_proj", + "model.layers.61.mlp.experts.229.down_proj", + "model.layers.61.mlp.experts.229.gate_proj", + "model.layers.61.mlp.experts.229.up_proj", + "model.layers.61.mlp.experts.23.down_proj", + "model.layers.61.mlp.experts.23.gate_proj", + "model.layers.61.mlp.experts.23.up_proj", + "model.layers.61.mlp.experts.230.down_proj", + "model.layers.61.mlp.experts.230.gate_proj", + "model.layers.61.mlp.experts.230.up_proj", + "model.layers.61.mlp.experts.231.down_proj", + "model.layers.61.mlp.experts.231.gate_proj", + "model.layers.61.mlp.experts.231.up_proj", + "model.layers.61.mlp.experts.232.down_proj", + "model.layers.61.mlp.experts.232.gate_proj", + "model.layers.61.mlp.experts.232.up_proj", + "model.layers.61.mlp.experts.233.down_proj", + "model.layers.61.mlp.experts.233.gate_proj", + "model.layers.61.mlp.experts.233.up_proj", + "model.layers.61.mlp.experts.234.down_proj", + "model.layers.61.mlp.experts.234.gate_proj", + "model.layers.61.mlp.experts.234.up_proj", + "model.layers.61.mlp.experts.235.down_proj", + "model.layers.61.mlp.experts.235.gate_proj", + "model.layers.61.mlp.experts.235.up_proj", + "model.layers.61.mlp.experts.236.down_proj", + "model.layers.61.mlp.experts.236.gate_proj", + "model.layers.61.mlp.experts.236.up_proj", + "model.layers.61.mlp.experts.237.down_proj", + "model.layers.61.mlp.experts.237.gate_proj", + "model.layers.61.mlp.experts.237.up_proj", + "model.layers.61.mlp.experts.238.down_proj", + "model.layers.61.mlp.experts.238.gate_proj", + "model.layers.61.mlp.experts.238.up_proj", + "model.layers.61.mlp.experts.239.down_proj", + "model.layers.61.mlp.experts.239.gate_proj", + "model.layers.61.mlp.experts.239.up_proj", + "model.layers.61.mlp.experts.24.down_proj", + "model.layers.61.mlp.experts.24.gate_proj", + "model.layers.61.mlp.experts.24.up_proj", + "model.layers.61.mlp.experts.240.down_proj", + "model.layers.61.mlp.experts.240.gate_proj", + "model.layers.61.mlp.experts.240.up_proj", + "model.layers.61.mlp.experts.241.down_proj", + "model.layers.61.mlp.experts.241.gate_proj", + "model.layers.61.mlp.experts.241.up_proj", + "model.layers.61.mlp.experts.242.down_proj", + "model.layers.61.mlp.experts.242.gate_proj", + "model.layers.61.mlp.experts.242.up_proj", + "model.layers.61.mlp.experts.243.down_proj", + "model.layers.61.mlp.experts.243.gate_proj", + "model.layers.61.mlp.experts.243.up_proj", + "model.layers.61.mlp.experts.244.down_proj", + "model.layers.61.mlp.experts.244.gate_proj", + "model.layers.61.mlp.experts.244.up_proj", + "model.layers.61.mlp.experts.245.down_proj", + "model.layers.61.mlp.experts.245.gate_proj", + "model.layers.61.mlp.experts.245.up_proj", + "model.layers.61.mlp.experts.246.down_proj", + "model.layers.61.mlp.experts.246.gate_proj", + "model.layers.61.mlp.experts.246.up_proj", + "model.layers.61.mlp.experts.247.down_proj", + "model.layers.61.mlp.experts.247.gate_proj", + "model.layers.61.mlp.experts.247.up_proj", + "model.layers.61.mlp.experts.248.down_proj", + "model.layers.61.mlp.experts.248.gate_proj", + "model.layers.61.mlp.experts.248.up_proj", + "model.layers.61.mlp.experts.249.down_proj", + "model.layers.61.mlp.experts.249.gate_proj", + "model.layers.61.mlp.experts.249.up_proj", + "model.layers.61.mlp.experts.25.down_proj", + "model.layers.61.mlp.experts.25.gate_proj", + "model.layers.61.mlp.experts.25.up_proj", + "model.layers.61.mlp.experts.250.down_proj", + "model.layers.61.mlp.experts.250.gate_proj", + "model.layers.61.mlp.experts.250.up_proj", + "model.layers.61.mlp.experts.251.down_proj", + "model.layers.61.mlp.experts.251.gate_proj", + "model.layers.61.mlp.experts.251.up_proj", + "model.layers.61.mlp.experts.252.down_proj", + "model.layers.61.mlp.experts.252.gate_proj", + "model.layers.61.mlp.experts.252.up_proj", + "model.layers.61.mlp.experts.253.down_proj", + "model.layers.61.mlp.experts.253.gate_proj", + "model.layers.61.mlp.experts.253.up_proj", + "model.layers.61.mlp.experts.254.down_proj", + "model.layers.61.mlp.experts.254.gate_proj", + "model.layers.61.mlp.experts.254.up_proj", + "model.layers.61.mlp.experts.255.down_proj", + "model.layers.61.mlp.experts.255.gate_proj", + "model.layers.61.mlp.experts.255.up_proj", + "model.layers.61.mlp.experts.26.down_proj", + "model.layers.61.mlp.experts.26.gate_proj", + "model.layers.61.mlp.experts.26.up_proj", + "model.layers.61.mlp.experts.27.down_proj", + "model.layers.61.mlp.experts.27.gate_proj", + "model.layers.61.mlp.experts.27.up_proj", + "model.layers.61.mlp.experts.28.down_proj", + "model.layers.61.mlp.experts.28.gate_proj", + "model.layers.61.mlp.experts.28.up_proj", + "model.layers.61.mlp.experts.29.down_proj", + "model.layers.61.mlp.experts.29.gate_proj", + "model.layers.61.mlp.experts.29.up_proj", + "model.layers.61.mlp.experts.3.down_proj", + "model.layers.61.mlp.experts.3.gate_proj", + "model.layers.61.mlp.experts.3.up_proj", + "model.layers.61.mlp.experts.30.down_proj", + "model.layers.61.mlp.experts.30.gate_proj", + "model.layers.61.mlp.experts.30.up_proj", + "model.layers.61.mlp.experts.31.down_proj", + "model.layers.61.mlp.experts.31.gate_proj", + "model.layers.61.mlp.experts.31.up_proj", + "model.layers.61.mlp.experts.32.down_proj", + "model.layers.61.mlp.experts.32.gate_proj", + "model.layers.61.mlp.experts.32.up_proj", + "model.layers.61.mlp.experts.33.down_proj", + "model.layers.61.mlp.experts.33.gate_proj", + "model.layers.61.mlp.experts.33.up_proj", + "model.layers.61.mlp.experts.34.down_proj", + "model.layers.61.mlp.experts.34.gate_proj", + "model.layers.61.mlp.experts.34.up_proj", + "model.layers.61.mlp.experts.35.down_proj", + "model.layers.61.mlp.experts.35.gate_proj", + "model.layers.61.mlp.experts.35.up_proj", + "model.layers.61.mlp.experts.36.down_proj", + "model.layers.61.mlp.experts.36.gate_proj", + "model.layers.61.mlp.experts.36.up_proj", + "model.layers.61.mlp.experts.37.down_proj", + "model.layers.61.mlp.experts.37.gate_proj", + "model.layers.61.mlp.experts.37.up_proj", + "model.layers.61.mlp.experts.38.down_proj", + "model.layers.61.mlp.experts.38.gate_proj", + "model.layers.61.mlp.experts.38.up_proj", + "model.layers.61.mlp.experts.39.down_proj", + "model.layers.61.mlp.experts.39.gate_proj", + "model.layers.61.mlp.experts.39.up_proj", + "model.layers.61.mlp.experts.4.down_proj", + "model.layers.61.mlp.experts.4.gate_proj", + "model.layers.61.mlp.experts.4.up_proj", + "model.layers.61.mlp.experts.40.down_proj", + "model.layers.61.mlp.experts.40.gate_proj", + "model.layers.61.mlp.experts.40.up_proj", + "model.layers.61.mlp.experts.41.down_proj", + "model.layers.61.mlp.experts.41.gate_proj", + "model.layers.61.mlp.experts.41.up_proj", + "model.layers.61.mlp.experts.42.down_proj", + "model.layers.61.mlp.experts.42.gate_proj", + "model.layers.61.mlp.experts.42.up_proj", + "model.layers.61.mlp.experts.43.down_proj", + "model.layers.61.mlp.experts.43.gate_proj", + "model.layers.61.mlp.experts.43.up_proj", + "model.layers.61.mlp.experts.44.down_proj", + "model.layers.61.mlp.experts.44.gate_proj", + "model.layers.61.mlp.experts.44.up_proj", + "model.layers.61.mlp.experts.45.down_proj", + "model.layers.61.mlp.experts.45.gate_proj", + "model.layers.61.mlp.experts.45.up_proj", + "model.layers.61.mlp.experts.46.down_proj", + "model.layers.61.mlp.experts.46.gate_proj", + "model.layers.61.mlp.experts.46.up_proj", + "model.layers.61.mlp.experts.47.down_proj", + "model.layers.61.mlp.experts.47.gate_proj", + "model.layers.61.mlp.experts.47.up_proj", + "model.layers.61.mlp.experts.48.down_proj", + "model.layers.61.mlp.experts.48.gate_proj", + "model.layers.61.mlp.experts.48.up_proj", + "model.layers.61.mlp.experts.49.down_proj", + "model.layers.61.mlp.experts.49.gate_proj", + "model.layers.61.mlp.experts.49.up_proj", + "model.layers.61.mlp.experts.5.down_proj", + "model.layers.61.mlp.experts.5.gate_proj", + "model.layers.61.mlp.experts.5.up_proj", + "model.layers.61.mlp.experts.50.down_proj", + "model.layers.61.mlp.experts.50.gate_proj", + "model.layers.61.mlp.experts.50.up_proj", + "model.layers.61.mlp.experts.51.down_proj", + "model.layers.61.mlp.experts.51.gate_proj", + "model.layers.61.mlp.experts.51.up_proj", + "model.layers.61.mlp.experts.52.down_proj", + "model.layers.61.mlp.experts.52.gate_proj", + "model.layers.61.mlp.experts.52.up_proj", + "model.layers.61.mlp.experts.53.down_proj", + "model.layers.61.mlp.experts.53.gate_proj", + "model.layers.61.mlp.experts.53.up_proj", + "model.layers.61.mlp.experts.54.down_proj", + "model.layers.61.mlp.experts.54.gate_proj", + "model.layers.61.mlp.experts.54.up_proj", + "model.layers.61.mlp.experts.55.down_proj", + "model.layers.61.mlp.experts.55.gate_proj", + "model.layers.61.mlp.experts.55.up_proj", + "model.layers.61.mlp.experts.56.down_proj", + "model.layers.61.mlp.experts.56.gate_proj", + "model.layers.61.mlp.experts.56.up_proj", + "model.layers.61.mlp.experts.57.down_proj", + "model.layers.61.mlp.experts.57.gate_proj", + "model.layers.61.mlp.experts.57.up_proj", + "model.layers.61.mlp.experts.58.down_proj", + "model.layers.61.mlp.experts.58.gate_proj", + "model.layers.61.mlp.experts.58.up_proj", + "model.layers.61.mlp.experts.59.down_proj", + "model.layers.61.mlp.experts.59.gate_proj", + "model.layers.61.mlp.experts.59.up_proj", + "model.layers.61.mlp.experts.6.down_proj", + "model.layers.61.mlp.experts.6.gate_proj", + "model.layers.61.mlp.experts.6.up_proj", + "model.layers.61.mlp.experts.60.down_proj", + "model.layers.61.mlp.experts.60.gate_proj", + "model.layers.61.mlp.experts.60.up_proj", + "model.layers.61.mlp.experts.61.down_proj", + "model.layers.61.mlp.experts.61.gate_proj", + "model.layers.61.mlp.experts.61.up_proj", + "model.layers.61.mlp.experts.62.down_proj", + "model.layers.61.mlp.experts.62.gate_proj", + "model.layers.61.mlp.experts.62.up_proj", + "model.layers.61.mlp.experts.63.down_proj", + "model.layers.61.mlp.experts.63.gate_proj", + "model.layers.61.mlp.experts.63.up_proj", + "model.layers.61.mlp.experts.64.down_proj", + "model.layers.61.mlp.experts.64.gate_proj", + "model.layers.61.mlp.experts.64.up_proj", + "model.layers.61.mlp.experts.65.down_proj", + "model.layers.61.mlp.experts.65.gate_proj", + "model.layers.61.mlp.experts.65.up_proj", + "model.layers.61.mlp.experts.66.down_proj", + "model.layers.61.mlp.experts.66.gate_proj", + "model.layers.61.mlp.experts.66.up_proj", + "model.layers.61.mlp.experts.67.down_proj", + "model.layers.61.mlp.experts.67.gate_proj", + "model.layers.61.mlp.experts.67.up_proj", + "model.layers.61.mlp.experts.68.down_proj", + "model.layers.61.mlp.experts.68.gate_proj", + "model.layers.61.mlp.experts.68.up_proj", + "model.layers.61.mlp.experts.69.down_proj", + "model.layers.61.mlp.experts.69.gate_proj", + "model.layers.61.mlp.experts.69.up_proj", + "model.layers.61.mlp.experts.7.down_proj", + "model.layers.61.mlp.experts.7.gate_proj", + "model.layers.61.mlp.experts.7.up_proj", + "model.layers.61.mlp.experts.70.down_proj", + "model.layers.61.mlp.experts.70.gate_proj", + "model.layers.61.mlp.experts.70.up_proj", + "model.layers.61.mlp.experts.71.down_proj", + "model.layers.61.mlp.experts.71.gate_proj", + "model.layers.61.mlp.experts.71.up_proj", + "model.layers.61.mlp.experts.72.down_proj", + "model.layers.61.mlp.experts.72.gate_proj", + "model.layers.61.mlp.experts.72.up_proj", + "model.layers.61.mlp.experts.73.down_proj", + "model.layers.61.mlp.experts.73.gate_proj", + "model.layers.61.mlp.experts.73.up_proj", + "model.layers.61.mlp.experts.74.down_proj", + "model.layers.61.mlp.experts.74.gate_proj", + "model.layers.61.mlp.experts.74.up_proj", + "model.layers.61.mlp.experts.75.down_proj", + "model.layers.61.mlp.experts.75.gate_proj", + "model.layers.61.mlp.experts.75.up_proj", + "model.layers.61.mlp.experts.76.down_proj", + "model.layers.61.mlp.experts.76.gate_proj", + "model.layers.61.mlp.experts.76.up_proj", + "model.layers.61.mlp.experts.77.down_proj", + "model.layers.61.mlp.experts.77.gate_proj", + "model.layers.61.mlp.experts.77.up_proj", + "model.layers.61.mlp.experts.78.down_proj", + "model.layers.61.mlp.experts.78.gate_proj", + "model.layers.61.mlp.experts.78.up_proj", + "model.layers.61.mlp.experts.79.down_proj", + "model.layers.61.mlp.experts.79.gate_proj", + "model.layers.61.mlp.experts.79.up_proj", + "model.layers.61.mlp.experts.8.down_proj", + "model.layers.61.mlp.experts.8.gate_proj", + "model.layers.61.mlp.experts.8.up_proj", + "model.layers.61.mlp.experts.80.down_proj", + "model.layers.61.mlp.experts.80.gate_proj", + "model.layers.61.mlp.experts.80.up_proj", + "model.layers.61.mlp.experts.81.down_proj", + "model.layers.61.mlp.experts.81.gate_proj", + "model.layers.61.mlp.experts.81.up_proj", + "model.layers.61.mlp.experts.82.down_proj", + "model.layers.61.mlp.experts.82.gate_proj", + "model.layers.61.mlp.experts.82.up_proj", + "model.layers.61.mlp.experts.83.down_proj", + "model.layers.61.mlp.experts.83.gate_proj", + "model.layers.61.mlp.experts.83.up_proj", + "model.layers.61.mlp.experts.84.down_proj", + "model.layers.61.mlp.experts.84.gate_proj", + "model.layers.61.mlp.experts.84.up_proj", + "model.layers.61.mlp.experts.85.down_proj", + "model.layers.61.mlp.experts.85.gate_proj", + "model.layers.61.mlp.experts.85.up_proj", + "model.layers.61.mlp.experts.86.down_proj", + "model.layers.61.mlp.experts.86.gate_proj", + "model.layers.61.mlp.experts.86.up_proj", + "model.layers.61.mlp.experts.87.down_proj", + "model.layers.61.mlp.experts.87.gate_proj", + "model.layers.61.mlp.experts.87.up_proj", + "model.layers.61.mlp.experts.88.down_proj", + "model.layers.61.mlp.experts.88.gate_proj", + "model.layers.61.mlp.experts.88.up_proj", + "model.layers.61.mlp.experts.89.down_proj", + "model.layers.61.mlp.experts.89.gate_proj", + "model.layers.61.mlp.experts.89.up_proj", + "model.layers.61.mlp.experts.9.down_proj", + "model.layers.61.mlp.experts.9.gate_proj", + "model.layers.61.mlp.experts.9.up_proj", + "model.layers.61.mlp.experts.90.down_proj", + "model.layers.61.mlp.experts.90.gate_proj", + "model.layers.61.mlp.experts.90.up_proj", + "model.layers.61.mlp.experts.91.down_proj", + "model.layers.61.mlp.experts.91.gate_proj", + "model.layers.61.mlp.experts.91.up_proj", + "model.layers.61.mlp.experts.92.down_proj", + "model.layers.61.mlp.experts.92.gate_proj", + "model.layers.61.mlp.experts.92.up_proj", + "model.layers.61.mlp.experts.93.down_proj", + "model.layers.61.mlp.experts.93.gate_proj", + "model.layers.61.mlp.experts.93.up_proj", + "model.layers.61.mlp.experts.94.down_proj", + "model.layers.61.mlp.experts.94.gate_proj", + "model.layers.61.mlp.experts.94.up_proj", + "model.layers.61.mlp.experts.95.down_proj", + "model.layers.61.mlp.experts.95.gate_proj", + "model.layers.61.mlp.experts.95.up_proj", + "model.layers.61.mlp.experts.96.down_proj", + "model.layers.61.mlp.experts.96.gate_proj", + "model.layers.61.mlp.experts.96.up_proj", + "model.layers.61.mlp.experts.97.down_proj", + "model.layers.61.mlp.experts.97.gate_proj", + "model.layers.61.mlp.experts.97.up_proj", + "model.layers.61.mlp.experts.98.down_proj", + "model.layers.61.mlp.experts.98.gate_proj", + "model.layers.61.mlp.experts.98.up_proj", + "model.layers.61.mlp.experts.99.down_proj", + "model.layers.61.mlp.experts.99.gate_proj", + "model.layers.61.mlp.experts.99.up_proj", + "model.layers.62.mlp.experts.0.down_proj", + "model.layers.62.mlp.experts.0.gate_proj", + "model.layers.62.mlp.experts.0.up_proj", + "model.layers.62.mlp.experts.1.down_proj", + "model.layers.62.mlp.experts.1.gate_proj", + "model.layers.62.mlp.experts.1.up_proj", + "model.layers.62.mlp.experts.10.down_proj", + "model.layers.62.mlp.experts.10.gate_proj", + "model.layers.62.mlp.experts.10.up_proj", + "model.layers.62.mlp.experts.100.down_proj", + "model.layers.62.mlp.experts.100.gate_proj", + "model.layers.62.mlp.experts.100.up_proj", + "model.layers.62.mlp.experts.101.down_proj", + "model.layers.62.mlp.experts.101.gate_proj", + "model.layers.62.mlp.experts.101.up_proj", + "model.layers.62.mlp.experts.102.down_proj", + "model.layers.62.mlp.experts.102.gate_proj", + "model.layers.62.mlp.experts.102.up_proj", + "model.layers.62.mlp.experts.103.down_proj", + "model.layers.62.mlp.experts.103.gate_proj", + "model.layers.62.mlp.experts.103.up_proj", + "model.layers.62.mlp.experts.104.down_proj", + "model.layers.62.mlp.experts.104.gate_proj", + "model.layers.62.mlp.experts.104.up_proj", + "model.layers.62.mlp.experts.105.down_proj", + "model.layers.62.mlp.experts.105.gate_proj", + "model.layers.62.mlp.experts.105.up_proj", + "model.layers.62.mlp.experts.106.down_proj", + "model.layers.62.mlp.experts.106.gate_proj", + "model.layers.62.mlp.experts.106.up_proj", + "model.layers.62.mlp.experts.107.down_proj", + "model.layers.62.mlp.experts.107.gate_proj", + "model.layers.62.mlp.experts.107.up_proj", + "model.layers.62.mlp.experts.108.down_proj", + "model.layers.62.mlp.experts.108.gate_proj", + "model.layers.62.mlp.experts.108.up_proj", + "model.layers.62.mlp.experts.109.down_proj", + "model.layers.62.mlp.experts.109.gate_proj", + "model.layers.62.mlp.experts.109.up_proj", + "model.layers.62.mlp.experts.11.down_proj", + "model.layers.62.mlp.experts.11.gate_proj", + "model.layers.62.mlp.experts.11.up_proj", + "model.layers.62.mlp.experts.110.down_proj", + "model.layers.62.mlp.experts.110.gate_proj", + "model.layers.62.mlp.experts.110.up_proj", + "model.layers.62.mlp.experts.111.down_proj", + "model.layers.62.mlp.experts.111.gate_proj", + "model.layers.62.mlp.experts.111.up_proj", + "model.layers.62.mlp.experts.112.down_proj", + "model.layers.62.mlp.experts.112.gate_proj", + "model.layers.62.mlp.experts.112.up_proj", + "model.layers.62.mlp.experts.113.down_proj", + "model.layers.62.mlp.experts.113.gate_proj", + "model.layers.62.mlp.experts.113.up_proj", + "model.layers.62.mlp.experts.114.down_proj", + "model.layers.62.mlp.experts.114.gate_proj", + "model.layers.62.mlp.experts.114.up_proj", + "model.layers.62.mlp.experts.115.down_proj", + "model.layers.62.mlp.experts.115.gate_proj", + "model.layers.62.mlp.experts.115.up_proj", + "model.layers.62.mlp.experts.116.down_proj", + "model.layers.62.mlp.experts.116.gate_proj", + "model.layers.62.mlp.experts.116.up_proj", + "model.layers.62.mlp.experts.117.down_proj", + "model.layers.62.mlp.experts.117.gate_proj", + "model.layers.62.mlp.experts.117.up_proj", + "model.layers.62.mlp.experts.118.down_proj", + "model.layers.62.mlp.experts.118.gate_proj", + "model.layers.62.mlp.experts.118.up_proj", + "model.layers.62.mlp.experts.119.down_proj", + "model.layers.62.mlp.experts.119.gate_proj", + "model.layers.62.mlp.experts.119.up_proj", + "model.layers.62.mlp.experts.12.down_proj", + "model.layers.62.mlp.experts.12.gate_proj", + "model.layers.62.mlp.experts.12.up_proj", + "model.layers.62.mlp.experts.120.down_proj", + "model.layers.62.mlp.experts.120.gate_proj", + "model.layers.62.mlp.experts.120.up_proj", + "model.layers.62.mlp.experts.121.down_proj", + "model.layers.62.mlp.experts.121.gate_proj", + "model.layers.62.mlp.experts.121.up_proj", + "model.layers.62.mlp.experts.122.down_proj", + "model.layers.62.mlp.experts.122.gate_proj", + "model.layers.62.mlp.experts.122.up_proj", + "model.layers.62.mlp.experts.123.down_proj", + "model.layers.62.mlp.experts.123.gate_proj", + "model.layers.62.mlp.experts.123.up_proj", + "model.layers.62.mlp.experts.124.down_proj", + "model.layers.62.mlp.experts.124.gate_proj", + "model.layers.62.mlp.experts.124.up_proj", + "model.layers.62.mlp.experts.125.down_proj", + "model.layers.62.mlp.experts.125.gate_proj", + "model.layers.62.mlp.experts.125.up_proj", + "model.layers.62.mlp.experts.126.down_proj", + "model.layers.62.mlp.experts.126.gate_proj", + "model.layers.62.mlp.experts.126.up_proj", + "model.layers.62.mlp.experts.127.down_proj", + "model.layers.62.mlp.experts.127.gate_proj", + "model.layers.62.mlp.experts.127.up_proj", + "model.layers.62.mlp.experts.128.down_proj", + "model.layers.62.mlp.experts.128.gate_proj", + "model.layers.62.mlp.experts.128.up_proj", + "model.layers.62.mlp.experts.129.down_proj", + "model.layers.62.mlp.experts.129.gate_proj", + "model.layers.62.mlp.experts.129.up_proj", + "model.layers.62.mlp.experts.13.down_proj", + "model.layers.62.mlp.experts.13.gate_proj", + "model.layers.62.mlp.experts.13.up_proj", + "model.layers.62.mlp.experts.130.down_proj", + "model.layers.62.mlp.experts.130.gate_proj", + "model.layers.62.mlp.experts.130.up_proj", + "model.layers.62.mlp.experts.131.down_proj", + "model.layers.62.mlp.experts.131.gate_proj", + "model.layers.62.mlp.experts.131.up_proj", + "model.layers.62.mlp.experts.132.down_proj", + "model.layers.62.mlp.experts.132.gate_proj", + "model.layers.62.mlp.experts.132.up_proj", + "model.layers.62.mlp.experts.133.down_proj", + "model.layers.62.mlp.experts.133.gate_proj", + "model.layers.62.mlp.experts.133.up_proj", + "model.layers.62.mlp.experts.134.down_proj", + "model.layers.62.mlp.experts.134.gate_proj", + "model.layers.62.mlp.experts.134.up_proj", + "model.layers.62.mlp.experts.135.down_proj", + "model.layers.62.mlp.experts.135.gate_proj", + "model.layers.62.mlp.experts.135.up_proj", + "model.layers.62.mlp.experts.136.down_proj", + "model.layers.62.mlp.experts.136.gate_proj", + "model.layers.62.mlp.experts.136.up_proj", + "model.layers.62.mlp.experts.137.down_proj", + "model.layers.62.mlp.experts.137.gate_proj", + "model.layers.62.mlp.experts.137.up_proj", + "model.layers.62.mlp.experts.138.down_proj", + "model.layers.62.mlp.experts.138.gate_proj", + "model.layers.62.mlp.experts.138.up_proj", + "model.layers.62.mlp.experts.139.down_proj", + "model.layers.62.mlp.experts.139.gate_proj", + "model.layers.62.mlp.experts.139.up_proj", + "model.layers.62.mlp.experts.14.down_proj", + "model.layers.62.mlp.experts.14.gate_proj", + "model.layers.62.mlp.experts.14.up_proj", + "model.layers.62.mlp.experts.140.down_proj", + "model.layers.62.mlp.experts.140.gate_proj", + "model.layers.62.mlp.experts.140.up_proj", + "model.layers.62.mlp.experts.141.down_proj", + "model.layers.62.mlp.experts.141.gate_proj", + "model.layers.62.mlp.experts.141.up_proj", + "model.layers.62.mlp.experts.142.down_proj", + "model.layers.62.mlp.experts.142.gate_proj", + "model.layers.62.mlp.experts.142.up_proj", + "model.layers.62.mlp.experts.143.down_proj", + "model.layers.62.mlp.experts.143.gate_proj", + "model.layers.62.mlp.experts.143.up_proj", + "model.layers.62.mlp.experts.144.down_proj", + "model.layers.62.mlp.experts.144.gate_proj", + "model.layers.62.mlp.experts.144.up_proj", + "model.layers.62.mlp.experts.145.down_proj", + "model.layers.62.mlp.experts.145.gate_proj", + "model.layers.62.mlp.experts.145.up_proj", + "model.layers.62.mlp.experts.146.down_proj", + "model.layers.62.mlp.experts.146.gate_proj", + "model.layers.62.mlp.experts.146.up_proj", + "model.layers.62.mlp.experts.147.down_proj", + "model.layers.62.mlp.experts.147.gate_proj", + "model.layers.62.mlp.experts.147.up_proj", + "model.layers.62.mlp.experts.148.down_proj", + "model.layers.62.mlp.experts.148.gate_proj", + "model.layers.62.mlp.experts.148.up_proj", + "model.layers.62.mlp.experts.149.down_proj", + "model.layers.62.mlp.experts.149.gate_proj", + "model.layers.62.mlp.experts.149.up_proj", + "model.layers.62.mlp.experts.15.down_proj", + "model.layers.62.mlp.experts.15.gate_proj", + "model.layers.62.mlp.experts.15.up_proj", + "model.layers.62.mlp.experts.150.down_proj", + "model.layers.62.mlp.experts.150.gate_proj", + "model.layers.62.mlp.experts.150.up_proj", + "model.layers.62.mlp.experts.151.down_proj", + "model.layers.62.mlp.experts.151.gate_proj", + "model.layers.62.mlp.experts.151.up_proj", + "model.layers.62.mlp.experts.152.down_proj", + "model.layers.62.mlp.experts.152.gate_proj", + "model.layers.62.mlp.experts.152.up_proj", + "model.layers.62.mlp.experts.153.down_proj", + "model.layers.62.mlp.experts.153.gate_proj", + "model.layers.62.mlp.experts.153.up_proj", + "model.layers.62.mlp.experts.154.down_proj", + "model.layers.62.mlp.experts.154.gate_proj", + "model.layers.62.mlp.experts.154.up_proj", + "model.layers.62.mlp.experts.155.down_proj", + "model.layers.62.mlp.experts.155.gate_proj", + "model.layers.62.mlp.experts.155.up_proj", + "model.layers.62.mlp.experts.156.down_proj", + "model.layers.62.mlp.experts.156.gate_proj", + "model.layers.62.mlp.experts.156.up_proj", + "model.layers.62.mlp.experts.157.down_proj", + "model.layers.62.mlp.experts.157.gate_proj", + "model.layers.62.mlp.experts.157.up_proj", + "model.layers.62.mlp.experts.158.down_proj", + "model.layers.62.mlp.experts.158.gate_proj", + "model.layers.62.mlp.experts.158.up_proj", + "model.layers.62.mlp.experts.159.down_proj", + "model.layers.62.mlp.experts.159.gate_proj", + "model.layers.62.mlp.experts.159.up_proj", + "model.layers.62.mlp.experts.16.down_proj", + "model.layers.62.mlp.experts.16.gate_proj", + "model.layers.62.mlp.experts.16.up_proj", + "model.layers.62.mlp.experts.160.down_proj", + "model.layers.62.mlp.experts.160.gate_proj", + "model.layers.62.mlp.experts.160.up_proj", + "model.layers.62.mlp.experts.161.down_proj", + "model.layers.62.mlp.experts.161.gate_proj", + "model.layers.62.mlp.experts.161.up_proj", + "model.layers.62.mlp.experts.162.down_proj", + "model.layers.62.mlp.experts.162.gate_proj", + "model.layers.62.mlp.experts.162.up_proj", + "model.layers.62.mlp.experts.163.down_proj", + "model.layers.62.mlp.experts.163.gate_proj", + "model.layers.62.mlp.experts.163.up_proj", + "model.layers.62.mlp.experts.164.down_proj", + "model.layers.62.mlp.experts.164.gate_proj", + "model.layers.62.mlp.experts.164.up_proj", + "model.layers.62.mlp.experts.165.down_proj", + "model.layers.62.mlp.experts.165.gate_proj", + "model.layers.62.mlp.experts.165.up_proj", + "model.layers.62.mlp.experts.166.down_proj", + "model.layers.62.mlp.experts.166.gate_proj", + "model.layers.62.mlp.experts.166.up_proj", + "model.layers.62.mlp.experts.167.down_proj", + "model.layers.62.mlp.experts.167.gate_proj", + "model.layers.62.mlp.experts.167.up_proj", + "model.layers.62.mlp.experts.168.down_proj", + "model.layers.62.mlp.experts.168.gate_proj", + "model.layers.62.mlp.experts.168.up_proj", + "model.layers.62.mlp.experts.169.down_proj", + "model.layers.62.mlp.experts.169.gate_proj", + "model.layers.62.mlp.experts.169.up_proj", + "model.layers.62.mlp.experts.17.down_proj", + "model.layers.62.mlp.experts.17.gate_proj", + "model.layers.62.mlp.experts.17.up_proj", + "model.layers.62.mlp.experts.170.down_proj", + "model.layers.62.mlp.experts.170.gate_proj", + "model.layers.62.mlp.experts.170.up_proj", + "model.layers.62.mlp.experts.171.down_proj", + "model.layers.62.mlp.experts.171.gate_proj", + "model.layers.62.mlp.experts.171.up_proj", + "model.layers.62.mlp.experts.172.down_proj", + "model.layers.62.mlp.experts.172.gate_proj", + "model.layers.62.mlp.experts.172.up_proj", + "model.layers.62.mlp.experts.173.down_proj", + "model.layers.62.mlp.experts.173.gate_proj", + "model.layers.62.mlp.experts.173.up_proj", + "model.layers.62.mlp.experts.174.down_proj", + "model.layers.62.mlp.experts.174.gate_proj", + "model.layers.62.mlp.experts.174.up_proj", + "model.layers.62.mlp.experts.175.down_proj", + "model.layers.62.mlp.experts.175.gate_proj", + "model.layers.62.mlp.experts.175.up_proj", + "model.layers.62.mlp.experts.176.down_proj", + "model.layers.62.mlp.experts.176.gate_proj", + "model.layers.62.mlp.experts.176.up_proj", + "model.layers.62.mlp.experts.177.down_proj", + "model.layers.62.mlp.experts.177.gate_proj", + "model.layers.62.mlp.experts.177.up_proj", + "model.layers.62.mlp.experts.178.down_proj", + "model.layers.62.mlp.experts.178.gate_proj", + "model.layers.62.mlp.experts.178.up_proj", + "model.layers.62.mlp.experts.179.down_proj", + "model.layers.62.mlp.experts.179.gate_proj", + "model.layers.62.mlp.experts.179.up_proj", + "model.layers.62.mlp.experts.18.down_proj", + "model.layers.62.mlp.experts.18.gate_proj", + "model.layers.62.mlp.experts.18.up_proj", + "model.layers.62.mlp.experts.180.down_proj", + "model.layers.62.mlp.experts.180.gate_proj", + "model.layers.62.mlp.experts.180.up_proj", + "model.layers.62.mlp.experts.181.down_proj", + "model.layers.62.mlp.experts.181.gate_proj", + "model.layers.62.mlp.experts.181.up_proj", + "model.layers.62.mlp.experts.182.down_proj", + "model.layers.62.mlp.experts.182.gate_proj", + "model.layers.62.mlp.experts.182.up_proj", + "model.layers.62.mlp.experts.183.down_proj", + "model.layers.62.mlp.experts.183.gate_proj", + "model.layers.62.mlp.experts.183.up_proj", + "model.layers.62.mlp.experts.184.down_proj", + "model.layers.62.mlp.experts.184.gate_proj", + "model.layers.62.mlp.experts.184.up_proj", + "model.layers.62.mlp.experts.185.down_proj", + "model.layers.62.mlp.experts.185.gate_proj", + "model.layers.62.mlp.experts.185.up_proj", + "model.layers.62.mlp.experts.186.down_proj", + "model.layers.62.mlp.experts.186.gate_proj", + "model.layers.62.mlp.experts.186.up_proj", + "model.layers.62.mlp.experts.187.down_proj", + "model.layers.62.mlp.experts.187.gate_proj", + "model.layers.62.mlp.experts.187.up_proj", + "model.layers.62.mlp.experts.188.down_proj", + "model.layers.62.mlp.experts.188.gate_proj", + "model.layers.62.mlp.experts.188.up_proj", + "model.layers.62.mlp.experts.189.down_proj", + "model.layers.62.mlp.experts.189.gate_proj", + "model.layers.62.mlp.experts.189.up_proj", + "model.layers.62.mlp.experts.19.down_proj", + "model.layers.62.mlp.experts.19.gate_proj", + "model.layers.62.mlp.experts.19.up_proj", + "model.layers.62.mlp.experts.190.down_proj", + "model.layers.62.mlp.experts.190.gate_proj", + "model.layers.62.mlp.experts.190.up_proj", + "model.layers.62.mlp.experts.191.down_proj", + "model.layers.62.mlp.experts.191.gate_proj", + "model.layers.62.mlp.experts.191.up_proj", + "model.layers.62.mlp.experts.192.down_proj", + "model.layers.62.mlp.experts.192.gate_proj", + "model.layers.62.mlp.experts.192.up_proj", + "model.layers.62.mlp.experts.193.down_proj", + "model.layers.62.mlp.experts.193.gate_proj", + "model.layers.62.mlp.experts.193.up_proj", + "model.layers.62.mlp.experts.194.down_proj", + "model.layers.62.mlp.experts.194.gate_proj", + "model.layers.62.mlp.experts.194.up_proj", + "model.layers.62.mlp.experts.195.down_proj", + "model.layers.62.mlp.experts.195.gate_proj", + "model.layers.62.mlp.experts.195.up_proj", + "model.layers.62.mlp.experts.196.down_proj", + "model.layers.62.mlp.experts.196.gate_proj", + "model.layers.62.mlp.experts.196.up_proj", + "model.layers.62.mlp.experts.197.down_proj", + "model.layers.62.mlp.experts.197.gate_proj", + "model.layers.62.mlp.experts.197.up_proj", + "model.layers.62.mlp.experts.198.down_proj", + "model.layers.62.mlp.experts.198.gate_proj", + "model.layers.62.mlp.experts.198.up_proj", + "model.layers.62.mlp.experts.199.down_proj", + "model.layers.62.mlp.experts.199.gate_proj", + "model.layers.62.mlp.experts.199.up_proj", + "model.layers.62.mlp.experts.2.down_proj", + "model.layers.62.mlp.experts.2.gate_proj", + "model.layers.62.mlp.experts.2.up_proj", + "model.layers.62.mlp.experts.20.down_proj", + "model.layers.62.mlp.experts.20.gate_proj", + "model.layers.62.mlp.experts.20.up_proj", + "model.layers.62.mlp.experts.200.down_proj", + "model.layers.62.mlp.experts.200.gate_proj", + "model.layers.62.mlp.experts.200.up_proj", + "model.layers.62.mlp.experts.201.down_proj", + "model.layers.62.mlp.experts.201.gate_proj", + "model.layers.62.mlp.experts.201.up_proj", + "model.layers.62.mlp.experts.202.down_proj", + "model.layers.62.mlp.experts.202.gate_proj", + "model.layers.62.mlp.experts.202.up_proj", + "model.layers.62.mlp.experts.203.down_proj", + "model.layers.62.mlp.experts.203.gate_proj", + "model.layers.62.mlp.experts.203.up_proj", + "model.layers.62.mlp.experts.204.down_proj", + "model.layers.62.mlp.experts.204.gate_proj", + "model.layers.62.mlp.experts.204.up_proj", + "model.layers.62.mlp.experts.205.down_proj", + "model.layers.62.mlp.experts.205.gate_proj", + "model.layers.62.mlp.experts.205.up_proj", + "model.layers.62.mlp.experts.206.down_proj", + "model.layers.62.mlp.experts.206.gate_proj", + "model.layers.62.mlp.experts.206.up_proj", + "model.layers.62.mlp.experts.207.down_proj", + "model.layers.62.mlp.experts.207.gate_proj", + "model.layers.62.mlp.experts.207.up_proj", + "model.layers.62.mlp.experts.208.down_proj", + "model.layers.62.mlp.experts.208.gate_proj", + "model.layers.62.mlp.experts.208.up_proj", + "model.layers.62.mlp.experts.209.down_proj", + "model.layers.62.mlp.experts.209.gate_proj", + "model.layers.62.mlp.experts.209.up_proj", + "model.layers.62.mlp.experts.21.down_proj", + "model.layers.62.mlp.experts.21.gate_proj", + "model.layers.62.mlp.experts.21.up_proj", + "model.layers.62.mlp.experts.210.down_proj", + "model.layers.62.mlp.experts.210.gate_proj", + "model.layers.62.mlp.experts.210.up_proj", + "model.layers.62.mlp.experts.211.down_proj", + "model.layers.62.mlp.experts.211.gate_proj", + "model.layers.62.mlp.experts.211.up_proj", + "model.layers.62.mlp.experts.212.down_proj", + "model.layers.62.mlp.experts.212.gate_proj", + "model.layers.62.mlp.experts.212.up_proj", + "model.layers.62.mlp.experts.213.down_proj", + "model.layers.62.mlp.experts.213.gate_proj", + "model.layers.62.mlp.experts.213.up_proj", + "model.layers.62.mlp.experts.214.down_proj", + "model.layers.62.mlp.experts.214.gate_proj", + "model.layers.62.mlp.experts.214.up_proj", + "model.layers.62.mlp.experts.215.down_proj", + "model.layers.62.mlp.experts.215.gate_proj", + "model.layers.62.mlp.experts.215.up_proj", + "model.layers.62.mlp.experts.216.down_proj", + "model.layers.62.mlp.experts.216.gate_proj", + "model.layers.62.mlp.experts.216.up_proj", + "model.layers.62.mlp.experts.217.down_proj", + "model.layers.62.mlp.experts.217.gate_proj", + "model.layers.62.mlp.experts.217.up_proj", + "model.layers.62.mlp.experts.218.down_proj", + "model.layers.62.mlp.experts.218.gate_proj", + "model.layers.62.mlp.experts.218.up_proj", + "model.layers.62.mlp.experts.219.down_proj", + "model.layers.62.mlp.experts.219.gate_proj", + "model.layers.62.mlp.experts.219.up_proj", + "model.layers.62.mlp.experts.22.down_proj", + "model.layers.62.mlp.experts.22.gate_proj", + "model.layers.62.mlp.experts.22.up_proj", + "model.layers.62.mlp.experts.220.down_proj", + "model.layers.62.mlp.experts.220.gate_proj", + "model.layers.62.mlp.experts.220.up_proj", + "model.layers.62.mlp.experts.221.down_proj", + "model.layers.62.mlp.experts.221.gate_proj", + "model.layers.62.mlp.experts.221.up_proj", + "model.layers.62.mlp.experts.222.down_proj", + "model.layers.62.mlp.experts.222.gate_proj", + "model.layers.62.mlp.experts.222.up_proj", + "model.layers.62.mlp.experts.223.down_proj", + "model.layers.62.mlp.experts.223.gate_proj", + "model.layers.62.mlp.experts.223.up_proj", + "model.layers.62.mlp.experts.224.down_proj", + "model.layers.62.mlp.experts.224.gate_proj", + "model.layers.62.mlp.experts.224.up_proj", + "model.layers.62.mlp.experts.225.down_proj", + "model.layers.62.mlp.experts.225.gate_proj", + "model.layers.62.mlp.experts.225.up_proj", + "model.layers.62.mlp.experts.226.down_proj", + "model.layers.62.mlp.experts.226.gate_proj", + "model.layers.62.mlp.experts.226.up_proj", + "model.layers.62.mlp.experts.227.down_proj", + "model.layers.62.mlp.experts.227.gate_proj", + "model.layers.62.mlp.experts.227.up_proj", + "model.layers.62.mlp.experts.228.down_proj", + "model.layers.62.mlp.experts.228.gate_proj", + "model.layers.62.mlp.experts.228.up_proj", + "model.layers.62.mlp.experts.229.down_proj", + "model.layers.62.mlp.experts.229.gate_proj", + "model.layers.62.mlp.experts.229.up_proj", + "model.layers.62.mlp.experts.23.down_proj", + "model.layers.62.mlp.experts.23.gate_proj", + "model.layers.62.mlp.experts.23.up_proj", + "model.layers.62.mlp.experts.230.down_proj", + "model.layers.62.mlp.experts.230.gate_proj", + "model.layers.62.mlp.experts.230.up_proj", + "model.layers.62.mlp.experts.231.down_proj", + "model.layers.62.mlp.experts.231.gate_proj", + "model.layers.62.mlp.experts.231.up_proj", + "model.layers.62.mlp.experts.232.down_proj", + "model.layers.62.mlp.experts.232.gate_proj", + "model.layers.62.mlp.experts.232.up_proj", + "model.layers.62.mlp.experts.233.down_proj", + "model.layers.62.mlp.experts.233.gate_proj", + "model.layers.62.mlp.experts.233.up_proj", + "model.layers.62.mlp.experts.234.down_proj", + "model.layers.62.mlp.experts.234.gate_proj", + "model.layers.62.mlp.experts.234.up_proj", + "model.layers.62.mlp.experts.235.down_proj", + "model.layers.62.mlp.experts.235.gate_proj", + "model.layers.62.mlp.experts.235.up_proj", + "model.layers.62.mlp.experts.236.down_proj", + "model.layers.62.mlp.experts.236.gate_proj", + "model.layers.62.mlp.experts.236.up_proj", + "model.layers.62.mlp.experts.237.down_proj", + "model.layers.62.mlp.experts.237.gate_proj", + "model.layers.62.mlp.experts.237.up_proj", + "model.layers.62.mlp.experts.238.down_proj", + "model.layers.62.mlp.experts.238.gate_proj", + "model.layers.62.mlp.experts.238.up_proj", + "model.layers.62.mlp.experts.239.down_proj", + "model.layers.62.mlp.experts.239.gate_proj", + "model.layers.62.mlp.experts.239.up_proj", + "model.layers.62.mlp.experts.24.down_proj", + "model.layers.62.mlp.experts.24.gate_proj", + "model.layers.62.mlp.experts.24.up_proj", + "model.layers.62.mlp.experts.240.down_proj", + "model.layers.62.mlp.experts.240.gate_proj", + "model.layers.62.mlp.experts.240.up_proj", + "model.layers.62.mlp.experts.241.down_proj", + "model.layers.62.mlp.experts.241.gate_proj", + "model.layers.62.mlp.experts.241.up_proj", + "model.layers.62.mlp.experts.242.down_proj", + "model.layers.62.mlp.experts.242.gate_proj", + "model.layers.62.mlp.experts.242.up_proj", + "model.layers.62.mlp.experts.243.down_proj", + "model.layers.62.mlp.experts.243.gate_proj", + "model.layers.62.mlp.experts.243.up_proj", + "model.layers.62.mlp.experts.244.down_proj", + "model.layers.62.mlp.experts.244.gate_proj", + "model.layers.62.mlp.experts.244.up_proj", + "model.layers.62.mlp.experts.245.down_proj", + "model.layers.62.mlp.experts.245.gate_proj", + "model.layers.62.mlp.experts.245.up_proj", + "model.layers.62.mlp.experts.246.down_proj", + "model.layers.62.mlp.experts.246.gate_proj", + "model.layers.62.mlp.experts.246.up_proj", + "model.layers.62.mlp.experts.247.down_proj", + "model.layers.62.mlp.experts.247.gate_proj", + "model.layers.62.mlp.experts.247.up_proj", + "model.layers.62.mlp.experts.248.down_proj", + "model.layers.62.mlp.experts.248.gate_proj", + "model.layers.62.mlp.experts.248.up_proj", + "model.layers.62.mlp.experts.249.down_proj", + "model.layers.62.mlp.experts.249.gate_proj", + "model.layers.62.mlp.experts.249.up_proj", + "model.layers.62.mlp.experts.25.down_proj", + "model.layers.62.mlp.experts.25.gate_proj", + "model.layers.62.mlp.experts.25.up_proj", + "model.layers.62.mlp.experts.250.down_proj", + "model.layers.62.mlp.experts.250.gate_proj", + "model.layers.62.mlp.experts.250.up_proj", + "model.layers.62.mlp.experts.251.down_proj", + "model.layers.62.mlp.experts.251.gate_proj", + "model.layers.62.mlp.experts.251.up_proj", + "model.layers.62.mlp.experts.252.down_proj", + "model.layers.62.mlp.experts.252.gate_proj", + "model.layers.62.mlp.experts.252.up_proj", + "model.layers.62.mlp.experts.253.down_proj", + "model.layers.62.mlp.experts.253.gate_proj", + "model.layers.62.mlp.experts.253.up_proj", + "model.layers.62.mlp.experts.254.down_proj", + "model.layers.62.mlp.experts.254.gate_proj", + "model.layers.62.mlp.experts.254.up_proj", + "model.layers.62.mlp.experts.255.down_proj", + "model.layers.62.mlp.experts.255.gate_proj", + "model.layers.62.mlp.experts.255.up_proj", + "model.layers.62.mlp.experts.26.down_proj", + "model.layers.62.mlp.experts.26.gate_proj", + "model.layers.62.mlp.experts.26.up_proj", + "model.layers.62.mlp.experts.27.down_proj", + "model.layers.62.mlp.experts.27.gate_proj", + "model.layers.62.mlp.experts.27.up_proj", + "model.layers.62.mlp.experts.28.down_proj", + "model.layers.62.mlp.experts.28.gate_proj", + "model.layers.62.mlp.experts.28.up_proj", + "model.layers.62.mlp.experts.29.down_proj", + "model.layers.62.mlp.experts.29.gate_proj", + "model.layers.62.mlp.experts.29.up_proj", + "model.layers.62.mlp.experts.3.down_proj", + "model.layers.62.mlp.experts.3.gate_proj", + "model.layers.62.mlp.experts.3.up_proj", + "model.layers.62.mlp.experts.30.down_proj", + "model.layers.62.mlp.experts.30.gate_proj", + "model.layers.62.mlp.experts.30.up_proj", + "model.layers.62.mlp.experts.31.down_proj", + "model.layers.62.mlp.experts.31.gate_proj", + "model.layers.62.mlp.experts.31.up_proj", + "model.layers.62.mlp.experts.32.down_proj", + "model.layers.62.mlp.experts.32.gate_proj", + "model.layers.62.mlp.experts.32.up_proj", + "model.layers.62.mlp.experts.33.down_proj", + "model.layers.62.mlp.experts.33.gate_proj", + "model.layers.62.mlp.experts.33.up_proj", + "model.layers.62.mlp.experts.34.down_proj", + "model.layers.62.mlp.experts.34.gate_proj", + "model.layers.62.mlp.experts.34.up_proj", + "model.layers.62.mlp.experts.35.down_proj", + "model.layers.62.mlp.experts.35.gate_proj", + "model.layers.62.mlp.experts.35.up_proj", + "model.layers.62.mlp.experts.36.down_proj", + "model.layers.62.mlp.experts.36.gate_proj", + "model.layers.62.mlp.experts.36.up_proj", + "model.layers.62.mlp.experts.37.down_proj", + "model.layers.62.mlp.experts.37.gate_proj", + "model.layers.62.mlp.experts.37.up_proj", + "model.layers.62.mlp.experts.38.down_proj", + "model.layers.62.mlp.experts.38.gate_proj", + "model.layers.62.mlp.experts.38.up_proj", + "model.layers.62.mlp.experts.39.down_proj", + "model.layers.62.mlp.experts.39.gate_proj", + "model.layers.62.mlp.experts.39.up_proj", + "model.layers.62.mlp.experts.4.down_proj", + "model.layers.62.mlp.experts.4.gate_proj", + "model.layers.62.mlp.experts.4.up_proj", + "model.layers.62.mlp.experts.40.down_proj", + "model.layers.62.mlp.experts.40.gate_proj", + "model.layers.62.mlp.experts.40.up_proj", + "model.layers.62.mlp.experts.41.down_proj", + "model.layers.62.mlp.experts.41.gate_proj", + "model.layers.62.mlp.experts.41.up_proj", + "model.layers.62.mlp.experts.42.down_proj", + "model.layers.62.mlp.experts.42.gate_proj", + "model.layers.62.mlp.experts.42.up_proj", + "model.layers.62.mlp.experts.43.down_proj", + "model.layers.62.mlp.experts.43.gate_proj", + "model.layers.62.mlp.experts.43.up_proj", + "model.layers.62.mlp.experts.44.down_proj", + "model.layers.62.mlp.experts.44.gate_proj", + "model.layers.62.mlp.experts.44.up_proj", + "model.layers.62.mlp.experts.45.down_proj", + "model.layers.62.mlp.experts.45.gate_proj", + "model.layers.62.mlp.experts.45.up_proj", + "model.layers.62.mlp.experts.46.down_proj", + "model.layers.62.mlp.experts.46.gate_proj", + "model.layers.62.mlp.experts.46.up_proj", + "model.layers.62.mlp.experts.47.down_proj", + "model.layers.62.mlp.experts.47.gate_proj", + "model.layers.62.mlp.experts.47.up_proj", + "model.layers.62.mlp.experts.48.down_proj", + "model.layers.62.mlp.experts.48.gate_proj", + "model.layers.62.mlp.experts.48.up_proj", + "model.layers.62.mlp.experts.49.down_proj", + "model.layers.62.mlp.experts.49.gate_proj", + "model.layers.62.mlp.experts.49.up_proj", + "model.layers.62.mlp.experts.5.down_proj", + "model.layers.62.mlp.experts.5.gate_proj", + "model.layers.62.mlp.experts.5.up_proj", + "model.layers.62.mlp.experts.50.down_proj", + "model.layers.62.mlp.experts.50.gate_proj", + "model.layers.62.mlp.experts.50.up_proj", + "model.layers.62.mlp.experts.51.down_proj", + "model.layers.62.mlp.experts.51.gate_proj", + "model.layers.62.mlp.experts.51.up_proj", + "model.layers.62.mlp.experts.52.down_proj", + "model.layers.62.mlp.experts.52.gate_proj", + "model.layers.62.mlp.experts.52.up_proj", + "model.layers.62.mlp.experts.53.down_proj", + "model.layers.62.mlp.experts.53.gate_proj", + "model.layers.62.mlp.experts.53.up_proj", + "model.layers.62.mlp.experts.54.down_proj", + "model.layers.62.mlp.experts.54.gate_proj", + "model.layers.62.mlp.experts.54.up_proj", + "model.layers.62.mlp.experts.55.down_proj", + "model.layers.62.mlp.experts.55.gate_proj", + "model.layers.62.mlp.experts.55.up_proj", + "model.layers.62.mlp.experts.56.down_proj", + "model.layers.62.mlp.experts.56.gate_proj", + "model.layers.62.mlp.experts.56.up_proj", + "model.layers.62.mlp.experts.57.down_proj", + "model.layers.62.mlp.experts.57.gate_proj", + "model.layers.62.mlp.experts.57.up_proj", + "model.layers.62.mlp.experts.58.down_proj", + "model.layers.62.mlp.experts.58.gate_proj", + "model.layers.62.mlp.experts.58.up_proj", + "model.layers.62.mlp.experts.59.down_proj", + "model.layers.62.mlp.experts.59.gate_proj", + "model.layers.62.mlp.experts.59.up_proj", + "model.layers.62.mlp.experts.6.down_proj", + "model.layers.62.mlp.experts.6.gate_proj", + "model.layers.62.mlp.experts.6.up_proj", + "model.layers.62.mlp.experts.60.down_proj", + "model.layers.62.mlp.experts.60.gate_proj", + "model.layers.62.mlp.experts.60.up_proj", + "model.layers.62.mlp.experts.61.down_proj", + "model.layers.62.mlp.experts.61.gate_proj", + "model.layers.62.mlp.experts.61.up_proj", + "model.layers.62.mlp.experts.62.down_proj", + "model.layers.62.mlp.experts.62.gate_proj", + "model.layers.62.mlp.experts.62.up_proj", + "model.layers.62.mlp.experts.63.down_proj", + "model.layers.62.mlp.experts.63.gate_proj", + "model.layers.62.mlp.experts.63.up_proj", + "model.layers.62.mlp.experts.64.down_proj", + "model.layers.62.mlp.experts.64.gate_proj", + "model.layers.62.mlp.experts.64.up_proj", + "model.layers.62.mlp.experts.65.down_proj", + "model.layers.62.mlp.experts.65.gate_proj", + "model.layers.62.mlp.experts.65.up_proj", + "model.layers.62.mlp.experts.66.down_proj", + "model.layers.62.mlp.experts.66.gate_proj", + "model.layers.62.mlp.experts.66.up_proj", + "model.layers.62.mlp.experts.67.down_proj", + "model.layers.62.mlp.experts.67.gate_proj", + "model.layers.62.mlp.experts.67.up_proj", + "model.layers.62.mlp.experts.68.down_proj", + "model.layers.62.mlp.experts.68.gate_proj", + "model.layers.62.mlp.experts.68.up_proj", + "model.layers.62.mlp.experts.69.down_proj", + "model.layers.62.mlp.experts.69.gate_proj", + "model.layers.62.mlp.experts.69.up_proj", + "model.layers.62.mlp.experts.7.down_proj", + "model.layers.62.mlp.experts.7.gate_proj", + "model.layers.62.mlp.experts.7.up_proj", + "model.layers.62.mlp.experts.70.down_proj", + "model.layers.62.mlp.experts.70.gate_proj", + "model.layers.62.mlp.experts.70.up_proj", + "model.layers.62.mlp.experts.71.down_proj", + "model.layers.62.mlp.experts.71.gate_proj", + "model.layers.62.mlp.experts.71.up_proj", + "model.layers.62.mlp.experts.72.down_proj", + "model.layers.62.mlp.experts.72.gate_proj", + "model.layers.62.mlp.experts.72.up_proj", + "model.layers.62.mlp.experts.73.down_proj", + "model.layers.62.mlp.experts.73.gate_proj", + "model.layers.62.mlp.experts.73.up_proj", + "model.layers.62.mlp.experts.74.down_proj", + "model.layers.62.mlp.experts.74.gate_proj", + "model.layers.62.mlp.experts.74.up_proj", + "model.layers.62.mlp.experts.75.down_proj", + "model.layers.62.mlp.experts.75.gate_proj", + "model.layers.62.mlp.experts.75.up_proj", + "model.layers.62.mlp.experts.76.down_proj", + "model.layers.62.mlp.experts.76.gate_proj", + "model.layers.62.mlp.experts.76.up_proj", + "model.layers.62.mlp.experts.77.down_proj", + "model.layers.62.mlp.experts.77.gate_proj", + "model.layers.62.mlp.experts.77.up_proj", + "model.layers.62.mlp.experts.78.down_proj", + "model.layers.62.mlp.experts.78.gate_proj", + "model.layers.62.mlp.experts.78.up_proj", + "model.layers.62.mlp.experts.79.down_proj", + "model.layers.62.mlp.experts.79.gate_proj", + "model.layers.62.mlp.experts.79.up_proj", + "model.layers.62.mlp.experts.8.down_proj", + "model.layers.62.mlp.experts.8.gate_proj", + "model.layers.62.mlp.experts.8.up_proj", + "model.layers.62.mlp.experts.80.down_proj", + "model.layers.62.mlp.experts.80.gate_proj", + "model.layers.62.mlp.experts.80.up_proj", + "model.layers.62.mlp.experts.81.down_proj", + "model.layers.62.mlp.experts.81.gate_proj", + "model.layers.62.mlp.experts.81.up_proj", + "model.layers.62.mlp.experts.82.down_proj", + "model.layers.62.mlp.experts.82.gate_proj", + "model.layers.62.mlp.experts.82.up_proj", + "model.layers.62.mlp.experts.83.down_proj", + "model.layers.62.mlp.experts.83.gate_proj", + "model.layers.62.mlp.experts.83.up_proj", + "model.layers.62.mlp.experts.84.down_proj", + "model.layers.62.mlp.experts.84.gate_proj", + "model.layers.62.mlp.experts.84.up_proj", + "model.layers.62.mlp.experts.85.down_proj", + "model.layers.62.mlp.experts.85.gate_proj", + "model.layers.62.mlp.experts.85.up_proj", + "model.layers.62.mlp.experts.86.down_proj", + "model.layers.62.mlp.experts.86.gate_proj", + "model.layers.62.mlp.experts.86.up_proj", + "model.layers.62.mlp.experts.87.down_proj", + "model.layers.62.mlp.experts.87.gate_proj", + "model.layers.62.mlp.experts.87.up_proj", + "model.layers.62.mlp.experts.88.down_proj", + "model.layers.62.mlp.experts.88.gate_proj", + "model.layers.62.mlp.experts.88.up_proj", + "model.layers.62.mlp.experts.89.down_proj", + "model.layers.62.mlp.experts.89.gate_proj", + "model.layers.62.mlp.experts.89.up_proj", + "model.layers.62.mlp.experts.9.down_proj", + "model.layers.62.mlp.experts.9.gate_proj", + "model.layers.62.mlp.experts.9.up_proj", + "model.layers.62.mlp.experts.90.down_proj", + "model.layers.62.mlp.experts.90.gate_proj", + "model.layers.62.mlp.experts.90.up_proj", + "model.layers.62.mlp.experts.91.down_proj", + "model.layers.62.mlp.experts.91.gate_proj", + "model.layers.62.mlp.experts.91.up_proj", + "model.layers.62.mlp.experts.92.down_proj", + "model.layers.62.mlp.experts.92.gate_proj", + "model.layers.62.mlp.experts.92.up_proj", + "model.layers.62.mlp.experts.93.down_proj", + "model.layers.62.mlp.experts.93.gate_proj", + "model.layers.62.mlp.experts.93.up_proj", + "model.layers.62.mlp.experts.94.down_proj", + "model.layers.62.mlp.experts.94.gate_proj", + "model.layers.62.mlp.experts.94.up_proj", + "model.layers.62.mlp.experts.95.down_proj", + "model.layers.62.mlp.experts.95.gate_proj", + "model.layers.62.mlp.experts.95.up_proj", + "model.layers.62.mlp.experts.96.down_proj", + "model.layers.62.mlp.experts.96.gate_proj", + "model.layers.62.mlp.experts.96.up_proj", + "model.layers.62.mlp.experts.97.down_proj", + "model.layers.62.mlp.experts.97.gate_proj", + "model.layers.62.mlp.experts.97.up_proj", + "model.layers.62.mlp.experts.98.down_proj", + "model.layers.62.mlp.experts.98.gate_proj", + "model.layers.62.mlp.experts.98.up_proj", + "model.layers.62.mlp.experts.99.down_proj", + "model.layers.62.mlp.experts.99.gate_proj", + "model.layers.62.mlp.experts.99.up_proj", + "model.layers.63.mlp.experts.0.down_proj", + "model.layers.63.mlp.experts.0.gate_proj", + "model.layers.63.mlp.experts.0.up_proj", + "model.layers.63.mlp.experts.1.down_proj", + "model.layers.63.mlp.experts.1.gate_proj", + "model.layers.63.mlp.experts.1.up_proj", + "model.layers.63.mlp.experts.10.down_proj", + "model.layers.63.mlp.experts.10.gate_proj", + "model.layers.63.mlp.experts.10.up_proj", + "model.layers.63.mlp.experts.100.down_proj", + "model.layers.63.mlp.experts.100.gate_proj", + "model.layers.63.mlp.experts.100.up_proj", + "model.layers.63.mlp.experts.101.down_proj", + "model.layers.63.mlp.experts.101.gate_proj", + "model.layers.63.mlp.experts.101.up_proj", + "model.layers.63.mlp.experts.102.down_proj", + "model.layers.63.mlp.experts.102.gate_proj", + "model.layers.63.mlp.experts.102.up_proj", + "model.layers.63.mlp.experts.103.down_proj", + "model.layers.63.mlp.experts.103.gate_proj", + "model.layers.63.mlp.experts.103.up_proj", + "model.layers.63.mlp.experts.104.down_proj", + "model.layers.63.mlp.experts.104.gate_proj", + "model.layers.63.mlp.experts.104.up_proj", + "model.layers.63.mlp.experts.105.down_proj", + "model.layers.63.mlp.experts.105.gate_proj", + "model.layers.63.mlp.experts.105.up_proj", + "model.layers.63.mlp.experts.106.down_proj", + "model.layers.63.mlp.experts.106.gate_proj", + "model.layers.63.mlp.experts.106.up_proj", + "model.layers.63.mlp.experts.107.down_proj", + "model.layers.63.mlp.experts.107.gate_proj", + "model.layers.63.mlp.experts.107.up_proj", + "model.layers.63.mlp.experts.108.down_proj", + "model.layers.63.mlp.experts.108.gate_proj", + "model.layers.63.mlp.experts.108.up_proj", + "model.layers.63.mlp.experts.109.down_proj", + "model.layers.63.mlp.experts.109.gate_proj", + "model.layers.63.mlp.experts.109.up_proj", + "model.layers.63.mlp.experts.11.down_proj", + "model.layers.63.mlp.experts.11.gate_proj", + "model.layers.63.mlp.experts.11.up_proj", + "model.layers.63.mlp.experts.110.down_proj", + "model.layers.63.mlp.experts.110.gate_proj", + "model.layers.63.mlp.experts.110.up_proj", + "model.layers.63.mlp.experts.111.down_proj", + "model.layers.63.mlp.experts.111.gate_proj", + "model.layers.63.mlp.experts.111.up_proj", + "model.layers.63.mlp.experts.112.down_proj", + "model.layers.63.mlp.experts.112.gate_proj", + "model.layers.63.mlp.experts.112.up_proj", + "model.layers.63.mlp.experts.113.down_proj", + "model.layers.63.mlp.experts.113.gate_proj", + "model.layers.63.mlp.experts.113.up_proj", + "model.layers.63.mlp.experts.114.down_proj", + "model.layers.63.mlp.experts.114.gate_proj", + "model.layers.63.mlp.experts.114.up_proj", + "model.layers.63.mlp.experts.115.down_proj", + "model.layers.63.mlp.experts.115.gate_proj", + "model.layers.63.mlp.experts.115.up_proj", + "model.layers.63.mlp.experts.116.down_proj", + "model.layers.63.mlp.experts.116.gate_proj", + "model.layers.63.mlp.experts.116.up_proj", + "model.layers.63.mlp.experts.117.down_proj", + "model.layers.63.mlp.experts.117.gate_proj", + "model.layers.63.mlp.experts.117.up_proj", + "model.layers.63.mlp.experts.118.down_proj", + "model.layers.63.mlp.experts.118.gate_proj", + "model.layers.63.mlp.experts.118.up_proj", + "model.layers.63.mlp.experts.119.down_proj", + "model.layers.63.mlp.experts.119.gate_proj", + "model.layers.63.mlp.experts.119.up_proj", + "model.layers.63.mlp.experts.12.down_proj", + "model.layers.63.mlp.experts.12.gate_proj", + "model.layers.63.mlp.experts.12.up_proj", + "model.layers.63.mlp.experts.120.down_proj", + "model.layers.63.mlp.experts.120.gate_proj", + "model.layers.63.mlp.experts.120.up_proj", + "model.layers.63.mlp.experts.121.down_proj", + "model.layers.63.mlp.experts.121.gate_proj", + "model.layers.63.mlp.experts.121.up_proj", + "model.layers.63.mlp.experts.122.down_proj", + "model.layers.63.mlp.experts.122.gate_proj", + "model.layers.63.mlp.experts.122.up_proj", + "model.layers.63.mlp.experts.123.down_proj", + "model.layers.63.mlp.experts.123.gate_proj", + "model.layers.63.mlp.experts.123.up_proj", + "model.layers.63.mlp.experts.124.down_proj", + "model.layers.63.mlp.experts.124.gate_proj", + "model.layers.63.mlp.experts.124.up_proj", + "model.layers.63.mlp.experts.125.down_proj", + "model.layers.63.mlp.experts.125.gate_proj", + "model.layers.63.mlp.experts.125.up_proj", + "model.layers.63.mlp.experts.126.down_proj", + "model.layers.63.mlp.experts.126.gate_proj", + "model.layers.63.mlp.experts.126.up_proj", + "model.layers.63.mlp.experts.127.down_proj", + "model.layers.63.mlp.experts.127.gate_proj", + "model.layers.63.mlp.experts.127.up_proj", + "model.layers.63.mlp.experts.128.down_proj", + "model.layers.63.mlp.experts.128.gate_proj", + "model.layers.63.mlp.experts.128.up_proj", + "model.layers.63.mlp.experts.129.down_proj", + "model.layers.63.mlp.experts.129.gate_proj", + "model.layers.63.mlp.experts.129.up_proj", + "model.layers.63.mlp.experts.13.down_proj", + "model.layers.63.mlp.experts.13.gate_proj", + "model.layers.63.mlp.experts.13.up_proj", + "model.layers.63.mlp.experts.130.down_proj", + "model.layers.63.mlp.experts.130.gate_proj", + "model.layers.63.mlp.experts.130.up_proj", + "model.layers.63.mlp.experts.131.down_proj", + "model.layers.63.mlp.experts.131.gate_proj", + "model.layers.63.mlp.experts.131.up_proj", + "model.layers.63.mlp.experts.132.down_proj", + "model.layers.63.mlp.experts.132.gate_proj", + "model.layers.63.mlp.experts.132.up_proj", + "model.layers.63.mlp.experts.133.down_proj", + "model.layers.63.mlp.experts.133.gate_proj", + "model.layers.63.mlp.experts.133.up_proj", + "model.layers.63.mlp.experts.134.down_proj", + "model.layers.63.mlp.experts.134.gate_proj", + "model.layers.63.mlp.experts.134.up_proj", + "model.layers.63.mlp.experts.135.down_proj", + "model.layers.63.mlp.experts.135.gate_proj", + "model.layers.63.mlp.experts.135.up_proj", + "model.layers.63.mlp.experts.136.down_proj", + "model.layers.63.mlp.experts.136.gate_proj", + "model.layers.63.mlp.experts.136.up_proj", + "model.layers.63.mlp.experts.137.down_proj", + "model.layers.63.mlp.experts.137.gate_proj", + "model.layers.63.mlp.experts.137.up_proj", + "model.layers.63.mlp.experts.138.down_proj", + "model.layers.63.mlp.experts.138.gate_proj", + "model.layers.63.mlp.experts.138.up_proj", + "model.layers.63.mlp.experts.139.down_proj", + "model.layers.63.mlp.experts.139.gate_proj", + "model.layers.63.mlp.experts.139.up_proj", + "model.layers.63.mlp.experts.14.down_proj", + "model.layers.63.mlp.experts.14.gate_proj", + "model.layers.63.mlp.experts.14.up_proj", + "model.layers.63.mlp.experts.140.down_proj", + "model.layers.63.mlp.experts.140.gate_proj", + "model.layers.63.mlp.experts.140.up_proj", + "model.layers.63.mlp.experts.141.down_proj", + "model.layers.63.mlp.experts.141.gate_proj", + "model.layers.63.mlp.experts.141.up_proj", + "model.layers.63.mlp.experts.142.down_proj", + "model.layers.63.mlp.experts.142.gate_proj", + "model.layers.63.mlp.experts.142.up_proj", + "model.layers.63.mlp.experts.143.down_proj", + "model.layers.63.mlp.experts.143.gate_proj", + "model.layers.63.mlp.experts.143.up_proj", + "model.layers.63.mlp.experts.144.down_proj", + "model.layers.63.mlp.experts.144.gate_proj", + "model.layers.63.mlp.experts.144.up_proj", + "model.layers.63.mlp.experts.145.down_proj", + "model.layers.63.mlp.experts.145.gate_proj", + "model.layers.63.mlp.experts.145.up_proj", + "model.layers.63.mlp.experts.146.down_proj", + "model.layers.63.mlp.experts.146.gate_proj", + "model.layers.63.mlp.experts.146.up_proj", + "model.layers.63.mlp.experts.147.down_proj", + "model.layers.63.mlp.experts.147.gate_proj", + "model.layers.63.mlp.experts.147.up_proj", + "model.layers.63.mlp.experts.148.down_proj", + "model.layers.63.mlp.experts.148.gate_proj", + "model.layers.63.mlp.experts.148.up_proj", + "model.layers.63.mlp.experts.149.down_proj", + "model.layers.63.mlp.experts.149.gate_proj", + "model.layers.63.mlp.experts.149.up_proj", + "model.layers.63.mlp.experts.15.down_proj", + "model.layers.63.mlp.experts.15.gate_proj", + "model.layers.63.mlp.experts.15.up_proj", + "model.layers.63.mlp.experts.150.down_proj", + "model.layers.63.mlp.experts.150.gate_proj", + "model.layers.63.mlp.experts.150.up_proj", + "model.layers.63.mlp.experts.151.down_proj", + "model.layers.63.mlp.experts.151.gate_proj", + "model.layers.63.mlp.experts.151.up_proj", + "model.layers.63.mlp.experts.152.down_proj", + "model.layers.63.mlp.experts.152.gate_proj", + "model.layers.63.mlp.experts.152.up_proj", + "model.layers.63.mlp.experts.153.down_proj", + "model.layers.63.mlp.experts.153.gate_proj", + "model.layers.63.mlp.experts.153.up_proj", + "model.layers.63.mlp.experts.154.down_proj", + "model.layers.63.mlp.experts.154.gate_proj", + "model.layers.63.mlp.experts.154.up_proj", + "model.layers.63.mlp.experts.155.down_proj", + "model.layers.63.mlp.experts.155.gate_proj", + "model.layers.63.mlp.experts.155.up_proj", + "model.layers.63.mlp.experts.156.down_proj", + "model.layers.63.mlp.experts.156.gate_proj", + "model.layers.63.mlp.experts.156.up_proj", + "model.layers.63.mlp.experts.157.down_proj", + "model.layers.63.mlp.experts.157.gate_proj", + "model.layers.63.mlp.experts.157.up_proj", + "model.layers.63.mlp.experts.158.down_proj", + "model.layers.63.mlp.experts.158.gate_proj", + "model.layers.63.mlp.experts.158.up_proj", + "model.layers.63.mlp.experts.159.down_proj", + "model.layers.63.mlp.experts.159.gate_proj", + "model.layers.63.mlp.experts.159.up_proj", + "model.layers.63.mlp.experts.16.down_proj", + "model.layers.63.mlp.experts.16.gate_proj", + "model.layers.63.mlp.experts.16.up_proj", + "model.layers.63.mlp.experts.160.down_proj", + "model.layers.63.mlp.experts.160.gate_proj", + "model.layers.63.mlp.experts.160.up_proj", + "model.layers.63.mlp.experts.161.down_proj", + "model.layers.63.mlp.experts.161.gate_proj", + "model.layers.63.mlp.experts.161.up_proj", + "model.layers.63.mlp.experts.162.down_proj", + "model.layers.63.mlp.experts.162.gate_proj", + "model.layers.63.mlp.experts.162.up_proj", + "model.layers.63.mlp.experts.163.down_proj", + "model.layers.63.mlp.experts.163.gate_proj", + "model.layers.63.mlp.experts.163.up_proj", + "model.layers.63.mlp.experts.164.down_proj", + "model.layers.63.mlp.experts.164.gate_proj", + "model.layers.63.mlp.experts.164.up_proj", + "model.layers.63.mlp.experts.165.down_proj", + "model.layers.63.mlp.experts.165.gate_proj", + "model.layers.63.mlp.experts.165.up_proj", + "model.layers.63.mlp.experts.166.down_proj", + "model.layers.63.mlp.experts.166.gate_proj", + "model.layers.63.mlp.experts.166.up_proj", + "model.layers.63.mlp.experts.167.down_proj", + "model.layers.63.mlp.experts.167.gate_proj", + "model.layers.63.mlp.experts.167.up_proj", + "model.layers.63.mlp.experts.168.down_proj", + "model.layers.63.mlp.experts.168.gate_proj", + "model.layers.63.mlp.experts.168.up_proj", + "model.layers.63.mlp.experts.169.down_proj", + "model.layers.63.mlp.experts.169.gate_proj", + "model.layers.63.mlp.experts.169.up_proj", + "model.layers.63.mlp.experts.17.down_proj", + "model.layers.63.mlp.experts.17.gate_proj", + "model.layers.63.mlp.experts.17.up_proj", + "model.layers.63.mlp.experts.170.down_proj", + "model.layers.63.mlp.experts.170.gate_proj", + "model.layers.63.mlp.experts.170.up_proj", + "model.layers.63.mlp.experts.171.down_proj", + "model.layers.63.mlp.experts.171.gate_proj", + "model.layers.63.mlp.experts.171.up_proj", + "model.layers.63.mlp.experts.172.down_proj", + "model.layers.63.mlp.experts.172.gate_proj", + "model.layers.63.mlp.experts.172.up_proj", + "model.layers.63.mlp.experts.173.down_proj", + "model.layers.63.mlp.experts.173.gate_proj", + "model.layers.63.mlp.experts.173.up_proj", + "model.layers.63.mlp.experts.174.down_proj", + "model.layers.63.mlp.experts.174.gate_proj", + "model.layers.63.mlp.experts.174.up_proj", + "model.layers.63.mlp.experts.175.down_proj", + "model.layers.63.mlp.experts.175.gate_proj", + "model.layers.63.mlp.experts.175.up_proj", + "model.layers.63.mlp.experts.176.down_proj", + "model.layers.63.mlp.experts.176.gate_proj", + "model.layers.63.mlp.experts.176.up_proj", + "model.layers.63.mlp.experts.177.down_proj", + "model.layers.63.mlp.experts.177.gate_proj", + "model.layers.63.mlp.experts.177.up_proj", + "model.layers.63.mlp.experts.178.down_proj", + "model.layers.63.mlp.experts.178.gate_proj", + "model.layers.63.mlp.experts.178.up_proj", + "model.layers.63.mlp.experts.179.down_proj", + "model.layers.63.mlp.experts.179.gate_proj", + "model.layers.63.mlp.experts.179.up_proj", + "model.layers.63.mlp.experts.18.down_proj", + "model.layers.63.mlp.experts.18.gate_proj", + "model.layers.63.mlp.experts.18.up_proj", + "model.layers.63.mlp.experts.180.down_proj", + "model.layers.63.mlp.experts.180.gate_proj", + "model.layers.63.mlp.experts.180.up_proj", + "model.layers.63.mlp.experts.181.down_proj", + "model.layers.63.mlp.experts.181.gate_proj", + "model.layers.63.mlp.experts.181.up_proj", + "model.layers.63.mlp.experts.182.down_proj", + "model.layers.63.mlp.experts.182.gate_proj", + "model.layers.63.mlp.experts.182.up_proj", + "model.layers.63.mlp.experts.183.down_proj", + "model.layers.63.mlp.experts.183.gate_proj", + "model.layers.63.mlp.experts.183.up_proj", + "model.layers.63.mlp.experts.184.down_proj", + "model.layers.63.mlp.experts.184.gate_proj", + "model.layers.63.mlp.experts.184.up_proj", + "model.layers.63.mlp.experts.185.down_proj", + "model.layers.63.mlp.experts.185.gate_proj", + "model.layers.63.mlp.experts.185.up_proj", + "model.layers.63.mlp.experts.186.down_proj", + "model.layers.63.mlp.experts.186.gate_proj", + "model.layers.63.mlp.experts.186.up_proj", + "model.layers.63.mlp.experts.187.down_proj", + "model.layers.63.mlp.experts.187.gate_proj", + "model.layers.63.mlp.experts.187.up_proj", + "model.layers.63.mlp.experts.188.down_proj", + "model.layers.63.mlp.experts.188.gate_proj", + "model.layers.63.mlp.experts.188.up_proj", + "model.layers.63.mlp.experts.189.down_proj", + "model.layers.63.mlp.experts.189.gate_proj", + "model.layers.63.mlp.experts.189.up_proj", + "model.layers.63.mlp.experts.19.down_proj", + "model.layers.63.mlp.experts.19.gate_proj", + "model.layers.63.mlp.experts.19.up_proj", + "model.layers.63.mlp.experts.190.down_proj", + "model.layers.63.mlp.experts.190.gate_proj", + "model.layers.63.mlp.experts.190.up_proj", + "model.layers.63.mlp.experts.191.down_proj", + "model.layers.63.mlp.experts.191.gate_proj", + "model.layers.63.mlp.experts.191.up_proj", + "model.layers.63.mlp.experts.192.down_proj", + "model.layers.63.mlp.experts.192.gate_proj", + "model.layers.63.mlp.experts.192.up_proj", + "model.layers.63.mlp.experts.193.down_proj", + "model.layers.63.mlp.experts.193.gate_proj", + "model.layers.63.mlp.experts.193.up_proj", + "model.layers.63.mlp.experts.194.down_proj", + "model.layers.63.mlp.experts.194.gate_proj", + "model.layers.63.mlp.experts.194.up_proj", + "model.layers.63.mlp.experts.195.down_proj", + "model.layers.63.mlp.experts.195.gate_proj", + "model.layers.63.mlp.experts.195.up_proj", + "model.layers.63.mlp.experts.196.down_proj", + "model.layers.63.mlp.experts.196.gate_proj", + "model.layers.63.mlp.experts.196.up_proj", + "model.layers.63.mlp.experts.197.down_proj", + "model.layers.63.mlp.experts.197.gate_proj", + "model.layers.63.mlp.experts.197.up_proj", + "model.layers.63.mlp.experts.198.down_proj", + "model.layers.63.mlp.experts.198.gate_proj", + "model.layers.63.mlp.experts.198.up_proj", + "model.layers.63.mlp.experts.199.down_proj", + "model.layers.63.mlp.experts.199.gate_proj", + "model.layers.63.mlp.experts.199.up_proj", + "model.layers.63.mlp.experts.2.down_proj", + "model.layers.63.mlp.experts.2.gate_proj", + "model.layers.63.mlp.experts.2.up_proj", + "model.layers.63.mlp.experts.20.down_proj", + "model.layers.63.mlp.experts.20.gate_proj", + "model.layers.63.mlp.experts.20.up_proj", + "model.layers.63.mlp.experts.200.down_proj", + "model.layers.63.mlp.experts.200.gate_proj", + "model.layers.63.mlp.experts.200.up_proj", + "model.layers.63.mlp.experts.201.down_proj", + "model.layers.63.mlp.experts.201.gate_proj", + "model.layers.63.mlp.experts.201.up_proj", + "model.layers.63.mlp.experts.202.down_proj", + "model.layers.63.mlp.experts.202.gate_proj", + "model.layers.63.mlp.experts.202.up_proj", + "model.layers.63.mlp.experts.203.down_proj", + "model.layers.63.mlp.experts.203.gate_proj", + "model.layers.63.mlp.experts.203.up_proj", + "model.layers.63.mlp.experts.204.down_proj", + "model.layers.63.mlp.experts.204.gate_proj", + "model.layers.63.mlp.experts.204.up_proj", + "model.layers.63.mlp.experts.205.down_proj", + "model.layers.63.mlp.experts.205.gate_proj", + "model.layers.63.mlp.experts.205.up_proj", + "model.layers.63.mlp.experts.206.down_proj", + "model.layers.63.mlp.experts.206.gate_proj", + "model.layers.63.mlp.experts.206.up_proj", + "model.layers.63.mlp.experts.207.down_proj", + "model.layers.63.mlp.experts.207.gate_proj", + "model.layers.63.mlp.experts.207.up_proj", + "model.layers.63.mlp.experts.208.down_proj", + "model.layers.63.mlp.experts.208.gate_proj", + "model.layers.63.mlp.experts.208.up_proj", + "model.layers.63.mlp.experts.209.down_proj", + "model.layers.63.mlp.experts.209.gate_proj", + "model.layers.63.mlp.experts.209.up_proj", + "model.layers.63.mlp.experts.21.down_proj", + "model.layers.63.mlp.experts.21.gate_proj", + "model.layers.63.mlp.experts.21.up_proj", + "model.layers.63.mlp.experts.210.down_proj", + "model.layers.63.mlp.experts.210.gate_proj", + "model.layers.63.mlp.experts.210.up_proj", + "model.layers.63.mlp.experts.211.down_proj", + "model.layers.63.mlp.experts.211.gate_proj", + "model.layers.63.mlp.experts.211.up_proj", + "model.layers.63.mlp.experts.212.down_proj", + "model.layers.63.mlp.experts.212.gate_proj", + "model.layers.63.mlp.experts.212.up_proj", + "model.layers.63.mlp.experts.213.down_proj", + "model.layers.63.mlp.experts.213.gate_proj", + "model.layers.63.mlp.experts.213.up_proj", + "model.layers.63.mlp.experts.214.down_proj", + "model.layers.63.mlp.experts.214.gate_proj", + "model.layers.63.mlp.experts.214.up_proj", + "model.layers.63.mlp.experts.215.down_proj", + "model.layers.63.mlp.experts.215.gate_proj", + "model.layers.63.mlp.experts.215.up_proj", + "model.layers.63.mlp.experts.216.down_proj", + "model.layers.63.mlp.experts.216.gate_proj", + "model.layers.63.mlp.experts.216.up_proj", + "model.layers.63.mlp.experts.217.down_proj", + "model.layers.63.mlp.experts.217.gate_proj", + "model.layers.63.mlp.experts.217.up_proj", + "model.layers.63.mlp.experts.218.down_proj", + "model.layers.63.mlp.experts.218.gate_proj", + "model.layers.63.mlp.experts.218.up_proj", + "model.layers.63.mlp.experts.219.down_proj", + "model.layers.63.mlp.experts.219.gate_proj", + "model.layers.63.mlp.experts.219.up_proj", + "model.layers.63.mlp.experts.22.down_proj", + "model.layers.63.mlp.experts.22.gate_proj", + "model.layers.63.mlp.experts.22.up_proj", + "model.layers.63.mlp.experts.220.down_proj", + "model.layers.63.mlp.experts.220.gate_proj", + "model.layers.63.mlp.experts.220.up_proj", + "model.layers.63.mlp.experts.221.down_proj", + "model.layers.63.mlp.experts.221.gate_proj", + "model.layers.63.mlp.experts.221.up_proj", + "model.layers.63.mlp.experts.222.down_proj", + "model.layers.63.mlp.experts.222.gate_proj", + "model.layers.63.mlp.experts.222.up_proj", + "model.layers.63.mlp.experts.223.down_proj", + "model.layers.63.mlp.experts.223.gate_proj", + "model.layers.63.mlp.experts.223.up_proj", + "model.layers.63.mlp.experts.224.down_proj", + "model.layers.63.mlp.experts.224.gate_proj", + "model.layers.63.mlp.experts.224.up_proj", + "model.layers.63.mlp.experts.225.down_proj", + "model.layers.63.mlp.experts.225.gate_proj", + "model.layers.63.mlp.experts.225.up_proj", + "model.layers.63.mlp.experts.226.down_proj", + "model.layers.63.mlp.experts.226.gate_proj", + "model.layers.63.mlp.experts.226.up_proj", + "model.layers.63.mlp.experts.227.down_proj", + "model.layers.63.mlp.experts.227.gate_proj", + "model.layers.63.mlp.experts.227.up_proj", + "model.layers.63.mlp.experts.228.down_proj", + "model.layers.63.mlp.experts.228.gate_proj", + "model.layers.63.mlp.experts.228.up_proj", + "model.layers.63.mlp.experts.229.down_proj", + "model.layers.63.mlp.experts.229.gate_proj", + "model.layers.63.mlp.experts.229.up_proj", + "model.layers.63.mlp.experts.23.down_proj", + "model.layers.63.mlp.experts.23.gate_proj", + "model.layers.63.mlp.experts.23.up_proj", + "model.layers.63.mlp.experts.230.down_proj", + "model.layers.63.mlp.experts.230.gate_proj", + "model.layers.63.mlp.experts.230.up_proj", + "model.layers.63.mlp.experts.231.down_proj", + "model.layers.63.mlp.experts.231.gate_proj", + "model.layers.63.mlp.experts.231.up_proj", + "model.layers.63.mlp.experts.232.down_proj", + "model.layers.63.mlp.experts.232.gate_proj", + "model.layers.63.mlp.experts.232.up_proj", + "model.layers.63.mlp.experts.233.down_proj", + "model.layers.63.mlp.experts.233.gate_proj", + "model.layers.63.mlp.experts.233.up_proj", + "model.layers.63.mlp.experts.234.down_proj", + "model.layers.63.mlp.experts.234.gate_proj", + "model.layers.63.mlp.experts.234.up_proj", + "model.layers.63.mlp.experts.235.down_proj", + "model.layers.63.mlp.experts.235.gate_proj", + "model.layers.63.mlp.experts.235.up_proj", + "model.layers.63.mlp.experts.236.down_proj", + "model.layers.63.mlp.experts.236.gate_proj", + "model.layers.63.mlp.experts.236.up_proj", + "model.layers.63.mlp.experts.237.down_proj", + "model.layers.63.mlp.experts.237.gate_proj", + "model.layers.63.mlp.experts.237.up_proj", + "model.layers.63.mlp.experts.238.down_proj", + "model.layers.63.mlp.experts.238.gate_proj", + "model.layers.63.mlp.experts.238.up_proj", + "model.layers.63.mlp.experts.239.down_proj", + "model.layers.63.mlp.experts.239.gate_proj", + "model.layers.63.mlp.experts.239.up_proj", + "model.layers.63.mlp.experts.24.down_proj", + "model.layers.63.mlp.experts.24.gate_proj", + "model.layers.63.mlp.experts.24.up_proj", + "model.layers.63.mlp.experts.240.down_proj", + "model.layers.63.mlp.experts.240.gate_proj", + "model.layers.63.mlp.experts.240.up_proj", + "model.layers.63.mlp.experts.241.down_proj", + "model.layers.63.mlp.experts.241.gate_proj", + "model.layers.63.mlp.experts.241.up_proj", + "model.layers.63.mlp.experts.242.down_proj", + "model.layers.63.mlp.experts.242.gate_proj", + "model.layers.63.mlp.experts.242.up_proj", + "model.layers.63.mlp.experts.243.down_proj", + "model.layers.63.mlp.experts.243.gate_proj", + "model.layers.63.mlp.experts.243.up_proj", + "model.layers.63.mlp.experts.244.down_proj", + "model.layers.63.mlp.experts.244.gate_proj", + "model.layers.63.mlp.experts.244.up_proj", + "model.layers.63.mlp.experts.245.down_proj", + "model.layers.63.mlp.experts.245.gate_proj", + "model.layers.63.mlp.experts.245.up_proj", + "model.layers.63.mlp.experts.246.down_proj", + "model.layers.63.mlp.experts.246.gate_proj", + "model.layers.63.mlp.experts.246.up_proj", + "model.layers.63.mlp.experts.247.down_proj", + "model.layers.63.mlp.experts.247.gate_proj", + "model.layers.63.mlp.experts.247.up_proj", + "model.layers.63.mlp.experts.248.down_proj", + "model.layers.63.mlp.experts.248.gate_proj", + "model.layers.63.mlp.experts.248.up_proj", + "model.layers.63.mlp.experts.249.down_proj", + "model.layers.63.mlp.experts.249.gate_proj", + "model.layers.63.mlp.experts.249.up_proj", + "model.layers.63.mlp.experts.25.down_proj", + "model.layers.63.mlp.experts.25.gate_proj", + "model.layers.63.mlp.experts.25.up_proj", + "model.layers.63.mlp.experts.250.down_proj", + "model.layers.63.mlp.experts.250.gate_proj", + "model.layers.63.mlp.experts.250.up_proj", + "model.layers.63.mlp.experts.251.down_proj", + "model.layers.63.mlp.experts.251.gate_proj", + "model.layers.63.mlp.experts.251.up_proj", + "model.layers.63.mlp.experts.252.down_proj", + "model.layers.63.mlp.experts.252.gate_proj", + "model.layers.63.mlp.experts.252.up_proj", + "model.layers.63.mlp.experts.253.down_proj", + "model.layers.63.mlp.experts.253.gate_proj", + "model.layers.63.mlp.experts.253.up_proj", + "model.layers.63.mlp.experts.254.down_proj", + "model.layers.63.mlp.experts.254.gate_proj", + "model.layers.63.mlp.experts.254.up_proj", + "model.layers.63.mlp.experts.255.down_proj", + "model.layers.63.mlp.experts.255.gate_proj", + "model.layers.63.mlp.experts.255.up_proj", + "model.layers.63.mlp.experts.26.down_proj", + "model.layers.63.mlp.experts.26.gate_proj", + "model.layers.63.mlp.experts.26.up_proj", + "model.layers.63.mlp.experts.27.down_proj", + "model.layers.63.mlp.experts.27.gate_proj", + "model.layers.63.mlp.experts.27.up_proj", + "model.layers.63.mlp.experts.28.down_proj", + "model.layers.63.mlp.experts.28.gate_proj", + "model.layers.63.mlp.experts.28.up_proj", + "model.layers.63.mlp.experts.29.down_proj", + "model.layers.63.mlp.experts.29.gate_proj", + "model.layers.63.mlp.experts.29.up_proj", + "model.layers.63.mlp.experts.3.down_proj", + "model.layers.63.mlp.experts.3.gate_proj", + "model.layers.63.mlp.experts.3.up_proj", + "model.layers.63.mlp.experts.30.down_proj", + "model.layers.63.mlp.experts.30.gate_proj", + "model.layers.63.mlp.experts.30.up_proj", + "model.layers.63.mlp.experts.31.down_proj", + "model.layers.63.mlp.experts.31.gate_proj", + "model.layers.63.mlp.experts.31.up_proj", + "model.layers.63.mlp.experts.32.down_proj", + "model.layers.63.mlp.experts.32.gate_proj", + "model.layers.63.mlp.experts.32.up_proj", + "model.layers.63.mlp.experts.33.down_proj", + "model.layers.63.mlp.experts.33.gate_proj", + "model.layers.63.mlp.experts.33.up_proj", + "model.layers.63.mlp.experts.34.down_proj", + "model.layers.63.mlp.experts.34.gate_proj", + "model.layers.63.mlp.experts.34.up_proj", + "model.layers.63.mlp.experts.35.down_proj", + "model.layers.63.mlp.experts.35.gate_proj", + "model.layers.63.mlp.experts.35.up_proj", + "model.layers.63.mlp.experts.36.down_proj", + "model.layers.63.mlp.experts.36.gate_proj", + "model.layers.63.mlp.experts.36.up_proj", + "model.layers.63.mlp.experts.37.down_proj", + "model.layers.63.mlp.experts.37.gate_proj", + "model.layers.63.mlp.experts.37.up_proj", + "model.layers.63.mlp.experts.38.down_proj", + "model.layers.63.mlp.experts.38.gate_proj", + "model.layers.63.mlp.experts.38.up_proj", + "model.layers.63.mlp.experts.39.down_proj", + "model.layers.63.mlp.experts.39.gate_proj", + "model.layers.63.mlp.experts.39.up_proj", + "model.layers.63.mlp.experts.4.down_proj", + "model.layers.63.mlp.experts.4.gate_proj", + "model.layers.63.mlp.experts.4.up_proj", + "model.layers.63.mlp.experts.40.down_proj", + "model.layers.63.mlp.experts.40.gate_proj", + "model.layers.63.mlp.experts.40.up_proj", + "model.layers.63.mlp.experts.41.down_proj", + "model.layers.63.mlp.experts.41.gate_proj", + "model.layers.63.mlp.experts.41.up_proj", + "model.layers.63.mlp.experts.42.down_proj", + "model.layers.63.mlp.experts.42.gate_proj", + "model.layers.63.mlp.experts.42.up_proj", + "model.layers.63.mlp.experts.43.down_proj", + "model.layers.63.mlp.experts.43.gate_proj", + "model.layers.63.mlp.experts.43.up_proj", + "model.layers.63.mlp.experts.44.down_proj", + "model.layers.63.mlp.experts.44.gate_proj", + "model.layers.63.mlp.experts.44.up_proj", + "model.layers.63.mlp.experts.45.down_proj", + "model.layers.63.mlp.experts.45.gate_proj", + "model.layers.63.mlp.experts.45.up_proj", + "model.layers.63.mlp.experts.46.down_proj", + "model.layers.63.mlp.experts.46.gate_proj", + "model.layers.63.mlp.experts.46.up_proj", + "model.layers.63.mlp.experts.47.down_proj", + "model.layers.63.mlp.experts.47.gate_proj", + "model.layers.63.mlp.experts.47.up_proj", + "model.layers.63.mlp.experts.48.down_proj", + "model.layers.63.mlp.experts.48.gate_proj", + "model.layers.63.mlp.experts.48.up_proj", + "model.layers.63.mlp.experts.49.down_proj", + "model.layers.63.mlp.experts.49.gate_proj", + "model.layers.63.mlp.experts.49.up_proj", + "model.layers.63.mlp.experts.5.down_proj", + "model.layers.63.mlp.experts.5.gate_proj", + "model.layers.63.mlp.experts.5.up_proj", + "model.layers.63.mlp.experts.50.down_proj", + "model.layers.63.mlp.experts.50.gate_proj", + "model.layers.63.mlp.experts.50.up_proj", + "model.layers.63.mlp.experts.51.down_proj", + "model.layers.63.mlp.experts.51.gate_proj", + "model.layers.63.mlp.experts.51.up_proj", + "model.layers.63.mlp.experts.52.down_proj", + "model.layers.63.mlp.experts.52.gate_proj", + "model.layers.63.mlp.experts.52.up_proj", + "model.layers.63.mlp.experts.53.down_proj", + "model.layers.63.mlp.experts.53.gate_proj", + "model.layers.63.mlp.experts.53.up_proj", + "model.layers.63.mlp.experts.54.down_proj", + "model.layers.63.mlp.experts.54.gate_proj", + "model.layers.63.mlp.experts.54.up_proj", + "model.layers.63.mlp.experts.55.down_proj", + "model.layers.63.mlp.experts.55.gate_proj", + "model.layers.63.mlp.experts.55.up_proj", + "model.layers.63.mlp.experts.56.down_proj", + "model.layers.63.mlp.experts.56.gate_proj", + "model.layers.63.mlp.experts.56.up_proj", + "model.layers.63.mlp.experts.57.down_proj", + "model.layers.63.mlp.experts.57.gate_proj", + "model.layers.63.mlp.experts.57.up_proj", + "model.layers.63.mlp.experts.58.down_proj", + "model.layers.63.mlp.experts.58.gate_proj", + "model.layers.63.mlp.experts.58.up_proj", + "model.layers.63.mlp.experts.59.down_proj", + "model.layers.63.mlp.experts.59.gate_proj", + "model.layers.63.mlp.experts.59.up_proj", + "model.layers.63.mlp.experts.6.down_proj", + "model.layers.63.mlp.experts.6.gate_proj", + "model.layers.63.mlp.experts.6.up_proj", + "model.layers.63.mlp.experts.60.down_proj", + "model.layers.63.mlp.experts.60.gate_proj", + "model.layers.63.mlp.experts.60.up_proj", + "model.layers.63.mlp.experts.61.down_proj", + "model.layers.63.mlp.experts.61.gate_proj", + "model.layers.63.mlp.experts.61.up_proj", + "model.layers.63.mlp.experts.62.down_proj", + "model.layers.63.mlp.experts.62.gate_proj", + "model.layers.63.mlp.experts.62.up_proj", + "model.layers.63.mlp.experts.63.down_proj", + "model.layers.63.mlp.experts.63.gate_proj", + "model.layers.63.mlp.experts.63.up_proj", + "model.layers.63.mlp.experts.64.down_proj", + "model.layers.63.mlp.experts.64.gate_proj", + "model.layers.63.mlp.experts.64.up_proj", + "model.layers.63.mlp.experts.65.down_proj", + "model.layers.63.mlp.experts.65.gate_proj", + "model.layers.63.mlp.experts.65.up_proj", + "model.layers.63.mlp.experts.66.down_proj", + "model.layers.63.mlp.experts.66.gate_proj", + "model.layers.63.mlp.experts.66.up_proj", + "model.layers.63.mlp.experts.67.down_proj", + "model.layers.63.mlp.experts.67.gate_proj", + "model.layers.63.mlp.experts.67.up_proj", + "model.layers.63.mlp.experts.68.down_proj", + "model.layers.63.mlp.experts.68.gate_proj", + "model.layers.63.mlp.experts.68.up_proj", + "model.layers.63.mlp.experts.69.down_proj", + "model.layers.63.mlp.experts.69.gate_proj", + "model.layers.63.mlp.experts.69.up_proj", + "model.layers.63.mlp.experts.7.down_proj", + "model.layers.63.mlp.experts.7.gate_proj", + "model.layers.63.mlp.experts.7.up_proj", + "model.layers.63.mlp.experts.70.down_proj", + "model.layers.63.mlp.experts.70.gate_proj", + "model.layers.63.mlp.experts.70.up_proj", + "model.layers.63.mlp.experts.71.down_proj", + "model.layers.63.mlp.experts.71.gate_proj", + "model.layers.63.mlp.experts.71.up_proj", + "model.layers.63.mlp.experts.72.down_proj", + "model.layers.63.mlp.experts.72.gate_proj", + "model.layers.63.mlp.experts.72.up_proj", + "model.layers.63.mlp.experts.73.down_proj", + "model.layers.63.mlp.experts.73.gate_proj", + "model.layers.63.mlp.experts.73.up_proj", + "model.layers.63.mlp.experts.74.down_proj", + "model.layers.63.mlp.experts.74.gate_proj", + "model.layers.63.mlp.experts.74.up_proj", + "model.layers.63.mlp.experts.75.down_proj", + "model.layers.63.mlp.experts.75.gate_proj", + "model.layers.63.mlp.experts.75.up_proj", + "model.layers.63.mlp.experts.76.down_proj", + "model.layers.63.mlp.experts.76.gate_proj", + "model.layers.63.mlp.experts.76.up_proj", + "model.layers.63.mlp.experts.77.down_proj", + "model.layers.63.mlp.experts.77.gate_proj", + "model.layers.63.mlp.experts.77.up_proj", + "model.layers.63.mlp.experts.78.down_proj", + "model.layers.63.mlp.experts.78.gate_proj", + "model.layers.63.mlp.experts.78.up_proj", + "model.layers.63.mlp.experts.79.down_proj", + "model.layers.63.mlp.experts.79.gate_proj", + "model.layers.63.mlp.experts.79.up_proj", + "model.layers.63.mlp.experts.8.down_proj", + "model.layers.63.mlp.experts.8.gate_proj", + "model.layers.63.mlp.experts.8.up_proj", + "model.layers.63.mlp.experts.80.down_proj", + "model.layers.63.mlp.experts.80.gate_proj", + "model.layers.63.mlp.experts.80.up_proj", + "model.layers.63.mlp.experts.81.down_proj", + "model.layers.63.mlp.experts.81.gate_proj", + "model.layers.63.mlp.experts.81.up_proj", + "model.layers.63.mlp.experts.82.down_proj", + "model.layers.63.mlp.experts.82.gate_proj", + "model.layers.63.mlp.experts.82.up_proj", + "model.layers.63.mlp.experts.83.down_proj", + "model.layers.63.mlp.experts.83.gate_proj", + "model.layers.63.mlp.experts.83.up_proj", + "model.layers.63.mlp.experts.84.down_proj", + "model.layers.63.mlp.experts.84.gate_proj", + "model.layers.63.mlp.experts.84.up_proj", + "model.layers.63.mlp.experts.85.down_proj", + "model.layers.63.mlp.experts.85.gate_proj", + "model.layers.63.mlp.experts.85.up_proj", + "model.layers.63.mlp.experts.86.down_proj", + "model.layers.63.mlp.experts.86.gate_proj", + "model.layers.63.mlp.experts.86.up_proj", + "model.layers.63.mlp.experts.87.down_proj", + "model.layers.63.mlp.experts.87.gate_proj", + "model.layers.63.mlp.experts.87.up_proj", + "model.layers.63.mlp.experts.88.down_proj", + "model.layers.63.mlp.experts.88.gate_proj", + "model.layers.63.mlp.experts.88.up_proj", + "model.layers.63.mlp.experts.89.down_proj", + "model.layers.63.mlp.experts.89.gate_proj", + "model.layers.63.mlp.experts.89.up_proj", + "model.layers.63.mlp.experts.9.down_proj", + "model.layers.63.mlp.experts.9.gate_proj", + "model.layers.63.mlp.experts.9.up_proj", + "model.layers.63.mlp.experts.90.down_proj", + "model.layers.63.mlp.experts.90.gate_proj", + "model.layers.63.mlp.experts.90.up_proj", + "model.layers.63.mlp.experts.91.down_proj", + "model.layers.63.mlp.experts.91.gate_proj", + "model.layers.63.mlp.experts.91.up_proj", + "model.layers.63.mlp.experts.92.down_proj", + "model.layers.63.mlp.experts.92.gate_proj", + "model.layers.63.mlp.experts.92.up_proj", + "model.layers.63.mlp.experts.93.down_proj", + "model.layers.63.mlp.experts.93.gate_proj", + "model.layers.63.mlp.experts.93.up_proj", + "model.layers.63.mlp.experts.94.down_proj", + "model.layers.63.mlp.experts.94.gate_proj", + "model.layers.63.mlp.experts.94.up_proj", + "model.layers.63.mlp.experts.95.down_proj", + "model.layers.63.mlp.experts.95.gate_proj", + "model.layers.63.mlp.experts.95.up_proj", + "model.layers.63.mlp.experts.96.down_proj", + "model.layers.63.mlp.experts.96.gate_proj", + "model.layers.63.mlp.experts.96.up_proj", + "model.layers.63.mlp.experts.97.down_proj", + "model.layers.63.mlp.experts.97.gate_proj", + "model.layers.63.mlp.experts.97.up_proj", + "model.layers.63.mlp.experts.98.down_proj", + "model.layers.63.mlp.experts.98.gate_proj", + "model.layers.63.mlp.experts.98.up_proj", + "model.layers.63.mlp.experts.99.down_proj", + "model.layers.63.mlp.experts.99.gate_proj", + "model.layers.63.mlp.experts.99.up_proj", + "model.layers.64.mlp.experts.0.down_proj", + "model.layers.64.mlp.experts.0.gate_proj", + "model.layers.64.mlp.experts.0.up_proj", + "model.layers.64.mlp.experts.1.down_proj", + "model.layers.64.mlp.experts.1.gate_proj", + "model.layers.64.mlp.experts.1.up_proj", + "model.layers.64.mlp.experts.10.down_proj", + "model.layers.64.mlp.experts.10.gate_proj", + "model.layers.64.mlp.experts.10.up_proj", + "model.layers.64.mlp.experts.100.down_proj", + "model.layers.64.mlp.experts.100.gate_proj", + "model.layers.64.mlp.experts.100.up_proj", + "model.layers.64.mlp.experts.101.down_proj", + "model.layers.64.mlp.experts.101.gate_proj", + "model.layers.64.mlp.experts.101.up_proj", + "model.layers.64.mlp.experts.102.down_proj", + "model.layers.64.mlp.experts.102.gate_proj", + "model.layers.64.mlp.experts.102.up_proj", + "model.layers.64.mlp.experts.103.down_proj", + "model.layers.64.mlp.experts.103.gate_proj", + "model.layers.64.mlp.experts.103.up_proj", + "model.layers.64.mlp.experts.104.down_proj", + "model.layers.64.mlp.experts.104.gate_proj", + "model.layers.64.mlp.experts.104.up_proj", + "model.layers.64.mlp.experts.105.down_proj", + "model.layers.64.mlp.experts.105.gate_proj", + "model.layers.64.mlp.experts.105.up_proj", + "model.layers.64.mlp.experts.106.down_proj", + "model.layers.64.mlp.experts.106.gate_proj", + "model.layers.64.mlp.experts.106.up_proj", + "model.layers.64.mlp.experts.107.down_proj", + "model.layers.64.mlp.experts.107.gate_proj", + "model.layers.64.mlp.experts.107.up_proj", + "model.layers.64.mlp.experts.108.down_proj", + "model.layers.64.mlp.experts.108.gate_proj", + "model.layers.64.mlp.experts.108.up_proj", + "model.layers.64.mlp.experts.109.down_proj", + "model.layers.64.mlp.experts.109.gate_proj", + "model.layers.64.mlp.experts.109.up_proj", + "model.layers.64.mlp.experts.11.down_proj", + "model.layers.64.mlp.experts.11.gate_proj", + "model.layers.64.mlp.experts.11.up_proj", + "model.layers.64.mlp.experts.110.down_proj", + "model.layers.64.mlp.experts.110.gate_proj", + "model.layers.64.mlp.experts.110.up_proj", + "model.layers.64.mlp.experts.111.down_proj", + "model.layers.64.mlp.experts.111.gate_proj", + "model.layers.64.mlp.experts.111.up_proj", + "model.layers.64.mlp.experts.112.down_proj", + "model.layers.64.mlp.experts.112.gate_proj", + "model.layers.64.mlp.experts.112.up_proj", + "model.layers.64.mlp.experts.113.down_proj", + "model.layers.64.mlp.experts.113.gate_proj", + "model.layers.64.mlp.experts.113.up_proj", + "model.layers.64.mlp.experts.114.down_proj", + "model.layers.64.mlp.experts.114.gate_proj", + "model.layers.64.mlp.experts.114.up_proj", + "model.layers.64.mlp.experts.115.down_proj", + "model.layers.64.mlp.experts.115.gate_proj", + "model.layers.64.mlp.experts.115.up_proj", + "model.layers.64.mlp.experts.116.down_proj", + "model.layers.64.mlp.experts.116.gate_proj", + "model.layers.64.mlp.experts.116.up_proj", + "model.layers.64.mlp.experts.117.down_proj", + "model.layers.64.mlp.experts.117.gate_proj", + "model.layers.64.mlp.experts.117.up_proj", + "model.layers.64.mlp.experts.118.down_proj", + "model.layers.64.mlp.experts.118.gate_proj", + "model.layers.64.mlp.experts.118.up_proj", + "model.layers.64.mlp.experts.119.down_proj", + "model.layers.64.mlp.experts.119.gate_proj", + "model.layers.64.mlp.experts.119.up_proj", + "model.layers.64.mlp.experts.12.down_proj", + "model.layers.64.mlp.experts.12.gate_proj", + "model.layers.64.mlp.experts.12.up_proj", + "model.layers.64.mlp.experts.120.down_proj", + "model.layers.64.mlp.experts.120.gate_proj", + "model.layers.64.mlp.experts.120.up_proj", + "model.layers.64.mlp.experts.121.down_proj", + "model.layers.64.mlp.experts.121.gate_proj", + "model.layers.64.mlp.experts.121.up_proj", + "model.layers.64.mlp.experts.122.down_proj", + "model.layers.64.mlp.experts.122.gate_proj", + "model.layers.64.mlp.experts.122.up_proj", + "model.layers.64.mlp.experts.123.down_proj", + "model.layers.64.mlp.experts.123.gate_proj", + "model.layers.64.mlp.experts.123.up_proj", + "model.layers.64.mlp.experts.124.down_proj", + "model.layers.64.mlp.experts.124.gate_proj", + "model.layers.64.mlp.experts.124.up_proj", + "model.layers.64.mlp.experts.125.down_proj", + "model.layers.64.mlp.experts.125.gate_proj", + "model.layers.64.mlp.experts.125.up_proj", + "model.layers.64.mlp.experts.126.down_proj", + "model.layers.64.mlp.experts.126.gate_proj", + "model.layers.64.mlp.experts.126.up_proj", + "model.layers.64.mlp.experts.127.down_proj", + "model.layers.64.mlp.experts.127.gate_proj", + "model.layers.64.mlp.experts.127.up_proj", + "model.layers.64.mlp.experts.128.down_proj", + "model.layers.64.mlp.experts.128.gate_proj", + "model.layers.64.mlp.experts.128.up_proj", + "model.layers.64.mlp.experts.129.down_proj", + "model.layers.64.mlp.experts.129.gate_proj", + "model.layers.64.mlp.experts.129.up_proj", + "model.layers.64.mlp.experts.13.down_proj", + "model.layers.64.mlp.experts.13.gate_proj", + "model.layers.64.mlp.experts.13.up_proj", + "model.layers.64.mlp.experts.130.down_proj", + "model.layers.64.mlp.experts.130.gate_proj", + "model.layers.64.mlp.experts.130.up_proj", + "model.layers.64.mlp.experts.131.down_proj", + "model.layers.64.mlp.experts.131.gate_proj", + "model.layers.64.mlp.experts.131.up_proj", + "model.layers.64.mlp.experts.132.down_proj", + "model.layers.64.mlp.experts.132.gate_proj", + "model.layers.64.mlp.experts.132.up_proj", + "model.layers.64.mlp.experts.133.down_proj", + "model.layers.64.mlp.experts.133.gate_proj", + "model.layers.64.mlp.experts.133.up_proj", + "model.layers.64.mlp.experts.134.down_proj", + "model.layers.64.mlp.experts.134.gate_proj", + "model.layers.64.mlp.experts.134.up_proj", + "model.layers.64.mlp.experts.135.down_proj", + "model.layers.64.mlp.experts.135.gate_proj", + "model.layers.64.mlp.experts.135.up_proj", + "model.layers.64.mlp.experts.136.down_proj", + "model.layers.64.mlp.experts.136.gate_proj", + "model.layers.64.mlp.experts.136.up_proj", + "model.layers.64.mlp.experts.137.down_proj", + "model.layers.64.mlp.experts.137.gate_proj", + "model.layers.64.mlp.experts.137.up_proj", + "model.layers.64.mlp.experts.138.down_proj", + "model.layers.64.mlp.experts.138.gate_proj", + "model.layers.64.mlp.experts.138.up_proj", + "model.layers.64.mlp.experts.139.down_proj", + "model.layers.64.mlp.experts.139.gate_proj", + "model.layers.64.mlp.experts.139.up_proj", + "model.layers.64.mlp.experts.14.down_proj", + "model.layers.64.mlp.experts.14.gate_proj", + "model.layers.64.mlp.experts.14.up_proj", + "model.layers.64.mlp.experts.140.down_proj", + "model.layers.64.mlp.experts.140.gate_proj", + "model.layers.64.mlp.experts.140.up_proj", + "model.layers.64.mlp.experts.141.down_proj", + "model.layers.64.mlp.experts.141.gate_proj", + "model.layers.64.mlp.experts.141.up_proj", + "model.layers.64.mlp.experts.142.down_proj", + "model.layers.64.mlp.experts.142.gate_proj", + "model.layers.64.mlp.experts.142.up_proj", + "model.layers.64.mlp.experts.143.down_proj", + "model.layers.64.mlp.experts.143.gate_proj", + "model.layers.64.mlp.experts.143.up_proj", + "model.layers.64.mlp.experts.144.down_proj", + "model.layers.64.mlp.experts.144.gate_proj", + "model.layers.64.mlp.experts.144.up_proj", + "model.layers.64.mlp.experts.145.down_proj", + "model.layers.64.mlp.experts.145.gate_proj", + "model.layers.64.mlp.experts.145.up_proj", + "model.layers.64.mlp.experts.146.down_proj", + "model.layers.64.mlp.experts.146.gate_proj", + "model.layers.64.mlp.experts.146.up_proj", + "model.layers.64.mlp.experts.147.down_proj", + "model.layers.64.mlp.experts.147.gate_proj", + "model.layers.64.mlp.experts.147.up_proj", + "model.layers.64.mlp.experts.148.down_proj", + "model.layers.64.mlp.experts.148.gate_proj", + "model.layers.64.mlp.experts.148.up_proj", + "model.layers.64.mlp.experts.149.down_proj", + "model.layers.64.mlp.experts.149.gate_proj", + "model.layers.64.mlp.experts.149.up_proj", + "model.layers.64.mlp.experts.15.down_proj", + "model.layers.64.mlp.experts.15.gate_proj", + "model.layers.64.mlp.experts.15.up_proj", + "model.layers.64.mlp.experts.150.down_proj", + "model.layers.64.mlp.experts.150.gate_proj", + "model.layers.64.mlp.experts.150.up_proj", + "model.layers.64.mlp.experts.151.down_proj", + "model.layers.64.mlp.experts.151.gate_proj", + "model.layers.64.mlp.experts.151.up_proj", + "model.layers.64.mlp.experts.152.down_proj", + "model.layers.64.mlp.experts.152.gate_proj", + "model.layers.64.mlp.experts.152.up_proj", + "model.layers.64.mlp.experts.153.down_proj", + "model.layers.64.mlp.experts.153.gate_proj", + "model.layers.64.mlp.experts.153.up_proj", + "model.layers.64.mlp.experts.154.down_proj", + "model.layers.64.mlp.experts.154.gate_proj", + "model.layers.64.mlp.experts.154.up_proj", + "model.layers.64.mlp.experts.155.down_proj", + "model.layers.64.mlp.experts.155.gate_proj", + "model.layers.64.mlp.experts.155.up_proj", + "model.layers.64.mlp.experts.156.down_proj", + "model.layers.64.mlp.experts.156.gate_proj", + "model.layers.64.mlp.experts.156.up_proj", + "model.layers.64.mlp.experts.157.down_proj", + "model.layers.64.mlp.experts.157.gate_proj", + "model.layers.64.mlp.experts.157.up_proj", + "model.layers.64.mlp.experts.158.down_proj", + "model.layers.64.mlp.experts.158.gate_proj", + "model.layers.64.mlp.experts.158.up_proj", + "model.layers.64.mlp.experts.159.down_proj", + "model.layers.64.mlp.experts.159.gate_proj", + "model.layers.64.mlp.experts.159.up_proj", + "model.layers.64.mlp.experts.16.down_proj", + "model.layers.64.mlp.experts.16.gate_proj", + "model.layers.64.mlp.experts.16.up_proj", + "model.layers.64.mlp.experts.160.down_proj", + "model.layers.64.mlp.experts.160.gate_proj", + "model.layers.64.mlp.experts.160.up_proj", + "model.layers.64.mlp.experts.161.down_proj", + "model.layers.64.mlp.experts.161.gate_proj", + "model.layers.64.mlp.experts.161.up_proj", + "model.layers.64.mlp.experts.162.down_proj", + "model.layers.64.mlp.experts.162.gate_proj", + "model.layers.64.mlp.experts.162.up_proj", + "model.layers.64.mlp.experts.163.down_proj", + "model.layers.64.mlp.experts.163.gate_proj", + "model.layers.64.mlp.experts.163.up_proj", + "model.layers.64.mlp.experts.164.down_proj", + "model.layers.64.mlp.experts.164.gate_proj", + "model.layers.64.mlp.experts.164.up_proj", + "model.layers.64.mlp.experts.165.down_proj", + "model.layers.64.mlp.experts.165.gate_proj", + "model.layers.64.mlp.experts.165.up_proj", + "model.layers.64.mlp.experts.166.down_proj", + "model.layers.64.mlp.experts.166.gate_proj", + "model.layers.64.mlp.experts.166.up_proj", + "model.layers.64.mlp.experts.167.down_proj", + "model.layers.64.mlp.experts.167.gate_proj", + "model.layers.64.mlp.experts.167.up_proj", + "model.layers.64.mlp.experts.168.down_proj", + "model.layers.64.mlp.experts.168.gate_proj", + "model.layers.64.mlp.experts.168.up_proj", + "model.layers.64.mlp.experts.169.down_proj", + "model.layers.64.mlp.experts.169.gate_proj", + "model.layers.64.mlp.experts.169.up_proj", + "model.layers.64.mlp.experts.17.down_proj", + "model.layers.64.mlp.experts.17.gate_proj", + "model.layers.64.mlp.experts.17.up_proj", + "model.layers.64.mlp.experts.170.down_proj", + "model.layers.64.mlp.experts.170.gate_proj", + "model.layers.64.mlp.experts.170.up_proj", + "model.layers.64.mlp.experts.171.down_proj", + "model.layers.64.mlp.experts.171.gate_proj", + "model.layers.64.mlp.experts.171.up_proj", + "model.layers.64.mlp.experts.172.down_proj", + "model.layers.64.mlp.experts.172.gate_proj", + "model.layers.64.mlp.experts.172.up_proj", + "model.layers.64.mlp.experts.173.down_proj", + "model.layers.64.mlp.experts.173.gate_proj", + "model.layers.64.mlp.experts.173.up_proj", + "model.layers.64.mlp.experts.174.down_proj", + "model.layers.64.mlp.experts.174.gate_proj", + "model.layers.64.mlp.experts.174.up_proj", + "model.layers.64.mlp.experts.175.down_proj", + "model.layers.64.mlp.experts.175.gate_proj", + "model.layers.64.mlp.experts.175.up_proj", + "model.layers.64.mlp.experts.176.down_proj", + "model.layers.64.mlp.experts.176.gate_proj", + "model.layers.64.mlp.experts.176.up_proj", + "model.layers.64.mlp.experts.177.down_proj", + "model.layers.64.mlp.experts.177.gate_proj", + "model.layers.64.mlp.experts.177.up_proj", + "model.layers.64.mlp.experts.178.down_proj", + "model.layers.64.mlp.experts.178.gate_proj", + "model.layers.64.mlp.experts.178.up_proj", + "model.layers.64.mlp.experts.179.down_proj", + "model.layers.64.mlp.experts.179.gate_proj", + "model.layers.64.mlp.experts.179.up_proj", + "model.layers.64.mlp.experts.18.down_proj", + "model.layers.64.mlp.experts.18.gate_proj", + "model.layers.64.mlp.experts.18.up_proj", + "model.layers.64.mlp.experts.180.down_proj", + "model.layers.64.mlp.experts.180.gate_proj", + "model.layers.64.mlp.experts.180.up_proj", + "model.layers.64.mlp.experts.181.down_proj", + "model.layers.64.mlp.experts.181.gate_proj", + "model.layers.64.mlp.experts.181.up_proj", + "model.layers.64.mlp.experts.182.down_proj", + "model.layers.64.mlp.experts.182.gate_proj", + "model.layers.64.mlp.experts.182.up_proj", + "model.layers.64.mlp.experts.183.down_proj", + "model.layers.64.mlp.experts.183.gate_proj", + "model.layers.64.mlp.experts.183.up_proj", + "model.layers.64.mlp.experts.184.down_proj", + "model.layers.64.mlp.experts.184.gate_proj", + "model.layers.64.mlp.experts.184.up_proj", + "model.layers.64.mlp.experts.185.down_proj", + "model.layers.64.mlp.experts.185.gate_proj", + "model.layers.64.mlp.experts.185.up_proj", + "model.layers.64.mlp.experts.186.down_proj", + "model.layers.64.mlp.experts.186.gate_proj", + "model.layers.64.mlp.experts.186.up_proj", + "model.layers.64.mlp.experts.187.down_proj", + "model.layers.64.mlp.experts.187.gate_proj", + "model.layers.64.mlp.experts.187.up_proj", + "model.layers.64.mlp.experts.188.down_proj", + "model.layers.64.mlp.experts.188.gate_proj", + "model.layers.64.mlp.experts.188.up_proj", + "model.layers.64.mlp.experts.189.down_proj", + "model.layers.64.mlp.experts.189.gate_proj", + "model.layers.64.mlp.experts.189.up_proj", + "model.layers.64.mlp.experts.19.down_proj", + "model.layers.64.mlp.experts.19.gate_proj", + "model.layers.64.mlp.experts.19.up_proj", + "model.layers.64.mlp.experts.190.down_proj", + "model.layers.64.mlp.experts.190.gate_proj", + "model.layers.64.mlp.experts.190.up_proj", + "model.layers.64.mlp.experts.191.down_proj", + "model.layers.64.mlp.experts.191.gate_proj", + "model.layers.64.mlp.experts.191.up_proj", + "model.layers.64.mlp.experts.192.down_proj", + "model.layers.64.mlp.experts.192.gate_proj", + "model.layers.64.mlp.experts.192.up_proj", + "model.layers.64.mlp.experts.193.down_proj", + "model.layers.64.mlp.experts.193.gate_proj", + "model.layers.64.mlp.experts.193.up_proj", + "model.layers.64.mlp.experts.194.down_proj", + "model.layers.64.mlp.experts.194.gate_proj", + "model.layers.64.mlp.experts.194.up_proj", + "model.layers.64.mlp.experts.195.down_proj", + "model.layers.64.mlp.experts.195.gate_proj", + "model.layers.64.mlp.experts.195.up_proj", + "model.layers.64.mlp.experts.196.down_proj", + "model.layers.64.mlp.experts.196.gate_proj", + "model.layers.64.mlp.experts.196.up_proj", + "model.layers.64.mlp.experts.197.down_proj", + "model.layers.64.mlp.experts.197.gate_proj", + "model.layers.64.mlp.experts.197.up_proj", + "model.layers.64.mlp.experts.198.down_proj", + "model.layers.64.mlp.experts.198.gate_proj", + "model.layers.64.mlp.experts.198.up_proj", + "model.layers.64.mlp.experts.199.down_proj", + "model.layers.64.mlp.experts.199.gate_proj", + "model.layers.64.mlp.experts.199.up_proj", + "model.layers.64.mlp.experts.2.down_proj", + "model.layers.64.mlp.experts.2.gate_proj", + "model.layers.64.mlp.experts.2.up_proj", + "model.layers.64.mlp.experts.20.down_proj", + "model.layers.64.mlp.experts.20.gate_proj", + "model.layers.64.mlp.experts.20.up_proj", + "model.layers.64.mlp.experts.200.down_proj", + "model.layers.64.mlp.experts.200.gate_proj", + "model.layers.64.mlp.experts.200.up_proj", + "model.layers.64.mlp.experts.201.down_proj", + "model.layers.64.mlp.experts.201.gate_proj", + "model.layers.64.mlp.experts.201.up_proj", + "model.layers.64.mlp.experts.202.down_proj", + "model.layers.64.mlp.experts.202.gate_proj", + "model.layers.64.mlp.experts.202.up_proj", + "model.layers.64.mlp.experts.203.down_proj", + "model.layers.64.mlp.experts.203.gate_proj", + "model.layers.64.mlp.experts.203.up_proj", + "model.layers.64.mlp.experts.204.down_proj", + "model.layers.64.mlp.experts.204.gate_proj", + "model.layers.64.mlp.experts.204.up_proj", + "model.layers.64.mlp.experts.205.down_proj", + "model.layers.64.mlp.experts.205.gate_proj", + "model.layers.64.mlp.experts.205.up_proj", + "model.layers.64.mlp.experts.206.down_proj", + "model.layers.64.mlp.experts.206.gate_proj", + "model.layers.64.mlp.experts.206.up_proj", + "model.layers.64.mlp.experts.207.down_proj", + "model.layers.64.mlp.experts.207.gate_proj", + "model.layers.64.mlp.experts.207.up_proj", + "model.layers.64.mlp.experts.208.down_proj", + "model.layers.64.mlp.experts.208.gate_proj", + "model.layers.64.mlp.experts.208.up_proj", + "model.layers.64.mlp.experts.209.down_proj", + "model.layers.64.mlp.experts.209.gate_proj", + "model.layers.64.mlp.experts.209.up_proj", + "model.layers.64.mlp.experts.21.down_proj", + "model.layers.64.mlp.experts.21.gate_proj", + "model.layers.64.mlp.experts.21.up_proj", + "model.layers.64.mlp.experts.210.down_proj", + "model.layers.64.mlp.experts.210.gate_proj", + "model.layers.64.mlp.experts.210.up_proj", + "model.layers.64.mlp.experts.211.down_proj", + "model.layers.64.mlp.experts.211.gate_proj", + "model.layers.64.mlp.experts.211.up_proj", + "model.layers.64.mlp.experts.212.down_proj", + "model.layers.64.mlp.experts.212.gate_proj", + "model.layers.64.mlp.experts.212.up_proj", + "model.layers.64.mlp.experts.213.down_proj", + "model.layers.64.mlp.experts.213.gate_proj", + "model.layers.64.mlp.experts.213.up_proj", + "model.layers.64.mlp.experts.214.down_proj", + "model.layers.64.mlp.experts.214.gate_proj", + "model.layers.64.mlp.experts.214.up_proj", + "model.layers.64.mlp.experts.215.down_proj", + "model.layers.64.mlp.experts.215.gate_proj", + "model.layers.64.mlp.experts.215.up_proj", + "model.layers.64.mlp.experts.216.down_proj", + "model.layers.64.mlp.experts.216.gate_proj", + "model.layers.64.mlp.experts.216.up_proj", + "model.layers.64.mlp.experts.217.down_proj", + "model.layers.64.mlp.experts.217.gate_proj", + "model.layers.64.mlp.experts.217.up_proj", + "model.layers.64.mlp.experts.218.down_proj", + "model.layers.64.mlp.experts.218.gate_proj", + "model.layers.64.mlp.experts.218.up_proj", + "model.layers.64.mlp.experts.219.down_proj", + "model.layers.64.mlp.experts.219.gate_proj", + "model.layers.64.mlp.experts.219.up_proj", + "model.layers.64.mlp.experts.22.down_proj", + "model.layers.64.mlp.experts.22.gate_proj", + "model.layers.64.mlp.experts.22.up_proj", + "model.layers.64.mlp.experts.220.down_proj", + "model.layers.64.mlp.experts.220.gate_proj", + "model.layers.64.mlp.experts.220.up_proj", + "model.layers.64.mlp.experts.221.down_proj", + "model.layers.64.mlp.experts.221.gate_proj", + "model.layers.64.mlp.experts.221.up_proj", + "model.layers.64.mlp.experts.222.down_proj", + "model.layers.64.mlp.experts.222.gate_proj", + "model.layers.64.mlp.experts.222.up_proj", + "model.layers.64.mlp.experts.223.down_proj", + "model.layers.64.mlp.experts.223.gate_proj", + "model.layers.64.mlp.experts.223.up_proj", + "model.layers.64.mlp.experts.224.down_proj", + "model.layers.64.mlp.experts.224.gate_proj", + "model.layers.64.mlp.experts.224.up_proj", + "model.layers.64.mlp.experts.225.down_proj", + "model.layers.64.mlp.experts.225.gate_proj", + "model.layers.64.mlp.experts.225.up_proj", + "model.layers.64.mlp.experts.226.down_proj", + "model.layers.64.mlp.experts.226.gate_proj", + "model.layers.64.mlp.experts.226.up_proj", + "model.layers.64.mlp.experts.227.down_proj", + "model.layers.64.mlp.experts.227.gate_proj", + "model.layers.64.mlp.experts.227.up_proj", + "model.layers.64.mlp.experts.228.down_proj", + "model.layers.64.mlp.experts.228.gate_proj", + "model.layers.64.mlp.experts.228.up_proj", + "model.layers.64.mlp.experts.229.down_proj", + "model.layers.64.mlp.experts.229.gate_proj", + "model.layers.64.mlp.experts.229.up_proj", + "model.layers.64.mlp.experts.23.down_proj", + "model.layers.64.mlp.experts.23.gate_proj", + "model.layers.64.mlp.experts.23.up_proj", + "model.layers.64.mlp.experts.230.down_proj", + "model.layers.64.mlp.experts.230.gate_proj", + "model.layers.64.mlp.experts.230.up_proj", + "model.layers.64.mlp.experts.231.down_proj", + "model.layers.64.mlp.experts.231.gate_proj", + "model.layers.64.mlp.experts.231.up_proj", + "model.layers.64.mlp.experts.232.down_proj", + "model.layers.64.mlp.experts.232.gate_proj", + "model.layers.64.mlp.experts.232.up_proj", + "model.layers.64.mlp.experts.233.down_proj", + "model.layers.64.mlp.experts.233.gate_proj", + "model.layers.64.mlp.experts.233.up_proj", + "model.layers.64.mlp.experts.234.down_proj", + "model.layers.64.mlp.experts.234.gate_proj", + "model.layers.64.mlp.experts.234.up_proj", + "model.layers.64.mlp.experts.235.down_proj", + "model.layers.64.mlp.experts.235.gate_proj", + "model.layers.64.mlp.experts.235.up_proj", + "model.layers.64.mlp.experts.236.down_proj", + "model.layers.64.mlp.experts.236.gate_proj", + "model.layers.64.mlp.experts.236.up_proj", + "model.layers.64.mlp.experts.237.down_proj", + "model.layers.64.mlp.experts.237.gate_proj", + "model.layers.64.mlp.experts.237.up_proj", + "model.layers.64.mlp.experts.238.down_proj", + "model.layers.64.mlp.experts.238.gate_proj", + "model.layers.64.mlp.experts.238.up_proj", + "model.layers.64.mlp.experts.239.down_proj", + "model.layers.64.mlp.experts.239.gate_proj", + "model.layers.64.mlp.experts.239.up_proj", + "model.layers.64.mlp.experts.24.down_proj", + "model.layers.64.mlp.experts.24.gate_proj", + "model.layers.64.mlp.experts.24.up_proj", + "model.layers.64.mlp.experts.240.down_proj", + "model.layers.64.mlp.experts.240.gate_proj", + "model.layers.64.mlp.experts.240.up_proj", + "model.layers.64.mlp.experts.241.down_proj", + "model.layers.64.mlp.experts.241.gate_proj", + "model.layers.64.mlp.experts.241.up_proj", + "model.layers.64.mlp.experts.242.down_proj", + "model.layers.64.mlp.experts.242.gate_proj", + "model.layers.64.mlp.experts.242.up_proj", + "model.layers.64.mlp.experts.243.down_proj", + "model.layers.64.mlp.experts.243.gate_proj", + "model.layers.64.mlp.experts.243.up_proj", + "model.layers.64.mlp.experts.244.down_proj", + "model.layers.64.mlp.experts.244.gate_proj", + "model.layers.64.mlp.experts.244.up_proj", + "model.layers.64.mlp.experts.245.down_proj", + "model.layers.64.mlp.experts.245.gate_proj", + "model.layers.64.mlp.experts.245.up_proj", + "model.layers.64.mlp.experts.246.down_proj", + "model.layers.64.mlp.experts.246.gate_proj", + "model.layers.64.mlp.experts.246.up_proj", + "model.layers.64.mlp.experts.247.down_proj", + "model.layers.64.mlp.experts.247.gate_proj", + "model.layers.64.mlp.experts.247.up_proj", + "model.layers.64.mlp.experts.248.down_proj", + "model.layers.64.mlp.experts.248.gate_proj", + "model.layers.64.mlp.experts.248.up_proj", + "model.layers.64.mlp.experts.249.down_proj", + "model.layers.64.mlp.experts.249.gate_proj", + "model.layers.64.mlp.experts.249.up_proj", + "model.layers.64.mlp.experts.25.down_proj", + "model.layers.64.mlp.experts.25.gate_proj", + "model.layers.64.mlp.experts.25.up_proj", + "model.layers.64.mlp.experts.250.down_proj", + "model.layers.64.mlp.experts.250.gate_proj", + "model.layers.64.mlp.experts.250.up_proj", + "model.layers.64.mlp.experts.251.down_proj", + "model.layers.64.mlp.experts.251.gate_proj", + "model.layers.64.mlp.experts.251.up_proj", + "model.layers.64.mlp.experts.252.down_proj", + "model.layers.64.mlp.experts.252.gate_proj", + "model.layers.64.mlp.experts.252.up_proj", + "model.layers.64.mlp.experts.253.down_proj", + "model.layers.64.mlp.experts.253.gate_proj", + "model.layers.64.mlp.experts.253.up_proj", + "model.layers.64.mlp.experts.254.down_proj", + "model.layers.64.mlp.experts.254.gate_proj", + "model.layers.64.mlp.experts.254.up_proj", + "model.layers.64.mlp.experts.255.down_proj", + "model.layers.64.mlp.experts.255.gate_proj", + "model.layers.64.mlp.experts.255.up_proj", + "model.layers.64.mlp.experts.26.down_proj", + "model.layers.64.mlp.experts.26.gate_proj", + "model.layers.64.mlp.experts.26.up_proj", + "model.layers.64.mlp.experts.27.down_proj", + "model.layers.64.mlp.experts.27.gate_proj", + "model.layers.64.mlp.experts.27.up_proj", + "model.layers.64.mlp.experts.28.down_proj", + "model.layers.64.mlp.experts.28.gate_proj", + "model.layers.64.mlp.experts.28.up_proj", + "model.layers.64.mlp.experts.29.down_proj", + "model.layers.64.mlp.experts.29.gate_proj", + "model.layers.64.mlp.experts.29.up_proj", + "model.layers.64.mlp.experts.3.down_proj", + "model.layers.64.mlp.experts.3.gate_proj", + "model.layers.64.mlp.experts.3.up_proj", + "model.layers.64.mlp.experts.30.down_proj", + "model.layers.64.mlp.experts.30.gate_proj", + "model.layers.64.mlp.experts.30.up_proj", + "model.layers.64.mlp.experts.31.down_proj", + "model.layers.64.mlp.experts.31.gate_proj", + "model.layers.64.mlp.experts.31.up_proj", + "model.layers.64.mlp.experts.32.down_proj", + "model.layers.64.mlp.experts.32.gate_proj", + "model.layers.64.mlp.experts.32.up_proj", + "model.layers.64.mlp.experts.33.down_proj", + "model.layers.64.mlp.experts.33.gate_proj", + "model.layers.64.mlp.experts.33.up_proj", + "model.layers.64.mlp.experts.34.down_proj", + "model.layers.64.mlp.experts.34.gate_proj", + "model.layers.64.mlp.experts.34.up_proj", + "model.layers.64.mlp.experts.35.down_proj", + "model.layers.64.mlp.experts.35.gate_proj", + "model.layers.64.mlp.experts.35.up_proj", + "model.layers.64.mlp.experts.36.down_proj", + "model.layers.64.mlp.experts.36.gate_proj", + "model.layers.64.mlp.experts.36.up_proj", + "model.layers.64.mlp.experts.37.down_proj", + "model.layers.64.mlp.experts.37.gate_proj", + "model.layers.64.mlp.experts.37.up_proj", + "model.layers.64.mlp.experts.38.down_proj", + "model.layers.64.mlp.experts.38.gate_proj", + "model.layers.64.mlp.experts.38.up_proj", + "model.layers.64.mlp.experts.39.down_proj", + "model.layers.64.mlp.experts.39.gate_proj", + "model.layers.64.mlp.experts.39.up_proj", + "model.layers.64.mlp.experts.4.down_proj", + "model.layers.64.mlp.experts.4.gate_proj", + "model.layers.64.mlp.experts.4.up_proj", + "model.layers.64.mlp.experts.40.down_proj", + "model.layers.64.mlp.experts.40.gate_proj", + "model.layers.64.mlp.experts.40.up_proj", + "model.layers.64.mlp.experts.41.down_proj", + "model.layers.64.mlp.experts.41.gate_proj", + "model.layers.64.mlp.experts.41.up_proj", + "model.layers.64.mlp.experts.42.down_proj", + "model.layers.64.mlp.experts.42.gate_proj", + "model.layers.64.mlp.experts.42.up_proj", + "model.layers.64.mlp.experts.43.down_proj", + "model.layers.64.mlp.experts.43.gate_proj", + "model.layers.64.mlp.experts.43.up_proj", + "model.layers.64.mlp.experts.44.down_proj", + "model.layers.64.mlp.experts.44.gate_proj", + "model.layers.64.mlp.experts.44.up_proj", + "model.layers.64.mlp.experts.45.down_proj", + "model.layers.64.mlp.experts.45.gate_proj", + "model.layers.64.mlp.experts.45.up_proj", + "model.layers.64.mlp.experts.46.down_proj", + "model.layers.64.mlp.experts.46.gate_proj", + "model.layers.64.mlp.experts.46.up_proj", + "model.layers.64.mlp.experts.47.down_proj", + "model.layers.64.mlp.experts.47.gate_proj", + "model.layers.64.mlp.experts.47.up_proj", + "model.layers.64.mlp.experts.48.down_proj", + "model.layers.64.mlp.experts.48.gate_proj", + "model.layers.64.mlp.experts.48.up_proj", + "model.layers.64.mlp.experts.49.down_proj", + "model.layers.64.mlp.experts.49.gate_proj", + "model.layers.64.mlp.experts.49.up_proj", + "model.layers.64.mlp.experts.5.down_proj", + "model.layers.64.mlp.experts.5.gate_proj", + "model.layers.64.mlp.experts.5.up_proj", + "model.layers.64.mlp.experts.50.down_proj", + "model.layers.64.mlp.experts.50.gate_proj", + "model.layers.64.mlp.experts.50.up_proj", + "model.layers.64.mlp.experts.51.down_proj", + "model.layers.64.mlp.experts.51.gate_proj", + "model.layers.64.mlp.experts.51.up_proj", + "model.layers.64.mlp.experts.52.down_proj", + "model.layers.64.mlp.experts.52.gate_proj", + "model.layers.64.mlp.experts.52.up_proj", + "model.layers.64.mlp.experts.53.down_proj", + "model.layers.64.mlp.experts.53.gate_proj", + "model.layers.64.mlp.experts.53.up_proj", + "model.layers.64.mlp.experts.54.down_proj", + "model.layers.64.mlp.experts.54.gate_proj", + "model.layers.64.mlp.experts.54.up_proj", + "model.layers.64.mlp.experts.55.down_proj", + "model.layers.64.mlp.experts.55.gate_proj", + "model.layers.64.mlp.experts.55.up_proj", + "model.layers.64.mlp.experts.56.down_proj", + "model.layers.64.mlp.experts.56.gate_proj", + "model.layers.64.mlp.experts.56.up_proj", + "model.layers.64.mlp.experts.57.down_proj", + "model.layers.64.mlp.experts.57.gate_proj", + "model.layers.64.mlp.experts.57.up_proj", + "model.layers.64.mlp.experts.58.down_proj", + "model.layers.64.mlp.experts.58.gate_proj", + "model.layers.64.mlp.experts.58.up_proj", + "model.layers.64.mlp.experts.59.down_proj", + "model.layers.64.mlp.experts.59.gate_proj", + "model.layers.64.mlp.experts.59.up_proj", + "model.layers.64.mlp.experts.6.down_proj", + "model.layers.64.mlp.experts.6.gate_proj", + "model.layers.64.mlp.experts.6.up_proj", + "model.layers.64.mlp.experts.60.down_proj", + "model.layers.64.mlp.experts.60.gate_proj", + "model.layers.64.mlp.experts.60.up_proj", + "model.layers.64.mlp.experts.61.down_proj", + "model.layers.64.mlp.experts.61.gate_proj", + "model.layers.64.mlp.experts.61.up_proj", + "model.layers.64.mlp.experts.62.down_proj", + "model.layers.64.mlp.experts.62.gate_proj", + "model.layers.64.mlp.experts.62.up_proj", + "model.layers.64.mlp.experts.63.down_proj", + "model.layers.64.mlp.experts.63.gate_proj", + "model.layers.64.mlp.experts.63.up_proj", + "model.layers.64.mlp.experts.64.down_proj", + "model.layers.64.mlp.experts.64.gate_proj", + "model.layers.64.mlp.experts.64.up_proj", + "model.layers.64.mlp.experts.65.down_proj", + "model.layers.64.mlp.experts.65.gate_proj", + "model.layers.64.mlp.experts.65.up_proj", + "model.layers.64.mlp.experts.66.down_proj", + "model.layers.64.mlp.experts.66.gate_proj", + "model.layers.64.mlp.experts.66.up_proj", + "model.layers.64.mlp.experts.67.down_proj", + "model.layers.64.mlp.experts.67.gate_proj", + "model.layers.64.mlp.experts.67.up_proj", + "model.layers.64.mlp.experts.68.down_proj", + "model.layers.64.mlp.experts.68.gate_proj", + "model.layers.64.mlp.experts.68.up_proj", + "model.layers.64.mlp.experts.69.down_proj", + "model.layers.64.mlp.experts.69.gate_proj", + "model.layers.64.mlp.experts.69.up_proj", + "model.layers.64.mlp.experts.7.down_proj", + "model.layers.64.mlp.experts.7.gate_proj", + "model.layers.64.mlp.experts.7.up_proj", + "model.layers.64.mlp.experts.70.down_proj", + "model.layers.64.mlp.experts.70.gate_proj", + "model.layers.64.mlp.experts.70.up_proj", + "model.layers.64.mlp.experts.71.down_proj", + "model.layers.64.mlp.experts.71.gate_proj", + "model.layers.64.mlp.experts.71.up_proj", + "model.layers.64.mlp.experts.72.down_proj", + "model.layers.64.mlp.experts.72.gate_proj", + "model.layers.64.mlp.experts.72.up_proj", + "model.layers.64.mlp.experts.73.down_proj", + "model.layers.64.mlp.experts.73.gate_proj", + "model.layers.64.mlp.experts.73.up_proj", + "model.layers.64.mlp.experts.74.down_proj", + "model.layers.64.mlp.experts.74.gate_proj", + "model.layers.64.mlp.experts.74.up_proj", + "model.layers.64.mlp.experts.75.down_proj", + "model.layers.64.mlp.experts.75.gate_proj", + "model.layers.64.mlp.experts.75.up_proj", + "model.layers.64.mlp.experts.76.down_proj", + "model.layers.64.mlp.experts.76.gate_proj", + "model.layers.64.mlp.experts.76.up_proj", + "model.layers.64.mlp.experts.77.down_proj", + "model.layers.64.mlp.experts.77.gate_proj", + "model.layers.64.mlp.experts.77.up_proj", + "model.layers.64.mlp.experts.78.down_proj", + "model.layers.64.mlp.experts.78.gate_proj", + "model.layers.64.mlp.experts.78.up_proj", + "model.layers.64.mlp.experts.79.down_proj", + "model.layers.64.mlp.experts.79.gate_proj", + "model.layers.64.mlp.experts.79.up_proj", + "model.layers.64.mlp.experts.8.down_proj", + "model.layers.64.mlp.experts.8.gate_proj", + "model.layers.64.mlp.experts.8.up_proj", + "model.layers.64.mlp.experts.80.down_proj", + "model.layers.64.mlp.experts.80.gate_proj", + "model.layers.64.mlp.experts.80.up_proj", + "model.layers.64.mlp.experts.81.down_proj", + "model.layers.64.mlp.experts.81.gate_proj", + "model.layers.64.mlp.experts.81.up_proj", + "model.layers.64.mlp.experts.82.down_proj", + "model.layers.64.mlp.experts.82.gate_proj", + "model.layers.64.mlp.experts.82.up_proj", + "model.layers.64.mlp.experts.83.down_proj", + "model.layers.64.mlp.experts.83.gate_proj", + "model.layers.64.mlp.experts.83.up_proj", + "model.layers.64.mlp.experts.84.down_proj", + "model.layers.64.mlp.experts.84.gate_proj", + "model.layers.64.mlp.experts.84.up_proj", + "model.layers.64.mlp.experts.85.down_proj", + "model.layers.64.mlp.experts.85.gate_proj", + "model.layers.64.mlp.experts.85.up_proj", + "model.layers.64.mlp.experts.86.down_proj", + "model.layers.64.mlp.experts.86.gate_proj", + "model.layers.64.mlp.experts.86.up_proj", + "model.layers.64.mlp.experts.87.down_proj", + "model.layers.64.mlp.experts.87.gate_proj", + "model.layers.64.mlp.experts.87.up_proj", + "model.layers.64.mlp.experts.88.down_proj", + "model.layers.64.mlp.experts.88.gate_proj", + "model.layers.64.mlp.experts.88.up_proj", + "model.layers.64.mlp.experts.89.down_proj", + "model.layers.64.mlp.experts.89.gate_proj", + "model.layers.64.mlp.experts.89.up_proj", + "model.layers.64.mlp.experts.9.down_proj", + "model.layers.64.mlp.experts.9.gate_proj", + "model.layers.64.mlp.experts.9.up_proj", + "model.layers.64.mlp.experts.90.down_proj", + "model.layers.64.mlp.experts.90.gate_proj", + "model.layers.64.mlp.experts.90.up_proj", + "model.layers.64.mlp.experts.91.down_proj", + "model.layers.64.mlp.experts.91.gate_proj", + "model.layers.64.mlp.experts.91.up_proj", + "model.layers.64.mlp.experts.92.down_proj", + "model.layers.64.mlp.experts.92.gate_proj", + "model.layers.64.mlp.experts.92.up_proj", + "model.layers.64.mlp.experts.93.down_proj", + "model.layers.64.mlp.experts.93.gate_proj", + "model.layers.64.mlp.experts.93.up_proj", + "model.layers.64.mlp.experts.94.down_proj", + "model.layers.64.mlp.experts.94.gate_proj", + "model.layers.64.mlp.experts.94.up_proj", + "model.layers.64.mlp.experts.95.down_proj", + "model.layers.64.mlp.experts.95.gate_proj", + "model.layers.64.mlp.experts.95.up_proj", + "model.layers.64.mlp.experts.96.down_proj", + "model.layers.64.mlp.experts.96.gate_proj", + "model.layers.64.mlp.experts.96.up_proj", + "model.layers.64.mlp.experts.97.down_proj", + "model.layers.64.mlp.experts.97.gate_proj", + "model.layers.64.mlp.experts.97.up_proj", + "model.layers.64.mlp.experts.98.down_proj", + "model.layers.64.mlp.experts.98.gate_proj", + "model.layers.64.mlp.experts.98.up_proj", + "model.layers.64.mlp.experts.99.down_proj", + "model.layers.64.mlp.experts.99.gate_proj", + "model.layers.64.mlp.experts.99.up_proj", + "model.layers.65.mlp.experts.0.down_proj", + "model.layers.65.mlp.experts.0.gate_proj", + "model.layers.65.mlp.experts.0.up_proj", + "model.layers.65.mlp.experts.1.down_proj", + "model.layers.65.mlp.experts.1.gate_proj", + "model.layers.65.mlp.experts.1.up_proj", + "model.layers.65.mlp.experts.10.down_proj", + "model.layers.65.mlp.experts.10.gate_proj", + "model.layers.65.mlp.experts.10.up_proj", + "model.layers.65.mlp.experts.100.down_proj", + "model.layers.65.mlp.experts.100.gate_proj", + "model.layers.65.mlp.experts.100.up_proj", + "model.layers.65.mlp.experts.101.down_proj", + "model.layers.65.mlp.experts.101.gate_proj", + "model.layers.65.mlp.experts.101.up_proj", + "model.layers.65.mlp.experts.102.down_proj", + "model.layers.65.mlp.experts.102.gate_proj", + "model.layers.65.mlp.experts.102.up_proj", + "model.layers.65.mlp.experts.103.down_proj", + "model.layers.65.mlp.experts.103.gate_proj", + "model.layers.65.mlp.experts.103.up_proj", + "model.layers.65.mlp.experts.104.down_proj", + "model.layers.65.mlp.experts.104.gate_proj", + "model.layers.65.mlp.experts.104.up_proj", + "model.layers.65.mlp.experts.105.down_proj", + "model.layers.65.mlp.experts.105.gate_proj", + "model.layers.65.mlp.experts.105.up_proj", + "model.layers.65.mlp.experts.106.down_proj", + "model.layers.65.mlp.experts.106.gate_proj", + "model.layers.65.mlp.experts.106.up_proj", + "model.layers.65.mlp.experts.107.down_proj", + "model.layers.65.mlp.experts.107.gate_proj", + "model.layers.65.mlp.experts.107.up_proj", + "model.layers.65.mlp.experts.108.down_proj", + "model.layers.65.mlp.experts.108.gate_proj", + "model.layers.65.mlp.experts.108.up_proj", + "model.layers.65.mlp.experts.109.down_proj", + "model.layers.65.mlp.experts.109.gate_proj", + "model.layers.65.mlp.experts.109.up_proj", + "model.layers.65.mlp.experts.11.down_proj", + "model.layers.65.mlp.experts.11.gate_proj", + "model.layers.65.mlp.experts.11.up_proj", + "model.layers.65.mlp.experts.110.down_proj", + "model.layers.65.mlp.experts.110.gate_proj", + "model.layers.65.mlp.experts.110.up_proj", + "model.layers.65.mlp.experts.111.down_proj", + "model.layers.65.mlp.experts.111.gate_proj", + "model.layers.65.mlp.experts.111.up_proj", + "model.layers.65.mlp.experts.112.down_proj", + "model.layers.65.mlp.experts.112.gate_proj", + "model.layers.65.mlp.experts.112.up_proj", + "model.layers.65.mlp.experts.113.down_proj", + "model.layers.65.mlp.experts.113.gate_proj", + "model.layers.65.mlp.experts.113.up_proj", + "model.layers.65.mlp.experts.114.down_proj", + "model.layers.65.mlp.experts.114.gate_proj", + "model.layers.65.mlp.experts.114.up_proj", + "model.layers.65.mlp.experts.115.down_proj", + "model.layers.65.mlp.experts.115.gate_proj", + "model.layers.65.mlp.experts.115.up_proj", + "model.layers.65.mlp.experts.116.down_proj", + "model.layers.65.mlp.experts.116.gate_proj", + "model.layers.65.mlp.experts.116.up_proj", + "model.layers.65.mlp.experts.117.down_proj", + "model.layers.65.mlp.experts.117.gate_proj", + "model.layers.65.mlp.experts.117.up_proj", + "model.layers.65.mlp.experts.118.down_proj", + "model.layers.65.mlp.experts.118.gate_proj", + "model.layers.65.mlp.experts.118.up_proj", + "model.layers.65.mlp.experts.119.down_proj", + "model.layers.65.mlp.experts.119.gate_proj", + "model.layers.65.mlp.experts.119.up_proj", + "model.layers.65.mlp.experts.12.down_proj", + "model.layers.65.mlp.experts.12.gate_proj", + "model.layers.65.mlp.experts.12.up_proj", + "model.layers.65.mlp.experts.120.down_proj", + "model.layers.65.mlp.experts.120.gate_proj", + "model.layers.65.mlp.experts.120.up_proj", + "model.layers.65.mlp.experts.121.down_proj", + "model.layers.65.mlp.experts.121.gate_proj", + "model.layers.65.mlp.experts.121.up_proj", + "model.layers.65.mlp.experts.122.down_proj", + "model.layers.65.mlp.experts.122.gate_proj", + "model.layers.65.mlp.experts.122.up_proj", + "model.layers.65.mlp.experts.123.down_proj", + "model.layers.65.mlp.experts.123.gate_proj", + "model.layers.65.mlp.experts.123.up_proj", + "model.layers.65.mlp.experts.124.down_proj", + "model.layers.65.mlp.experts.124.gate_proj", + "model.layers.65.mlp.experts.124.up_proj", + "model.layers.65.mlp.experts.125.down_proj", + "model.layers.65.mlp.experts.125.gate_proj", + "model.layers.65.mlp.experts.125.up_proj", + "model.layers.65.mlp.experts.126.down_proj", + "model.layers.65.mlp.experts.126.gate_proj", + "model.layers.65.mlp.experts.126.up_proj", + "model.layers.65.mlp.experts.127.down_proj", + "model.layers.65.mlp.experts.127.gate_proj", + "model.layers.65.mlp.experts.127.up_proj", + "model.layers.65.mlp.experts.128.down_proj", + "model.layers.65.mlp.experts.128.gate_proj", + "model.layers.65.mlp.experts.128.up_proj", + "model.layers.65.mlp.experts.129.down_proj", + "model.layers.65.mlp.experts.129.gate_proj", + "model.layers.65.mlp.experts.129.up_proj", + "model.layers.65.mlp.experts.13.down_proj", + "model.layers.65.mlp.experts.13.gate_proj", + "model.layers.65.mlp.experts.13.up_proj", + "model.layers.65.mlp.experts.130.down_proj", + "model.layers.65.mlp.experts.130.gate_proj", + "model.layers.65.mlp.experts.130.up_proj", + "model.layers.65.mlp.experts.131.down_proj", + "model.layers.65.mlp.experts.131.gate_proj", + "model.layers.65.mlp.experts.131.up_proj", + "model.layers.65.mlp.experts.132.down_proj", + "model.layers.65.mlp.experts.132.gate_proj", + "model.layers.65.mlp.experts.132.up_proj", + "model.layers.65.mlp.experts.133.down_proj", + "model.layers.65.mlp.experts.133.gate_proj", + "model.layers.65.mlp.experts.133.up_proj", + "model.layers.65.mlp.experts.134.down_proj", + "model.layers.65.mlp.experts.134.gate_proj", + "model.layers.65.mlp.experts.134.up_proj", + "model.layers.65.mlp.experts.135.down_proj", + "model.layers.65.mlp.experts.135.gate_proj", + "model.layers.65.mlp.experts.135.up_proj", + "model.layers.65.mlp.experts.136.down_proj", + "model.layers.65.mlp.experts.136.gate_proj", + "model.layers.65.mlp.experts.136.up_proj", + "model.layers.65.mlp.experts.137.down_proj", + "model.layers.65.mlp.experts.137.gate_proj", + "model.layers.65.mlp.experts.137.up_proj", + "model.layers.65.mlp.experts.138.down_proj", + "model.layers.65.mlp.experts.138.gate_proj", + "model.layers.65.mlp.experts.138.up_proj", + "model.layers.65.mlp.experts.139.down_proj", + "model.layers.65.mlp.experts.139.gate_proj", + "model.layers.65.mlp.experts.139.up_proj", + "model.layers.65.mlp.experts.14.down_proj", + "model.layers.65.mlp.experts.14.gate_proj", + "model.layers.65.mlp.experts.14.up_proj", + "model.layers.65.mlp.experts.140.down_proj", + "model.layers.65.mlp.experts.140.gate_proj", + "model.layers.65.mlp.experts.140.up_proj", + "model.layers.65.mlp.experts.141.down_proj", + "model.layers.65.mlp.experts.141.gate_proj", + "model.layers.65.mlp.experts.141.up_proj", + "model.layers.65.mlp.experts.142.down_proj", + "model.layers.65.mlp.experts.142.gate_proj", + "model.layers.65.mlp.experts.142.up_proj", + "model.layers.65.mlp.experts.143.down_proj", + "model.layers.65.mlp.experts.143.gate_proj", + "model.layers.65.mlp.experts.143.up_proj", + "model.layers.65.mlp.experts.144.down_proj", + "model.layers.65.mlp.experts.144.gate_proj", + "model.layers.65.mlp.experts.144.up_proj", + "model.layers.65.mlp.experts.145.down_proj", + "model.layers.65.mlp.experts.145.gate_proj", + "model.layers.65.mlp.experts.145.up_proj", + "model.layers.65.mlp.experts.146.down_proj", + "model.layers.65.mlp.experts.146.gate_proj", + "model.layers.65.mlp.experts.146.up_proj", + "model.layers.65.mlp.experts.147.down_proj", + "model.layers.65.mlp.experts.147.gate_proj", + "model.layers.65.mlp.experts.147.up_proj", + "model.layers.65.mlp.experts.148.down_proj", + "model.layers.65.mlp.experts.148.gate_proj", + "model.layers.65.mlp.experts.148.up_proj", + "model.layers.65.mlp.experts.149.down_proj", + "model.layers.65.mlp.experts.149.gate_proj", + "model.layers.65.mlp.experts.149.up_proj", + "model.layers.65.mlp.experts.15.down_proj", + "model.layers.65.mlp.experts.15.gate_proj", + "model.layers.65.mlp.experts.15.up_proj", + "model.layers.65.mlp.experts.150.down_proj", + "model.layers.65.mlp.experts.150.gate_proj", + "model.layers.65.mlp.experts.150.up_proj", + "model.layers.65.mlp.experts.151.down_proj", + "model.layers.65.mlp.experts.151.gate_proj", + "model.layers.65.mlp.experts.151.up_proj", + "model.layers.65.mlp.experts.152.down_proj", + "model.layers.65.mlp.experts.152.gate_proj", + "model.layers.65.mlp.experts.152.up_proj", + "model.layers.65.mlp.experts.153.down_proj", + "model.layers.65.mlp.experts.153.gate_proj", + "model.layers.65.mlp.experts.153.up_proj", + "model.layers.65.mlp.experts.154.down_proj", + "model.layers.65.mlp.experts.154.gate_proj", + "model.layers.65.mlp.experts.154.up_proj", + "model.layers.65.mlp.experts.155.down_proj", + "model.layers.65.mlp.experts.155.gate_proj", + "model.layers.65.mlp.experts.155.up_proj", + "model.layers.65.mlp.experts.156.down_proj", + "model.layers.65.mlp.experts.156.gate_proj", + "model.layers.65.mlp.experts.156.up_proj", + "model.layers.65.mlp.experts.157.down_proj", + "model.layers.65.mlp.experts.157.gate_proj", + "model.layers.65.mlp.experts.157.up_proj", + "model.layers.65.mlp.experts.158.down_proj", + "model.layers.65.mlp.experts.158.gate_proj", + "model.layers.65.mlp.experts.158.up_proj", + "model.layers.65.mlp.experts.159.down_proj", + "model.layers.65.mlp.experts.159.gate_proj", + "model.layers.65.mlp.experts.159.up_proj", + "model.layers.65.mlp.experts.16.down_proj", + "model.layers.65.mlp.experts.16.gate_proj", + "model.layers.65.mlp.experts.16.up_proj", + "model.layers.65.mlp.experts.160.down_proj", + "model.layers.65.mlp.experts.160.gate_proj", + "model.layers.65.mlp.experts.160.up_proj", + "model.layers.65.mlp.experts.161.down_proj", + "model.layers.65.mlp.experts.161.gate_proj", + "model.layers.65.mlp.experts.161.up_proj", + "model.layers.65.mlp.experts.162.down_proj", + "model.layers.65.mlp.experts.162.gate_proj", + "model.layers.65.mlp.experts.162.up_proj", + "model.layers.65.mlp.experts.163.down_proj", + "model.layers.65.mlp.experts.163.gate_proj", + "model.layers.65.mlp.experts.163.up_proj", + "model.layers.65.mlp.experts.164.down_proj", + "model.layers.65.mlp.experts.164.gate_proj", + "model.layers.65.mlp.experts.164.up_proj", + "model.layers.65.mlp.experts.165.down_proj", + "model.layers.65.mlp.experts.165.gate_proj", + "model.layers.65.mlp.experts.165.up_proj", + "model.layers.65.mlp.experts.166.down_proj", + "model.layers.65.mlp.experts.166.gate_proj", + "model.layers.65.mlp.experts.166.up_proj", + "model.layers.65.mlp.experts.167.down_proj", + "model.layers.65.mlp.experts.167.gate_proj", + "model.layers.65.mlp.experts.167.up_proj", + "model.layers.65.mlp.experts.168.down_proj", + "model.layers.65.mlp.experts.168.gate_proj", + "model.layers.65.mlp.experts.168.up_proj", + "model.layers.65.mlp.experts.169.down_proj", + "model.layers.65.mlp.experts.169.gate_proj", + "model.layers.65.mlp.experts.169.up_proj", + "model.layers.65.mlp.experts.17.down_proj", + "model.layers.65.mlp.experts.17.gate_proj", + "model.layers.65.mlp.experts.17.up_proj", + "model.layers.65.mlp.experts.170.down_proj", + "model.layers.65.mlp.experts.170.gate_proj", + "model.layers.65.mlp.experts.170.up_proj", + "model.layers.65.mlp.experts.171.down_proj", + "model.layers.65.mlp.experts.171.gate_proj", + "model.layers.65.mlp.experts.171.up_proj", + "model.layers.65.mlp.experts.172.down_proj", + "model.layers.65.mlp.experts.172.gate_proj", + "model.layers.65.mlp.experts.172.up_proj", + "model.layers.65.mlp.experts.173.down_proj", + "model.layers.65.mlp.experts.173.gate_proj", + "model.layers.65.mlp.experts.173.up_proj", + "model.layers.65.mlp.experts.174.down_proj", + "model.layers.65.mlp.experts.174.gate_proj", + "model.layers.65.mlp.experts.174.up_proj", + "model.layers.65.mlp.experts.175.down_proj", + "model.layers.65.mlp.experts.175.gate_proj", + "model.layers.65.mlp.experts.175.up_proj", + "model.layers.65.mlp.experts.176.down_proj", + "model.layers.65.mlp.experts.176.gate_proj", + "model.layers.65.mlp.experts.176.up_proj", + "model.layers.65.mlp.experts.177.down_proj", + "model.layers.65.mlp.experts.177.gate_proj", + "model.layers.65.mlp.experts.177.up_proj", + "model.layers.65.mlp.experts.178.down_proj", + "model.layers.65.mlp.experts.178.gate_proj", + "model.layers.65.mlp.experts.178.up_proj", + "model.layers.65.mlp.experts.179.down_proj", + "model.layers.65.mlp.experts.179.gate_proj", + "model.layers.65.mlp.experts.179.up_proj", + "model.layers.65.mlp.experts.18.down_proj", + "model.layers.65.mlp.experts.18.gate_proj", + "model.layers.65.mlp.experts.18.up_proj", + "model.layers.65.mlp.experts.180.down_proj", + "model.layers.65.mlp.experts.180.gate_proj", + "model.layers.65.mlp.experts.180.up_proj", + "model.layers.65.mlp.experts.181.down_proj", + "model.layers.65.mlp.experts.181.gate_proj", + "model.layers.65.mlp.experts.181.up_proj", + "model.layers.65.mlp.experts.182.down_proj", + "model.layers.65.mlp.experts.182.gate_proj", + "model.layers.65.mlp.experts.182.up_proj", + "model.layers.65.mlp.experts.183.down_proj", + "model.layers.65.mlp.experts.183.gate_proj", + "model.layers.65.mlp.experts.183.up_proj", + "model.layers.65.mlp.experts.184.down_proj", + "model.layers.65.mlp.experts.184.gate_proj", + "model.layers.65.mlp.experts.184.up_proj", + "model.layers.65.mlp.experts.185.down_proj", + "model.layers.65.mlp.experts.185.gate_proj", + "model.layers.65.mlp.experts.185.up_proj", + "model.layers.65.mlp.experts.186.down_proj", + "model.layers.65.mlp.experts.186.gate_proj", + "model.layers.65.mlp.experts.186.up_proj", + "model.layers.65.mlp.experts.187.down_proj", + "model.layers.65.mlp.experts.187.gate_proj", + "model.layers.65.mlp.experts.187.up_proj", + "model.layers.65.mlp.experts.188.down_proj", + "model.layers.65.mlp.experts.188.gate_proj", + "model.layers.65.mlp.experts.188.up_proj", + "model.layers.65.mlp.experts.189.down_proj", + "model.layers.65.mlp.experts.189.gate_proj", + "model.layers.65.mlp.experts.189.up_proj", + "model.layers.65.mlp.experts.19.down_proj", + "model.layers.65.mlp.experts.19.gate_proj", + "model.layers.65.mlp.experts.19.up_proj", + "model.layers.65.mlp.experts.190.down_proj", + "model.layers.65.mlp.experts.190.gate_proj", + "model.layers.65.mlp.experts.190.up_proj", + "model.layers.65.mlp.experts.191.down_proj", + "model.layers.65.mlp.experts.191.gate_proj", + "model.layers.65.mlp.experts.191.up_proj", + "model.layers.65.mlp.experts.192.down_proj", + "model.layers.65.mlp.experts.192.gate_proj", + "model.layers.65.mlp.experts.192.up_proj", + "model.layers.65.mlp.experts.193.down_proj", + "model.layers.65.mlp.experts.193.gate_proj", + "model.layers.65.mlp.experts.193.up_proj", + "model.layers.65.mlp.experts.194.down_proj", + "model.layers.65.mlp.experts.194.gate_proj", + "model.layers.65.mlp.experts.194.up_proj", + "model.layers.65.mlp.experts.195.down_proj", + "model.layers.65.mlp.experts.195.gate_proj", + "model.layers.65.mlp.experts.195.up_proj", + "model.layers.65.mlp.experts.196.down_proj", + "model.layers.65.mlp.experts.196.gate_proj", + "model.layers.65.mlp.experts.196.up_proj", + "model.layers.65.mlp.experts.197.down_proj", + "model.layers.65.mlp.experts.197.gate_proj", + "model.layers.65.mlp.experts.197.up_proj", + "model.layers.65.mlp.experts.198.down_proj", + "model.layers.65.mlp.experts.198.gate_proj", + "model.layers.65.mlp.experts.198.up_proj", + "model.layers.65.mlp.experts.199.down_proj", + "model.layers.65.mlp.experts.199.gate_proj", + "model.layers.65.mlp.experts.199.up_proj", + "model.layers.65.mlp.experts.2.down_proj", + "model.layers.65.mlp.experts.2.gate_proj", + "model.layers.65.mlp.experts.2.up_proj", + "model.layers.65.mlp.experts.20.down_proj", + "model.layers.65.mlp.experts.20.gate_proj", + "model.layers.65.mlp.experts.20.up_proj", + "model.layers.65.mlp.experts.200.down_proj", + "model.layers.65.mlp.experts.200.gate_proj", + "model.layers.65.mlp.experts.200.up_proj", + "model.layers.65.mlp.experts.201.down_proj", + "model.layers.65.mlp.experts.201.gate_proj", + "model.layers.65.mlp.experts.201.up_proj", + "model.layers.65.mlp.experts.202.down_proj", + "model.layers.65.mlp.experts.202.gate_proj", + "model.layers.65.mlp.experts.202.up_proj", + "model.layers.65.mlp.experts.203.down_proj", + "model.layers.65.mlp.experts.203.gate_proj", + "model.layers.65.mlp.experts.203.up_proj", + "model.layers.65.mlp.experts.204.down_proj", + "model.layers.65.mlp.experts.204.gate_proj", + "model.layers.65.mlp.experts.204.up_proj", + "model.layers.65.mlp.experts.205.down_proj", + "model.layers.65.mlp.experts.205.gate_proj", + "model.layers.65.mlp.experts.205.up_proj", + "model.layers.65.mlp.experts.206.down_proj", + "model.layers.65.mlp.experts.206.gate_proj", + "model.layers.65.mlp.experts.206.up_proj", + "model.layers.65.mlp.experts.207.down_proj", + "model.layers.65.mlp.experts.207.gate_proj", + "model.layers.65.mlp.experts.207.up_proj", + "model.layers.65.mlp.experts.208.down_proj", + "model.layers.65.mlp.experts.208.gate_proj", + "model.layers.65.mlp.experts.208.up_proj", + "model.layers.65.mlp.experts.209.down_proj", + "model.layers.65.mlp.experts.209.gate_proj", + "model.layers.65.mlp.experts.209.up_proj", + "model.layers.65.mlp.experts.21.down_proj", + "model.layers.65.mlp.experts.21.gate_proj", + "model.layers.65.mlp.experts.21.up_proj", + "model.layers.65.mlp.experts.210.down_proj", + "model.layers.65.mlp.experts.210.gate_proj", + "model.layers.65.mlp.experts.210.up_proj", + "model.layers.65.mlp.experts.211.down_proj", + "model.layers.65.mlp.experts.211.gate_proj", + "model.layers.65.mlp.experts.211.up_proj", + "model.layers.65.mlp.experts.212.down_proj", + "model.layers.65.mlp.experts.212.gate_proj", + "model.layers.65.mlp.experts.212.up_proj", + "model.layers.65.mlp.experts.213.down_proj", + "model.layers.65.mlp.experts.213.gate_proj", + "model.layers.65.mlp.experts.213.up_proj", + "model.layers.65.mlp.experts.214.down_proj", + "model.layers.65.mlp.experts.214.gate_proj", + "model.layers.65.mlp.experts.214.up_proj", + "model.layers.65.mlp.experts.215.down_proj", + "model.layers.65.mlp.experts.215.gate_proj", + "model.layers.65.mlp.experts.215.up_proj", + "model.layers.65.mlp.experts.216.down_proj", + "model.layers.65.mlp.experts.216.gate_proj", + "model.layers.65.mlp.experts.216.up_proj", + "model.layers.65.mlp.experts.217.down_proj", + "model.layers.65.mlp.experts.217.gate_proj", + "model.layers.65.mlp.experts.217.up_proj", + "model.layers.65.mlp.experts.218.down_proj", + "model.layers.65.mlp.experts.218.gate_proj", + "model.layers.65.mlp.experts.218.up_proj", + "model.layers.65.mlp.experts.219.down_proj", + "model.layers.65.mlp.experts.219.gate_proj", + "model.layers.65.mlp.experts.219.up_proj", + "model.layers.65.mlp.experts.22.down_proj", + "model.layers.65.mlp.experts.22.gate_proj", + "model.layers.65.mlp.experts.22.up_proj", + "model.layers.65.mlp.experts.220.down_proj", + "model.layers.65.mlp.experts.220.gate_proj", + "model.layers.65.mlp.experts.220.up_proj", + "model.layers.65.mlp.experts.221.down_proj", + "model.layers.65.mlp.experts.221.gate_proj", + "model.layers.65.mlp.experts.221.up_proj", + "model.layers.65.mlp.experts.222.down_proj", + "model.layers.65.mlp.experts.222.gate_proj", + "model.layers.65.mlp.experts.222.up_proj", + "model.layers.65.mlp.experts.223.down_proj", + "model.layers.65.mlp.experts.223.gate_proj", + "model.layers.65.mlp.experts.223.up_proj", + "model.layers.65.mlp.experts.224.down_proj", + "model.layers.65.mlp.experts.224.gate_proj", + "model.layers.65.mlp.experts.224.up_proj", + "model.layers.65.mlp.experts.225.down_proj", + "model.layers.65.mlp.experts.225.gate_proj", + "model.layers.65.mlp.experts.225.up_proj", + "model.layers.65.mlp.experts.226.down_proj", + "model.layers.65.mlp.experts.226.gate_proj", + "model.layers.65.mlp.experts.226.up_proj", + "model.layers.65.mlp.experts.227.down_proj", + "model.layers.65.mlp.experts.227.gate_proj", + "model.layers.65.mlp.experts.227.up_proj", + "model.layers.65.mlp.experts.228.down_proj", + "model.layers.65.mlp.experts.228.gate_proj", + "model.layers.65.mlp.experts.228.up_proj", + "model.layers.65.mlp.experts.229.down_proj", + "model.layers.65.mlp.experts.229.gate_proj", + "model.layers.65.mlp.experts.229.up_proj", + "model.layers.65.mlp.experts.23.down_proj", + "model.layers.65.mlp.experts.23.gate_proj", + "model.layers.65.mlp.experts.23.up_proj", + "model.layers.65.mlp.experts.230.down_proj", + "model.layers.65.mlp.experts.230.gate_proj", + "model.layers.65.mlp.experts.230.up_proj", + "model.layers.65.mlp.experts.231.down_proj", + "model.layers.65.mlp.experts.231.gate_proj", + "model.layers.65.mlp.experts.231.up_proj", + "model.layers.65.mlp.experts.232.down_proj", + "model.layers.65.mlp.experts.232.gate_proj", + "model.layers.65.mlp.experts.232.up_proj", + "model.layers.65.mlp.experts.233.down_proj", + "model.layers.65.mlp.experts.233.gate_proj", + "model.layers.65.mlp.experts.233.up_proj", + "model.layers.65.mlp.experts.234.down_proj", + "model.layers.65.mlp.experts.234.gate_proj", + "model.layers.65.mlp.experts.234.up_proj", + "model.layers.65.mlp.experts.235.down_proj", + "model.layers.65.mlp.experts.235.gate_proj", + "model.layers.65.mlp.experts.235.up_proj", + "model.layers.65.mlp.experts.236.down_proj", + "model.layers.65.mlp.experts.236.gate_proj", + "model.layers.65.mlp.experts.236.up_proj", + "model.layers.65.mlp.experts.237.down_proj", + "model.layers.65.mlp.experts.237.gate_proj", + "model.layers.65.mlp.experts.237.up_proj", + "model.layers.65.mlp.experts.238.down_proj", + "model.layers.65.mlp.experts.238.gate_proj", + "model.layers.65.mlp.experts.238.up_proj", + "model.layers.65.mlp.experts.239.down_proj", + "model.layers.65.mlp.experts.239.gate_proj", + "model.layers.65.mlp.experts.239.up_proj", + "model.layers.65.mlp.experts.24.down_proj", + "model.layers.65.mlp.experts.24.gate_proj", + "model.layers.65.mlp.experts.24.up_proj", + "model.layers.65.mlp.experts.240.down_proj", + "model.layers.65.mlp.experts.240.gate_proj", + "model.layers.65.mlp.experts.240.up_proj", + "model.layers.65.mlp.experts.241.down_proj", + "model.layers.65.mlp.experts.241.gate_proj", + "model.layers.65.mlp.experts.241.up_proj", + "model.layers.65.mlp.experts.242.down_proj", + "model.layers.65.mlp.experts.242.gate_proj", + "model.layers.65.mlp.experts.242.up_proj", + "model.layers.65.mlp.experts.243.down_proj", + "model.layers.65.mlp.experts.243.gate_proj", + "model.layers.65.mlp.experts.243.up_proj", + "model.layers.65.mlp.experts.244.down_proj", + "model.layers.65.mlp.experts.244.gate_proj", + "model.layers.65.mlp.experts.244.up_proj", + "model.layers.65.mlp.experts.245.down_proj", + "model.layers.65.mlp.experts.245.gate_proj", + "model.layers.65.mlp.experts.245.up_proj", + "model.layers.65.mlp.experts.246.down_proj", + "model.layers.65.mlp.experts.246.gate_proj", + "model.layers.65.mlp.experts.246.up_proj", + "model.layers.65.mlp.experts.247.down_proj", + "model.layers.65.mlp.experts.247.gate_proj", + "model.layers.65.mlp.experts.247.up_proj", + "model.layers.65.mlp.experts.248.down_proj", + "model.layers.65.mlp.experts.248.gate_proj", + "model.layers.65.mlp.experts.248.up_proj", + "model.layers.65.mlp.experts.249.down_proj", + "model.layers.65.mlp.experts.249.gate_proj", + "model.layers.65.mlp.experts.249.up_proj", + "model.layers.65.mlp.experts.25.down_proj", + "model.layers.65.mlp.experts.25.gate_proj", + "model.layers.65.mlp.experts.25.up_proj", + "model.layers.65.mlp.experts.250.down_proj", + "model.layers.65.mlp.experts.250.gate_proj", + "model.layers.65.mlp.experts.250.up_proj", + "model.layers.65.mlp.experts.251.down_proj", + "model.layers.65.mlp.experts.251.gate_proj", + "model.layers.65.mlp.experts.251.up_proj", + "model.layers.65.mlp.experts.252.down_proj", + "model.layers.65.mlp.experts.252.gate_proj", + "model.layers.65.mlp.experts.252.up_proj", + "model.layers.65.mlp.experts.253.down_proj", + "model.layers.65.mlp.experts.253.gate_proj", + "model.layers.65.mlp.experts.253.up_proj", + "model.layers.65.mlp.experts.254.down_proj", + "model.layers.65.mlp.experts.254.gate_proj", + "model.layers.65.mlp.experts.254.up_proj", + "model.layers.65.mlp.experts.255.down_proj", + "model.layers.65.mlp.experts.255.gate_proj", + "model.layers.65.mlp.experts.255.up_proj", + "model.layers.65.mlp.experts.26.down_proj", + "model.layers.65.mlp.experts.26.gate_proj", + "model.layers.65.mlp.experts.26.up_proj", + "model.layers.65.mlp.experts.27.down_proj", + "model.layers.65.mlp.experts.27.gate_proj", + "model.layers.65.mlp.experts.27.up_proj", + "model.layers.65.mlp.experts.28.down_proj", + "model.layers.65.mlp.experts.28.gate_proj", + "model.layers.65.mlp.experts.28.up_proj", + "model.layers.65.mlp.experts.29.down_proj", + "model.layers.65.mlp.experts.29.gate_proj", + "model.layers.65.mlp.experts.29.up_proj", + "model.layers.65.mlp.experts.3.down_proj", + "model.layers.65.mlp.experts.3.gate_proj", + "model.layers.65.mlp.experts.3.up_proj", + "model.layers.65.mlp.experts.30.down_proj", + "model.layers.65.mlp.experts.30.gate_proj", + "model.layers.65.mlp.experts.30.up_proj", + "model.layers.65.mlp.experts.31.down_proj", + "model.layers.65.mlp.experts.31.gate_proj", + "model.layers.65.mlp.experts.31.up_proj", + "model.layers.65.mlp.experts.32.down_proj", + "model.layers.65.mlp.experts.32.gate_proj", + "model.layers.65.mlp.experts.32.up_proj", + "model.layers.65.mlp.experts.33.down_proj", + "model.layers.65.mlp.experts.33.gate_proj", + "model.layers.65.mlp.experts.33.up_proj", + "model.layers.65.mlp.experts.34.down_proj", + "model.layers.65.mlp.experts.34.gate_proj", + "model.layers.65.mlp.experts.34.up_proj", + "model.layers.65.mlp.experts.35.down_proj", + "model.layers.65.mlp.experts.35.gate_proj", + "model.layers.65.mlp.experts.35.up_proj", + "model.layers.65.mlp.experts.36.down_proj", + "model.layers.65.mlp.experts.36.gate_proj", + "model.layers.65.mlp.experts.36.up_proj", + "model.layers.65.mlp.experts.37.down_proj", + "model.layers.65.mlp.experts.37.gate_proj", + "model.layers.65.mlp.experts.37.up_proj", + "model.layers.65.mlp.experts.38.down_proj", + "model.layers.65.mlp.experts.38.gate_proj", + "model.layers.65.mlp.experts.38.up_proj", + "model.layers.65.mlp.experts.39.down_proj", + "model.layers.65.mlp.experts.39.gate_proj", + "model.layers.65.mlp.experts.39.up_proj", + "model.layers.65.mlp.experts.4.down_proj", + "model.layers.65.mlp.experts.4.gate_proj", + "model.layers.65.mlp.experts.4.up_proj", + "model.layers.65.mlp.experts.40.down_proj", + "model.layers.65.mlp.experts.40.gate_proj", + "model.layers.65.mlp.experts.40.up_proj", + "model.layers.65.mlp.experts.41.down_proj", + "model.layers.65.mlp.experts.41.gate_proj", + "model.layers.65.mlp.experts.41.up_proj", + "model.layers.65.mlp.experts.42.down_proj", + "model.layers.65.mlp.experts.42.gate_proj", + "model.layers.65.mlp.experts.42.up_proj", + "model.layers.65.mlp.experts.43.down_proj", + "model.layers.65.mlp.experts.43.gate_proj", + "model.layers.65.mlp.experts.43.up_proj", + "model.layers.65.mlp.experts.44.down_proj", + "model.layers.65.mlp.experts.44.gate_proj", + "model.layers.65.mlp.experts.44.up_proj", + "model.layers.65.mlp.experts.45.down_proj", + "model.layers.65.mlp.experts.45.gate_proj", + "model.layers.65.mlp.experts.45.up_proj", + "model.layers.65.mlp.experts.46.down_proj", + "model.layers.65.mlp.experts.46.gate_proj", + "model.layers.65.mlp.experts.46.up_proj", + "model.layers.65.mlp.experts.47.down_proj", + "model.layers.65.mlp.experts.47.gate_proj", + "model.layers.65.mlp.experts.47.up_proj", + "model.layers.65.mlp.experts.48.down_proj", + "model.layers.65.mlp.experts.48.gate_proj", + "model.layers.65.mlp.experts.48.up_proj", + "model.layers.65.mlp.experts.49.down_proj", + "model.layers.65.mlp.experts.49.gate_proj", + "model.layers.65.mlp.experts.49.up_proj", + "model.layers.65.mlp.experts.5.down_proj", + "model.layers.65.mlp.experts.5.gate_proj", + "model.layers.65.mlp.experts.5.up_proj", + "model.layers.65.mlp.experts.50.down_proj", + "model.layers.65.mlp.experts.50.gate_proj", + "model.layers.65.mlp.experts.50.up_proj", + "model.layers.65.mlp.experts.51.down_proj", + "model.layers.65.mlp.experts.51.gate_proj", + "model.layers.65.mlp.experts.51.up_proj", + "model.layers.65.mlp.experts.52.down_proj", + "model.layers.65.mlp.experts.52.gate_proj", + "model.layers.65.mlp.experts.52.up_proj", + "model.layers.65.mlp.experts.53.down_proj", + "model.layers.65.mlp.experts.53.gate_proj", + "model.layers.65.mlp.experts.53.up_proj", + "model.layers.65.mlp.experts.54.down_proj", + "model.layers.65.mlp.experts.54.gate_proj", + "model.layers.65.mlp.experts.54.up_proj", + "model.layers.65.mlp.experts.55.down_proj", + "model.layers.65.mlp.experts.55.gate_proj", + "model.layers.65.mlp.experts.55.up_proj", + "model.layers.65.mlp.experts.56.down_proj", + "model.layers.65.mlp.experts.56.gate_proj", + "model.layers.65.mlp.experts.56.up_proj", + "model.layers.65.mlp.experts.57.down_proj", + "model.layers.65.mlp.experts.57.gate_proj", + "model.layers.65.mlp.experts.57.up_proj", + "model.layers.65.mlp.experts.58.down_proj", + "model.layers.65.mlp.experts.58.gate_proj", + "model.layers.65.mlp.experts.58.up_proj", + "model.layers.65.mlp.experts.59.down_proj", + "model.layers.65.mlp.experts.59.gate_proj", + "model.layers.65.mlp.experts.59.up_proj", + "model.layers.65.mlp.experts.6.down_proj", + "model.layers.65.mlp.experts.6.gate_proj", + "model.layers.65.mlp.experts.6.up_proj", + "model.layers.65.mlp.experts.60.down_proj", + "model.layers.65.mlp.experts.60.gate_proj", + "model.layers.65.mlp.experts.60.up_proj", + "model.layers.65.mlp.experts.61.down_proj", + "model.layers.65.mlp.experts.61.gate_proj", + "model.layers.65.mlp.experts.61.up_proj", + "model.layers.65.mlp.experts.62.down_proj", + "model.layers.65.mlp.experts.62.gate_proj", + "model.layers.65.mlp.experts.62.up_proj", + "model.layers.65.mlp.experts.63.down_proj", + "model.layers.65.mlp.experts.63.gate_proj", + "model.layers.65.mlp.experts.63.up_proj", + "model.layers.65.mlp.experts.64.down_proj", + "model.layers.65.mlp.experts.64.gate_proj", + "model.layers.65.mlp.experts.64.up_proj", + "model.layers.65.mlp.experts.65.down_proj", + "model.layers.65.mlp.experts.65.gate_proj", + "model.layers.65.mlp.experts.65.up_proj", + "model.layers.65.mlp.experts.66.down_proj", + "model.layers.65.mlp.experts.66.gate_proj", + "model.layers.65.mlp.experts.66.up_proj", + "model.layers.65.mlp.experts.67.down_proj", + "model.layers.65.mlp.experts.67.gate_proj", + "model.layers.65.mlp.experts.67.up_proj", + "model.layers.65.mlp.experts.68.down_proj", + "model.layers.65.mlp.experts.68.gate_proj", + "model.layers.65.mlp.experts.68.up_proj", + "model.layers.65.mlp.experts.69.down_proj", + "model.layers.65.mlp.experts.69.gate_proj", + "model.layers.65.mlp.experts.69.up_proj", + "model.layers.65.mlp.experts.7.down_proj", + "model.layers.65.mlp.experts.7.gate_proj", + "model.layers.65.mlp.experts.7.up_proj", + "model.layers.65.mlp.experts.70.down_proj", + "model.layers.65.mlp.experts.70.gate_proj", + "model.layers.65.mlp.experts.70.up_proj", + "model.layers.65.mlp.experts.71.down_proj", + "model.layers.65.mlp.experts.71.gate_proj", + "model.layers.65.mlp.experts.71.up_proj", + "model.layers.65.mlp.experts.72.down_proj", + "model.layers.65.mlp.experts.72.gate_proj", + "model.layers.65.mlp.experts.72.up_proj", + "model.layers.65.mlp.experts.73.down_proj", + "model.layers.65.mlp.experts.73.gate_proj", + "model.layers.65.mlp.experts.73.up_proj", + "model.layers.65.mlp.experts.74.down_proj", + "model.layers.65.mlp.experts.74.gate_proj", + "model.layers.65.mlp.experts.74.up_proj", + "model.layers.65.mlp.experts.75.down_proj", + "model.layers.65.mlp.experts.75.gate_proj", + "model.layers.65.mlp.experts.75.up_proj", + "model.layers.65.mlp.experts.76.down_proj", + "model.layers.65.mlp.experts.76.gate_proj", + "model.layers.65.mlp.experts.76.up_proj", + "model.layers.65.mlp.experts.77.down_proj", + "model.layers.65.mlp.experts.77.gate_proj", + "model.layers.65.mlp.experts.77.up_proj", + "model.layers.65.mlp.experts.78.down_proj", + "model.layers.65.mlp.experts.78.gate_proj", + "model.layers.65.mlp.experts.78.up_proj", + "model.layers.65.mlp.experts.79.down_proj", + "model.layers.65.mlp.experts.79.gate_proj", + "model.layers.65.mlp.experts.79.up_proj", + "model.layers.65.mlp.experts.8.down_proj", + "model.layers.65.mlp.experts.8.gate_proj", + "model.layers.65.mlp.experts.8.up_proj", + "model.layers.65.mlp.experts.80.down_proj", + "model.layers.65.mlp.experts.80.gate_proj", + "model.layers.65.mlp.experts.80.up_proj", + "model.layers.65.mlp.experts.81.down_proj", + "model.layers.65.mlp.experts.81.gate_proj", + "model.layers.65.mlp.experts.81.up_proj", + "model.layers.65.mlp.experts.82.down_proj", + "model.layers.65.mlp.experts.82.gate_proj", + "model.layers.65.mlp.experts.82.up_proj", + "model.layers.65.mlp.experts.83.down_proj", + "model.layers.65.mlp.experts.83.gate_proj", + "model.layers.65.mlp.experts.83.up_proj", + "model.layers.65.mlp.experts.84.down_proj", + "model.layers.65.mlp.experts.84.gate_proj", + "model.layers.65.mlp.experts.84.up_proj", + "model.layers.65.mlp.experts.85.down_proj", + "model.layers.65.mlp.experts.85.gate_proj", + "model.layers.65.mlp.experts.85.up_proj", + "model.layers.65.mlp.experts.86.down_proj", + "model.layers.65.mlp.experts.86.gate_proj", + "model.layers.65.mlp.experts.86.up_proj", + "model.layers.65.mlp.experts.87.down_proj", + "model.layers.65.mlp.experts.87.gate_proj", + "model.layers.65.mlp.experts.87.up_proj", + "model.layers.65.mlp.experts.88.down_proj", + "model.layers.65.mlp.experts.88.gate_proj", + "model.layers.65.mlp.experts.88.up_proj", + "model.layers.65.mlp.experts.89.down_proj", + "model.layers.65.mlp.experts.89.gate_proj", + "model.layers.65.mlp.experts.89.up_proj", + "model.layers.65.mlp.experts.9.down_proj", + "model.layers.65.mlp.experts.9.gate_proj", + "model.layers.65.mlp.experts.9.up_proj", + "model.layers.65.mlp.experts.90.down_proj", + "model.layers.65.mlp.experts.90.gate_proj", + "model.layers.65.mlp.experts.90.up_proj", + "model.layers.65.mlp.experts.91.down_proj", + "model.layers.65.mlp.experts.91.gate_proj", + "model.layers.65.mlp.experts.91.up_proj", + "model.layers.65.mlp.experts.92.down_proj", + "model.layers.65.mlp.experts.92.gate_proj", + "model.layers.65.mlp.experts.92.up_proj", + "model.layers.65.mlp.experts.93.down_proj", + "model.layers.65.mlp.experts.93.gate_proj", + "model.layers.65.mlp.experts.93.up_proj", + "model.layers.65.mlp.experts.94.down_proj", + "model.layers.65.mlp.experts.94.gate_proj", + "model.layers.65.mlp.experts.94.up_proj", + "model.layers.65.mlp.experts.95.down_proj", + "model.layers.65.mlp.experts.95.gate_proj", + "model.layers.65.mlp.experts.95.up_proj", + "model.layers.65.mlp.experts.96.down_proj", + "model.layers.65.mlp.experts.96.gate_proj", + "model.layers.65.mlp.experts.96.up_proj", + "model.layers.65.mlp.experts.97.down_proj", + "model.layers.65.mlp.experts.97.gate_proj", + "model.layers.65.mlp.experts.97.up_proj", + "model.layers.65.mlp.experts.98.down_proj", + "model.layers.65.mlp.experts.98.gate_proj", + "model.layers.65.mlp.experts.98.up_proj", + "model.layers.65.mlp.experts.99.down_proj", + "model.layers.65.mlp.experts.99.gate_proj", + "model.layers.65.mlp.experts.99.up_proj", + "model.layers.66.mlp.experts.0.down_proj", + "model.layers.66.mlp.experts.0.gate_proj", + "model.layers.66.mlp.experts.0.up_proj", + "model.layers.66.mlp.experts.1.down_proj", + "model.layers.66.mlp.experts.1.gate_proj", + "model.layers.66.mlp.experts.1.up_proj", + "model.layers.66.mlp.experts.10.down_proj", + "model.layers.66.mlp.experts.10.gate_proj", + "model.layers.66.mlp.experts.10.up_proj", + "model.layers.66.mlp.experts.100.down_proj", + "model.layers.66.mlp.experts.100.gate_proj", + "model.layers.66.mlp.experts.100.up_proj", + "model.layers.66.mlp.experts.101.down_proj", + "model.layers.66.mlp.experts.101.gate_proj", + "model.layers.66.mlp.experts.101.up_proj", + "model.layers.66.mlp.experts.102.down_proj", + "model.layers.66.mlp.experts.102.gate_proj", + "model.layers.66.mlp.experts.102.up_proj", + "model.layers.66.mlp.experts.103.down_proj", + "model.layers.66.mlp.experts.103.gate_proj", + "model.layers.66.mlp.experts.103.up_proj", + "model.layers.66.mlp.experts.104.down_proj", + "model.layers.66.mlp.experts.104.gate_proj", + "model.layers.66.mlp.experts.104.up_proj", + "model.layers.66.mlp.experts.105.down_proj", + "model.layers.66.mlp.experts.105.gate_proj", + "model.layers.66.mlp.experts.105.up_proj", + "model.layers.66.mlp.experts.106.down_proj", + "model.layers.66.mlp.experts.106.gate_proj", + "model.layers.66.mlp.experts.106.up_proj", + "model.layers.66.mlp.experts.107.down_proj", + "model.layers.66.mlp.experts.107.gate_proj", + "model.layers.66.mlp.experts.107.up_proj", + "model.layers.66.mlp.experts.108.down_proj", + "model.layers.66.mlp.experts.108.gate_proj", + "model.layers.66.mlp.experts.108.up_proj", + "model.layers.66.mlp.experts.109.down_proj", + "model.layers.66.mlp.experts.109.gate_proj", + "model.layers.66.mlp.experts.109.up_proj", + "model.layers.66.mlp.experts.11.down_proj", + "model.layers.66.mlp.experts.11.gate_proj", + "model.layers.66.mlp.experts.11.up_proj", + "model.layers.66.mlp.experts.110.down_proj", + "model.layers.66.mlp.experts.110.gate_proj", + "model.layers.66.mlp.experts.110.up_proj", + "model.layers.66.mlp.experts.111.down_proj", + "model.layers.66.mlp.experts.111.gate_proj", + "model.layers.66.mlp.experts.111.up_proj", + "model.layers.66.mlp.experts.112.down_proj", + "model.layers.66.mlp.experts.112.gate_proj", + "model.layers.66.mlp.experts.112.up_proj", + "model.layers.66.mlp.experts.113.down_proj", + "model.layers.66.mlp.experts.113.gate_proj", + "model.layers.66.mlp.experts.113.up_proj", + "model.layers.66.mlp.experts.114.down_proj", + "model.layers.66.mlp.experts.114.gate_proj", + "model.layers.66.mlp.experts.114.up_proj", + "model.layers.66.mlp.experts.115.down_proj", + "model.layers.66.mlp.experts.115.gate_proj", + "model.layers.66.mlp.experts.115.up_proj", + "model.layers.66.mlp.experts.116.down_proj", + "model.layers.66.mlp.experts.116.gate_proj", + "model.layers.66.mlp.experts.116.up_proj", + "model.layers.66.mlp.experts.117.down_proj", + "model.layers.66.mlp.experts.117.gate_proj", + "model.layers.66.mlp.experts.117.up_proj", + "model.layers.66.mlp.experts.118.down_proj", + "model.layers.66.mlp.experts.118.gate_proj", + "model.layers.66.mlp.experts.118.up_proj", + "model.layers.66.mlp.experts.119.down_proj", + "model.layers.66.mlp.experts.119.gate_proj", + "model.layers.66.mlp.experts.119.up_proj", + "model.layers.66.mlp.experts.12.down_proj", + "model.layers.66.mlp.experts.12.gate_proj", + "model.layers.66.mlp.experts.12.up_proj", + "model.layers.66.mlp.experts.120.down_proj", + "model.layers.66.mlp.experts.120.gate_proj", + "model.layers.66.mlp.experts.120.up_proj", + "model.layers.66.mlp.experts.121.down_proj", + "model.layers.66.mlp.experts.121.gate_proj", + "model.layers.66.mlp.experts.121.up_proj", + "model.layers.66.mlp.experts.122.down_proj", + "model.layers.66.mlp.experts.122.gate_proj", + "model.layers.66.mlp.experts.122.up_proj", + "model.layers.66.mlp.experts.123.down_proj", + "model.layers.66.mlp.experts.123.gate_proj", + "model.layers.66.mlp.experts.123.up_proj", + "model.layers.66.mlp.experts.124.down_proj", + "model.layers.66.mlp.experts.124.gate_proj", + "model.layers.66.mlp.experts.124.up_proj", + "model.layers.66.mlp.experts.125.down_proj", + "model.layers.66.mlp.experts.125.gate_proj", + "model.layers.66.mlp.experts.125.up_proj", + "model.layers.66.mlp.experts.126.down_proj", + "model.layers.66.mlp.experts.126.gate_proj", + "model.layers.66.mlp.experts.126.up_proj", + "model.layers.66.mlp.experts.127.down_proj", + "model.layers.66.mlp.experts.127.gate_proj", + "model.layers.66.mlp.experts.127.up_proj", + "model.layers.66.mlp.experts.128.down_proj", + "model.layers.66.mlp.experts.128.gate_proj", + "model.layers.66.mlp.experts.128.up_proj", + "model.layers.66.mlp.experts.129.down_proj", + "model.layers.66.mlp.experts.129.gate_proj", + "model.layers.66.mlp.experts.129.up_proj", + "model.layers.66.mlp.experts.13.down_proj", + "model.layers.66.mlp.experts.13.gate_proj", + "model.layers.66.mlp.experts.13.up_proj", + "model.layers.66.mlp.experts.130.down_proj", + "model.layers.66.mlp.experts.130.gate_proj", + "model.layers.66.mlp.experts.130.up_proj", + "model.layers.66.mlp.experts.131.down_proj", + "model.layers.66.mlp.experts.131.gate_proj", + "model.layers.66.mlp.experts.131.up_proj", + "model.layers.66.mlp.experts.132.down_proj", + "model.layers.66.mlp.experts.132.gate_proj", + "model.layers.66.mlp.experts.132.up_proj", + "model.layers.66.mlp.experts.133.down_proj", + "model.layers.66.mlp.experts.133.gate_proj", + "model.layers.66.mlp.experts.133.up_proj", + "model.layers.66.mlp.experts.134.down_proj", + "model.layers.66.mlp.experts.134.gate_proj", + "model.layers.66.mlp.experts.134.up_proj", + "model.layers.66.mlp.experts.135.down_proj", + "model.layers.66.mlp.experts.135.gate_proj", + "model.layers.66.mlp.experts.135.up_proj", + "model.layers.66.mlp.experts.136.down_proj", + "model.layers.66.mlp.experts.136.gate_proj", + "model.layers.66.mlp.experts.136.up_proj", + "model.layers.66.mlp.experts.137.down_proj", + "model.layers.66.mlp.experts.137.gate_proj", + "model.layers.66.mlp.experts.137.up_proj", + "model.layers.66.mlp.experts.138.down_proj", + "model.layers.66.mlp.experts.138.gate_proj", + "model.layers.66.mlp.experts.138.up_proj", + "model.layers.66.mlp.experts.139.down_proj", + "model.layers.66.mlp.experts.139.gate_proj", + "model.layers.66.mlp.experts.139.up_proj", + "model.layers.66.mlp.experts.14.down_proj", + "model.layers.66.mlp.experts.14.gate_proj", + "model.layers.66.mlp.experts.14.up_proj", + "model.layers.66.mlp.experts.140.down_proj", + "model.layers.66.mlp.experts.140.gate_proj", + "model.layers.66.mlp.experts.140.up_proj", + "model.layers.66.mlp.experts.141.down_proj", + "model.layers.66.mlp.experts.141.gate_proj", + "model.layers.66.mlp.experts.141.up_proj", + "model.layers.66.mlp.experts.142.down_proj", + "model.layers.66.mlp.experts.142.gate_proj", + "model.layers.66.mlp.experts.142.up_proj", + "model.layers.66.mlp.experts.143.down_proj", + "model.layers.66.mlp.experts.143.gate_proj", + "model.layers.66.mlp.experts.143.up_proj", + "model.layers.66.mlp.experts.144.down_proj", + "model.layers.66.mlp.experts.144.gate_proj", + "model.layers.66.mlp.experts.144.up_proj", + "model.layers.66.mlp.experts.145.down_proj", + "model.layers.66.mlp.experts.145.gate_proj", + "model.layers.66.mlp.experts.145.up_proj", + "model.layers.66.mlp.experts.146.down_proj", + "model.layers.66.mlp.experts.146.gate_proj", + "model.layers.66.mlp.experts.146.up_proj", + "model.layers.66.mlp.experts.147.down_proj", + "model.layers.66.mlp.experts.147.gate_proj", + "model.layers.66.mlp.experts.147.up_proj", + "model.layers.66.mlp.experts.148.down_proj", + "model.layers.66.mlp.experts.148.gate_proj", + "model.layers.66.mlp.experts.148.up_proj", + "model.layers.66.mlp.experts.149.down_proj", + "model.layers.66.mlp.experts.149.gate_proj", + "model.layers.66.mlp.experts.149.up_proj", + "model.layers.66.mlp.experts.15.down_proj", + "model.layers.66.mlp.experts.15.gate_proj", + "model.layers.66.mlp.experts.15.up_proj", + "model.layers.66.mlp.experts.150.down_proj", + "model.layers.66.mlp.experts.150.gate_proj", + "model.layers.66.mlp.experts.150.up_proj", + "model.layers.66.mlp.experts.151.down_proj", + "model.layers.66.mlp.experts.151.gate_proj", + "model.layers.66.mlp.experts.151.up_proj", + "model.layers.66.mlp.experts.152.down_proj", + "model.layers.66.mlp.experts.152.gate_proj", + "model.layers.66.mlp.experts.152.up_proj", + "model.layers.66.mlp.experts.153.down_proj", + "model.layers.66.mlp.experts.153.gate_proj", + "model.layers.66.mlp.experts.153.up_proj", + "model.layers.66.mlp.experts.154.down_proj", + "model.layers.66.mlp.experts.154.gate_proj", + "model.layers.66.mlp.experts.154.up_proj", + "model.layers.66.mlp.experts.155.down_proj", + "model.layers.66.mlp.experts.155.gate_proj", + "model.layers.66.mlp.experts.155.up_proj", + "model.layers.66.mlp.experts.156.down_proj", + "model.layers.66.mlp.experts.156.gate_proj", + "model.layers.66.mlp.experts.156.up_proj", + "model.layers.66.mlp.experts.157.down_proj", + "model.layers.66.mlp.experts.157.gate_proj", + "model.layers.66.mlp.experts.157.up_proj", + "model.layers.66.mlp.experts.158.down_proj", + "model.layers.66.mlp.experts.158.gate_proj", + "model.layers.66.mlp.experts.158.up_proj", + "model.layers.66.mlp.experts.159.down_proj", + "model.layers.66.mlp.experts.159.gate_proj", + "model.layers.66.mlp.experts.159.up_proj", + "model.layers.66.mlp.experts.16.down_proj", + "model.layers.66.mlp.experts.16.gate_proj", + "model.layers.66.mlp.experts.16.up_proj", + "model.layers.66.mlp.experts.160.down_proj", + "model.layers.66.mlp.experts.160.gate_proj", + "model.layers.66.mlp.experts.160.up_proj", + "model.layers.66.mlp.experts.161.down_proj", + "model.layers.66.mlp.experts.161.gate_proj", + "model.layers.66.mlp.experts.161.up_proj", + "model.layers.66.mlp.experts.162.down_proj", + "model.layers.66.mlp.experts.162.gate_proj", + "model.layers.66.mlp.experts.162.up_proj", + "model.layers.66.mlp.experts.163.down_proj", + "model.layers.66.mlp.experts.163.gate_proj", + "model.layers.66.mlp.experts.163.up_proj", + "model.layers.66.mlp.experts.164.down_proj", + "model.layers.66.mlp.experts.164.gate_proj", + "model.layers.66.mlp.experts.164.up_proj", + "model.layers.66.mlp.experts.165.down_proj", + "model.layers.66.mlp.experts.165.gate_proj", + "model.layers.66.mlp.experts.165.up_proj", + "model.layers.66.mlp.experts.166.down_proj", + "model.layers.66.mlp.experts.166.gate_proj", + "model.layers.66.mlp.experts.166.up_proj", + "model.layers.66.mlp.experts.167.down_proj", + "model.layers.66.mlp.experts.167.gate_proj", + "model.layers.66.mlp.experts.167.up_proj", + "model.layers.66.mlp.experts.168.down_proj", + "model.layers.66.mlp.experts.168.gate_proj", + "model.layers.66.mlp.experts.168.up_proj", + "model.layers.66.mlp.experts.169.down_proj", + "model.layers.66.mlp.experts.169.gate_proj", + "model.layers.66.mlp.experts.169.up_proj", + "model.layers.66.mlp.experts.17.down_proj", + "model.layers.66.mlp.experts.17.gate_proj", + "model.layers.66.mlp.experts.17.up_proj", + "model.layers.66.mlp.experts.170.down_proj", + "model.layers.66.mlp.experts.170.gate_proj", + "model.layers.66.mlp.experts.170.up_proj", + "model.layers.66.mlp.experts.171.down_proj", + "model.layers.66.mlp.experts.171.gate_proj", + "model.layers.66.mlp.experts.171.up_proj", + "model.layers.66.mlp.experts.172.down_proj", + "model.layers.66.mlp.experts.172.gate_proj", + "model.layers.66.mlp.experts.172.up_proj", + "model.layers.66.mlp.experts.173.down_proj", + "model.layers.66.mlp.experts.173.gate_proj", + "model.layers.66.mlp.experts.173.up_proj", + "model.layers.66.mlp.experts.174.down_proj", + "model.layers.66.mlp.experts.174.gate_proj", + "model.layers.66.mlp.experts.174.up_proj", + "model.layers.66.mlp.experts.175.down_proj", + "model.layers.66.mlp.experts.175.gate_proj", + "model.layers.66.mlp.experts.175.up_proj", + "model.layers.66.mlp.experts.176.down_proj", + "model.layers.66.mlp.experts.176.gate_proj", + "model.layers.66.mlp.experts.176.up_proj", + "model.layers.66.mlp.experts.177.down_proj", + "model.layers.66.mlp.experts.177.gate_proj", + "model.layers.66.mlp.experts.177.up_proj", + "model.layers.66.mlp.experts.178.down_proj", + "model.layers.66.mlp.experts.178.gate_proj", + "model.layers.66.mlp.experts.178.up_proj", + "model.layers.66.mlp.experts.179.down_proj", + "model.layers.66.mlp.experts.179.gate_proj", + "model.layers.66.mlp.experts.179.up_proj", + "model.layers.66.mlp.experts.18.down_proj", + "model.layers.66.mlp.experts.18.gate_proj", + "model.layers.66.mlp.experts.18.up_proj", + "model.layers.66.mlp.experts.180.down_proj", + "model.layers.66.mlp.experts.180.gate_proj", + "model.layers.66.mlp.experts.180.up_proj", + "model.layers.66.mlp.experts.181.down_proj", + "model.layers.66.mlp.experts.181.gate_proj", + "model.layers.66.mlp.experts.181.up_proj", + "model.layers.66.mlp.experts.182.down_proj", + "model.layers.66.mlp.experts.182.gate_proj", + "model.layers.66.mlp.experts.182.up_proj", + "model.layers.66.mlp.experts.183.down_proj", + "model.layers.66.mlp.experts.183.gate_proj", + "model.layers.66.mlp.experts.183.up_proj", + "model.layers.66.mlp.experts.184.down_proj", + "model.layers.66.mlp.experts.184.gate_proj", + "model.layers.66.mlp.experts.184.up_proj", + "model.layers.66.mlp.experts.185.down_proj", + "model.layers.66.mlp.experts.185.gate_proj", + "model.layers.66.mlp.experts.185.up_proj", + "model.layers.66.mlp.experts.186.down_proj", + "model.layers.66.mlp.experts.186.gate_proj", + "model.layers.66.mlp.experts.186.up_proj", + "model.layers.66.mlp.experts.187.down_proj", + "model.layers.66.mlp.experts.187.gate_proj", + "model.layers.66.mlp.experts.187.up_proj", + "model.layers.66.mlp.experts.188.down_proj", + "model.layers.66.mlp.experts.188.gate_proj", + "model.layers.66.mlp.experts.188.up_proj", + "model.layers.66.mlp.experts.189.down_proj", + "model.layers.66.mlp.experts.189.gate_proj", + "model.layers.66.mlp.experts.189.up_proj", + "model.layers.66.mlp.experts.19.down_proj", + "model.layers.66.mlp.experts.19.gate_proj", + "model.layers.66.mlp.experts.19.up_proj", + "model.layers.66.mlp.experts.190.down_proj", + "model.layers.66.mlp.experts.190.gate_proj", + "model.layers.66.mlp.experts.190.up_proj", + "model.layers.66.mlp.experts.191.down_proj", + "model.layers.66.mlp.experts.191.gate_proj", + "model.layers.66.mlp.experts.191.up_proj", + "model.layers.66.mlp.experts.192.down_proj", + "model.layers.66.mlp.experts.192.gate_proj", + "model.layers.66.mlp.experts.192.up_proj", + "model.layers.66.mlp.experts.193.down_proj", + "model.layers.66.mlp.experts.193.gate_proj", + "model.layers.66.mlp.experts.193.up_proj", + "model.layers.66.mlp.experts.194.down_proj", + "model.layers.66.mlp.experts.194.gate_proj", + "model.layers.66.mlp.experts.194.up_proj", + "model.layers.66.mlp.experts.195.down_proj", + "model.layers.66.mlp.experts.195.gate_proj", + "model.layers.66.mlp.experts.195.up_proj", + "model.layers.66.mlp.experts.196.down_proj", + "model.layers.66.mlp.experts.196.gate_proj", + "model.layers.66.mlp.experts.196.up_proj", + "model.layers.66.mlp.experts.197.down_proj", + "model.layers.66.mlp.experts.197.gate_proj", + "model.layers.66.mlp.experts.197.up_proj", + "model.layers.66.mlp.experts.198.down_proj", + "model.layers.66.mlp.experts.198.gate_proj", + "model.layers.66.mlp.experts.198.up_proj", + "model.layers.66.mlp.experts.199.down_proj", + "model.layers.66.mlp.experts.199.gate_proj", + "model.layers.66.mlp.experts.199.up_proj", + "model.layers.66.mlp.experts.2.down_proj", + "model.layers.66.mlp.experts.2.gate_proj", + "model.layers.66.mlp.experts.2.up_proj", + "model.layers.66.mlp.experts.20.down_proj", + "model.layers.66.mlp.experts.20.gate_proj", + "model.layers.66.mlp.experts.20.up_proj", + "model.layers.66.mlp.experts.200.down_proj", + "model.layers.66.mlp.experts.200.gate_proj", + "model.layers.66.mlp.experts.200.up_proj", + "model.layers.66.mlp.experts.201.down_proj", + "model.layers.66.mlp.experts.201.gate_proj", + "model.layers.66.mlp.experts.201.up_proj", + "model.layers.66.mlp.experts.202.down_proj", + "model.layers.66.mlp.experts.202.gate_proj", + "model.layers.66.mlp.experts.202.up_proj", + "model.layers.66.mlp.experts.203.down_proj", + "model.layers.66.mlp.experts.203.gate_proj", + "model.layers.66.mlp.experts.203.up_proj", + "model.layers.66.mlp.experts.204.down_proj", + "model.layers.66.mlp.experts.204.gate_proj", + "model.layers.66.mlp.experts.204.up_proj", + "model.layers.66.mlp.experts.205.down_proj", + "model.layers.66.mlp.experts.205.gate_proj", + "model.layers.66.mlp.experts.205.up_proj", + "model.layers.66.mlp.experts.206.down_proj", + "model.layers.66.mlp.experts.206.gate_proj", + "model.layers.66.mlp.experts.206.up_proj", + "model.layers.66.mlp.experts.207.down_proj", + "model.layers.66.mlp.experts.207.gate_proj", + "model.layers.66.mlp.experts.207.up_proj", + "model.layers.66.mlp.experts.208.down_proj", + "model.layers.66.mlp.experts.208.gate_proj", + "model.layers.66.mlp.experts.208.up_proj", + "model.layers.66.mlp.experts.209.down_proj", + "model.layers.66.mlp.experts.209.gate_proj", + "model.layers.66.mlp.experts.209.up_proj", + "model.layers.66.mlp.experts.21.down_proj", + "model.layers.66.mlp.experts.21.gate_proj", + "model.layers.66.mlp.experts.21.up_proj", + "model.layers.66.mlp.experts.210.down_proj", + "model.layers.66.mlp.experts.210.gate_proj", + "model.layers.66.mlp.experts.210.up_proj", + "model.layers.66.mlp.experts.211.down_proj", + "model.layers.66.mlp.experts.211.gate_proj", + "model.layers.66.mlp.experts.211.up_proj", + "model.layers.66.mlp.experts.212.down_proj", + "model.layers.66.mlp.experts.212.gate_proj", + "model.layers.66.mlp.experts.212.up_proj", + "model.layers.66.mlp.experts.213.down_proj", + "model.layers.66.mlp.experts.213.gate_proj", + "model.layers.66.mlp.experts.213.up_proj", + "model.layers.66.mlp.experts.214.down_proj", + "model.layers.66.mlp.experts.214.gate_proj", + "model.layers.66.mlp.experts.214.up_proj", + "model.layers.66.mlp.experts.215.down_proj", + "model.layers.66.mlp.experts.215.gate_proj", + "model.layers.66.mlp.experts.215.up_proj", + "model.layers.66.mlp.experts.216.down_proj", + "model.layers.66.mlp.experts.216.gate_proj", + "model.layers.66.mlp.experts.216.up_proj", + "model.layers.66.mlp.experts.217.down_proj", + "model.layers.66.mlp.experts.217.gate_proj", + "model.layers.66.mlp.experts.217.up_proj", + "model.layers.66.mlp.experts.218.down_proj", + "model.layers.66.mlp.experts.218.gate_proj", + "model.layers.66.mlp.experts.218.up_proj", + "model.layers.66.mlp.experts.219.down_proj", + "model.layers.66.mlp.experts.219.gate_proj", + "model.layers.66.mlp.experts.219.up_proj", + "model.layers.66.mlp.experts.22.down_proj", + "model.layers.66.mlp.experts.22.gate_proj", + "model.layers.66.mlp.experts.22.up_proj", + "model.layers.66.mlp.experts.220.down_proj", + "model.layers.66.mlp.experts.220.gate_proj", + "model.layers.66.mlp.experts.220.up_proj", + "model.layers.66.mlp.experts.221.down_proj", + "model.layers.66.mlp.experts.221.gate_proj", + "model.layers.66.mlp.experts.221.up_proj", + "model.layers.66.mlp.experts.222.down_proj", + "model.layers.66.mlp.experts.222.gate_proj", + "model.layers.66.mlp.experts.222.up_proj", + "model.layers.66.mlp.experts.223.down_proj", + "model.layers.66.mlp.experts.223.gate_proj", + "model.layers.66.mlp.experts.223.up_proj", + "model.layers.66.mlp.experts.224.down_proj", + "model.layers.66.mlp.experts.224.gate_proj", + "model.layers.66.mlp.experts.224.up_proj", + "model.layers.66.mlp.experts.225.down_proj", + "model.layers.66.mlp.experts.225.gate_proj", + "model.layers.66.mlp.experts.225.up_proj", + "model.layers.66.mlp.experts.226.down_proj", + "model.layers.66.mlp.experts.226.gate_proj", + "model.layers.66.mlp.experts.226.up_proj", + "model.layers.66.mlp.experts.227.down_proj", + "model.layers.66.mlp.experts.227.gate_proj", + "model.layers.66.mlp.experts.227.up_proj", + "model.layers.66.mlp.experts.228.down_proj", + "model.layers.66.mlp.experts.228.gate_proj", + "model.layers.66.mlp.experts.228.up_proj", + "model.layers.66.mlp.experts.229.down_proj", + "model.layers.66.mlp.experts.229.gate_proj", + "model.layers.66.mlp.experts.229.up_proj", + "model.layers.66.mlp.experts.23.down_proj", + "model.layers.66.mlp.experts.23.gate_proj", + "model.layers.66.mlp.experts.23.up_proj", + "model.layers.66.mlp.experts.230.down_proj", + "model.layers.66.mlp.experts.230.gate_proj", + "model.layers.66.mlp.experts.230.up_proj", + "model.layers.66.mlp.experts.231.down_proj", + "model.layers.66.mlp.experts.231.gate_proj", + "model.layers.66.mlp.experts.231.up_proj", + "model.layers.66.mlp.experts.232.down_proj", + "model.layers.66.mlp.experts.232.gate_proj", + "model.layers.66.mlp.experts.232.up_proj", + "model.layers.66.mlp.experts.233.down_proj", + "model.layers.66.mlp.experts.233.gate_proj", + "model.layers.66.mlp.experts.233.up_proj", + "model.layers.66.mlp.experts.234.down_proj", + "model.layers.66.mlp.experts.234.gate_proj", + "model.layers.66.mlp.experts.234.up_proj", + "model.layers.66.mlp.experts.235.down_proj", + "model.layers.66.mlp.experts.235.gate_proj", + "model.layers.66.mlp.experts.235.up_proj", + "model.layers.66.mlp.experts.236.down_proj", + "model.layers.66.mlp.experts.236.gate_proj", + "model.layers.66.mlp.experts.236.up_proj", + "model.layers.66.mlp.experts.237.down_proj", + "model.layers.66.mlp.experts.237.gate_proj", + "model.layers.66.mlp.experts.237.up_proj", + "model.layers.66.mlp.experts.238.down_proj", + "model.layers.66.mlp.experts.238.gate_proj", + "model.layers.66.mlp.experts.238.up_proj", + "model.layers.66.mlp.experts.239.down_proj", + "model.layers.66.mlp.experts.239.gate_proj", + "model.layers.66.mlp.experts.239.up_proj", + "model.layers.66.mlp.experts.24.down_proj", + "model.layers.66.mlp.experts.24.gate_proj", + "model.layers.66.mlp.experts.24.up_proj", + "model.layers.66.mlp.experts.240.down_proj", + "model.layers.66.mlp.experts.240.gate_proj", + "model.layers.66.mlp.experts.240.up_proj", + "model.layers.66.mlp.experts.241.down_proj", + "model.layers.66.mlp.experts.241.gate_proj", + "model.layers.66.mlp.experts.241.up_proj", + "model.layers.66.mlp.experts.242.down_proj", + "model.layers.66.mlp.experts.242.gate_proj", + "model.layers.66.mlp.experts.242.up_proj", + "model.layers.66.mlp.experts.243.down_proj", + "model.layers.66.mlp.experts.243.gate_proj", + "model.layers.66.mlp.experts.243.up_proj", + "model.layers.66.mlp.experts.244.down_proj", + "model.layers.66.mlp.experts.244.gate_proj", + "model.layers.66.mlp.experts.244.up_proj", + "model.layers.66.mlp.experts.245.down_proj", + "model.layers.66.mlp.experts.245.gate_proj", + "model.layers.66.mlp.experts.245.up_proj", + "model.layers.66.mlp.experts.246.down_proj", + "model.layers.66.mlp.experts.246.gate_proj", + "model.layers.66.mlp.experts.246.up_proj", + "model.layers.66.mlp.experts.247.down_proj", + "model.layers.66.mlp.experts.247.gate_proj", + "model.layers.66.mlp.experts.247.up_proj", + "model.layers.66.mlp.experts.248.down_proj", + "model.layers.66.mlp.experts.248.gate_proj", + "model.layers.66.mlp.experts.248.up_proj", + "model.layers.66.mlp.experts.249.down_proj", + "model.layers.66.mlp.experts.249.gate_proj", + "model.layers.66.mlp.experts.249.up_proj", + "model.layers.66.mlp.experts.25.down_proj", + "model.layers.66.mlp.experts.25.gate_proj", + "model.layers.66.mlp.experts.25.up_proj", + "model.layers.66.mlp.experts.250.down_proj", + "model.layers.66.mlp.experts.250.gate_proj", + "model.layers.66.mlp.experts.250.up_proj", + "model.layers.66.mlp.experts.251.down_proj", + "model.layers.66.mlp.experts.251.gate_proj", + "model.layers.66.mlp.experts.251.up_proj", + "model.layers.66.mlp.experts.252.down_proj", + "model.layers.66.mlp.experts.252.gate_proj", + "model.layers.66.mlp.experts.252.up_proj", + "model.layers.66.mlp.experts.253.down_proj", + "model.layers.66.mlp.experts.253.gate_proj", + "model.layers.66.mlp.experts.253.up_proj", + "model.layers.66.mlp.experts.254.down_proj", + "model.layers.66.mlp.experts.254.gate_proj", + "model.layers.66.mlp.experts.254.up_proj", + "model.layers.66.mlp.experts.255.down_proj", + "model.layers.66.mlp.experts.255.gate_proj", + "model.layers.66.mlp.experts.255.up_proj", + "model.layers.66.mlp.experts.26.down_proj", + "model.layers.66.mlp.experts.26.gate_proj", + "model.layers.66.mlp.experts.26.up_proj", + "model.layers.66.mlp.experts.27.down_proj", + "model.layers.66.mlp.experts.27.gate_proj", + "model.layers.66.mlp.experts.27.up_proj", + "model.layers.66.mlp.experts.28.down_proj", + "model.layers.66.mlp.experts.28.gate_proj", + "model.layers.66.mlp.experts.28.up_proj", + "model.layers.66.mlp.experts.29.down_proj", + "model.layers.66.mlp.experts.29.gate_proj", + "model.layers.66.mlp.experts.29.up_proj", + "model.layers.66.mlp.experts.3.down_proj", + "model.layers.66.mlp.experts.3.gate_proj", + "model.layers.66.mlp.experts.3.up_proj", + "model.layers.66.mlp.experts.30.down_proj", + "model.layers.66.mlp.experts.30.gate_proj", + "model.layers.66.mlp.experts.30.up_proj", + "model.layers.66.mlp.experts.31.down_proj", + "model.layers.66.mlp.experts.31.gate_proj", + "model.layers.66.mlp.experts.31.up_proj", + "model.layers.66.mlp.experts.32.down_proj", + "model.layers.66.mlp.experts.32.gate_proj", + "model.layers.66.mlp.experts.32.up_proj", + "model.layers.66.mlp.experts.33.down_proj", + "model.layers.66.mlp.experts.33.gate_proj", + "model.layers.66.mlp.experts.33.up_proj", + "model.layers.66.mlp.experts.34.down_proj", + "model.layers.66.mlp.experts.34.gate_proj", + "model.layers.66.mlp.experts.34.up_proj", + "model.layers.66.mlp.experts.35.down_proj", + "model.layers.66.mlp.experts.35.gate_proj", + "model.layers.66.mlp.experts.35.up_proj", + "model.layers.66.mlp.experts.36.down_proj", + "model.layers.66.mlp.experts.36.gate_proj", + "model.layers.66.mlp.experts.36.up_proj", + "model.layers.66.mlp.experts.37.down_proj", + "model.layers.66.mlp.experts.37.gate_proj", + "model.layers.66.mlp.experts.37.up_proj", + "model.layers.66.mlp.experts.38.down_proj", + "model.layers.66.mlp.experts.38.gate_proj", + "model.layers.66.mlp.experts.38.up_proj", + "model.layers.66.mlp.experts.39.down_proj", + "model.layers.66.mlp.experts.39.gate_proj", + "model.layers.66.mlp.experts.39.up_proj", + "model.layers.66.mlp.experts.4.down_proj", + "model.layers.66.mlp.experts.4.gate_proj", + "model.layers.66.mlp.experts.4.up_proj", + "model.layers.66.mlp.experts.40.down_proj", + "model.layers.66.mlp.experts.40.gate_proj", + "model.layers.66.mlp.experts.40.up_proj", + "model.layers.66.mlp.experts.41.down_proj", + "model.layers.66.mlp.experts.41.gate_proj", + "model.layers.66.mlp.experts.41.up_proj", + "model.layers.66.mlp.experts.42.down_proj", + "model.layers.66.mlp.experts.42.gate_proj", + "model.layers.66.mlp.experts.42.up_proj", + "model.layers.66.mlp.experts.43.down_proj", + "model.layers.66.mlp.experts.43.gate_proj", + "model.layers.66.mlp.experts.43.up_proj", + "model.layers.66.mlp.experts.44.down_proj", + "model.layers.66.mlp.experts.44.gate_proj", + "model.layers.66.mlp.experts.44.up_proj", + "model.layers.66.mlp.experts.45.down_proj", + "model.layers.66.mlp.experts.45.gate_proj", + "model.layers.66.mlp.experts.45.up_proj", + "model.layers.66.mlp.experts.46.down_proj", + "model.layers.66.mlp.experts.46.gate_proj", + "model.layers.66.mlp.experts.46.up_proj", + "model.layers.66.mlp.experts.47.down_proj", + "model.layers.66.mlp.experts.47.gate_proj", + "model.layers.66.mlp.experts.47.up_proj", + "model.layers.66.mlp.experts.48.down_proj", + "model.layers.66.mlp.experts.48.gate_proj", + "model.layers.66.mlp.experts.48.up_proj", + "model.layers.66.mlp.experts.49.down_proj", + "model.layers.66.mlp.experts.49.gate_proj", + "model.layers.66.mlp.experts.49.up_proj", + "model.layers.66.mlp.experts.5.down_proj", + "model.layers.66.mlp.experts.5.gate_proj", + "model.layers.66.mlp.experts.5.up_proj", + "model.layers.66.mlp.experts.50.down_proj", + "model.layers.66.mlp.experts.50.gate_proj", + "model.layers.66.mlp.experts.50.up_proj", + "model.layers.66.mlp.experts.51.down_proj", + "model.layers.66.mlp.experts.51.gate_proj", + "model.layers.66.mlp.experts.51.up_proj", + "model.layers.66.mlp.experts.52.down_proj", + "model.layers.66.mlp.experts.52.gate_proj", + "model.layers.66.mlp.experts.52.up_proj", + "model.layers.66.mlp.experts.53.down_proj", + "model.layers.66.mlp.experts.53.gate_proj", + "model.layers.66.mlp.experts.53.up_proj", + "model.layers.66.mlp.experts.54.down_proj", + "model.layers.66.mlp.experts.54.gate_proj", + "model.layers.66.mlp.experts.54.up_proj", + "model.layers.66.mlp.experts.55.down_proj", + "model.layers.66.mlp.experts.55.gate_proj", + "model.layers.66.mlp.experts.55.up_proj", + "model.layers.66.mlp.experts.56.down_proj", + "model.layers.66.mlp.experts.56.gate_proj", + "model.layers.66.mlp.experts.56.up_proj", + "model.layers.66.mlp.experts.57.down_proj", + "model.layers.66.mlp.experts.57.gate_proj", + "model.layers.66.mlp.experts.57.up_proj", + "model.layers.66.mlp.experts.58.down_proj", + "model.layers.66.mlp.experts.58.gate_proj", + "model.layers.66.mlp.experts.58.up_proj", + "model.layers.66.mlp.experts.59.down_proj", + "model.layers.66.mlp.experts.59.gate_proj", + "model.layers.66.mlp.experts.59.up_proj", + "model.layers.66.mlp.experts.6.down_proj", + "model.layers.66.mlp.experts.6.gate_proj", + "model.layers.66.mlp.experts.6.up_proj", + "model.layers.66.mlp.experts.60.down_proj", + "model.layers.66.mlp.experts.60.gate_proj", + "model.layers.66.mlp.experts.60.up_proj", + "model.layers.66.mlp.experts.61.down_proj", + "model.layers.66.mlp.experts.61.gate_proj", + "model.layers.66.mlp.experts.61.up_proj", + "model.layers.66.mlp.experts.62.down_proj", + "model.layers.66.mlp.experts.62.gate_proj", + "model.layers.66.mlp.experts.62.up_proj", + "model.layers.66.mlp.experts.63.down_proj", + "model.layers.66.mlp.experts.63.gate_proj", + "model.layers.66.mlp.experts.63.up_proj", + "model.layers.66.mlp.experts.64.down_proj", + "model.layers.66.mlp.experts.64.gate_proj", + "model.layers.66.mlp.experts.64.up_proj", + "model.layers.66.mlp.experts.65.down_proj", + "model.layers.66.mlp.experts.65.gate_proj", + "model.layers.66.mlp.experts.65.up_proj", + "model.layers.66.mlp.experts.66.down_proj", + "model.layers.66.mlp.experts.66.gate_proj", + "model.layers.66.mlp.experts.66.up_proj", + "model.layers.66.mlp.experts.67.down_proj", + "model.layers.66.mlp.experts.67.gate_proj", + "model.layers.66.mlp.experts.67.up_proj", + "model.layers.66.mlp.experts.68.down_proj", + "model.layers.66.mlp.experts.68.gate_proj", + "model.layers.66.mlp.experts.68.up_proj", + "model.layers.66.mlp.experts.69.down_proj", + "model.layers.66.mlp.experts.69.gate_proj", + "model.layers.66.mlp.experts.69.up_proj", + "model.layers.66.mlp.experts.7.down_proj", + "model.layers.66.mlp.experts.7.gate_proj", + "model.layers.66.mlp.experts.7.up_proj", + "model.layers.66.mlp.experts.70.down_proj", + "model.layers.66.mlp.experts.70.gate_proj", + "model.layers.66.mlp.experts.70.up_proj", + "model.layers.66.mlp.experts.71.down_proj", + "model.layers.66.mlp.experts.71.gate_proj", + "model.layers.66.mlp.experts.71.up_proj", + "model.layers.66.mlp.experts.72.down_proj", + "model.layers.66.mlp.experts.72.gate_proj", + "model.layers.66.mlp.experts.72.up_proj", + "model.layers.66.mlp.experts.73.down_proj", + "model.layers.66.mlp.experts.73.gate_proj", + "model.layers.66.mlp.experts.73.up_proj", + "model.layers.66.mlp.experts.74.down_proj", + "model.layers.66.mlp.experts.74.gate_proj", + "model.layers.66.mlp.experts.74.up_proj", + "model.layers.66.mlp.experts.75.down_proj", + "model.layers.66.mlp.experts.75.gate_proj", + "model.layers.66.mlp.experts.75.up_proj", + "model.layers.66.mlp.experts.76.down_proj", + "model.layers.66.mlp.experts.76.gate_proj", + "model.layers.66.mlp.experts.76.up_proj", + "model.layers.66.mlp.experts.77.down_proj", + "model.layers.66.mlp.experts.77.gate_proj", + "model.layers.66.mlp.experts.77.up_proj", + "model.layers.66.mlp.experts.78.down_proj", + "model.layers.66.mlp.experts.78.gate_proj", + "model.layers.66.mlp.experts.78.up_proj", + "model.layers.66.mlp.experts.79.down_proj", + "model.layers.66.mlp.experts.79.gate_proj", + "model.layers.66.mlp.experts.79.up_proj", + "model.layers.66.mlp.experts.8.down_proj", + "model.layers.66.mlp.experts.8.gate_proj", + "model.layers.66.mlp.experts.8.up_proj", + "model.layers.66.mlp.experts.80.down_proj", + "model.layers.66.mlp.experts.80.gate_proj", + "model.layers.66.mlp.experts.80.up_proj", + "model.layers.66.mlp.experts.81.down_proj", + "model.layers.66.mlp.experts.81.gate_proj", + "model.layers.66.mlp.experts.81.up_proj", + "model.layers.66.mlp.experts.82.down_proj", + "model.layers.66.mlp.experts.82.gate_proj", + "model.layers.66.mlp.experts.82.up_proj", + "model.layers.66.mlp.experts.83.down_proj", + "model.layers.66.mlp.experts.83.gate_proj", + "model.layers.66.mlp.experts.83.up_proj", + "model.layers.66.mlp.experts.84.down_proj", + "model.layers.66.mlp.experts.84.gate_proj", + "model.layers.66.mlp.experts.84.up_proj", + "model.layers.66.mlp.experts.85.down_proj", + "model.layers.66.mlp.experts.85.gate_proj", + "model.layers.66.mlp.experts.85.up_proj", + "model.layers.66.mlp.experts.86.down_proj", + "model.layers.66.mlp.experts.86.gate_proj", + "model.layers.66.mlp.experts.86.up_proj", + "model.layers.66.mlp.experts.87.down_proj", + "model.layers.66.mlp.experts.87.gate_proj", + "model.layers.66.mlp.experts.87.up_proj", + "model.layers.66.mlp.experts.88.down_proj", + "model.layers.66.mlp.experts.88.gate_proj", + "model.layers.66.mlp.experts.88.up_proj", + "model.layers.66.mlp.experts.89.down_proj", + "model.layers.66.mlp.experts.89.gate_proj", + "model.layers.66.mlp.experts.89.up_proj", + "model.layers.66.mlp.experts.9.down_proj", + "model.layers.66.mlp.experts.9.gate_proj", + "model.layers.66.mlp.experts.9.up_proj", + "model.layers.66.mlp.experts.90.down_proj", + "model.layers.66.mlp.experts.90.gate_proj", + "model.layers.66.mlp.experts.90.up_proj", + "model.layers.66.mlp.experts.91.down_proj", + "model.layers.66.mlp.experts.91.gate_proj", + "model.layers.66.mlp.experts.91.up_proj", + "model.layers.66.mlp.experts.92.down_proj", + "model.layers.66.mlp.experts.92.gate_proj", + "model.layers.66.mlp.experts.92.up_proj", + "model.layers.66.mlp.experts.93.down_proj", + "model.layers.66.mlp.experts.93.gate_proj", + "model.layers.66.mlp.experts.93.up_proj", + "model.layers.66.mlp.experts.94.down_proj", + "model.layers.66.mlp.experts.94.gate_proj", + "model.layers.66.mlp.experts.94.up_proj", + "model.layers.66.mlp.experts.95.down_proj", + "model.layers.66.mlp.experts.95.gate_proj", + "model.layers.66.mlp.experts.95.up_proj", + "model.layers.66.mlp.experts.96.down_proj", + "model.layers.66.mlp.experts.96.gate_proj", + "model.layers.66.mlp.experts.96.up_proj", + "model.layers.66.mlp.experts.97.down_proj", + "model.layers.66.mlp.experts.97.gate_proj", + "model.layers.66.mlp.experts.97.up_proj", + "model.layers.66.mlp.experts.98.down_proj", + "model.layers.66.mlp.experts.98.gate_proj", + "model.layers.66.mlp.experts.98.up_proj", + "model.layers.66.mlp.experts.99.down_proj", + "model.layers.66.mlp.experts.99.gate_proj", + "model.layers.66.mlp.experts.99.up_proj", + "model.layers.67.mlp.experts.0.down_proj", + "model.layers.67.mlp.experts.0.gate_proj", + "model.layers.67.mlp.experts.0.up_proj", + "model.layers.67.mlp.experts.1.down_proj", + "model.layers.67.mlp.experts.1.gate_proj", + "model.layers.67.mlp.experts.1.up_proj", + "model.layers.67.mlp.experts.10.down_proj", + "model.layers.67.mlp.experts.10.gate_proj", + "model.layers.67.mlp.experts.10.up_proj", + "model.layers.67.mlp.experts.100.down_proj", + "model.layers.67.mlp.experts.100.gate_proj", + "model.layers.67.mlp.experts.100.up_proj", + "model.layers.67.mlp.experts.101.down_proj", + "model.layers.67.mlp.experts.101.gate_proj", + "model.layers.67.mlp.experts.101.up_proj", + "model.layers.67.mlp.experts.102.down_proj", + "model.layers.67.mlp.experts.102.gate_proj", + "model.layers.67.mlp.experts.102.up_proj", + "model.layers.67.mlp.experts.103.down_proj", + "model.layers.67.mlp.experts.103.gate_proj", + "model.layers.67.mlp.experts.103.up_proj", + "model.layers.67.mlp.experts.104.down_proj", + "model.layers.67.mlp.experts.104.gate_proj", + "model.layers.67.mlp.experts.104.up_proj", + "model.layers.67.mlp.experts.105.down_proj", + "model.layers.67.mlp.experts.105.gate_proj", + "model.layers.67.mlp.experts.105.up_proj", + "model.layers.67.mlp.experts.106.down_proj", + "model.layers.67.mlp.experts.106.gate_proj", + "model.layers.67.mlp.experts.106.up_proj", + "model.layers.67.mlp.experts.107.down_proj", + "model.layers.67.mlp.experts.107.gate_proj", + "model.layers.67.mlp.experts.107.up_proj", + "model.layers.67.mlp.experts.108.down_proj", + "model.layers.67.mlp.experts.108.gate_proj", + "model.layers.67.mlp.experts.108.up_proj", + "model.layers.67.mlp.experts.109.down_proj", + "model.layers.67.mlp.experts.109.gate_proj", + "model.layers.67.mlp.experts.109.up_proj", + "model.layers.67.mlp.experts.11.down_proj", + "model.layers.67.mlp.experts.11.gate_proj", + "model.layers.67.mlp.experts.11.up_proj", + "model.layers.67.mlp.experts.110.down_proj", + "model.layers.67.mlp.experts.110.gate_proj", + "model.layers.67.mlp.experts.110.up_proj", + "model.layers.67.mlp.experts.111.down_proj", + "model.layers.67.mlp.experts.111.gate_proj", + "model.layers.67.mlp.experts.111.up_proj", + "model.layers.67.mlp.experts.112.down_proj", + "model.layers.67.mlp.experts.112.gate_proj", + "model.layers.67.mlp.experts.112.up_proj", + "model.layers.67.mlp.experts.113.down_proj", + "model.layers.67.mlp.experts.113.gate_proj", + "model.layers.67.mlp.experts.113.up_proj", + "model.layers.67.mlp.experts.114.down_proj", + "model.layers.67.mlp.experts.114.gate_proj", + "model.layers.67.mlp.experts.114.up_proj", + "model.layers.67.mlp.experts.115.down_proj", + "model.layers.67.mlp.experts.115.gate_proj", + "model.layers.67.mlp.experts.115.up_proj", + "model.layers.67.mlp.experts.116.down_proj", + "model.layers.67.mlp.experts.116.gate_proj", + "model.layers.67.mlp.experts.116.up_proj", + "model.layers.67.mlp.experts.117.down_proj", + "model.layers.67.mlp.experts.117.gate_proj", + "model.layers.67.mlp.experts.117.up_proj", + "model.layers.67.mlp.experts.118.down_proj", + "model.layers.67.mlp.experts.118.gate_proj", + "model.layers.67.mlp.experts.118.up_proj", + "model.layers.67.mlp.experts.119.down_proj", + "model.layers.67.mlp.experts.119.gate_proj", + "model.layers.67.mlp.experts.119.up_proj", + "model.layers.67.mlp.experts.12.down_proj", + "model.layers.67.mlp.experts.12.gate_proj", + "model.layers.67.mlp.experts.12.up_proj", + "model.layers.67.mlp.experts.120.down_proj", + "model.layers.67.mlp.experts.120.gate_proj", + "model.layers.67.mlp.experts.120.up_proj", + "model.layers.67.mlp.experts.121.down_proj", + "model.layers.67.mlp.experts.121.gate_proj", + "model.layers.67.mlp.experts.121.up_proj", + "model.layers.67.mlp.experts.122.down_proj", + "model.layers.67.mlp.experts.122.gate_proj", + "model.layers.67.mlp.experts.122.up_proj", + "model.layers.67.mlp.experts.123.down_proj", + "model.layers.67.mlp.experts.123.gate_proj", + "model.layers.67.mlp.experts.123.up_proj", + "model.layers.67.mlp.experts.124.down_proj", + "model.layers.67.mlp.experts.124.gate_proj", + "model.layers.67.mlp.experts.124.up_proj", + "model.layers.67.mlp.experts.125.down_proj", + "model.layers.67.mlp.experts.125.gate_proj", + "model.layers.67.mlp.experts.125.up_proj", + "model.layers.67.mlp.experts.126.down_proj", + "model.layers.67.mlp.experts.126.gate_proj", + "model.layers.67.mlp.experts.126.up_proj", + "model.layers.67.mlp.experts.127.down_proj", + "model.layers.67.mlp.experts.127.gate_proj", + "model.layers.67.mlp.experts.127.up_proj", + "model.layers.67.mlp.experts.128.down_proj", + "model.layers.67.mlp.experts.128.gate_proj", + "model.layers.67.mlp.experts.128.up_proj", + "model.layers.67.mlp.experts.129.down_proj", + "model.layers.67.mlp.experts.129.gate_proj", + "model.layers.67.mlp.experts.129.up_proj", + "model.layers.67.mlp.experts.13.down_proj", + "model.layers.67.mlp.experts.13.gate_proj", + "model.layers.67.mlp.experts.13.up_proj", + "model.layers.67.mlp.experts.130.down_proj", + "model.layers.67.mlp.experts.130.gate_proj", + "model.layers.67.mlp.experts.130.up_proj", + "model.layers.67.mlp.experts.131.down_proj", + "model.layers.67.mlp.experts.131.gate_proj", + "model.layers.67.mlp.experts.131.up_proj", + "model.layers.67.mlp.experts.132.down_proj", + "model.layers.67.mlp.experts.132.gate_proj", + "model.layers.67.mlp.experts.132.up_proj", + "model.layers.67.mlp.experts.133.down_proj", + "model.layers.67.mlp.experts.133.gate_proj", + "model.layers.67.mlp.experts.133.up_proj", + "model.layers.67.mlp.experts.134.down_proj", + "model.layers.67.mlp.experts.134.gate_proj", + "model.layers.67.mlp.experts.134.up_proj", + "model.layers.67.mlp.experts.135.down_proj", + "model.layers.67.mlp.experts.135.gate_proj", + "model.layers.67.mlp.experts.135.up_proj", + "model.layers.67.mlp.experts.136.down_proj", + "model.layers.67.mlp.experts.136.gate_proj", + "model.layers.67.mlp.experts.136.up_proj", + "model.layers.67.mlp.experts.137.down_proj", + "model.layers.67.mlp.experts.137.gate_proj", + "model.layers.67.mlp.experts.137.up_proj", + "model.layers.67.mlp.experts.138.down_proj", + "model.layers.67.mlp.experts.138.gate_proj", + "model.layers.67.mlp.experts.138.up_proj", + "model.layers.67.mlp.experts.139.down_proj", + "model.layers.67.mlp.experts.139.gate_proj", + "model.layers.67.mlp.experts.139.up_proj", + "model.layers.67.mlp.experts.14.down_proj", + "model.layers.67.mlp.experts.14.gate_proj", + "model.layers.67.mlp.experts.14.up_proj", + "model.layers.67.mlp.experts.140.down_proj", + "model.layers.67.mlp.experts.140.gate_proj", + "model.layers.67.mlp.experts.140.up_proj", + "model.layers.67.mlp.experts.141.down_proj", + "model.layers.67.mlp.experts.141.gate_proj", + "model.layers.67.mlp.experts.141.up_proj", + "model.layers.67.mlp.experts.142.down_proj", + "model.layers.67.mlp.experts.142.gate_proj", + "model.layers.67.mlp.experts.142.up_proj", + "model.layers.67.mlp.experts.143.down_proj", + "model.layers.67.mlp.experts.143.gate_proj", + "model.layers.67.mlp.experts.143.up_proj", + "model.layers.67.mlp.experts.144.down_proj", + "model.layers.67.mlp.experts.144.gate_proj", + "model.layers.67.mlp.experts.144.up_proj", + "model.layers.67.mlp.experts.145.down_proj", + "model.layers.67.mlp.experts.145.gate_proj", + "model.layers.67.mlp.experts.145.up_proj", + "model.layers.67.mlp.experts.146.down_proj", + "model.layers.67.mlp.experts.146.gate_proj", + "model.layers.67.mlp.experts.146.up_proj", + "model.layers.67.mlp.experts.147.down_proj", + "model.layers.67.mlp.experts.147.gate_proj", + "model.layers.67.mlp.experts.147.up_proj", + "model.layers.67.mlp.experts.148.down_proj", + "model.layers.67.mlp.experts.148.gate_proj", + "model.layers.67.mlp.experts.148.up_proj", + "model.layers.67.mlp.experts.149.down_proj", + "model.layers.67.mlp.experts.149.gate_proj", + "model.layers.67.mlp.experts.149.up_proj", + "model.layers.67.mlp.experts.15.down_proj", + "model.layers.67.mlp.experts.15.gate_proj", + "model.layers.67.mlp.experts.15.up_proj", + "model.layers.67.mlp.experts.150.down_proj", + "model.layers.67.mlp.experts.150.gate_proj", + "model.layers.67.mlp.experts.150.up_proj", + "model.layers.67.mlp.experts.151.down_proj", + "model.layers.67.mlp.experts.151.gate_proj", + "model.layers.67.mlp.experts.151.up_proj", + "model.layers.67.mlp.experts.152.down_proj", + "model.layers.67.mlp.experts.152.gate_proj", + "model.layers.67.mlp.experts.152.up_proj", + "model.layers.67.mlp.experts.153.down_proj", + "model.layers.67.mlp.experts.153.gate_proj", + "model.layers.67.mlp.experts.153.up_proj", + "model.layers.67.mlp.experts.154.down_proj", + "model.layers.67.mlp.experts.154.gate_proj", + "model.layers.67.mlp.experts.154.up_proj", + "model.layers.67.mlp.experts.155.down_proj", + "model.layers.67.mlp.experts.155.gate_proj", + "model.layers.67.mlp.experts.155.up_proj", + "model.layers.67.mlp.experts.156.down_proj", + "model.layers.67.mlp.experts.156.gate_proj", + "model.layers.67.mlp.experts.156.up_proj", + "model.layers.67.mlp.experts.157.down_proj", + "model.layers.67.mlp.experts.157.gate_proj", + "model.layers.67.mlp.experts.157.up_proj", + "model.layers.67.mlp.experts.158.down_proj", + "model.layers.67.mlp.experts.158.gate_proj", + "model.layers.67.mlp.experts.158.up_proj", + "model.layers.67.mlp.experts.159.down_proj", + "model.layers.67.mlp.experts.159.gate_proj", + "model.layers.67.mlp.experts.159.up_proj", + "model.layers.67.mlp.experts.16.down_proj", + "model.layers.67.mlp.experts.16.gate_proj", + "model.layers.67.mlp.experts.16.up_proj", + "model.layers.67.mlp.experts.160.down_proj", + "model.layers.67.mlp.experts.160.gate_proj", + "model.layers.67.mlp.experts.160.up_proj", + "model.layers.67.mlp.experts.161.down_proj", + "model.layers.67.mlp.experts.161.gate_proj", + "model.layers.67.mlp.experts.161.up_proj", + "model.layers.67.mlp.experts.162.down_proj", + "model.layers.67.mlp.experts.162.gate_proj", + "model.layers.67.mlp.experts.162.up_proj", + "model.layers.67.mlp.experts.163.down_proj", + "model.layers.67.mlp.experts.163.gate_proj", + "model.layers.67.mlp.experts.163.up_proj", + "model.layers.67.mlp.experts.164.down_proj", + "model.layers.67.mlp.experts.164.gate_proj", + "model.layers.67.mlp.experts.164.up_proj", + "model.layers.67.mlp.experts.165.down_proj", + "model.layers.67.mlp.experts.165.gate_proj", + "model.layers.67.mlp.experts.165.up_proj", + "model.layers.67.mlp.experts.166.down_proj", + "model.layers.67.mlp.experts.166.gate_proj", + "model.layers.67.mlp.experts.166.up_proj", + "model.layers.67.mlp.experts.167.down_proj", + "model.layers.67.mlp.experts.167.gate_proj", + "model.layers.67.mlp.experts.167.up_proj", + "model.layers.67.mlp.experts.168.down_proj", + "model.layers.67.mlp.experts.168.gate_proj", + "model.layers.67.mlp.experts.168.up_proj", + "model.layers.67.mlp.experts.169.down_proj", + "model.layers.67.mlp.experts.169.gate_proj", + "model.layers.67.mlp.experts.169.up_proj", + "model.layers.67.mlp.experts.17.down_proj", + "model.layers.67.mlp.experts.17.gate_proj", + "model.layers.67.mlp.experts.17.up_proj", + "model.layers.67.mlp.experts.170.down_proj", + "model.layers.67.mlp.experts.170.gate_proj", + "model.layers.67.mlp.experts.170.up_proj", + "model.layers.67.mlp.experts.171.down_proj", + "model.layers.67.mlp.experts.171.gate_proj", + "model.layers.67.mlp.experts.171.up_proj", + "model.layers.67.mlp.experts.172.down_proj", + "model.layers.67.mlp.experts.172.gate_proj", + "model.layers.67.mlp.experts.172.up_proj", + "model.layers.67.mlp.experts.173.down_proj", + "model.layers.67.mlp.experts.173.gate_proj", + "model.layers.67.mlp.experts.173.up_proj", + "model.layers.67.mlp.experts.174.down_proj", + "model.layers.67.mlp.experts.174.gate_proj", + "model.layers.67.mlp.experts.174.up_proj", + "model.layers.67.mlp.experts.175.down_proj", + "model.layers.67.mlp.experts.175.gate_proj", + "model.layers.67.mlp.experts.175.up_proj", + "model.layers.67.mlp.experts.176.down_proj", + "model.layers.67.mlp.experts.176.gate_proj", + "model.layers.67.mlp.experts.176.up_proj", + "model.layers.67.mlp.experts.177.down_proj", + "model.layers.67.mlp.experts.177.gate_proj", + "model.layers.67.mlp.experts.177.up_proj", + "model.layers.67.mlp.experts.178.down_proj", + "model.layers.67.mlp.experts.178.gate_proj", + "model.layers.67.mlp.experts.178.up_proj", + "model.layers.67.mlp.experts.179.down_proj", + "model.layers.67.mlp.experts.179.gate_proj", + "model.layers.67.mlp.experts.179.up_proj", + "model.layers.67.mlp.experts.18.down_proj", + "model.layers.67.mlp.experts.18.gate_proj", + "model.layers.67.mlp.experts.18.up_proj", + "model.layers.67.mlp.experts.180.down_proj", + "model.layers.67.mlp.experts.180.gate_proj", + "model.layers.67.mlp.experts.180.up_proj", + "model.layers.67.mlp.experts.181.down_proj", + "model.layers.67.mlp.experts.181.gate_proj", + "model.layers.67.mlp.experts.181.up_proj", + "model.layers.67.mlp.experts.182.down_proj", + "model.layers.67.mlp.experts.182.gate_proj", + "model.layers.67.mlp.experts.182.up_proj", + "model.layers.67.mlp.experts.183.down_proj", + "model.layers.67.mlp.experts.183.gate_proj", + "model.layers.67.mlp.experts.183.up_proj", + "model.layers.67.mlp.experts.184.down_proj", + "model.layers.67.mlp.experts.184.gate_proj", + "model.layers.67.mlp.experts.184.up_proj", + "model.layers.67.mlp.experts.185.down_proj", + "model.layers.67.mlp.experts.185.gate_proj", + "model.layers.67.mlp.experts.185.up_proj", + "model.layers.67.mlp.experts.186.down_proj", + "model.layers.67.mlp.experts.186.gate_proj", + "model.layers.67.mlp.experts.186.up_proj", + "model.layers.67.mlp.experts.187.down_proj", + "model.layers.67.mlp.experts.187.gate_proj", + "model.layers.67.mlp.experts.187.up_proj", + "model.layers.67.mlp.experts.188.down_proj", + "model.layers.67.mlp.experts.188.gate_proj", + "model.layers.67.mlp.experts.188.up_proj", + "model.layers.67.mlp.experts.189.down_proj", + "model.layers.67.mlp.experts.189.gate_proj", + "model.layers.67.mlp.experts.189.up_proj", + "model.layers.67.mlp.experts.19.down_proj", + "model.layers.67.mlp.experts.19.gate_proj", + "model.layers.67.mlp.experts.19.up_proj", + "model.layers.67.mlp.experts.190.down_proj", + "model.layers.67.mlp.experts.190.gate_proj", + "model.layers.67.mlp.experts.190.up_proj", + "model.layers.67.mlp.experts.191.down_proj", + "model.layers.67.mlp.experts.191.gate_proj", + "model.layers.67.mlp.experts.191.up_proj", + "model.layers.67.mlp.experts.192.down_proj", + "model.layers.67.mlp.experts.192.gate_proj", + "model.layers.67.mlp.experts.192.up_proj", + "model.layers.67.mlp.experts.193.down_proj", + "model.layers.67.mlp.experts.193.gate_proj", + "model.layers.67.mlp.experts.193.up_proj", + "model.layers.67.mlp.experts.194.down_proj", + "model.layers.67.mlp.experts.194.gate_proj", + "model.layers.67.mlp.experts.194.up_proj", + "model.layers.67.mlp.experts.195.down_proj", + "model.layers.67.mlp.experts.195.gate_proj", + "model.layers.67.mlp.experts.195.up_proj", + "model.layers.67.mlp.experts.196.down_proj", + "model.layers.67.mlp.experts.196.gate_proj", + "model.layers.67.mlp.experts.196.up_proj", + "model.layers.67.mlp.experts.197.down_proj", + "model.layers.67.mlp.experts.197.gate_proj", + "model.layers.67.mlp.experts.197.up_proj", + "model.layers.67.mlp.experts.198.down_proj", + "model.layers.67.mlp.experts.198.gate_proj", + "model.layers.67.mlp.experts.198.up_proj", + "model.layers.67.mlp.experts.199.down_proj", + "model.layers.67.mlp.experts.199.gate_proj", + "model.layers.67.mlp.experts.199.up_proj", + "model.layers.67.mlp.experts.2.down_proj", + "model.layers.67.mlp.experts.2.gate_proj", + "model.layers.67.mlp.experts.2.up_proj", + "model.layers.67.mlp.experts.20.down_proj", + "model.layers.67.mlp.experts.20.gate_proj", + "model.layers.67.mlp.experts.20.up_proj", + "model.layers.67.mlp.experts.200.down_proj", + "model.layers.67.mlp.experts.200.gate_proj", + "model.layers.67.mlp.experts.200.up_proj", + "model.layers.67.mlp.experts.201.down_proj", + "model.layers.67.mlp.experts.201.gate_proj", + "model.layers.67.mlp.experts.201.up_proj", + "model.layers.67.mlp.experts.202.down_proj", + "model.layers.67.mlp.experts.202.gate_proj", + "model.layers.67.mlp.experts.202.up_proj", + "model.layers.67.mlp.experts.203.down_proj", + "model.layers.67.mlp.experts.203.gate_proj", + "model.layers.67.mlp.experts.203.up_proj", + "model.layers.67.mlp.experts.204.down_proj", + "model.layers.67.mlp.experts.204.gate_proj", + "model.layers.67.mlp.experts.204.up_proj", + "model.layers.67.mlp.experts.205.down_proj", + "model.layers.67.mlp.experts.205.gate_proj", + "model.layers.67.mlp.experts.205.up_proj", + "model.layers.67.mlp.experts.206.down_proj", + "model.layers.67.mlp.experts.206.gate_proj", + "model.layers.67.mlp.experts.206.up_proj", + "model.layers.67.mlp.experts.207.down_proj", + "model.layers.67.mlp.experts.207.gate_proj", + "model.layers.67.mlp.experts.207.up_proj", + "model.layers.67.mlp.experts.208.down_proj", + "model.layers.67.mlp.experts.208.gate_proj", + "model.layers.67.mlp.experts.208.up_proj", + "model.layers.67.mlp.experts.209.down_proj", + "model.layers.67.mlp.experts.209.gate_proj", + "model.layers.67.mlp.experts.209.up_proj", + "model.layers.67.mlp.experts.21.down_proj", + "model.layers.67.mlp.experts.21.gate_proj", + "model.layers.67.mlp.experts.21.up_proj", + "model.layers.67.mlp.experts.210.down_proj", + "model.layers.67.mlp.experts.210.gate_proj", + "model.layers.67.mlp.experts.210.up_proj", + "model.layers.67.mlp.experts.211.down_proj", + "model.layers.67.mlp.experts.211.gate_proj", + "model.layers.67.mlp.experts.211.up_proj", + "model.layers.67.mlp.experts.212.down_proj", + "model.layers.67.mlp.experts.212.gate_proj", + "model.layers.67.mlp.experts.212.up_proj", + "model.layers.67.mlp.experts.213.down_proj", + "model.layers.67.mlp.experts.213.gate_proj", + "model.layers.67.mlp.experts.213.up_proj", + "model.layers.67.mlp.experts.214.down_proj", + "model.layers.67.mlp.experts.214.gate_proj", + "model.layers.67.mlp.experts.214.up_proj", + "model.layers.67.mlp.experts.215.down_proj", + "model.layers.67.mlp.experts.215.gate_proj", + "model.layers.67.mlp.experts.215.up_proj", + "model.layers.67.mlp.experts.216.down_proj", + "model.layers.67.mlp.experts.216.gate_proj", + "model.layers.67.mlp.experts.216.up_proj", + "model.layers.67.mlp.experts.217.down_proj", + "model.layers.67.mlp.experts.217.gate_proj", + "model.layers.67.mlp.experts.217.up_proj", + "model.layers.67.mlp.experts.218.down_proj", + "model.layers.67.mlp.experts.218.gate_proj", + "model.layers.67.mlp.experts.218.up_proj", + "model.layers.67.mlp.experts.219.down_proj", + "model.layers.67.mlp.experts.219.gate_proj", + "model.layers.67.mlp.experts.219.up_proj", + "model.layers.67.mlp.experts.22.down_proj", + "model.layers.67.mlp.experts.22.gate_proj", + "model.layers.67.mlp.experts.22.up_proj", + "model.layers.67.mlp.experts.220.down_proj", + "model.layers.67.mlp.experts.220.gate_proj", + "model.layers.67.mlp.experts.220.up_proj", + "model.layers.67.mlp.experts.221.down_proj", + "model.layers.67.mlp.experts.221.gate_proj", + "model.layers.67.mlp.experts.221.up_proj", + "model.layers.67.mlp.experts.222.down_proj", + "model.layers.67.mlp.experts.222.gate_proj", + "model.layers.67.mlp.experts.222.up_proj", + "model.layers.67.mlp.experts.223.down_proj", + "model.layers.67.mlp.experts.223.gate_proj", + "model.layers.67.mlp.experts.223.up_proj", + "model.layers.67.mlp.experts.224.down_proj", + "model.layers.67.mlp.experts.224.gate_proj", + "model.layers.67.mlp.experts.224.up_proj", + "model.layers.67.mlp.experts.225.down_proj", + "model.layers.67.mlp.experts.225.gate_proj", + "model.layers.67.mlp.experts.225.up_proj", + "model.layers.67.mlp.experts.226.down_proj", + "model.layers.67.mlp.experts.226.gate_proj", + "model.layers.67.mlp.experts.226.up_proj", + "model.layers.67.mlp.experts.227.down_proj", + "model.layers.67.mlp.experts.227.gate_proj", + "model.layers.67.mlp.experts.227.up_proj", + "model.layers.67.mlp.experts.228.down_proj", + "model.layers.67.mlp.experts.228.gate_proj", + "model.layers.67.mlp.experts.228.up_proj", + "model.layers.67.mlp.experts.229.down_proj", + "model.layers.67.mlp.experts.229.gate_proj", + "model.layers.67.mlp.experts.229.up_proj", + "model.layers.67.mlp.experts.23.down_proj", + "model.layers.67.mlp.experts.23.gate_proj", + "model.layers.67.mlp.experts.23.up_proj", + "model.layers.67.mlp.experts.230.down_proj", + "model.layers.67.mlp.experts.230.gate_proj", + "model.layers.67.mlp.experts.230.up_proj", + "model.layers.67.mlp.experts.231.down_proj", + "model.layers.67.mlp.experts.231.gate_proj", + "model.layers.67.mlp.experts.231.up_proj", + "model.layers.67.mlp.experts.232.down_proj", + "model.layers.67.mlp.experts.232.gate_proj", + "model.layers.67.mlp.experts.232.up_proj", + "model.layers.67.mlp.experts.233.down_proj", + "model.layers.67.mlp.experts.233.gate_proj", + "model.layers.67.mlp.experts.233.up_proj", + "model.layers.67.mlp.experts.234.down_proj", + "model.layers.67.mlp.experts.234.gate_proj", + "model.layers.67.mlp.experts.234.up_proj", + "model.layers.67.mlp.experts.235.down_proj", + "model.layers.67.mlp.experts.235.gate_proj", + "model.layers.67.mlp.experts.235.up_proj", + "model.layers.67.mlp.experts.236.down_proj", + "model.layers.67.mlp.experts.236.gate_proj", + "model.layers.67.mlp.experts.236.up_proj", + "model.layers.67.mlp.experts.237.down_proj", + "model.layers.67.mlp.experts.237.gate_proj", + "model.layers.67.mlp.experts.237.up_proj", + "model.layers.67.mlp.experts.238.down_proj", + "model.layers.67.mlp.experts.238.gate_proj", + "model.layers.67.mlp.experts.238.up_proj", + "model.layers.67.mlp.experts.239.down_proj", + "model.layers.67.mlp.experts.239.gate_proj", + "model.layers.67.mlp.experts.239.up_proj", + "model.layers.67.mlp.experts.24.down_proj", + "model.layers.67.mlp.experts.24.gate_proj", + "model.layers.67.mlp.experts.24.up_proj", + "model.layers.67.mlp.experts.240.down_proj", + "model.layers.67.mlp.experts.240.gate_proj", + "model.layers.67.mlp.experts.240.up_proj", + "model.layers.67.mlp.experts.241.down_proj", + "model.layers.67.mlp.experts.241.gate_proj", + "model.layers.67.mlp.experts.241.up_proj", + "model.layers.67.mlp.experts.242.down_proj", + "model.layers.67.mlp.experts.242.gate_proj", + "model.layers.67.mlp.experts.242.up_proj", + "model.layers.67.mlp.experts.243.down_proj", + "model.layers.67.mlp.experts.243.gate_proj", + "model.layers.67.mlp.experts.243.up_proj", + "model.layers.67.mlp.experts.244.down_proj", + "model.layers.67.mlp.experts.244.gate_proj", + "model.layers.67.mlp.experts.244.up_proj", + "model.layers.67.mlp.experts.245.down_proj", + "model.layers.67.mlp.experts.245.gate_proj", + "model.layers.67.mlp.experts.245.up_proj", + "model.layers.67.mlp.experts.246.down_proj", + "model.layers.67.mlp.experts.246.gate_proj", + "model.layers.67.mlp.experts.246.up_proj", + "model.layers.67.mlp.experts.247.down_proj", + "model.layers.67.mlp.experts.247.gate_proj", + "model.layers.67.mlp.experts.247.up_proj", + "model.layers.67.mlp.experts.248.down_proj", + "model.layers.67.mlp.experts.248.gate_proj", + "model.layers.67.mlp.experts.248.up_proj", + "model.layers.67.mlp.experts.249.down_proj", + "model.layers.67.mlp.experts.249.gate_proj", + "model.layers.67.mlp.experts.249.up_proj", + "model.layers.67.mlp.experts.25.down_proj", + "model.layers.67.mlp.experts.25.gate_proj", + "model.layers.67.mlp.experts.25.up_proj", + "model.layers.67.mlp.experts.250.down_proj", + "model.layers.67.mlp.experts.250.gate_proj", + "model.layers.67.mlp.experts.250.up_proj", + "model.layers.67.mlp.experts.251.down_proj", + "model.layers.67.mlp.experts.251.gate_proj", + "model.layers.67.mlp.experts.251.up_proj", + "model.layers.67.mlp.experts.252.down_proj", + "model.layers.67.mlp.experts.252.gate_proj", + "model.layers.67.mlp.experts.252.up_proj", + "model.layers.67.mlp.experts.253.down_proj", + "model.layers.67.mlp.experts.253.gate_proj", + "model.layers.67.mlp.experts.253.up_proj", + "model.layers.67.mlp.experts.254.down_proj", + "model.layers.67.mlp.experts.254.gate_proj", + "model.layers.67.mlp.experts.254.up_proj", + "model.layers.67.mlp.experts.255.down_proj", + "model.layers.67.mlp.experts.255.gate_proj", + "model.layers.67.mlp.experts.255.up_proj", + "model.layers.67.mlp.experts.26.down_proj", + "model.layers.67.mlp.experts.26.gate_proj", + "model.layers.67.mlp.experts.26.up_proj", + "model.layers.67.mlp.experts.27.down_proj", + "model.layers.67.mlp.experts.27.gate_proj", + "model.layers.67.mlp.experts.27.up_proj", + "model.layers.67.mlp.experts.28.down_proj", + "model.layers.67.mlp.experts.28.gate_proj", + "model.layers.67.mlp.experts.28.up_proj", + "model.layers.67.mlp.experts.29.down_proj", + "model.layers.67.mlp.experts.29.gate_proj", + "model.layers.67.mlp.experts.29.up_proj", + "model.layers.67.mlp.experts.3.down_proj", + "model.layers.67.mlp.experts.3.gate_proj", + "model.layers.67.mlp.experts.3.up_proj", + "model.layers.67.mlp.experts.30.down_proj", + "model.layers.67.mlp.experts.30.gate_proj", + "model.layers.67.mlp.experts.30.up_proj", + "model.layers.67.mlp.experts.31.down_proj", + "model.layers.67.mlp.experts.31.gate_proj", + "model.layers.67.mlp.experts.31.up_proj", + "model.layers.67.mlp.experts.32.down_proj", + "model.layers.67.mlp.experts.32.gate_proj", + "model.layers.67.mlp.experts.32.up_proj", + "model.layers.67.mlp.experts.33.down_proj", + "model.layers.67.mlp.experts.33.gate_proj", + "model.layers.67.mlp.experts.33.up_proj", + "model.layers.67.mlp.experts.34.down_proj", + "model.layers.67.mlp.experts.34.gate_proj", + "model.layers.67.mlp.experts.34.up_proj", + "model.layers.67.mlp.experts.35.down_proj", + "model.layers.67.mlp.experts.35.gate_proj", + "model.layers.67.mlp.experts.35.up_proj", + "model.layers.67.mlp.experts.36.down_proj", + "model.layers.67.mlp.experts.36.gate_proj", + "model.layers.67.mlp.experts.36.up_proj", + "model.layers.67.mlp.experts.37.down_proj", + "model.layers.67.mlp.experts.37.gate_proj", + "model.layers.67.mlp.experts.37.up_proj", + "model.layers.67.mlp.experts.38.down_proj", + "model.layers.67.mlp.experts.38.gate_proj", + "model.layers.67.mlp.experts.38.up_proj", + "model.layers.67.mlp.experts.39.down_proj", + "model.layers.67.mlp.experts.39.gate_proj", + "model.layers.67.mlp.experts.39.up_proj", + "model.layers.67.mlp.experts.4.down_proj", + "model.layers.67.mlp.experts.4.gate_proj", + "model.layers.67.mlp.experts.4.up_proj", + "model.layers.67.mlp.experts.40.down_proj", + "model.layers.67.mlp.experts.40.gate_proj", + "model.layers.67.mlp.experts.40.up_proj", + "model.layers.67.mlp.experts.41.down_proj", + "model.layers.67.mlp.experts.41.gate_proj", + "model.layers.67.mlp.experts.41.up_proj", + "model.layers.67.mlp.experts.42.down_proj", + "model.layers.67.mlp.experts.42.gate_proj", + "model.layers.67.mlp.experts.42.up_proj", + "model.layers.67.mlp.experts.43.down_proj", + "model.layers.67.mlp.experts.43.gate_proj", + "model.layers.67.mlp.experts.43.up_proj", + "model.layers.67.mlp.experts.44.down_proj", + "model.layers.67.mlp.experts.44.gate_proj", + "model.layers.67.mlp.experts.44.up_proj", + "model.layers.67.mlp.experts.45.down_proj", + "model.layers.67.mlp.experts.45.gate_proj", + "model.layers.67.mlp.experts.45.up_proj", + "model.layers.67.mlp.experts.46.down_proj", + "model.layers.67.mlp.experts.46.gate_proj", + "model.layers.67.mlp.experts.46.up_proj", + "model.layers.67.mlp.experts.47.down_proj", + "model.layers.67.mlp.experts.47.gate_proj", + "model.layers.67.mlp.experts.47.up_proj", + "model.layers.67.mlp.experts.48.down_proj", + "model.layers.67.mlp.experts.48.gate_proj", + "model.layers.67.mlp.experts.48.up_proj", + "model.layers.67.mlp.experts.49.down_proj", + "model.layers.67.mlp.experts.49.gate_proj", + "model.layers.67.mlp.experts.49.up_proj", + "model.layers.67.mlp.experts.5.down_proj", + "model.layers.67.mlp.experts.5.gate_proj", + "model.layers.67.mlp.experts.5.up_proj", + "model.layers.67.mlp.experts.50.down_proj", + "model.layers.67.mlp.experts.50.gate_proj", + "model.layers.67.mlp.experts.50.up_proj", + "model.layers.67.mlp.experts.51.down_proj", + "model.layers.67.mlp.experts.51.gate_proj", + "model.layers.67.mlp.experts.51.up_proj", + "model.layers.67.mlp.experts.52.down_proj", + "model.layers.67.mlp.experts.52.gate_proj", + "model.layers.67.mlp.experts.52.up_proj", + "model.layers.67.mlp.experts.53.down_proj", + "model.layers.67.mlp.experts.53.gate_proj", + "model.layers.67.mlp.experts.53.up_proj", + "model.layers.67.mlp.experts.54.down_proj", + "model.layers.67.mlp.experts.54.gate_proj", + "model.layers.67.mlp.experts.54.up_proj", + "model.layers.67.mlp.experts.55.down_proj", + "model.layers.67.mlp.experts.55.gate_proj", + "model.layers.67.mlp.experts.55.up_proj", + "model.layers.67.mlp.experts.56.down_proj", + "model.layers.67.mlp.experts.56.gate_proj", + "model.layers.67.mlp.experts.56.up_proj", + "model.layers.67.mlp.experts.57.down_proj", + "model.layers.67.mlp.experts.57.gate_proj", + "model.layers.67.mlp.experts.57.up_proj", + "model.layers.67.mlp.experts.58.down_proj", + "model.layers.67.mlp.experts.58.gate_proj", + "model.layers.67.mlp.experts.58.up_proj", + "model.layers.67.mlp.experts.59.down_proj", + "model.layers.67.mlp.experts.59.gate_proj", + "model.layers.67.mlp.experts.59.up_proj", + "model.layers.67.mlp.experts.6.down_proj", + "model.layers.67.mlp.experts.6.gate_proj", + "model.layers.67.mlp.experts.6.up_proj", + "model.layers.67.mlp.experts.60.down_proj", + "model.layers.67.mlp.experts.60.gate_proj", + "model.layers.67.mlp.experts.60.up_proj", + "model.layers.67.mlp.experts.61.down_proj", + "model.layers.67.mlp.experts.61.gate_proj", + "model.layers.67.mlp.experts.61.up_proj", + "model.layers.67.mlp.experts.62.down_proj", + "model.layers.67.mlp.experts.62.gate_proj", + "model.layers.67.mlp.experts.62.up_proj", + "model.layers.67.mlp.experts.63.down_proj", + "model.layers.67.mlp.experts.63.gate_proj", + "model.layers.67.mlp.experts.63.up_proj", + "model.layers.67.mlp.experts.64.down_proj", + "model.layers.67.mlp.experts.64.gate_proj", + "model.layers.67.mlp.experts.64.up_proj", + "model.layers.67.mlp.experts.65.down_proj", + "model.layers.67.mlp.experts.65.gate_proj", + "model.layers.67.mlp.experts.65.up_proj", + "model.layers.67.mlp.experts.66.down_proj", + "model.layers.67.mlp.experts.66.gate_proj", + "model.layers.67.mlp.experts.66.up_proj", + "model.layers.67.mlp.experts.67.down_proj", + "model.layers.67.mlp.experts.67.gate_proj", + "model.layers.67.mlp.experts.67.up_proj", + "model.layers.67.mlp.experts.68.down_proj", + "model.layers.67.mlp.experts.68.gate_proj", + "model.layers.67.mlp.experts.68.up_proj", + "model.layers.67.mlp.experts.69.down_proj", + "model.layers.67.mlp.experts.69.gate_proj", + "model.layers.67.mlp.experts.69.up_proj", + "model.layers.67.mlp.experts.7.down_proj", + "model.layers.67.mlp.experts.7.gate_proj", + "model.layers.67.mlp.experts.7.up_proj", + "model.layers.67.mlp.experts.70.down_proj", + "model.layers.67.mlp.experts.70.gate_proj", + "model.layers.67.mlp.experts.70.up_proj", + "model.layers.67.mlp.experts.71.down_proj", + "model.layers.67.mlp.experts.71.gate_proj", + "model.layers.67.mlp.experts.71.up_proj", + "model.layers.67.mlp.experts.72.down_proj", + "model.layers.67.mlp.experts.72.gate_proj", + "model.layers.67.mlp.experts.72.up_proj", + "model.layers.67.mlp.experts.73.down_proj", + "model.layers.67.mlp.experts.73.gate_proj", + "model.layers.67.mlp.experts.73.up_proj", + "model.layers.67.mlp.experts.74.down_proj", + "model.layers.67.mlp.experts.74.gate_proj", + "model.layers.67.mlp.experts.74.up_proj", + "model.layers.67.mlp.experts.75.down_proj", + "model.layers.67.mlp.experts.75.gate_proj", + "model.layers.67.mlp.experts.75.up_proj", + "model.layers.67.mlp.experts.76.down_proj", + "model.layers.67.mlp.experts.76.gate_proj", + "model.layers.67.mlp.experts.76.up_proj", + "model.layers.67.mlp.experts.77.down_proj", + "model.layers.67.mlp.experts.77.gate_proj", + "model.layers.67.mlp.experts.77.up_proj", + "model.layers.67.mlp.experts.78.down_proj", + "model.layers.67.mlp.experts.78.gate_proj", + "model.layers.67.mlp.experts.78.up_proj", + "model.layers.67.mlp.experts.79.down_proj", + "model.layers.67.mlp.experts.79.gate_proj", + "model.layers.67.mlp.experts.79.up_proj", + "model.layers.67.mlp.experts.8.down_proj", + "model.layers.67.mlp.experts.8.gate_proj", + "model.layers.67.mlp.experts.8.up_proj", + "model.layers.67.mlp.experts.80.down_proj", + "model.layers.67.mlp.experts.80.gate_proj", + "model.layers.67.mlp.experts.80.up_proj", + "model.layers.67.mlp.experts.81.down_proj", + "model.layers.67.mlp.experts.81.gate_proj", + "model.layers.67.mlp.experts.81.up_proj", + "model.layers.67.mlp.experts.82.down_proj", + "model.layers.67.mlp.experts.82.gate_proj", + "model.layers.67.mlp.experts.82.up_proj", + "model.layers.67.mlp.experts.83.down_proj", + "model.layers.67.mlp.experts.83.gate_proj", + "model.layers.67.mlp.experts.83.up_proj", + "model.layers.67.mlp.experts.84.down_proj", + "model.layers.67.mlp.experts.84.gate_proj", + "model.layers.67.mlp.experts.84.up_proj", + "model.layers.67.mlp.experts.85.down_proj", + "model.layers.67.mlp.experts.85.gate_proj", + "model.layers.67.mlp.experts.85.up_proj", + "model.layers.67.mlp.experts.86.down_proj", + "model.layers.67.mlp.experts.86.gate_proj", + "model.layers.67.mlp.experts.86.up_proj", + "model.layers.67.mlp.experts.87.down_proj", + "model.layers.67.mlp.experts.87.gate_proj", + "model.layers.67.mlp.experts.87.up_proj", + "model.layers.67.mlp.experts.88.down_proj", + "model.layers.67.mlp.experts.88.gate_proj", + "model.layers.67.mlp.experts.88.up_proj", + "model.layers.67.mlp.experts.89.down_proj", + "model.layers.67.mlp.experts.89.gate_proj", + "model.layers.67.mlp.experts.89.up_proj", + "model.layers.67.mlp.experts.9.down_proj", + "model.layers.67.mlp.experts.9.gate_proj", + "model.layers.67.mlp.experts.9.up_proj", + "model.layers.67.mlp.experts.90.down_proj", + "model.layers.67.mlp.experts.90.gate_proj", + "model.layers.67.mlp.experts.90.up_proj", + "model.layers.67.mlp.experts.91.down_proj", + "model.layers.67.mlp.experts.91.gate_proj", + "model.layers.67.mlp.experts.91.up_proj", + "model.layers.67.mlp.experts.92.down_proj", + "model.layers.67.mlp.experts.92.gate_proj", + "model.layers.67.mlp.experts.92.up_proj", + "model.layers.67.mlp.experts.93.down_proj", + "model.layers.67.mlp.experts.93.gate_proj", + "model.layers.67.mlp.experts.93.up_proj", + "model.layers.67.mlp.experts.94.down_proj", + "model.layers.67.mlp.experts.94.gate_proj", + "model.layers.67.mlp.experts.94.up_proj", + "model.layers.67.mlp.experts.95.down_proj", + "model.layers.67.mlp.experts.95.gate_proj", + "model.layers.67.mlp.experts.95.up_proj", + "model.layers.67.mlp.experts.96.down_proj", + "model.layers.67.mlp.experts.96.gate_proj", + "model.layers.67.mlp.experts.96.up_proj", + "model.layers.67.mlp.experts.97.down_proj", + "model.layers.67.mlp.experts.97.gate_proj", + "model.layers.67.mlp.experts.97.up_proj", + "model.layers.67.mlp.experts.98.down_proj", + "model.layers.67.mlp.experts.98.gate_proj", + "model.layers.67.mlp.experts.98.up_proj", + "model.layers.67.mlp.experts.99.down_proj", + "model.layers.67.mlp.experts.99.gate_proj", + "model.layers.67.mlp.experts.99.up_proj", + "model.layers.68.mlp.experts.0.down_proj", + "model.layers.68.mlp.experts.0.gate_proj", + "model.layers.68.mlp.experts.0.up_proj", + "model.layers.68.mlp.experts.1.down_proj", + "model.layers.68.mlp.experts.1.gate_proj", + "model.layers.68.mlp.experts.1.up_proj", + "model.layers.68.mlp.experts.10.down_proj", + "model.layers.68.mlp.experts.10.gate_proj", + "model.layers.68.mlp.experts.10.up_proj", + "model.layers.68.mlp.experts.100.down_proj", + "model.layers.68.mlp.experts.100.gate_proj", + "model.layers.68.mlp.experts.100.up_proj", + "model.layers.68.mlp.experts.101.down_proj", + "model.layers.68.mlp.experts.101.gate_proj", + "model.layers.68.mlp.experts.101.up_proj", + "model.layers.68.mlp.experts.102.down_proj", + "model.layers.68.mlp.experts.102.gate_proj", + "model.layers.68.mlp.experts.102.up_proj", + "model.layers.68.mlp.experts.103.down_proj", + "model.layers.68.mlp.experts.103.gate_proj", + "model.layers.68.mlp.experts.103.up_proj", + "model.layers.68.mlp.experts.104.down_proj", + "model.layers.68.mlp.experts.104.gate_proj", + "model.layers.68.mlp.experts.104.up_proj", + "model.layers.68.mlp.experts.105.down_proj", + "model.layers.68.mlp.experts.105.gate_proj", + "model.layers.68.mlp.experts.105.up_proj", + "model.layers.68.mlp.experts.106.down_proj", + "model.layers.68.mlp.experts.106.gate_proj", + "model.layers.68.mlp.experts.106.up_proj", + "model.layers.68.mlp.experts.107.down_proj", + "model.layers.68.mlp.experts.107.gate_proj", + "model.layers.68.mlp.experts.107.up_proj", + "model.layers.68.mlp.experts.108.down_proj", + "model.layers.68.mlp.experts.108.gate_proj", + "model.layers.68.mlp.experts.108.up_proj", + "model.layers.68.mlp.experts.109.down_proj", + "model.layers.68.mlp.experts.109.gate_proj", + "model.layers.68.mlp.experts.109.up_proj", + "model.layers.68.mlp.experts.11.down_proj", + "model.layers.68.mlp.experts.11.gate_proj", + "model.layers.68.mlp.experts.11.up_proj", + "model.layers.68.mlp.experts.110.down_proj", + "model.layers.68.mlp.experts.110.gate_proj", + "model.layers.68.mlp.experts.110.up_proj", + "model.layers.68.mlp.experts.111.down_proj", + "model.layers.68.mlp.experts.111.gate_proj", + "model.layers.68.mlp.experts.111.up_proj", + "model.layers.68.mlp.experts.112.down_proj", + "model.layers.68.mlp.experts.112.gate_proj", + "model.layers.68.mlp.experts.112.up_proj", + "model.layers.68.mlp.experts.113.down_proj", + "model.layers.68.mlp.experts.113.gate_proj", + "model.layers.68.mlp.experts.113.up_proj", + "model.layers.68.mlp.experts.114.down_proj", + "model.layers.68.mlp.experts.114.gate_proj", + "model.layers.68.mlp.experts.114.up_proj", + "model.layers.68.mlp.experts.115.down_proj", + "model.layers.68.mlp.experts.115.gate_proj", + "model.layers.68.mlp.experts.115.up_proj", + "model.layers.68.mlp.experts.116.down_proj", + "model.layers.68.mlp.experts.116.gate_proj", + "model.layers.68.mlp.experts.116.up_proj", + "model.layers.68.mlp.experts.117.down_proj", + "model.layers.68.mlp.experts.117.gate_proj", + "model.layers.68.mlp.experts.117.up_proj", + "model.layers.68.mlp.experts.118.down_proj", + "model.layers.68.mlp.experts.118.gate_proj", + "model.layers.68.mlp.experts.118.up_proj", + "model.layers.68.mlp.experts.119.down_proj", + "model.layers.68.mlp.experts.119.gate_proj", + "model.layers.68.mlp.experts.119.up_proj", + "model.layers.68.mlp.experts.12.down_proj", + "model.layers.68.mlp.experts.12.gate_proj", + "model.layers.68.mlp.experts.12.up_proj", + "model.layers.68.mlp.experts.120.down_proj", + "model.layers.68.mlp.experts.120.gate_proj", + "model.layers.68.mlp.experts.120.up_proj", + "model.layers.68.mlp.experts.121.down_proj", + "model.layers.68.mlp.experts.121.gate_proj", + "model.layers.68.mlp.experts.121.up_proj", + "model.layers.68.mlp.experts.122.down_proj", + "model.layers.68.mlp.experts.122.gate_proj", + "model.layers.68.mlp.experts.122.up_proj", + "model.layers.68.mlp.experts.123.down_proj", + "model.layers.68.mlp.experts.123.gate_proj", + "model.layers.68.mlp.experts.123.up_proj", + "model.layers.68.mlp.experts.124.down_proj", + "model.layers.68.mlp.experts.124.gate_proj", + "model.layers.68.mlp.experts.124.up_proj", + "model.layers.68.mlp.experts.125.down_proj", + "model.layers.68.mlp.experts.125.gate_proj", + "model.layers.68.mlp.experts.125.up_proj", + "model.layers.68.mlp.experts.126.down_proj", + "model.layers.68.mlp.experts.126.gate_proj", + "model.layers.68.mlp.experts.126.up_proj", + "model.layers.68.mlp.experts.127.down_proj", + "model.layers.68.mlp.experts.127.gate_proj", + "model.layers.68.mlp.experts.127.up_proj", + "model.layers.68.mlp.experts.128.down_proj", + "model.layers.68.mlp.experts.128.gate_proj", + "model.layers.68.mlp.experts.128.up_proj", + "model.layers.68.mlp.experts.129.down_proj", + "model.layers.68.mlp.experts.129.gate_proj", + "model.layers.68.mlp.experts.129.up_proj", + "model.layers.68.mlp.experts.13.down_proj", + "model.layers.68.mlp.experts.13.gate_proj", + "model.layers.68.mlp.experts.13.up_proj", + "model.layers.68.mlp.experts.130.down_proj", + "model.layers.68.mlp.experts.130.gate_proj", + "model.layers.68.mlp.experts.130.up_proj", + "model.layers.68.mlp.experts.131.down_proj", + "model.layers.68.mlp.experts.131.gate_proj", + "model.layers.68.mlp.experts.131.up_proj", + "model.layers.68.mlp.experts.132.down_proj", + "model.layers.68.mlp.experts.132.gate_proj", + "model.layers.68.mlp.experts.132.up_proj", + "model.layers.68.mlp.experts.133.down_proj", + "model.layers.68.mlp.experts.133.gate_proj", + "model.layers.68.mlp.experts.133.up_proj", + "model.layers.68.mlp.experts.134.down_proj", + "model.layers.68.mlp.experts.134.gate_proj", + "model.layers.68.mlp.experts.134.up_proj", + "model.layers.68.mlp.experts.135.down_proj", + "model.layers.68.mlp.experts.135.gate_proj", + "model.layers.68.mlp.experts.135.up_proj", + "model.layers.68.mlp.experts.136.down_proj", + "model.layers.68.mlp.experts.136.gate_proj", + "model.layers.68.mlp.experts.136.up_proj", + "model.layers.68.mlp.experts.137.down_proj", + "model.layers.68.mlp.experts.137.gate_proj", + "model.layers.68.mlp.experts.137.up_proj", + "model.layers.68.mlp.experts.138.down_proj", + "model.layers.68.mlp.experts.138.gate_proj", + "model.layers.68.mlp.experts.138.up_proj", + "model.layers.68.mlp.experts.139.down_proj", + "model.layers.68.mlp.experts.139.gate_proj", + "model.layers.68.mlp.experts.139.up_proj", + "model.layers.68.mlp.experts.14.down_proj", + "model.layers.68.mlp.experts.14.gate_proj", + "model.layers.68.mlp.experts.14.up_proj", + "model.layers.68.mlp.experts.140.down_proj", + "model.layers.68.mlp.experts.140.gate_proj", + "model.layers.68.mlp.experts.140.up_proj", + "model.layers.68.mlp.experts.141.down_proj", + "model.layers.68.mlp.experts.141.gate_proj", + "model.layers.68.mlp.experts.141.up_proj", + "model.layers.68.mlp.experts.142.down_proj", + "model.layers.68.mlp.experts.142.gate_proj", + "model.layers.68.mlp.experts.142.up_proj", + "model.layers.68.mlp.experts.143.down_proj", + "model.layers.68.mlp.experts.143.gate_proj", + "model.layers.68.mlp.experts.143.up_proj", + "model.layers.68.mlp.experts.144.down_proj", + "model.layers.68.mlp.experts.144.gate_proj", + "model.layers.68.mlp.experts.144.up_proj", + "model.layers.68.mlp.experts.145.down_proj", + "model.layers.68.mlp.experts.145.gate_proj", + "model.layers.68.mlp.experts.145.up_proj", + "model.layers.68.mlp.experts.146.down_proj", + "model.layers.68.mlp.experts.146.gate_proj", + "model.layers.68.mlp.experts.146.up_proj", + "model.layers.68.mlp.experts.147.down_proj", + "model.layers.68.mlp.experts.147.gate_proj", + "model.layers.68.mlp.experts.147.up_proj", + "model.layers.68.mlp.experts.148.down_proj", + "model.layers.68.mlp.experts.148.gate_proj", + "model.layers.68.mlp.experts.148.up_proj", + "model.layers.68.mlp.experts.149.down_proj", + "model.layers.68.mlp.experts.149.gate_proj", + "model.layers.68.mlp.experts.149.up_proj", + "model.layers.68.mlp.experts.15.down_proj", + "model.layers.68.mlp.experts.15.gate_proj", + "model.layers.68.mlp.experts.15.up_proj", + "model.layers.68.mlp.experts.150.down_proj", + "model.layers.68.mlp.experts.150.gate_proj", + "model.layers.68.mlp.experts.150.up_proj", + "model.layers.68.mlp.experts.151.down_proj", + "model.layers.68.mlp.experts.151.gate_proj", + "model.layers.68.mlp.experts.151.up_proj", + "model.layers.68.mlp.experts.152.down_proj", + "model.layers.68.mlp.experts.152.gate_proj", + "model.layers.68.mlp.experts.152.up_proj", + "model.layers.68.mlp.experts.153.down_proj", + "model.layers.68.mlp.experts.153.gate_proj", + "model.layers.68.mlp.experts.153.up_proj", + "model.layers.68.mlp.experts.154.down_proj", + "model.layers.68.mlp.experts.154.gate_proj", + "model.layers.68.mlp.experts.154.up_proj", + "model.layers.68.mlp.experts.155.down_proj", + "model.layers.68.mlp.experts.155.gate_proj", + "model.layers.68.mlp.experts.155.up_proj", + "model.layers.68.mlp.experts.156.down_proj", + "model.layers.68.mlp.experts.156.gate_proj", + "model.layers.68.mlp.experts.156.up_proj", + "model.layers.68.mlp.experts.157.down_proj", + "model.layers.68.mlp.experts.157.gate_proj", + "model.layers.68.mlp.experts.157.up_proj", + "model.layers.68.mlp.experts.158.down_proj", + "model.layers.68.mlp.experts.158.gate_proj", + "model.layers.68.mlp.experts.158.up_proj", + "model.layers.68.mlp.experts.159.down_proj", + "model.layers.68.mlp.experts.159.gate_proj", + "model.layers.68.mlp.experts.159.up_proj", + "model.layers.68.mlp.experts.16.down_proj", + "model.layers.68.mlp.experts.16.gate_proj", + "model.layers.68.mlp.experts.16.up_proj", + "model.layers.68.mlp.experts.160.down_proj", + "model.layers.68.mlp.experts.160.gate_proj", + "model.layers.68.mlp.experts.160.up_proj", + "model.layers.68.mlp.experts.161.down_proj", + "model.layers.68.mlp.experts.161.gate_proj", + "model.layers.68.mlp.experts.161.up_proj", + "model.layers.68.mlp.experts.162.down_proj", + "model.layers.68.mlp.experts.162.gate_proj", + "model.layers.68.mlp.experts.162.up_proj", + "model.layers.68.mlp.experts.163.down_proj", + "model.layers.68.mlp.experts.163.gate_proj", + "model.layers.68.mlp.experts.163.up_proj", + "model.layers.68.mlp.experts.164.down_proj", + "model.layers.68.mlp.experts.164.gate_proj", + "model.layers.68.mlp.experts.164.up_proj", + "model.layers.68.mlp.experts.165.down_proj", + "model.layers.68.mlp.experts.165.gate_proj", + "model.layers.68.mlp.experts.165.up_proj", + "model.layers.68.mlp.experts.166.down_proj", + "model.layers.68.mlp.experts.166.gate_proj", + "model.layers.68.mlp.experts.166.up_proj", + "model.layers.68.mlp.experts.167.down_proj", + "model.layers.68.mlp.experts.167.gate_proj", + "model.layers.68.mlp.experts.167.up_proj", + "model.layers.68.mlp.experts.168.down_proj", + "model.layers.68.mlp.experts.168.gate_proj", + "model.layers.68.mlp.experts.168.up_proj", + "model.layers.68.mlp.experts.169.down_proj", + "model.layers.68.mlp.experts.169.gate_proj", + "model.layers.68.mlp.experts.169.up_proj", + "model.layers.68.mlp.experts.17.down_proj", + "model.layers.68.mlp.experts.17.gate_proj", + "model.layers.68.mlp.experts.17.up_proj", + "model.layers.68.mlp.experts.170.down_proj", + "model.layers.68.mlp.experts.170.gate_proj", + "model.layers.68.mlp.experts.170.up_proj", + "model.layers.68.mlp.experts.171.down_proj", + "model.layers.68.mlp.experts.171.gate_proj", + "model.layers.68.mlp.experts.171.up_proj", + "model.layers.68.mlp.experts.172.down_proj", + "model.layers.68.mlp.experts.172.gate_proj", + "model.layers.68.mlp.experts.172.up_proj", + "model.layers.68.mlp.experts.173.down_proj", + "model.layers.68.mlp.experts.173.gate_proj", + "model.layers.68.mlp.experts.173.up_proj", + "model.layers.68.mlp.experts.174.down_proj", + "model.layers.68.mlp.experts.174.gate_proj", + "model.layers.68.mlp.experts.174.up_proj", + "model.layers.68.mlp.experts.175.down_proj", + "model.layers.68.mlp.experts.175.gate_proj", + "model.layers.68.mlp.experts.175.up_proj", + "model.layers.68.mlp.experts.176.down_proj", + "model.layers.68.mlp.experts.176.gate_proj", + "model.layers.68.mlp.experts.176.up_proj", + "model.layers.68.mlp.experts.177.down_proj", + "model.layers.68.mlp.experts.177.gate_proj", + "model.layers.68.mlp.experts.177.up_proj", + "model.layers.68.mlp.experts.178.down_proj", + "model.layers.68.mlp.experts.178.gate_proj", + "model.layers.68.mlp.experts.178.up_proj", + "model.layers.68.mlp.experts.179.down_proj", + "model.layers.68.mlp.experts.179.gate_proj", + "model.layers.68.mlp.experts.179.up_proj", + "model.layers.68.mlp.experts.18.down_proj", + "model.layers.68.mlp.experts.18.gate_proj", + "model.layers.68.mlp.experts.18.up_proj", + "model.layers.68.mlp.experts.180.down_proj", + "model.layers.68.mlp.experts.180.gate_proj", + "model.layers.68.mlp.experts.180.up_proj", + "model.layers.68.mlp.experts.181.down_proj", + "model.layers.68.mlp.experts.181.gate_proj", + "model.layers.68.mlp.experts.181.up_proj", + "model.layers.68.mlp.experts.182.down_proj", + "model.layers.68.mlp.experts.182.gate_proj", + "model.layers.68.mlp.experts.182.up_proj", + "model.layers.68.mlp.experts.183.down_proj", + "model.layers.68.mlp.experts.183.gate_proj", + "model.layers.68.mlp.experts.183.up_proj", + "model.layers.68.mlp.experts.184.down_proj", + "model.layers.68.mlp.experts.184.gate_proj", + "model.layers.68.mlp.experts.184.up_proj", + "model.layers.68.mlp.experts.185.down_proj", + "model.layers.68.mlp.experts.185.gate_proj", + "model.layers.68.mlp.experts.185.up_proj", + "model.layers.68.mlp.experts.186.down_proj", + "model.layers.68.mlp.experts.186.gate_proj", + "model.layers.68.mlp.experts.186.up_proj", + "model.layers.68.mlp.experts.187.down_proj", + "model.layers.68.mlp.experts.187.gate_proj", + "model.layers.68.mlp.experts.187.up_proj", + "model.layers.68.mlp.experts.188.down_proj", + "model.layers.68.mlp.experts.188.gate_proj", + "model.layers.68.mlp.experts.188.up_proj", + "model.layers.68.mlp.experts.189.down_proj", + "model.layers.68.mlp.experts.189.gate_proj", + "model.layers.68.mlp.experts.189.up_proj", + "model.layers.68.mlp.experts.19.down_proj", + "model.layers.68.mlp.experts.19.gate_proj", + "model.layers.68.mlp.experts.19.up_proj", + "model.layers.68.mlp.experts.190.down_proj", + "model.layers.68.mlp.experts.190.gate_proj", + "model.layers.68.mlp.experts.190.up_proj", + "model.layers.68.mlp.experts.191.down_proj", + "model.layers.68.mlp.experts.191.gate_proj", + "model.layers.68.mlp.experts.191.up_proj", + "model.layers.68.mlp.experts.192.down_proj", + "model.layers.68.mlp.experts.192.gate_proj", + "model.layers.68.mlp.experts.192.up_proj", + "model.layers.68.mlp.experts.193.down_proj", + "model.layers.68.mlp.experts.193.gate_proj", + "model.layers.68.mlp.experts.193.up_proj", + "model.layers.68.mlp.experts.194.down_proj", + "model.layers.68.mlp.experts.194.gate_proj", + "model.layers.68.mlp.experts.194.up_proj", + "model.layers.68.mlp.experts.195.down_proj", + "model.layers.68.mlp.experts.195.gate_proj", + "model.layers.68.mlp.experts.195.up_proj", + "model.layers.68.mlp.experts.196.down_proj", + "model.layers.68.mlp.experts.196.gate_proj", + "model.layers.68.mlp.experts.196.up_proj", + "model.layers.68.mlp.experts.197.down_proj", + "model.layers.68.mlp.experts.197.gate_proj", + "model.layers.68.mlp.experts.197.up_proj", + "model.layers.68.mlp.experts.198.down_proj", + "model.layers.68.mlp.experts.198.gate_proj", + "model.layers.68.mlp.experts.198.up_proj", + "model.layers.68.mlp.experts.199.down_proj", + "model.layers.68.mlp.experts.199.gate_proj", + "model.layers.68.mlp.experts.199.up_proj", + "model.layers.68.mlp.experts.2.down_proj", + "model.layers.68.mlp.experts.2.gate_proj", + "model.layers.68.mlp.experts.2.up_proj", + "model.layers.68.mlp.experts.20.down_proj", + "model.layers.68.mlp.experts.20.gate_proj", + "model.layers.68.mlp.experts.20.up_proj", + "model.layers.68.mlp.experts.200.down_proj", + "model.layers.68.mlp.experts.200.gate_proj", + "model.layers.68.mlp.experts.200.up_proj", + "model.layers.68.mlp.experts.201.down_proj", + "model.layers.68.mlp.experts.201.gate_proj", + "model.layers.68.mlp.experts.201.up_proj", + "model.layers.68.mlp.experts.202.down_proj", + "model.layers.68.mlp.experts.202.gate_proj", + "model.layers.68.mlp.experts.202.up_proj", + "model.layers.68.mlp.experts.203.down_proj", + "model.layers.68.mlp.experts.203.gate_proj", + "model.layers.68.mlp.experts.203.up_proj", + "model.layers.68.mlp.experts.204.down_proj", + "model.layers.68.mlp.experts.204.gate_proj", + "model.layers.68.mlp.experts.204.up_proj", + "model.layers.68.mlp.experts.205.down_proj", + "model.layers.68.mlp.experts.205.gate_proj", + "model.layers.68.mlp.experts.205.up_proj", + "model.layers.68.mlp.experts.206.down_proj", + "model.layers.68.mlp.experts.206.gate_proj", + "model.layers.68.mlp.experts.206.up_proj", + "model.layers.68.mlp.experts.207.down_proj", + "model.layers.68.mlp.experts.207.gate_proj", + "model.layers.68.mlp.experts.207.up_proj", + "model.layers.68.mlp.experts.208.down_proj", + "model.layers.68.mlp.experts.208.gate_proj", + "model.layers.68.mlp.experts.208.up_proj", + "model.layers.68.mlp.experts.209.down_proj", + "model.layers.68.mlp.experts.209.gate_proj", + "model.layers.68.mlp.experts.209.up_proj", + "model.layers.68.mlp.experts.21.down_proj", + "model.layers.68.mlp.experts.21.gate_proj", + "model.layers.68.mlp.experts.21.up_proj", + "model.layers.68.mlp.experts.210.down_proj", + "model.layers.68.mlp.experts.210.gate_proj", + "model.layers.68.mlp.experts.210.up_proj", + "model.layers.68.mlp.experts.211.down_proj", + "model.layers.68.mlp.experts.211.gate_proj", + "model.layers.68.mlp.experts.211.up_proj", + "model.layers.68.mlp.experts.212.down_proj", + "model.layers.68.mlp.experts.212.gate_proj", + "model.layers.68.mlp.experts.212.up_proj", + "model.layers.68.mlp.experts.213.down_proj", + "model.layers.68.mlp.experts.213.gate_proj", + "model.layers.68.mlp.experts.213.up_proj", + "model.layers.68.mlp.experts.214.down_proj", + "model.layers.68.mlp.experts.214.gate_proj", + "model.layers.68.mlp.experts.214.up_proj", + "model.layers.68.mlp.experts.215.down_proj", + "model.layers.68.mlp.experts.215.gate_proj", + "model.layers.68.mlp.experts.215.up_proj", + "model.layers.68.mlp.experts.216.down_proj", + "model.layers.68.mlp.experts.216.gate_proj", + "model.layers.68.mlp.experts.216.up_proj", + "model.layers.68.mlp.experts.217.down_proj", + "model.layers.68.mlp.experts.217.gate_proj", + "model.layers.68.mlp.experts.217.up_proj", + "model.layers.68.mlp.experts.218.down_proj", + "model.layers.68.mlp.experts.218.gate_proj", + "model.layers.68.mlp.experts.218.up_proj", + "model.layers.68.mlp.experts.219.down_proj", + "model.layers.68.mlp.experts.219.gate_proj", + "model.layers.68.mlp.experts.219.up_proj", + "model.layers.68.mlp.experts.22.down_proj", + "model.layers.68.mlp.experts.22.gate_proj", + "model.layers.68.mlp.experts.22.up_proj", + "model.layers.68.mlp.experts.220.down_proj", + "model.layers.68.mlp.experts.220.gate_proj", + "model.layers.68.mlp.experts.220.up_proj", + "model.layers.68.mlp.experts.221.down_proj", + "model.layers.68.mlp.experts.221.gate_proj", + "model.layers.68.mlp.experts.221.up_proj", + "model.layers.68.mlp.experts.222.down_proj", + "model.layers.68.mlp.experts.222.gate_proj", + "model.layers.68.mlp.experts.222.up_proj", + "model.layers.68.mlp.experts.223.down_proj", + "model.layers.68.mlp.experts.223.gate_proj", + "model.layers.68.mlp.experts.223.up_proj", + "model.layers.68.mlp.experts.224.down_proj", + "model.layers.68.mlp.experts.224.gate_proj", + "model.layers.68.mlp.experts.224.up_proj", + "model.layers.68.mlp.experts.225.down_proj", + "model.layers.68.mlp.experts.225.gate_proj", + "model.layers.68.mlp.experts.225.up_proj", + "model.layers.68.mlp.experts.226.down_proj", + "model.layers.68.mlp.experts.226.gate_proj", + "model.layers.68.mlp.experts.226.up_proj", + "model.layers.68.mlp.experts.227.down_proj", + "model.layers.68.mlp.experts.227.gate_proj", + "model.layers.68.mlp.experts.227.up_proj", + "model.layers.68.mlp.experts.228.down_proj", + "model.layers.68.mlp.experts.228.gate_proj", + "model.layers.68.mlp.experts.228.up_proj", + "model.layers.68.mlp.experts.229.down_proj", + "model.layers.68.mlp.experts.229.gate_proj", + "model.layers.68.mlp.experts.229.up_proj", + "model.layers.68.mlp.experts.23.down_proj", + "model.layers.68.mlp.experts.23.gate_proj", + "model.layers.68.mlp.experts.23.up_proj", + "model.layers.68.mlp.experts.230.down_proj", + "model.layers.68.mlp.experts.230.gate_proj", + "model.layers.68.mlp.experts.230.up_proj", + "model.layers.68.mlp.experts.231.down_proj", + "model.layers.68.mlp.experts.231.gate_proj", + "model.layers.68.mlp.experts.231.up_proj", + "model.layers.68.mlp.experts.232.down_proj", + "model.layers.68.mlp.experts.232.gate_proj", + "model.layers.68.mlp.experts.232.up_proj", + "model.layers.68.mlp.experts.233.down_proj", + "model.layers.68.mlp.experts.233.gate_proj", + "model.layers.68.mlp.experts.233.up_proj", + "model.layers.68.mlp.experts.234.down_proj", + "model.layers.68.mlp.experts.234.gate_proj", + "model.layers.68.mlp.experts.234.up_proj", + "model.layers.68.mlp.experts.235.down_proj", + "model.layers.68.mlp.experts.235.gate_proj", + "model.layers.68.mlp.experts.235.up_proj", + "model.layers.68.mlp.experts.236.down_proj", + "model.layers.68.mlp.experts.236.gate_proj", + "model.layers.68.mlp.experts.236.up_proj", + "model.layers.68.mlp.experts.237.down_proj", + "model.layers.68.mlp.experts.237.gate_proj", + "model.layers.68.mlp.experts.237.up_proj", + "model.layers.68.mlp.experts.238.down_proj", + "model.layers.68.mlp.experts.238.gate_proj", + "model.layers.68.mlp.experts.238.up_proj", + "model.layers.68.mlp.experts.239.down_proj", + "model.layers.68.mlp.experts.239.gate_proj", + "model.layers.68.mlp.experts.239.up_proj", + "model.layers.68.mlp.experts.24.down_proj", + "model.layers.68.mlp.experts.24.gate_proj", + "model.layers.68.mlp.experts.24.up_proj", + "model.layers.68.mlp.experts.240.down_proj", + "model.layers.68.mlp.experts.240.gate_proj", + "model.layers.68.mlp.experts.240.up_proj", + "model.layers.68.mlp.experts.241.down_proj", + "model.layers.68.mlp.experts.241.gate_proj", + "model.layers.68.mlp.experts.241.up_proj", + "model.layers.68.mlp.experts.242.down_proj", + "model.layers.68.mlp.experts.242.gate_proj", + "model.layers.68.mlp.experts.242.up_proj", + "model.layers.68.mlp.experts.243.down_proj", + "model.layers.68.mlp.experts.243.gate_proj", + "model.layers.68.mlp.experts.243.up_proj", + "model.layers.68.mlp.experts.244.down_proj", + "model.layers.68.mlp.experts.244.gate_proj", + "model.layers.68.mlp.experts.244.up_proj", + "model.layers.68.mlp.experts.245.down_proj", + "model.layers.68.mlp.experts.245.gate_proj", + "model.layers.68.mlp.experts.245.up_proj", + "model.layers.68.mlp.experts.246.down_proj", + "model.layers.68.mlp.experts.246.gate_proj", + "model.layers.68.mlp.experts.246.up_proj", + "model.layers.68.mlp.experts.247.down_proj", + "model.layers.68.mlp.experts.247.gate_proj", + "model.layers.68.mlp.experts.247.up_proj", + "model.layers.68.mlp.experts.248.down_proj", + "model.layers.68.mlp.experts.248.gate_proj", + "model.layers.68.mlp.experts.248.up_proj", + "model.layers.68.mlp.experts.249.down_proj", + "model.layers.68.mlp.experts.249.gate_proj", + "model.layers.68.mlp.experts.249.up_proj", + "model.layers.68.mlp.experts.25.down_proj", + "model.layers.68.mlp.experts.25.gate_proj", + "model.layers.68.mlp.experts.25.up_proj", + "model.layers.68.mlp.experts.250.down_proj", + "model.layers.68.mlp.experts.250.gate_proj", + "model.layers.68.mlp.experts.250.up_proj", + "model.layers.68.mlp.experts.251.down_proj", + "model.layers.68.mlp.experts.251.gate_proj", + "model.layers.68.mlp.experts.251.up_proj", + "model.layers.68.mlp.experts.252.down_proj", + "model.layers.68.mlp.experts.252.gate_proj", + "model.layers.68.mlp.experts.252.up_proj", + "model.layers.68.mlp.experts.253.down_proj", + "model.layers.68.mlp.experts.253.gate_proj", + "model.layers.68.mlp.experts.253.up_proj", + "model.layers.68.mlp.experts.254.down_proj", + "model.layers.68.mlp.experts.254.gate_proj", + "model.layers.68.mlp.experts.254.up_proj", + "model.layers.68.mlp.experts.255.down_proj", + "model.layers.68.mlp.experts.255.gate_proj", + "model.layers.68.mlp.experts.255.up_proj", + "model.layers.68.mlp.experts.26.down_proj", + "model.layers.68.mlp.experts.26.gate_proj", + "model.layers.68.mlp.experts.26.up_proj", + "model.layers.68.mlp.experts.27.down_proj", + "model.layers.68.mlp.experts.27.gate_proj", + "model.layers.68.mlp.experts.27.up_proj", + "model.layers.68.mlp.experts.28.down_proj", + "model.layers.68.mlp.experts.28.gate_proj", + "model.layers.68.mlp.experts.28.up_proj", + "model.layers.68.mlp.experts.29.down_proj", + "model.layers.68.mlp.experts.29.gate_proj", + "model.layers.68.mlp.experts.29.up_proj", + "model.layers.68.mlp.experts.3.down_proj", + "model.layers.68.mlp.experts.3.gate_proj", + "model.layers.68.mlp.experts.3.up_proj", + "model.layers.68.mlp.experts.30.down_proj", + "model.layers.68.mlp.experts.30.gate_proj", + "model.layers.68.mlp.experts.30.up_proj", + "model.layers.68.mlp.experts.31.down_proj", + "model.layers.68.mlp.experts.31.gate_proj", + "model.layers.68.mlp.experts.31.up_proj", + "model.layers.68.mlp.experts.32.down_proj", + "model.layers.68.mlp.experts.32.gate_proj", + "model.layers.68.mlp.experts.32.up_proj", + "model.layers.68.mlp.experts.33.down_proj", + "model.layers.68.mlp.experts.33.gate_proj", + "model.layers.68.mlp.experts.33.up_proj", + "model.layers.68.mlp.experts.34.down_proj", + "model.layers.68.mlp.experts.34.gate_proj", + "model.layers.68.mlp.experts.34.up_proj", + "model.layers.68.mlp.experts.35.down_proj", + "model.layers.68.mlp.experts.35.gate_proj", + "model.layers.68.mlp.experts.35.up_proj", + "model.layers.68.mlp.experts.36.down_proj", + "model.layers.68.mlp.experts.36.gate_proj", + "model.layers.68.mlp.experts.36.up_proj", + "model.layers.68.mlp.experts.37.down_proj", + "model.layers.68.mlp.experts.37.gate_proj", + "model.layers.68.mlp.experts.37.up_proj", + "model.layers.68.mlp.experts.38.down_proj", + "model.layers.68.mlp.experts.38.gate_proj", + "model.layers.68.mlp.experts.38.up_proj", + "model.layers.68.mlp.experts.39.down_proj", + "model.layers.68.mlp.experts.39.gate_proj", + "model.layers.68.mlp.experts.39.up_proj", + "model.layers.68.mlp.experts.4.down_proj", + "model.layers.68.mlp.experts.4.gate_proj", + "model.layers.68.mlp.experts.4.up_proj", + "model.layers.68.mlp.experts.40.down_proj", + "model.layers.68.mlp.experts.40.gate_proj", + "model.layers.68.mlp.experts.40.up_proj", + "model.layers.68.mlp.experts.41.down_proj", + "model.layers.68.mlp.experts.41.gate_proj", + "model.layers.68.mlp.experts.41.up_proj", + "model.layers.68.mlp.experts.42.down_proj", + "model.layers.68.mlp.experts.42.gate_proj", + "model.layers.68.mlp.experts.42.up_proj", + "model.layers.68.mlp.experts.43.down_proj", + "model.layers.68.mlp.experts.43.gate_proj", + "model.layers.68.mlp.experts.43.up_proj", + "model.layers.68.mlp.experts.44.down_proj", + "model.layers.68.mlp.experts.44.gate_proj", + "model.layers.68.mlp.experts.44.up_proj", + "model.layers.68.mlp.experts.45.down_proj", + "model.layers.68.mlp.experts.45.gate_proj", + "model.layers.68.mlp.experts.45.up_proj", + "model.layers.68.mlp.experts.46.down_proj", + "model.layers.68.mlp.experts.46.gate_proj", + "model.layers.68.mlp.experts.46.up_proj", + "model.layers.68.mlp.experts.47.down_proj", + "model.layers.68.mlp.experts.47.gate_proj", + "model.layers.68.mlp.experts.47.up_proj", + "model.layers.68.mlp.experts.48.down_proj", + "model.layers.68.mlp.experts.48.gate_proj", + "model.layers.68.mlp.experts.48.up_proj", + "model.layers.68.mlp.experts.49.down_proj", + "model.layers.68.mlp.experts.49.gate_proj", + "model.layers.68.mlp.experts.49.up_proj", + "model.layers.68.mlp.experts.5.down_proj", + "model.layers.68.mlp.experts.5.gate_proj", + "model.layers.68.mlp.experts.5.up_proj", + "model.layers.68.mlp.experts.50.down_proj", + "model.layers.68.mlp.experts.50.gate_proj", + "model.layers.68.mlp.experts.50.up_proj", + "model.layers.68.mlp.experts.51.down_proj", + "model.layers.68.mlp.experts.51.gate_proj", + "model.layers.68.mlp.experts.51.up_proj", + "model.layers.68.mlp.experts.52.down_proj", + "model.layers.68.mlp.experts.52.gate_proj", + "model.layers.68.mlp.experts.52.up_proj", + "model.layers.68.mlp.experts.53.down_proj", + "model.layers.68.mlp.experts.53.gate_proj", + "model.layers.68.mlp.experts.53.up_proj", + "model.layers.68.mlp.experts.54.down_proj", + "model.layers.68.mlp.experts.54.gate_proj", + "model.layers.68.mlp.experts.54.up_proj", + "model.layers.68.mlp.experts.55.down_proj", + "model.layers.68.mlp.experts.55.gate_proj", + "model.layers.68.mlp.experts.55.up_proj", + "model.layers.68.mlp.experts.56.down_proj", + "model.layers.68.mlp.experts.56.gate_proj", + "model.layers.68.mlp.experts.56.up_proj", + "model.layers.68.mlp.experts.57.down_proj", + "model.layers.68.mlp.experts.57.gate_proj", + "model.layers.68.mlp.experts.57.up_proj", + "model.layers.68.mlp.experts.58.down_proj", + "model.layers.68.mlp.experts.58.gate_proj", + "model.layers.68.mlp.experts.58.up_proj", + "model.layers.68.mlp.experts.59.down_proj", + "model.layers.68.mlp.experts.59.gate_proj", + "model.layers.68.mlp.experts.59.up_proj", + "model.layers.68.mlp.experts.6.down_proj", + "model.layers.68.mlp.experts.6.gate_proj", + "model.layers.68.mlp.experts.6.up_proj", + "model.layers.68.mlp.experts.60.down_proj", + "model.layers.68.mlp.experts.60.gate_proj", + "model.layers.68.mlp.experts.60.up_proj", + "model.layers.68.mlp.experts.61.down_proj", + "model.layers.68.mlp.experts.61.gate_proj", + "model.layers.68.mlp.experts.61.up_proj", + "model.layers.68.mlp.experts.62.down_proj", + "model.layers.68.mlp.experts.62.gate_proj", + "model.layers.68.mlp.experts.62.up_proj", + "model.layers.68.mlp.experts.63.down_proj", + "model.layers.68.mlp.experts.63.gate_proj", + "model.layers.68.mlp.experts.63.up_proj", + "model.layers.68.mlp.experts.64.down_proj", + "model.layers.68.mlp.experts.64.gate_proj", + "model.layers.68.mlp.experts.64.up_proj", + "model.layers.68.mlp.experts.65.down_proj", + "model.layers.68.mlp.experts.65.gate_proj", + "model.layers.68.mlp.experts.65.up_proj", + "model.layers.68.mlp.experts.66.down_proj", + "model.layers.68.mlp.experts.66.gate_proj", + "model.layers.68.mlp.experts.66.up_proj", + "model.layers.68.mlp.experts.67.down_proj", + "model.layers.68.mlp.experts.67.gate_proj", + "model.layers.68.mlp.experts.67.up_proj", + "model.layers.68.mlp.experts.68.down_proj", + "model.layers.68.mlp.experts.68.gate_proj", + "model.layers.68.mlp.experts.68.up_proj", + "model.layers.68.mlp.experts.69.down_proj", + "model.layers.68.mlp.experts.69.gate_proj", + "model.layers.68.mlp.experts.69.up_proj", + "model.layers.68.mlp.experts.7.down_proj", + "model.layers.68.mlp.experts.7.gate_proj", + "model.layers.68.mlp.experts.7.up_proj", + "model.layers.68.mlp.experts.70.down_proj", + "model.layers.68.mlp.experts.70.gate_proj", + "model.layers.68.mlp.experts.70.up_proj", + "model.layers.68.mlp.experts.71.down_proj", + "model.layers.68.mlp.experts.71.gate_proj", + "model.layers.68.mlp.experts.71.up_proj", + "model.layers.68.mlp.experts.72.down_proj", + "model.layers.68.mlp.experts.72.gate_proj", + "model.layers.68.mlp.experts.72.up_proj", + "model.layers.68.mlp.experts.73.down_proj", + "model.layers.68.mlp.experts.73.gate_proj", + "model.layers.68.mlp.experts.73.up_proj", + "model.layers.68.mlp.experts.74.down_proj", + "model.layers.68.mlp.experts.74.gate_proj", + "model.layers.68.mlp.experts.74.up_proj", + "model.layers.68.mlp.experts.75.down_proj", + "model.layers.68.mlp.experts.75.gate_proj", + "model.layers.68.mlp.experts.75.up_proj", + "model.layers.68.mlp.experts.76.down_proj", + "model.layers.68.mlp.experts.76.gate_proj", + "model.layers.68.mlp.experts.76.up_proj", + "model.layers.68.mlp.experts.77.down_proj", + "model.layers.68.mlp.experts.77.gate_proj", + "model.layers.68.mlp.experts.77.up_proj", + "model.layers.68.mlp.experts.78.down_proj", + "model.layers.68.mlp.experts.78.gate_proj", + "model.layers.68.mlp.experts.78.up_proj", + "model.layers.68.mlp.experts.79.down_proj", + "model.layers.68.mlp.experts.79.gate_proj", + "model.layers.68.mlp.experts.79.up_proj", + "model.layers.68.mlp.experts.8.down_proj", + "model.layers.68.mlp.experts.8.gate_proj", + "model.layers.68.mlp.experts.8.up_proj", + "model.layers.68.mlp.experts.80.down_proj", + "model.layers.68.mlp.experts.80.gate_proj", + "model.layers.68.mlp.experts.80.up_proj", + "model.layers.68.mlp.experts.81.down_proj", + "model.layers.68.mlp.experts.81.gate_proj", + "model.layers.68.mlp.experts.81.up_proj", + "model.layers.68.mlp.experts.82.down_proj", + "model.layers.68.mlp.experts.82.gate_proj", + "model.layers.68.mlp.experts.82.up_proj", + "model.layers.68.mlp.experts.83.down_proj", + "model.layers.68.mlp.experts.83.gate_proj", + "model.layers.68.mlp.experts.83.up_proj", + "model.layers.68.mlp.experts.84.down_proj", + "model.layers.68.mlp.experts.84.gate_proj", + "model.layers.68.mlp.experts.84.up_proj", + "model.layers.68.mlp.experts.85.down_proj", + "model.layers.68.mlp.experts.85.gate_proj", + "model.layers.68.mlp.experts.85.up_proj", + "model.layers.68.mlp.experts.86.down_proj", + "model.layers.68.mlp.experts.86.gate_proj", + "model.layers.68.mlp.experts.86.up_proj", + "model.layers.68.mlp.experts.87.down_proj", + "model.layers.68.mlp.experts.87.gate_proj", + "model.layers.68.mlp.experts.87.up_proj", + "model.layers.68.mlp.experts.88.down_proj", + "model.layers.68.mlp.experts.88.gate_proj", + "model.layers.68.mlp.experts.88.up_proj", + "model.layers.68.mlp.experts.89.down_proj", + "model.layers.68.mlp.experts.89.gate_proj", + "model.layers.68.mlp.experts.89.up_proj", + "model.layers.68.mlp.experts.9.down_proj", + "model.layers.68.mlp.experts.9.gate_proj", + "model.layers.68.mlp.experts.9.up_proj", + "model.layers.68.mlp.experts.90.down_proj", + "model.layers.68.mlp.experts.90.gate_proj", + "model.layers.68.mlp.experts.90.up_proj", + "model.layers.68.mlp.experts.91.down_proj", + "model.layers.68.mlp.experts.91.gate_proj", + "model.layers.68.mlp.experts.91.up_proj", + "model.layers.68.mlp.experts.92.down_proj", + "model.layers.68.mlp.experts.92.gate_proj", + "model.layers.68.mlp.experts.92.up_proj", + "model.layers.68.mlp.experts.93.down_proj", + "model.layers.68.mlp.experts.93.gate_proj", + "model.layers.68.mlp.experts.93.up_proj", + "model.layers.68.mlp.experts.94.down_proj", + "model.layers.68.mlp.experts.94.gate_proj", + "model.layers.68.mlp.experts.94.up_proj", + "model.layers.68.mlp.experts.95.down_proj", + "model.layers.68.mlp.experts.95.gate_proj", + "model.layers.68.mlp.experts.95.up_proj", + "model.layers.68.mlp.experts.96.down_proj", + "model.layers.68.mlp.experts.96.gate_proj", + "model.layers.68.mlp.experts.96.up_proj", + "model.layers.68.mlp.experts.97.down_proj", + "model.layers.68.mlp.experts.97.gate_proj", + "model.layers.68.mlp.experts.97.up_proj", + "model.layers.68.mlp.experts.98.down_proj", + "model.layers.68.mlp.experts.98.gate_proj", + "model.layers.68.mlp.experts.98.up_proj", + "model.layers.68.mlp.experts.99.down_proj", + "model.layers.68.mlp.experts.99.gate_proj", + "model.layers.68.mlp.experts.99.up_proj", + "model.layers.69.mlp.experts.0.down_proj", + "model.layers.69.mlp.experts.0.gate_proj", + "model.layers.69.mlp.experts.0.up_proj", + "model.layers.69.mlp.experts.1.down_proj", + "model.layers.69.mlp.experts.1.gate_proj", + "model.layers.69.mlp.experts.1.up_proj", + "model.layers.69.mlp.experts.10.down_proj", + "model.layers.69.mlp.experts.10.gate_proj", + "model.layers.69.mlp.experts.10.up_proj", + "model.layers.69.mlp.experts.100.down_proj", + "model.layers.69.mlp.experts.100.gate_proj", + "model.layers.69.mlp.experts.100.up_proj", + "model.layers.69.mlp.experts.101.down_proj", + "model.layers.69.mlp.experts.101.gate_proj", + "model.layers.69.mlp.experts.101.up_proj", + "model.layers.69.mlp.experts.102.down_proj", + "model.layers.69.mlp.experts.102.gate_proj", + "model.layers.69.mlp.experts.102.up_proj", + "model.layers.69.mlp.experts.103.down_proj", + "model.layers.69.mlp.experts.103.gate_proj", + "model.layers.69.mlp.experts.103.up_proj", + "model.layers.69.mlp.experts.104.down_proj", + "model.layers.69.mlp.experts.104.gate_proj", + "model.layers.69.mlp.experts.104.up_proj", + "model.layers.69.mlp.experts.105.down_proj", + "model.layers.69.mlp.experts.105.gate_proj", + "model.layers.69.mlp.experts.105.up_proj", + "model.layers.69.mlp.experts.106.down_proj", + "model.layers.69.mlp.experts.106.gate_proj", + "model.layers.69.mlp.experts.106.up_proj", + "model.layers.69.mlp.experts.107.down_proj", + "model.layers.69.mlp.experts.107.gate_proj", + "model.layers.69.mlp.experts.107.up_proj", + "model.layers.69.mlp.experts.108.down_proj", + "model.layers.69.mlp.experts.108.gate_proj", + "model.layers.69.mlp.experts.108.up_proj", + "model.layers.69.mlp.experts.109.down_proj", + "model.layers.69.mlp.experts.109.gate_proj", + "model.layers.69.mlp.experts.109.up_proj", + "model.layers.69.mlp.experts.11.down_proj", + "model.layers.69.mlp.experts.11.gate_proj", + "model.layers.69.mlp.experts.11.up_proj", + "model.layers.69.mlp.experts.110.down_proj", + "model.layers.69.mlp.experts.110.gate_proj", + "model.layers.69.mlp.experts.110.up_proj", + "model.layers.69.mlp.experts.111.down_proj", + "model.layers.69.mlp.experts.111.gate_proj", + "model.layers.69.mlp.experts.111.up_proj", + "model.layers.69.mlp.experts.112.down_proj", + "model.layers.69.mlp.experts.112.gate_proj", + "model.layers.69.mlp.experts.112.up_proj", + "model.layers.69.mlp.experts.113.down_proj", + "model.layers.69.mlp.experts.113.gate_proj", + "model.layers.69.mlp.experts.113.up_proj", + "model.layers.69.mlp.experts.114.down_proj", + "model.layers.69.mlp.experts.114.gate_proj", + "model.layers.69.mlp.experts.114.up_proj", + "model.layers.69.mlp.experts.115.down_proj", + "model.layers.69.mlp.experts.115.gate_proj", + "model.layers.69.mlp.experts.115.up_proj", + "model.layers.69.mlp.experts.116.down_proj", + "model.layers.69.mlp.experts.116.gate_proj", + "model.layers.69.mlp.experts.116.up_proj", + "model.layers.69.mlp.experts.117.down_proj", + "model.layers.69.mlp.experts.117.gate_proj", + "model.layers.69.mlp.experts.117.up_proj", + "model.layers.69.mlp.experts.118.down_proj", + "model.layers.69.mlp.experts.118.gate_proj", + "model.layers.69.mlp.experts.118.up_proj", + "model.layers.69.mlp.experts.119.down_proj", + "model.layers.69.mlp.experts.119.gate_proj", + "model.layers.69.mlp.experts.119.up_proj", + "model.layers.69.mlp.experts.12.down_proj", + "model.layers.69.mlp.experts.12.gate_proj", + "model.layers.69.mlp.experts.12.up_proj", + "model.layers.69.mlp.experts.120.down_proj", + "model.layers.69.mlp.experts.120.gate_proj", + "model.layers.69.mlp.experts.120.up_proj", + "model.layers.69.mlp.experts.121.down_proj", + "model.layers.69.mlp.experts.121.gate_proj", + "model.layers.69.mlp.experts.121.up_proj", + "model.layers.69.mlp.experts.122.down_proj", + "model.layers.69.mlp.experts.122.gate_proj", + "model.layers.69.mlp.experts.122.up_proj", + "model.layers.69.mlp.experts.123.down_proj", + "model.layers.69.mlp.experts.123.gate_proj", + "model.layers.69.mlp.experts.123.up_proj", + "model.layers.69.mlp.experts.124.down_proj", + "model.layers.69.mlp.experts.124.gate_proj", + "model.layers.69.mlp.experts.124.up_proj", + "model.layers.69.mlp.experts.125.down_proj", + "model.layers.69.mlp.experts.125.gate_proj", + "model.layers.69.mlp.experts.125.up_proj", + "model.layers.69.mlp.experts.126.down_proj", + "model.layers.69.mlp.experts.126.gate_proj", + "model.layers.69.mlp.experts.126.up_proj", + "model.layers.69.mlp.experts.127.down_proj", + "model.layers.69.mlp.experts.127.gate_proj", + "model.layers.69.mlp.experts.127.up_proj", + "model.layers.69.mlp.experts.128.down_proj", + "model.layers.69.mlp.experts.128.gate_proj", + "model.layers.69.mlp.experts.128.up_proj", + "model.layers.69.mlp.experts.129.down_proj", + "model.layers.69.mlp.experts.129.gate_proj", + "model.layers.69.mlp.experts.129.up_proj", + "model.layers.69.mlp.experts.13.down_proj", + "model.layers.69.mlp.experts.13.gate_proj", + "model.layers.69.mlp.experts.13.up_proj", + "model.layers.69.mlp.experts.130.down_proj", + "model.layers.69.mlp.experts.130.gate_proj", + "model.layers.69.mlp.experts.130.up_proj", + "model.layers.69.mlp.experts.131.down_proj", + "model.layers.69.mlp.experts.131.gate_proj", + "model.layers.69.mlp.experts.131.up_proj", + "model.layers.69.mlp.experts.132.down_proj", + "model.layers.69.mlp.experts.132.gate_proj", + "model.layers.69.mlp.experts.132.up_proj", + "model.layers.69.mlp.experts.133.down_proj", + "model.layers.69.mlp.experts.133.gate_proj", + "model.layers.69.mlp.experts.133.up_proj", + "model.layers.69.mlp.experts.134.down_proj", + "model.layers.69.mlp.experts.134.gate_proj", + "model.layers.69.mlp.experts.134.up_proj", + "model.layers.69.mlp.experts.135.down_proj", + "model.layers.69.mlp.experts.135.gate_proj", + "model.layers.69.mlp.experts.135.up_proj", + "model.layers.69.mlp.experts.136.down_proj", + "model.layers.69.mlp.experts.136.gate_proj", + "model.layers.69.mlp.experts.136.up_proj", + "model.layers.69.mlp.experts.137.down_proj", + "model.layers.69.mlp.experts.137.gate_proj", + "model.layers.69.mlp.experts.137.up_proj", + "model.layers.69.mlp.experts.138.down_proj", + "model.layers.69.mlp.experts.138.gate_proj", + "model.layers.69.mlp.experts.138.up_proj", + "model.layers.69.mlp.experts.139.down_proj", + "model.layers.69.mlp.experts.139.gate_proj", + "model.layers.69.mlp.experts.139.up_proj", + "model.layers.69.mlp.experts.14.down_proj", + "model.layers.69.mlp.experts.14.gate_proj", + "model.layers.69.mlp.experts.14.up_proj", + "model.layers.69.mlp.experts.140.down_proj", + "model.layers.69.mlp.experts.140.gate_proj", + "model.layers.69.mlp.experts.140.up_proj", + "model.layers.69.mlp.experts.141.down_proj", + "model.layers.69.mlp.experts.141.gate_proj", + "model.layers.69.mlp.experts.141.up_proj", + "model.layers.69.mlp.experts.142.down_proj", + "model.layers.69.mlp.experts.142.gate_proj", + "model.layers.69.mlp.experts.142.up_proj", + "model.layers.69.mlp.experts.143.down_proj", + "model.layers.69.mlp.experts.143.gate_proj", + "model.layers.69.mlp.experts.143.up_proj", + "model.layers.69.mlp.experts.144.down_proj", + "model.layers.69.mlp.experts.144.gate_proj", + "model.layers.69.mlp.experts.144.up_proj", + "model.layers.69.mlp.experts.145.down_proj", + "model.layers.69.mlp.experts.145.gate_proj", + "model.layers.69.mlp.experts.145.up_proj", + "model.layers.69.mlp.experts.146.down_proj", + "model.layers.69.mlp.experts.146.gate_proj", + "model.layers.69.mlp.experts.146.up_proj", + "model.layers.69.mlp.experts.147.down_proj", + "model.layers.69.mlp.experts.147.gate_proj", + "model.layers.69.mlp.experts.147.up_proj", + "model.layers.69.mlp.experts.148.down_proj", + "model.layers.69.mlp.experts.148.gate_proj", + "model.layers.69.mlp.experts.148.up_proj", + "model.layers.69.mlp.experts.149.down_proj", + "model.layers.69.mlp.experts.149.gate_proj", + "model.layers.69.mlp.experts.149.up_proj", + "model.layers.69.mlp.experts.15.down_proj", + "model.layers.69.mlp.experts.15.gate_proj", + "model.layers.69.mlp.experts.15.up_proj", + "model.layers.69.mlp.experts.150.down_proj", + "model.layers.69.mlp.experts.150.gate_proj", + "model.layers.69.mlp.experts.150.up_proj", + "model.layers.69.mlp.experts.151.down_proj", + "model.layers.69.mlp.experts.151.gate_proj", + "model.layers.69.mlp.experts.151.up_proj", + "model.layers.69.mlp.experts.152.down_proj", + "model.layers.69.mlp.experts.152.gate_proj", + "model.layers.69.mlp.experts.152.up_proj", + "model.layers.69.mlp.experts.153.down_proj", + "model.layers.69.mlp.experts.153.gate_proj", + "model.layers.69.mlp.experts.153.up_proj", + "model.layers.69.mlp.experts.154.down_proj", + "model.layers.69.mlp.experts.154.gate_proj", + "model.layers.69.mlp.experts.154.up_proj", + "model.layers.69.mlp.experts.155.down_proj", + "model.layers.69.mlp.experts.155.gate_proj", + "model.layers.69.mlp.experts.155.up_proj", + "model.layers.69.mlp.experts.156.down_proj", + "model.layers.69.mlp.experts.156.gate_proj", + "model.layers.69.mlp.experts.156.up_proj", + "model.layers.69.mlp.experts.157.down_proj", + "model.layers.69.mlp.experts.157.gate_proj", + "model.layers.69.mlp.experts.157.up_proj", + "model.layers.69.mlp.experts.158.down_proj", + "model.layers.69.mlp.experts.158.gate_proj", + "model.layers.69.mlp.experts.158.up_proj", + "model.layers.69.mlp.experts.159.down_proj", + "model.layers.69.mlp.experts.159.gate_proj", + "model.layers.69.mlp.experts.159.up_proj", + "model.layers.69.mlp.experts.16.down_proj", + "model.layers.69.mlp.experts.16.gate_proj", + "model.layers.69.mlp.experts.16.up_proj", + "model.layers.69.mlp.experts.160.down_proj", + "model.layers.69.mlp.experts.160.gate_proj", + "model.layers.69.mlp.experts.160.up_proj", + "model.layers.69.mlp.experts.161.down_proj", + "model.layers.69.mlp.experts.161.gate_proj", + "model.layers.69.mlp.experts.161.up_proj", + "model.layers.69.mlp.experts.162.down_proj", + "model.layers.69.mlp.experts.162.gate_proj", + "model.layers.69.mlp.experts.162.up_proj", + "model.layers.69.mlp.experts.163.down_proj", + "model.layers.69.mlp.experts.163.gate_proj", + "model.layers.69.mlp.experts.163.up_proj", + "model.layers.69.mlp.experts.164.down_proj", + "model.layers.69.mlp.experts.164.gate_proj", + "model.layers.69.mlp.experts.164.up_proj", + "model.layers.69.mlp.experts.165.down_proj", + "model.layers.69.mlp.experts.165.gate_proj", + "model.layers.69.mlp.experts.165.up_proj", + "model.layers.69.mlp.experts.166.down_proj", + "model.layers.69.mlp.experts.166.gate_proj", + "model.layers.69.mlp.experts.166.up_proj", + "model.layers.69.mlp.experts.167.down_proj", + "model.layers.69.mlp.experts.167.gate_proj", + "model.layers.69.mlp.experts.167.up_proj", + "model.layers.69.mlp.experts.168.down_proj", + "model.layers.69.mlp.experts.168.gate_proj", + "model.layers.69.mlp.experts.168.up_proj", + "model.layers.69.mlp.experts.169.down_proj", + "model.layers.69.mlp.experts.169.gate_proj", + "model.layers.69.mlp.experts.169.up_proj", + "model.layers.69.mlp.experts.17.down_proj", + "model.layers.69.mlp.experts.17.gate_proj", + "model.layers.69.mlp.experts.17.up_proj", + "model.layers.69.mlp.experts.170.down_proj", + "model.layers.69.mlp.experts.170.gate_proj", + "model.layers.69.mlp.experts.170.up_proj", + "model.layers.69.mlp.experts.171.down_proj", + "model.layers.69.mlp.experts.171.gate_proj", + "model.layers.69.mlp.experts.171.up_proj", + "model.layers.69.mlp.experts.172.down_proj", + "model.layers.69.mlp.experts.172.gate_proj", + "model.layers.69.mlp.experts.172.up_proj", + "model.layers.69.mlp.experts.173.down_proj", + "model.layers.69.mlp.experts.173.gate_proj", + "model.layers.69.mlp.experts.173.up_proj", + "model.layers.69.mlp.experts.174.down_proj", + "model.layers.69.mlp.experts.174.gate_proj", + "model.layers.69.mlp.experts.174.up_proj", + "model.layers.69.mlp.experts.175.down_proj", + "model.layers.69.mlp.experts.175.gate_proj", + "model.layers.69.mlp.experts.175.up_proj", + "model.layers.69.mlp.experts.176.down_proj", + "model.layers.69.mlp.experts.176.gate_proj", + "model.layers.69.mlp.experts.176.up_proj", + "model.layers.69.mlp.experts.177.down_proj", + "model.layers.69.mlp.experts.177.gate_proj", + "model.layers.69.mlp.experts.177.up_proj", + "model.layers.69.mlp.experts.178.down_proj", + "model.layers.69.mlp.experts.178.gate_proj", + "model.layers.69.mlp.experts.178.up_proj", + "model.layers.69.mlp.experts.179.down_proj", + "model.layers.69.mlp.experts.179.gate_proj", + "model.layers.69.mlp.experts.179.up_proj", + "model.layers.69.mlp.experts.18.down_proj", + "model.layers.69.mlp.experts.18.gate_proj", + "model.layers.69.mlp.experts.18.up_proj", + "model.layers.69.mlp.experts.180.down_proj", + "model.layers.69.mlp.experts.180.gate_proj", + "model.layers.69.mlp.experts.180.up_proj", + "model.layers.69.mlp.experts.181.down_proj", + "model.layers.69.mlp.experts.181.gate_proj", + "model.layers.69.mlp.experts.181.up_proj", + "model.layers.69.mlp.experts.182.down_proj", + "model.layers.69.mlp.experts.182.gate_proj", + "model.layers.69.mlp.experts.182.up_proj", + "model.layers.69.mlp.experts.183.down_proj", + "model.layers.69.mlp.experts.183.gate_proj", + "model.layers.69.mlp.experts.183.up_proj", + "model.layers.69.mlp.experts.184.down_proj", + "model.layers.69.mlp.experts.184.gate_proj", + "model.layers.69.mlp.experts.184.up_proj", + "model.layers.69.mlp.experts.185.down_proj", + "model.layers.69.mlp.experts.185.gate_proj", + "model.layers.69.mlp.experts.185.up_proj", + "model.layers.69.mlp.experts.186.down_proj", + "model.layers.69.mlp.experts.186.gate_proj", + "model.layers.69.mlp.experts.186.up_proj", + "model.layers.69.mlp.experts.187.down_proj", + "model.layers.69.mlp.experts.187.gate_proj", + "model.layers.69.mlp.experts.187.up_proj", + "model.layers.69.mlp.experts.188.down_proj", + "model.layers.69.mlp.experts.188.gate_proj", + "model.layers.69.mlp.experts.188.up_proj", + "model.layers.69.mlp.experts.189.down_proj", + "model.layers.69.mlp.experts.189.gate_proj", + "model.layers.69.mlp.experts.189.up_proj", + "model.layers.69.mlp.experts.19.down_proj", + "model.layers.69.mlp.experts.19.gate_proj", + "model.layers.69.mlp.experts.19.up_proj", + "model.layers.69.mlp.experts.190.down_proj", + "model.layers.69.mlp.experts.190.gate_proj", + "model.layers.69.mlp.experts.190.up_proj", + "model.layers.69.mlp.experts.191.down_proj", + "model.layers.69.mlp.experts.191.gate_proj", + "model.layers.69.mlp.experts.191.up_proj", + "model.layers.69.mlp.experts.192.down_proj", + "model.layers.69.mlp.experts.192.gate_proj", + "model.layers.69.mlp.experts.192.up_proj", + "model.layers.69.mlp.experts.193.down_proj", + "model.layers.69.mlp.experts.193.gate_proj", + "model.layers.69.mlp.experts.193.up_proj", + "model.layers.69.mlp.experts.194.down_proj", + "model.layers.69.mlp.experts.194.gate_proj", + "model.layers.69.mlp.experts.194.up_proj", + "model.layers.69.mlp.experts.195.down_proj", + "model.layers.69.mlp.experts.195.gate_proj", + "model.layers.69.mlp.experts.195.up_proj", + "model.layers.69.mlp.experts.196.down_proj", + "model.layers.69.mlp.experts.196.gate_proj", + "model.layers.69.mlp.experts.196.up_proj", + "model.layers.69.mlp.experts.197.down_proj", + "model.layers.69.mlp.experts.197.gate_proj", + "model.layers.69.mlp.experts.197.up_proj", + "model.layers.69.mlp.experts.198.down_proj", + "model.layers.69.mlp.experts.198.gate_proj", + "model.layers.69.mlp.experts.198.up_proj", + "model.layers.69.mlp.experts.199.down_proj", + "model.layers.69.mlp.experts.199.gate_proj", + "model.layers.69.mlp.experts.199.up_proj", + "model.layers.69.mlp.experts.2.down_proj", + "model.layers.69.mlp.experts.2.gate_proj", + "model.layers.69.mlp.experts.2.up_proj", + "model.layers.69.mlp.experts.20.down_proj", + "model.layers.69.mlp.experts.20.gate_proj", + "model.layers.69.mlp.experts.20.up_proj", + "model.layers.69.mlp.experts.200.down_proj", + "model.layers.69.mlp.experts.200.gate_proj", + "model.layers.69.mlp.experts.200.up_proj", + "model.layers.69.mlp.experts.201.down_proj", + "model.layers.69.mlp.experts.201.gate_proj", + "model.layers.69.mlp.experts.201.up_proj", + "model.layers.69.mlp.experts.202.down_proj", + "model.layers.69.mlp.experts.202.gate_proj", + "model.layers.69.mlp.experts.202.up_proj", + "model.layers.69.mlp.experts.203.down_proj", + "model.layers.69.mlp.experts.203.gate_proj", + "model.layers.69.mlp.experts.203.up_proj", + "model.layers.69.mlp.experts.204.down_proj", + "model.layers.69.mlp.experts.204.gate_proj", + "model.layers.69.mlp.experts.204.up_proj", + "model.layers.69.mlp.experts.205.down_proj", + "model.layers.69.mlp.experts.205.gate_proj", + "model.layers.69.mlp.experts.205.up_proj", + "model.layers.69.mlp.experts.206.down_proj", + "model.layers.69.mlp.experts.206.gate_proj", + "model.layers.69.mlp.experts.206.up_proj", + "model.layers.69.mlp.experts.207.down_proj", + "model.layers.69.mlp.experts.207.gate_proj", + "model.layers.69.mlp.experts.207.up_proj", + "model.layers.69.mlp.experts.208.down_proj", + "model.layers.69.mlp.experts.208.gate_proj", + "model.layers.69.mlp.experts.208.up_proj", + "model.layers.69.mlp.experts.209.down_proj", + "model.layers.69.mlp.experts.209.gate_proj", + "model.layers.69.mlp.experts.209.up_proj", + "model.layers.69.mlp.experts.21.down_proj", + "model.layers.69.mlp.experts.21.gate_proj", + "model.layers.69.mlp.experts.21.up_proj", + "model.layers.69.mlp.experts.210.down_proj", + "model.layers.69.mlp.experts.210.gate_proj", + "model.layers.69.mlp.experts.210.up_proj", + "model.layers.69.mlp.experts.211.down_proj", + "model.layers.69.mlp.experts.211.gate_proj", + "model.layers.69.mlp.experts.211.up_proj", + "model.layers.69.mlp.experts.212.down_proj", + "model.layers.69.mlp.experts.212.gate_proj", + "model.layers.69.mlp.experts.212.up_proj", + "model.layers.69.mlp.experts.213.down_proj", + "model.layers.69.mlp.experts.213.gate_proj", + "model.layers.69.mlp.experts.213.up_proj", + "model.layers.69.mlp.experts.214.down_proj", + "model.layers.69.mlp.experts.214.gate_proj", + "model.layers.69.mlp.experts.214.up_proj", + "model.layers.69.mlp.experts.215.down_proj", + "model.layers.69.mlp.experts.215.gate_proj", + "model.layers.69.mlp.experts.215.up_proj", + "model.layers.69.mlp.experts.216.down_proj", + "model.layers.69.mlp.experts.216.gate_proj", + "model.layers.69.mlp.experts.216.up_proj", + "model.layers.69.mlp.experts.217.down_proj", + "model.layers.69.mlp.experts.217.gate_proj", + "model.layers.69.mlp.experts.217.up_proj", + "model.layers.69.mlp.experts.218.down_proj", + "model.layers.69.mlp.experts.218.gate_proj", + "model.layers.69.mlp.experts.218.up_proj", + "model.layers.69.mlp.experts.219.down_proj", + "model.layers.69.mlp.experts.219.gate_proj", + "model.layers.69.mlp.experts.219.up_proj", + "model.layers.69.mlp.experts.22.down_proj", + "model.layers.69.mlp.experts.22.gate_proj", + "model.layers.69.mlp.experts.22.up_proj", + "model.layers.69.mlp.experts.220.down_proj", + "model.layers.69.mlp.experts.220.gate_proj", + "model.layers.69.mlp.experts.220.up_proj", + "model.layers.69.mlp.experts.221.down_proj", + "model.layers.69.mlp.experts.221.gate_proj", + "model.layers.69.mlp.experts.221.up_proj", + "model.layers.69.mlp.experts.222.down_proj", + "model.layers.69.mlp.experts.222.gate_proj", + "model.layers.69.mlp.experts.222.up_proj", + "model.layers.69.mlp.experts.223.down_proj", + "model.layers.69.mlp.experts.223.gate_proj", + "model.layers.69.mlp.experts.223.up_proj", + "model.layers.69.mlp.experts.224.down_proj", + "model.layers.69.mlp.experts.224.gate_proj", + "model.layers.69.mlp.experts.224.up_proj", + "model.layers.69.mlp.experts.225.down_proj", + "model.layers.69.mlp.experts.225.gate_proj", + "model.layers.69.mlp.experts.225.up_proj", + "model.layers.69.mlp.experts.226.down_proj", + "model.layers.69.mlp.experts.226.gate_proj", + "model.layers.69.mlp.experts.226.up_proj", + "model.layers.69.mlp.experts.227.down_proj", + "model.layers.69.mlp.experts.227.gate_proj", + "model.layers.69.mlp.experts.227.up_proj", + "model.layers.69.mlp.experts.228.down_proj", + "model.layers.69.mlp.experts.228.gate_proj", + "model.layers.69.mlp.experts.228.up_proj", + "model.layers.69.mlp.experts.229.down_proj", + "model.layers.69.mlp.experts.229.gate_proj", + "model.layers.69.mlp.experts.229.up_proj", + "model.layers.69.mlp.experts.23.down_proj", + "model.layers.69.mlp.experts.23.gate_proj", + "model.layers.69.mlp.experts.23.up_proj", + "model.layers.69.mlp.experts.230.down_proj", + "model.layers.69.mlp.experts.230.gate_proj", + "model.layers.69.mlp.experts.230.up_proj", + "model.layers.69.mlp.experts.231.down_proj", + "model.layers.69.mlp.experts.231.gate_proj", + "model.layers.69.mlp.experts.231.up_proj", + "model.layers.69.mlp.experts.232.down_proj", + "model.layers.69.mlp.experts.232.gate_proj", + "model.layers.69.mlp.experts.232.up_proj", + "model.layers.69.mlp.experts.233.down_proj", + "model.layers.69.mlp.experts.233.gate_proj", + "model.layers.69.mlp.experts.233.up_proj", + "model.layers.69.mlp.experts.234.down_proj", + "model.layers.69.mlp.experts.234.gate_proj", + "model.layers.69.mlp.experts.234.up_proj", + "model.layers.69.mlp.experts.235.down_proj", + "model.layers.69.mlp.experts.235.gate_proj", + "model.layers.69.mlp.experts.235.up_proj", + "model.layers.69.mlp.experts.236.down_proj", + "model.layers.69.mlp.experts.236.gate_proj", + "model.layers.69.mlp.experts.236.up_proj", + "model.layers.69.mlp.experts.237.down_proj", + "model.layers.69.mlp.experts.237.gate_proj", + "model.layers.69.mlp.experts.237.up_proj", + "model.layers.69.mlp.experts.238.down_proj", + "model.layers.69.mlp.experts.238.gate_proj", + "model.layers.69.mlp.experts.238.up_proj", + "model.layers.69.mlp.experts.239.down_proj", + "model.layers.69.mlp.experts.239.gate_proj", + "model.layers.69.mlp.experts.239.up_proj", + "model.layers.69.mlp.experts.24.down_proj", + "model.layers.69.mlp.experts.24.gate_proj", + "model.layers.69.mlp.experts.24.up_proj", + "model.layers.69.mlp.experts.240.down_proj", + "model.layers.69.mlp.experts.240.gate_proj", + "model.layers.69.mlp.experts.240.up_proj", + "model.layers.69.mlp.experts.241.down_proj", + "model.layers.69.mlp.experts.241.gate_proj", + "model.layers.69.mlp.experts.241.up_proj", + "model.layers.69.mlp.experts.242.down_proj", + "model.layers.69.mlp.experts.242.gate_proj", + "model.layers.69.mlp.experts.242.up_proj", + "model.layers.69.mlp.experts.243.down_proj", + "model.layers.69.mlp.experts.243.gate_proj", + "model.layers.69.mlp.experts.243.up_proj", + "model.layers.69.mlp.experts.244.down_proj", + "model.layers.69.mlp.experts.244.gate_proj", + "model.layers.69.mlp.experts.244.up_proj", + "model.layers.69.mlp.experts.245.down_proj", + "model.layers.69.mlp.experts.245.gate_proj", + "model.layers.69.mlp.experts.245.up_proj", + "model.layers.69.mlp.experts.246.down_proj", + "model.layers.69.mlp.experts.246.gate_proj", + "model.layers.69.mlp.experts.246.up_proj", + "model.layers.69.mlp.experts.247.down_proj", + "model.layers.69.mlp.experts.247.gate_proj", + "model.layers.69.mlp.experts.247.up_proj", + "model.layers.69.mlp.experts.248.down_proj", + "model.layers.69.mlp.experts.248.gate_proj", + "model.layers.69.mlp.experts.248.up_proj", + "model.layers.69.mlp.experts.249.down_proj", + "model.layers.69.mlp.experts.249.gate_proj", + "model.layers.69.mlp.experts.249.up_proj", + "model.layers.69.mlp.experts.25.down_proj", + "model.layers.69.mlp.experts.25.gate_proj", + "model.layers.69.mlp.experts.25.up_proj", + "model.layers.69.mlp.experts.250.down_proj", + "model.layers.69.mlp.experts.250.gate_proj", + "model.layers.69.mlp.experts.250.up_proj", + "model.layers.69.mlp.experts.251.down_proj", + "model.layers.69.mlp.experts.251.gate_proj", + "model.layers.69.mlp.experts.251.up_proj", + "model.layers.69.mlp.experts.252.down_proj", + "model.layers.69.mlp.experts.252.gate_proj", + "model.layers.69.mlp.experts.252.up_proj", + "model.layers.69.mlp.experts.253.down_proj", + "model.layers.69.mlp.experts.253.gate_proj", + "model.layers.69.mlp.experts.253.up_proj", + "model.layers.69.mlp.experts.254.down_proj", + "model.layers.69.mlp.experts.254.gate_proj", + "model.layers.69.mlp.experts.254.up_proj", + "model.layers.69.mlp.experts.255.down_proj", + "model.layers.69.mlp.experts.255.gate_proj", + "model.layers.69.mlp.experts.255.up_proj", + "model.layers.69.mlp.experts.26.down_proj", + "model.layers.69.mlp.experts.26.gate_proj", + "model.layers.69.mlp.experts.26.up_proj", + "model.layers.69.mlp.experts.27.down_proj", + "model.layers.69.mlp.experts.27.gate_proj", + "model.layers.69.mlp.experts.27.up_proj", + "model.layers.69.mlp.experts.28.down_proj", + "model.layers.69.mlp.experts.28.gate_proj", + "model.layers.69.mlp.experts.28.up_proj", + "model.layers.69.mlp.experts.29.down_proj", + "model.layers.69.mlp.experts.29.gate_proj", + "model.layers.69.mlp.experts.29.up_proj", + "model.layers.69.mlp.experts.3.down_proj", + "model.layers.69.mlp.experts.3.gate_proj", + "model.layers.69.mlp.experts.3.up_proj", + "model.layers.69.mlp.experts.30.down_proj", + "model.layers.69.mlp.experts.30.gate_proj", + "model.layers.69.mlp.experts.30.up_proj", + "model.layers.69.mlp.experts.31.down_proj", + "model.layers.69.mlp.experts.31.gate_proj", + "model.layers.69.mlp.experts.31.up_proj", + "model.layers.69.mlp.experts.32.down_proj", + "model.layers.69.mlp.experts.32.gate_proj", + "model.layers.69.mlp.experts.32.up_proj", + "model.layers.69.mlp.experts.33.down_proj", + "model.layers.69.mlp.experts.33.gate_proj", + "model.layers.69.mlp.experts.33.up_proj", + "model.layers.69.mlp.experts.34.down_proj", + "model.layers.69.mlp.experts.34.gate_proj", + "model.layers.69.mlp.experts.34.up_proj", + "model.layers.69.mlp.experts.35.down_proj", + "model.layers.69.mlp.experts.35.gate_proj", + "model.layers.69.mlp.experts.35.up_proj", + "model.layers.69.mlp.experts.36.down_proj", + "model.layers.69.mlp.experts.36.gate_proj", + "model.layers.69.mlp.experts.36.up_proj", + "model.layers.69.mlp.experts.37.down_proj", + "model.layers.69.mlp.experts.37.gate_proj", + "model.layers.69.mlp.experts.37.up_proj", + "model.layers.69.mlp.experts.38.down_proj", + "model.layers.69.mlp.experts.38.gate_proj", + "model.layers.69.mlp.experts.38.up_proj", + "model.layers.69.mlp.experts.39.down_proj", + "model.layers.69.mlp.experts.39.gate_proj", + "model.layers.69.mlp.experts.39.up_proj", + "model.layers.69.mlp.experts.4.down_proj", + "model.layers.69.mlp.experts.4.gate_proj", + "model.layers.69.mlp.experts.4.up_proj", + "model.layers.69.mlp.experts.40.down_proj", + "model.layers.69.mlp.experts.40.gate_proj", + "model.layers.69.mlp.experts.40.up_proj", + "model.layers.69.mlp.experts.41.down_proj", + "model.layers.69.mlp.experts.41.gate_proj", + "model.layers.69.mlp.experts.41.up_proj", + "model.layers.69.mlp.experts.42.down_proj", + "model.layers.69.mlp.experts.42.gate_proj", + "model.layers.69.mlp.experts.42.up_proj", + "model.layers.69.mlp.experts.43.down_proj", + "model.layers.69.mlp.experts.43.gate_proj", + "model.layers.69.mlp.experts.43.up_proj", + "model.layers.69.mlp.experts.44.down_proj", + "model.layers.69.mlp.experts.44.gate_proj", + "model.layers.69.mlp.experts.44.up_proj", + "model.layers.69.mlp.experts.45.down_proj", + "model.layers.69.mlp.experts.45.gate_proj", + "model.layers.69.mlp.experts.45.up_proj", + "model.layers.69.mlp.experts.46.down_proj", + "model.layers.69.mlp.experts.46.gate_proj", + "model.layers.69.mlp.experts.46.up_proj", + "model.layers.69.mlp.experts.47.down_proj", + "model.layers.69.mlp.experts.47.gate_proj", + "model.layers.69.mlp.experts.47.up_proj", + "model.layers.69.mlp.experts.48.down_proj", + "model.layers.69.mlp.experts.48.gate_proj", + "model.layers.69.mlp.experts.48.up_proj", + "model.layers.69.mlp.experts.49.down_proj", + "model.layers.69.mlp.experts.49.gate_proj", + "model.layers.69.mlp.experts.49.up_proj", + "model.layers.69.mlp.experts.5.down_proj", + "model.layers.69.mlp.experts.5.gate_proj", + "model.layers.69.mlp.experts.5.up_proj", + "model.layers.69.mlp.experts.50.down_proj", + "model.layers.69.mlp.experts.50.gate_proj", + "model.layers.69.mlp.experts.50.up_proj", + "model.layers.69.mlp.experts.51.down_proj", + "model.layers.69.mlp.experts.51.gate_proj", + "model.layers.69.mlp.experts.51.up_proj", + "model.layers.69.mlp.experts.52.down_proj", + "model.layers.69.mlp.experts.52.gate_proj", + "model.layers.69.mlp.experts.52.up_proj", + "model.layers.69.mlp.experts.53.down_proj", + "model.layers.69.mlp.experts.53.gate_proj", + "model.layers.69.mlp.experts.53.up_proj", + "model.layers.69.mlp.experts.54.down_proj", + "model.layers.69.mlp.experts.54.gate_proj", + "model.layers.69.mlp.experts.54.up_proj", + "model.layers.69.mlp.experts.55.down_proj", + "model.layers.69.mlp.experts.55.gate_proj", + "model.layers.69.mlp.experts.55.up_proj", + "model.layers.69.mlp.experts.56.down_proj", + "model.layers.69.mlp.experts.56.gate_proj", + "model.layers.69.mlp.experts.56.up_proj", + "model.layers.69.mlp.experts.57.down_proj", + "model.layers.69.mlp.experts.57.gate_proj", + "model.layers.69.mlp.experts.57.up_proj", + "model.layers.69.mlp.experts.58.down_proj", + "model.layers.69.mlp.experts.58.gate_proj", + "model.layers.69.mlp.experts.58.up_proj", + "model.layers.69.mlp.experts.59.down_proj", + "model.layers.69.mlp.experts.59.gate_proj", + "model.layers.69.mlp.experts.59.up_proj", + "model.layers.69.mlp.experts.6.down_proj", + "model.layers.69.mlp.experts.6.gate_proj", + "model.layers.69.mlp.experts.6.up_proj", + "model.layers.69.mlp.experts.60.down_proj", + "model.layers.69.mlp.experts.60.gate_proj", + "model.layers.69.mlp.experts.60.up_proj", + "model.layers.69.mlp.experts.61.down_proj", + "model.layers.69.mlp.experts.61.gate_proj", + "model.layers.69.mlp.experts.61.up_proj", + "model.layers.69.mlp.experts.62.down_proj", + "model.layers.69.mlp.experts.62.gate_proj", + "model.layers.69.mlp.experts.62.up_proj", + "model.layers.69.mlp.experts.63.down_proj", + "model.layers.69.mlp.experts.63.gate_proj", + "model.layers.69.mlp.experts.63.up_proj", + "model.layers.69.mlp.experts.64.down_proj", + "model.layers.69.mlp.experts.64.gate_proj", + "model.layers.69.mlp.experts.64.up_proj", + "model.layers.69.mlp.experts.65.down_proj", + "model.layers.69.mlp.experts.65.gate_proj", + "model.layers.69.mlp.experts.65.up_proj", + "model.layers.69.mlp.experts.66.down_proj", + "model.layers.69.mlp.experts.66.gate_proj", + "model.layers.69.mlp.experts.66.up_proj", + "model.layers.69.mlp.experts.67.down_proj", + "model.layers.69.mlp.experts.67.gate_proj", + "model.layers.69.mlp.experts.67.up_proj", + "model.layers.69.mlp.experts.68.down_proj", + "model.layers.69.mlp.experts.68.gate_proj", + "model.layers.69.mlp.experts.68.up_proj", + "model.layers.69.mlp.experts.69.down_proj", + "model.layers.69.mlp.experts.69.gate_proj", + "model.layers.69.mlp.experts.69.up_proj", + "model.layers.69.mlp.experts.7.down_proj", + "model.layers.69.mlp.experts.7.gate_proj", + "model.layers.69.mlp.experts.7.up_proj", + "model.layers.69.mlp.experts.70.down_proj", + "model.layers.69.mlp.experts.70.gate_proj", + "model.layers.69.mlp.experts.70.up_proj", + "model.layers.69.mlp.experts.71.down_proj", + "model.layers.69.mlp.experts.71.gate_proj", + "model.layers.69.mlp.experts.71.up_proj", + "model.layers.69.mlp.experts.72.down_proj", + "model.layers.69.mlp.experts.72.gate_proj", + "model.layers.69.mlp.experts.72.up_proj", + "model.layers.69.mlp.experts.73.down_proj", + "model.layers.69.mlp.experts.73.gate_proj", + "model.layers.69.mlp.experts.73.up_proj", + "model.layers.69.mlp.experts.74.down_proj", + "model.layers.69.mlp.experts.74.gate_proj", + "model.layers.69.mlp.experts.74.up_proj", + "model.layers.69.mlp.experts.75.down_proj", + "model.layers.69.mlp.experts.75.gate_proj", + "model.layers.69.mlp.experts.75.up_proj", + "model.layers.69.mlp.experts.76.down_proj", + "model.layers.69.mlp.experts.76.gate_proj", + "model.layers.69.mlp.experts.76.up_proj", + "model.layers.69.mlp.experts.77.down_proj", + "model.layers.69.mlp.experts.77.gate_proj", + "model.layers.69.mlp.experts.77.up_proj", + "model.layers.69.mlp.experts.78.down_proj", + "model.layers.69.mlp.experts.78.gate_proj", + "model.layers.69.mlp.experts.78.up_proj", + "model.layers.69.mlp.experts.79.down_proj", + "model.layers.69.mlp.experts.79.gate_proj", + "model.layers.69.mlp.experts.79.up_proj", + "model.layers.69.mlp.experts.8.down_proj", + "model.layers.69.mlp.experts.8.gate_proj", + "model.layers.69.mlp.experts.8.up_proj", + "model.layers.69.mlp.experts.80.down_proj", + "model.layers.69.mlp.experts.80.gate_proj", + "model.layers.69.mlp.experts.80.up_proj", + "model.layers.69.mlp.experts.81.down_proj", + "model.layers.69.mlp.experts.81.gate_proj", + "model.layers.69.mlp.experts.81.up_proj", + "model.layers.69.mlp.experts.82.down_proj", + "model.layers.69.mlp.experts.82.gate_proj", + "model.layers.69.mlp.experts.82.up_proj", + "model.layers.69.mlp.experts.83.down_proj", + "model.layers.69.mlp.experts.83.gate_proj", + "model.layers.69.mlp.experts.83.up_proj", + "model.layers.69.mlp.experts.84.down_proj", + "model.layers.69.mlp.experts.84.gate_proj", + "model.layers.69.mlp.experts.84.up_proj", + "model.layers.69.mlp.experts.85.down_proj", + "model.layers.69.mlp.experts.85.gate_proj", + "model.layers.69.mlp.experts.85.up_proj", + "model.layers.69.mlp.experts.86.down_proj", + "model.layers.69.mlp.experts.86.gate_proj", + "model.layers.69.mlp.experts.86.up_proj", + "model.layers.69.mlp.experts.87.down_proj", + "model.layers.69.mlp.experts.87.gate_proj", + "model.layers.69.mlp.experts.87.up_proj", + "model.layers.69.mlp.experts.88.down_proj", + "model.layers.69.mlp.experts.88.gate_proj", + "model.layers.69.mlp.experts.88.up_proj", + "model.layers.69.mlp.experts.89.down_proj", + "model.layers.69.mlp.experts.89.gate_proj", + "model.layers.69.mlp.experts.89.up_proj", + "model.layers.69.mlp.experts.9.down_proj", + "model.layers.69.mlp.experts.9.gate_proj", + "model.layers.69.mlp.experts.9.up_proj", + "model.layers.69.mlp.experts.90.down_proj", + "model.layers.69.mlp.experts.90.gate_proj", + "model.layers.69.mlp.experts.90.up_proj", + "model.layers.69.mlp.experts.91.down_proj", + "model.layers.69.mlp.experts.91.gate_proj", + "model.layers.69.mlp.experts.91.up_proj", + "model.layers.69.mlp.experts.92.down_proj", + "model.layers.69.mlp.experts.92.gate_proj", + "model.layers.69.mlp.experts.92.up_proj", + "model.layers.69.mlp.experts.93.down_proj", + "model.layers.69.mlp.experts.93.gate_proj", + "model.layers.69.mlp.experts.93.up_proj", + "model.layers.69.mlp.experts.94.down_proj", + "model.layers.69.mlp.experts.94.gate_proj", + "model.layers.69.mlp.experts.94.up_proj", + "model.layers.69.mlp.experts.95.down_proj", + "model.layers.69.mlp.experts.95.gate_proj", + "model.layers.69.mlp.experts.95.up_proj", + "model.layers.69.mlp.experts.96.down_proj", + "model.layers.69.mlp.experts.96.gate_proj", + "model.layers.69.mlp.experts.96.up_proj", + "model.layers.69.mlp.experts.97.down_proj", + "model.layers.69.mlp.experts.97.gate_proj", + "model.layers.69.mlp.experts.97.up_proj", + "model.layers.69.mlp.experts.98.down_proj", + "model.layers.69.mlp.experts.98.gate_proj", + "model.layers.69.mlp.experts.98.up_proj", + "model.layers.69.mlp.experts.99.down_proj", + "model.layers.69.mlp.experts.99.gate_proj", + "model.layers.69.mlp.experts.99.up_proj", + "model.layers.7.mlp.experts.0.down_proj", + "model.layers.7.mlp.experts.0.gate_proj", + "model.layers.7.mlp.experts.0.up_proj", + "model.layers.7.mlp.experts.1.down_proj", + "model.layers.7.mlp.experts.1.gate_proj", + "model.layers.7.mlp.experts.1.up_proj", + "model.layers.7.mlp.experts.10.down_proj", + "model.layers.7.mlp.experts.10.gate_proj", + "model.layers.7.mlp.experts.10.up_proj", + "model.layers.7.mlp.experts.100.down_proj", + "model.layers.7.mlp.experts.100.gate_proj", + "model.layers.7.mlp.experts.100.up_proj", + "model.layers.7.mlp.experts.101.down_proj", + "model.layers.7.mlp.experts.101.gate_proj", + "model.layers.7.mlp.experts.101.up_proj", + "model.layers.7.mlp.experts.102.down_proj", + "model.layers.7.mlp.experts.102.gate_proj", + "model.layers.7.mlp.experts.102.up_proj", + "model.layers.7.mlp.experts.103.down_proj", + "model.layers.7.mlp.experts.103.gate_proj", + "model.layers.7.mlp.experts.103.up_proj", + "model.layers.7.mlp.experts.104.down_proj", + "model.layers.7.mlp.experts.104.gate_proj", + "model.layers.7.mlp.experts.104.up_proj", + "model.layers.7.mlp.experts.105.down_proj", + "model.layers.7.mlp.experts.105.gate_proj", + "model.layers.7.mlp.experts.105.up_proj", + "model.layers.7.mlp.experts.106.down_proj", + "model.layers.7.mlp.experts.106.gate_proj", + "model.layers.7.mlp.experts.106.up_proj", + "model.layers.7.mlp.experts.107.down_proj", + "model.layers.7.mlp.experts.107.gate_proj", + "model.layers.7.mlp.experts.107.up_proj", + "model.layers.7.mlp.experts.108.down_proj", + "model.layers.7.mlp.experts.108.gate_proj", + "model.layers.7.mlp.experts.108.up_proj", + "model.layers.7.mlp.experts.109.down_proj", + "model.layers.7.mlp.experts.109.gate_proj", + "model.layers.7.mlp.experts.109.up_proj", + "model.layers.7.mlp.experts.11.down_proj", + "model.layers.7.mlp.experts.11.gate_proj", + "model.layers.7.mlp.experts.11.up_proj", + "model.layers.7.mlp.experts.110.down_proj", + "model.layers.7.mlp.experts.110.gate_proj", + "model.layers.7.mlp.experts.110.up_proj", + "model.layers.7.mlp.experts.111.down_proj", + "model.layers.7.mlp.experts.111.gate_proj", + "model.layers.7.mlp.experts.111.up_proj", + "model.layers.7.mlp.experts.112.down_proj", + "model.layers.7.mlp.experts.112.gate_proj", + "model.layers.7.mlp.experts.112.up_proj", + "model.layers.7.mlp.experts.113.down_proj", + "model.layers.7.mlp.experts.113.gate_proj", + "model.layers.7.mlp.experts.113.up_proj", + "model.layers.7.mlp.experts.114.down_proj", + "model.layers.7.mlp.experts.114.gate_proj", + "model.layers.7.mlp.experts.114.up_proj", + "model.layers.7.mlp.experts.115.down_proj", + "model.layers.7.mlp.experts.115.gate_proj", + "model.layers.7.mlp.experts.115.up_proj", + "model.layers.7.mlp.experts.116.down_proj", + "model.layers.7.mlp.experts.116.gate_proj", + "model.layers.7.mlp.experts.116.up_proj", + "model.layers.7.mlp.experts.117.down_proj", + "model.layers.7.mlp.experts.117.gate_proj", + "model.layers.7.mlp.experts.117.up_proj", + "model.layers.7.mlp.experts.118.down_proj", + "model.layers.7.mlp.experts.118.gate_proj", + "model.layers.7.mlp.experts.118.up_proj", + "model.layers.7.mlp.experts.119.down_proj", + "model.layers.7.mlp.experts.119.gate_proj", + "model.layers.7.mlp.experts.119.up_proj", + "model.layers.7.mlp.experts.12.down_proj", + "model.layers.7.mlp.experts.12.gate_proj", + "model.layers.7.mlp.experts.12.up_proj", + "model.layers.7.mlp.experts.120.down_proj", + "model.layers.7.mlp.experts.120.gate_proj", + "model.layers.7.mlp.experts.120.up_proj", + "model.layers.7.mlp.experts.121.down_proj", + "model.layers.7.mlp.experts.121.gate_proj", + "model.layers.7.mlp.experts.121.up_proj", + "model.layers.7.mlp.experts.122.down_proj", + "model.layers.7.mlp.experts.122.gate_proj", + "model.layers.7.mlp.experts.122.up_proj", + "model.layers.7.mlp.experts.123.down_proj", + "model.layers.7.mlp.experts.123.gate_proj", + "model.layers.7.mlp.experts.123.up_proj", + "model.layers.7.mlp.experts.124.down_proj", + "model.layers.7.mlp.experts.124.gate_proj", + "model.layers.7.mlp.experts.124.up_proj", + "model.layers.7.mlp.experts.125.down_proj", + "model.layers.7.mlp.experts.125.gate_proj", + "model.layers.7.mlp.experts.125.up_proj", + "model.layers.7.mlp.experts.126.down_proj", + "model.layers.7.mlp.experts.126.gate_proj", + "model.layers.7.mlp.experts.126.up_proj", + "model.layers.7.mlp.experts.127.down_proj", + "model.layers.7.mlp.experts.127.gate_proj", + "model.layers.7.mlp.experts.127.up_proj", + "model.layers.7.mlp.experts.128.down_proj", + "model.layers.7.mlp.experts.128.gate_proj", + "model.layers.7.mlp.experts.128.up_proj", + "model.layers.7.mlp.experts.129.down_proj", + "model.layers.7.mlp.experts.129.gate_proj", + "model.layers.7.mlp.experts.129.up_proj", + "model.layers.7.mlp.experts.13.down_proj", + "model.layers.7.mlp.experts.13.gate_proj", + "model.layers.7.mlp.experts.13.up_proj", + "model.layers.7.mlp.experts.130.down_proj", + "model.layers.7.mlp.experts.130.gate_proj", + "model.layers.7.mlp.experts.130.up_proj", + "model.layers.7.mlp.experts.131.down_proj", + "model.layers.7.mlp.experts.131.gate_proj", + "model.layers.7.mlp.experts.131.up_proj", + "model.layers.7.mlp.experts.132.down_proj", + "model.layers.7.mlp.experts.132.gate_proj", + "model.layers.7.mlp.experts.132.up_proj", + "model.layers.7.mlp.experts.133.down_proj", + "model.layers.7.mlp.experts.133.gate_proj", + "model.layers.7.mlp.experts.133.up_proj", + "model.layers.7.mlp.experts.134.down_proj", + "model.layers.7.mlp.experts.134.gate_proj", + "model.layers.7.mlp.experts.134.up_proj", + "model.layers.7.mlp.experts.135.down_proj", + "model.layers.7.mlp.experts.135.gate_proj", + "model.layers.7.mlp.experts.135.up_proj", + "model.layers.7.mlp.experts.136.down_proj", + "model.layers.7.mlp.experts.136.gate_proj", + "model.layers.7.mlp.experts.136.up_proj", + "model.layers.7.mlp.experts.137.down_proj", + "model.layers.7.mlp.experts.137.gate_proj", + "model.layers.7.mlp.experts.137.up_proj", + "model.layers.7.mlp.experts.138.down_proj", + "model.layers.7.mlp.experts.138.gate_proj", + "model.layers.7.mlp.experts.138.up_proj", + "model.layers.7.mlp.experts.139.down_proj", + "model.layers.7.mlp.experts.139.gate_proj", + "model.layers.7.mlp.experts.139.up_proj", + "model.layers.7.mlp.experts.14.down_proj", + "model.layers.7.mlp.experts.14.gate_proj", + "model.layers.7.mlp.experts.14.up_proj", + "model.layers.7.mlp.experts.140.down_proj", + "model.layers.7.mlp.experts.140.gate_proj", + "model.layers.7.mlp.experts.140.up_proj", + "model.layers.7.mlp.experts.141.down_proj", + "model.layers.7.mlp.experts.141.gate_proj", + "model.layers.7.mlp.experts.141.up_proj", + "model.layers.7.mlp.experts.142.down_proj", + "model.layers.7.mlp.experts.142.gate_proj", + "model.layers.7.mlp.experts.142.up_proj", + "model.layers.7.mlp.experts.143.down_proj", + "model.layers.7.mlp.experts.143.gate_proj", + "model.layers.7.mlp.experts.143.up_proj", + "model.layers.7.mlp.experts.144.down_proj", + "model.layers.7.mlp.experts.144.gate_proj", + "model.layers.7.mlp.experts.144.up_proj", + "model.layers.7.mlp.experts.145.down_proj", + "model.layers.7.mlp.experts.145.gate_proj", + "model.layers.7.mlp.experts.145.up_proj", + "model.layers.7.mlp.experts.146.down_proj", + "model.layers.7.mlp.experts.146.gate_proj", + "model.layers.7.mlp.experts.146.up_proj", + "model.layers.7.mlp.experts.147.down_proj", + "model.layers.7.mlp.experts.147.gate_proj", + "model.layers.7.mlp.experts.147.up_proj", + "model.layers.7.mlp.experts.148.down_proj", + "model.layers.7.mlp.experts.148.gate_proj", + "model.layers.7.mlp.experts.148.up_proj", + "model.layers.7.mlp.experts.149.down_proj", + "model.layers.7.mlp.experts.149.gate_proj", + "model.layers.7.mlp.experts.149.up_proj", + "model.layers.7.mlp.experts.15.down_proj", + "model.layers.7.mlp.experts.15.gate_proj", + "model.layers.7.mlp.experts.15.up_proj", + "model.layers.7.mlp.experts.150.down_proj", + "model.layers.7.mlp.experts.150.gate_proj", + "model.layers.7.mlp.experts.150.up_proj", + "model.layers.7.mlp.experts.151.down_proj", + "model.layers.7.mlp.experts.151.gate_proj", + "model.layers.7.mlp.experts.151.up_proj", + "model.layers.7.mlp.experts.152.down_proj", + "model.layers.7.mlp.experts.152.gate_proj", + "model.layers.7.mlp.experts.152.up_proj", + "model.layers.7.mlp.experts.153.down_proj", + "model.layers.7.mlp.experts.153.gate_proj", + "model.layers.7.mlp.experts.153.up_proj", + "model.layers.7.mlp.experts.154.down_proj", + "model.layers.7.mlp.experts.154.gate_proj", + "model.layers.7.mlp.experts.154.up_proj", + "model.layers.7.mlp.experts.155.down_proj", + "model.layers.7.mlp.experts.155.gate_proj", + "model.layers.7.mlp.experts.155.up_proj", + "model.layers.7.mlp.experts.156.down_proj", + "model.layers.7.mlp.experts.156.gate_proj", + "model.layers.7.mlp.experts.156.up_proj", + "model.layers.7.mlp.experts.157.down_proj", + "model.layers.7.mlp.experts.157.gate_proj", + "model.layers.7.mlp.experts.157.up_proj", + "model.layers.7.mlp.experts.158.down_proj", + "model.layers.7.mlp.experts.158.gate_proj", + "model.layers.7.mlp.experts.158.up_proj", + "model.layers.7.mlp.experts.159.down_proj", + "model.layers.7.mlp.experts.159.gate_proj", + "model.layers.7.mlp.experts.159.up_proj", + "model.layers.7.mlp.experts.16.down_proj", + "model.layers.7.mlp.experts.16.gate_proj", + "model.layers.7.mlp.experts.16.up_proj", + "model.layers.7.mlp.experts.160.down_proj", + "model.layers.7.mlp.experts.160.gate_proj", + "model.layers.7.mlp.experts.160.up_proj", + "model.layers.7.mlp.experts.161.down_proj", + "model.layers.7.mlp.experts.161.gate_proj", + "model.layers.7.mlp.experts.161.up_proj", + "model.layers.7.mlp.experts.162.down_proj", + "model.layers.7.mlp.experts.162.gate_proj", + "model.layers.7.mlp.experts.162.up_proj", + "model.layers.7.mlp.experts.163.down_proj", + "model.layers.7.mlp.experts.163.gate_proj", + "model.layers.7.mlp.experts.163.up_proj", + "model.layers.7.mlp.experts.164.down_proj", + "model.layers.7.mlp.experts.164.gate_proj", + "model.layers.7.mlp.experts.164.up_proj", + "model.layers.7.mlp.experts.165.down_proj", + "model.layers.7.mlp.experts.165.gate_proj", + "model.layers.7.mlp.experts.165.up_proj", + "model.layers.7.mlp.experts.166.down_proj", + "model.layers.7.mlp.experts.166.gate_proj", + "model.layers.7.mlp.experts.166.up_proj", + "model.layers.7.mlp.experts.167.down_proj", + "model.layers.7.mlp.experts.167.gate_proj", + "model.layers.7.mlp.experts.167.up_proj", + "model.layers.7.mlp.experts.168.down_proj", + "model.layers.7.mlp.experts.168.gate_proj", + "model.layers.7.mlp.experts.168.up_proj", + "model.layers.7.mlp.experts.169.down_proj", + "model.layers.7.mlp.experts.169.gate_proj", + "model.layers.7.mlp.experts.169.up_proj", + "model.layers.7.mlp.experts.17.down_proj", + "model.layers.7.mlp.experts.17.gate_proj", + "model.layers.7.mlp.experts.17.up_proj", + "model.layers.7.mlp.experts.170.down_proj", + "model.layers.7.mlp.experts.170.gate_proj", + "model.layers.7.mlp.experts.170.up_proj", + "model.layers.7.mlp.experts.171.down_proj", + "model.layers.7.mlp.experts.171.gate_proj", + "model.layers.7.mlp.experts.171.up_proj", + "model.layers.7.mlp.experts.172.down_proj", + "model.layers.7.mlp.experts.172.gate_proj", + "model.layers.7.mlp.experts.172.up_proj", + "model.layers.7.mlp.experts.173.down_proj", + "model.layers.7.mlp.experts.173.gate_proj", + "model.layers.7.mlp.experts.173.up_proj", + "model.layers.7.mlp.experts.174.down_proj", + "model.layers.7.mlp.experts.174.gate_proj", + "model.layers.7.mlp.experts.174.up_proj", + "model.layers.7.mlp.experts.175.down_proj", + "model.layers.7.mlp.experts.175.gate_proj", + "model.layers.7.mlp.experts.175.up_proj", + "model.layers.7.mlp.experts.176.down_proj", + "model.layers.7.mlp.experts.176.gate_proj", + "model.layers.7.mlp.experts.176.up_proj", + "model.layers.7.mlp.experts.177.down_proj", + "model.layers.7.mlp.experts.177.gate_proj", + "model.layers.7.mlp.experts.177.up_proj", + "model.layers.7.mlp.experts.178.down_proj", + "model.layers.7.mlp.experts.178.gate_proj", + "model.layers.7.mlp.experts.178.up_proj", + "model.layers.7.mlp.experts.179.down_proj", + "model.layers.7.mlp.experts.179.gate_proj", + "model.layers.7.mlp.experts.179.up_proj", + "model.layers.7.mlp.experts.18.down_proj", + "model.layers.7.mlp.experts.18.gate_proj", + "model.layers.7.mlp.experts.18.up_proj", + "model.layers.7.mlp.experts.180.down_proj", + "model.layers.7.mlp.experts.180.gate_proj", + "model.layers.7.mlp.experts.180.up_proj", + "model.layers.7.mlp.experts.181.down_proj", + "model.layers.7.mlp.experts.181.gate_proj", + "model.layers.7.mlp.experts.181.up_proj", + "model.layers.7.mlp.experts.182.down_proj", + "model.layers.7.mlp.experts.182.gate_proj", + "model.layers.7.mlp.experts.182.up_proj", + "model.layers.7.mlp.experts.183.down_proj", + "model.layers.7.mlp.experts.183.gate_proj", + "model.layers.7.mlp.experts.183.up_proj", + "model.layers.7.mlp.experts.184.down_proj", + "model.layers.7.mlp.experts.184.gate_proj", + "model.layers.7.mlp.experts.184.up_proj", + "model.layers.7.mlp.experts.185.down_proj", + "model.layers.7.mlp.experts.185.gate_proj", + "model.layers.7.mlp.experts.185.up_proj", + "model.layers.7.mlp.experts.186.down_proj", + "model.layers.7.mlp.experts.186.gate_proj", + "model.layers.7.mlp.experts.186.up_proj", + "model.layers.7.mlp.experts.187.down_proj", + "model.layers.7.mlp.experts.187.gate_proj", + "model.layers.7.mlp.experts.187.up_proj", + "model.layers.7.mlp.experts.188.down_proj", + "model.layers.7.mlp.experts.188.gate_proj", + "model.layers.7.mlp.experts.188.up_proj", + "model.layers.7.mlp.experts.189.down_proj", + "model.layers.7.mlp.experts.189.gate_proj", + "model.layers.7.mlp.experts.189.up_proj", + "model.layers.7.mlp.experts.19.down_proj", + "model.layers.7.mlp.experts.19.gate_proj", + "model.layers.7.mlp.experts.19.up_proj", + "model.layers.7.mlp.experts.190.down_proj", + "model.layers.7.mlp.experts.190.gate_proj", + "model.layers.7.mlp.experts.190.up_proj", + "model.layers.7.mlp.experts.191.down_proj", + "model.layers.7.mlp.experts.191.gate_proj", + "model.layers.7.mlp.experts.191.up_proj", + "model.layers.7.mlp.experts.192.down_proj", + "model.layers.7.mlp.experts.192.gate_proj", + "model.layers.7.mlp.experts.192.up_proj", + "model.layers.7.mlp.experts.193.down_proj", + "model.layers.7.mlp.experts.193.gate_proj", + "model.layers.7.mlp.experts.193.up_proj", + "model.layers.7.mlp.experts.194.down_proj", + "model.layers.7.mlp.experts.194.gate_proj", + "model.layers.7.mlp.experts.194.up_proj", + "model.layers.7.mlp.experts.195.down_proj", + "model.layers.7.mlp.experts.195.gate_proj", + "model.layers.7.mlp.experts.195.up_proj", + "model.layers.7.mlp.experts.196.down_proj", + "model.layers.7.mlp.experts.196.gate_proj", + "model.layers.7.mlp.experts.196.up_proj", + "model.layers.7.mlp.experts.197.down_proj", + "model.layers.7.mlp.experts.197.gate_proj", + "model.layers.7.mlp.experts.197.up_proj", + "model.layers.7.mlp.experts.198.down_proj", + "model.layers.7.mlp.experts.198.gate_proj", + "model.layers.7.mlp.experts.198.up_proj", + "model.layers.7.mlp.experts.199.down_proj", + "model.layers.7.mlp.experts.199.gate_proj", + "model.layers.7.mlp.experts.199.up_proj", + "model.layers.7.mlp.experts.2.down_proj", + "model.layers.7.mlp.experts.2.gate_proj", + "model.layers.7.mlp.experts.2.up_proj", + "model.layers.7.mlp.experts.20.down_proj", + "model.layers.7.mlp.experts.20.gate_proj", + "model.layers.7.mlp.experts.20.up_proj", + "model.layers.7.mlp.experts.200.down_proj", + "model.layers.7.mlp.experts.200.gate_proj", + "model.layers.7.mlp.experts.200.up_proj", + "model.layers.7.mlp.experts.201.down_proj", + "model.layers.7.mlp.experts.201.gate_proj", + "model.layers.7.mlp.experts.201.up_proj", + "model.layers.7.mlp.experts.202.down_proj", + "model.layers.7.mlp.experts.202.gate_proj", + "model.layers.7.mlp.experts.202.up_proj", + "model.layers.7.mlp.experts.203.down_proj", + "model.layers.7.mlp.experts.203.gate_proj", + "model.layers.7.mlp.experts.203.up_proj", + "model.layers.7.mlp.experts.204.down_proj", + "model.layers.7.mlp.experts.204.gate_proj", + "model.layers.7.mlp.experts.204.up_proj", + "model.layers.7.mlp.experts.205.down_proj", + "model.layers.7.mlp.experts.205.gate_proj", + "model.layers.7.mlp.experts.205.up_proj", + "model.layers.7.mlp.experts.206.down_proj", + "model.layers.7.mlp.experts.206.gate_proj", + "model.layers.7.mlp.experts.206.up_proj", + "model.layers.7.mlp.experts.207.down_proj", + "model.layers.7.mlp.experts.207.gate_proj", + "model.layers.7.mlp.experts.207.up_proj", + "model.layers.7.mlp.experts.208.down_proj", + "model.layers.7.mlp.experts.208.gate_proj", + "model.layers.7.mlp.experts.208.up_proj", + "model.layers.7.mlp.experts.209.down_proj", + "model.layers.7.mlp.experts.209.gate_proj", + "model.layers.7.mlp.experts.209.up_proj", + "model.layers.7.mlp.experts.21.down_proj", + "model.layers.7.mlp.experts.21.gate_proj", + "model.layers.7.mlp.experts.21.up_proj", + "model.layers.7.mlp.experts.210.down_proj", + "model.layers.7.mlp.experts.210.gate_proj", + "model.layers.7.mlp.experts.210.up_proj", + "model.layers.7.mlp.experts.211.down_proj", + "model.layers.7.mlp.experts.211.gate_proj", + "model.layers.7.mlp.experts.211.up_proj", + "model.layers.7.mlp.experts.212.down_proj", + "model.layers.7.mlp.experts.212.gate_proj", + "model.layers.7.mlp.experts.212.up_proj", + "model.layers.7.mlp.experts.213.down_proj", + "model.layers.7.mlp.experts.213.gate_proj", + "model.layers.7.mlp.experts.213.up_proj", + "model.layers.7.mlp.experts.214.down_proj", + "model.layers.7.mlp.experts.214.gate_proj", + "model.layers.7.mlp.experts.214.up_proj", + "model.layers.7.mlp.experts.215.down_proj", + "model.layers.7.mlp.experts.215.gate_proj", + "model.layers.7.mlp.experts.215.up_proj", + "model.layers.7.mlp.experts.216.down_proj", + "model.layers.7.mlp.experts.216.gate_proj", + "model.layers.7.mlp.experts.216.up_proj", + "model.layers.7.mlp.experts.217.down_proj", + "model.layers.7.mlp.experts.217.gate_proj", + "model.layers.7.mlp.experts.217.up_proj", + "model.layers.7.mlp.experts.218.down_proj", + "model.layers.7.mlp.experts.218.gate_proj", + "model.layers.7.mlp.experts.218.up_proj", + "model.layers.7.mlp.experts.219.down_proj", + "model.layers.7.mlp.experts.219.gate_proj", + "model.layers.7.mlp.experts.219.up_proj", + "model.layers.7.mlp.experts.22.down_proj", + "model.layers.7.mlp.experts.22.gate_proj", + "model.layers.7.mlp.experts.22.up_proj", + "model.layers.7.mlp.experts.220.down_proj", + "model.layers.7.mlp.experts.220.gate_proj", + "model.layers.7.mlp.experts.220.up_proj", + "model.layers.7.mlp.experts.221.down_proj", + "model.layers.7.mlp.experts.221.gate_proj", + "model.layers.7.mlp.experts.221.up_proj", + "model.layers.7.mlp.experts.222.down_proj", + "model.layers.7.mlp.experts.222.gate_proj", + "model.layers.7.mlp.experts.222.up_proj", + "model.layers.7.mlp.experts.223.down_proj", + "model.layers.7.mlp.experts.223.gate_proj", + "model.layers.7.mlp.experts.223.up_proj", + "model.layers.7.mlp.experts.224.down_proj", + "model.layers.7.mlp.experts.224.gate_proj", + "model.layers.7.mlp.experts.224.up_proj", + "model.layers.7.mlp.experts.225.down_proj", + "model.layers.7.mlp.experts.225.gate_proj", + "model.layers.7.mlp.experts.225.up_proj", + "model.layers.7.mlp.experts.226.down_proj", + "model.layers.7.mlp.experts.226.gate_proj", + "model.layers.7.mlp.experts.226.up_proj", + "model.layers.7.mlp.experts.227.down_proj", + "model.layers.7.mlp.experts.227.gate_proj", + "model.layers.7.mlp.experts.227.up_proj", + "model.layers.7.mlp.experts.228.down_proj", + "model.layers.7.mlp.experts.228.gate_proj", + "model.layers.7.mlp.experts.228.up_proj", + "model.layers.7.mlp.experts.229.down_proj", + "model.layers.7.mlp.experts.229.gate_proj", + "model.layers.7.mlp.experts.229.up_proj", + "model.layers.7.mlp.experts.23.down_proj", + "model.layers.7.mlp.experts.23.gate_proj", + "model.layers.7.mlp.experts.23.up_proj", + "model.layers.7.mlp.experts.230.down_proj", + "model.layers.7.mlp.experts.230.gate_proj", + "model.layers.7.mlp.experts.230.up_proj", + "model.layers.7.mlp.experts.231.down_proj", + "model.layers.7.mlp.experts.231.gate_proj", + "model.layers.7.mlp.experts.231.up_proj", + "model.layers.7.mlp.experts.232.down_proj", + "model.layers.7.mlp.experts.232.gate_proj", + "model.layers.7.mlp.experts.232.up_proj", + "model.layers.7.mlp.experts.233.down_proj", + "model.layers.7.mlp.experts.233.gate_proj", + "model.layers.7.mlp.experts.233.up_proj", + "model.layers.7.mlp.experts.234.down_proj", + "model.layers.7.mlp.experts.234.gate_proj", + "model.layers.7.mlp.experts.234.up_proj", + "model.layers.7.mlp.experts.235.down_proj", + "model.layers.7.mlp.experts.235.gate_proj", + "model.layers.7.mlp.experts.235.up_proj", + "model.layers.7.mlp.experts.236.down_proj", + "model.layers.7.mlp.experts.236.gate_proj", + "model.layers.7.mlp.experts.236.up_proj", + "model.layers.7.mlp.experts.237.down_proj", + "model.layers.7.mlp.experts.237.gate_proj", + "model.layers.7.mlp.experts.237.up_proj", + "model.layers.7.mlp.experts.238.down_proj", + "model.layers.7.mlp.experts.238.gate_proj", + "model.layers.7.mlp.experts.238.up_proj", + "model.layers.7.mlp.experts.239.down_proj", + "model.layers.7.mlp.experts.239.gate_proj", + "model.layers.7.mlp.experts.239.up_proj", + "model.layers.7.mlp.experts.24.down_proj", + "model.layers.7.mlp.experts.24.gate_proj", + "model.layers.7.mlp.experts.24.up_proj", + "model.layers.7.mlp.experts.240.down_proj", + "model.layers.7.mlp.experts.240.gate_proj", + "model.layers.7.mlp.experts.240.up_proj", + "model.layers.7.mlp.experts.241.down_proj", + "model.layers.7.mlp.experts.241.gate_proj", + "model.layers.7.mlp.experts.241.up_proj", + "model.layers.7.mlp.experts.242.down_proj", + "model.layers.7.mlp.experts.242.gate_proj", + "model.layers.7.mlp.experts.242.up_proj", + "model.layers.7.mlp.experts.243.down_proj", + "model.layers.7.mlp.experts.243.gate_proj", + "model.layers.7.mlp.experts.243.up_proj", + "model.layers.7.mlp.experts.244.down_proj", + "model.layers.7.mlp.experts.244.gate_proj", + "model.layers.7.mlp.experts.244.up_proj", + "model.layers.7.mlp.experts.245.down_proj", + "model.layers.7.mlp.experts.245.gate_proj", + "model.layers.7.mlp.experts.245.up_proj", + "model.layers.7.mlp.experts.246.down_proj", + "model.layers.7.mlp.experts.246.gate_proj", + "model.layers.7.mlp.experts.246.up_proj", + "model.layers.7.mlp.experts.247.down_proj", + "model.layers.7.mlp.experts.247.gate_proj", + "model.layers.7.mlp.experts.247.up_proj", + "model.layers.7.mlp.experts.248.down_proj", + "model.layers.7.mlp.experts.248.gate_proj", + "model.layers.7.mlp.experts.248.up_proj", + "model.layers.7.mlp.experts.249.down_proj", + "model.layers.7.mlp.experts.249.gate_proj", + "model.layers.7.mlp.experts.249.up_proj", + "model.layers.7.mlp.experts.25.down_proj", + "model.layers.7.mlp.experts.25.gate_proj", + "model.layers.7.mlp.experts.25.up_proj", + "model.layers.7.mlp.experts.250.down_proj", + "model.layers.7.mlp.experts.250.gate_proj", + "model.layers.7.mlp.experts.250.up_proj", + "model.layers.7.mlp.experts.251.down_proj", + "model.layers.7.mlp.experts.251.gate_proj", + "model.layers.7.mlp.experts.251.up_proj", + "model.layers.7.mlp.experts.252.down_proj", + "model.layers.7.mlp.experts.252.gate_proj", + "model.layers.7.mlp.experts.252.up_proj", + "model.layers.7.mlp.experts.253.down_proj", + "model.layers.7.mlp.experts.253.gate_proj", + "model.layers.7.mlp.experts.253.up_proj", + "model.layers.7.mlp.experts.254.down_proj", + "model.layers.7.mlp.experts.254.gate_proj", + "model.layers.7.mlp.experts.254.up_proj", + "model.layers.7.mlp.experts.255.down_proj", + "model.layers.7.mlp.experts.255.gate_proj", + "model.layers.7.mlp.experts.255.up_proj", + "model.layers.7.mlp.experts.26.down_proj", + "model.layers.7.mlp.experts.26.gate_proj", + "model.layers.7.mlp.experts.26.up_proj", + "model.layers.7.mlp.experts.27.down_proj", + "model.layers.7.mlp.experts.27.gate_proj", + "model.layers.7.mlp.experts.27.up_proj", + "model.layers.7.mlp.experts.28.down_proj", + "model.layers.7.mlp.experts.28.gate_proj", + "model.layers.7.mlp.experts.28.up_proj", + "model.layers.7.mlp.experts.29.down_proj", + "model.layers.7.mlp.experts.29.gate_proj", + "model.layers.7.mlp.experts.29.up_proj", + "model.layers.7.mlp.experts.3.down_proj", + "model.layers.7.mlp.experts.3.gate_proj", + "model.layers.7.mlp.experts.3.up_proj", + "model.layers.7.mlp.experts.30.down_proj", + "model.layers.7.mlp.experts.30.gate_proj", + "model.layers.7.mlp.experts.30.up_proj", + "model.layers.7.mlp.experts.31.down_proj", + "model.layers.7.mlp.experts.31.gate_proj", + "model.layers.7.mlp.experts.31.up_proj", + "model.layers.7.mlp.experts.32.down_proj", + "model.layers.7.mlp.experts.32.gate_proj", + "model.layers.7.mlp.experts.32.up_proj", + "model.layers.7.mlp.experts.33.down_proj", + "model.layers.7.mlp.experts.33.gate_proj", + "model.layers.7.mlp.experts.33.up_proj", + "model.layers.7.mlp.experts.34.down_proj", + "model.layers.7.mlp.experts.34.gate_proj", + "model.layers.7.mlp.experts.34.up_proj", + "model.layers.7.mlp.experts.35.down_proj", + "model.layers.7.mlp.experts.35.gate_proj", + "model.layers.7.mlp.experts.35.up_proj", + "model.layers.7.mlp.experts.36.down_proj", + "model.layers.7.mlp.experts.36.gate_proj", + "model.layers.7.mlp.experts.36.up_proj", + "model.layers.7.mlp.experts.37.down_proj", + "model.layers.7.mlp.experts.37.gate_proj", + "model.layers.7.mlp.experts.37.up_proj", + "model.layers.7.mlp.experts.38.down_proj", + "model.layers.7.mlp.experts.38.gate_proj", + "model.layers.7.mlp.experts.38.up_proj", + "model.layers.7.mlp.experts.39.down_proj", + "model.layers.7.mlp.experts.39.gate_proj", + "model.layers.7.mlp.experts.39.up_proj", + "model.layers.7.mlp.experts.4.down_proj", + "model.layers.7.mlp.experts.4.gate_proj", + "model.layers.7.mlp.experts.4.up_proj", + "model.layers.7.mlp.experts.40.down_proj", + "model.layers.7.mlp.experts.40.gate_proj", + "model.layers.7.mlp.experts.40.up_proj", + "model.layers.7.mlp.experts.41.down_proj", + "model.layers.7.mlp.experts.41.gate_proj", + "model.layers.7.mlp.experts.41.up_proj", + "model.layers.7.mlp.experts.42.down_proj", + "model.layers.7.mlp.experts.42.gate_proj", + "model.layers.7.mlp.experts.42.up_proj", + "model.layers.7.mlp.experts.43.down_proj", + "model.layers.7.mlp.experts.43.gate_proj", + "model.layers.7.mlp.experts.43.up_proj", + "model.layers.7.mlp.experts.44.down_proj", + "model.layers.7.mlp.experts.44.gate_proj", + "model.layers.7.mlp.experts.44.up_proj", + "model.layers.7.mlp.experts.45.down_proj", + "model.layers.7.mlp.experts.45.gate_proj", + "model.layers.7.mlp.experts.45.up_proj", + "model.layers.7.mlp.experts.46.down_proj", + "model.layers.7.mlp.experts.46.gate_proj", + "model.layers.7.mlp.experts.46.up_proj", + "model.layers.7.mlp.experts.47.down_proj", + "model.layers.7.mlp.experts.47.gate_proj", + "model.layers.7.mlp.experts.47.up_proj", + "model.layers.7.mlp.experts.48.down_proj", + "model.layers.7.mlp.experts.48.gate_proj", + "model.layers.7.mlp.experts.48.up_proj", + "model.layers.7.mlp.experts.49.down_proj", + "model.layers.7.mlp.experts.49.gate_proj", + "model.layers.7.mlp.experts.49.up_proj", + "model.layers.7.mlp.experts.5.down_proj", + "model.layers.7.mlp.experts.5.gate_proj", + "model.layers.7.mlp.experts.5.up_proj", + "model.layers.7.mlp.experts.50.down_proj", + "model.layers.7.mlp.experts.50.gate_proj", + "model.layers.7.mlp.experts.50.up_proj", + "model.layers.7.mlp.experts.51.down_proj", + "model.layers.7.mlp.experts.51.gate_proj", + "model.layers.7.mlp.experts.51.up_proj", + "model.layers.7.mlp.experts.52.down_proj", + "model.layers.7.mlp.experts.52.gate_proj", + "model.layers.7.mlp.experts.52.up_proj", + "model.layers.7.mlp.experts.53.down_proj", + "model.layers.7.mlp.experts.53.gate_proj", + "model.layers.7.mlp.experts.53.up_proj", + "model.layers.7.mlp.experts.54.down_proj", + "model.layers.7.mlp.experts.54.gate_proj", + "model.layers.7.mlp.experts.54.up_proj", + "model.layers.7.mlp.experts.55.down_proj", + "model.layers.7.mlp.experts.55.gate_proj", + "model.layers.7.mlp.experts.55.up_proj", + "model.layers.7.mlp.experts.56.down_proj", + "model.layers.7.mlp.experts.56.gate_proj", + "model.layers.7.mlp.experts.56.up_proj", + "model.layers.7.mlp.experts.57.down_proj", + "model.layers.7.mlp.experts.57.gate_proj", + "model.layers.7.mlp.experts.57.up_proj", + "model.layers.7.mlp.experts.58.down_proj", + "model.layers.7.mlp.experts.58.gate_proj", + "model.layers.7.mlp.experts.58.up_proj", + "model.layers.7.mlp.experts.59.down_proj", + "model.layers.7.mlp.experts.59.gate_proj", + "model.layers.7.mlp.experts.59.up_proj", + "model.layers.7.mlp.experts.6.down_proj", + "model.layers.7.mlp.experts.6.gate_proj", + "model.layers.7.mlp.experts.6.up_proj", + "model.layers.7.mlp.experts.60.down_proj", + "model.layers.7.mlp.experts.60.gate_proj", + "model.layers.7.mlp.experts.60.up_proj", + "model.layers.7.mlp.experts.61.down_proj", + "model.layers.7.mlp.experts.61.gate_proj", + "model.layers.7.mlp.experts.61.up_proj", + "model.layers.7.mlp.experts.62.down_proj", + "model.layers.7.mlp.experts.62.gate_proj", + "model.layers.7.mlp.experts.62.up_proj", + "model.layers.7.mlp.experts.63.down_proj", + "model.layers.7.mlp.experts.63.gate_proj", + "model.layers.7.mlp.experts.63.up_proj", + "model.layers.7.mlp.experts.64.down_proj", + "model.layers.7.mlp.experts.64.gate_proj", + "model.layers.7.mlp.experts.64.up_proj", + "model.layers.7.mlp.experts.65.down_proj", + "model.layers.7.mlp.experts.65.gate_proj", + "model.layers.7.mlp.experts.65.up_proj", + "model.layers.7.mlp.experts.66.down_proj", + "model.layers.7.mlp.experts.66.gate_proj", + "model.layers.7.mlp.experts.66.up_proj", + "model.layers.7.mlp.experts.67.down_proj", + "model.layers.7.mlp.experts.67.gate_proj", + "model.layers.7.mlp.experts.67.up_proj", + "model.layers.7.mlp.experts.68.down_proj", + "model.layers.7.mlp.experts.68.gate_proj", + "model.layers.7.mlp.experts.68.up_proj", + "model.layers.7.mlp.experts.69.down_proj", + "model.layers.7.mlp.experts.69.gate_proj", + "model.layers.7.mlp.experts.69.up_proj", + "model.layers.7.mlp.experts.7.down_proj", + "model.layers.7.mlp.experts.7.gate_proj", + "model.layers.7.mlp.experts.7.up_proj", + "model.layers.7.mlp.experts.70.down_proj", + "model.layers.7.mlp.experts.70.gate_proj", + "model.layers.7.mlp.experts.70.up_proj", + "model.layers.7.mlp.experts.71.down_proj", + "model.layers.7.mlp.experts.71.gate_proj", + "model.layers.7.mlp.experts.71.up_proj", + "model.layers.7.mlp.experts.72.down_proj", + "model.layers.7.mlp.experts.72.gate_proj", + "model.layers.7.mlp.experts.72.up_proj", + "model.layers.7.mlp.experts.73.down_proj", + "model.layers.7.mlp.experts.73.gate_proj", + "model.layers.7.mlp.experts.73.up_proj", + "model.layers.7.mlp.experts.74.down_proj", + "model.layers.7.mlp.experts.74.gate_proj", + "model.layers.7.mlp.experts.74.up_proj", + "model.layers.7.mlp.experts.75.down_proj", + "model.layers.7.mlp.experts.75.gate_proj", + "model.layers.7.mlp.experts.75.up_proj", + "model.layers.7.mlp.experts.76.down_proj", + "model.layers.7.mlp.experts.76.gate_proj", + "model.layers.7.mlp.experts.76.up_proj", + "model.layers.7.mlp.experts.77.down_proj", + "model.layers.7.mlp.experts.77.gate_proj", + "model.layers.7.mlp.experts.77.up_proj", + "model.layers.7.mlp.experts.78.down_proj", + "model.layers.7.mlp.experts.78.gate_proj", + "model.layers.7.mlp.experts.78.up_proj", + "model.layers.7.mlp.experts.79.down_proj", + "model.layers.7.mlp.experts.79.gate_proj", + "model.layers.7.mlp.experts.79.up_proj", + "model.layers.7.mlp.experts.8.down_proj", + "model.layers.7.mlp.experts.8.gate_proj", + "model.layers.7.mlp.experts.8.up_proj", + "model.layers.7.mlp.experts.80.down_proj", + "model.layers.7.mlp.experts.80.gate_proj", + "model.layers.7.mlp.experts.80.up_proj", + "model.layers.7.mlp.experts.81.down_proj", + "model.layers.7.mlp.experts.81.gate_proj", + "model.layers.7.mlp.experts.81.up_proj", + "model.layers.7.mlp.experts.82.down_proj", + "model.layers.7.mlp.experts.82.gate_proj", + "model.layers.7.mlp.experts.82.up_proj", + "model.layers.7.mlp.experts.83.down_proj", + "model.layers.7.mlp.experts.83.gate_proj", + "model.layers.7.mlp.experts.83.up_proj", + "model.layers.7.mlp.experts.84.down_proj", + "model.layers.7.mlp.experts.84.gate_proj", + "model.layers.7.mlp.experts.84.up_proj", + "model.layers.7.mlp.experts.85.down_proj", + "model.layers.7.mlp.experts.85.gate_proj", + "model.layers.7.mlp.experts.85.up_proj", + "model.layers.7.mlp.experts.86.down_proj", + "model.layers.7.mlp.experts.86.gate_proj", + "model.layers.7.mlp.experts.86.up_proj", + "model.layers.7.mlp.experts.87.down_proj", + "model.layers.7.mlp.experts.87.gate_proj", + "model.layers.7.mlp.experts.87.up_proj", + "model.layers.7.mlp.experts.88.down_proj", + "model.layers.7.mlp.experts.88.gate_proj", + "model.layers.7.mlp.experts.88.up_proj", + "model.layers.7.mlp.experts.89.down_proj", + "model.layers.7.mlp.experts.89.gate_proj", + "model.layers.7.mlp.experts.89.up_proj", + "model.layers.7.mlp.experts.9.down_proj", + "model.layers.7.mlp.experts.9.gate_proj", + "model.layers.7.mlp.experts.9.up_proj", + "model.layers.7.mlp.experts.90.down_proj", + "model.layers.7.mlp.experts.90.gate_proj", + "model.layers.7.mlp.experts.90.up_proj", + "model.layers.7.mlp.experts.91.down_proj", + "model.layers.7.mlp.experts.91.gate_proj", + "model.layers.7.mlp.experts.91.up_proj", + "model.layers.7.mlp.experts.92.down_proj", + "model.layers.7.mlp.experts.92.gate_proj", + "model.layers.7.mlp.experts.92.up_proj", + "model.layers.7.mlp.experts.93.down_proj", + "model.layers.7.mlp.experts.93.gate_proj", + "model.layers.7.mlp.experts.93.up_proj", + "model.layers.7.mlp.experts.94.down_proj", + "model.layers.7.mlp.experts.94.gate_proj", + "model.layers.7.mlp.experts.94.up_proj", + "model.layers.7.mlp.experts.95.down_proj", + "model.layers.7.mlp.experts.95.gate_proj", + "model.layers.7.mlp.experts.95.up_proj", + "model.layers.7.mlp.experts.96.down_proj", + "model.layers.7.mlp.experts.96.gate_proj", + "model.layers.7.mlp.experts.96.up_proj", + "model.layers.7.mlp.experts.97.down_proj", + "model.layers.7.mlp.experts.97.gate_proj", + "model.layers.7.mlp.experts.97.up_proj", + "model.layers.7.mlp.experts.98.down_proj", + "model.layers.7.mlp.experts.98.gate_proj", + "model.layers.7.mlp.experts.98.up_proj", + "model.layers.7.mlp.experts.99.down_proj", + "model.layers.7.mlp.experts.99.gate_proj", + "model.layers.7.mlp.experts.99.up_proj", + "model.layers.70.mlp.experts.0.down_proj", + "model.layers.70.mlp.experts.0.gate_proj", + "model.layers.70.mlp.experts.0.up_proj", + "model.layers.70.mlp.experts.1.down_proj", + "model.layers.70.mlp.experts.1.gate_proj", + "model.layers.70.mlp.experts.1.up_proj", + "model.layers.70.mlp.experts.10.down_proj", + "model.layers.70.mlp.experts.10.gate_proj", + "model.layers.70.mlp.experts.10.up_proj", + "model.layers.70.mlp.experts.100.down_proj", + "model.layers.70.mlp.experts.100.gate_proj", + "model.layers.70.mlp.experts.100.up_proj", + "model.layers.70.mlp.experts.101.down_proj", + "model.layers.70.mlp.experts.101.gate_proj", + "model.layers.70.mlp.experts.101.up_proj", + "model.layers.70.mlp.experts.102.down_proj", + "model.layers.70.mlp.experts.102.gate_proj", + "model.layers.70.mlp.experts.102.up_proj", + "model.layers.70.mlp.experts.103.down_proj", + "model.layers.70.mlp.experts.103.gate_proj", + "model.layers.70.mlp.experts.103.up_proj", + "model.layers.70.mlp.experts.104.down_proj", + "model.layers.70.mlp.experts.104.gate_proj", + "model.layers.70.mlp.experts.104.up_proj", + "model.layers.70.mlp.experts.105.down_proj", + "model.layers.70.mlp.experts.105.gate_proj", + "model.layers.70.mlp.experts.105.up_proj", + "model.layers.70.mlp.experts.106.down_proj", + "model.layers.70.mlp.experts.106.gate_proj", + "model.layers.70.mlp.experts.106.up_proj", + "model.layers.70.mlp.experts.107.down_proj", + "model.layers.70.mlp.experts.107.gate_proj", + "model.layers.70.mlp.experts.107.up_proj", + "model.layers.70.mlp.experts.108.down_proj", + "model.layers.70.mlp.experts.108.gate_proj", + "model.layers.70.mlp.experts.108.up_proj", + "model.layers.70.mlp.experts.109.down_proj", + "model.layers.70.mlp.experts.109.gate_proj", + "model.layers.70.mlp.experts.109.up_proj", + "model.layers.70.mlp.experts.11.down_proj", + "model.layers.70.mlp.experts.11.gate_proj", + "model.layers.70.mlp.experts.11.up_proj", + "model.layers.70.mlp.experts.110.down_proj", + "model.layers.70.mlp.experts.110.gate_proj", + "model.layers.70.mlp.experts.110.up_proj", + "model.layers.70.mlp.experts.111.down_proj", + "model.layers.70.mlp.experts.111.gate_proj", + "model.layers.70.mlp.experts.111.up_proj", + "model.layers.70.mlp.experts.112.down_proj", + "model.layers.70.mlp.experts.112.gate_proj", + "model.layers.70.mlp.experts.112.up_proj", + "model.layers.70.mlp.experts.113.down_proj", + "model.layers.70.mlp.experts.113.gate_proj", + "model.layers.70.mlp.experts.113.up_proj", + "model.layers.70.mlp.experts.114.down_proj", + "model.layers.70.mlp.experts.114.gate_proj", + "model.layers.70.mlp.experts.114.up_proj", + "model.layers.70.mlp.experts.115.down_proj", + "model.layers.70.mlp.experts.115.gate_proj", + "model.layers.70.mlp.experts.115.up_proj", + "model.layers.70.mlp.experts.116.down_proj", + "model.layers.70.mlp.experts.116.gate_proj", + "model.layers.70.mlp.experts.116.up_proj", + "model.layers.70.mlp.experts.117.down_proj", + "model.layers.70.mlp.experts.117.gate_proj", + "model.layers.70.mlp.experts.117.up_proj", + "model.layers.70.mlp.experts.118.down_proj", + "model.layers.70.mlp.experts.118.gate_proj", + "model.layers.70.mlp.experts.118.up_proj", + "model.layers.70.mlp.experts.119.down_proj", + "model.layers.70.mlp.experts.119.gate_proj", + "model.layers.70.mlp.experts.119.up_proj", + "model.layers.70.mlp.experts.12.down_proj", + "model.layers.70.mlp.experts.12.gate_proj", + "model.layers.70.mlp.experts.12.up_proj", + "model.layers.70.mlp.experts.120.down_proj", + "model.layers.70.mlp.experts.120.gate_proj", + "model.layers.70.mlp.experts.120.up_proj", + "model.layers.70.mlp.experts.121.down_proj", + "model.layers.70.mlp.experts.121.gate_proj", + "model.layers.70.mlp.experts.121.up_proj", + "model.layers.70.mlp.experts.122.down_proj", + "model.layers.70.mlp.experts.122.gate_proj", + "model.layers.70.mlp.experts.122.up_proj", + "model.layers.70.mlp.experts.123.down_proj", + "model.layers.70.mlp.experts.123.gate_proj", + "model.layers.70.mlp.experts.123.up_proj", + "model.layers.70.mlp.experts.124.down_proj", + "model.layers.70.mlp.experts.124.gate_proj", + "model.layers.70.mlp.experts.124.up_proj", + "model.layers.70.mlp.experts.125.down_proj", + "model.layers.70.mlp.experts.125.gate_proj", + "model.layers.70.mlp.experts.125.up_proj", + "model.layers.70.mlp.experts.126.down_proj", + "model.layers.70.mlp.experts.126.gate_proj", + "model.layers.70.mlp.experts.126.up_proj", + "model.layers.70.mlp.experts.127.down_proj", + "model.layers.70.mlp.experts.127.gate_proj", + "model.layers.70.mlp.experts.127.up_proj", + "model.layers.70.mlp.experts.128.down_proj", + "model.layers.70.mlp.experts.128.gate_proj", + "model.layers.70.mlp.experts.128.up_proj", + "model.layers.70.mlp.experts.129.down_proj", + "model.layers.70.mlp.experts.129.gate_proj", + "model.layers.70.mlp.experts.129.up_proj", + "model.layers.70.mlp.experts.13.down_proj", + "model.layers.70.mlp.experts.13.gate_proj", + "model.layers.70.mlp.experts.13.up_proj", + "model.layers.70.mlp.experts.130.down_proj", + "model.layers.70.mlp.experts.130.gate_proj", + "model.layers.70.mlp.experts.130.up_proj", + "model.layers.70.mlp.experts.131.down_proj", + "model.layers.70.mlp.experts.131.gate_proj", + "model.layers.70.mlp.experts.131.up_proj", + "model.layers.70.mlp.experts.132.down_proj", + "model.layers.70.mlp.experts.132.gate_proj", + "model.layers.70.mlp.experts.132.up_proj", + "model.layers.70.mlp.experts.133.down_proj", + "model.layers.70.mlp.experts.133.gate_proj", + "model.layers.70.mlp.experts.133.up_proj", + "model.layers.70.mlp.experts.134.down_proj", + "model.layers.70.mlp.experts.134.gate_proj", + "model.layers.70.mlp.experts.134.up_proj", + "model.layers.70.mlp.experts.135.down_proj", + "model.layers.70.mlp.experts.135.gate_proj", + "model.layers.70.mlp.experts.135.up_proj", + "model.layers.70.mlp.experts.136.down_proj", + "model.layers.70.mlp.experts.136.gate_proj", + "model.layers.70.mlp.experts.136.up_proj", + "model.layers.70.mlp.experts.137.down_proj", + "model.layers.70.mlp.experts.137.gate_proj", + "model.layers.70.mlp.experts.137.up_proj", + "model.layers.70.mlp.experts.138.down_proj", + "model.layers.70.mlp.experts.138.gate_proj", + "model.layers.70.mlp.experts.138.up_proj", + "model.layers.70.mlp.experts.139.down_proj", + "model.layers.70.mlp.experts.139.gate_proj", + "model.layers.70.mlp.experts.139.up_proj", + "model.layers.70.mlp.experts.14.down_proj", + "model.layers.70.mlp.experts.14.gate_proj", + "model.layers.70.mlp.experts.14.up_proj", + "model.layers.70.mlp.experts.140.down_proj", + "model.layers.70.mlp.experts.140.gate_proj", + "model.layers.70.mlp.experts.140.up_proj", + "model.layers.70.mlp.experts.141.down_proj", + "model.layers.70.mlp.experts.141.gate_proj", + "model.layers.70.mlp.experts.141.up_proj", + "model.layers.70.mlp.experts.142.down_proj", + "model.layers.70.mlp.experts.142.gate_proj", + "model.layers.70.mlp.experts.142.up_proj", + "model.layers.70.mlp.experts.143.down_proj", + "model.layers.70.mlp.experts.143.gate_proj", + "model.layers.70.mlp.experts.143.up_proj", + "model.layers.70.mlp.experts.144.down_proj", + "model.layers.70.mlp.experts.144.gate_proj", + "model.layers.70.mlp.experts.144.up_proj", + "model.layers.70.mlp.experts.145.down_proj", + "model.layers.70.mlp.experts.145.gate_proj", + "model.layers.70.mlp.experts.145.up_proj", + "model.layers.70.mlp.experts.146.down_proj", + "model.layers.70.mlp.experts.146.gate_proj", + "model.layers.70.mlp.experts.146.up_proj", + "model.layers.70.mlp.experts.147.down_proj", + "model.layers.70.mlp.experts.147.gate_proj", + "model.layers.70.mlp.experts.147.up_proj", + "model.layers.70.mlp.experts.148.down_proj", + "model.layers.70.mlp.experts.148.gate_proj", + "model.layers.70.mlp.experts.148.up_proj", + "model.layers.70.mlp.experts.149.down_proj", + "model.layers.70.mlp.experts.149.gate_proj", + "model.layers.70.mlp.experts.149.up_proj", + "model.layers.70.mlp.experts.15.down_proj", + "model.layers.70.mlp.experts.15.gate_proj", + "model.layers.70.mlp.experts.15.up_proj", + "model.layers.70.mlp.experts.150.down_proj", + "model.layers.70.mlp.experts.150.gate_proj", + "model.layers.70.mlp.experts.150.up_proj", + "model.layers.70.mlp.experts.151.down_proj", + "model.layers.70.mlp.experts.151.gate_proj", + "model.layers.70.mlp.experts.151.up_proj", + "model.layers.70.mlp.experts.152.down_proj", + "model.layers.70.mlp.experts.152.gate_proj", + "model.layers.70.mlp.experts.152.up_proj", + "model.layers.70.mlp.experts.153.down_proj", + "model.layers.70.mlp.experts.153.gate_proj", + "model.layers.70.mlp.experts.153.up_proj", + "model.layers.70.mlp.experts.154.down_proj", + "model.layers.70.mlp.experts.154.gate_proj", + "model.layers.70.mlp.experts.154.up_proj", + "model.layers.70.mlp.experts.155.down_proj", + "model.layers.70.mlp.experts.155.gate_proj", + "model.layers.70.mlp.experts.155.up_proj", + "model.layers.70.mlp.experts.156.down_proj", + "model.layers.70.mlp.experts.156.gate_proj", + "model.layers.70.mlp.experts.156.up_proj", + "model.layers.70.mlp.experts.157.down_proj", + "model.layers.70.mlp.experts.157.gate_proj", + "model.layers.70.mlp.experts.157.up_proj", + "model.layers.70.mlp.experts.158.down_proj", + "model.layers.70.mlp.experts.158.gate_proj", + "model.layers.70.mlp.experts.158.up_proj", + "model.layers.70.mlp.experts.159.down_proj", + "model.layers.70.mlp.experts.159.gate_proj", + "model.layers.70.mlp.experts.159.up_proj", + "model.layers.70.mlp.experts.16.down_proj", + "model.layers.70.mlp.experts.16.gate_proj", + "model.layers.70.mlp.experts.16.up_proj", + "model.layers.70.mlp.experts.160.down_proj", + "model.layers.70.mlp.experts.160.gate_proj", + "model.layers.70.mlp.experts.160.up_proj", + "model.layers.70.mlp.experts.161.down_proj", + "model.layers.70.mlp.experts.161.gate_proj", + "model.layers.70.mlp.experts.161.up_proj", + "model.layers.70.mlp.experts.162.down_proj", + "model.layers.70.mlp.experts.162.gate_proj", + "model.layers.70.mlp.experts.162.up_proj", + "model.layers.70.mlp.experts.163.down_proj", + "model.layers.70.mlp.experts.163.gate_proj", + "model.layers.70.mlp.experts.163.up_proj", + "model.layers.70.mlp.experts.164.down_proj", + "model.layers.70.mlp.experts.164.gate_proj", + "model.layers.70.mlp.experts.164.up_proj", + "model.layers.70.mlp.experts.165.down_proj", + "model.layers.70.mlp.experts.165.gate_proj", + "model.layers.70.mlp.experts.165.up_proj", + "model.layers.70.mlp.experts.166.down_proj", + "model.layers.70.mlp.experts.166.gate_proj", + "model.layers.70.mlp.experts.166.up_proj", + "model.layers.70.mlp.experts.167.down_proj", + "model.layers.70.mlp.experts.167.gate_proj", + "model.layers.70.mlp.experts.167.up_proj", + "model.layers.70.mlp.experts.168.down_proj", + "model.layers.70.mlp.experts.168.gate_proj", + "model.layers.70.mlp.experts.168.up_proj", + "model.layers.70.mlp.experts.169.down_proj", + "model.layers.70.mlp.experts.169.gate_proj", + "model.layers.70.mlp.experts.169.up_proj", + "model.layers.70.mlp.experts.17.down_proj", + "model.layers.70.mlp.experts.17.gate_proj", + "model.layers.70.mlp.experts.17.up_proj", + "model.layers.70.mlp.experts.170.down_proj", + "model.layers.70.mlp.experts.170.gate_proj", + "model.layers.70.mlp.experts.170.up_proj", + "model.layers.70.mlp.experts.171.down_proj", + "model.layers.70.mlp.experts.171.gate_proj", + "model.layers.70.mlp.experts.171.up_proj", + "model.layers.70.mlp.experts.172.down_proj", + "model.layers.70.mlp.experts.172.gate_proj", + "model.layers.70.mlp.experts.172.up_proj", + "model.layers.70.mlp.experts.173.down_proj", + "model.layers.70.mlp.experts.173.gate_proj", + "model.layers.70.mlp.experts.173.up_proj", + "model.layers.70.mlp.experts.174.down_proj", + "model.layers.70.mlp.experts.174.gate_proj", + "model.layers.70.mlp.experts.174.up_proj", + "model.layers.70.mlp.experts.175.down_proj", + "model.layers.70.mlp.experts.175.gate_proj", + "model.layers.70.mlp.experts.175.up_proj", + "model.layers.70.mlp.experts.176.down_proj", + "model.layers.70.mlp.experts.176.gate_proj", + "model.layers.70.mlp.experts.176.up_proj", + "model.layers.70.mlp.experts.177.down_proj", + "model.layers.70.mlp.experts.177.gate_proj", + "model.layers.70.mlp.experts.177.up_proj", + "model.layers.70.mlp.experts.178.down_proj", + "model.layers.70.mlp.experts.178.gate_proj", + "model.layers.70.mlp.experts.178.up_proj", + "model.layers.70.mlp.experts.179.down_proj", + "model.layers.70.mlp.experts.179.gate_proj", + "model.layers.70.mlp.experts.179.up_proj", + "model.layers.70.mlp.experts.18.down_proj", + "model.layers.70.mlp.experts.18.gate_proj", + "model.layers.70.mlp.experts.18.up_proj", + "model.layers.70.mlp.experts.180.down_proj", + "model.layers.70.mlp.experts.180.gate_proj", + "model.layers.70.mlp.experts.180.up_proj", + "model.layers.70.mlp.experts.181.down_proj", + "model.layers.70.mlp.experts.181.gate_proj", + "model.layers.70.mlp.experts.181.up_proj", + "model.layers.70.mlp.experts.182.down_proj", + "model.layers.70.mlp.experts.182.gate_proj", + "model.layers.70.mlp.experts.182.up_proj", + "model.layers.70.mlp.experts.183.down_proj", + "model.layers.70.mlp.experts.183.gate_proj", + "model.layers.70.mlp.experts.183.up_proj", + "model.layers.70.mlp.experts.184.down_proj", + "model.layers.70.mlp.experts.184.gate_proj", + "model.layers.70.mlp.experts.184.up_proj", + "model.layers.70.mlp.experts.185.down_proj", + "model.layers.70.mlp.experts.185.gate_proj", + "model.layers.70.mlp.experts.185.up_proj", + "model.layers.70.mlp.experts.186.down_proj", + "model.layers.70.mlp.experts.186.gate_proj", + "model.layers.70.mlp.experts.186.up_proj", + "model.layers.70.mlp.experts.187.down_proj", + "model.layers.70.mlp.experts.187.gate_proj", + "model.layers.70.mlp.experts.187.up_proj", + "model.layers.70.mlp.experts.188.down_proj", + "model.layers.70.mlp.experts.188.gate_proj", + "model.layers.70.mlp.experts.188.up_proj", + "model.layers.70.mlp.experts.189.down_proj", + "model.layers.70.mlp.experts.189.gate_proj", + "model.layers.70.mlp.experts.189.up_proj", + "model.layers.70.mlp.experts.19.down_proj", + "model.layers.70.mlp.experts.19.gate_proj", + "model.layers.70.mlp.experts.19.up_proj", + "model.layers.70.mlp.experts.190.down_proj", + "model.layers.70.mlp.experts.190.gate_proj", + "model.layers.70.mlp.experts.190.up_proj", + "model.layers.70.mlp.experts.191.down_proj", + "model.layers.70.mlp.experts.191.gate_proj", + "model.layers.70.mlp.experts.191.up_proj", + "model.layers.70.mlp.experts.192.down_proj", + "model.layers.70.mlp.experts.192.gate_proj", + "model.layers.70.mlp.experts.192.up_proj", + "model.layers.70.mlp.experts.193.down_proj", + "model.layers.70.mlp.experts.193.gate_proj", + "model.layers.70.mlp.experts.193.up_proj", + "model.layers.70.mlp.experts.194.down_proj", + "model.layers.70.mlp.experts.194.gate_proj", + "model.layers.70.mlp.experts.194.up_proj", + "model.layers.70.mlp.experts.195.down_proj", + "model.layers.70.mlp.experts.195.gate_proj", + "model.layers.70.mlp.experts.195.up_proj", + "model.layers.70.mlp.experts.196.down_proj", + "model.layers.70.mlp.experts.196.gate_proj", + "model.layers.70.mlp.experts.196.up_proj", + "model.layers.70.mlp.experts.197.down_proj", + "model.layers.70.mlp.experts.197.gate_proj", + "model.layers.70.mlp.experts.197.up_proj", + "model.layers.70.mlp.experts.198.down_proj", + "model.layers.70.mlp.experts.198.gate_proj", + "model.layers.70.mlp.experts.198.up_proj", + "model.layers.70.mlp.experts.199.down_proj", + "model.layers.70.mlp.experts.199.gate_proj", + "model.layers.70.mlp.experts.199.up_proj", + "model.layers.70.mlp.experts.2.down_proj", + "model.layers.70.mlp.experts.2.gate_proj", + "model.layers.70.mlp.experts.2.up_proj", + "model.layers.70.mlp.experts.20.down_proj", + "model.layers.70.mlp.experts.20.gate_proj", + "model.layers.70.mlp.experts.20.up_proj", + "model.layers.70.mlp.experts.200.down_proj", + "model.layers.70.mlp.experts.200.gate_proj", + "model.layers.70.mlp.experts.200.up_proj", + "model.layers.70.mlp.experts.201.down_proj", + "model.layers.70.mlp.experts.201.gate_proj", + "model.layers.70.mlp.experts.201.up_proj", + "model.layers.70.mlp.experts.202.down_proj", + "model.layers.70.mlp.experts.202.gate_proj", + "model.layers.70.mlp.experts.202.up_proj", + "model.layers.70.mlp.experts.203.down_proj", + "model.layers.70.mlp.experts.203.gate_proj", + "model.layers.70.mlp.experts.203.up_proj", + "model.layers.70.mlp.experts.204.down_proj", + "model.layers.70.mlp.experts.204.gate_proj", + "model.layers.70.mlp.experts.204.up_proj", + "model.layers.70.mlp.experts.205.down_proj", + "model.layers.70.mlp.experts.205.gate_proj", + "model.layers.70.mlp.experts.205.up_proj", + "model.layers.70.mlp.experts.206.down_proj", + "model.layers.70.mlp.experts.206.gate_proj", + "model.layers.70.mlp.experts.206.up_proj", + "model.layers.70.mlp.experts.207.down_proj", + "model.layers.70.mlp.experts.207.gate_proj", + "model.layers.70.mlp.experts.207.up_proj", + "model.layers.70.mlp.experts.208.down_proj", + "model.layers.70.mlp.experts.208.gate_proj", + "model.layers.70.mlp.experts.208.up_proj", + "model.layers.70.mlp.experts.209.down_proj", + "model.layers.70.mlp.experts.209.gate_proj", + "model.layers.70.mlp.experts.209.up_proj", + "model.layers.70.mlp.experts.21.down_proj", + "model.layers.70.mlp.experts.21.gate_proj", + "model.layers.70.mlp.experts.21.up_proj", + "model.layers.70.mlp.experts.210.down_proj", + "model.layers.70.mlp.experts.210.gate_proj", + "model.layers.70.mlp.experts.210.up_proj", + "model.layers.70.mlp.experts.211.down_proj", + "model.layers.70.mlp.experts.211.gate_proj", + "model.layers.70.mlp.experts.211.up_proj", + "model.layers.70.mlp.experts.212.down_proj", + "model.layers.70.mlp.experts.212.gate_proj", + "model.layers.70.mlp.experts.212.up_proj", + "model.layers.70.mlp.experts.213.down_proj", + "model.layers.70.mlp.experts.213.gate_proj", + "model.layers.70.mlp.experts.213.up_proj", + "model.layers.70.mlp.experts.214.down_proj", + "model.layers.70.mlp.experts.214.gate_proj", + "model.layers.70.mlp.experts.214.up_proj", + "model.layers.70.mlp.experts.215.down_proj", + "model.layers.70.mlp.experts.215.gate_proj", + "model.layers.70.mlp.experts.215.up_proj", + "model.layers.70.mlp.experts.216.down_proj", + "model.layers.70.mlp.experts.216.gate_proj", + "model.layers.70.mlp.experts.216.up_proj", + "model.layers.70.mlp.experts.217.down_proj", + "model.layers.70.mlp.experts.217.gate_proj", + "model.layers.70.mlp.experts.217.up_proj", + "model.layers.70.mlp.experts.218.down_proj", + "model.layers.70.mlp.experts.218.gate_proj", + "model.layers.70.mlp.experts.218.up_proj", + "model.layers.70.mlp.experts.219.down_proj", + "model.layers.70.mlp.experts.219.gate_proj", + "model.layers.70.mlp.experts.219.up_proj", + "model.layers.70.mlp.experts.22.down_proj", + "model.layers.70.mlp.experts.22.gate_proj", + "model.layers.70.mlp.experts.22.up_proj", + "model.layers.70.mlp.experts.220.down_proj", + "model.layers.70.mlp.experts.220.gate_proj", + "model.layers.70.mlp.experts.220.up_proj", + "model.layers.70.mlp.experts.221.down_proj", + "model.layers.70.mlp.experts.221.gate_proj", + "model.layers.70.mlp.experts.221.up_proj", + "model.layers.70.mlp.experts.222.down_proj", + "model.layers.70.mlp.experts.222.gate_proj", + "model.layers.70.mlp.experts.222.up_proj", + "model.layers.70.mlp.experts.223.down_proj", + "model.layers.70.mlp.experts.223.gate_proj", + "model.layers.70.mlp.experts.223.up_proj", + "model.layers.70.mlp.experts.224.down_proj", + "model.layers.70.mlp.experts.224.gate_proj", + "model.layers.70.mlp.experts.224.up_proj", + "model.layers.70.mlp.experts.225.down_proj", + "model.layers.70.mlp.experts.225.gate_proj", + "model.layers.70.mlp.experts.225.up_proj", + "model.layers.70.mlp.experts.226.down_proj", + "model.layers.70.mlp.experts.226.gate_proj", + "model.layers.70.mlp.experts.226.up_proj", + "model.layers.70.mlp.experts.227.down_proj", + "model.layers.70.mlp.experts.227.gate_proj", + "model.layers.70.mlp.experts.227.up_proj", + "model.layers.70.mlp.experts.228.down_proj", + "model.layers.70.mlp.experts.228.gate_proj", + "model.layers.70.mlp.experts.228.up_proj", + "model.layers.70.mlp.experts.229.down_proj", + "model.layers.70.mlp.experts.229.gate_proj", + "model.layers.70.mlp.experts.229.up_proj", + "model.layers.70.mlp.experts.23.down_proj", + "model.layers.70.mlp.experts.23.gate_proj", + "model.layers.70.mlp.experts.23.up_proj", + "model.layers.70.mlp.experts.230.down_proj", + "model.layers.70.mlp.experts.230.gate_proj", + "model.layers.70.mlp.experts.230.up_proj", + "model.layers.70.mlp.experts.231.down_proj", + "model.layers.70.mlp.experts.231.gate_proj", + "model.layers.70.mlp.experts.231.up_proj", + "model.layers.70.mlp.experts.232.down_proj", + "model.layers.70.mlp.experts.232.gate_proj", + "model.layers.70.mlp.experts.232.up_proj", + "model.layers.70.mlp.experts.233.down_proj", + "model.layers.70.mlp.experts.233.gate_proj", + "model.layers.70.mlp.experts.233.up_proj", + "model.layers.70.mlp.experts.234.down_proj", + "model.layers.70.mlp.experts.234.gate_proj", + "model.layers.70.mlp.experts.234.up_proj", + "model.layers.70.mlp.experts.235.down_proj", + "model.layers.70.mlp.experts.235.gate_proj", + "model.layers.70.mlp.experts.235.up_proj", + "model.layers.70.mlp.experts.236.down_proj", + "model.layers.70.mlp.experts.236.gate_proj", + "model.layers.70.mlp.experts.236.up_proj", + "model.layers.70.mlp.experts.237.down_proj", + "model.layers.70.mlp.experts.237.gate_proj", + "model.layers.70.mlp.experts.237.up_proj", + "model.layers.70.mlp.experts.238.down_proj", + "model.layers.70.mlp.experts.238.gate_proj", + "model.layers.70.mlp.experts.238.up_proj", + "model.layers.70.mlp.experts.239.down_proj", + "model.layers.70.mlp.experts.239.gate_proj", + "model.layers.70.mlp.experts.239.up_proj", + "model.layers.70.mlp.experts.24.down_proj", + "model.layers.70.mlp.experts.24.gate_proj", + "model.layers.70.mlp.experts.24.up_proj", + "model.layers.70.mlp.experts.240.down_proj", + "model.layers.70.mlp.experts.240.gate_proj", + "model.layers.70.mlp.experts.240.up_proj", + "model.layers.70.mlp.experts.241.down_proj", + "model.layers.70.mlp.experts.241.gate_proj", + "model.layers.70.mlp.experts.241.up_proj", + "model.layers.70.mlp.experts.242.down_proj", + "model.layers.70.mlp.experts.242.gate_proj", + "model.layers.70.mlp.experts.242.up_proj", + "model.layers.70.mlp.experts.243.down_proj", + "model.layers.70.mlp.experts.243.gate_proj", + "model.layers.70.mlp.experts.243.up_proj", + "model.layers.70.mlp.experts.244.down_proj", + "model.layers.70.mlp.experts.244.gate_proj", + "model.layers.70.mlp.experts.244.up_proj", + "model.layers.70.mlp.experts.245.down_proj", + "model.layers.70.mlp.experts.245.gate_proj", + "model.layers.70.mlp.experts.245.up_proj", + "model.layers.70.mlp.experts.246.down_proj", + "model.layers.70.mlp.experts.246.gate_proj", + "model.layers.70.mlp.experts.246.up_proj", + "model.layers.70.mlp.experts.247.down_proj", + "model.layers.70.mlp.experts.247.gate_proj", + "model.layers.70.mlp.experts.247.up_proj", + "model.layers.70.mlp.experts.248.down_proj", + "model.layers.70.mlp.experts.248.gate_proj", + "model.layers.70.mlp.experts.248.up_proj", + "model.layers.70.mlp.experts.249.down_proj", + "model.layers.70.mlp.experts.249.gate_proj", + "model.layers.70.mlp.experts.249.up_proj", + "model.layers.70.mlp.experts.25.down_proj", + "model.layers.70.mlp.experts.25.gate_proj", + "model.layers.70.mlp.experts.25.up_proj", + "model.layers.70.mlp.experts.250.down_proj", + "model.layers.70.mlp.experts.250.gate_proj", + "model.layers.70.mlp.experts.250.up_proj", + "model.layers.70.mlp.experts.251.down_proj", + "model.layers.70.mlp.experts.251.gate_proj", + "model.layers.70.mlp.experts.251.up_proj", + "model.layers.70.mlp.experts.252.down_proj", + "model.layers.70.mlp.experts.252.gate_proj", + "model.layers.70.mlp.experts.252.up_proj", + "model.layers.70.mlp.experts.253.down_proj", + "model.layers.70.mlp.experts.253.gate_proj", + "model.layers.70.mlp.experts.253.up_proj", + "model.layers.70.mlp.experts.254.down_proj", + "model.layers.70.mlp.experts.254.gate_proj", + "model.layers.70.mlp.experts.254.up_proj", + "model.layers.70.mlp.experts.255.down_proj", + "model.layers.70.mlp.experts.255.gate_proj", + "model.layers.70.mlp.experts.255.up_proj", + "model.layers.70.mlp.experts.26.down_proj", + "model.layers.70.mlp.experts.26.gate_proj", + "model.layers.70.mlp.experts.26.up_proj", + "model.layers.70.mlp.experts.27.down_proj", + "model.layers.70.mlp.experts.27.gate_proj", + "model.layers.70.mlp.experts.27.up_proj", + "model.layers.70.mlp.experts.28.down_proj", + "model.layers.70.mlp.experts.28.gate_proj", + "model.layers.70.mlp.experts.28.up_proj", + "model.layers.70.mlp.experts.29.down_proj", + "model.layers.70.mlp.experts.29.gate_proj", + "model.layers.70.mlp.experts.29.up_proj", + "model.layers.70.mlp.experts.3.down_proj", + "model.layers.70.mlp.experts.3.gate_proj", + "model.layers.70.mlp.experts.3.up_proj", + "model.layers.70.mlp.experts.30.down_proj", + "model.layers.70.mlp.experts.30.gate_proj", + "model.layers.70.mlp.experts.30.up_proj", + "model.layers.70.mlp.experts.31.down_proj", + "model.layers.70.mlp.experts.31.gate_proj", + "model.layers.70.mlp.experts.31.up_proj", + "model.layers.70.mlp.experts.32.down_proj", + "model.layers.70.mlp.experts.32.gate_proj", + "model.layers.70.mlp.experts.32.up_proj", + "model.layers.70.mlp.experts.33.down_proj", + "model.layers.70.mlp.experts.33.gate_proj", + "model.layers.70.mlp.experts.33.up_proj", + "model.layers.70.mlp.experts.34.down_proj", + "model.layers.70.mlp.experts.34.gate_proj", + "model.layers.70.mlp.experts.34.up_proj", + "model.layers.70.mlp.experts.35.down_proj", + "model.layers.70.mlp.experts.35.gate_proj", + "model.layers.70.mlp.experts.35.up_proj", + "model.layers.70.mlp.experts.36.down_proj", + "model.layers.70.mlp.experts.36.gate_proj", + "model.layers.70.mlp.experts.36.up_proj", + "model.layers.70.mlp.experts.37.down_proj", + "model.layers.70.mlp.experts.37.gate_proj", + "model.layers.70.mlp.experts.37.up_proj", + "model.layers.70.mlp.experts.38.down_proj", + "model.layers.70.mlp.experts.38.gate_proj", + "model.layers.70.mlp.experts.38.up_proj", + "model.layers.70.mlp.experts.39.down_proj", + "model.layers.70.mlp.experts.39.gate_proj", + "model.layers.70.mlp.experts.39.up_proj", + "model.layers.70.mlp.experts.4.down_proj", + "model.layers.70.mlp.experts.4.gate_proj", + "model.layers.70.mlp.experts.4.up_proj", + "model.layers.70.mlp.experts.40.down_proj", + "model.layers.70.mlp.experts.40.gate_proj", + "model.layers.70.mlp.experts.40.up_proj", + "model.layers.70.mlp.experts.41.down_proj", + "model.layers.70.mlp.experts.41.gate_proj", + "model.layers.70.mlp.experts.41.up_proj", + "model.layers.70.mlp.experts.42.down_proj", + "model.layers.70.mlp.experts.42.gate_proj", + "model.layers.70.mlp.experts.42.up_proj", + "model.layers.70.mlp.experts.43.down_proj", + "model.layers.70.mlp.experts.43.gate_proj", + "model.layers.70.mlp.experts.43.up_proj", + "model.layers.70.mlp.experts.44.down_proj", + "model.layers.70.mlp.experts.44.gate_proj", + "model.layers.70.mlp.experts.44.up_proj", + "model.layers.70.mlp.experts.45.down_proj", + "model.layers.70.mlp.experts.45.gate_proj", + "model.layers.70.mlp.experts.45.up_proj", + "model.layers.70.mlp.experts.46.down_proj", + "model.layers.70.mlp.experts.46.gate_proj", + "model.layers.70.mlp.experts.46.up_proj", + "model.layers.70.mlp.experts.47.down_proj", + "model.layers.70.mlp.experts.47.gate_proj", + "model.layers.70.mlp.experts.47.up_proj", + "model.layers.70.mlp.experts.48.down_proj", + "model.layers.70.mlp.experts.48.gate_proj", + "model.layers.70.mlp.experts.48.up_proj", + "model.layers.70.mlp.experts.49.down_proj", + "model.layers.70.mlp.experts.49.gate_proj", + "model.layers.70.mlp.experts.49.up_proj", + "model.layers.70.mlp.experts.5.down_proj", + "model.layers.70.mlp.experts.5.gate_proj", + "model.layers.70.mlp.experts.5.up_proj", + "model.layers.70.mlp.experts.50.down_proj", + "model.layers.70.mlp.experts.50.gate_proj", + "model.layers.70.mlp.experts.50.up_proj", + "model.layers.70.mlp.experts.51.down_proj", + "model.layers.70.mlp.experts.51.gate_proj", + "model.layers.70.mlp.experts.51.up_proj", + "model.layers.70.mlp.experts.52.down_proj", + "model.layers.70.mlp.experts.52.gate_proj", + "model.layers.70.mlp.experts.52.up_proj", + "model.layers.70.mlp.experts.53.down_proj", + "model.layers.70.mlp.experts.53.gate_proj", + "model.layers.70.mlp.experts.53.up_proj", + "model.layers.70.mlp.experts.54.down_proj", + "model.layers.70.mlp.experts.54.gate_proj", + "model.layers.70.mlp.experts.54.up_proj", + "model.layers.70.mlp.experts.55.down_proj", + "model.layers.70.mlp.experts.55.gate_proj", + "model.layers.70.mlp.experts.55.up_proj", + "model.layers.70.mlp.experts.56.down_proj", + "model.layers.70.mlp.experts.56.gate_proj", + "model.layers.70.mlp.experts.56.up_proj", + "model.layers.70.mlp.experts.57.down_proj", + "model.layers.70.mlp.experts.57.gate_proj", + "model.layers.70.mlp.experts.57.up_proj", + "model.layers.70.mlp.experts.58.down_proj", + "model.layers.70.mlp.experts.58.gate_proj", + "model.layers.70.mlp.experts.58.up_proj", + "model.layers.70.mlp.experts.59.down_proj", + "model.layers.70.mlp.experts.59.gate_proj", + "model.layers.70.mlp.experts.59.up_proj", + "model.layers.70.mlp.experts.6.down_proj", + "model.layers.70.mlp.experts.6.gate_proj", + "model.layers.70.mlp.experts.6.up_proj", + "model.layers.70.mlp.experts.60.down_proj", + "model.layers.70.mlp.experts.60.gate_proj", + "model.layers.70.mlp.experts.60.up_proj", + "model.layers.70.mlp.experts.61.down_proj", + "model.layers.70.mlp.experts.61.gate_proj", + "model.layers.70.mlp.experts.61.up_proj", + "model.layers.70.mlp.experts.62.down_proj", + "model.layers.70.mlp.experts.62.gate_proj", + "model.layers.70.mlp.experts.62.up_proj", + "model.layers.70.mlp.experts.63.down_proj", + "model.layers.70.mlp.experts.63.gate_proj", + "model.layers.70.mlp.experts.63.up_proj", + "model.layers.70.mlp.experts.64.down_proj", + "model.layers.70.mlp.experts.64.gate_proj", + "model.layers.70.mlp.experts.64.up_proj", + "model.layers.70.mlp.experts.65.down_proj", + "model.layers.70.mlp.experts.65.gate_proj", + "model.layers.70.mlp.experts.65.up_proj", + "model.layers.70.mlp.experts.66.down_proj", + "model.layers.70.mlp.experts.66.gate_proj", + "model.layers.70.mlp.experts.66.up_proj", + "model.layers.70.mlp.experts.67.down_proj", + "model.layers.70.mlp.experts.67.gate_proj", + "model.layers.70.mlp.experts.67.up_proj", + "model.layers.70.mlp.experts.68.down_proj", + "model.layers.70.mlp.experts.68.gate_proj", + "model.layers.70.mlp.experts.68.up_proj", + "model.layers.70.mlp.experts.69.down_proj", + "model.layers.70.mlp.experts.69.gate_proj", + "model.layers.70.mlp.experts.69.up_proj", + "model.layers.70.mlp.experts.7.down_proj", + "model.layers.70.mlp.experts.7.gate_proj", + "model.layers.70.mlp.experts.7.up_proj", + "model.layers.70.mlp.experts.70.down_proj", + "model.layers.70.mlp.experts.70.gate_proj", + "model.layers.70.mlp.experts.70.up_proj", + "model.layers.70.mlp.experts.71.down_proj", + "model.layers.70.mlp.experts.71.gate_proj", + "model.layers.70.mlp.experts.71.up_proj", + "model.layers.70.mlp.experts.72.down_proj", + "model.layers.70.mlp.experts.72.gate_proj", + "model.layers.70.mlp.experts.72.up_proj", + "model.layers.70.mlp.experts.73.down_proj", + "model.layers.70.mlp.experts.73.gate_proj", + "model.layers.70.mlp.experts.73.up_proj", + "model.layers.70.mlp.experts.74.down_proj", + "model.layers.70.mlp.experts.74.gate_proj", + "model.layers.70.mlp.experts.74.up_proj", + "model.layers.70.mlp.experts.75.down_proj", + "model.layers.70.mlp.experts.75.gate_proj", + "model.layers.70.mlp.experts.75.up_proj", + "model.layers.70.mlp.experts.76.down_proj", + "model.layers.70.mlp.experts.76.gate_proj", + "model.layers.70.mlp.experts.76.up_proj", + "model.layers.70.mlp.experts.77.down_proj", + "model.layers.70.mlp.experts.77.gate_proj", + "model.layers.70.mlp.experts.77.up_proj", + "model.layers.70.mlp.experts.78.down_proj", + "model.layers.70.mlp.experts.78.gate_proj", + "model.layers.70.mlp.experts.78.up_proj", + "model.layers.70.mlp.experts.79.down_proj", + "model.layers.70.mlp.experts.79.gate_proj", + "model.layers.70.mlp.experts.79.up_proj", + "model.layers.70.mlp.experts.8.down_proj", + "model.layers.70.mlp.experts.8.gate_proj", + "model.layers.70.mlp.experts.8.up_proj", + "model.layers.70.mlp.experts.80.down_proj", + "model.layers.70.mlp.experts.80.gate_proj", + "model.layers.70.mlp.experts.80.up_proj", + "model.layers.70.mlp.experts.81.down_proj", + "model.layers.70.mlp.experts.81.gate_proj", + "model.layers.70.mlp.experts.81.up_proj", + "model.layers.70.mlp.experts.82.down_proj", + "model.layers.70.mlp.experts.82.gate_proj", + "model.layers.70.mlp.experts.82.up_proj", + "model.layers.70.mlp.experts.83.down_proj", + "model.layers.70.mlp.experts.83.gate_proj", + "model.layers.70.mlp.experts.83.up_proj", + "model.layers.70.mlp.experts.84.down_proj", + "model.layers.70.mlp.experts.84.gate_proj", + "model.layers.70.mlp.experts.84.up_proj", + "model.layers.70.mlp.experts.85.down_proj", + "model.layers.70.mlp.experts.85.gate_proj", + "model.layers.70.mlp.experts.85.up_proj", + "model.layers.70.mlp.experts.86.down_proj", + "model.layers.70.mlp.experts.86.gate_proj", + "model.layers.70.mlp.experts.86.up_proj", + "model.layers.70.mlp.experts.87.down_proj", + "model.layers.70.mlp.experts.87.gate_proj", + "model.layers.70.mlp.experts.87.up_proj", + "model.layers.70.mlp.experts.88.down_proj", + "model.layers.70.mlp.experts.88.gate_proj", + "model.layers.70.mlp.experts.88.up_proj", + "model.layers.70.mlp.experts.89.down_proj", + "model.layers.70.mlp.experts.89.gate_proj", + "model.layers.70.mlp.experts.89.up_proj", + "model.layers.70.mlp.experts.9.down_proj", + "model.layers.70.mlp.experts.9.gate_proj", + "model.layers.70.mlp.experts.9.up_proj", + "model.layers.70.mlp.experts.90.down_proj", + "model.layers.70.mlp.experts.90.gate_proj", + "model.layers.70.mlp.experts.90.up_proj", + "model.layers.70.mlp.experts.91.down_proj", + "model.layers.70.mlp.experts.91.gate_proj", + "model.layers.70.mlp.experts.91.up_proj", + "model.layers.70.mlp.experts.92.down_proj", + "model.layers.70.mlp.experts.92.gate_proj", + "model.layers.70.mlp.experts.92.up_proj", + "model.layers.70.mlp.experts.93.down_proj", + "model.layers.70.mlp.experts.93.gate_proj", + "model.layers.70.mlp.experts.93.up_proj", + "model.layers.70.mlp.experts.94.down_proj", + "model.layers.70.mlp.experts.94.gate_proj", + "model.layers.70.mlp.experts.94.up_proj", + "model.layers.70.mlp.experts.95.down_proj", + "model.layers.70.mlp.experts.95.gate_proj", + "model.layers.70.mlp.experts.95.up_proj", + "model.layers.70.mlp.experts.96.down_proj", + "model.layers.70.mlp.experts.96.gate_proj", + "model.layers.70.mlp.experts.96.up_proj", + "model.layers.70.mlp.experts.97.down_proj", + "model.layers.70.mlp.experts.97.gate_proj", + "model.layers.70.mlp.experts.97.up_proj", + "model.layers.70.mlp.experts.98.down_proj", + "model.layers.70.mlp.experts.98.gate_proj", + "model.layers.70.mlp.experts.98.up_proj", + "model.layers.70.mlp.experts.99.down_proj", + "model.layers.70.mlp.experts.99.gate_proj", + "model.layers.70.mlp.experts.99.up_proj", + "model.layers.71.mlp.experts.0.down_proj", + "model.layers.71.mlp.experts.0.gate_proj", + "model.layers.71.mlp.experts.0.up_proj", + "model.layers.71.mlp.experts.1.down_proj", + "model.layers.71.mlp.experts.1.gate_proj", + "model.layers.71.mlp.experts.1.up_proj", + "model.layers.71.mlp.experts.10.down_proj", + "model.layers.71.mlp.experts.10.gate_proj", + "model.layers.71.mlp.experts.10.up_proj", + "model.layers.71.mlp.experts.100.down_proj", + "model.layers.71.mlp.experts.100.gate_proj", + "model.layers.71.mlp.experts.100.up_proj", + "model.layers.71.mlp.experts.101.down_proj", + "model.layers.71.mlp.experts.101.gate_proj", + "model.layers.71.mlp.experts.101.up_proj", + "model.layers.71.mlp.experts.102.down_proj", + "model.layers.71.mlp.experts.102.gate_proj", + "model.layers.71.mlp.experts.102.up_proj", + "model.layers.71.mlp.experts.103.down_proj", + "model.layers.71.mlp.experts.103.gate_proj", + "model.layers.71.mlp.experts.103.up_proj", + "model.layers.71.mlp.experts.104.down_proj", + "model.layers.71.mlp.experts.104.gate_proj", + "model.layers.71.mlp.experts.104.up_proj", + "model.layers.71.mlp.experts.105.down_proj", + "model.layers.71.mlp.experts.105.gate_proj", + "model.layers.71.mlp.experts.105.up_proj", + "model.layers.71.mlp.experts.106.down_proj", + "model.layers.71.mlp.experts.106.gate_proj", + "model.layers.71.mlp.experts.106.up_proj", + "model.layers.71.mlp.experts.107.down_proj", + "model.layers.71.mlp.experts.107.gate_proj", + "model.layers.71.mlp.experts.107.up_proj", + "model.layers.71.mlp.experts.108.down_proj", + "model.layers.71.mlp.experts.108.gate_proj", + "model.layers.71.mlp.experts.108.up_proj", + "model.layers.71.mlp.experts.109.down_proj", + "model.layers.71.mlp.experts.109.gate_proj", + "model.layers.71.mlp.experts.109.up_proj", + "model.layers.71.mlp.experts.11.down_proj", + "model.layers.71.mlp.experts.11.gate_proj", + "model.layers.71.mlp.experts.11.up_proj", + "model.layers.71.mlp.experts.110.down_proj", + "model.layers.71.mlp.experts.110.gate_proj", + "model.layers.71.mlp.experts.110.up_proj", + "model.layers.71.mlp.experts.111.down_proj", + "model.layers.71.mlp.experts.111.gate_proj", + "model.layers.71.mlp.experts.111.up_proj", + "model.layers.71.mlp.experts.112.down_proj", + "model.layers.71.mlp.experts.112.gate_proj", + "model.layers.71.mlp.experts.112.up_proj", + "model.layers.71.mlp.experts.113.down_proj", + "model.layers.71.mlp.experts.113.gate_proj", + "model.layers.71.mlp.experts.113.up_proj", + "model.layers.71.mlp.experts.114.down_proj", + "model.layers.71.mlp.experts.114.gate_proj", + "model.layers.71.mlp.experts.114.up_proj", + "model.layers.71.mlp.experts.115.down_proj", + "model.layers.71.mlp.experts.115.gate_proj", + "model.layers.71.mlp.experts.115.up_proj", + "model.layers.71.mlp.experts.116.down_proj", + "model.layers.71.mlp.experts.116.gate_proj", + "model.layers.71.mlp.experts.116.up_proj", + "model.layers.71.mlp.experts.117.down_proj", + "model.layers.71.mlp.experts.117.gate_proj", + "model.layers.71.mlp.experts.117.up_proj", + "model.layers.71.mlp.experts.118.down_proj", + "model.layers.71.mlp.experts.118.gate_proj", + "model.layers.71.mlp.experts.118.up_proj", + "model.layers.71.mlp.experts.119.down_proj", + "model.layers.71.mlp.experts.119.gate_proj", + "model.layers.71.mlp.experts.119.up_proj", + "model.layers.71.mlp.experts.12.down_proj", + "model.layers.71.mlp.experts.12.gate_proj", + "model.layers.71.mlp.experts.12.up_proj", + "model.layers.71.mlp.experts.120.down_proj", + "model.layers.71.mlp.experts.120.gate_proj", + "model.layers.71.mlp.experts.120.up_proj", + "model.layers.71.mlp.experts.121.down_proj", + "model.layers.71.mlp.experts.121.gate_proj", + "model.layers.71.mlp.experts.121.up_proj", + "model.layers.71.mlp.experts.122.down_proj", + "model.layers.71.mlp.experts.122.gate_proj", + "model.layers.71.mlp.experts.122.up_proj", + "model.layers.71.mlp.experts.123.down_proj", + "model.layers.71.mlp.experts.123.gate_proj", + "model.layers.71.mlp.experts.123.up_proj", + "model.layers.71.mlp.experts.124.down_proj", + "model.layers.71.mlp.experts.124.gate_proj", + "model.layers.71.mlp.experts.124.up_proj", + "model.layers.71.mlp.experts.125.down_proj", + "model.layers.71.mlp.experts.125.gate_proj", + "model.layers.71.mlp.experts.125.up_proj", + "model.layers.71.mlp.experts.126.down_proj", + "model.layers.71.mlp.experts.126.gate_proj", + "model.layers.71.mlp.experts.126.up_proj", + "model.layers.71.mlp.experts.127.down_proj", + "model.layers.71.mlp.experts.127.gate_proj", + "model.layers.71.mlp.experts.127.up_proj", + "model.layers.71.mlp.experts.128.down_proj", + "model.layers.71.mlp.experts.128.gate_proj", + "model.layers.71.mlp.experts.128.up_proj", + "model.layers.71.mlp.experts.129.down_proj", + "model.layers.71.mlp.experts.129.gate_proj", + "model.layers.71.mlp.experts.129.up_proj", + "model.layers.71.mlp.experts.13.down_proj", + "model.layers.71.mlp.experts.13.gate_proj", + "model.layers.71.mlp.experts.13.up_proj", + "model.layers.71.mlp.experts.130.down_proj", + "model.layers.71.mlp.experts.130.gate_proj", + "model.layers.71.mlp.experts.130.up_proj", + "model.layers.71.mlp.experts.131.down_proj", + "model.layers.71.mlp.experts.131.gate_proj", + "model.layers.71.mlp.experts.131.up_proj", + "model.layers.71.mlp.experts.132.down_proj", + "model.layers.71.mlp.experts.132.gate_proj", + "model.layers.71.mlp.experts.132.up_proj", + "model.layers.71.mlp.experts.133.down_proj", + "model.layers.71.mlp.experts.133.gate_proj", + "model.layers.71.mlp.experts.133.up_proj", + "model.layers.71.mlp.experts.134.down_proj", + "model.layers.71.mlp.experts.134.gate_proj", + "model.layers.71.mlp.experts.134.up_proj", + "model.layers.71.mlp.experts.135.down_proj", + "model.layers.71.mlp.experts.135.gate_proj", + "model.layers.71.mlp.experts.135.up_proj", + "model.layers.71.mlp.experts.136.down_proj", + "model.layers.71.mlp.experts.136.gate_proj", + "model.layers.71.mlp.experts.136.up_proj", + "model.layers.71.mlp.experts.137.down_proj", + "model.layers.71.mlp.experts.137.gate_proj", + "model.layers.71.mlp.experts.137.up_proj", + "model.layers.71.mlp.experts.138.down_proj", + "model.layers.71.mlp.experts.138.gate_proj", + "model.layers.71.mlp.experts.138.up_proj", + "model.layers.71.mlp.experts.139.down_proj", + "model.layers.71.mlp.experts.139.gate_proj", + "model.layers.71.mlp.experts.139.up_proj", + "model.layers.71.mlp.experts.14.down_proj", + "model.layers.71.mlp.experts.14.gate_proj", + "model.layers.71.mlp.experts.14.up_proj", + "model.layers.71.mlp.experts.140.down_proj", + "model.layers.71.mlp.experts.140.gate_proj", + "model.layers.71.mlp.experts.140.up_proj", + "model.layers.71.mlp.experts.141.down_proj", + "model.layers.71.mlp.experts.141.gate_proj", + "model.layers.71.mlp.experts.141.up_proj", + "model.layers.71.mlp.experts.142.down_proj", + "model.layers.71.mlp.experts.142.gate_proj", + "model.layers.71.mlp.experts.142.up_proj", + "model.layers.71.mlp.experts.143.down_proj", + "model.layers.71.mlp.experts.143.gate_proj", + "model.layers.71.mlp.experts.143.up_proj", + "model.layers.71.mlp.experts.144.down_proj", + "model.layers.71.mlp.experts.144.gate_proj", + "model.layers.71.mlp.experts.144.up_proj", + "model.layers.71.mlp.experts.145.down_proj", + "model.layers.71.mlp.experts.145.gate_proj", + "model.layers.71.mlp.experts.145.up_proj", + "model.layers.71.mlp.experts.146.down_proj", + "model.layers.71.mlp.experts.146.gate_proj", + "model.layers.71.mlp.experts.146.up_proj", + "model.layers.71.mlp.experts.147.down_proj", + "model.layers.71.mlp.experts.147.gate_proj", + "model.layers.71.mlp.experts.147.up_proj", + "model.layers.71.mlp.experts.148.down_proj", + "model.layers.71.mlp.experts.148.gate_proj", + "model.layers.71.mlp.experts.148.up_proj", + "model.layers.71.mlp.experts.149.down_proj", + "model.layers.71.mlp.experts.149.gate_proj", + "model.layers.71.mlp.experts.149.up_proj", + "model.layers.71.mlp.experts.15.down_proj", + "model.layers.71.mlp.experts.15.gate_proj", + "model.layers.71.mlp.experts.15.up_proj", + "model.layers.71.mlp.experts.150.down_proj", + "model.layers.71.mlp.experts.150.gate_proj", + "model.layers.71.mlp.experts.150.up_proj", + "model.layers.71.mlp.experts.151.down_proj", + "model.layers.71.mlp.experts.151.gate_proj", + "model.layers.71.mlp.experts.151.up_proj", + "model.layers.71.mlp.experts.152.down_proj", + "model.layers.71.mlp.experts.152.gate_proj", + "model.layers.71.mlp.experts.152.up_proj", + "model.layers.71.mlp.experts.153.down_proj", + "model.layers.71.mlp.experts.153.gate_proj", + "model.layers.71.mlp.experts.153.up_proj", + "model.layers.71.mlp.experts.154.down_proj", + "model.layers.71.mlp.experts.154.gate_proj", + "model.layers.71.mlp.experts.154.up_proj", + "model.layers.71.mlp.experts.155.down_proj", + "model.layers.71.mlp.experts.155.gate_proj", + "model.layers.71.mlp.experts.155.up_proj", + "model.layers.71.mlp.experts.156.down_proj", + "model.layers.71.mlp.experts.156.gate_proj", + "model.layers.71.mlp.experts.156.up_proj", + "model.layers.71.mlp.experts.157.down_proj", + "model.layers.71.mlp.experts.157.gate_proj", + "model.layers.71.mlp.experts.157.up_proj", + "model.layers.71.mlp.experts.158.down_proj", + "model.layers.71.mlp.experts.158.gate_proj", + "model.layers.71.mlp.experts.158.up_proj", + "model.layers.71.mlp.experts.159.down_proj", + "model.layers.71.mlp.experts.159.gate_proj", + "model.layers.71.mlp.experts.159.up_proj", + "model.layers.71.mlp.experts.16.down_proj", + "model.layers.71.mlp.experts.16.gate_proj", + "model.layers.71.mlp.experts.16.up_proj", + "model.layers.71.mlp.experts.160.down_proj", + "model.layers.71.mlp.experts.160.gate_proj", + "model.layers.71.mlp.experts.160.up_proj", + "model.layers.71.mlp.experts.161.down_proj", + "model.layers.71.mlp.experts.161.gate_proj", + "model.layers.71.mlp.experts.161.up_proj", + "model.layers.71.mlp.experts.162.down_proj", + "model.layers.71.mlp.experts.162.gate_proj", + "model.layers.71.mlp.experts.162.up_proj", + "model.layers.71.mlp.experts.163.down_proj", + "model.layers.71.mlp.experts.163.gate_proj", + "model.layers.71.mlp.experts.163.up_proj", + "model.layers.71.mlp.experts.164.down_proj", + "model.layers.71.mlp.experts.164.gate_proj", + "model.layers.71.mlp.experts.164.up_proj", + "model.layers.71.mlp.experts.165.down_proj", + "model.layers.71.mlp.experts.165.gate_proj", + "model.layers.71.mlp.experts.165.up_proj", + "model.layers.71.mlp.experts.166.down_proj", + "model.layers.71.mlp.experts.166.gate_proj", + "model.layers.71.mlp.experts.166.up_proj", + "model.layers.71.mlp.experts.167.down_proj", + "model.layers.71.mlp.experts.167.gate_proj", + "model.layers.71.mlp.experts.167.up_proj", + "model.layers.71.mlp.experts.168.down_proj", + "model.layers.71.mlp.experts.168.gate_proj", + "model.layers.71.mlp.experts.168.up_proj", + "model.layers.71.mlp.experts.169.down_proj", + "model.layers.71.mlp.experts.169.gate_proj", + "model.layers.71.mlp.experts.169.up_proj", + "model.layers.71.mlp.experts.17.down_proj", + "model.layers.71.mlp.experts.17.gate_proj", + "model.layers.71.mlp.experts.17.up_proj", + "model.layers.71.mlp.experts.170.down_proj", + "model.layers.71.mlp.experts.170.gate_proj", + "model.layers.71.mlp.experts.170.up_proj", + "model.layers.71.mlp.experts.171.down_proj", + "model.layers.71.mlp.experts.171.gate_proj", + "model.layers.71.mlp.experts.171.up_proj", + "model.layers.71.mlp.experts.172.down_proj", + "model.layers.71.mlp.experts.172.gate_proj", + "model.layers.71.mlp.experts.172.up_proj", + "model.layers.71.mlp.experts.173.down_proj", + "model.layers.71.mlp.experts.173.gate_proj", + "model.layers.71.mlp.experts.173.up_proj", + "model.layers.71.mlp.experts.174.down_proj", + "model.layers.71.mlp.experts.174.gate_proj", + "model.layers.71.mlp.experts.174.up_proj", + "model.layers.71.mlp.experts.175.down_proj", + "model.layers.71.mlp.experts.175.gate_proj", + "model.layers.71.mlp.experts.175.up_proj", + "model.layers.71.mlp.experts.176.down_proj", + "model.layers.71.mlp.experts.176.gate_proj", + "model.layers.71.mlp.experts.176.up_proj", + "model.layers.71.mlp.experts.177.down_proj", + "model.layers.71.mlp.experts.177.gate_proj", + "model.layers.71.mlp.experts.177.up_proj", + "model.layers.71.mlp.experts.178.down_proj", + "model.layers.71.mlp.experts.178.gate_proj", + "model.layers.71.mlp.experts.178.up_proj", + "model.layers.71.mlp.experts.179.down_proj", + "model.layers.71.mlp.experts.179.gate_proj", + "model.layers.71.mlp.experts.179.up_proj", + "model.layers.71.mlp.experts.18.down_proj", + "model.layers.71.mlp.experts.18.gate_proj", + "model.layers.71.mlp.experts.18.up_proj", + "model.layers.71.mlp.experts.180.down_proj", + "model.layers.71.mlp.experts.180.gate_proj", + "model.layers.71.mlp.experts.180.up_proj", + "model.layers.71.mlp.experts.181.down_proj", + "model.layers.71.mlp.experts.181.gate_proj", + "model.layers.71.mlp.experts.181.up_proj", + "model.layers.71.mlp.experts.182.down_proj", + "model.layers.71.mlp.experts.182.gate_proj", + "model.layers.71.mlp.experts.182.up_proj", + "model.layers.71.mlp.experts.183.down_proj", + "model.layers.71.mlp.experts.183.gate_proj", + "model.layers.71.mlp.experts.183.up_proj", + "model.layers.71.mlp.experts.184.down_proj", + "model.layers.71.mlp.experts.184.gate_proj", + "model.layers.71.mlp.experts.184.up_proj", + "model.layers.71.mlp.experts.185.down_proj", + "model.layers.71.mlp.experts.185.gate_proj", + "model.layers.71.mlp.experts.185.up_proj", + "model.layers.71.mlp.experts.186.down_proj", + "model.layers.71.mlp.experts.186.gate_proj", + "model.layers.71.mlp.experts.186.up_proj", + "model.layers.71.mlp.experts.187.down_proj", + "model.layers.71.mlp.experts.187.gate_proj", + "model.layers.71.mlp.experts.187.up_proj", + "model.layers.71.mlp.experts.188.down_proj", + "model.layers.71.mlp.experts.188.gate_proj", + "model.layers.71.mlp.experts.188.up_proj", + "model.layers.71.mlp.experts.189.down_proj", + "model.layers.71.mlp.experts.189.gate_proj", + "model.layers.71.mlp.experts.189.up_proj", + "model.layers.71.mlp.experts.19.down_proj", + "model.layers.71.mlp.experts.19.gate_proj", + "model.layers.71.mlp.experts.19.up_proj", + "model.layers.71.mlp.experts.190.down_proj", + "model.layers.71.mlp.experts.190.gate_proj", + "model.layers.71.mlp.experts.190.up_proj", + "model.layers.71.mlp.experts.191.down_proj", + "model.layers.71.mlp.experts.191.gate_proj", + "model.layers.71.mlp.experts.191.up_proj", + "model.layers.71.mlp.experts.192.down_proj", + "model.layers.71.mlp.experts.192.gate_proj", + "model.layers.71.mlp.experts.192.up_proj", + "model.layers.71.mlp.experts.193.down_proj", + "model.layers.71.mlp.experts.193.gate_proj", + "model.layers.71.mlp.experts.193.up_proj", + "model.layers.71.mlp.experts.194.down_proj", + "model.layers.71.mlp.experts.194.gate_proj", + "model.layers.71.mlp.experts.194.up_proj", + "model.layers.71.mlp.experts.195.down_proj", + "model.layers.71.mlp.experts.195.gate_proj", + "model.layers.71.mlp.experts.195.up_proj", + "model.layers.71.mlp.experts.196.down_proj", + "model.layers.71.mlp.experts.196.gate_proj", + "model.layers.71.mlp.experts.196.up_proj", + "model.layers.71.mlp.experts.197.down_proj", + "model.layers.71.mlp.experts.197.gate_proj", + "model.layers.71.mlp.experts.197.up_proj", + "model.layers.71.mlp.experts.198.down_proj", + "model.layers.71.mlp.experts.198.gate_proj", + "model.layers.71.mlp.experts.198.up_proj", + "model.layers.71.mlp.experts.199.down_proj", + "model.layers.71.mlp.experts.199.gate_proj", + "model.layers.71.mlp.experts.199.up_proj", + "model.layers.71.mlp.experts.2.down_proj", + "model.layers.71.mlp.experts.2.gate_proj", + "model.layers.71.mlp.experts.2.up_proj", + "model.layers.71.mlp.experts.20.down_proj", + "model.layers.71.mlp.experts.20.gate_proj", + "model.layers.71.mlp.experts.20.up_proj", + "model.layers.71.mlp.experts.200.down_proj", + "model.layers.71.mlp.experts.200.gate_proj", + "model.layers.71.mlp.experts.200.up_proj", + "model.layers.71.mlp.experts.201.down_proj", + "model.layers.71.mlp.experts.201.gate_proj", + "model.layers.71.mlp.experts.201.up_proj", + "model.layers.71.mlp.experts.202.down_proj", + "model.layers.71.mlp.experts.202.gate_proj", + "model.layers.71.mlp.experts.202.up_proj", + "model.layers.71.mlp.experts.203.down_proj", + "model.layers.71.mlp.experts.203.gate_proj", + "model.layers.71.mlp.experts.203.up_proj", + "model.layers.71.mlp.experts.204.down_proj", + "model.layers.71.mlp.experts.204.gate_proj", + "model.layers.71.mlp.experts.204.up_proj", + "model.layers.71.mlp.experts.205.down_proj", + "model.layers.71.mlp.experts.205.gate_proj", + "model.layers.71.mlp.experts.205.up_proj", + "model.layers.71.mlp.experts.206.down_proj", + "model.layers.71.mlp.experts.206.gate_proj", + "model.layers.71.mlp.experts.206.up_proj", + "model.layers.71.mlp.experts.207.down_proj", + "model.layers.71.mlp.experts.207.gate_proj", + "model.layers.71.mlp.experts.207.up_proj", + "model.layers.71.mlp.experts.208.down_proj", + "model.layers.71.mlp.experts.208.gate_proj", + "model.layers.71.mlp.experts.208.up_proj", + "model.layers.71.mlp.experts.209.down_proj", + "model.layers.71.mlp.experts.209.gate_proj", + "model.layers.71.mlp.experts.209.up_proj", + "model.layers.71.mlp.experts.21.down_proj", + "model.layers.71.mlp.experts.21.gate_proj", + "model.layers.71.mlp.experts.21.up_proj", + "model.layers.71.mlp.experts.210.down_proj", + "model.layers.71.mlp.experts.210.gate_proj", + "model.layers.71.mlp.experts.210.up_proj", + "model.layers.71.mlp.experts.211.down_proj", + "model.layers.71.mlp.experts.211.gate_proj", + "model.layers.71.mlp.experts.211.up_proj", + "model.layers.71.mlp.experts.212.down_proj", + "model.layers.71.mlp.experts.212.gate_proj", + "model.layers.71.mlp.experts.212.up_proj", + "model.layers.71.mlp.experts.213.down_proj", + "model.layers.71.mlp.experts.213.gate_proj", + "model.layers.71.mlp.experts.213.up_proj", + "model.layers.71.mlp.experts.214.down_proj", + "model.layers.71.mlp.experts.214.gate_proj", + "model.layers.71.mlp.experts.214.up_proj", + "model.layers.71.mlp.experts.215.down_proj", + "model.layers.71.mlp.experts.215.gate_proj", + "model.layers.71.mlp.experts.215.up_proj", + "model.layers.71.mlp.experts.216.down_proj", + "model.layers.71.mlp.experts.216.gate_proj", + "model.layers.71.mlp.experts.216.up_proj", + "model.layers.71.mlp.experts.217.down_proj", + "model.layers.71.mlp.experts.217.gate_proj", + "model.layers.71.mlp.experts.217.up_proj", + "model.layers.71.mlp.experts.218.down_proj", + "model.layers.71.mlp.experts.218.gate_proj", + "model.layers.71.mlp.experts.218.up_proj", + "model.layers.71.mlp.experts.219.down_proj", + "model.layers.71.mlp.experts.219.gate_proj", + "model.layers.71.mlp.experts.219.up_proj", + "model.layers.71.mlp.experts.22.down_proj", + "model.layers.71.mlp.experts.22.gate_proj", + "model.layers.71.mlp.experts.22.up_proj", + "model.layers.71.mlp.experts.220.down_proj", + "model.layers.71.mlp.experts.220.gate_proj", + "model.layers.71.mlp.experts.220.up_proj", + "model.layers.71.mlp.experts.221.down_proj", + "model.layers.71.mlp.experts.221.gate_proj", + "model.layers.71.mlp.experts.221.up_proj", + "model.layers.71.mlp.experts.222.down_proj", + "model.layers.71.mlp.experts.222.gate_proj", + "model.layers.71.mlp.experts.222.up_proj", + "model.layers.71.mlp.experts.223.down_proj", + "model.layers.71.mlp.experts.223.gate_proj", + "model.layers.71.mlp.experts.223.up_proj", + "model.layers.71.mlp.experts.224.down_proj", + "model.layers.71.mlp.experts.224.gate_proj", + "model.layers.71.mlp.experts.224.up_proj", + "model.layers.71.mlp.experts.225.down_proj", + "model.layers.71.mlp.experts.225.gate_proj", + "model.layers.71.mlp.experts.225.up_proj", + "model.layers.71.mlp.experts.226.down_proj", + "model.layers.71.mlp.experts.226.gate_proj", + "model.layers.71.mlp.experts.226.up_proj", + "model.layers.71.mlp.experts.227.down_proj", + "model.layers.71.mlp.experts.227.gate_proj", + "model.layers.71.mlp.experts.227.up_proj", + "model.layers.71.mlp.experts.228.down_proj", + "model.layers.71.mlp.experts.228.gate_proj", + "model.layers.71.mlp.experts.228.up_proj", + "model.layers.71.mlp.experts.229.down_proj", + "model.layers.71.mlp.experts.229.gate_proj", + "model.layers.71.mlp.experts.229.up_proj", + "model.layers.71.mlp.experts.23.down_proj", + "model.layers.71.mlp.experts.23.gate_proj", + "model.layers.71.mlp.experts.23.up_proj", + "model.layers.71.mlp.experts.230.down_proj", + "model.layers.71.mlp.experts.230.gate_proj", + "model.layers.71.mlp.experts.230.up_proj", + "model.layers.71.mlp.experts.231.down_proj", + "model.layers.71.mlp.experts.231.gate_proj", + "model.layers.71.mlp.experts.231.up_proj", + "model.layers.71.mlp.experts.232.down_proj", + "model.layers.71.mlp.experts.232.gate_proj", + "model.layers.71.mlp.experts.232.up_proj", + "model.layers.71.mlp.experts.233.down_proj", + "model.layers.71.mlp.experts.233.gate_proj", + "model.layers.71.mlp.experts.233.up_proj", + "model.layers.71.mlp.experts.234.down_proj", + "model.layers.71.mlp.experts.234.gate_proj", + "model.layers.71.mlp.experts.234.up_proj", + "model.layers.71.mlp.experts.235.down_proj", + "model.layers.71.mlp.experts.235.gate_proj", + "model.layers.71.mlp.experts.235.up_proj", + "model.layers.71.mlp.experts.236.down_proj", + "model.layers.71.mlp.experts.236.gate_proj", + "model.layers.71.mlp.experts.236.up_proj", + "model.layers.71.mlp.experts.237.down_proj", + "model.layers.71.mlp.experts.237.gate_proj", + "model.layers.71.mlp.experts.237.up_proj", + "model.layers.71.mlp.experts.238.down_proj", + "model.layers.71.mlp.experts.238.gate_proj", + "model.layers.71.mlp.experts.238.up_proj", + "model.layers.71.mlp.experts.239.down_proj", + "model.layers.71.mlp.experts.239.gate_proj", + "model.layers.71.mlp.experts.239.up_proj", + "model.layers.71.mlp.experts.24.down_proj", + "model.layers.71.mlp.experts.24.gate_proj", + "model.layers.71.mlp.experts.24.up_proj", + "model.layers.71.mlp.experts.240.down_proj", + "model.layers.71.mlp.experts.240.gate_proj", + "model.layers.71.mlp.experts.240.up_proj", + "model.layers.71.mlp.experts.241.down_proj", + "model.layers.71.mlp.experts.241.gate_proj", + "model.layers.71.mlp.experts.241.up_proj", + "model.layers.71.mlp.experts.242.down_proj", + "model.layers.71.mlp.experts.242.gate_proj", + "model.layers.71.mlp.experts.242.up_proj", + "model.layers.71.mlp.experts.243.down_proj", + "model.layers.71.mlp.experts.243.gate_proj", + "model.layers.71.mlp.experts.243.up_proj", + "model.layers.71.mlp.experts.244.down_proj", + "model.layers.71.mlp.experts.244.gate_proj", + "model.layers.71.mlp.experts.244.up_proj", + "model.layers.71.mlp.experts.245.down_proj", + "model.layers.71.mlp.experts.245.gate_proj", + "model.layers.71.mlp.experts.245.up_proj", + "model.layers.71.mlp.experts.246.down_proj", + "model.layers.71.mlp.experts.246.gate_proj", + "model.layers.71.mlp.experts.246.up_proj", + "model.layers.71.mlp.experts.247.down_proj", + "model.layers.71.mlp.experts.247.gate_proj", + "model.layers.71.mlp.experts.247.up_proj", + "model.layers.71.mlp.experts.248.down_proj", + "model.layers.71.mlp.experts.248.gate_proj", + "model.layers.71.mlp.experts.248.up_proj", + "model.layers.71.mlp.experts.249.down_proj", + "model.layers.71.mlp.experts.249.gate_proj", + "model.layers.71.mlp.experts.249.up_proj", + "model.layers.71.mlp.experts.25.down_proj", + "model.layers.71.mlp.experts.25.gate_proj", + "model.layers.71.mlp.experts.25.up_proj", + "model.layers.71.mlp.experts.250.down_proj", + "model.layers.71.mlp.experts.250.gate_proj", + "model.layers.71.mlp.experts.250.up_proj", + "model.layers.71.mlp.experts.251.down_proj", + "model.layers.71.mlp.experts.251.gate_proj", + "model.layers.71.mlp.experts.251.up_proj", + "model.layers.71.mlp.experts.252.down_proj", + "model.layers.71.mlp.experts.252.gate_proj", + "model.layers.71.mlp.experts.252.up_proj", + "model.layers.71.mlp.experts.253.down_proj", + "model.layers.71.mlp.experts.253.gate_proj", + "model.layers.71.mlp.experts.253.up_proj", + "model.layers.71.mlp.experts.254.down_proj", + "model.layers.71.mlp.experts.254.gate_proj", + "model.layers.71.mlp.experts.254.up_proj", + "model.layers.71.mlp.experts.255.down_proj", + "model.layers.71.mlp.experts.255.gate_proj", + "model.layers.71.mlp.experts.255.up_proj", + "model.layers.71.mlp.experts.26.down_proj", + "model.layers.71.mlp.experts.26.gate_proj", + "model.layers.71.mlp.experts.26.up_proj", + "model.layers.71.mlp.experts.27.down_proj", + "model.layers.71.mlp.experts.27.gate_proj", + "model.layers.71.mlp.experts.27.up_proj", + "model.layers.71.mlp.experts.28.down_proj", + "model.layers.71.mlp.experts.28.gate_proj", + "model.layers.71.mlp.experts.28.up_proj", + "model.layers.71.mlp.experts.29.down_proj", + "model.layers.71.mlp.experts.29.gate_proj", + "model.layers.71.mlp.experts.29.up_proj", + "model.layers.71.mlp.experts.3.down_proj", + "model.layers.71.mlp.experts.3.gate_proj", + "model.layers.71.mlp.experts.3.up_proj", + "model.layers.71.mlp.experts.30.down_proj", + "model.layers.71.mlp.experts.30.gate_proj", + "model.layers.71.mlp.experts.30.up_proj", + "model.layers.71.mlp.experts.31.down_proj", + "model.layers.71.mlp.experts.31.gate_proj", + "model.layers.71.mlp.experts.31.up_proj", + "model.layers.71.mlp.experts.32.down_proj", + "model.layers.71.mlp.experts.32.gate_proj", + "model.layers.71.mlp.experts.32.up_proj", + "model.layers.71.mlp.experts.33.down_proj", + "model.layers.71.mlp.experts.33.gate_proj", + "model.layers.71.mlp.experts.33.up_proj", + "model.layers.71.mlp.experts.34.down_proj", + "model.layers.71.mlp.experts.34.gate_proj", + "model.layers.71.mlp.experts.34.up_proj", + "model.layers.71.mlp.experts.35.down_proj", + "model.layers.71.mlp.experts.35.gate_proj", + "model.layers.71.mlp.experts.35.up_proj", + "model.layers.71.mlp.experts.36.down_proj", + "model.layers.71.mlp.experts.36.gate_proj", + "model.layers.71.mlp.experts.36.up_proj", + "model.layers.71.mlp.experts.37.down_proj", + "model.layers.71.mlp.experts.37.gate_proj", + "model.layers.71.mlp.experts.37.up_proj", + "model.layers.71.mlp.experts.38.down_proj", + "model.layers.71.mlp.experts.38.gate_proj", + "model.layers.71.mlp.experts.38.up_proj", + "model.layers.71.mlp.experts.39.down_proj", + "model.layers.71.mlp.experts.39.gate_proj", + "model.layers.71.mlp.experts.39.up_proj", + "model.layers.71.mlp.experts.4.down_proj", + "model.layers.71.mlp.experts.4.gate_proj", + "model.layers.71.mlp.experts.4.up_proj", + "model.layers.71.mlp.experts.40.down_proj", + "model.layers.71.mlp.experts.40.gate_proj", + "model.layers.71.mlp.experts.40.up_proj", + "model.layers.71.mlp.experts.41.down_proj", + "model.layers.71.mlp.experts.41.gate_proj", + "model.layers.71.mlp.experts.41.up_proj", + "model.layers.71.mlp.experts.42.down_proj", + "model.layers.71.mlp.experts.42.gate_proj", + "model.layers.71.mlp.experts.42.up_proj", + "model.layers.71.mlp.experts.43.down_proj", + "model.layers.71.mlp.experts.43.gate_proj", + "model.layers.71.mlp.experts.43.up_proj", + "model.layers.71.mlp.experts.44.down_proj", + "model.layers.71.mlp.experts.44.gate_proj", + "model.layers.71.mlp.experts.44.up_proj", + "model.layers.71.mlp.experts.45.down_proj", + "model.layers.71.mlp.experts.45.gate_proj", + "model.layers.71.mlp.experts.45.up_proj", + "model.layers.71.mlp.experts.46.down_proj", + "model.layers.71.mlp.experts.46.gate_proj", + "model.layers.71.mlp.experts.46.up_proj", + "model.layers.71.mlp.experts.47.down_proj", + "model.layers.71.mlp.experts.47.gate_proj", + "model.layers.71.mlp.experts.47.up_proj", + "model.layers.71.mlp.experts.48.down_proj", + "model.layers.71.mlp.experts.48.gate_proj", + "model.layers.71.mlp.experts.48.up_proj", + "model.layers.71.mlp.experts.49.down_proj", + "model.layers.71.mlp.experts.49.gate_proj", + "model.layers.71.mlp.experts.49.up_proj", + "model.layers.71.mlp.experts.5.down_proj", + "model.layers.71.mlp.experts.5.gate_proj", + "model.layers.71.mlp.experts.5.up_proj", + "model.layers.71.mlp.experts.50.down_proj", + "model.layers.71.mlp.experts.50.gate_proj", + "model.layers.71.mlp.experts.50.up_proj", + "model.layers.71.mlp.experts.51.down_proj", + "model.layers.71.mlp.experts.51.gate_proj", + "model.layers.71.mlp.experts.51.up_proj", + "model.layers.71.mlp.experts.52.down_proj", + "model.layers.71.mlp.experts.52.gate_proj", + "model.layers.71.mlp.experts.52.up_proj", + "model.layers.71.mlp.experts.53.down_proj", + "model.layers.71.mlp.experts.53.gate_proj", + "model.layers.71.mlp.experts.53.up_proj", + "model.layers.71.mlp.experts.54.down_proj", + "model.layers.71.mlp.experts.54.gate_proj", + "model.layers.71.mlp.experts.54.up_proj", + "model.layers.71.mlp.experts.55.down_proj", + "model.layers.71.mlp.experts.55.gate_proj", + "model.layers.71.mlp.experts.55.up_proj", + "model.layers.71.mlp.experts.56.down_proj", + "model.layers.71.mlp.experts.56.gate_proj", + "model.layers.71.mlp.experts.56.up_proj", + "model.layers.71.mlp.experts.57.down_proj", + "model.layers.71.mlp.experts.57.gate_proj", + "model.layers.71.mlp.experts.57.up_proj", + "model.layers.71.mlp.experts.58.down_proj", + "model.layers.71.mlp.experts.58.gate_proj", + "model.layers.71.mlp.experts.58.up_proj", + "model.layers.71.mlp.experts.59.down_proj", + "model.layers.71.mlp.experts.59.gate_proj", + "model.layers.71.mlp.experts.59.up_proj", + "model.layers.71.mlp.experts.6.down_proj", + "model.layers.71.mlp.experts.6.gate_proj", + "model.layers.71.mlp.experts.6.up_proj", + "model.layers.71.mlp.experts.60.down_proj", + "model.layers.71.mlp.experts.60.gate_proj", + "model.layers.71.mlp.experts.60.up_proj", + "model.layers.71.mlp.experts.61.down_proj", + "model.layers.71.mlp.experts.61.gate_proj", + "model.layers.71.mlp.experts.61.up_proj", + "model.layers.71.mlp.experts.62.down_proj", + "model.layers.71.mlp.experts.62.gate_proj", + "model.layers.71.mlp.experts.62.up_proj", + "model.layers.71.mlp.experts.63.down_proj", + "model.layers.71.mlp.experts.63.gate_proj", + "model.layers.71.mlp.experts.63.up_proj", + "model.layers.71.mlp.experts.64.down_proj", + "model.layers.71.mlp.experts.64.gate_proj", + "model.layers.71.mlp.experts.64.up_proj", + "model.layers.71.mlp.experts.65.down_proj", + "model.layers.71.mlp.experts.65.gate_proj", + "model.layers.71.mlp.experts.65.up_proj", + "model.layers.71.mlp.experts.66.down_proj", + "model.layers.71.mlp.experts.66.gate_proj", + "model.layers.71.mlp.experts.66.up_proj", + "model.layers.71.mlp.experts.67.down_proj", + "model.layers.71.mlp.experts.67.gate_proj", + "model.layers.71.mlp.experts.67.up_proj", + "model.layers.71.mlp.experts.68.down_proj", + "model.layers.71.mlp.experts.68.gate_proj", + "model.layers.71.mlp.experts.68.up_proj", + "model.layers.71.mlp.experts.69.down_proj", + "model.layers.71.mlp.experts.69.gate_proj", + "model.layers.71.mlp.experts.69.up_proj", + "model.layers.71.mlp.experts.7.down_proj", + "model.layers.71.mlp.experts.7.gate_proj", + "model.layers.71.mlp.experts.7.up_proj", + "model.layers.71.mlp.experts.70.down_proj", + "model.layers.71.mlp.experts.70.gate_proj", + "model.layers.71.mlp.experts.70.up_proj", + "model.layers.71.mlp.experts.71.down_proj", + "model.layers.71.mlp.experts.71.gate_proj", + "model.layers.71.mlp.experts.71.up_proj", + "model.layers.71.mlp.experts.72.down_proj", + "model.layers.71.mlp.experts.72.gate_proj", + "model.layers.71.mlp.experts.72.up_proj", + "model.layers.71.mlp.experts.73.down_proj", + "model.layers.71.mlp.experts.73.gate_proj", + "model.layers.71.mlp.experts.73.up_proj", + "model.layers.71.mlp.experts.74.down_proj", + "model.layers.71.mlp.experts.74.gate_proj", + "model.layers.71.mlp.experts.74.up_proj", + "model.layers.71.mlp.experts.75.down_proj", + "model.layers.71.mlp.experts.75.gate_proj", + "model.layers.71.mlp.experts.75.up_proj", + "model.layers.71.mlp.experts.76.down_proj", + "model.layers.71.mlp.experts.76.gate_proj", + "model.layers.71.mlp.experts.76.up_proj", + "model.layers.71.mlp.experts.77.down_proj", + "model.layers.71.mlp.experts.77.gate_proj", + "model.layers.71.mlp.experts.77.up_proj", + "model.layers.71.mlp.experts.78.down_proj", + "model.layers.71.mlp.experts.78.gate_proj", + "model.layers.71.mlp.experts.78.up_proj", + "model.layers.71.mlp.experts.79.down_proj", + "model.layers.71.mlp.experts.79.gate_proj", + "model.layers.71.mlp.experts.79.up_proj", + "model.layers.71.mlp.experts.8.down_proj", + "model.layers.71.mlp.experts.8.gate_proj", + "model.layers.71.mlp.experts.8.up_proj", + "model.layers.71.mlp.experts.80.down_proj", + "model.layers.71.mlp.experts.80.gate_proj", + "model.layers.71.mlp.experts.80.up_proj", + "model.layers.71.mlp.experts.81.down_proj", + "model.layers.71.mlp.experts.81.gate_proj", + "model.layers.71.mlp.experts.81.up_proj", + "model.layers.71.mlp.experts.82.down_proj", + "model.layers.71.mlp.experts.82.gate_proj", + "model.layers.71.mlp.experts.82.up_proj", + "model.layers.71.mlp.experts.83.down_proj", + "model.layers.71.mlp.experts.83.gate_proj", + "model.layers.71.mlp.experts.83.up_proj", + "model.layers.71.mlp.experts.84.down_proj", + "model.layers.71.mlp.experts.84.gate_proj", + "model.layers.71.mlp.experts.84.up_proj", + "model.layers.71.mlp.experts.85.down_proj", + "model.layers.71.mlp.experts.85.gate_proj", + "model.layers.71.mlp.experts.85.up_proj", + "model.layers.71.mlp.experts.86.down_proj", + "model.layers.71.mlp.experts.86.gate_proj", + "model.layers.71.mlp.experts.86.up_proj", + "model.layers.71.mlp.experts.87.down_proj", + "model.layers.71.mlp.experts.87.gate_proj", + "model.layers.71.mlp.experts.87.up_proj", + "model.layers.71.mlp.experts.88.down_proj", + "model.layers.71.mlp.experts.88.gate_proj", + "model.layers.71.mlp.experts.88.up_proj", + "model.layers.71.mlp.experts.89.down_proj", + "model.layers.71.mlp.experts.89.gate_proj", + "model.layers.71.mlp.experts.89.up_proj", + "model.layers.71.mlp.experts.9.down_proj", + "model.layers.71.mlp.experts.9.gate_proj", + "model.layers.71.mlp.experts.9.up_proj", + "model.layers.71.mlp.experts.90.down_proj", + "model.layers.71.mlp.experts.90.gate_proj", + "model.layers.71.mlp.experts.90.up_proj", + "model.layers.71.mlp.experts.91.down_proj", + "model.layers.71.mlp.experts.91.gate_proj", + "model.layers.71.mlp.experts.91.up_proj", + "model.layers.71.mlp.experts.92.down_proj", + "model.layers.71.mlp.experts.92.gate_proj", + "model.layers.71.mlp.experts.92.up_proj", + "model.layers.71.mlp.experts.93.down_proj", + "model.layers.71.mlp.experts.93.gate_proj", + "model.layers.71.mlp.experts.93.up_proj", + "model.layers.71.mlp.experts.94.down_proj", + "model.layers.71.mlp.experts.94.gate_proj", + "model.layers.71.mlp.experts.94.up_proj", + "model.layers.71.mlp.experts.95.down_proj", + "model.layers.71.mlp.experts.95.gate_proj", + "model.layers.71.mlp.experts.95.up_proj", + "model.layers.71.mlp.experts.96.down_proj", + "model.layers.71.mlp.experts.96.gate_proj", + "model.layers.71.mlp.experts.96.up_proj", + "model.layers.71.mlp.experts.97.down_proj", + "model.layers.71.mlp.experts.97.gate_proj", + "model.layers.71.mlp.experts.97.up_proj", + "model.layers.71.mlp.experts.98.down_proj", + "model.layers.71.mlp.experts.98.gate_proj", + "model.layers.71.mlp.experts.98.up_proj", + "model.layers.71.mlp.experts.99.down_proj", + "model.layers.71.mlp.experts.99.gate_proj", + "model.layers.71.mlp.experts.99.up_proj", + "model.layers.72.mlp.experts.0.down_proj", + "model.layers.72.mlp.experts.0.gate_proj", + "model.layers.72.mlp.experts.0.up_proj", + "model.layers.72.mlp.experts.1.down_proj", + "model.layers.72.mlp.experts.1.gate_proj", + "model.layers.72.mlp.experts.1.up_proj", + "model.layers.72.mlp.experts.10.down_proj", + "model.layers.72.mlp.experts.10.gate_proj", + "model.layers.72.mlp.experts.10.up_proj", + "model.layers.72.mlp.experts.100.down_proj", + "model.layers.72.mlp.experts.100.gate_proj", + "model.layers.72.mlp.experts.100.up_proj", + "model.layers.72.mlp.experts.101.down_proj", + "model.layers.72.mlp.experts.101.gate_proj", + "model.layers.72.mlp.experts.101.up_proj", + "model.layers.72.mlp.experts.102.down_proj", + "model.layers.72.mlp.experts.102.gate_proj", + "model.layers.72.mlp.experts.102.up_proj", + "model.layers.72.mlp.experts.103.down_proj", + "model.layers.72.mlp.experts.103.gate_proj", + "model.layers.72.mlp.experts.103.up_proj", + "model.layers.72.mlp.experts.104.down_proj", + "model.layers.72.mlp.experts.104.gate_proj", + "model.layers.72.mlp.experts.104.up_proj", + "model.layers.72.mlp.experts.105.down_proj", + "model.layers.72.mlp.experts.105.gate_proj", + "model.layers.72.mlp.experts.105.up_proj", + "model.layers.72.mlp.experts.106.down_proj", + "model.layers.72.mlp.experts.106.gate_proj", + "model.layers.72.mlp.experts.106.up_proj", + "model.layers.72.mlp.experts.107.down_proj", + "model.layers.72.mlp.experts.107.gate_proj", + "model.layers.72.mlp.experts.107.up_proj", + "model.layers.72.mlp.experts.108.down_proj", + "model.layers.72.mlp.experts.108.gate_proj", + "model.layers.72.mlp.experts.108.up_proj", + "model.layers.72.mlp.experts.109.down_proj", + "model.layers.72.mlp.experts.109.gate_proj", + "model.layers.72.mlp.experts.109.up_proj", + "model.layers.72.mlp.experts.11.down_proj", + "model.layers.72.mlp.experts.11.gate_proj", + "model.layers.72.mlp.experts.11.up_proj", + "model.layers.72.mlp.experts.110.down_proj", + "model.layers.72.mlp.experts.110.gate_proj", + "model.layers.72.mlp.experts.110.up_proj", + "model.layers.72.mlp.experts.111.down_proj", + "model.layers.72.mlp.experts.111.gate_proj", + "model.layers.72.mlp.experts.111.up_proj", + "model.layers.72.mlp.experts.112.down_proj", + "model.layers.72.mlp.experts.112.gate_proj", + "model.layers.72.mlp.experts.112.up_proj", + "model.layers.72.mlp.experts.113.down_proj", + "model.layers.72.mlp.experts.113.gate_proj", + "model.layers.72.mlp.experts.113.up_proj", + "model.layers.72.mlp.experts.114.down_proj", + "model.layers.72.mlp.experts.114.gate_proj", + "model.layers.72.mlp.experts.114.up_proj", + "model.layers.72.mlp.experts.115.down_proj", + "model.layers.72.mlp.experts.115.gate_proj", + "model.layers.72.mlp.experts.115.up_proj", + "model.layers.72.mlp.experts.116.down_proj", + "model.layers.72.mlp.experts.116.gate_proj", + "model.layers.72.mlp.experts.116.up_proj", + "model.layers.72.mlp.experts.117.down_proj", + "model.layers.72.mlp.experts.117.gate_proj", + "model.layers.72.mlp.experts.117.up_proj", + "model.layers.72.mlp.experts.118.down_proj", + "model.layers.72.mlp.experts.118.gate_proj", + "model.layers.72.mlp.experts.118.up_proj", + "model.layers.72.mlp.experts.119.down_proj", + "model.layers.72.mlp.experts.119.gate_proj", + "model.layers.72.mlp.experts.119.up_proj", + "model.layers.72.mlp.experts.12.down_proj", + "model.layers.72.mlp.experts.12.gate_proj", + "model.layers.72.mlp.experts.12.up_proj", + "model.layers.72.mlp.experts.120.down_proj", + "model.layers.72.mlp.experts.120.gate_proj", + "model.layers.72.mlp.experts.120.up_proj", + "model.layers.72.mlp.experts.121.down_proj", + "model.layers.72.mlp.experts.121.gate_proj", + "model.layers.72.mlp.experts.121.up_proj", + "model.layers.72.mlp.experts.122.down_proj", + "model.layers.72.mlp.experts.122.gate_proj", + "model.layers.72.mlp.experts.122.up_proj", + "model.layers.72.mlp.experts.123.down_proj", + "model.layers.72.mlp.experts.123.gate_proj", + "model.layers.72.mlp.experts.123.up_proj", + "model.layers.72.mlp.experts.124.down_proj", + "model.layers.72.mlp.experts.124.gate_proj", + "model.layers.72.mlp.experts.124.up_proj", + "model.layers.72.mlp.experts.125.down_proj", + "model.layers.72.mlp.experts.125.gate_proj", + "model.layers.72.mlp.experts.125.up_proj", + "model.layers.72.mlp.experts.126.down_proj", + "model.layers.72.mlp.experts.126.gate_proj", + "model.layers.72.mlp.experts.126.up_proj", + "model.layers.72.mlp.experts.127.down_proj", + "model.layers.72.mlp.experts.127.gate_proj", + "model.layers.72.mlp.experts.127.up_proj", + "model.layers.72.mlp.experts.128.down_proj", + "model.layers.72.mlp.experts.128.gate_proj", + "model.layers.72.mlp.experts.128.up_proj", + "model.layers.72.mlp.experts.129.down_proj", + "model.layers.72.mlp.experts.129.gate_proj", + "model.layers.72.mlp.experts.129.up_proj", + "model.layers.72.mlp.experts.13.down_proj", + "model.layers.72.mlp.experts.13.gate_proj", + "model.layers.72.mlp.experts.13.up_proj", + "model.layers.72.mlp.experts.130.down_proj", + "model.layers.72.mlp.experts.130.gate_proj", + "model.layers.72.mlp.experts.130.up_proj", + "model.layers.72.mlp.experts.131.down_proj", + "model.layers.72.mlp.experts.131.gate_proj", + "model.layers.72.mlp.experts.131.up_proj", + "model.layers.72.mlp.experts.132.down_proj", + "model.layers.72.mlp.experts.132.gate_proj", + "model.layers.72.mlp.experts.132.up_proj", + "model.layers.72.mlp.experts.133.down_proj", + "model.layers.72.mlp.experts.133.gate_proj", + "model.layers.72.mlp.experts.133.up_proj", + "model.layers.72.mlp.experts.134.down_proj", + "model.layers.72.mlp.experts.134.gate_proj", + "model.layers.72.mlp.experts.134.up_proj", + "model.layers.72.mlp.experts.135.down_proj", + "model.layers.72.mlp.experts.135.gate_proj", + "model.layers.72.mlp.experts.135.up_proj", + "model.layers.72.mlp.experts.136.down_proj", + "model.layers.72.mlp.experts.136.gate_proj", + "model.layers.72.mlp.experts.136.up_proj", + "model.layers.72.mlp.experts.137.down_proj", + "model.layers.72.mlp.experts.137.gate_proj", + "model.layers.72.mlp.experts.137.up_proj", + "model.layers.72.mlp.experts.138.down_proj", + "model.layers.72.mlp.experts.138.gate_proj", + "model.layers.72.mlp.experts.138.up_proj", + "model.layers.72.mlp.experts.139.down_proj", + "model.layers.72.mlp.experts.139.gate_proj", + "model.layers.72.mlp.experts.139.up_proj", + "model.layers.72.mlp.experts.14.down_proj", + "model.layers.72.mlp.experts.14.gate_proj", + "model.layers.72.mlp.experts.14.up_proj", + "model.layers.72.mlp.experts.140.down_proj", + "model.layers.72.mlp.experts.140.gate_proj", + "model.layers.72.mlp.experts.140.up_proj", + "model.layers.72.mlp.experts.141.down_proj", + "model.layers.72.mlp.experts.141.gate_proj", + "model.layers.72.mlp.experts.141.up_proj", + "model.layers.72.mlp.experts.142.down_proj", + "model.layers.72.mlp.experts.142.gate_proj", + "model.layers.72.mlp.experts.142.up_proj", + "model.layers.72.mlp.experts.143.down_proj", + "model.layers.72.mlp.experts.143.gate_proj", + "model.layers.72.mlp.experts.143.up_proj", + "model.layers.72.mlp.experts.144.down_proj", + "model.layers.72.mlp.experts.144.gate_proj", + "model.layers.72.mlp.experts.144.up_proj", + "model.layers.72.mlp.experts.145.down_proj", + "model.layers.72.mlp.experts.145.gate_proj", + "model.layers.72.mlp.experts.145.up_proj", + "model.layers.72.mlp.experts.146.down_proj", + "model.layers.72.mlp.experts.146.gate_proj", + "model.layers.72.mlp.experts.146.up_proj", + "model.layers.72.mlp.experts.147.down_proj", + "model.layers.72.mlp.experts.147.gate_proj", + "model.layers.72.mlp.experts.147.up_proj", + "model.layers.72.mlp.experts.148.down_proj", + "model.layers.72.mlp.experts.148.gate_proj", + "model.layers.72.mlp.experts.148.up_proj", + "model.layers.72.mlp.experts.149.down_proj", + "model.layers.72.mlp.experts.149.gate_proj", + "model.layers.72.mlp.experts.149.up_proj", + "model.layers.72.mlp.experts.15.down_proj", + "model.layers.72.mlp.experts.15.gate_proj", + "model.layers.72.mlp.experts.15.up_proj", + "model.layers.72.mlp.experts.150.down_proj", + "model.layers.72.mlp.experts.150.gate_proj", + "model.layers.72.mlp.experts.150.up_proj", + "model.layers.72.mlp.experts.151.down_proj", + "model.layers.72.mlp.experts.151.gate_proj", + "model.layers.72.mlp.experts.151.up_proj", + "model.layers.72.mlp.experts.152.down_proj", + "model.layers.72.mlp.experts.152.gate_proj", + "model.layers.72.mlp.experts.152.up_proj", + "model.layers.72.mlp.experts.153.down_proj", + "model.layers.72.mlp.experts.153.gate_proj", + "model.layers.72.mlp.experts.153.up_proj", + "model.layers.72.mlp.experts.154.down_proj", + "model.layers.72.mlp.experts.154.gate_proj", + "model.layers.72.mlp.experts.154.up_proj", + "model.layers.72.mlp.experts.155.down_proj", + "model.layers.72.mlp.experts.155.gate_proj", + "model.layers.72.mlp.experts.155.up_proj", + "model.layers.72.mlp.experts.156.down_proj", + "model.layers.72.mlp.experts.156.gate_proj", + "model.layers.72.mlp.experts.156.up_proj", + "model.layers.72.mlp.experts.157.down_proj", + "model.layers.72.mlp.experts.157.gate_proj", + "model.layers.72.mlp.experts.157.up_proj", + "model.layers.72.mlp.experts.158.down_proj", + "model.layers.72.mlp.experts.158.gate_proj", + "model.layers.72.mlp.experts.158.up_proj", + "model.layers.72.mlp.experts.159.down_proj", + "model.layers.72.mlp.experts.159.gate_proj", + "model.layers.72.mlp.experts.159.up_proj", + "model.layers.72.mlp.experts.16.down_proj", + "model.layers.72.mlp.experts.16.gate_proj", + "model.layers.72.mlp.experts.16.up_proj", + "model.layers.72.mlp.experts.160.down_proj", + "model.layers.72.mlp.experts.160.gate_proj", + "model.layers.72.mlp.experts.160.up_proj", + "model.layers.72.mlp.experts.161.down_proj", + "model.layers.72.mlp.experts.161.gate_proj", + "model.layers.72.mlp.experts.161.up_proj", + "model.layers.72.mlp.experts.162.down_proj", + "model.layers.72.mlp.experts.162.gate_proj", + "model.layers.72.mlp.experts.162.up_proj", + "model.layers.72.mlp.experts.163.down_proj", + "model.layers.72.mlp.experts.163.gate_proj", + "model.layers.72.mlp.experts.163.up_proj", + "model.layers.72.mlp.experts.164.down_proj", + "model.layers.72.mlp.experts.164.gate_proj", + "model.layers.72.mlp.experts.164.up_proj", + "model.layers.72.mlp.experts.165.down_proj", + "model.layers.72.mlp.experts.165.gate_proj", + "model.layers.72.mlp.experts.165.up_proj", + "model.layers.72.mlp.experts.166.down_proj", + "model.layers.72.mlp.experts.166.gate_proj", + "model.layers.72.mlp.experts.166.up_proj", + "model.layers.72.mlp.experts.167.down_proj", + "model.layers.72.mlp.experts.167.gate_proj", + "model.layers.72.mlp.experts.167.up_proj", + "model.layers.72.mlp.experts.168.down_proj", + "model.layers.72.mlp.experts.168.gate_proj", + "model.layers.72.mlp.experts.168.up_proj", + "model.layers.72.mlp.experts.169.down_proj", + "model.layers.72.mlp.experts.169.gate_proj", + "model.layers.72.mlp.experts.169.up_proj", + "model.layers.72.mlp.experts.17.down_proj", + "model.layers.72.mlp.experts.17.gate_proj", + "model.layers.72.mlp.experts.17.up_proj", + "model.layers.72.mlp.experts.170.down_proj", + "model.layers.72.mlp.experts.170.gate_proj", + "model.layers.72.mlp.experts.170.up_proj", + "model.layers.72.mlp.experts.171.down_proj", + "model.layers.72.mlp.experts.171.gate_proj", + "model.layers.72.mlp.experts.171.up_proj", + "model.layers.72.mlp.experts.172.down_proj", + "model.layers.72.mlp.experts.172.gate_proj", + "model.layers.72.mlp.experts.172.up_proj", + "model.layers.72.mlp.experts.173.down_proj", + "model.layers.72.mlp.experts.173.gate_proj", + "model.layers.72.mlp.experts.173.up_proj", + "model.layers.72.mlp.experts.174.down_proj", + "model.layers.72.mlp.experts.174.gate_proj", + "model.layers.72.mlp.experts.174.up_proj", + "model.layers.72.mlp.experts.175.down_proj", + "model.layers.72.mlp.experts.175.gate_proj", + "model.layers.72.mlp.experts.175.up_proj", + "model.layers.72.mlp.experts.176.down_proj", + "model.layers.72.mlp.experts.176.gate_proj", + "model.layers.72.mlp.experts.176.up_proj", + "model.layers.72.mlp.experts.177.down_proj", + "model.layers.72.mlp.experts.177.gate_proj", + "model.layers.72.mlp.experts.177.up_proj", + "model.layers.72.mlp.experts.178.down_proj", + "model.layers.72.mlp.experts.178.gate_proj", + "model.layers.72.mlp.experts.178.up_proj", + "model.layers.72.mlp.experts.179.down_proj", + "model.layers.72.mlp.experts.179.gate_proj", + "model.layers.72.mlp.experts.179.up_proj", + "model.layers.72.mlp.experts.18.down_proj", + "model.layers.72.mlp.experts.18.gate_proj", + "model.layers.72.mlp.experts.18.up_proj", + "model.layers.72.mlp.experts.180.down_proj", + "model.layers.72.mlp.experts.180.gate_proj", + "model.layers.72.mlp.experts.180.up_proj", + "model.layers.72.mlp.experts.181.down_proj", + "model.layers.72.mlp.experts.181.gate_proj", + "model.layers.72.mlp.experts.181.up_proj", + "model.layers.72.mlp.experts.182.down_proj", + "model.layers.72.mlp.experts.182.gate_proj", + "model.layers.72.mlp.experts.182.up_proj", + "model.layers.72.mlp.experts.183.down_proj", + "model.layers.72.mlp.experts.183.gate_proj", + "model.layers.72.mlp.experts.183.up_proj", + "model.layers.72.mlp.experts.184.down_proj", + "model.layers.72.mlp.experts.184.gate_proj", + "model.layers.72.mlp.experts.184.up_proj", + "model.layers.72.mlp.experts.185.down_proj", + "model.layers.72.mlp.experts.185.gate_proj", + "model.layers.72.mlp.experts.185.up_proj", + "model.layers.72.mlp.experts.186.down_proj", + "model.layers.72.mlp.experts.186.gate_proj", + "model.layers.72.mlp.experts.186.up_proj", + "model.layers.72.mlp.experts.187.down_proj", + "model.layers.72.mlp.experts.187.gate_proj", + "model.layers.72.mlp.experts.187.up_proj", + "model.layers.72.mlp.experts.188.down_proj", + "model.layers.72.mlp.experts.188.gate_proj", + "model.layers.72.mlp.experts.188.up_proj", + "model.layers.72.mlp.experts.189.down_proj", + "model.layers.72.mlp.experts.189.gate_proj", + "model.layers.72.mlp.experts.189.up_proj", + "model.layers.72.mlp.experts.19.down_proj", + "model.layers.72.mlp.experts.19.gate_proj", + "model.layers.72.mlp.experts.19.up_proj", + "model.layers.72.mlp.experts.190.down_proj", + "model.layers.72.mlp.experts.190.gate_proj", + "model.layers.72.mlp.experts.190.up_proj", + "model.layers.72.mlp.experts.191.down_proj", + "model.layers.72.mlp.experts.191.gate_proj", + "model.layers.72.mlp.experts.191.up_proj", + "model.layers.72.mlp.experts.192.down_proj", + "model.layers.72.mlp.experts.192.gate_proj", + "model.layers.72.mlp.experts.192.up_proj", + "model.layers.72.mlp.experts.193.down_proj", + "model.layers.72.mlp.experts.193.gate_proj", + "model.layers.72.mlp.experts.193.up_proj", + "model.layers.72.mlp.experts.194.down_proj", + "model.layers.72.mlp.experts.194.gate_proj", + "model.layers.72.mlp.experts.194.up_proj", + "model.layers.72.mlp.experts.195.down_proj", + "model.layers.72.mlp.experts.195.gate_proj", + "model.layers.72.mlp.experts.195.up_proj", + "model.layers.72.mlp.experts.196.down_proj", + "model.layers.72.mlp.experts.196.gate_proj", + "model.layers.72.mlp.experts.196.up_proj", + "model.layers.72.mlp.experts.197.down_proj", + "model.layers.72.mlp.experts.197.gate_proj", + "model.layers.72.mlp.experts.197.up_proj", + "model.layers.72.mlp.experts.198.down_proj", + "model.layers.72.mlp.experts.198.gate_proj", + "model.layers.72.mlp.experts.198.up_proj", + "model.layers.72.mlp.experts.199.down_proj", + "model.layers.72.mlp.experts.199.gate_proj", + "model.layers.72.mlp.experts.199.up_proj", + "model.layers.72.mlp.experts.2.down_proj", + "model.layers.72.mlp.experts.2.gate_proj", + "model.layers.72.mlp.experts.2.up_proj", + "model.layers.72.mlp.experts.20.down_proj", + "model.layers.72.mlp.experts.20.gate_proj", + "model.layers.72.mlp.experts.20.up_proj", + "model.layers.72.mlp.experts.200.down_proj", + "model.layers.72.mlp.experts.200.gate_proj", + "model.layers.72.mlp.experts.200.up_proj", + "model.layers.72.mlp.experts.201.down_proj", + "model.layers.72.mlp.experts.201.gate_proj", + "model.layers.72.mlp.experts.201.up_proj", + "model.layers.72.mlp.experts.202.down_proj", + "model.layers.72.mlp.experts.202.gate_proj", + "model.layers.72.mlp.experts.202.up_proj", + "model.layers.72.mlp.experts.203.down_proj", + "model.layers.72.mlp.experts.203.gate_proj", + "model.layers.72.mlp.experts.203.up_proj", + "model.layers.72.mlp.experts.204.down_proj", + "model.layers.72.mlp.experts.204.gate_proj", + "model.layers.72.mlp.experts.204.up_proj", + "model.layers.72.mlp.experts.205.down_proj", + "model.layers.72.mlp.experts.205.gate_proj", + "model.layers.72.mlp.experts.205.up_proj", + "model.layers.72.mlp.experts.206.down_proj", + "model.layers.72.mlp.experts.206.gate_proj", + "model.layers.72.mlp.experts.206.up_proj", + "model.layers.72.mlp.experts.207.down_proj", + "model.layers.72.mlp.experts.207.gate_proj", + "model.layers.72.mlp.experts.207.up_proj", + "model.layers.72.mlp.experts.208.down_proj", + "model.layers.72.mlp.experts.208.gate_proj", + "model.layers.72.mlp.experts.208.up_proj", + "model.layers.72.mlp.experts.209.down_proj", + "model.layers.72.mlp.experts.209.gate_proj", + "model.layers.72.mlp.experts.209.up_proj", + "model.layers.72.mlp.experts.21.down_proj", + "model.layers.72.mlp.experts.21.gate_proj", + "model.layers.72.mlp.experts.21.up_proj", + "model.layers.72.mlp.experts.210.down_proj", + "model.layers.72.mlp.experts.210.gate_proj", + "model.layers.72.mlp.experts.210.up_proj", + "model.layers.72.mlp.experts.211.down_proj", + "model.layers.72.mlp.experts.211.gate_proj", + "model.layers.72.mlp.experts.211.up_proj", + "model.layers.72.mlp.experts.212.down_proj", + "model.layers.72.mlp.experts.212.gate_proj", + "model.layers.72.mlp.experts.212.up_proj", + "model.layers.72.mlp.experts.213.down_proj", + "model.layers.72.mlp.experts.213.gate_proj", + "model.layers.72.mlp.experts.213.up_proj", + "model.layers.72.mlp.experts.214.down_proj", + "model.layers.72.mlp.experts.214.gate_proj", + "model.layers.72.mlp.experts.214.up_proj", + "model.layers.72.mlp.experts.215.down_proj", + "model.layers.72.mlp.experts.215.gate_proj", + "model.layers.72.mlp.experts.215.up_proj", + "model.layers.72.mlp.experts.216.down_proj", + "model.layers.72.mlp.experts.216.gate_proj", + "model.layers.72.mlp.experts.216.up_proj", + "model.layers.72.mlp.experts.217.down_proj", + "model.layers.72.mlp.experts.217.gate_proj", + "model.layers.72.mlp.experts.217.up_proj", + "model.layers.72.mlp.experts.218.down_proj", + "model.layers.72.mlp.experts.218.gate_proj", + "model.layers.72.mlp.experts.218.up_proj", + "model.layers.72.mlp.experts.219.down_proj", + "model.layers.72.mlp.experts.219.gate_proj", + "model.layers.72.mlp.experts.219.up_proj", + "model.layers.72.mlp.experts.22.down_proj", + "model.layers.72.mlp.experts.22.gate_proj", + "model.layers.72.mlp.experts.22.up_proj", + "model.layers.72.mlp.experts.220.down_proj", + "model.layers.72.mlp.experts.220.gate_proj", + "model.layers.72.mlp.experts.220.up_proj", + "model.layers.72.mlp.experts.221.down_proj", + "model.layers.72.mlp.experts.221.gate_proj", + "model.layers.72.mlp.experts.221.up_proj", + "model.layers.72.mlp.experts.222.down_proj", + "model.layers.72.mlp.experts.222.gate_proj", + "model.layers.72.mlp.experts.222.up_proj", + "model.layers.72.mlp.experts.223.down_proj", + "model.layers.72.mlp.experts.223.gate_proj", + "model.layers.72.mlp.experts.223.up_proj", + "model.layers.72.mlp.experts.224.down_proj", + "model.layers.72.mlp.experts.224.gate_proj", + "model.layers.72.mlp.experts.224.up_proj", + "model.layers.72.mlp.experts.225.down_proj", + "model.layers.72.mlp.experts.225.gate_proj", + "model.layers.72.mlp.experts.225.up_proj", + "model.layers.72.mlp.experts.226.down_proj", + "model.layers.72.mlp.experts.226.gate_proj", + "model.layers.72.mlp.experts.226.up_proj", + "model.layers.72.mlp.experts.227.down_proj", + "model.layers.72.mlp.experts.227.gate_proj", + "model.layers.72.mlp.experts.227.up_proj", + "model.layers.72.mlp.experts.228.down_proj", + "model.layers.72.mlp.experts.228.gate_proj", + "model.layers.72.mlp.experts.228.up_proj", + "model.layers.72.mlp.experts.229.down_proj", + "model.layers.72.mlp.experts.229.gate_proj", + "model.layers.72.mlp.experts.229.up_proj", + "model.layers.72.mlp.experts.23.down_proj", + "model.layers.72.mlp.experts.23.gate_proj", + "model.layers.72.mlp.experts.23.up_proj", + "model.layers.72.mlp.experts.230.down_proj", + "model.layers.72.mlp.experts.230.gate_proj", + "model.layers.72.mlp.experts.230.up_proj", + "model.layers.72.mlp.experts.231.down_proj", + "model.layers.72.mlp.experts.231.gate_proj", + "model.layers.72.mlp.experts.231.up_proj", + "model.layers.72.mlp.experts.232.down_proj", + "model.layers.72.mlp.experts.232.gate_proj", + "model.layers.72.mlp.experts.232.up_proj", + "model.layers.72.mlp.experts.233.down_proj", + "model.layers.72.mlp.experts.233.gate_proj", + "model.layers.72.mlp.experts.233.up_proj", + "model.layers.72.mlp.experts.234.down_proj", + "model.layers.72.mlp.experts.234.gate_proj", + "model.layers.72.mlp.experts.234.up_proj", + "model.layers.72.mlp.experts.235.down_proj", + "model.layers.72.mlp.experts.235.gate_proj", + "model.layers.72.mlp.experts.235.up_proj", + "model.layers.72.mlp.experts.236.down_proj", + "model.layers.72.mlp.experts.236.gate_proj", + "model.layers.72.mlp.experts.236.up_proj", + "model.layers.72.mlp.experts.237.down_proj", + "model.layers.72.mlp.experts.237.gate_proj", + "model.layers.72.mlp.experts.237.up_proj", + "model.layers.72.mlp.experts.238.down_proj", + "model.layers.72.mlp.experts.238.gate_proj", + "model.layers.72.mlp.experts.238.up_proj", + "model.layers.72.mlp.experts.239.down_proj", + "model.layers.72.mlp.experts.239.gate_proj", + "model.layers.72.mlp.experts.239.up_proj", + "model.layers.72.mlp.experts.24.down_proj", + "model.layers.72.mlp.experts.24.gate_proj", + "model.layers.72.mlp.experts.24.up_proj", + "model.layers.72.mlp.experts.240.down_proj", + "model.layers.72.mlp.experts.240.gate_proj", + "model.layers.72.mlp.experts.240.up_proj", + "model.layers.72.mlp.experts.241.down_proj", + "model.layers.72.mlp.experts.241.gate_proj", + "model.layers.72.mlp.experts.241.up_proj", + "model.layers.72.mlp.experts.242.down_proj", + "model.layers.72.mlp.experts.242.gate_proj", + "model.layers.72.mlp.experts.242.up_proj", + "model.layers.72.mlp.experts.243.down_proj", + "model.layers.72.mlp.experts.243.gate_proj", + "model.layers.72.mlp.experts.243.up_proj", + "model.layers.72.mlp.experts.244.down_proj", + "model.layers.72.mlp.experts.244.gate_proj", + "model.layers.72.mlp.experts.244.up_proj", + "model.layers.72.mlp.experts.245.down_proj", + "model.layers.72.mlp.experts.245.gate_proj", + "model.layers.72.mlp.experts.245.up_proj", + "model.layers.72.mlp.experts.246.down_proj", + "model.layers.72.mlp.experts.246.gate_proj", + "model.layers.72.mlp.experts.246.up_proj", + "model.layers.72.mlp.experts.247.down_proj", + "model.layers.72.mlp.experts.247.gate_proj", + "model.layers.72.mlp.experts.247.up_proj", + "model.layers.72.mlp.experts.248.down_proj", + "model.layers.72.mlp.experts.248.gate_proj", + "model.layers.72.mlp.experts.248.up_proj", + "model.layers.72.mlp.experts.249.down_proj", + "model.layers.72.mlp.experts.249.gate_proj", + "model.layers.72.mlp.experts.249.up_proj", + "model.layers.72.mlp.experts.25.down_proj", + "model.layers.72.mlp.experts.25.gate_proj", + "model.layers.72.mlp.experts.25.up_proj", + "model.layers.72.mlp.experts.250.down_proj", + "model.layers.72.mlp.experts.250.gate_proj", + "model.layers.72.mlp.experts.250.up_proj", + "model.layers.72.mlp.experts.251.down_proj", + "model.layers.72.mlp.experts.251.gate_proj", + "model.layers.72.mlp.experts.251.up_proj", + "model.layers.72.mlp.experts.252.down_proj", + "model.layers.72.mlp.experts.252.gate_proj", + "model.layers.72.mlp.experts.252.up_proj", + "model.layers.72.mlp.experts.253.down_proj", + "model.layers.72.mlp.experts.253.gate_proj", + "model.layers.72.mlp.experts.253.up_proj", + "model.layers.72.mlp.experts.254.down_proj", + "model.layers.72.mlp.experts.254.gate_proj", + "model.layers.72.mlp.experts.254.up_proj", + "model.layers.72.mlp.experts.255.down_proj", + "model.layers.72.mlp.experts.255.gate_proj", + "model.layers.72.mlp.experts.255.up_proj", + "model.layers.72.mlp.experts.26.down_proj", + "model.layers.72.mlp.experts.26.gate_proj", + "model.layers.72.mlp.experts.26.up_proj", + "model.layers.72.mlp.experts.27.down_proj", + "model.layers.72.mlp.experts.27.gate_proj", + "model.layers.72.mlp.experts.27.up_proj", + "model.layers.72.mlp.experts.28.down_proj", + "model.layers.72.mlp.experts.28.gate_proj", + "model.layers.72.mlp.experts.28.up_proj", + "model.layers.72.mlp.experts.29.down_proj", + "model.layers.72.mlp.experts.29.gate_proj", + "model.layers.72.mlp.experts.29.up_proj", + "model.layers.72.mlp.experts.3.down_proj", + "model.layers.72.mlp.experts.3.gate_proj", + "model.layers.72.mlp.experts.3.up_proj", + "model.layers.72.mlp.experts.30.down_proj", + "model.layers.72.mlp.experts.30.gate_proj", + "model.layers.72.mlp.experts.30.up_proj", + "model.layers.72.mlp.experts.31.down_proj", + "model.layers.72.mlp.experts.31.gate_proj", + "model.layers.72.mlp.experts.31.up_proj", + "model.layers.72.mlp.experts.32.down_proj", + "model.layers.72.mlp.experts.32.gate_proj", + "model.layers.72.mlp.experts.32.up_proj", + "model.layers.72.mlp.experts.33.down_proj", + "model.layers.72.mlp.experts.33.gate_proj", + "model.layers.72.mlp.experts.33.up_proj", + "model.layers.72.mlp.experts.34.down_proj", + "model.layers.72.mlp.experts.34.gate_proj", + "model.layers.72.mlp.experts.34.up_proj", + "model.layers.72.mlp.experts.35.down_proj", + "model.layers.72.mlp.experts.35.gate_proj", + "model.layers.72.mlp.experts.35.up_proj", + "model.layers.72.mlp.experts.36.down_proj", + "model.layers.72.mlp.experts.36.gate_proj", + "model.layers.72.mlp.experts.36.up_proj", + "model.layers.72.mlp.experts.37.down_proj", + "model.layers.72.mlp.experts.37.gate_proj", + "model.layers.72.mlp.experts.37.up_proj", + "model.layers.72.mlp.experts.38.down_proj", + "model.layers.72.mlp.experts.38.gate_proj", + "model.layers.72.mlp.experts.38.up_proj", + "model.layers.72.mlp.experts.39.down_proj", + "model.layers.72.mlp.experts.39.gate_proj", + "model.layers.72.mlp.experts.39.up_proj", + "model.layers.72.mlp.experts.4.down_proj", + "model.layers.72.mlp.experts.4.gate_proj", + "model.layers.72.mlp.experts.4.up_proj", + "model.layers.72.mlp.experts.40.down_proj", + "model.layers.72.mlp.experts.40.gate_proj", + "model.layers.72.mlp.experts.40.up_proj", + "model.layers.72.mlp.experts.41.down_proj", + "model.layers.72.mlp.experts.41.gate_proj", + "model.layers.72.mlp.experts.41.up_proj", + "model.layers.72.mlp.experts.42.down_proj", + "model.layers.72.mlp.experts.42.gate_proj", + "model.layers.72.mlp.experts.42.up_proj", + "model.layers.72.mlp.experts.43.down_proj", + "model.layers.72.mlp.experts.43.gate_proj", + "model.layers.72.mlp.experts.43.up_proj", + "model.layers.72.mlp.experts.44.down_proj", + "model.layers.72.mlp.experts.44.gate_proj", + "model.layers.72.mlp.experts.44.up_proj", + "model.layers.72.mlp.experts.45.down_proj", + "model.layers.72.mlp.experts.45.gate_proj", + "model.layers.72.mlp.experts.45.up_proj", + "model.layers.72.mlp.experts.46.down_proj", + "model.layers.72.mlp.experts.46.gate_proj", + "model.layers.72.mlp.experts.46.up_proj", + "model.layers.72.mlp.experts.47.down_proj", + "model.layers.72.mlp.experts.47.gate_proj", + "model.layers.72.mlp.experts.47.up_proj", + "model.layers.72.mlp.experts.48.down_proj", + "model.layers.72.mlp.experts.48.gate_proj", + "model.layers.72.mlp.experts.48.up_proj", + "model.layers.72.mlp.experts.49.down_proj", + "model.layers.72.mlp.experts.49.gate_proj", + "model.layers.72.mlp.experts.49.up_proj", + "model.layers.72.mlp.experts.5.down_proj", + "model.layers.72.mlp.experts.5.gate_proj", + "model.layers.72.mlp.experts.5.up_proj", + "model.layers.72.mlp.experts.50.down_proj", + "model.layers.72.mlp.experts.50.gate_proj", + "model.layers.72.mlp.experts.50.up_proj", + "model.layers.72.mlp.experts.51.down_proj", + "model.layers.72.mlp.experts.51.gate_proj", + "model.layers.72.mlp.experts.51.up_proj", + "model.layers.72.mlp.experts.52.down_proj", + "model.layers.72.mlp.experts.52.gate_proj", + "model.layers.72.mlp.experts.52.up_proj", + "model.layers.72.mlp.experts.53.down_proj", + "model.layers.72.mlp.experts.53.gate_proj", + "model.layers.72.mlp.experts.53.up_proj", + "model.layers.72.mlp.experts.54.down_proj", + "model.layers.72.mlp.experts.54.gate_proj", + "model.layers.72.mlp.experts.54.up_proj", + "model.layers.72.mlp.experts.55.down_proj", + "model.layers.72.mlp.experts.55.gate_proj", + "model.layers.72.mlp.experts.55.up_proj", + "model.layers.72.mlp.experts.56.down_proj", + "model.layers.72.mlp.experts.56.gate_proj", + "model.layers.72.mlp.experts.56.up_proj", + "model.layers.72.mlp.experts.57.down_proj", + "model.layers.72.mlp.experts.57.gate_proj", + "model.layers.72.mlp.experts.57.up_proj", + "model.layers.72.mlp.experts.58.down_proj", + "model.layers.72.mlp.experts.58.gate_proj", + "model.layers.72.mlp.experts.58.up_proj", + "model.layers.72.mlp.experts.59.down_proj", + "model.layers.72.mlp.experts.59.gate_proj", + "model.layers.72.mlp.experts.59.up_proj", + "model.layers.72.mlp.experts.6.down_proj", + "model.layers.72.mlp.experts.6.gate_proj", + "model.layers.72.mlp.experts.6.up_proj", + "model.layers.72.mlp.experts.60.down_proj", + "model.layers.72.mlp.experts.60.gate_proj", + "model.layers.72.mlp.experts.60.up_proj", + "model.layers.72.mlp.experts.61.down_proj", + "model.layers.72.mlp.experts.61.gate_proj", + "model.layers.72.mlp.experts.61.up_proj", + "model.layers.72.mlp.experts.62.down_proj", + "model.layers.72.mlp.experts.62.gate_proj", + "model.layers.72.mlp.experts.62.up_proj", + "model.layers.72.mlp.experts.63.down_proj", + "model.layers.72.mlp.experts.63.gate_proj", + "model.layers.72.mlp.experts.63.up_proj", + "model.layers.72.mlp.experts.64.down_proj", + "model.layers.72.mlp.experts.64.gate_proj", + "model.layers.72.mlp.experts.64.up_proj", + "model.layers.72.mlp.experts.65.down_proj", + "model.layers.72.mlp.experts.65.gate_proj", + "model.layers.72.mlp.experts.65.up_proj", + "model.layers.72.mlp.experts.66.down_proj", + "model.layers.72.mlp.experts.66.gate_proj", + "model.layers.72.mlp.experts.66.up_proj", + "model.layers.72.mlp.experts.67.down_proj", + "model.layers.72.mlp.experts.67.gate_proj", + "model.layers.72.mlp.experts.67.up_proj", + "model.layers.72.mlp.experts.68.down_proj", + "model.layers.72.mlp.experts.68.gate_proj", + "model.layers.72.mlp.experts.68.up_proj", + "model.layers.72.mlp.experts.69.down_proj", + "model.layers.72.mlp.experts.69.gate_proj", + "model.layers.72.mlp.experts.69.up_proj", + "model.layers.72.mlp.experts.7.down_proj", + "model.layers.72.mlp.experts.7.gate_proj", + "model.layers.72.mlp.experts.7.up_proj", + "model.layers.72.mlp.experts.70.down_proj", + "model.layers.72.mlp.experts.70.gate_proj", + "model.layers.72.mlp.experts.70.up_proj", + "model.layers.72.mlp.experts.71.down_proj", + "model.layers.72.mlp.experts.71.gate_proj", + "model.layers.72.mlp.experts.71.up_proj", + "model.layers.72.mlp.experts.72.down_proj", + "model.layers.72.mlp.experts.72.gate_proj", + "model.layers.72.mlp.experts.72.up_proj", + "model.layers.72.mlp.experts.73.down_proj", + "model.layers.72.mlp.experts.73.gate_proj", + "model.layers.72.mlp.experts.73.up_proj", + "model.layers.72.mlp.experts.74.down_proj", + "model.layers.72.mlp.experts.74.gate_proj", + "model.layers.72.mlp.experts.74.up_proj", + "model.layers.72.mlp.experts.75.down_proj", + "model.layers.72.mlp.experts.75.gate_proj", + "model.layers.72.mlp.experts.75.up_proj", + "model.layers.72.mlp.experts.76.down_proj", + "model.layers.72.mlp.experts.76.gate_proj", + "model.layers.72.mlp.experts.76.up_proj", + "model.layers.72.mlp.experts.77.down_proj", + "model.layers.72.mlp.experts.77.gate_proj", + "model.layers.72.mlp.experts.77.up_proj", + "model.layers.72.mlp.experts.78.down_proj", + "model.layers.72.mlp.experts.78.gate_proj", + "model.layers.72.mlp.experts.78.up_proj", + "model.layers.72.mlp.experts.79.down_proj", + "model.layers.72.mlp.experts.79.gate_proj", + "model.layers.72.mlp.experts.79.up_proj", + "model.layers.72.mlp.experts.8.down_proj", + "model.layers.72.mlp.experts.8.gate_proj", + "model.layers.72.mlp.experts.8.up_proj", + "model.layers.72.mlp.experts.80.down_proj", + "model.layers.72.mlp.experts.80.gate_proj", + "model.layers.72.mlp.experts.80.up_proj", + "model.layers.72.mlp.experts.81.down_proj", + "model.layers.72.mlp.experts.81.gate_proj", + "model.layers.72.mlp.experts.81.up_proj", + "model.layers.72.mlp.experts.82.down_proj", + "model.layers.72.mlp.experts.82.gate_proj", + "model.layers.72.mlp.experts.82.up_proj", + "model.layers.72.mlp.experts.83.down_proj", + "model.layers.72.mlp.experts.83.gate_proj", + "model.layers.72.mlp.experts.83.up_proj", + "model.layers.72.mlp.experts.84.down_proj", + "model.layers.72.mlp.experts.84.gate_proj", + "model.layers.72.mlp.experts.84.up_proj", + "model.layers.72.mlp.experts.85.down_proj", + "model.layers.72.mlp.experts.85.gate_proj", + "model.layers.72.mlp.experts.85.up_proj", + "model.layers.72.mlp.experts.86.down_proj", + "model.layers.72.mlp.experts.86.gate_proj", + "model.layers.72.mlp.experts.86.up_proj", + "model.layers.72.mlp.experts.87.down_proj", + "model.layers.72.mlp.experts.87.gate_proj", + "model.layers.72.mlp.experts.87.up_proj", + "model.layers.72.mlp.experts.88.down_proj", + "model.layers.72.mlp.experts.88.gate_proj", + "model.layers.72.mlp.experts.88.up_proj", + "model.layers.72.mlp.experts.89.down_proj", + "model.layers.72.mlp.experts.89.gate_proj", + "model.layers.72.mlp.experts.89.up_proj", + "model.layers.72.mlp.experts.9.down_proj", + "model.layers.72.mlp.experts.9.gate_proj", + "model.layers.72.mlp.experts.9.up_proj", + "model.layers.72.mlp.experts.90.down_proj", + "model.layers.72.mlp.experts.90.gate_proj", + "model.layers.72.mlp.experts.90.up_proj", + "model.layers.72.mlp.experts.91.down_proj", + "model.layers.72.mlp.experts.91.gate_proj", + "model.layers.72.mlp.experts.91.up_proj", + "model.layers.72.mlp.experts.92.down_proj", + "model.layers.72.mlp.experts.92.gate_proj", + "model.layers.72.mlp.experts.92.up_proj", + "model.layers.72.mlp.experts.93.down_proj", + "model.layers.72.mlp.experts.93.gate_proj", + "model.layers.72.mlp.experts.93.up_proj", + "model.layers.72.mlp.experts.94.down_proj", + "model.layers.72.mlp.experts.94.gate_proj", + "model.layers.72.mlp.experts.94.up_proj", + "model.layers.72.mlp.experts.95.down_proj", + "model.layers.72.mlp.experts.95.gate_proj", + "model.layers.72.mlp.experts.95.up_proj", + "model.layers.72.mlp.experts.96.down_proj", + "model.layers.72.mlp.experts.96.gate_proj", + "model.layers.72.mlp.experts.96.up_proj", + "model.layers.72.mlp.experts.97.down_proj", + "model.layers.72.mlp.experts.97.gate_proj", + "model.layers.72.mlp.experts.97.up_proj", + "model.layers.72.mlp.experts.98.down_proj", + "model.layers.72.mlp.experts.98.gate_proj", + "model.layers.72.mlp.experts.98.up_proj", + "model.layers.72.mlp.experts.99.down_proj", + "model.layers.72.mlp.experts.99.gate_proj", + "model.layers.72.mlp.experts.99.up_proj", + "model.layers.73.mlp.experts.0.down_proj", + "model.layers.73.mlp.experts.0.gate_proj", + "model.layers.73.mlp.experts.0.up_proj", + "model.layers.73.mlp.experts.1.down_proj", + "model.layers.73.mlp.experts.1.gate_proj", + "model.layers.73.mlp.experts.1.up_proj", + "model.layers.73.mlp.experts.10.down_proj", + "model.layers.73.mlp.experts.10.gate_proj", + "model.layers.73.mlp.experts.10.up_proj", + "model.layers.73.mlp.experts.100.down_proj", + "model.layers.73.mlp.experts.100.gate_proj", + "model.layers.73.mlp.experts.100.up_proj", + "model.layers.73.mlp.experts.101.down_proj", + "model.layers.73.mlp.experts.101.gate_proj", + "model.layers.73.mlp.experts.101.up_proj", + "model.layers.73.mlp.experts.102.down_proj", + "model.layers.73.mlp.experts.102.gate_proj", + "model.layers.73.mlp.experts.102.up_proj", + "model.layers.73.mlp.experts.103.down_proj", + "model.layers.73.mlp.experts.103.gate_proj", + "model.layers.73.mlp.experts.103.up_proj", + "model.layers.73.mlp.experts.104.down_proj", + "model.layers.73.mlp.experts.104.gate_proj", + "model.layers.73.mlp.experts.104.up_proj", + "model.layers.73.mlp.experts.105.down_proj", + "model.layers.73.mlp.experts.105.gate_proj", + "model.layers.73.mlp.experts.105.up_proj", + "model.layers.73.mlp.experts.106.down_proj", + "model.layers.73.mlp.experts.106.gate_proj", + "model.layers.73.mlp.experts.106.up_proj", + "model.layers.73.mlp.experts.107.down_proj", + "model.layers.73.mlp.experts.107.gate_proj", + "model.layers.73.mlp.experts.107.up_proj", + "model.layers.73.mlp.experts.108.down_proj", + "model.layers.73.mlp.experts.108.gate_proj", + "model.layers.73.mlp.experts.108.up_proj", + "model.layers.73.mlp.experts.109.down_proj", + "model.layers.73.mlp.experts.109.gate_proj", + "model.layers.73.mlp.experts.109.up_proj", + "model.layers.73.mlp.experts.11.down_proj", + "model.layers.73.mlp.experts.11.gate_proj", + "model.layers.73.mlp.experts.11.up_proj", + "model.layers.73.mlp.experts.110.down_proj", + "model.layers.73.mlp.experts.110.gate_proj", + "model.layers.73.mlp.experts.110.up_proj", + "model.layers.73.mlp.experts.111.down_proj", + "model.layers.73.mlp.experts.111.gate_proj", + "model.layers.73.mlp.experts.111.up_proj", + "model.layers.73.mlp.experts.112.down_proj", + "model.layers.73.mlp.experts.112.gate_proj", + "model.layers.73.mlp.experts.112.up_proj", + "model.layers.73.mlp.experts.113.down_proj", + "model.layers.73.mlp.experts.113.gate_proj", + "model.layers.73.mlp.experts.113.up_proj", + "model.layers.73.mlp.experts.114.down_proj", + "model.layers.73.mlp.experts.114.gate_proj", + "model.layers.73.mlp.experts.114.up_proj", + "model.layers.73.mlp.experts.115.down_proj", + "model.layers.73.mlp.experts.115.gate_proj", + "model.layers.73.mlp.experts.115.up_proj", + "model.layers.73.mlp.experts.116.down_proj", + "model.layers.73.mlp.experts.116.gate_proj", + "model.layers.73.mlp.experts.116.up_proj", + "model.layers.73.mlp.experts.117.down_proj", + "model.layers.73.mlp.experts.117.gate_proj", + "model.layers.73.mlp.experts.117.up_proj", + "model.layers.73.mlp.experts.118.down_proj", + "model.layers.73.mlp.experts.118.gate_proj", + "model.layers.73.mlp.experts.118.up_proj", + "model.layers.73.mlp.experts.119.down_proj", + "model.layers.73.mlp.experts.119.gate_proj", + "model.layers.73.mlp.experts.119.up_proj", + "model.layers.73.mlp.experts.12.down_proj", + "model.layers.73.mlp.experts.12.gate_proj", + "model.layers.73.mlp.experts.12.up_proj", + "model.layers.73.mlp.experts.120.down_proj", + "model.layers.73.mlp.experts.120.gate_proj", + "model.layers.73.mlp.experts.120.up_proj", + "model.layers.73.mlp.experts.121.down_proj", + "model.layers.73.mlp.experts.121.gate_proj", + "model.layers.73.mlp.experts.121.up_proj", + "model.layers.73.mlp.experts.122.down_proj", + "model.layers.73.mlp.experts.122.gate_proj", + "model.layers.73.mlp.experts.122.up_proj", + "model.layers.73.mlp.experts.123.down_proj", + "model.layers.73.mlp.experts.123.gate_proj", + "model.layers.73.mlp.experts.123.up_proj", + "model.layers.73.mlp.experts.124.down_proj", + "model.layers.73.mlp.experts.124.gate_proj", + "model.layers.73.mlp.experts.124.up_proj", + "model.layers.73.mlp.experts.125.down_proj", + "model.layers.73.mlp.experts.125.gate_proj", + "model.layers.73.mlp.experts.125.up_proj", + "model.layers.73.mlp.experts.126.down_proj", + "model.layers.73.mlp.experts.126.gate_proj", + "model.layers.73.mlp.experts.126.up_proj", + "model.layers.73.mlp.experts.127.down_proj", + "model.layers.73.mlp.experts.127.gate_proj", + "model.layers.73.mlp.experts.127.up_proj", + "model.layers.73.mlp.experts.128.down_proj", + "model.layers.73.mlp.experts.128.gate_proj", + "model.layers.73.mlp.experts.128.up_proj", + "model.layers.73.mlp.experts.129.down_proj", + "model.layers.73.mlp.experts.129.gate_proj", + "model.layers.73.mlp.experts.129.up_proj", + "model.layers.73.mlp.experts.13.down_proj", + "model.layers.73.mlp.experts.13.gate_proj", + "model.layers.73.mlp.experts.13.up_proj", + "model.layers.73.mlp.experts.130.down_proj", + "model.layers.73.mlp.experts.130.gate_proj", + "model.layers.73.mlp.experts.130.up_proj", + "model.layers.73.mlp.experts.131.down_proj", + "model.layers.73.mlp.experts.131.gate_proj", + "model.layers.73.mlp.experts.131.up_proj", + "model.layers.73.mlp.experts.132.down_proj", + "model.layers.73.mlp.experts.132.gate_proj", + "model.layers.73.mlp.experts.132.up_proj", + "model.layers.73.mlp.experts.133.down_proj", + "model.layers.73.mlp.experts.133.gate_proj", + "model.layers.73.mlp.experts.133.up_proj", + "model.layers.73.mlp.experts.134.down_proj", + "model.layers.73.mlp.experts.134.gate_proj", + "model.layers.73.mlp.experts.134.up_proj", + "model.layers.73.mlp.experts.135.down_proj", + "model.layers.73.mlp.experts.135.gate_proj", + "model.layers.73.mlp.experts.135.up_proj", + "model.layers.73.mlp.experts.136.down_proj", + "model.layers.73.mlp.experts.136.gate_proj", + "model.layers.73.mlp.experts.136.up_proj", + "model.layers.73.mlp.experts.137.down_proj", + "model.layers.73.mlp.experts.137.gate_proj", + "model.layers.73.mlp.experts.137.up_proj", + "model.layers.73.mlp.experts.138.down_proj", + "model.layers.73.mlp.experts.138.gate_proj", + "model.layers.73.mlp.experts.138.up_proj", + "model.layers.73.mlp.experts.139.down_proj", + "model.layers.73.mlp.experts.139.gate_proj", + "model.layers.73.mlp.experts.139.up_proj", + "model.layers.73.mlp.experts.14.down_proj", + "model.layers.73.mlp.experts.14.gate_proj", + "model.layers.73.mlp.experts.14.up_proj", + "model.layers.73.mlp.experts.140.down_proj", + "model.layers.73.mlp.experts.140.gate_proj", + "model.layers.73.mlp.experts.140.up_proj", + "model.layers.73.mlp.experts.141.down_proj", + "model.layers.73.mlp.experts.141.gate_proj", + "model.layers.73.mlp.experts.141.up_proj", + "model.layers.73.mlp.experts.142.down_proj", + "model.layers.73.mlp.experts.142.gate_proj", + "model.layers.73.mlp.experts.142.up_proj", + "model.layers.73.mlp.experts.143.down_proj", + "model.layers.73.mlp.experts.143.gate_proj", + "model.layers.73.mlp.experts.143.up_proj", + "model.layers.73.mlp.experts.144.down_proj", + "model.layers.73.mlp.experts.144.gate_proj", + "model.layers.73.mlp.experts.144.up_proj", + "model.layers.73.mlp.experts.145.down_proj", + "model.layers.73.mlp.experts.145.gate_proj", + "model.layers.73.mlp.experts.145.up_proj", + "model.layers.73.mlp.experts.146.down_proj", + "model.layers.73.mlp.experts.146.gate_proj", + "model.layers.73.mlp.experts.146.up_proj", + "model.layers.73.mlp.experts.147.down_proj", + "model.layers.73.mlp.experts.147.gate_proj", + "model.layers.73.mlp.experts.147.up_proj", + "model.layers.73.mlp.experts.148.down_proj", + "model.layers.73.mlp.experts.148.gate_proj", + "model.layers.73.mlp.experts.148.up_proj", + "model.layers.73.mlp.experts.149.down_proj", + "model.layers.73.mlp.experts.149.gate_proj", + "model.layers.73.mlp.experts.149.up_proj", + "model.layers.73.mlp.experts.15.down_proj", + "model.layers.73.mlp.experts.15.gate_proj", + "model.layers.73.mlp.experts.15.up_proj", + "model.layers.73.mlp.experts.150.down_proj", + "model.layers.73.mlp.experts.150.gate_proj", + "model.layers.73.mlp.experts.150.up_proj", + "model.layers.73.mlp.experts.151.down_proj", + "model.layers.73.mlp.experts.151.gate_proj", + "model.layers.73.mlp.experts.151.up_proj", + "model.layers.73.mlp.experts.152.down_proj", + "model.layers.73.mlp.experts.152.gate_proj", + "model.layers.73.mlp.experts.152.up_proj", + "model.layers.73.mlp.experts.153.down_proj", + "model.layers.73.mlp.experts.153.gate_proj", + "model.layers.73.mlp.experts.153.up_proj", + "model.layers.73.mlp.experts.154.down_proj", + "model.layers.73.mlp.experts.154.gate_proj", + "model.layers.73.mlp.experts.154.up_proj", + "model.layers.73.mlp.experts.155.down_proj", + "model.layers.73.mlp.experts.155.gate_proj", + "model.layers.73.mlp.experts.155.up_proj", + "model.layers.73.mlp.experts.156.down_proj", + "model.layers.73.mlp.experts.156.gate_proj", + "model.layers.73.mlp.experts.156.up_proj", + "model.layers.73.mlp.experts.157.down_proj", + "model.layers.73.mlp.experts.157.gate_proj", + "model.layers.73.mlp.experts.157.up_proj", + "model.layers.73.mlp.experts.158.down_proj", + "model.layers.73.mlp.experts.158.gate_proj", + "model.layers.73.mlp.experts.158.up_proj", + "model.layers.73.mlp.experts.159.down_proj", + "model.layers.73.mlp.experts.159.gate_proj", + "model.layers.73.mlp.experts.159.up_proj", + "model.layers.73.mlp.experts.16.down_proj", + "model.layers.73.mlp.experts.16.gate_proj", + "model.layers.73.mlp.experts.16.up_proj", + "model.layers.73.mlp.experts.160.down_proj", + "model.layers.73.mlp.experts.160.gate_proj", + "model.layers.73.mlp.experts.160.up_proj", + "model.layers.73.mlp.experts.161.down_proj", + "model.layers.73.mlp.experts.161.gate_proj", + "model.layers.73.mlp.experts.161.up_proj", + "model.layers.73.mlp.experts.162.down_proj", + "model.layers.73.mlp.experts.162.gate_proj", + "model.layers.73.mlp.experts.162.up_proj", + "model.layers.73.mlp.experts.163.down_proj", + "model.layers.73.mlp.experts.163.gate_proj", + "model.layers.73.mlp.experts.163.up_proj", + "model.layers.73.mlp.experts.164.down_proj", + "model.layers.73.mlp.experts.164.gate_proj", + "model.layers.73.mlp.experts.164.up_proj", + "model.layers.73.mlp.experts.165.down_proj", + "model.layers.73.mlp.experts.165.gate_proj", + "model.layers.73.mlp.experts.165.up_proj", + "model.layers.73.mlp.experts.166.down_proj", + "model.layers.73.mlp.experts.166.gate_proj", + "model.layers.73.mlp.experts.166.up_proj", + "model.layers.73.mlp.experts.167.down_proj", + "model.layers.73.mlp.experts.167.gate_proj", + "model.layers.73.mlp.experts.167.up_proj", + "model.layers.73.mlp.experts.168.down_proj", + "model.layers.73.mlp.experts.168.gate_proj", + "model.layers.73.mlp.experts.168.up_proj", + "model.layers.73.mlp.experts.169.down_proj", + "model.layers.73.mlp.experts.169.gate_proj", + "model.layers.73.mlp.experts.169.up_proj", + "model.layers.73.mlp.experts.17.down_proj", + "model.layers.73.mlp.experts.17.gate_proj", + "model.layers.73.mlp.experts.17.up_proj", + "model.layers.73.mlp.experts.170.down_proj", + "model.layers.73.mlp.experts.170.gate_proj", + "model.layers.73.mlp.experts.170.up_proj", + "model.layers.73.mlp.experts.171.down_proj", + "model.layers.73.mlp.experts.171.gate_proj", + "model.layers.73.mlp.experts.171.up_proj", + "model.layers.73.mlp.experts.172.down_proj", + "model.layers.73.mlp.experts.172.gate_proj", + "model.layers.73.mlp.experts.172.up_proj", + "model.layers.73.mlp.experts.173.down_proj", + "model.layers.73.mlp.experts.173.gate_proj", + "model.layers.73.mlp.experts.173.up_proj", + "model.layers.73.mlp.experts.174.down_proj", + "model.layers.73.mlp.experts.174.gate_proj", + "model.layers.73.mlp.experts.174.up_proj", + "model.layers.73.mlp.experts.175.down_proj", + "model.layers.73.mlp.experts.175.gate_proj", + "model.layers.73.mlp.experts.175.up_proj", + "model.layers.73.mlp.experts.176.down_proj", + "model.layers.73.mlp.experts.176.gate_proj", + "model.layers.73.mlp.experts.176.up_proj", + "model.layers.73.mlp.experts.177.down_proj", + "model.layers.73.mlp.experts.177.gate_proj", + "model.layers.73.mlp.experts.177.up_proj", + "model.layers.73.mlp.experts.178.down_proj", + "model.layers.73.mlp.experts.178.gate_proj", + "model.layers.73.mlp.experts.178.up_proj", + "model.layers.73.mlp.experts.179.down_proj", + "model.layers.73.mlp.experts.179.gate_proj", + "model.layers.73.mlp.experts.179.up_proj", + "model.layers.73.mlp.experts.18.down_proj", + "model.layers.73.mlp.experts.18.gate_proj", + "model.layers.73.mlp.experts.18.up_proj", + "model.layers.73.mlp.experts.180.down_proj", + "model.layers.73.mlp.experts.180.gate_proj", + "model.layers.73.mlp.experts.180.up_proj", + "model.layers.73.mlp.experts.181.down_proj", + "model.layers.73.mlp.experts.181.gate_proj", + "model.layers.73.mlp.experts.181.up_proj", + "model.layers.73.mlp.experts.182.down_proj", + "model.layers.73.mlp.experts.182.gate_proj", + "model.layers.73.mlp.experts.182.up_proj", + "model.layers.73.mlp.experts.183.down_proj", + "model.layers.73.mlp.experts.183.gate_proj", + "model.layers.73.mlp.experts.183.up_proj", + "model.layers.73.mlp.experts.184.down_proj", + "model.layers.73.mlp.experts.184.gate_proj", + "model.layers.73.mlp.experts.184.up_proj", + "model.layers.73.mlp.experts.185.down_proj", + "model.layers.73.mlp.experts.185.gate_proj", + "model.layers.73.mlp.experts.185.up_proj", + "model.layers.73.mlp.experts.186.down_proj", + "model.layers.73.mlp.experts.186.gate_proj", + "model.layers.73.mlp.experts.186.up_proj", + "model.layers.73.mlp.experts.187.down_proj", + "model.layers.73.mlp.experts.187.gate_proj", + "model.layers.73.mlp.experts.187.up_proj", + "model.layers.73.mlp.experts.188.down_proj", + "model.layers.73.mlp.experts.188.gate_proj", + "model.layers.73.mlp.experts.188.up_proj", + "model.layers.73.mlp.experts.189.down_proj", + "model.layers.73.mlp.experts.189.gate_proj", + "model.layers.73.mlp.experts.189.up_proj", + "model.layers.73.mlp.experts.19.down_proj", + "model.layers.73.mlp.experts.19.gate_proj", + "model.layers.73.mlp.experts.19.up_proj", + "model.layers.73.mlp.experts.190.down_proj", + "model.layers.73.mlp.experts.190.gate_proj", + "model.layers.73.mlp.experts.190.up_proj", + "model.layers.73.mlp.experts.191.down_proj", + "model.layers.73.mlp.experts.191.gate_proj", + "model.layers.73.mlp.experts.191.up_proj", + "model.layers.73.mlp.experts.192.down_proj", + "model.layers.73.mlp.experts.192.gate_proj", + "model.layers.73.mlp.experts.192.up_proj", + "model.layers.73.mlp.experts.193.down_proj", + "model.layers.73.mlp.experts.193.gate_proj", + "model.layers.73.mlp.experts.193.up_proj", + "model.layers.73.mlp.experts.194.down_proj", + "model.layers.73.mlp.experts.194.gate_proj", + "model.layers.73.mlp.experts.194.up_proj", + "model.layers.73.mlp.experts.195.down_proj", + "model.layers.73.mlp.experts.195.gate_proj", + "model.layers.73.mlp.experts.195.up_proj", + "model.layers.73.mlp.experts.196.down_proj", + "model.layers.73.mlp.experts.196.gate_proj", + "model.layers.73.mlp.experts.196.up_proj", + "model.layers.73.mlp.experts.197.down_proj", + "model.layers.73.mlp.experts.197.gate_proj", + "model.layers.73.mlp.experts.197.up_proj", + "model.layers.73.mlp.experts.198.down_proj", + "model.layers.73.mlp.experts.198.gate_proj", + "model.layers.73.mlp.experts.198.up_proj", + "model.layers.73.mlp.experts.199.down_proj", + "model.layers.73.mlp.experts.199.gate_proj", + "model.layers.73.mlp.experts.199.up_proj", + "model.layers.73.mlp.experts.2.down_proj", + "model.layers.73.mlp.experts.2.gate_proj", + "model.layers.73.mlp.experts.2.up_proj", + "model.layers.73.mlp.experts.20.down_proj", + "model.layers.73.mlp.experts.20.gate_proj", + "model.layers.73.mlp.experts.20.up_proj", + "model.layers.73.mlp.experts.200.down_proj", + "model.layers.73.mlp.experts.200.gate_proj", + "model.layers.73.mlp.experts.200.up_proj", + "model.layers.73.mlp.experts.201.down_proj", + "model.layers.73.mlp.experts.201.gate_proj", + "model.layers.73.mlp.experts.201.up_proj", + "model.layers.73.mlp.experts.202.down_proj", + "model.layers.73.mlp.experts.202.gate_proj", + "model.layers.73.mlp.experts.202.up_proj", + "model.layers.73.mlp.experts.203.down_proj", + "model.layers.73.mlp.experts.203.gate_proj", + "model.layers.73.mlp.experts.203.up_proj", + "model.layers.73.mlp.experts.204.down_proj", + "model.layers.73.mlp.experts.204.gate_proj", + "model.layers.73.mlp.experts.204.up_proj", + "model.layers.73.mlp.experts.205.down_proj", + "model.layers.73.mlp.experts.205.gate_proj", + "model.layers.73.mlp.experts.205.up_proj", + "model.layers.73.mlp.experts.206.down_proj", + "model.layers.73.mlp.experts.206.gate_proj", + "model.layers.73.mlp.experts.206.up_proj", + "model.layers.73.mlp.experts.207.down_proj", + "model.layers.73.mlp.experts.207.gate_proj", + "model.layers.73.mlp.experts.207.up_proj", + "model.layers.73.mlp.experts.208.down_proj", + "model.layers.73.mlp.experts.208.gate_proj", + "model.layers.73.mlp.experts.208.up_proj", + "model.layers.73.mlp.experts.209.down_proj", + "model.layers.73.mlp.experts.209.gate_proj", + "model.layers.73.mlp.experts.209.up_proj", + "model.layers.73.mlp.experts.21.down_proj", + "model.layers.73.mlp.experts.21.gate_proj", + "model.layers.73.mlp.experts.21.up_proj", + "model.layers.73.mlp.experts.210.down_proj", + "model.layers.73.mlp.experts.210.gate_proj", + "model.layers.73.mlp.experts.210.up_proj", + "model.layers.73.mlp.experts.211.down_proj", + "model.layers.73.mlp.experts.211.gate_proj", + "model.layers.73.mlp.experts.211.up_proj", + "model.layers.73.mlp.experts.212.down_proj", + "model.layers.73.mlp.experts.212.gate_proj", + "model.layers.73.mlp.experts.212.up_proj", + "model.layers.73.mlp.experts.213.down_proj", + "model.layers.73.mlp.experts.213.gate_proj", + "model.layers.73.mlp.experts.213.up_proj", + "model.layers.73.mlp.experts.214.down_proj", + "model.layers.73.mlp.experts.214.gate_proj", + "model.layers.73.mlp.experts.214.up_proj", + "model.layers.73.mlp.experts.215.down_proj", + "model.layers.73.mlp.experts.215.gate_proj", + "model.layers.73.mlp.experts.215.up_proj", + "model.layers.73.mlp.experts.216.down_proj", + "model.layers.73.mlp.experts.216.gate_proj", + "model.layers.73.mlp.experts.216.up_proj", + "model.layers.73.mlp.experts.217.down_proj", + "model.layers.73.mlp.experts.217.gate_proj", + "model.layers.73.mlp.experts.217.up_proj", + "model.layers.73.mlp.experts.218.down_proj", + "model.layers.73.mlp.experts.218.gate_proj", + "model.layers.73.mlp.experts.218.up_proj", + "model.layers.73.mlp.experts.219.down_proj", + "model.layers.73.mlp.experts.219.gate_proj", + "model.layers.73.mlp.experts.219.up_proj", + "model.layers.73.mlp.experts.22.down_proj", + "model.layers.73.mlp.experts.22.gate_proj", + "model.layers.73.mlp.experts.22.up_proj", + "model.layers.73.mlp.experts.220.down_proj", + "model.layers.73.mlp.experts.220.gate_proj", + "model.layers.73.mlp.experts.220.up_proj", + "model.layers.73.mlp.experts.221.down_proj", + "model.layers.73.mlp.experts.221.gate_proj", + "model.layers.73.mlp.experts.221.up_proj", + "model.layers.73.mlp.experts.222.down_proj", + "model.layers.73.mlp.experts.222.gate_proj", + "model.layers.73.mlp.experts.222.up_proj", + "model.layers.73.mlp.experts.223.down_proj", + "model.layers.73.mlp.experts.223.gate_proj", + "model.layers.73.mlp.experts.223.up_proj", + "model.layers.73.mlp.experts.224.down_proj", + "model.layers.73.mlp.experts.224.gate_proj", + "model.layers.73.mlp.experts.224.up_proj", + "model.layers.73.mlp.experts.225.down_proj", + "model.layers.73.mlp.experts.225.gate_proj", + "model.layers.73.mlp.experts.225.up_proj", + "model.layers.73.mlp.experts.226.down_proj", + "model.layers.73.mlp.experts.226.gate_proj", + "model.layers.73.mlp.experts.226.up_proj", + "model.layers.73.mlp.experts.227.down_proj", + "model.layers.73.mlp.experts.227.gate_proj", + "model.layers.73.mlp.experts.227.up_proj", + "model.layers.73.mlp.experts.228.down_proj", + "model.layers.73.mlp.experts.228.gate_proj", + "model.layers.73.mlp.experts.228.up_proj", + "model.layers.73.mlp.experts.229.down_proj", + "model.layers.73.mlp.experts.229.gate_proj", + "model.layers.73.mlp.experts.229.up_proj", + "model.layers.73.mlp.experts.23.down_proj", + "model.layers.73.mlp.experts.23.gate_proj", + "model.layers.73.mlp.experts.23.up_proj", + "model.layers.73.mlp.experts.230.down_proj", + "model.layers.73.mlp.experts.230.gate_proj", + "model.layers.73.mlp.experts.230.up_proj", + "model.layers.73.mlp.experts.231.down_proj", + "model.layers.73.mlp.experts.231.gate_proj", + "model.layers.73.mlp.experts.231.up_proj", + "model.layers.73.mlp.experts.232.down_proj", + "model.layers.73.mlp.experts.232.gate_proj", + "model.layers.73.mlp.experts.232.up_proj", + "model.layers.73.mlp.experts.233.down_proj", + "model.layers.73.mlp.experts.233.gate_proj", + "model.layers.73.mlp.experts.233.up_proj", + "model.layers.73.mlp.experts.234.down_proj", + "model.layers.73.mlp.experts.234.gate_proj", + "model.layers.73.mlp.experts.234.up_proj", + "model.layers.73.mlp.experts.235.down_proj", + "model.layers.73.mlp.experts.235.gate_proj", + "model.layers.73.mlp.experts.235.up_proj", + "model.layers.73.mlp.experts.236.down_proj", + "model.layers.73.mlp.experts.236.gate_proj", + "model.layers.73.mlp.experts.236.up_proj", + "model.layers.73.mlp.experts.237.down_proj", + "model.layers.73.mlp.experts.237.gate_proj", + "model.layers.73.mlp.experts.237.up_proj", + "model.layers.73.mlp.experts.238.down_proj", + "model.layers.73.mlp.experts.238.gate_proj", + "model.layers.73.mlp.experts.238.up_proj", + "model.layers.73.mlp.experts.239.down_proj", + "model.layers.73.mlp.experts.239.gate_proj", + "model.layers.73.mlp.experts.239.up_proj", + "model.layers.73.mlp.experts.24.down_proj", + "model.layers.73.mlp.experts.24.gate_proj", + "model.layers.73.mlp.experts.24.up_proj", + "model.layers.73.mlp.experts.240.down_proj", + "model.layers.73.mlp.experts.240.gate_proj", + "model.layers.73.mlp.experts.240.up_proj", + "model.layers.73.mlp.experts.241.down_proj", + "model.layers.73.mlp.experts.241.gate_proj", + "model.layers.73.mlp.experts.241.up_proj", + "model.layers.73.mlp.experts.242.down_proj", + "model.layers.73.mlp.experts.242.gate_proj", + "model.layers.73.mlp.experts.242.up_proj", + "model.layers.73.mlp.experts.243.down_proj", + "model.layers.73.mlp.experts.243.gate_proj", + "model.layers.73.mlp.experts.243.up_proj", + "model.layers.73.mlp.experts.244.down_proj", + "model.layers.73.mlp.experts.244.gate_proj", + "model.layers.73.mlp.experts.244.up_proj", + "model.layers.73.mlp.experts.245.down_proj", + "model.layers.73.mlp.experts.245.gate_proj", + "model.layers.73.mlp.experts.245.up_proj", + "model.layers.73.mlp.experts.246.down_proj", + "model.layers.73.mlp.experts.246.gate_proj", + "model.layers.73.mlp.experts.246.up_proj", + "model.layers.73.mlp.experts.247.down_proj", + "model.layers.73.mlp.experts.247.gate_proj", + "model.layers.73.mlp.experts.247.up_proj", + "model.layers.73.mlp.experts.248.down_proj", + "model.layers.73.mlp.experts.248.gate_proj", + "model.layers.73.mlp.experts.248.up_proj", + "model.layers.73.mlp.experts.249.down_proj", + "model.layers.73.mlp.experts.249.gate_proj", + "model.layers.73.mlp.experts.249.up_proj", + "model.layers.73.mlp.experts.25.down_proj", + "model.layers.73.mlp.experts.25.gate_proj", + "model.layers.73.mlp.experts.25.up_proj", + "model.layers.73.mlp.experts.250.down_proj", + "model.layers.73.mlp.experts.250.gate_proj", + "model.layers.73.mlp.experts.250.up_proj", + "model.layers.73.mlp.experts.251.down_proj", + "model.layers.73.mlp.experts.251.gate_proj", + "model.layers.73.mlp.experts.251.up_proj", + "model.layers.73.mlp.experts.252.down_proj", + "model.layers.73.mlp.experts.252.gate_proj", + "model.layers.73.mlp.experts.252.up_proj", + "model.layers.73.mlp.experts.253.down_proj", + "model.layers.73.mlp.experts.253.gate_proj", + "model.layers.73.mlp.experts.253.up_proj", + "model.layers.73.mlp.experts.254.down_proj", + "model.layers.73.mlp.experts.254.gate_proj", + "model.layers.73.mlp.experts.254.up_proj", + "model.layers.73.mlp.experts.255.down_proj", + "model.layers.73.mlp.experts.255.gate_proj", + "model.layers.73.mlp.experts.255.up_proj", + "model.layers.73.mlp.experts.26.down_proj", + "model.layers.73.mlp.experts.26.gate_proj", + "model.layers.73.mlp.experts.26.up_proj", + "model.layers.73.mlp.experts.27.down_proj", + "model.layers.73.mlp.experts.27.gate_proj", + "model.layers.73.mlp.experts.27.up_proj", + "model.layers.73.mlp.experts.28.down_proj", + "model.layers.73.mlp.experts.28.gate_proj", + "model.layers.73.mlp.experts.28.up_proj", + "model.layers.73.mlp.experts.29.down_proj", + "model.layers.73.mlp.experts.29.gate_proj", + "model.layers.73.mlp.experts.29.up_proj", + "model.layers.73.mlp.experts.3.down_proj", + "model.layers.73.mlp.experts.3.gate_proj", + "model.layers.73.mlp.experts.3.up_proj", + "model.layers.73.mlp.experts.30.down_proj", + "model.layers.73.mlp.experts.30.gate_proj", + "model.layers.73.mlp.experts.30.up_proj", + "model.layers.73.mlp.experts.31.down_proj", + "model.layers.73.mlp.experts.31.gate_proj", + "model.layers.73.mlp.experts.31.up_proj", + "model.layers.73.mlp.experts.32.down_proj", + "model.layers.73.mlp.experts.32.gate_proj", + "model.layers.73.mlp.experts.32.up_proj", + "model.layers.73.mlp.experts.33.down_proj", + "model.layers.73.mlp.experts.33.gate_proj", + "model.layers.73.mlp.experts.33.up_proj", + "model.layers.73.mlp.experts.34.down_proj", + "model.layers.73.mlp.experts.34.gate_proj", + "model.layers.73.mlp.experts.34.up_proj", + "model.layers.73.mlp.experts.35.down_proj", + "model.layers.73.mlp.experts.35.gate_proj", + "model.layers.73.mlp.experts.35.up_proj", + "model.layers.73.mlp.experts.36.down_proj", + "model.layers.73.mlp.experts.36.gate_proj", + "model.layers.73.mlp.experts.36.up_proj", + "model.layers.73.mlp.experts.37.down_proj", + "model.layers.73.mlp.experts.37.gate_proj", + "model.layers.73.mlp.experts.37.up_proj", + "model.layers.73.mlp.experts.38.down_proj", + "model.layers.73.mlp.experts.38.gate_proj", + "model.layers.73.mlp.experts.38.up_proj", + "model.layers.73.mlp.experts.39.down_proj", + "model.layers.73.mlp.experts.39.gate_proj", + "model.layers.73.mlp.experts.39.up_proj", + "model.layers.73.mlp.experts.4.down_proj", + "model.layers.73.mlp.experts.4.gate_proj", + "model.layers.73.mlp.experts.4.up_proj", + "model.layers.73.mlp.experts.40.down_proj", + "model.layers.73.mlp.experts.40.gate_proj", + "model.layers.73.mlp.experts.40.up_proj", + "model.layers.73.mlp.experts.41.down_proj", + "model.layers.73.mlp.experts.41.gate_proj", + "model.layers.73.mlp.experts.41.up_proj", + "model.layers.73.mlp.experts.42.down_proj", + "model.layers.73.mlp.experts.42.gate_proj", + "model.layers.73.mlp.experts.42.up_proj", + "model.layers.73.mlp.experts.43.down_proj", + "model.layers.73.mlp.experts.43.gate_proj", + "model.layers.73.mlp.experts.43.up_proj", + "model.layers.73.mlp.experts.44.down_proj", + "model.layers.73.mlp.experts.44.gate_proj", + "model.layers.73.mlp.experts.44.up_proj", + "model.layers.73.mlp.experts.45.down_proj", + "model.layers.73.mlp.experts.45.gate_proj", + "model.layers.73.mlp.experts.45.up_proj", + "model.layers.73.mlp.experts.46.down_proj", + "model.layers.73.mlp.experts.46.gate_proj", + "model.layers.73.mlp.experts.46.up_proj", + "model.layers.73.mlp.experts.47.down_proj", + "model.layers.73.mlp.experts.47.gate_proj", + "model.layers.73.mlp.experts.47.up_proj", + "model.layers.73.mlp.experts.48.down_proj", + "model.layers.73.mlp.experts.48.gate_proj", + "model.layers.73.mlp.experts.48.up_proj", + "model.layers.73.mlp.experts.49.down_proj", + "model.layers.73.mlp.experts.49.gate_proj", + "model.layers.73.mlp.experts.49.up_proj", + "model.layers.73.mlp.experts.5.down_proj", + "model.layers.73.mlp.experts.5.gate_proj", + "model.layers.73.mlp.experts.5.up_proj", + "model.layers.73.mlp.experts.50.down_proj", + "model.layers.73.mlp.experts.50.gate_proj", + "model.layers.73.mlp.experts.50.up_proj", + "model.layers.73.mlp.experts.51.down_proj", + "model.layers.73.mlp.experts.51.gate_proj", + "model.layers.73.mlp.experts.51.up_proj", + "model.layers.73.mlp.experts.52.down_proj", + "model.layers.73.mlp.experts.52.gate_proj", + "model.layers.73.mlp.experts.52.up_proj", + "model.layers.73.mlp.experts.53.down_proj", + "model.layers.73.mlp.experts.53.gate_proj", + "model.layers.73.mlp.experts.53.up_proj", + "model.layers.73.mlp.experts.54.down_proj", + "model.layers.73.mlp.experts.54.gate_proj", + "model.layers.73.mlp.experts.54.up_proj", + "model.layers.73.mlp.experts.55.down_proj", + "model.layers.73.mlp.experts.55.gate_proj", + "model.layers.73.mlp.experts.55.up_proj", + "model.layers.73.mlp.experts.56.down_proj", + "model.layers.73.mlp.experts.56.gate_proj", + "model.layers.73.mlp.experts.56.up_proj", + "model.layers.73.mlp.experts.57.down_proj", + "model.layers.73.mlp.experts.57.gate_proj", + "model.layers.73.mlp.experts.57.up_proj", + "model.layers.73.mlp.experts.58.down_proj", + "model.layers.73.mlp.experts.58.gate_proj", + "model.layers.73.mlp.experts.58.up_proj", + "model.layers.73.mlp.experts.59.down_proj", + "model.layers.73.mlp.experts.59.gate_proj", + "model.layers.73.mlp.experts.59.up_proj", + "model.layers.73.mlp.experts.6.down_proj", + "model.layers.73.mlp.experts.6.gate_proj", + "model.layers.73.mlp.experts.6.up_proj", + "model.layers.73.mlp.experts.60.down_proj", + "model.layers.73.mlp.experts.60.gate_proj", + "model.layers.73.mlp.experts.60.up_proj", + "model.layers.73.mlp.experts.61.down_proj", + "model.layers.73.mlp.experts.61.gate_proj", + "model.layers.73.mlp.experts.61.up_proj", + "model.layers.73.mlp.experts.62.down_proj", + "model.layers.73.mlp.experts.62.gate_proj", + "model.layers.73.mlp.experts.62.up_proj", + "model.layers.73.mlp.experts.63.down_proj", + "model.layers.73.mlp.experts.63.gate_proj", + "model.layers.73.mlp.experts.63.up_proj", + "model.layers.73.mlp.experts.64.down_proj", + "model.layers.73.mlp.experts.64.gate_proj", + "model.layers.73.mlp.experts.64.up_proj", + "model.layers.73.mlp.experts.65.down_proj", + "model.layers.73.mlp.experts.65.gate_proj", + "model.layers.73.mlp.experts.65.up_proj", + "model.layers.73.mlp.experts.66.down_proj", + "model.layers.73.mlp.experts.66.gate_proj", + "model.layers.73.mlp.experts.66.up_proj", + "model.layers.73.mlp.experts.67.down_proj", + "model.layers.73.mlp.experts.67.gate_proj", + "model.layers.73.mlp.experts.67.up_proj", + "model.layers.73.mlp.experts.68.down_proj", + "model.layers.73.mlp.experts.68.gate_proj", + "model.layers.73.mlp.experts.68.up_proj", + "model.layers.73.mlp.experts.69.down_proj", + "model.layers.73.mlp.experts.69.gate_proj", + "model.layers.73.mlp.experts.69.up_proj", + "model.layers.73.mlp.experts.7.down_proj", + "model.layers.73.mlp.experts.7.gate_proj", + "model.layers.73.mlp.experts.7.up_proj", + "model.layers.73.mlp.experts.70.down_proj", + "model.layers.73.mlp.experts.70.gate_proj", + "model.layers.73.mlp.experts.70.up_proj", + "model.layers.73.mlp.experts.71.down_proj", + "model.layers.73.mlp.experts.71.gate_proj", + "model.layers.73.mlp.experts.71.up_proj", + "model.layers.73.mlp.experts.72.down_proj", + "model.layers.73.mlp.experts.72.gate_proj", + "model.layers.73.mlp.experts.72.up_proj", + "model.layers.73.mlp.experts.73.down_proj", + "model.layers.73.mlp.experts.73.gate_proj", + "model.layers.73.mlp.experts.73.up_proj", + "model.layers.73.mlp.experts.74.down_proj", + "model.layers.73.mlp.experts.74.gate_proj", + "model.layers.73.mlp.experts.74.up_proj", + "model.layers.73.mlp.experts.75.down_proj", + "model.layers.73.mlp.experts.75.gate_proj", + "model.layers.73.mlp.experts.75.up_proj", + "model.layers.73.mlp.experts.76.down_proj", + "model.layers.73.mlp.experts.76.gate_proj", + "model.layers.73.mlp.experts.76.up_proj", + "model.layers.73.mlp.experts.77.down_proj", + "model.layers.73.mlp.experts.77.gate_proj", + "model.layers.73.mlp.experts.77.up_proj", + "model.layers.73.mlp.experts.78.down_proj", + "model.layers.73.mlp.experts.78.gate_proj", + "model.layers.73.mlp.experts.78.up_proj", + "model.layers.73.mlp.experts.79.down_proj", + "model.layers.73.mlp.experts.79.gate_proj", + "model.layers.73.mlp.experts.79.up_proj", + "model.layers.73.mlp.experts.8.down_proj", + "model.layers.73.mlp.experts.8.gate_proj", + "model.layers.73.mlp.experts.8.up_proj", + "model.layers.73.mlp.experts.80.down_proj", + "model.layers.73.mlp.experts.80.gate_proj", + "model.layers.73.mlp.experts.80.up_proj", + "model.layers.73.mlp.experts.81.down_proj", + "model.layers.73.mlp.experts.81.gate_proj", + "model.layers.73.mlp.experts.81.up_proj", + "model.layers.73.mlp.experts.82.down_proj", + "model.layers.73.mlp.experts.82.gate_proj", + "model.layers.73.mlp.experts.82.up_proj", + "model.layers.73.mlp.experts.83.down_proj", + "model.layers.73.mlp.experts.83.gate_proj", + "model.layers.73.mlp.experts.83.up_proj", + "model.layers.73.mlp.experts.84.down_proj", + "model.layers.73.mlp.experts.84.gate_proj", + "model.layers.73.mlp.experts.84.up_proj", + "model.layers.73.mlp.experts.85.down_proj", + "model.layers.73.mlp.experts.85.gate_proj", + "model.layers.73.mlp.experts.85.up_proj", + "model.layers.73.mlp.experts.86.down_proj", + "model.layers.73.mlp.experts.86.gate_proj", + "model.layers.73.mlp.experts.86.up_proj", + "model.layers.73.mlp.experts.87.down_proj", + "model.layers.73.mlp.experts.87.gate_proj", + "model.layers.73.mlp.experts.87.up_proj", + "model.layers.73.mlp.experts.88.down_proj", + "model.layers.73.mlp.experts.88.gate_proj", + "model.layers.73.mlp.experts.88.up_proj", + "model.layers.73.mlp.experts.89.down_proj", + "model.layers.73.mlp.experts.89.gate_proj", + "model.layers.73.mlp.experts.89.up_proj", + "model.layers.73.mlp.experts.9.down_proj", + "model.layers.73.mlp.experts.9.gate_proj", + "model.layers.73.mlp.experts.9.up_proj", + "model.layers.73.mlp.experts.90.down_proj", + "model.layers.73.mlp.experts.90.gate_proj", + "model.layers.73.mlp.experts.90.up_proj", + "model.layers.73.mlp.experts.91.down_proj", + "model.layers.73.mlp.experts.91.gate_proj", + "model.layers.73.mlp.experts.91.up_proj", + "model.layers.73.mlp.experts.92.down_proj", + "model.layers.73.mlp.experts.92.gate_proj", + "model.layers.73.mlp.experts.92.up_proj", + "model.layers.73.mlp.experts.93.down_proj", + "model.layers.73.mlp.experts.93.gate_proj", + "model.layers.73.mlp.experts.93.up_proj", + "model.layers.73.mlp.experts.94.down_proj", + "model.layers.73.mlp.experts.94.gate_proj", + "model.layers.73.mlp.experts.94.up_proj", + "model.layers.73.mlp.experts.95.down_proj", + "model.layers.73.mlp.experts.95.gate_proj", + "model.layers.73.mlp.experts.95.up_proj", + "model.layers.73.mlp.experts.96.down_proj", + "model.layers.73.mlp.experts.96.gate_proj", + "model.layers.73.mlp.experts.96.up_proj", + "model.layers.73.mlp.experts.97.down_proj", + "model.layers.73.mlp.experts.97.gate_proj", + "model.layers.73.mlp.experts.97.up_proj", + "model.layers.73.mlp.experts.98.down_proj", + "model.layers.73.mlp.experts.98.gate_proj", + "model.layers.73.mlp.experts.98.up_proj", + "model.layers.73.mlp.experts.99.down_proj", + "model.layers.73.mlp.experts.99.gate_proj", + "model.layers.73.mlp.experts.99.up_proj", + "model.layers.74.mlp.experts.0.down_proj", + "model.layers.74.mlp.experts.0.gate_proj", + "model.layers.74.mlp.experts.0.up_proj", + "model.layers.74.mlp.experts.1.down_proj", + "model.layers.74.mlp.experts.1.gate_proj", + "model.layers.74.mlp.experts.1.up_proj", + "model.layers.74.mlp.experts.10.down_proj", + "model.layers.74.mlp.experts.10.gate_proj", + "model.layers.74.mlp.experts.10.up_proj", + "model.layers.74.mlp.experts.100.down_proj", + "model.layers.74.mlp.experts.100.gate_proj", + "model.layers.74.mlp.experts.100.up_proj", + "model.layers.74.mlp.experts.101.down_proj", + "model.layers.74.mlp.experts.101.gate_proj", + "model.layers.74.mlp.experts.101.up_proj", + "model.layers.74.mlp.experts.102.down_proj", + "model.layers.74.mlp.experts.102.gate_proj", + "model.layers.74.mlp.experts.102.up_proj", + "model.layers.74.mlp.experts.103.down_proj", + "model.layers.74.mlp.experts.103.gate_proj", + "model.layers.74.mlp.experts.103.up_proj", + "model.layers.74.mlp.experts.104.down_proj", + "model.layers.74.mlp.experts.104.gate_proj", + "model.layers.74.mlp.experts.104.up_proj", + "model.layers.74.mlp.experts.105.down_proj", + "model.layers.74.mlp.experts.105.gate_proj", + "model.layers.74.mlp.experts.105.up_proj", + "model.layers.74.mlp.experts.106.down_proj", + "model.layers.74.mlp.experts.106.gate_proj", + "model.layers.74.mlp.experts.106.up_proj", + "model.layers.74.mlp.experts.107.down_proj", + "model.layers.74.mlp.experts.107.gate_proj", + "model.layers.74.mlp.experts.107.up_proj", + "model.layers.74.mlp.experts.108.down_proj", + "model.layers.74.mlp.experts.108.gate_proj", + "model.layers.74.mlp.experts.108.up_proj", + "model.layers.74.mlp.experts.109.down_proj", + "model.layers.74.mlp.experts.109.gate_proj", + "model.layers.74.mlp.experts.109.up_proj", + "model.layers.74.mlp.experts.11.down_proj", + "model.layers.74.mlp.experts.11.gate_proj", + "model.layers.74.mlp.experts.11.up_proj", + "model.layers.74.mlp.experts.110.down_proj", + "model.layers.74.mlp.experts.110.gate_proj", + "model.layers.74.mlp.experts.110.up_proj", + "model.layers.74.mlp.experts.111.down_proj", + "model.layers.74.mlp.experts.111.gate_proj", + "model.layers.74.mlp.experts.111.up_proj", + "model.layers.74.mlp.experts.112.down_proj", + "model.layers.74.mlp.experts.112.gate_proj", + "model.layers.74.mlp.experts.112.up_proj", + "model.layers.74.mlp.experts.113.down_proj", + "model.layers.74.mlp.experts.113.gate_proj", + "model.layers.74.mlp.experts.113.up_proj", + "model.layers.74.mlp.experts.114.down_proj", + "model.layers.74.mlp.experts.114.gate_proj", + "model.layers.74.mlp.experts.114.up_proj", + "model.layers.74.mlp.experts.115.down_proj", + "model.layers.74.mlp.experts.115.gate_proj", + "model.layers.74.mlp.experts.115.up_proj", + "model.layers.74.mlp.experts.116.down_proj", + "model.layers.74.mlp.experts.116.gate_proj", + "model.layers.74.mlp.experts.116.up_proj", + "model.layers.74.mlp.experts.117.down_proj", + "model.layers.74.mlp.experts.117.gate_proj", + "model.layers.74.mlp.experts.117.up_proj", + "model.layers.74.mlp.experts.118.down_proj", + "model.layers.74.mlp.experts.118.gate_proj", + "model.layers.74.mlp.experts.118.up_proj", + "model.layers.74.mlp.experts.119.down_proj", + "model.layers.74.mlp.experts.119.gate_proj", + "model.layers.74.mlp.experts.119.up_proj", + "model.layers.74.mlp.experts.12.down_proj", + "model.layers.74.mlp.experts.12.gate_proj", + "model.layers.74.mlp.experts.12.up_proj", + "model.layers.74.mlp.experts.120.down_proj", + "model.layers.74.mlp.experts.120.gate_proj", + "model.layers.74.mlp.experts.120.up_proj", + "model.layers.74.mlp.experts.121.down_proj", + "model.layers.74.mlp.experts.121.gate_proj", + "model.layers.74.mlp.experts.121.up_proj", + "model.layers.74.mlp.experts.122.down_proj", + "model.layers.74.mlp.experts.122.gate_proj", + "model.layers.74.mlp.experts.122.up_proj", + "model.layers.74.mlp.experts.123.down_proj", + "model.layers.74.mlp.experts.123.gate_proj", + "model.layers.74.mlp.experts.123.up_proj", + "model.layers.74.mlp.experts.124.down_proj", + "model.layers.74.mlp.experts.124.gate_proj", + "model.layers.74.mlp.experts.124.up_proj", + "model.layers.74.mlp.experts.125.down_proj", + "model.layers.74.mlp.experts.125.gate_proj", + "model.layers.74.mlp.experts.125.up_proj", + "model.layers.74.mlp.experts.126.down_proj", + "model.layers.74.mlp.experts.126.gate_proj", + "model.layers.74.mlp.experts.126.up_proj", + "model.layers.74.mlp.experts.127.down_proj", + "model.layers.74.mlp.experts.127.gate_proj", + "model.layers.74.mlp.experts.127.up_proj", + "model.layers.74.mlp.experts.128.down_proj", + "model.layers.74.mlp.experts.128.gate_proj", + "model.layers.74.mlp.experts.128.up_proj", + "model.layers.74.mlp.experts.129.down_proj", + "model.layers.74.mlp.experts.129.gate_proj", + "model.layers.74.mlp.experts.129.up_proj", + "model.layers.74.mlp.experts.13.down_proj", + "model.layers.74.mlp.experts.13.gate_proj", + "model.layers.74.mlp.experts.13.up_proj", + "model.layers.74.mlp.experts.130.down_proj", + "model.layers.74.mlp.experts.130.gate_proj", + "model.layers.74.mlp.experts.130.up_proj", + "model.layers.74.mlp.experts.131.down_proj", + "model.layers.74.mlp.experts.131.gate_proj", + "model.layers.74.mlp.experts.131.up_proj", + "model.layers.74.mlp.experts.132.down_proj", + "model.layers.74.mlp.experts.132.gate_proj", + "model.layers.74.mlp.experts.132.up_proj", + "model.layers.74.mlp.experts.133.down_proj", + "model.layers.74.mlp.experts.133.gate_proj", + "model.layers.74.mlp.experts.133.up_proj", + "model.layers.74.mlp.experts.134.down_proj", + "model.layers.74.mlp.experts.134.gate_proj", + "model.layers.74.mlp.experts.134.up_proj", + "model.layers.74.mlp.experts.135.down_proj", + "model.layers.74.mlp.experts.135.gate_proj", + "model.layers.74.mlp.experts.135.up_proj", + "model.layers.74.mlp.experts.136.down_proj", + "model.layers.74.mlp.experts.136.gate_proj", + "model.layers.74.mlp.experts.136.up_proj", + "model.layers.74.mlp.experts.137.down_proj", + "model.layers.74.mlp.experts.137.gate_proj", + "model.layers.74.mlp.experts.137.up_proj", + "model.layers.74.mlp.experts.138.down_proj", + "model.layers.74.mlp.experts.138.gate_proj", + "model.layers.74.mlp.experts.138.up_proj", + "model.layers.74.mlp.experts.139.down_proj", + "model.layers.74.mlp.experts.139.gate_proj", + "model.layers.74.mlp.experts.139.up_proj", + "model.layers.74.mlp.experts.14.down_proj", + "model.layers.74.mlp.experts.14.gate_proj", + "model.layers.74.mlp.experts.14.up_proj", + "model.layers.74.mlp.experts.140.down_proj", + "model.layers.74.mlp.experts.140.gate_proj", + "model.layers.74.mlp.experts.140.up_proj", + "model.layers.74.mlp.experts.141.down_proj", + "model.layers.74.mlp.experts.141.gate_proj", + "model.layers.74.mlp.experts.141.up_proj", + "model.layers.74.mlp.experts.142.down_proj", + "model.layers.74.mlp.experts.142.gate_proj", + "model.layers.74.mlp.experts.142.up_proj", + "model.layers.74.mlp.experts.143.down_proj", + "model.layers.74.mlp.experts.143.gate_proj", + "model.layers.74.mlp.experts.143.up_proj", + "model.layers.74.mlp.experts.144.down_proj", + "model.layers.74.mlp.experts.144.gate_proj", + "model.layers.74.mlp.experts.144.up_proj", + "model.layers.74.mlp.experts.145.down_proj", + "model.layers.74.mlp.experts.145.gate_proj", + "model.layers.74.mlp.experts.145.up_proj", + "model.layers.74.mlp.experts.146.down_proj", + "model.layers.74.mlp.experts.146.gate_proj", + "model.layers.74.mlp.experts.146.up_proj", + "model.layers.74.mlp.experts.147.down_proj", + "model.layers.74.mlp.experts.147.gate_proj", + "model.layers.74.mlp.experts.147.up_proj", + "model.layers.74.mlp.experts.148.down_proj", + "model.layers.74.mlp.experts.148.gate_proj", + "model.layers.74.mlp.experts.148.up_proj", + "model.layers.74.mlp.experts.149.down_proj", + "model.layers.74.mlp.experts.149.gate_proj", + "model.layers.74.mlp.experts.149.up_proj", + "model.layers.74.mlp.experts.15.down_proj", + "model.layers.74.mlp.experts.15.gate_proj", + "model.layers.74.mlp.experts.15.up_proj", + "model.layers.74.mlp.experts.150.down_proj", + "model.layers.74.mlp.experts.150.gate_proj", + "model.layers.74.mlp.experts.150.up_proj", + "model.layers.74.mlp.experts.151.down_proj", + "model.layers.74.mlp.experts.151.gate_proj", + "model.layers.74.mlp.experts.151.up_proj", + "model.layers.74.mlp.experts.152.down_proj", + "model.layers.74.mlp.experts.152.gate_proj", + "model.layers.74.mlp.experts.152.up_proj", + "model.layers.74.mlp.experts.153.down_proj", + "model.layers.74.mlp.experts.153.gate_proj", + "model.layers.74.mlp.experts.153.up_proj", + "model.layers.74.mlp.experts.154.down_proj", + "model.layers.74.mlp.experts.154.gate_proj", + "model.layers.74.mlp.experts.154.up_proj", + "model.layers.74.mlp.experts.155.down_proj", + "model.layers.74.mlp.experts.155.gate_proj", + "model.layers.74.mlp.experts.155.up_proj", + "model.layers.74.mlp.experts.156.down_proj", + "model.layers.74.mlp.experts.156.gate_proj", + "model.layers.74.mlp.experts.156.up_proj", + "model.layers.74.mlp.experts.157.down_proj", + "model.layers.74.mlp.experts.157.gate_proj", + "model.layers.74.mlp.experts.157.up_proj", + "model.layers.74.mlp.experts.158.down_proj", + "model.layers.74.mlp.experts.158.gate_proj", + "model.layers.74.mlp.experts.158.up_proj", + "model.layers.74.mlp.experts.159.down_proj", + "model.layers.74.mlp.experts.159.gate_proj", + "model.layers.74.mlp.experts.159.up_proj", + "model.layers.74.mlp.experts.16.down_proj", + "model.layers.74.mlp.experts.16.gate_proj", + "model.layers.74.mlp.experts.16.up_proj", + "model.layers.74.mlp.experts.160.down_proj", + "model.layers.74.mlp.experts.160.gate_proj", + "model.layers.74.mlp.experts.160.up_proj", + "model.layers.74.mlp.experts.161.down_proj", + "model.layers.74.mlp.experts.161.gate_proj", + "model.layers.74.mlp.experts.161.up_proj", + "model.layers.74.mlp.experts.162.down_proj", + "model.layers.74.mlp.experts.162.gate_proj", + "model.layers.74.mlp.experts.162.up_proj", + "model.layers.74.mlp.experts.163.down_proj", + "model.layers.74.mlp.experts.163.gate_proj", + "model.layers.74.mlp.experts.163.up_proj", + "model.layers.74.mlp.experts.164.down_proj", + "model.layers.74.mlp.experts.164.gate_proj", + "model.layers.74.mlp.experts.164.up_proj", + "model.layers.74.mlp.experts.165.down_proj", + "model.layers.74.mlp.experts.165.gate_proj", + "model.layers.74.mlp.experts.165.up_proj", + "model.layers.74.mlp.experts.166.down_proj", + "model.layers.74.mlp.experts.166.gate_proj", + "model.layers.74.mlp.experts.166.up_proj", + "model.layers.74.mlp.experts.167.down_proj", + "model.layers.74.mlp.experts.167.gate_proj", + "model.layers.74.mlp.experts.167.up_proj", + "model.layers.74.mlp.experts.168.down_proj", + "model.layers.74.mlp.experts.168.gate_proj", + "model.layers.74.mlp.experts.168.up_proj", + "model.layers.74.mlp.experts.169.down_proj", + "model.layers.74.mlp.experts.169.gate_proj", + "model.layers.74.mlp.experts.169.up_proj", + "model.layers.74.mlp.experts.17.down_proj", + "model.layers.74.mlp.experts.17.gate_proj", + "model.layers.74.mlp.experts.17.up_proj", + "model.layers.74.mlp.experts.170.down_proj", + "model.layers.74.mlp.experts.170.gate_proj", + "model.layers.74.mlp.experts.170.up_proj", + "model.layers.74.mlp.experts.171.down_proj", + "model.layers.74.mlp.experts.171.gate_proj", + "model.layers.74.mlp.experts.171.up_proj", + "model.layers.74.mlp.experts.172.down_proj", + "model.layers.74.mlp.experts.172.gate_proj", + "model.layers.74.mlp.experts.172.up_proj", + "model.layers.74.mlp.experts.173.down_proj", + "model.layers.74.mlp.experts.173.gate_proj", + "model.layers.74.mlp.experts.173.up_proj", + "model.layers.74.mlp.experts.174.down_proj", + "model.layers.74.mlp.experts.174.gate_proj", + "model.layers.74.mlp.experts.174.up_proj", + "model.layers.74.mlp.experts.175.down_proj", + "model.layers.74.mlp.experts.175.gate_proj", + "model.layers.74.mlp.experts.175.up_proj", + "model.layers.74.mlp.experts.176.down_proj", + "model.layers.74.mlp.experts.176.gate_proj", + "model.layers.74.mlp.experts.176.up_proj", + "model.layers.74.mlp.experts.177.down_proj", + "model.layers.74.mlp.experts.177.gate_proj", + "model.layers.74.mlp.experts.177.up_proj", + "model.layers.74.mlp.experts.178.down_proj", + "model.layers.74.mlp.experts.178.gate_proj", + "model.layers.74.mlp.experts.178.up_proj", + "model.layers.74.mlp.experts.179.down_proj", + "model.layers.74.mlp.experts.179.gate_proj", + "model.layers.74.mlp.experts.179.up_proj", + "model.layers.74.mlp.experts.18.down_proj", + "model.layers.74.mlp.experts.18.gate_proj", + "model.layers.74.mlp.experts.18.up_proj", + "model.layers.74.mlp.experts.180.down_proj", + "model.layers.74.mlp.experts.180.gate_proj", + "model.layers.74.mlp.experts.180.up_proj", + "model.layers.74.mlp.experts.181.down_proj", + "model.layers.74.mlp.experts.181.gate_proj", + "model.layers.74.mlp.experts.181.up_proj", + "model.layers.74.mlp.experts.182.down_proj", + "model.layers.74.mlp.experts.182.gate_proj", + "model.layers.74.mlp.experts.182.up_proj", + "model.layers.74.mlp.experts.183.down_proj", + "model.layers.74.mlp.experts.183.gate_proj", + "model.layers.74.mlp.experts.183.up_proj", + "model.layers.74.mlp.experts.184.down_proj", + "model.layers.74.mlp.experts.184.gate_proj", + "model.layers.74.mlp.experts.184.up_proj", + "model.layers.74.mlp.experts.185.down_proj", + "model.layers.74.mlp.experts.185.gate_proj", + "model.layers.74.mlp.experts.185.up_proj", + "model.layers.74.mlp.experts.186.down_proj", + "model.layers.74.mlp.experts.186.gate_proj", + "model.layers.74.mlp.experts.186.up_proj", + "model.layers.74.mlp.experts.187.down_proj", + "model.layers.74.mlp.experts.187.gate_proj", + "model.layers.74.mlp.experts.187.up_proj", + "model.layers.74.mlp.experts.188.down_proj", + "model.layers.74.mlp.experts.188.gate_proj", + "model.layers.74.mlp.experts.188.up_proj", + "model.layers.74.mlp.experts.189.down_proj", + "model.layers.74.mlp.experts.189.gate_proj", + "model.layers.74.mlp.experts.189.up_proj", + "model.layers.74.mlp.experts.19.down_proj", + "model.layers.74.mlp.experts.19.gate_proj", + "model.layers.74.mlp.experts.19.up_proj", + "model.layers.74.mlp.experts.190.down_proj", + "model.layers.74.mlp.experts.190.gate_proj", + "model.layers.74.mlp.experts.190.up_proj", + "model.layers.74.mlp.experts.191.down_proj", + "model.layers.74.mlp.experts.191.gate_proj", + "model.layers.74.mlp.experts.191.up_proj", + "model.layers.74.mlp.experts.192.down_proj", + "model.layers.74.mlp.experts.192.gate_proj", + "model.layers.74.mlp.experts.192.up_proj", + "model.layers.74.mlp.experts.193.down_proj", + "model.layers.74.mlp.experts.193.gate_proj", + "model.layers.74.mlp.experts.193.up_proj", + "model.layers.74.mlp.experts.194.down_proj", + "model.layers.74.mlp.experts.194.gate_proj", + "model.layers.74.mlp.experts.194.up_proj", + "model.layers.74.mlp.experts.195.down_proj", + "model.layers.74.mlp.experts.195.gate_proj", + "model.layers.74.mlp.experts.195.up_proj", + "model.layers.74.mlp.experts.196.down_proj", + "model.layers.74.mlp.experts.196.gate_proj", + "model.layers.74.mlp.experts.196.up_proj", + "model.layers.74.mlp.experts.197.down_proj", + "model.layers.74.mlp.experts.197.gate_proj", + "model.layers.74.mlp.experts.197.up_proj", + "model.layers.74.mlp.experts.198.down_proj", + "model.layers.74.mlp.experts.198.gate_proj", + "model.layers.74.mlp.experts.198.up_proj", + "model.layers.74.mlp.experts.199.down_proj", + "model.layers.74.mlp.experts.199.gate_proj", + "model.layers.74.mlp.experts.199.up_proj", + "model.layers.74.mlp.experts.2.down_proj", + "model.layers.74.mlp.experts.2.gate_proj", + "model.layers.74.mlp.experts.2.up_proj", + "model.layers.74.mlp.experts.20.down_proj", + "model.layers.74.mlp.experts.20.gate_proj", + "model.layers.74.mlp.experts.20.up_proj", + "model.layers.74.mlp.experts.200.down_proj", + "model.layers.74.mlp.experts.200.gate_proj", + "model.layers.74.mlp.experts.200.up_proj", + "model.layers.74.mlp.experts.201.down_proj", + "model.layers.74.mlp.experts.201.gate_proj", + "model.layers.74.mlp.experts.201.up_proj", + "model.layers.74.mlp.experts.202.down_proj", + "model.layers.74.mlp.experts.202.gate_proj", + "model.layers.74.mlp.experts.202.up_proj", + "model.layers.74.mlp.experts.203.down_proj", + "model.layers.74.mlp.experts.203.gate_proj", + "model.layers.74.mlp.experts.203.up_proj", + "model.layers.74.mlp.experts.204.down_proj", + "model.layers.74.mlp.experts.204.gate_proj", + "model.layers.74.mlp.experts.204.up_proj", + "model.layers.74.mlp.experts.205.down_proj", + "model.layers.74.mlp.experts.205.gate_proj", + "model.layers.74.mlp.experts.205.up_proj", + "model.layers.74.mlp.experts.206.down_proj", + "model.layers.74.mlp.experts.206.gate_proj", + "model.layers.74.mlp.experts.206.up_proj", + "model.layers.74.mlp.experts.207.down_proj", + "model.layers.74.mlp.experts.207.gate_proj", + "model.layers.74.mlp.experts.207.up_proj", + "model.layers.74.mlp.experts.208.down_proj", + "model.layers.74.mlp.experts.208.gate_proj", + "model.layers.74.mlp.experts.208.up_proj", + "model.layers.74.mlp.experts.209.down_proj", + "model.layers.74.mlp.experts.209.gate_proj", + "model.layers.74.mlp.experts.209.up_proj", + "model.layers.74.mlp.experts.21.down_proj", + "model.layers.74.mlp.experts.21.gate_proj", + "model.layers.74.mlp.experts.21.up_proj", + "model.layers.74.mlp.experts.210.down_proj", + "model.layers.74.mlp.experts.210.gate_proj", + "model.layers.74.mlp.experts.210.up_proj", + "model.layers.74.mlp.experts.211.down_proj", + "model.layers.74.mlp.experts.211.gate_proj", + "model.layers.74.mlp.experts.211.up_proj", + "model.layers.74.mlp.experts.212.down_proj", + "model.layers.74.mlp.experts.212.gate_proj", + "model.layers.74.mlp.experts.212.up_proj", + "model.layers.74.mlp.experts.213.down_proj", + "model.layers.74.mlp.experts.213.gate_proj", + "model.layers.74.mlp.experts.213.up_proj", + "model.layers.74.mlp.experts.214.down_proj", + "model.layers.74.mlp.experts.214.gate_proj", + "model.layers.74.mlp.experts.214.up_proj", + "model.layers.74.mlp.experts.215.down_proj", + "model.layers.74.mlp.experts.215.gate_proj", + "model.layers.74.mlp.experts.215.up_proj", + "model.layers.74.mlp.experts.216.down_proj", + "model.layers.74.mlp.experts.216.gate_proj", + "model.layers.74.mlp.experts.216.up_proj", + "model.layers.74.mlp.experts.217.down_proj", + "model.layers.74.mlp.experts.217.gate_proj", + "model.layers.74.mlp.experts.217.up_proj", + "model.layers.74.mlp.experts.218.down_proj", + "model.layers.74.mlp.experts.218.gate_proj", + "model.layers.74.mlp.experts.218.up_proj", + "model.layers.74.mlp.experts.219.down_proj", + "model.layers.74.mlp.experts.219.gate_proj", + "model.layers.74.mlp.experts.219.up_proj", + "model.layers.74.mlp.experts.22.down_proj", + "model.layers.74.mlp.experts.22.gate_proj", + "model.layers.74.mlp.experts.22.up_proj", + "model.layers.74.mlp.experts.220.down_proj", + "model.layers.74.mlp.experts.220.gate_proj", + "model.layers.74.mlp.experts.220.up_proj", + "model.layers.74.mlp.experts.221.down_proj", + "model.layers.74.mlp.experts.221.gate_proj", + "model.layers.74.mlp.experts.221.up_proj", + "model.layers.74.mlp.experts.222.down_proj", + "model.layers.74.mlp.experts.222.gate_proj", + "model.layers.74.mlp.experts.222.up_proj", + "model.layers.74.mlp.experts.223.down_proj", + "model.layers.74.mlp.experts.223.gate_proj", + "model.layers.74.mlp.experts.223.up_proj", + "model.layers.74.mlp.experts.224.down_proj", + "model.layers.74.mlp.experts.224.gate_proj", + "model.layers.74.mlp.experts.224.up_proj", + "model.layers.74.mlp.experts.225.down_proj", + "model.layers.74.mlp.experts.225.gate_proj", + "model.layers.74.mlp.experts.225.up_proj", + "model.layers.74.mlp.experts.226.down_proj", + "model.layers.74.mlp.experts.226.gate_proj", + "model.layers.74.mlp.experts.226.up_proj", + "model.layers.74.mlp.experts.227.down_proj", + "model.layers.74.mlp.experts.227.gate_proj", + "model.layers.74.mlp.experts.227.up_proj", + "model.layers.74.mlp.experts.228.down_proj", + "model.layers.74.mlp.experts.228.gate_proj", + "model.layers.74.mlp.experts.228.up_proj", + "model.layers.74.mlp.experts.229.down_proj", + "model.layers.74.mlp.experts.229.gate_proj", + "model.layers.74.mlp.experts.229.up_proj", + "model.layers.74.mlp.experts.23.down_proj", + "model.layers.74.mlp.experts.23.gate_proj", + "model.layers.74.mlp.experts.23.up_proj", + "model.layers.74.mlp.experts.230.down_proj", + "model.layers.74.mlp.experts.230.gate_proj", + "model.layers.74.mlp.experts.230.up_proj", + "model.layers.74.mlp.experts.231.down_proj", + "model.layers.74.mlp.experts.231.gate_proj", + "model.layers.74.mlp.experts.231.up_proj", + "model.layers.74.mlp.experts.232.down_proj", + "model.layers.74.mlp.experts.232.gate_proj", + "model.layers.74.mlp.experts.232.up_proj", + "model.layers.74.mlp.experts.233.down_proj", + "model.layers.74.mlp.experts.233.gate_proj", + "model.layers.74.mlp.experts.233.up_proj", + "model.layers.74.mlp.experts.234.down_proj", + "model.layers.74.mlp.experts.234.gate_proj", + "model.layers.74.mlp.experts.234.up_proj", + "model.layers.74.mlp.experts.235.down_proj", + "model.layers.74.mlp.experts.235.gate_proj", + "model.layers.74.mlp.experts.235.up_proj", + "model.layers.74.mlp.experts.236.down_proj", + "model.layers.74.mlp.experts.236.gate_proj", + "model.layers.74.mlp.experts.236.up_proj", + "model.layers.74.mlp.experts.237.down_proj", + "model.layers.74.mlp.experts.237.gate_proj", + "model.layers.74.mlp.experts.237.up_proj", + "model.layers.74.mlp.experts.238.down_proj", + "model.layers.74.mlp.experts.238.gate_proj", + "model.layers.74.mlp.experts.238.up_proj", + "model.layers.74.mlp.experts.239.down_proj", + "model.layers.74.mlp.experts.239.gate_proj", + "model.layers.74.mlp.experts.239.up_proj", + "model.layers.74.mlp.experts.24.down_proj", + "model.layers.74.mlp.experts.24.gate_proj", + "model.layers.74.mlp.experts.24.up_proj", + "model.layers.74.mlp.experts.240.down_proj", + "model.layers.74.mlp.experts.240.gate_proj", + "model.layers.74.mlp.experts.240.up_proj", + "model.layers.74.mlp.experts.241.down_proj", + "model.layers.74.mlp.experts.241.gate_proj", + "model.layers.74.mlp.experts.241.up_proj", + "model.layers.74.mlp.experts.242.down_proj", + "model.layers.74.mlp.experts.242.gate_proj", + "model.layers.74.mlp.experts.242.up_proj", + "model.layers.74.mlp.experts.243.down_proj", + "model.layers.74.mlp.experts.243.gate_proj", + "model.layers.74.mlp.experts.243.up_proj", + "model.layers.74.mlp.experts.244.down_proj", + "model.layers.74.mlp.experts.244.gate_proj", + "model.layers.74.mlp.experts.244.up_proj", + "model.layers.74.mlp.experts.245.down_proj", + "model.layers.74.mlp.experts.245.gate_proj", + "model.layers.74.mlp.experts.245.up_proj", + "model.layers.74.mlp.experts.246.down_proj", + "model.layers.74.mlp.experts.246.gate_proj", + "model.layers.74.mlp.experts.246.up_proj", + "model.layers.74.mlp.experts.247.down_proj", + "model.layers.74.mlp.experts.247.gate_proj", + "model.layers.74.mlp.experts.247.up_proj", + "model.layers.74.mlp.experts.248.down_proj", + "model.layers.74.mlp.experts.248.gate_proj", + "model.layers.74.mlp.experts.248.up_proj", + "model.layers.74.mlp.experts.249.down_proj", + "model.layers.74.mlp.experts.249.gate_proj", + "model.layers.74.mlp.experts.249.up_proj", + "model.layers.74.mlp.experts.25.down_proj", + "model.layers.74.mlp.experts.25.gate_proj", + "model.layers.74.mlp.experts.25.up_proj", + "model.layers.74.mlp.experts.250.down_proj", + "model.layers.74.mlp.experts.250.gate_proj", + "model.layers.74.mlp.experts.250.up_proj", + "model.layers.74.mlp.experts.251.down_proj", + "model.layers.74.mlp.experts.251.gate_proj", + "model.layers.74.mlp.experts.251.up_proj", + "model.layers.74.mlp.experts.252.down_proj", + "model.layers.74.mlp.experts.252.gate_proj", + "model.layers.74.mlp.experts.252.up_proj", + "model.layers.74.mlp.experts.253.down_proj", + "model.layers.74.mlp.experts.253.gate_proj", + "model.layers.74.mlp.experts.253.up_proj", + "model.layers.74.mlp.experts.254.down_proj", + "model.layers.74.mlp.experts.254.gate_proj", + "model.layers.74.mlp.experts.254.up_proj", + "model.layers.74.mlp.experts.255.down_proj", + "model.layers.74.mlp.experts.255.gate_proj", + "model.layers.74.mlp.experts.255.up_proj", + "model.layers.74.mlp.experts.26.down_proj", + "model.layers.74.mlp.experts.26.gate_proj", + "model.layers.74.mlp.experts.26.up_proj", + "model.layers.74.mlp.experts.27.down_proj", + "model.layers.74.mlp.experts.27.gate_proj", + "model.layers.74.mlp.experts.27.up_proj", + "model.layers.74.mlp.experts.28.down_proj", + "model.layers.74.mlp.experts.28.gate_proj", + "model.layers.74.mlp.experts.28.up_proj", + "model.layers.74.mlp.experts.29.down_proj", + "model.layers.74.mlp.experts.29.gate_proj", + "model.layers.74.mlp.experts.29.up_proj", + "model.layers.74.mlp.experts.3.down_proj", + "model.layers.74.mlp.experts.3.gate_proj", + "model.layers.74.mlp.experts.3.up_proj", + "model.layers.74.mlp.experts.30.down_proj", + "model.layers.74.mlp.experts.30.gate_proj", + "model.layers.74.mlp.experts.30.up_proj", + "model.layers.74.mlp.experts.31.down_proj", + "model.layers.74.mlp.experts.31.gate_proj", + "model.layers.74.mlp.experts.31.up_proj", + "model.layers.74.mlp.experts.32.down_proj", + "model.layers.74.mlp.experts.32.gate_proj", + "model.layers.74.mlp.experts.32.up_proj", + "model.layers.74.mlp.experts.33.down_proj", + "model.layers.74.mlp.experts.33.gate_proj", + "model.layers.74.mlp.experts.33.up_proj", + "model.layers.74.mlp.experts.34.down_proj", + "model.layers.74.mlp.experts.34.gate_proj", + "model.layers.74.mlp.experts.34.up_proj", + "model.layers.74.mlp.experts.35.down_proj", + "model.layers.74.mlp.experts.35.gate_proj", + "model.layers.74.mlp.experts.35.up_proj", + "model.layers.74.mlp.experts.36.down_proj", + "model.layers.74.mlp.experts.36.gate_proj", + "model.layers.74.mlp.experts.36.up_proj", + "model.layers.74.mlp.experts.37.down_proj", + "model.layers.74.mlp.experts.37.gate_proj", + "model.layers.74.mlp.experts.37.up_proj", + "model.layers.74.mlp.experts.38.down_proj", + "model.layers.74.mlp.experts.38.gate_proj", + "model.layers.74.mlp.experts.38.up_proj", + "model.layers.74.mlp.experts.39.down_proj", + "model.layers.74.mlp.experts.39.gate_proj", + "model.layers.74.mlp.experts.39.up_proj", + "model.layers.74.mlp.experts.4.down_proj", + "model.layers.74.mlp.experts.4.gate_proj", + "model.layers.74.mlp.experts.4.up_proj", + "model.layers.74.mlp.experts.40.down_proj", + "model.layers.74.mlp.experts.40.gate_proj", + "model.layers.74.mlp.experts.40.up_proj", + "model.layers.74.mlp.experts.41.down_proj", + "model.layers.74.mlp.experts.41.gate_proj", + "model.layers.74.mlp.experts.41.up_proj", + "model.layers.74.mlp.experts.42.down_proj", + "model.layers.74.mlp.experts.42.gate_proj", + "model.layers.74.mlp.experts.42.up_proj", + "model.layers.74.mlp.experts.43.down_proj", + "model.layers.74.mlp.experts.43.gate_proj", + "model.layers.74.mlp.experts.43.up_proj", + "model.layers.74.mlp.experts.44.down_proj", + "model.layers.74.mlp.experts.44.gate_proj", + "model.layers.74.mlp.experts.44.up_proj", + "model.layers.74.mlp.experts.45.down_proj", + "model.layers.74.mlp.experts.45.gate_proj", + "model.layers.74.mlp.experts.45.up_proj", + "model.layers.74.mlp.experts.46.down_proj", + "model.layers.74.mlp.experts.46.gate_proj", + "model.layers.74.mlp.experts.46.up_proj", + "model.layers.74.mlp.experts.47.down_proj", + "model.layers.74.mlp.experts.47.gate_proj", + "model.layers.74.mlp.experts.47.up_proj", + "model.layers.74.mlp.experts.48.down_proj", + "model.layers.74.mlp.experts.48.gate_proj", + "model.layers.74.mlp.experts.48.up_proj", + "model.layers.74.mlp.experts.49.down_proj", + "model.layers.74.mlp.experts.49.gate_proj", + "model.layers.74.mlp.experts.49.up_proj", + "model.layers.74.mlp.experts.5.down_proj", + "model.layers.74.mlp.experts.5.gate_proj", + "model.layers.74.mlp.experts.5.up_proj", + "model.layers.74.mlp.experts.50.down_proj", + "model.layers.74.mlp.experts.50.gate_proj", + "model.layers.74.mlp.experts.50.up_proj", + "model.layers.74.mlp.experts.51.down_proj", + "model.layers.74.mlp.experts.51.gate_proj", + "model.layers.74.mlp.experts.51.up_proj", + "model.layers.74.mlp.experts.52.down_proj", + "model.layers.74.mlp.experts.52.gate_proj", + "model.layers.74.mlp.experts.52.up_proj", + "model.layers.74.mlp.experts.53.down_proj", + "model.layers.74.mlp.experts.53.gate_proj", + "model.layers.74.mlp.experts.53.up_proj", + "model.layers.74.mlp.experts.54.down_proj", + "model.layers.74.mlp.experts.54.gate_proj", + "model.layers.74.mlp.experts.54.up_proj", + "model.layers.74.mlp.experts.55.down_proj", + "model.layers.74.mlp.experts.55.gate_proj", + "model.layers.74.mlp.experts.55.up_proj", + "model.layers.74.mlp.experts.56.down_proj", + "model.layers.74.mlp.experts.56.gate_proj", + "model.layers.74.mlp.experts.56.up_proj", + "model.layers.74.mlp.experts.57.down_proj", + "model.layers.74.mlp.experts.57.gate_proj", + "model.layers.74.mlp.experts.57.up_proj", + "model.layers.74.mlp.experts.58.down_proj", + "model.layers.74.mlp.experts.58.gate_proj", + "model.layers.74.mlp.experts.58.up_proj", + "model.layers.74.mlp.experts.59.down_proj", + "model.layers.74.mlp.experts.59.gate_proj", + "model.layers.74.mlp.experts.59.up_proj", + "model.layers.74.mlp.experts.6.down_proj", + "model.layers.74.mlp.experts.6.gate_proj", + "model.layers.74.mlp.experts.6.up_proj", + "model.layers.74.mlp.experts.60.down_proj", + "model.layers.74.mlp.experts.60.gate_proj", + "model.layers.74.mlp.experts.60.up_proj", + "model.layers.74.mlp.experts.61.down_proj", + "model.layers.74.mlp.experts.61.gate_proj", + "model.layers.74.mlp.experts.61.up_proj", + "model.layers.74.mlp.experts.62.down_proj", + "model.layers.74.mlp.experts.62.gate_proj", + "model.layers.74.mlp.experts.62.up_proj", + "model.layers.74.mlp.experts.63.down_proj", + "model.layers.74.mlp.experts.63.gate_proj", + "model.layers.74.mlp.experts.63.up_proj", + "model.layers.74.mlp.experts.64.down_proj", + "model.layers.74.mlp.experts.64.gate_proj", + "model.layers.74.mlp.experts.64.up_proj", + "model.layers.74.mlp.experts.65.down_proj", + "model.layers.74.mlp.experts.65.gate_proj", + "model.layers.74.mlp.experts.65.up_proj", + "model.layers.74.mlp.experts.66.down_proj", + "model.layers.74.mlp.experts.66.gate_proj", + "model.layers.74.mlp.experts.66.up_proj", + "model.layers.74.mlp.experts.67.down_proj", + "model.layers.74.mlp.experts.67.gate_proj", + "model.layers.74.mlp.experts.67.up_proj", + "model.layers.74.mlp.experts.68.down_proj", + "model.layers.74.mlp.experts.68.gate_proj", + "model.layers.74.mlp.experts.68.up_proj", + "model.layers.74.mlp.experts.69.down_proj", + "model.layers.74.mlp.experts.69.gate_proj", + "model.layers.74.mlp.experts.69.up_proj", + "model.layers.74.mlp.experts.7.down_proj", + "model.layers.74.mlp.experts.7.gate_proj", + "model.layers.74.mlp.experts.7.up_proj", + "model.layers.74.mlp.experts.70.down_proj", + "model.layers.74.mlp.experts.70.gate_proj", + "model.layers.74.mlp.experts.70.up_proj", + "model.layers.74.mlp.experts.71.down_proj", + "model.layers.74.mlp.experts.71.gate_proj", + "model.layers.74.mlp.experts.71.up_proj", + "model.layers.74.mlp.experts.72.down_proj", + "model.layers.74.mlp.experts.72.gate_proj", + "model.layers.74.mlp.experts.72.up_proj", + "model.layers.74.mlp.experts.73.down_proj", + "model.layers.74.mlp.experts.73.gate_proj", + "model.layers.74.mlp.experts.73.up_proj", + "model.layers.74.mlp.experts.74.down_proj", + "model.layers.74.mlp.experts.74.gate_proj", + "model.layers.74.mlp.experts.74.up_proj", + "model.layers.74.mlp.experts.75.down_proj", + "model.layers.74.mlp.experts.75.gate_proj", + "model.layers.74.mlp.experts.75.up_proj", + "model.layers.74.mlp.experts.76.down_proj", + "model.layers.74.mlp.experts.76.gate_proj", + "model.layers.74.mlp.experts.76.up_proj", + "model.layers.74.mlp.experts.77.down_proj", + "model.layers.74.mlp.experts.77.gate_proj", + "model.layers.74.mlp.experts.77.up_proj", + "model.layers.74.mlp.experts.78.down_proj", + "model.layers.74.mlp.experts.78.gate_proj", + "model.layers.74.mlp.experts.78.up_proj", + "model.layers.74.mlp.experts.79.down_proj", + "model.layers.74.mlp.experts.79.gate_proj", + "model.layers.74.mlp.experts.79.up_proj", + "model.layers.74.mlp.experts.8.down_proj", + "model.layers.74.mlp.experts.8.gate_proj", + "model.layers.74.mlp.experts.8.up_proj", + "model.layers.74.mlp.experts.80.down_proj", + "model.layers.74.mlp.experts.80.gate_proj", + "model.layers.74.mlp.experts.80.up_proj", + "model.layers.74.mlp.experts.81.down_proj", + "model.layers.74.mlp.experts.81.gate_proj", + "model.layers.74.mlp.experts.81.up_proj", + "model.layers.74.mlp.experts.82.down_proj", + "model.layers.74.mlp.experts.82.gate_proj", + "model.layers.74.mlp.experts.82.up_proj", + "model.layers.74.mlp.experts.83.down_proj", + "model.layers.74.mlp.experts.83.gate_proj", + "model.layers.74.mlp.experts.83.up_proj", + "model.layers.74.mlp.experts.84.down_proj", + "model.layers.74.mlp.experts.84.gate_proj", + "model.layers.74.mlp.experts.84.up_proj", + "model.layers.74.mlp.experts.85.down_proj", + "model.layers.74.mlp.experts.85.gate_proj", + "model.layers.74.mlp.experts.85.up_proj", + "model.layers.74.mlp.experts.86.down_proj", + "model.layers.74.mlp.experts.86.gate_proj", + "model.layers.74.mlp.experts.86.up_proj", + "model.layers.74.mlp.experts.87.down_proj", + "model.layers.74.mlp.experts.87.gate_proj", + "model.layers.74.mlp.experts.87.up_proj", + "model.layers.74.mlp.experts.88.down_proj", + "model.layers.74.mlp.experts.88.gate_proj", + "model.layers.74.mlp.experts.88.up_proj", + "model.layers.74.mlp.experts.89.down_proj", + "model.layers.74.mlp.experts.89.gate_proj", + "model.layers.74.mlp.experts.89.up_proj", + "model.layers.74.mlp.experts.9.down_proj", + "model.layers.74.mlp.experts.9.gate_proj", + "model.layers.74.mlp.experts.9.up_proj", + "model.layers.74.mlp.experts.90.down_proj", + "model.layers.74.mlp.experts.90.gate_proj", + "model.layers.74.mlp.experts.90.up_proj", + "model.layers.74.mlp.experts.91.down_proj", + "model.layers.74.mlp.experts.91.gate_proj", + "model.layers.74.mlp.experts.91.up_proj", + "model.layers.74.mlp.experts.92.down_proj", + "model.layers.74.mlp.experts.92.gate_proj", + "model.layers.74.mlp.experts.92.up_proj", + "model.layers.74.mlp.experts.93.down_proj", + "model.layers.74.mlp.experts.93.gate_proj", + "model.layers.74.mlp.experts.93.up_proj", + "model.layers.74.mlp.experts.94.down_proj", + "model.layers.74.mlp.experts.94.gate_proj", + "model.layers.74.mlp.experts.94.up_proj", + "model.layers.74.mlp.experts.95.down_proj", + "model.layers.74.mlp.experts.95.gate_proj", + "model.layers.74.mlp.experts.95.up_proj", + "model.layers.74.mlp.experts.96.down_proj", + "model.layers.74.mlp.experts.96.gate_proj", + "model.layers.74.mlp.experts.96.up_proj", + "model.layers.74.mlp.experts.97.down_proj", + "model.layers.74.mlp.experts.97.gate_proj", + "model.layers.74.mlp.experts.97.up_proj", + "model.layers.74.mlp.experts.98.down_proj", + "model.layers.74.mlp.experts.98.gate_proj", + "model.layers.74.mlp.experts.98.up_proj", + "model.layers.74.mlp.experts.99.down_proj", + "model.layers.74.mlp.experts.99.gate_proj", + "model.layers.74.mlp.experts.99.up_proj", + "model.layers.75.mlp.experts.0.down_proj", + "model.layers.75.mlp.experts.0.gate_proj", + "model.layers.75.mlp.experts.0.up_proj", + "model.layers.75.mlp.experts.1.down_proj", + "model.layers.75.mlp.experts.1.gate_proj", + "model.layers.75.mlp.experts.1.up_proj", + "model.layers.75.mlp.experts.10.down_proj", + "model.layers.75.mlp.experts.10.gate_proj", + "model.layers.75.mlp.experts.10.up_proj", + "model.layers.75.mlp.experts.100.down_proj", + "model.layers.75.mlp.experts.100.gate_proj", + "model.layers.75.mlp.experts.100.up_proj", + "model.layers.75.mlp.experts.101.down_proj", + "model.layers.75.mlp.experts.101.gate_proj", + "model.layers.75.mlp.experts.101.up_proj", + "model.layers.75.mlp.experts.102.down_proj", + "model.layers.75.mlp.experts.102.gate_proj", + "model.layers.75.mlp.experts.102.up_proj", + "model.layers.75.mlp.experts.103.down_proj", + "model.layers.75.mlp.experts.103.gate_proj", + "model.layers.75.mlp.experts.103.up_proj", + "model.layers.75.mlp.experts.104.down_proj", + "model.layers.75.mlp.experts.104.gate_proj", + "model.layers.75.mlp.experts.104.up_proj", + "model.layers.75.mlp.experts.105.down_proj", + "model.layers.75.mlp.experts.105.gate_proj", + "model.layers.75.mlp.experts.105.up_proj", + "model.layers.75.mlp.experts.106.down_proj", + "model.layers.75.mlp.experts.106.gate_proj", + "model.layers.75.mlp.experts.106.up_proj", + "model.layers.75.mlp.experts.107.down_proj", + "model.layers.75.mlp.experts.107.gate_proj", + "model.layers.75.mlp.experts.107.up_proj", + "model.layers.75.mlp.experts.108.down_proj", + "model.layers.75.mlp.experts.108.gate_proj", + "model.layers.75.mlp.experts.108.up_proj", + "model.layers.75.mlp.experts.109.down_proj", + "model.layers.75.mlp.experts.109.gate_proj", + "model.layers.75.mlp.experts.109.up_proj", + "model.layers.75.mlp.experts.11.down_proj", + "model.layers.75.mlp.experts.11.gate_proj", + "model.layers.75.mlp.experts.11.up_proj", + "model.layers.75.mlp.experts.110.down_proj", + "model.layers.75.mlp.experts.110.gate_proj", + "model.layers.75.mlp.experts.110.up_proj", + "model.layers.75.mlp.experts.111.down_proj", + "model.layers.75.mlp.experts.111.gate_proj", + "model.layers.75.mlp.experts.111.up_proj", + "model.layers.75.mlp.experts.112.down_proj", + "model.layers.75.mlp.experts.112.gate_proj", + "model.layers.75.mlp.experts.112.up_proj", + "model.layers.75.mlp.experts.113.down_proj", + "model.layers.75.mlp.experts.113.gate_proj", + "model.layers.75.mlp.experts.113.up_proj", + "model.layers.75.mlp.experts.114.down_proj", + "model.layers.75.mlp.experts.114.gate_proj", + "model.layers.75.mlp.experts.114.up_proj", + "model.layers.75.mlp.experts.115.down_proj", + "model.layers.75.mlp.experts.115.gate_proj", + "model.layers.75.mlp.experts.115.up_proj", + "model.layers.75.mlp.experts.116.down_proj", + "model.layers.75.mlp.experts.116.gate_proj", + "model.layers.75.mlp.experts.116.up_proj", + "model.layers.75.mlp.experts.117.down_proj", + "model.layers.75.mlp.experts.117.gate_proj", + "model.layers.75.mlp.experts.117.up_proj", + "model.layers.75.mlp.experts.118.down_proj", + "model.layers.75.mlp.experts.118.gate_proj", + "model.layers.75.mlp.experts.118.up_proj", + "model.layers.75.mlp.experts.119.down_proj", + "model.layers.75.mlp.experts.119.gate_proj", + "model.layers.75.mlp.experts.119.up_proj", + "model.layers.75.mlp.experts.12.down_proj", + "model.layers.75.mlp.experts.12.gate_proj", + "model.layers.75.mlp.experts.12.up_proj", + "model.layers.75.mlp.experts.120.down_proj", + "model.layers.75.mlp.experts.120.gate_proj", + "model.layers.75.mlp.experts.120.up_proj", + "model.layers.75.mlp.experts.121.down_proj", + "model.layers.75.mlp.experts.121.gate_proj", + "model.layers.75.mlp.experts.121.up_proj", + "model.layers.75.mlp.experts.122.down_proj", + "model.layers.75.mlp.experts.122.gate_proj", + "model.layers.75.mlp.experts.122.up_proj", + "model.layers.75.mlp.experts.123.down_proj", + "model.layers.75.mlp.experts.123.gate_proj", + "model.layers.75.mlp.experts.123.up_proj", + "model.layers.75.mlp.experts.124.down_proj", + "model.layers.75.mlp.experts.124.gate_proj", + "model.layers.75.mlp.experts.124.up_proj", + "model.layers.75.mlp.experts.125.down_proj", + "model.layers.75.mlp.experts.125.gate_proj", + "model.layers.75.mlp.experts.125.up_proj", + "model.layers.75.mlp.experts.126.down_proj", + "model.layers.75.mlp.experts.126.gate_proj", + "model.layers.75.mlp.experts.126.up_proj", + "model.layers.75.mlp.experts.127.down_proj", + "model.layers.75.mlp.experts.127.gate_proj", + "model.layers.75.mlp.experts.127.up_proj", + "model.layers.75.mlp.experts.128.down_proj", + "model.layers.75.mlp.experts.128.gate_proj", + "model.layers.75.mlp.experts.128.up_proj", + "model.layers.75.mlp.experts.129.down_proj", + "model.layers.75.mlp.experts.129.gate_proj", + "model.layers.75.mlp.experts.129.up_proj", + "model.layers.75.mlp.experts.13.down_proj", + "model.layers.75.mlp.experts.13.gate_proj", + "model.layers.75.mlp.experts.13.up_proj", + "model.layers.75.mlp.experts.130.down_proj", + "model.layers.75.mlp.experts.130.gate_proj", + "model.layers.75.mlp.experts.130.up_proj", + "model.layers.75.mlp.experts.131.down_proj", + "model.layers.75.mlp.experts.131.gate_proj", + "model.layers.75.mlp.experts.131.up_proj", + "model.layers.75.mlp.experts.132.down_proj", + "model.layers.75.mlp.experts.132.gate_proj", + "model.layers.75.mlp.experts.132.up_proj", + "model.layers.75.mlp.experts.133.down_proj", + "model.layers.75.mlp.experts.133.gate_proj", + "model.layers.75.mlp.experts.133.up_proj", + "model.layers.75.mlp.experts.134.down_proj", + "model.layers.75.mlp.experts.134.gate_proj", + "model.layers.75.mlp.experts.134.up_proj", + "model.layers.75.mlp.experts.135.down_proj", + "model.layers.75.mlp.experts.135.gate_proj", + "model.layers.75.mlp.experts.135.up_proj", + "model.layers.75.mlp.experts.136.down_proj", + "model.layers.75.mlp.experts.136.gate_proj", + "model.layers.75.mlp.experts.136.up_proj", + "model.layers.75.mlp.experts.137.down_proj", + "model.layers.75.mlp.experts.137.gate_proj", + "model.layers.75.mlp.experts.137.up_proj", + "model.layers.75.mlp.experts.138.down_proj", + "model.layers.75.mlp.experts.138.gate_proj", + "model.layers.75.mlp.experts.138.up_proj", + "model.layers.75.mlp.experts.139.down_proj", + "model.layers.75.mlp.experts.139.gate_proj", + "model.layers.75.mlp.experts.139.up_proj", + "model.layers.75.mlp.experts.14.down_proj", + "model.layers.75.mlp.experts.14.gate_proj", + "model.layers.75.mlp.experts.14.up_proj", + "model.layers.75.mlp.experts.140.down_proj", + "model.layers.75.mlp.experts.140.gate_proj", + "model.layers.75.mlp.experts.140.up_proj", + "model.layers.75.mlp.experts.141.down_proj", + "model.layers.75.mlp.experts.141.gate_proj", + "model.layers.75.mlp.experts.141.up_proj", + "model.layers.75.mlp.experts.142.down_proj", + "model.layers.75.mlp.experts.142.gate_proj", + "model.layers.75.mlp.experts.142.up_proj", + "model.layers.75.mlp.experts.143.down_proj", + "model.layers.75.mlp.experts.143.gate_proj", + "model.layers.75.mlp.experts.143.up_proj", + "model.layers.75.mlp.experts.144.down_proj", + "model.layers.75.mlp.experts.144.gate_proj", + "model.layers.75.mlp.experts.144.up_proj", + "model.layers.75.mlp.experts.145.down_proj", + "model.layers.75.mlp.experts.145.gate_proj", + "model.layers.75.mlp.experts.145.up_proj", + "model.layers.75.mlp.experts.146.down_proj", + "model.layers.75.mlp.experts.146.gate_proj", + "model.layers.75.mlp.experts.146.up_proj", + "model.layers.75.mlp.experts.147.down_proj", + "model.layers.75.mlp.experts.147.gate_proj", + "model.layers.75.mlp.experts.147.up_proj", + "model.layers.75.mlp.experts.148.down_proj", + "model.layers.75.mlp.experts.148.gate_proj", + "model.layers.75.mlp.experts.148.up_proj", + "model.layers.75.mlp.experts.149.down_proj", + "model.layers.75.mlp.experts.149.gate_proj", + "model.layers.75.mlp.experts.149.up_proj", + "model.layers.75.mlp.experts.15.down_proj", + "model.layers.75.mlp.experts.15.gate_proj", + "model.layers.75.mlp.experts.15.up_proj", + "model.layers.75.mlp.experts.150.down_proj", + "model.layers.75.mlp.experts.150.gate_proj", + "model.layers.75.mlp.experts.150.up_proj", + "model.layers.75.mlp.experts.151.down_proj", + "model.layers.75.mlp.experts.151.gate_proj", + "model.layers.75.mlp.experts.151.up_proj", + "model.layers.75.mlp.experts.152.down_proj", + "model.layers.75.mlp.experts.152.gate_proj", + "model.layers.75.mlp.experts.152.up_proj", + "model.layers.75.mlp.experts.153.down_proj", + "model.layers.75.mlp.experts.153.gate_proj", + "model.layers.75.mlp.experts.153.up_proj", + "model.layers.75.mlp.experts.154.down_proj", + "model.layers.75.mlp.experts.154.gate_proj", + "model.layers.75.mlp.experts.154.up_proj", + "model.layers.75.mlp.experts.155.down_proj", + "model.layers.75.mlp.experts.155.gate_proj", + "model.layers.75.mlp.experts.155.up_proj", + "model.layers.75.mlp.experts.156.down_proj", + "model.layers.75.mlp.experts.156.gate_proj", + "model.layers.75.mlp.experts.156.up_proj", + "model.layers.75.mlp.experts.157.down_proj", + "model.layers.75.mlp.experts.157.gate_proj", + "model.layers.75.mlp.experts.157.up_proj", + "model.layers.75.mlp.experts.158.down_proj", + "model.layers.75.mlp.experts.158.gate_proj", + "model.layers.75.mlp.experts.158.up_proj", + "model.layers.75.mlp.experts.159.down_proj", + "model.layers.75.mlp.experts.159.gate_proj", + "model.layers.75.mlp.experts.159.up_proj", + "model.layers.75.mlp.experts.16.down_proj", + "model.layers.75.mlp.experts.16.gate_proj", + "model.layers.75.mlp.experts.16.up_proj", + "model.layers.75.mlp.experts.160.down_proj", + "model.layers.75.mlp.experts.160.gate_proj", + "model.layers.75.mlp.experts.160.up_proj", + "model.layers.75.mlp.experts.161.down_proj", + "model.layers.75.mlp.experts.161.gate_proj", + "model.layers.75.mlp.experts.161.up_proj", + "model.layers.75.mlp.experts.162.down_proj", + "model.layers.75.mlp.experts.162.gate_proj", + "model.layers.75.mlp.experts.162.up_proj", + "model.layers.75.mlp.experts.163.down_proj", + "model.layers.75.mlp.experts.163.gate_proj", + "model.layers.75.mlp.experts.163.up_proj", + "model.layers.75.mlp.experts.164.down_proj", + "model.layers.75.mlp.experts.164.gate_proj", + "model.layers.75.mlp.experts.164.up_proj", + "model.layers.75.mlp.experts.165.down_proj", + "model.layers.75.mlp.experts.165.gate_proj", + "model.layers.75.mlp.experts.165.up_proj", + "model.layers.75.mlp.experts.166.down_proj", + "model.layers.75.mlp.experts.166.gate_proj", + "model.layers.75.mlp.experts.166.up_proj", + "model.layers.75.mlp.experts.167.down_proj", + "model.layers.75.mlp.experts.167.gate_proj", + "model.layers.75.mlp.experts.167.up_proj", + "model.layers.75.mlp.experts.168.down_proj", + "model.layers.75.mlp.experts.168.gate_proj", + "model.layers.75.mlp.experts.168.up_proj", + "model.layers.75.mlp.experts.169.down_proj", + "model.layers.75.mlp.experts.169.gate_proj", + "model.layers.75.mlp.experts.169.up_proj", + "model.layers.75.mlp.experts.17.down_proj", + "model.layers.75.mlp.experts.17.gate_proj", + "model.layers.75.mlp.experts.17.up_proj", + "model.layers.75.mlp.experts.170.down_proj", + "model.layers.75.mlp.experts.170.gate_proj", + "model.layers.75.mlp.experts.170.up_proj", + "model.layers.75.mlp.experts.171.down_proj", + "model.layers.75.mlp.experts.171.gate_proj", + "model.layers.75.mlp.experts.171.up_proj", + "model.layers.75.mlp.experts.172.down_proj", + "model.layers.75.mlp.experts.172.gate_proj", + "model.layers.75.mlp.experts.172.up_proj", + "model.layers.75.mlp.experts.173.down_proj", + "model.layers.75.mlp.experts.173.gate_proj", + "model.layers.75.mlp.experts.173.up_proj", + "model.layers.75.mlp.experts.174.down_proj", + "model.layers.75.mlp.experts.174.gate_proj", + "model.layers.75.mlp.experts.174.up_proj", + "model.layers.75.mlp.experts.175.down_proj", + "model.layers.75.mlp.experts.175.gate_proj", + "model.layers.75.mlp.experts.175.up_proj", + "model.layers.75.mlp.experts.176.down_proj", + "model.layers.75.mlp.experts.176.gate_proj", + "model.layers.75.mlp.experts.176.up_proj", + "model.layers.75.mlp.experts.177.down_proj", + "model.layers.75.mlp.experts.177.gate_proj", + "model.layers.75.mlp.experts.177.up_proj", + "model.layers.75.mlp.experts.178.down_proj", + "model.layers.75.mlp.experts.178.gate_proj", + "model.layers.75.mlp.experts.178.up_proj", + "model.layers.75.mlp.experts.179.down_proj", + "model.layers.75.mlp.experts.179.gate_proj", + "model.layers.75.mlp.experts.179.up_proj", + "model.layers.75.mlp.experts.18.down_proj", + "model.layers.75.mlp.experts.18.gate_proj", + "model.layers.75.mlp.experts.18.up_proj", + "model.layers.75.mlp.experts.180.down_proj", + "model.layers.75.mlp.experts.180.gate_proj", + "model.layers.75.mlp.experts.180.up_proj", + "model.layers.75.mlp.experts.181.down_proj", + "model.layers.75.mlp.experts.181.gate_proj", + "model.layers.75.mlp.experts.181.up_proj", + "model.layers.75.mlp.experts.182.down_proj", + "model.layers.75.mlp.experts.182.gate_proj", + "model.layers.75.mlp.experts.182.up_proj", + "model.layers.75.mlp.experts.183.down_proj", + "model.layers.75.mlp.experts.183.gate_proj", + "model.layers.75.mlp.experts.183.up_proj", + "model.layers.75.mlp.experts.184.down_proj", + "model.layers.75.mlp.experts.184.gate_proj", + "model.layers.75.mlp.experts.184.up_proj", + "model.layers.75.mlp.experts.185.down_proj", + "model.layers.75.mlp.experts.185.gate_proj", + "model.layers.75.mlp.experts.185.up_proj", + "model.layers.75.mlp.experts.186.down_proj", + "model.layers.75.mlp.experts.186.gate_proj", + "model.layers.75.mlp.experts.186.up_proj", + "model.layers.75.mlp.experts.187.down_proj", + "model.layers.75.mlp.experts.187.gate_proj", + "model.layers.75.mlp.experts.187.up_proj", + "model.layers.75.mlp.experts.188.down_proj", + "model.layers.75.mlp.experts.188.gate_proj", + "model.layers.75.mlp.experts.188.up_proj", + "model.layers.75.mlp.experts.189.down_proj", + "model.layers.75.mlp.experts.189.gate_proj", + "model.layers.75.mlp.experts.189.up_proj", + "model.layers.75.mlp.experts.19.down_proj", + "model.layers.75.mlp.experts.19.gate_proj", + "model.layers.75.mlp.experts.19.up_proj", + "model.layers.75.mlp.experts.190.down_proj", + "model.layers.75.mlp.experts.190.gate_proj", + "model.layers.75.mlp.experts.190.up_proj", + "model.layers.75.mlp.experts.191.down_proj", + "model.layers.75.mlp.experts.191.gate_proj", + "model.layers.75.mlp.experts.191.up_proj", + "model.layers.75.mlp.experts.192.down_proj", + "model.layers.75.mlp.experts.192.gate_proj", + "model.layers.75.mlp.experts.192.up_proj", + "model.layers.75.mlp.experts.193.down_proj", + "model.layers.75.mlp.experts.193.gate_proj", + "model.layers.75.mlp.experts.193.up_proj", + "model.layers.75.mlp.experts.194.down_proj", + "model.layers.75.mlp.experts.194.gate_proj", + "model.layers.75.mlp.experts.194.up_proj", + "model.layers.75.mlp.experts.195.down_proj", + "model.layers.75.mlp.experts.195.gate_proj", + "model.layers.75.mlp.experts.195.up_proj", + "model.layers.75.mlp.experts.196.down_proj", + "model.layers.75.mlp.experts.196.gate_proj", + "model.layers.75.mlp.experts.196.up_proj", + "model.layers.75.mlp.experts.197.down_proj", + "model.layers.75.mlp.experts.197.gate_proj", + "model.layers.75.mlp.experts.197.up_proj", + "model.layers.75.mlp.experts.198.down_proj", + "model.layers.75.mlp.experts.198.gate_proj", + "model.layers.75.mlp.experts.198.up_proj", + "model.layers.75.mlp.experts.199.down_proj", + "model.layers.75.mlp.experts.199.gate_proj", + "model.layers.75.mlp.experts.199.up_proj", + "model.layers.75.mlp.experts.2.down_proj", + "model.layers.75.mlp.experts.2.gate_proj", + "model.layers.75.mlp.experts.2.up_proj", + "model.layers.75.mlp.experts.20.down_proj", + "model.layers.75.mlp.experts.20.gate_proj", + "model.layers.75.mlp.experts.20.up_proj", + "model.layers.75.mlp.experts.200.down_proj", + "model.layers.75.mlp.experts.200.gate_proj", + "model.layers.75.mlp.experts.200.up_proj", + "model.layers.75.mlp.experts.201.down_proj", + "model.layers.75.mlp.experts.201.gate_proj", + "model.layers.75.mlp.experts.201.up_proj", + "model.layers.75.mlp.experts.202.down_proj", + "model.layers.75.mlp.experts.202.gate_proj", + "model.layers.75.mlp.experts.202.up_proj", + "model.layers.75.mlp.experts.203.down_proj", + "model.layers.75.mlp.experts.203.gate_proj", + "model.layers.75.mlp.experts.203.up_proj", + "model.layers.75.mlp.experts.204.down_proj", + "model.layers.75.mlp.experts.204.gate_proj", + "model.layers.75.mlp.experts.204.up_proj", + "model.layers.75.mlp.experts.205.down_proj", + "model.layers.75.mlp.experts.205.gate_proj", + "model.layers.75.mlp.experts.205.up_proj", + "model.layers.75.mlp.experts.206.down_proj", + "model.layers.75.mlp.experts.206.gate_proj", + "model.layers.75.mlp.experts.206.up_proj", + "model.layers.75.mlp.experts.207.down_proj", + "model.layers.75.mlp.experts.207.gate_proj", + "model.layers.75.mlp.experts.207.up_proj", + "model.layers.75.mlp.experts.208.down_proj", + "model.layers.75.mlp.experts.208.gate_proj", + "model.layers.75.mlp.experts.208.up_proj", + "model.layers.75.mlp.experts.209.down_proj", + "model.layers.75.mlp.experts.209.gate_proj", + "model.layers.75.mlp.experts.209.up_proj", + "model.layers.75.mlp.experts.21.down_proj", + "model.layers.75.mlp.experts.21.gate_proj", + "model.layers.75.mlp.experts.21.up_proj", + "model.layers.75.mlp.experts.210.down_proj", + "model.layers.75.mlp.experts.210.gate_proj", + "model.layers.75.mlp.experts.210.up_proj", + "model.layers.75.mlp.experts.211.down_proj", + "model.layers.75.mlp.experts.211.gate_proj", + "model.layers.75.mlp.experts.211.up_proj", + "model.layers.75.mlp.experts.212.down_proj", + "model.layers.75.mlp.experts.212.gate_proj", + "model.layers.75.mlp.experts.212.up_proj", + "model.layers.75.mlp.experts.213.down_proj", + "model.layers.75.mlp.experts.213.gate_proj", + "model.layers.75.mlp.experts.213.up_proj", + "model.layers.75.mlp.experts.214.down_proj", + "model.layers.75.mlp.experts.214.gate_proj", + "model.layers.75.mlp.experts.214.up_proj", + "model.layers.75.mlp.experts.215.down_proj", + "model.layers.75.mlp.experts.215.gate_proj", + "model.layers.75.mlp.experts.215.up_proj", + "model.layers.75.mlp.experts.216.down_proj", + "model.layers.75.mlp.experts.216.gate_proj", + "model.layers.75.mlp.experts.216.up_proj", + "model.layers.75.mlp.experts.217.down_proj", + "model.layers.75.mlp.experts.217.gate_proj", + "model.layers.75.mlp.experts.217.up_proj", + "model.layers.75.mlp.experts.218.down_proj", + "model.layers.75.mlp.experts.218.gate_proj", + "model.layers.75.mlp.experts.218.up_proj", + "model.layers.75.mlp.experts.219.down_proj", + "model.layers.75.mlp.experts.219.gate_proj", + "model.layers.75.mlp.experts.219.up_proj", + "model.layers.75.mlp.experts.22.down_proj", + "model.layers.75.mlp.experts.22.gate_proj", + "model.layers.75.mlp.experts.22.up_proj", + "model.layers.75.mlp.experts.220.down_proj", + "model.layers.75.mlp.experts.220.gate_proj", + "model.layers.75.mlp.experts.220.up_proj", + "model.layers.75.mlp.experts.221.down_proj", + "model.layers.75.mlp.experts.221.gate_proj", + "model.layers.75.mlp.experts.221.up_proj", + "model.layers.75.mlp.experts.222.down_proj", + "model.layers.75.mlp.experts.222.gate_proj", + "model.layers.75.mlp.experts.222.up_proj", + "model.layers.75.mlp.experts.223.down_proj", + "model.layers.75.mlp.experts.223.gate_proj", + "model.layers.75.mlp.experts.223.up_proj", + "model.layers.75.mlp.experts.224.down_proj", + "model.layers.75.mlp.experts.224.gate_proj", + "model.layers.75.mlp.experts.224.up_proj", + "model.layers.75.mlp.experts.225.down_proj", + "model.layers.75.mlp.experts.225.gate_proj", + "model.layers.75.mlp.experts.225.up_proj", + "model.layers.75.mlp.experts.226.down_proj", + "model.layers.75.mlp.experts.226.gate_proj", + "model.layers.75.mlp.experts.226.up_proj", + "model.layers.75.mlp.experts.227.down_proj", + "model.layers.75.mlp.experts.227.gate_proj", + "model.layers.75.mlp.experts.227.up_proj", + "model.layers.75.mlp.experts.228.down_proj", + "model.layers.75.mlp.experts.228.gate_proj", + "model.layers.75.mlp.experts.228.up_proj", + "model.layers.75.mlp.experts.229.down_proj", + "model.layers.75.mlp.experts.229.gate_proj", + "model.layers.75.mlp.experts.229.up_proj", + "model.layers.75.mlp.experts.23.down_proj", + "model.layers.75.mlp.experts.23.gate_proj", + "model.layers.75.mlp.experts.23.up_proj", + "model.layers.75.mlp.experts.230.down_proj", + "model.layers.75.mlp.experts.230.gate_proj", + "model.layers.75.mlp.experts.230.up_proj", + "model.layers.75.mlp.experts.231.down_proj", + "model.layers.75.mlp.experts.231.gate_proj", + "model.layers.75.mlp.experts.231.up_proj", + "model.layers.75.mlp.experts.232.down_proj", + "model.layers.75.mlp.experts.232.gate_proj", + "model.layers.75.mlp.experts.232.up_proj", + "model.layers.75.mlp.experts.233.down_proj", + "model.layers.75.mlp.experts.233.gate_proj", + "model.layers.75.mlp.experts.233.up_proj", + "model.layers.75.mlp.experts.234.down_proj", + "model.layers.75.mlp.experts.234.gate_proj", + "model.layers.75.mlp.experts.234.up_proj", + "model.layers.75.mlp.experts.235.down_proj", + "model.layers.75.mlp.experts.235.gate_proj", + "model.layers.75.mlp.experts.235.up_proj", + "model.layers.75.mlp.experts.236.down_proj", + "model.layers.75.mlp.experts.236.gate_proj", + "model.layers.75.mlp.experts.236.up_proj", + "model.layers.75.mlp.experts.237.down_proj", + "model.layers.75.mlp.experts.237.gate_proj", + "model.layers.75.mlp.experts.237.up_proj", + "model.layers.75.mlp.experts.238.down_proj", + "model.layers.75.mlp.experts.238.gate_proj", + "model.layers.75.mlp.experts.238.up_proj", + "model.layers.75.mlp.experts.239.down_proj", + "model.layers.75.mlp.experts.239.gate_proj", + "model.layers.75.mlp.experts.239.up_proj", + "model.layers.75.mlp.experts.24.down_proj", + "model.layers.75.mlp.experts.24.gate_proj", + "model.layers.75.mlp.experts.24.up_proj", + "model.layers.75.mlp.experts.240.down_proj", + "model.layers.75.mlp.experts.240.gate_proj", + "model.layers.75.mlp.experts.240.up_proj", + "model.layers.75.mlp.experts.241.down_proj", + "model.layers.75.mlp.experts.241.gate_proj", + "model.layers.75.mlp.experts.241.up_proj", + "model.layers.75.mlp.experts.242.down_proj", + "model.layers.75.mlp.experts.242.gate_proj", + "model.layers.75.mlp.experts.242.up_proj", + "model.layers.75.mlp.experts.243.down_proj", + "model.layers.75.mlp.experts.243.gate_proj", + "model.layers.75.mlp.experts.243.up_proj", + "model.layers.75.mlp.experts.244.down_proj", + "model.layers.75.mlp.experts.244.gate_proj", + "model.layers.75.mlp.experts.244.up_proj", + "model.layers.75.mlp.experts.245.down_proj", + "model.layers.75.mlp.experts.245.gate_proj", + "model.layers.75.mlp.experts.245.up_proj", + "model.layers.75.mlp.experts.246.down_proj", + "model.layers.75.mlp.experts.246.gate_proj", + "model.layers.75.mlp.experts.246.up_proj", + "model.layers.75.mlp.experts.247.down_proj", + "model.layers.75.mlp.experts.247.gate_proj", + "model.layers.75.mlp.experts.247.up_proj", + "model.layers.75.mlp.experts.248.down_proj", + "model.layers.75.mlp.experts.248.gate_proj", + "model.layers.75.mlp.experts.248.up_proj", + "model.layers.75.mlp.experts.249.down_proj", + "model.layers.75.mlp.experts.249.gate_proj", + "model.layers.75.mlp.experts.249.up_proj", + "model.layers.75.mlp.experts.25.down_proj", + "model.layers.75.mlp.experts.25.gate_proj", + "model.layers.75.mlp.experts.25.up_proj", + "model.layers.75.mlp.experts.250.down_proj", + "model.layers.75.mlp.experts.250.gate_proj", + "model.layers.75.mlp.experts.250.up_proj", + "model.layers.75.mlp.experts.251.down_proj", + "model.layers.75.mlp.experts.251.gate_proj", + "model.layers.75.mlp.experts.251.up_proj", + "model.layers.75.mlp.experts.252.down_proj", + "model.layers.75.mlp.experts.252.gate_proj", + "model.layers.75.mlp.experts.252.up_proj", + "model.layers.75.mlp.experts.253.down_proj", + "model.layers.75.mlp.experts.253.gate_proj", + "model.layers.75.mlp.experts.253.up_proj", + "model.layers.75.mlp.experts.254.down_proj", + "model.layers.75.mlp.experts.254.gate_proj", + "model.layers.75.mlp.experts.254.up_proj", + "model.layers.75.mlp.experts.255.down_proj", + "model.layers.75.mlp.experts.255.gate_proj", + "model.layers.75.mlp.experts.255.up_proj", + "model.layers.75.mlp.experts.26.down_proj", + "model.layers.75.mlp.experts.26.gate_proj", + "model.layers.75.mlp.experts.26.up_proj", + "model.layers.75.mlp.experts.27.down_proj", + "model.layers.75.mlp.experts.27.gate_proj", + "model.layers.75.mlp.experts.27.up_proj", + "model.layers.75.mlp.experts.28.down_proj", + "model.layers.75.mlp.experts.28.gate_proj", + "model.layers.75.mlp.experts.28.up_proj", + "model.layers.75.mlp.experts.29.down_proj", + "model.layers.75.mlp.experts.29.gate_proj", + "model.layers.75.mlp.experts.29.up_proj", + "model.layers.75.mlp.experts.3.down_proj", + "model.layers.75.mlp.experts.3.gate_proj", + "model.layers.75.mlp.experts.3.up_proj", + "model.layers.75.mlp.experts.30.down_proj", + "model.layers.75.mlp.experts.30.gate_proj", + "model.layers.75.mlp.experts.30.up_proj", + "model.layers.75.mlp.experts.31.down_proj", + "model.layers.75.mlp.experts.31.gate_proj", + "model.layers.75.mlp.experts.31.up_proj", + "model.layers.75.mlp.experts.32.down_proj", + "model.layers.75.mlp.experts.32.gate_proj", + "model.layers.75.mlp.experts.32.up_proj", + "model.layers.75.mlp.experts.33.down_proj", + "model.layers.75.mlp.experts.33.gate_proj", + "model.layers.75.mlp.experts.33.up_proj", + "model.layers.75.mlp.experts.34.down_proj", + "model.layers.75.mlp.experts.34.gate_proj", + "model.layers.75.mlp.experts.34.up_proj", + "model.layers.75.mlp.experts.35.down_proj", + "model.layers.75.mlp.experts.35.gate_proj", + "model.layers.75.mlp.experts.35.up_proj", + "model.layers.75.mlp.experts.36.down_proj", + "model.layers.75.mlp.experts.36.gate_proj", + "model.layers.75.mlp.experts.36.up_proj", + "model.layers.75.mlp.experts.37.down_proj", + "model.layers.75.mlp.experts.37.gate_proj", + "model.layers.75.mlp.experts.37.up_proj", + "model.layers.75.mlp.experts.38.down_proj", + "model.layers.75.mlp.experts.38.gate_proj", + "model.layers.75.mlp.experts.38.up_proj", + "model.layers.75.mlp.experts.39.down_proj", + "model.layers.75.mlp.experts.39.gate_proj", + "model.layers.75.mlp.experts.39.up_proj", + "model.layers.75.mlp.experts.4.down_proj", + "model.layers.75.mlp.experts.4.gate_proj", + "model.layers.75.mlp.experts.4.up_proj", + "model.layers.75.mlp.experts.40.down_proj", + "model.layers.75.mlp.experts.40.gate_proj", + "model.layers.75.mlp.experts.40.up_proj", + "model.layers.75.mlp.experts.41.down_proj", + "model.layers.75.mlp.experts.41.gate_proj", + "model.layers.75.mlp.experts.41.up_proj", + "model.layers.75.mlp.experts.42.down_proj", + "model.layers.75.mlp.experts.42.gate_proj", + "model.layers.75.mlp.experts.42.up_proj", + "model.layers.75.mlp.experts.43.down_proj", + "model.layers.75.mlp.experts.43.gate_proj", + "model.layers.75.mlp.experts.43.up_proj", + "model.layers.75.mlp.experts.44.down_proj", + "model.layers.75.mlp.experts.44.gate_proj", + "model.layers.75.mlp.experts.44.up_proj", + "model.layers.75.mlp.experts.45.down_proj", + "model.layers.75.mlp.experts.45.gate_proj", + "model.layers.75.mlp.experts.45.up_proj", + "model.layers.75.mlp.experts.46.down_proj", + "model.layers.75.mlp.experts.46.gate_proj", + "model.layers.75.mlp.experts.46.up_proj", + "model.layers.75.mlp.experts.47.down_proj", + "model.layers.75.mlp.experts.47.gate_proj", + "model.layers.75.mlp.experts.47.up_proj", + "model.layers.75.mlp.experts.48.down_proj", + "model.layers.75.mlp.experts.48.gate_proj", + "model.layers.75.mlp.experts.48.up_proj", + "model.layers.75.mlp.experts.49.down_proj", + "model.layers.75.mlp.experts.49.gate_proj", + "model.layers.75.mlp.experts.49.up_proj", + "model.layers.75.mlp.experts.5.down_proj", + "model.layers.75.mlp.experts.5.gate_proj", + "model.layers.75.mlp.experts.5.up_proj", + "model.layers.75.mlp.experts.50.down_proj", + "model.layers.75.mlp.experts.50.gate_proj", + "model.layers.75.mlp.experts.50.up_proj", + "model.layers.75.mlp.experts.51.down_proj", + "model.layers.75.mlp.experts.51.gate_proj", + "model.layers.75.mlp.experts.51.up_proj", + "model.layers.75.mlp.experts.52.down_proj", + "model.layers.75.mlp.experts.52.gate_proj", + "model.layers.75.mlp.experts.52.up_proj", + "model.layers.75.mlp.experts.53.down_proj", + "model.layers.75.mlp.experts.53.gate_proj", + "model.layers.75.mlp.experts.53.up_proj", + "model.layers.75.mlp.experts.54.down_proj", + "model.layers.75.mlp.experts.54.gate_proj", + "model.layers.75.mlp.experts.54.up_proj", + "model.layers.75.mlp.experts.55.down_proj", + "model.layers.75.mlp.experts.55.gate_proj", + "model.layers.75.mlp.experts.55.up_proj", + "model.layers.75.mlp.experts.56.down_proj", + "model.layers.75.mlp.experts.56.gate_proj", + "model.layers.75.mlp.experts.56.up_proj", + "model.layers.75.mlp.experts.57.down_proj", + "model.layers.75.mlp.experts.57.gate_proj", + "model.layers.75.mlp.experts.57.up_proj", + "model.layers.75.mlp.experts.58.down_proj", + "model.layers.75.mlp.experts.58.gate_proj", + "model.layers.75.mlp.experts.58.up_proj", + "model.layers.75.mlp.experts.59.down_proj", + "model.layers.75.mlp.experts.59.gate_proj", + "model.layers.75.mlp.experts.59.up_proj", + "model.layers.75.mlp.experts.6.down_proj", + "model.layers.75.mlp.experts.6.gate_proj", + "model.layers.75.mlp.experts.6.up_proj", + "model.layers.75.mlp.experts.60.down_proj", + "model.layers.75.mlp.experts.60.gate_proj", + "model.layers.75.mlp.experts.60.up_proj", + "model.layers.75.mlp.experts.61.down_proj", + "model.layers.75.mlp.experts.61.gate_proj", + "model.layers.75.mlp.experts.61.up_proj", + "model.layers.75.mlp.experts.62.down_proj", + "model.layers.75.mlp.experts.62.gate_proj", + "model.layers.75.mlp.experts.62.up_proj", + "model.layers.75.mlp.experts.63.down_proj", + "model.layers.75.mlp.experts.63.gate_proj", + "model.layers.75.mlp.experts.63.up_proj", + "model.layers.75.mlp.experts.64.down_proj", + "model.layers.75.mlp.experts.64.gate_proj", + "model.layers.75.mlp.experts.64.up_proj", + "model.layers.75.mlp.experts.65.down_proj", + "model.layers.75.mlp.experts.65.gate_proj", + "model.layers.75.mlp.experts.65.up_proj", + "model.layers.75.mlp.experts.66.down_proj", + "model.layers.75.mlp.experts.66.gate_proj", + "model.layers.75.mlp.experts.66.up_proj", + "model.layers.75.mlp.experts.67.down_proj", + "model.layers.75.mlp.experts.67.gate_proj", + "model.layers.75.mlp.experts.67.up_proj", + "model.layers.75.mlp.experts.68.down_proj", + "model.layers.75.mlp.experts.68.gate_proj", + "model.layers.75.mlp.experts.68.up_proj", + "model.layers.75.mlp.experts.69.down_proj", + "model.layers.75.mlp.experts.69.gate_proj", + "model.layers.75.mlp.experts.69.up_proj", + "model.layers.75.mlp.experts.7.down_proj", + "model.layers.75.mlp.experts.7.gate_proj", + "model.layers.75.mlp.experts.7.up_proj", + "model.layers.75.mlp.experts.70.down_proj", + "model.layers.75.mlp.experts.70.gate_proj", + "model.layers.75.mlp.experts.70.up_proj", + "model.layers.75.mlp.experts.71.down_proj", + "model.layers.75.mlp.experts.71.gate_proj", + "model.layers.75.mlp.experts.71.up_proj", + "model.layers.75.mlp.experts.72.down_proj", + "model.layers.75.mlp.experts.72.gate_proj", + "model.layers.75.mlp.experts.72.up_proj", + "model.layers.75.mlp.experts.73.down_proj", + "model.layers.75.mlp.experts.73.gate_proj", + "model.layers.75.mlp.experts.73.up_proj", + "model.layers.75.mlp.experts.74.down_proj", + "model.layers.75.mlp.experts.74.gate_proj", + "model.layers.75.mlp.experts.74.up_proj", + "model.layers.75.mlp.experts.75.down_proj", + "model.layers.75.mlp.experts.75.gate_proj", + "model.layers.75.mlp.experts.75.up_proj", + "model.layers.75.mlp.experts.76.down_proj", + "model.layers.75.mlp.experts.76.gate_proj", + "model.layers.75.mlp.experts.76.up_proj", + "model.layers.75.mlp.experts.77.down_proj", + "model.layers.75.mlp.experts.77.gate_proj", + "model.layers.75.mlp.experts.77.up_proj", + "model.layers.75.mlp.experts.78.down_proj", + "model.layers.75.mlp.experts.78.gate_proj", + "model.layers.75.mlp.experts.78.up_proj", + "model.layers.75.mlp.experts.79.down_proj", + "model.layers.75.mlp.experts.79.gate_proj", + "model.layers.75.mlp.experts.79.up_proj", + "model.layers.75.mlp.experts.8.down_proj", + "model.layers.75.mlp.experts.8.gate_proj", + "model.layers.75.mlp.experts.8.up_proj", + "model.layers.75.mlp.experts.80.down_proj", + "model.layers.75.mlp.experts.80.gate_proj", + "model.layers.75.mlp.experts.80.up_proj", + "model.layers.75.mlp.experts.81.down_proj", + "model.layers.75.mlp.experts.81.gate_proj", + "model.layers.75.mlp.experts.81.up_proj", + "model.layers.75.mlp.experts.82.down_proj", + "model.layers.75.mlp.experts.82.gate_proj", + "model.layers.75.mlp.experts.82.up_proj", + "model.layers.75.mlp.experts.83.down_proj", + "model.layers.75.mlp.experts.83.gate_proj", + "model.layers.75.mlp.experts.83.up_proj", + "model.layers.75.mlp.experts.84.down_proj", + "model.layers.75.mlp.experts.84.gate_proj", + "model.layers.75.mlp.experts.84.up_proj", + "model.layers.75.mlp.experts.85.down_proj", + "model.layers.75.mlp.experts.85.gate_proj", + "model.layers.75.mlp.experts.85.up_proj", + "model.layers.75.mlp.experts.86.down_proj", + "model.layers.75.mlp.experts.86.gate_proj", + "model.layers.75.mlp.experts.86.up_proj", + "model.layers.75.mlp.experts.87.down_proj", + "model.layers.75.mlp.experts.87.gate_proj", + "model.layers.75.mlp.experts.87.up_proj", + "model.layers.75.mlp.experts.88.down_proj", + "model.layers.75.mlp.experts.88.gate_proj", + "model.layers.75.mlp.experts.88.up_proj", + "model.layers.75.mlp.experts.89.down_proj", + "model.layers.75.mlp.experts.89.gate_proj", + "model.layers.75.mlp.experts.89.up_proj", + "model.layers.75.mlp.experts.9.down_proj", + "model.layers.75.mlp.experts.9.gate_proj", + "model.layers.75.mlp.experts.9.up_proj", + "model.layers.75.mlp.experts.90.down_proj", + "model.layers.75.mlp.experts.90.gate_proj", + "model.layers.75.mlp.experts.90.up_proj", + "model.layers.75.mlp.experts.91.down_proj", + "model.layers.75.mlp.experts.91.gate_proj", + "model.layers.75.mlp.experts.91.up_proj", + "model.layers.75.mlp.experts.92.down_proj", + "model.layers.75.mlp.experts.92.gate_proj", + "model.layers.75.mlp.experts.92.up_proj", + "model.layers.75.mlp.experts.93.down_proj", + "model.layers.75.mlp.experts.93.gate_proj", + "model.layers.75.mlp.experts.93.up_proj", + "model.layers.75.mlp.experts.94.down_proj", + "model.layers.75.mlp.experts.94.gate_proj", + "model.layers.75.mlp.experts.94.up_proj", + "model.layers.75.mlp.experts.95.down_proj", + "model.layers.75.mlp.experts.95.gate_proj", + "model.layers.75.mlp.experts.95.up_proj", + "model.layers.75.mlp.experts.96.down_proj", + "model.layers.75.mlp.experts.96.gate_proj", + "model.layers.75.mlp.experts.96.up_proj", + "model.layers.75.mlp.experts.97.down_proj", + "model.layers.75.mlp.experts.97.gate_proj", + "model.layers.75.mlp.experts.97.up_proj", + "model.layers.75.mlp.experts.98.down_proj", + "model.layers.75.mlp.experts.98.gate_proj", + "model.layers.75.mlp.experts.98.up_proj", + "model.layers.75.mlp.experts.99.down_proj", + "model.layers.75.mlp.experts.99.gate_proj", + "model.layers.75.mlp.experts.99.up_proj", + "model.layers.76.mlp.experts.0.down_proj", + "model.layers.76.mlp.experts.0.gate_proj", + "model.layers.76.mlp.experts.0.up_proj", + "model.layers.76.mlp.experts.1.down_proj", + "model.layers.76.mlp.experts.1.gate_proj", + "model.layers.76.mlp.experts.1.up_proj", + "model.layers.76.mlp.experts.10.down_proj", + "model.layers.76.mlp.experts.10.gate_proj", + "model.layers.76.mlp.experts.10.up_proj", + "model.layers.76.mlp.experts.100.down_proj", + "model.layers.76.mlp.experts.100.gate_proj", + "model.layers.76.mlp.experts.100.up_proj", + "model.layers.76.mlp.experts.101.down_proj", + "model.layers.76.mlp.experts.101.gate_proj", + "model.layers.76.mlp.experts.101.up_proj", + "model.layers.76.mlp.experts.102.down_proj", + "model.layers.76.mlp.experts.102.gate_proj", + "model.layers.76.mlp.experts.102.up_proj", + "model.layers.76.mlp.experts.103.down_proj", + "model.layers.76.mlp.experts.103.gate_proj", + "model.layers.76.mlp.experts.103.up_proj", + "model.layers.76.mlp.experts.104.down_proj", + "model.layers.76.mlp.experts.104.gate_proj", + "model.layers.76.mlp.experts.104.up_proj", + "model.layers.76.mlp.experts.105.down_proj", + "model.layers.76.mlp.experts.105.gate_proj", + "model.layers.76.mlp.experts.105.up_proj", + "model.layers.76.mlp.experts.106.down_proj", + "model.layers.76.mlp.experts.106.gate_proj", + "model.layers.76.mlp.experts.106.up_proj", + "model.layers.76.mlp.experts.107.down_proj", + "model.layers.76.mlp.experts.107.gate_proj", + "model.layers.76.mlp.experts.107.up_proj", + "model.layers.76.mlp.experts.108.down_proj", + "model.layers.76.mlp.experts.108.gate_proj", + "model.layers.76.mlp.experts.108.up_proj", + "model.layers.76.mlp.experts.109.down_proj", + "model.layers.76.mlp.experts.109.gate_proj", + "model.layers.76.mlp.experts.109.up_proj", + "model.layers.76.mlp.experts.11.down_proj", + "model.layers.76.mlp.experts.11.gate_proj", + "model.layers.76.mlp.experts.11.up_proj", + "model.layers.76.mlp.experts.110.down_proj", + "model.layers.76.mlp.experts.110.gate_proj", + "model.layers.76.mlp.experts.110.up_proj", + "model.layers.76.mlp.experts.111.down_proj", + "model.layers.76.mlp.experts.111.gate_proj", + "model.layers.76.mlp.experts.111.up_proj", + "model.layers.76.mlp.experts.112.down_proj", + "model.layers.76.mlp.experts.112.gate_proj", + "model.layers.76.mlp.experts.112.up_proj", + "model.layers.76.mlp.experts.113.down_proj", + "model.layers.76.mlp.experts.113.gate_proj", + "model.layers.76.mlp.experts.113.up_proj", + "model.layers.76.mlp.experts.114.down_proj", + "model.layers.76.mlp.experts.114.gate_proj", + "model.layers.76.mlp.experts.114.up_proj", + "model.layers.76.mlp.experts.115.down_proj", + "model.layers.76.mlp.experts.115.gate_proj", + "model.layers.76.mlp.experts.115.up_proj", + "model.layers.76.mlp.experts.116.down_proj", + "model.layers.76.mlp.experts.116.gate_proj", + "model.layers.76.mlp.experts.116.up_proj", + "model.layers.76.mlp.experts.117.down_proj", + "model.layers.76.mlp.experts.117.gate_proj", + "model.layers.76.mlp.experts.117.up_proj", + "model.layers.76.mlp.experts.118.down_proj", + "model.layers.76.mlp.experts.118.gate_proj", + "model.layers.76.mlp.experts.118.up_proj", + "model.layers.76.mlp.experts.119.down_proj", + "model.layers.76.mlp.experts.119.gate_proj", + "model.layers.76.mlp.experts.119.up_proj", + "model.layers.76.mlp.experts.12.down_proj", + "model.layers.76.mlp.experts.12.gate_proj", + "model.layers.76.mlp.experts.12.up_proj", + "model.layers.76.mlp.experts.120.down_proj", + "model.layers.76.mlp.experts.120.gate_proj", + "model.layers.76.mlp.experts.120.up_proj", + "model.layers.76.mlp.experts.121.down_proj", + "model.layers.76.mlp.experts.121.gate_proj", + "model.layers.76.mlp.experts.121.up_proj", + "model.layers.76.mlp.experts.122.down_proj", + "model.layers.76.mlp.experts.122.gate_proj", + "model.layers.76.mlp.experts.122.up_proj", + "model.layers.76.mlp.experts.123.down_proj", + "model.layers.76.mlp.experts.123.gate_proj", + "model.layers.76.mlp.experts.123.up_proj", + "model.layers.76.mlp.experts.124.down_proj", + "model.layers.76.mlp.experts.124.gate_proj", + "model.layers.76.mlp.experts.124.up_proj", + "model.layers.76.mlp.experts.125.down_proj", + "model.layers.76.mlp.experts.125.gate_proj", + "model.layers.76.mlp.experts.125.up_proj", + "model.layers.76.mlp.experts.126.down_proj", + "model.layers.76.mlp.experts.126.gate_proj", + "model.layers.76.mlp.experts.126.up_proj", + "model.layers.76.mlp.experts.127.down_proj", + "model.layers.76.mlp.experts.127.gate_proj", + "model.layers.76.mlp.experts.127.up_proj", + "model.layers.76.mlp.experts.128.down_proj", + "model.layers.76.mlp.experts.128.gate_proj", + "model.layers.76.mlp.experts.128.up_proj", + "model.layers.76.mlp.experts.129.down_proj", + "model.layers.76.mlp.experts.129.gate_proj", + "model.layers.76.mlp.experts.129.up_proj", + "model.layers.76.mlp.experts.13.down_proj", + "model.layers.76.mlp.experts.13.gate_proj", + "model.layers.76.mlp.experts.13.up_proj", + "model.layers.76.mlp.experts.130.down_proj", + "model.layers.76.mlp.experts.130.gate_proj", + "model.layers.76.mlp.experts.130.up_proj", + "model.layers.76.mlp.experts.131.down_proj", + "model.layers.76.mlp.experts.131.gate_proj", + "model.layers.76.mlp.experts.131.up_proj", + "model.layers.76.mlp.experts.132.down_proj", + "model.layers.76.mlp.experts.132.gate_proj", + "model.layers.76.mlp.experts.132.up_proj", + "model.layers.76.mlp.experts.133.down_proj", + "model.layers.76.mlp.experts.133.gate_proj", + "model.layers.76.mlp.experts.133.up_proj", + "model.layers.76.mlp.experts.134.down_proj", + "model.layers.76.mlp.experts.134.gate_proj", + "model.layers.76.mlp.experts.134.up_proj", + "model.layers.76.mlp.experts.135.down_proj", + "model.layers.76.mlp.experts.135.gate_proj", + "model.layers.76.mlp.experts.135.up_proj", + "model.layers.76.mlp.experts.136.down_proj", + "model.layers.76.mlp.experts.136.gate_proj", + "model.layers.76.mlp.experts.136.up_proj", + "model.layers.76.mlp.experts.137.down_proj", + "model.layers.76.mlp.experts.137.gate_proj", + "model.layers.76.mlp.experts.137.up_proj", + "model.layers.76.mlp.experts.138.down_proj", + "model.layers.76.mlp.experts.138.gate_proj", + "model.layers.76.mlp.experts.138.up_proj", + "model.layers.76.mlp.experts.139.down_proj", + "model.layers.76.mlp.experts.139.gate_proj", + "model.layers.76.mlp.experts.139.up_proj", + "model.layers.76.mlp.experts.14.down_proj", + "model.layers.76.mlp.experts.14.gate_proj", + "model.layers.76.mlp.experts.14.up_proj", + "model.layers.76.mlp.experts.140.down_proj", + "model.layers.76.mlp.experts.140.gate_proj", + "model.layers.76.mlp.experts.140.up_proj", + "model.layers.76.mlp.experts.141.down_proj", + "model.layers.76.mlp.experts.141.gate_proj", + "model.layers.76.mlp.experts.141.up_proj", + "model.layers.76.mlp.experts.142.down_proj", + "model.layers.76.mlp.experts.142.gate_proj", + "model.layers.76.mlp.experts.142.up_proj", + "model.layers.76.mlp.experts.143.down_proj", + "model.layers.76.mlp.experts.143.gate_proj", + "model.layers.76.mlp.experts.143.up_proj", + "model.layers.76.mlp.experts.144.down_proj", + "model.layers.76.mlp.experts.144.gate_proj", + "model.layers.76.mlp.experts.144.up_proj", + "model.layers.76.mlp.experts.145.down_proj", + "model.layers.76.mlp.experts.145.gate_proj", + "model.layers.76.mlp.experts.145.up_proj", + "model.layers.76.mlp.experts.146.down_proj", + "model.layers.76.mlp.experts.146.gate_proj", + "model.layers.76.mlp.experts.146.up_proj", + "model.layers.76.mlp.experts.147.down_proj", + "model.layers.76.mlp.experts.147.gate_proj", + "model.layers.76.mlp.experts.147.up_proj", + "model.layers.76.mlp.experts.148.down_proj", + "model.layers.76.mlp.experts.148.gate_proj", + "model.layers.76.mlp.experts.148.up_proj", + "model.layers.76.mlp.experts.149.down_proj", + "model.layers.76.mlp.experts.149.gate_proj", + "model.layers.76.mlp.experts.149.up_proj", + "model.layers.76.mlp.experts.15.down_proj", + "model.layers.76.mlp.experts.15.gate_proj", + "model.layers.76.mlp.experts.15.up_proj", + "model.layers.76.mlp.experts.150.down_proj", + "model.layers.76.mlp.experts.150.gate_proj", + "model.layers.76.mlp.experts.150.up_proj", + "model.layers.76.mlp.experts.151.down_proj", + "model.layers.76.mlp.experts.151.gate_proj", + "model.layers.76.mlp.experts.151.up_proj", + "model.layers.76.mlp.experts.152.down_proj", + "model.layers.76.mlp.experts.152.gate_proj", + "model.layers.76.mlp.experts.152.up_proj", + "model.layers.76.mlp.experts.153.down_proj", + "model.layers.76.mlp.experts.153.gate_proj", + "model.layers.76.mlp.experts.153.up_proj", + "model.layers.76.mlp.experts.154.down_proj", + "model.layers.76.mlp.experts.154.gate_proj", + "model.layers.76.mlp.experts.154.up_proj", + "model.layers.76.mlp.experts.155.down_proj", + "model.layers.76.mlp.experts.155.gate_proj", + "model.layers.76.mlp.experts.155.up_proj", + "model.layers.76.mlp.experts.156.down_proj", + "model.layers.76.mlp.experts.156.gate_proj", + "model.layers.76.mlp.experts.156.up_proj", + "model.layers.76.mlp.experts.157.down_proj", + "model.layers.76.mlp.experts.157.gate_proj", + "model.layers.76.mlp.experts.157.up_proj", + "model.layers.76.mlp.experts.158.down_proj", + "model.layers.76.mlp.experts.158.gate_proj", + "model.layers.76.mlp.experts.158.up_proj", + "model.layers.76.mlp.experts.159.down_proj", + "model.layers.76.mlp.experts.159.gate_proj", + "model.layers.76.mlp.experts.159.up_proj", + "model.layers.76.mlp.experts.16.down_proj", + "model.layers.76.mlp.experts.16.gate_proj", + "model.layers.76.mlp.experts.16.up_proj", + "model.layers.76.mlp.experts.160.down_proj", + "model.layers.76.mlp.experts.160.gate_proj", + "model.layers.76.mlp.experts.160.up_proj", + "model.layers.76.mlp.experts.161.down_proj", + "model.layers.76.mlp.experts.161.gate_proj", + "model.layers.76.mlp.experts.161.up_proj", + "model.layers.76.mlp.experts.162.down_proj", + "model.layers.76.mlp.experts.162.gate_proj", + "model.layers.76.mlp.experts.162.up_proj", + "model.layers.76.mlp.experts.163.down_proj", + "model.layers.76.mlp.experts.163.gate_proj", + "model.layers.76.mlp.experts.163.up_proj", + "model.layers.76.mlp.experts.164.down_proj", + "model.layers.76.mlp.experts.164.gate_proj", + "model.layers.76.mlp.experts.164.up_proj", + "model.layers.76.mlp.experts.165.down_proj", + "model.layers.76.mlp.experts.165.gate_proj", + "model.layers.76.mlp.experts.165.up_proj", + "model.layers.76.mlp.experts.166.down_proj", + "model.layers.76.mlp.experts.166.gate_proj", + "model.layers.76.mlp.experts.166.up_proj", + "model.layers.76.mlp.experts.167.down_proj", + "model.layers.76.mlp.experts.167.gate_proj", + "model.layers.76.mlp.experts.167.up_proj", + "model.layers.76.mlp.experts.168.down_proj", + "model.layers.76.mlp.experts.168.gate_proj", + "model.layers.76.mlp.experts.168.up_proj", + "model.layers.76.mlp.experts.169.down_proj", + "model.layers.76.mlp.experts.169.gate_proj", + "model.layers.76.mlp.experts.169.up_proj", + "model.layers.76.mlp.experts.17.down_proj", + "model.layers.76.mlp.experts.17.gate_proj", + "model.layers.76.mlp.experts.17.up_proj", + "model.layers.76.mlp.experts.170.down_proj", + "model.layers.76.mlp.experts.170.gate_proj", + "model.layers.76.mlp.experts.170.up_proj", + "model.layers.76.mlp.experts.171.down_proj", + "model.layers.76.mlp.experts.171.gate_proj", + "model.layers.76.mlp.experts.171.up_proj", + "model.layers.76.mlp.experts.172.down_proj", + "model.layers.76.mlp.experts.172.gate_proj", + "model.layers.76.mlp.experts.172.up_proj", + "model.layers.76.mlp.experts.173.down_proj", + "model.layers.76.mlp.experts.173.gate_proj", + "model.layers.76.mlp.experts.173.up_proj", + "model.layers.76.mlp.experts.174.down_proj", + "model.layers.76.mlp.experts.174.gate_proj", + "model.layers.76.mlp.experts.174.up_proj", + "model.layers.76.mlp.experts.175.down_proj", + "model.layers.76.mlp.experts.175.gate_proj", + "model.layers.76.mlp.experts.175.up_proj", + "model.layers.76.mlp.experts.176.down_proj", + "model.layers.76.mlp.experts.176.gate_proj", + "model.layers.76.mlp.experts.176.up_proj", + "model.layers.76.mlp.experts.177.down_proj", + "model.layers.76.mlp.experts.177.gate_proj", + "model.layers.76.mlp.experts.177.up_proj", + "model.layers.76.mlp.experts.178.down_proj", + "model.layers.76.mlp.experts.178.gate_proj", + "model.layers.76.mlp.experts.178.up_proj", + "model.layers.76.mlp.experts.179.down_proj", + "model.layers.76.mlp.experts.179.gate_proj", + "model.layers.76.mlp.experts.179.up_proj", + "model.layers.76.mlp.experts.18.down_proj", + "model.layers.76.mlp.experts.18.gate_proj", + "model.layers.76.mlp.experts.18.up_proj", + "model.layers.76.mlp.experts.180.down_proj", + "model.layers.76.mlp.experts.180.gate_proj", + "model.layers.76.mlp.experts.180.up_proj", + "model.layers.76.mlp.experts.181.down_proj", + "model.layers.76.mlp.experts.181.gate_proj", + "model.layers.76.mlp.experts.181.up_proj", + "model.layers.76.mlp.experts.182.down_proj", + "model.layers.76.mlp.experts.182.gate_proj", + "model.layers.76.mlp.experts.182.up_proj", + "model.layers.76.mlp.experts.183.down_proj", + "model.layers.76.mlp.experts.183.gate_proj", + "model.layers.76.mlp.experts.183.up_proj", + "model.layers.76.mlp.experts.184.down_proj", + "model.layers.76.mlp.experts.184.gate_proj", + "model.layers.76.mlp.experts.184.up_proj", + "model.layers.76.mlp.experts.185.down_proj", + "model.layers.76.mlp.experts.185.gate_proj", + "model.layers.76.mlp.experts.185.up_proj", + "model.layers.76.mlp.experts.186.down_proj", + "model.layers.76.mlp.experts.186.gate_proj", + "model.layers.76.mlp.experts.186.up_proj", + "model.layers.76.mlp.experts.187.down_proj", + "model.layers.76.mlp.experts.187.gate_proj", + "model.layers.76.mlp.experts.187.up_proj", + "model.layers.76.mlp.experts.188.down_proj", + "model.layers.76.mlp.experts.188.gate_proj", + "model.layers.76.mlp.experts.188.up_proj", + "model.layers.76.mlp.experts.189.down_proj", + "model.layers.76.mlp.experts.189.gate_proj", + "model.layers.76.mlp.experts.189.up_proj", + "model.layers.76.mlp.experts.19.down_proj", + "model.layers.76.mlp.experts.19.gate_proj", + "model.layers.76.mlp.experts.19.up_proj", + "model.layers.76.mlp.experts.190.down_proj", + "model.layers.76.mlp.experts.190.gate_proj", + "model.layers.76.mlp.experts.190.up_proj", + "model.layers.76.mlp.experts.191.down_proj", + "model.layers.76.mlp.experts.191.gate_proj", + "model.layers.76.mlp.experts.191.up_proj", + "model.layers.76.mlp.experts.192.down_proj", + "model.layers.76.mlp.experts.192.gate_proj", + "model.layers.76.mlp.experts.192.up_proj", + "model.layers.76.mlp.experts.193.down_proj", + "model.layers.76.mlp.experts.193.gate_proj", + "model.layers.76.mlp.experts.193.up_proj", + "model.layers.76.mlp.experts.194.down_proj", + "model.layers.76.mlp.experts.194.gate_proj", + "model.layers.76.mlp.experts.194.up_proj", + "model.layers.76.mlp.experts.195.down_proj", + "model.layers.76.mlp.experts.195.gate_proj", + "model.layers.76.mlp.experts.195.up_proj", + "model.layers.76.mlp.experts.196.down_proj", + "model.layers.76.mlp.experts.196.gate_proj", + "model.layers.76.mlp.experts.196.up_proj", + "model.layers.76.mlp.experts.197.down_proj", + "model.layers.76.mlp.experts.197.gate_proj", + "model.layers.76.mlp.experts.197.up_proj", + "model.layers.76.mlp.experts.198.down_proj", + "model.layers.76.mlp.experts.198.gate_proj", + "model.layers.76.mlp.experts.198.up_proj", + "model.layers.76.mlp.experts.199.down_proj", + "model.layers.76.mlp.experts.199.gate_proj", + "model.layers.76.mlp.experts.199.up_proj", + "model.layers.76.mlp.experts.2.down_proj", + "model.layers.76.mlp.experts.2.gate_proj", + "model.layers.76.mlp.experts.2.up_proj", + "model.layers.76.mlp.experts.20.down_proj", + "model.layers.76.mlp.experts.20.gate_proj", + "model.layers.76.mlp.experts.20.up_proj", + "model.layers.76.mlp.experts.200.down_proj", + "model.layers.76.mlp.experts.200.gate_proj", + "model.layers.76.mlp.experts.200.up_proj", + "model.layers.76.mlp.experts.201.down_proj", + "model.layers.76.mlp.experts.201.gate_proj", + "model.layers.76.mlp.experts.201.up_proj", + "model.layers.76.mlp.experts.202.down_proj", + "model.layers.76.mlp.experts.202.gate_proj", + "model.layers.76.mlp.experts.202.up_proj", + "model.layers.76.mlp.experts.203.down_proj", + "model.layers.76.mlp.experts.203.gate_proj", + "model.layers.76.mlp.experts.203.up_proj", + "model.layers.76.mlp.experts.204.down_proj", + "model.layers.76.mlp.experts.204.gate_proj", + "model.layers.76.mlp.experts.204.up_proj", + "model.layers.76.mlp.experts.205.down_proj", + "model.layers.76.mlp.experts.205.gate_proj", + "model.layers.76.mlp.experts.205.up_proj", + "model.layers.76.mlp.experts.206.down_proj", + "model.layers.76.mlp.experts.206.gate_proj", + "model.layers.76.mlp.experts.206.up_proj", + "model.layers.76.mlp.experts.207.down_proj", + "model.layers.76.mlp.experts.207.gate_proj", + "model.layers.76.mlp.experts.207.up_proj", + "model.layers.76.mlp.experts.208.down_proj", + "model.layers.76.mlp.experts.208.gate_proj", + "model.layers.76.mlp.experts.208.up_proj", + "model.layers.76.mlp.experts.209.down_proj", + "model.layers.76.mlp.experts.209.gate_proj", + "model.layers.76.mlp.experts.209.up_proj", + "model.layers.76.mlp.experts.21.down_proj", + "model.layers.76.mlp.experts.21.gate_proj", + "model.layers.76.mlp.experts.21.up_proj", + "model.layers.76.mlp.experts.210.down_proj", + "model.layers.76.mlp.experts.210.gate_proj", + "model.layers.76.mlp.experts.210.up_proj", + "model.layers.76.mlp.experts.211.down_proj", + "model.layers.76.mlp.experts.211.gate_proj", + "model.layers.76.mlp.experts.211.up_proj", + "model.layers.76.mlp.experts.212.down_proj", + "model.layers.76.mlp.experts.212.gate_proj", + "model.layers.76.mlp.experts.212.up_proj", + "model.layers.76.mlp.experts.213.down_proj", + "model.layers.76.mlp.experts.213.gate_proj", + "model.layers.76.mlp.experts.213.up_proj", + "model.layers.76.mlp.experts.214.down_proj", + "model.layers.76.mlp.experts.214.gate_proj", + "model.layers.76.mlp.experts.214.up_proj", + "model.layers.76.mlp.experts.215.down_proj", + "model.layers.76.mlp.experts.215.gate_proj", + "model.layers.76.mlp.experts.215.up_proj", + "model.layers.76.mlp.experts.216.down_proj", + "model.layers.76.mlp.experts.216.gate_proj", + "model.layers.76.mlp.experts.216.up_proj", + "model.layers.76.mlp.experts.217.down_proj", + "model.layers.76.mlp.experts.217.gate_proj", + "model.layers.76.mlp.experts.217.up_proj", + "model.layers.76.mlp.experts.218.down_proj", + "model.layers.76.mlp.experts.218.gate_proj", + "model.layers.76.mlp.experts.218.up_proj", + "model.layers.76.mlp.experts.219.down_proj", + "model.layers.76.mlp.experts.219.gate_proj", + "model.layers.76.mlp.experts.219.up_proj", + "model.layers.76.mlp.experts.22.down_proj", + "model.layers.76.mlp.experts.22.gate_proj", + "model.layers.76.mlp.experts.22.up_proj", + "model.layers.76.mlp.experts.220.down_proj", + "model.layers.76.mlp.experts.220.gate_proj", + "model.layers.76.mlp.experts.220.up_proj", + "model.layers.76.mlp.experts.221.down_proj", + "model.layers.76.mlp.experts.221.gate_proj", + "model.layers.76.mlp.experts.221.up_proj", + "model.layers.76.mlp.experts.222.down_proj", + "model.layers.76.mlp.experts.222.gate_proj", + "model.layers.76.mlp.experts.222.up_proj", + "model.layers.76.mlp.experts.223.down_proj", + "model.layers.76.mlp.experts.223.gate_proj", + "model.layers.76.mlp.experts.223.up_proj", + "model.layers.76.mlp.experts.224.down_proj", + "model.layers.76.mlp.experts.224.gate_proj", + "model.layers.76.mlp.experts.224.up_proj", + "model.layers.76.mlp.experts.225.down_proj", + "model.layers.76.mlp.experts.225.gate_proj", + "model.layers.76.mlp.experts.225.up_proj", + "model.layers.76.mlp.experts.226.down_proj", + "model.layers.76.mlp.experts.226.gate_proj", + "model.layers.76.mlp.experts.226.up_proj", + "model.layers.76.mlp.experts.227.down_proj", + "model.layers.76.mlp.experts.227.gate_proj", + "model.layers.76.mlp.experts.227.up_proj", + "model.layers.76.mlp.experts.228.down_proj", + "model.layers.76.mlp.experts.228.gate_proj", + "model.layers.76.mlp.experts.228.up_proj", + "model.layers.76.mlp.experts.229.down_proj", + "model.layers.76.mlp.experts.229.gate_proj", + "model.layers.76.mlp.experts.229.up_proj", + "model.layers.76.mlp.experts.23.down_proj", + "model.layers.76.mlp.experts.23.gate_proj", + "model.layers.76.mlp.experts.23.up_proj", + "model.layers.76.mlp.experts.230.down_proj", + "model.layers.76.mlp.experts.230.gate_proj", + "model.layers.76.mlp.experts.230.up_proj", + "model.layers.76.mlp.experts.231.down_proj", + "model.layers.76.mlp.experts.231.gate_proj", + "model.layers.76.mlp.experts.231.up_proj", + "model.layers.76.mlp.experts.232.down_proj", + "model.layers.76.mlp.experts.232.gate_proj", + "model.layers.76.mlp.experts.232.up_proj", + "model.layers.76.mlp.experts.233.down_proj", + "model.layers.76.mlp.experts.233.gate_proj", + "model.layers.76.mlp.experts.233.up_proj", + "model.layers.76.mlp.experts.234.down_proj", + "model.layers.76.mlp.experts.234.gate_proj", + "model.layers.76.mlp.experts.234.up_proj", + "model.layers.76.mlp.experts.235.down_proj", + "model.layers.76.mlp.experts.235.gate_proj", + "model.layers.76.mlp.experts.235.up_proj", + "model.layers.76.mlp.experts.236.down_proj", + "model.layers.76.mlp.experts.236.gate_proj", + "model.layers.76.mlp.experts.236.up_proj", + "model.layers.76.mlp.experts.237.down_proj", + "model.layers.76.mlp.experts.237.gate_proj", + "model.layers.76.mlp.experts.237.up_proj", + "model.layers.76.mlp.experts.238.down_proj", + "model.layers.76.mlp.experts.238.gate_proj", + "model.layers.76.mlp.experts.238.up_proj", + "model.layers.76.mlp.experts.239.down_proj", + "model.layers.76.mlp.experts.239.gate_proj", + "model.layers.76.mlp.experts.239.up_proj", + "model.layers.76.mlp.experts.24.down_proj", + "model.layers.76.mlp.experts.24.gate_proj", + "model.layers.76.mlp.experts.24.up_proj", + "model.layers.76.mlp.experts.240.down_proj", + "model.layers.76.mlp.experts.240.gate_proj", + "model.layers.76.mlp.experts.240.up_proj", + "model.layers.76.mlp.experts.241.down_proj", + "model.layers.76.mlp.experts.241.gate_proj", + "model.layers.76.mlp.experts.241.up_proj", + "model.layers.76.mlp.experts.242.down_proj", + "model.layers.76.mlp.experts.242.gate_proj", + "model.layers.76.mlp.experts.242.up_proj", + "model.layers.76.mlp.experts.243.down_proj", + "model.layers.76.mlp.experts.243.gate_proj", + "model.layers.76.mlp.experts.243.up_proj", + "model.layers.76.mlp.experts.244.down_proj", + "model.layers.76.mlp.experts.244.gate_proj", + "model.layers.76.mlp.experts.244.up_proj", + "model.layers.76.mlp.experts.245.down_proj", + "model.layers.76.mlp.experts.245.gate_proj", + "model.layers.76.mlp.experts.245.up_proj", + "model.layers.76.mlp.experts.246.down_proj", + "model.layers.76.mlp.experts.246.gate_proj", + "model.layers.76.mlp.experts.246.up_proj", + "model.layers.76.mlp.experts.247.down_proj", + "model.layers.76.mlp.experts.247.gate_proj", + "model.layers.76.mlp.experts.247.up_proj", + "model.layers.76.mlp.experts.248.down_proj", + "model.layers.76.mlp.experts.248.gate_proj", + "model.layers.76.mlp.experts.248.up_proj", + "model.layers.76.mlp.experts.249.down_proj", + "model.layers.76.mlp.experts.249.gate_proj", + "model.layers.76.mlp.experts.249.up_proj", + "model.layers.76.mlp.experts.25.down_proj", + "model.layers.76.mlp.experts.25.gate_proj", + "model.layers.76.mlp.experts.25.up_proj", + "model.layers.76.mlp.experts.250.down_proj", + "model.layers.76.mlp.experts.250.gate_proj", + "model.layers.76.mlp.experts.250.up_proj", + "model.layers.76.mlp.experts.251.down_proj", + "model.layers.76.mlp.experts.251.gate_proj", + "model.layers.76.mlp.experts.251.up_proj", + "model.layers.76.mlp.experts.252.down_proj", + "model.layers.76.mlp.experts.252.gate_proj", + "model.layers.76.mlp.experts.252.up_proj", + "model.layers.76.mlp.experts.253.down_proj", + "model.layers.76.mlp.experts.253.gate_proj", + "model.layers.76.mlp.experts.253.up_proj", + "model.layers.76.mlp.experts.254.down_proj", + "model.layers.76.mlp.experts.254.gate_proj", + "model.layers.76.mlp.experts.254.up_proj", + "model.layers.76.mlp.experts.255.down_proj", + "model.layers.76.mlp.experts.255.gate_proj", + "model.layers.76.mlp.experts.255.up_proj", + "model.layers.76.mlp.experts.26.down_proj", + "model.layers.76.mlp.experts.26.gate_proj", + "model.layers.76.mlp.experts.26.up_proj", + "model.layers.76.mlp.experts.27.down_proj", + "model.layers.76.mlp.experts.27.gate_proj", + "model.layers.76.mlp.experts.27.up_proj", + "model.layers.76.mlp.experts.28.down_proj", + "model.layers.76.mlp.experts.28.gate_proj", + "model.layers.76.mlp.experts.28.up_proj", + "model.layers.76.mlp.experts.29.down_proj", + "model.layers.76.mlp.experts.29.gate_proj", + "model.layers.76.mlp.experts.29.up_proj", + "model.layers.76.mlp.experts.3.down_proj", + "model.layers.76.mlp.experts.3.gate_proj", + "model.layers.76.mlp.experts.3.up_proj", + "model.layers.76.mlp.experts.30.down_proj", + "model.layers.76.mlp.experts.30.gate_proj", + "model.layers.76.mlp.experts.30.up_proj", + "model.layers.76.mlp.experts.31.down_proj", + "model.layers.76.mlp.experts.31.gate_proj", + "model.layers.76.mlp.experts.31.up_proj", + "model.layers.76.mlp.experts.32.down_proj", + "model.layers.76.mlp.experts.32.gate_proj", + "model.layers.76.mlp.experts.32.up_proj", + "model.layers.76.mlp.experts.33.down_proj", + "model.layers.76.mlp.experts.33.gate_proj", + "model.layers.76.mlp.experts.33.up_proj", + "model.layers.76.mlp.experts.34.down_proj", + "model.layers.76.mlp.experts.34.gate_proj", + "model.layers.76.mlp.experts.34.up_proj", + "model.layers.76.mlp.experts.35.down_proj", + "model.layers.76.mlp.experts.35.gate_proj", + "model.layers.76.mlp.experts.35.up_proj", + "model.layers.76.mlp.experts.36.down_proj", + "model.layers.76.mlp.experts.36.gate_proj", + "model.layers.76.mlp.experts.36.up_proj", + "model.layers.76.mlp.experts.37.down_proj", + "model.layers.76.mlp.experts.37.gate_proj", + "model.layers.76.mlp.experts.37.up_proj", + "model.layers.76.mlp.experts.38.down_proj", + "model.layers.76.mlp.experts.38.gate_proj", + "model.layers.76.mlp.experts.38.up_proj", + "model.layers.76.mlp.experts.39.down_proj", + "model.layers.76.mlp.experts.39.gate_proj", + "model.layers.76.mlp.experts.39.up_proj", + "model.layers.76.mlp.experts.4.down_proj", + "model.layers.76.mlp.experts.4.gate_proj", + "model.layers.76.mlp.experts.4.up_proj", + "model.layers.76.mlp.experts.40.down_proj", + "model.layers.76.mlp.experts.40.gate_proj", + "model.layers.76.mlp.experts.40.up_proj", + "model.layers.76.mlp.experts.41.down_proj", + "model.layers.76.mlp.experts.41.gate_proj", + "model.layers.76.mlp.experts.41.up_proj", + "model.layers.76.mlp.experts.42.down_proj", + "model.layers.76.mlp.experts.42.gate_proj", + "model.layers.76.mlp.experts.42.up_proj", + "model.layers.76.mlp.experts.43.down_proj", + "model.layers.76.mlp.experts.43.gate_proj", + "model.layers.76.mlp.experts.43.up_proj", + "model.layers.76.mlp.experts.44.down_proj", + "model.layers.76.mlp.experts.44.gate_proj", + "model.layers.76.mlp.experts.44.up_proj", + "model.layers.76.mlp.experts.45.down_proj", + "model.layers.76.mlp.experts.45.gate_proj", + "model.layers.76.mlp.experts.45.up_proj", + "model.layers.76.mlp.experts.46.down_proj", + "model.layers.76.mlp.experts.46.gate_proj", + "model.layers.76.mlp.experts.46.up_proj", + "model.layers.76.mlp.experts.47.down_proj", + "model.layers.76.mlp.experts.47.gate_proj", + "model.layers.76.mlp.experts.47.up_proj", + "model.layers.76.mlp.experts.48.down_proj", + "model.layers.76.mlp.experts.48.gate_proj", + "model.layers.76.mlp.experts.48.up_proj", + "model.layers.76.mlp.experts.49.down_proj", + "model.layers.76.mlp.experts.49.gate_proj", + "model.layers.76.mlp.experts.49.up_proj", + "model.layers.76.mlp.experts.5.down_proj", + "model.layers.76.mlp.experts.5.gate_proj", + "model.layers.76.mlp.experts.5.up_proj", + "model.layers.76.mlp.experts.50.down_proj", + "model.layers.76.mlp.experts.50.gate_proj", + "model.layers.76.mlp.experts.50.up_proj", + "model.layers.76.mlp.experts.51.down_proj", + "model.layers.76.mlp.experts.51.gate_proj", + "model.layers.76.mlp.experts.51.up_proj", + "model.layers.76.mlp.experts.52.down_proj", + "model.layers.76.mlp.experts.52.gate_proj", + "model.layers.76.mlp.experts.52.up_proj", + "model.layers.76.mlp.experts.53.down_proj", + "model.layers.76.mlp.experts.53.gate_proj", + "model.layers.76.mlp.experts.53.up_proj", + "model.layers.76.mlp.experts.54.down_proj", + "model.layers.76.mlp.experts.54.gate_proj", + "model.layers.76.mlp.experts.54.up_proj", + "model.layers.76.mlp.experts.55.down_proj", + "model.layers.76.mlp.experts.55.gate_proj", + "model.layers.76.mlp.experts.55.up_proj", + "model.layers.76.mlp.experts.56.down_proj", + "model.layers.76.mlp.experts.56.gate_proj", + "model.layers.76.mlp.experts.56.up_proj", + "model.layers.76.mlp.experts.57.down_proj", + "model.layers.76.mlp.experts.57.gate_proj", + "model.layers.76.mlp.experts.57.up_proj", + "model.layers.76.mlp.experts.58.down_proj", + "model.layers.76.mlp.experts.58.gate_proj", + "model.layers.76.mlp.experts.58.up_proj", + "model.layers.76.mlp.experts.59.down_proj", + "model.layers.76.mlp.experts.59.gate_proj", + "model.layers.76.mlp.experts.59.up_proj", + "model.layers.76.mlp.experts.6.down_proj", + "model.layers.76.mlp.experts.6.gate_proj", + "model.layers.76.mlp.experts.6.up_proj", + "model.layers.76.mlp.experts.60.down_proj", + "model.layers.76.mlp.experts.60.gate_proj", + "model.layers.76.mlp.experts.60.up_proj", + "model.layers.76.mlp.experts.61.down_proj", + "model.layers.76.mlp.experts.61.gate_proj", + "model.layers.76.mlp.experts.61.up_proj", + "model.layers.76.mlp.experts.62.down_proj", + "model.layers.76.mlp.experts.62.gate_proj", + "model.layers.76.mlp.experts.62.up_proj", + "model.layers.76.mlp.experts.63.down_proj", + "model.layers.76.mlp.experts.63.gate_proj", + "model.layers.76.mlp.experts.63.up_proj", + "model.layers.76.mlp.experts.64.down_proj", + "model.layers.76.mlp.experts.64.gate_proj", + "model.layers.76.mlp.experts.64.up_proj", + "model.layers.76.mlp.experts.65.down_proj", + "model.layers.76.mlp.experts.65.gate_proj", + "model.layers.76.mlp.experts.65.up_proj", + "model.layers.76.mlp.experts.66.down_proj", + "model.layers.76.mlp.experts.66.gate_proj", + "model.layers.76.mlp.experts.66.up_proj", + "model.layers.76.mlp.experts.67.down_proj", + "model.layers.76.mlp.experts.67.gate_proj", + "model.layers.76.mlp.experts.67.up_proj", + "model.layers.76.mlp.experts.68.down_proj", + "model.layers.76.mlp.experts.68.gate_proj", + "model.layers.76.mlp.experts.68.up_proj", + "model.layers.76.mlp.experts.69.down_proj", + "model.layers.76.mlp.experts.69.gate_proj", + "model.layers.76.mlp.experts.69.up_proj", + "model.layers.76.mlp.experts.7.down_proj", + "model.layers.76.mlp.experts.7.gate_proj", + "model.layers.76.mlp.experts.7.up_proj", + "model.layers.76.mlp.experts.70.down_proj", + "model.layers.76.mlp.experts.70.gate_proj", + "model.layers.76.mlp.experts.70.up_proj", + "model.layers.76.mlp.experts.71.down_proj", + "model.layers.76.mlp.experts.71.gate_proj", + "model.layers.76.mlp.experts.71.up_proj", + "model.layers.76.mlp.experts.72.down_proj", + "model.layers.76.mlp.experts.72.gate_proj", + "model.layers.76.mlp.experts.72.up_proj", + "model.layers.76.mlp.experts.73.down_proj", + "model.layers.76.mlp.experts.73.gate_proj", + "model.layers.76.mlp.experts.73.up_proj", + "model.layers.76.mlp.experts.74.down_proj", + "model.layers.76.mlp.experts.74.gate_proj", + "model.layers.76.mlp.experts.74.up_proj", + "model.layers.76.mlp.experts.75.down_proj", + "model.layers.76.mlp.experts.75.gate_proj", + "model.layers.76.mlp.experts.75.up_proj", + "model.layers.76.mlp.experts.76.down_proj", + "model.layers.76.mlp.experts.76.gate_proj", + "model.layers.76.mlp.experts.76.up_proj", + "model.layers.76.mlp.experts.77.down_proj", + "model.layers.76.mlp.experts.77.gate_proj", + "model.layers.76.mlp.experts.77.up_proj", + "model.layers.76.mlp.experts.78.down_proj", + "model.layers.76.mlp.experts.78.gate_proj", + "model.layers.76.mlp.experts.78.up_proj", + "model.layers.76.mlp.experts.79.down_proj", + "model.layers.76.mlp.experts.79.gate_proj", + "model.layers.76.mlp.experts.79.up_proj", + "model.layers.76.mlp.experts.8.down_proj", + "model.layers.76.mlp.experts.8.gate_proj", + "model.layers.76.mlp.experts.8.up_proj", + "model.layers.76.mlp.experts.80.down_proj", + "model.layers.76.mlp.experts.80.gate_proj", + "model.layers.76.mlp.experts.80.up_proj", + "model.layers.76.mlp.experts.81.down_proj", + "model.layers.76.mlp.experts.81.gate_proj", + "model.layers.76.mlp.experts.81.up_proj", + "model.layers.76.mlp.experts.82.down_proj", + "model.layers.76.mlp.experts.82.gate_proj", + "model.layers.76.mlp.experts.82.up_proj", + "model.layers.76.mlp.experts.83.down_proj", + "model.layers.76.mlp.experts.83.gate_proj", + "model.layers.76.mlp.experts.83.up_proj", + "model.layers.76.mlp.experts.84.down_proj", + "model.layers.76.mlp.experts.84.gate_proj", + "model.layers.76.mlp.experts.84.up_proj", + "model.layers.76.mlp.experts.85.down_proj", + "model.layers.76.mlp.experts.85.gate_proj", + "model.layers.76.mlp.experts.85.up_proj", + "model.layers.76.mlp.experts.86.down_proj", + "model.layers.76.mlp.experts.86.gate_proj", + "model.layers.76.mlp.experts.86.up_proj", + "model.layers.76.mlp.experts.87.down_proj", + "model.layers.76.mlp.experts.87.gate_proj", + "model.layers.76.mlp.experts.87.up_proj", + "model.layers.76.mlp.experts.88.down_proj", + "model.layers.76.mlp.experts.88.gate_proj", + "model.layers.76.mlp.experts.88.up_proj", + "model.layers.76.mlp.experts.89.down_proj", + "model.layers.76.mlp.experts.89.gate_proj", + "model.layers.76.mlp.experts.89.up_proj", + "model.layers.76.mlp.experts.9.down_proj", + "model.layers.76.mlp.experts.9.gate_proj", + "model.layers.76.mlp.experts.9.up_proj", + "model.layers.76.mlp.experts.90.down_proj", + "model.layers.76.mlp.experts.90.gate_proj", + "model.layers.76.mlp.experts.90.up_proj", + "model.layers.76.mlp.experts.91.down_proj", + "model.layers.76.mlp.experts.91.gate_proj", + "model.layers.76.mlp.experts.91.up_proj", + "model.layers.76.mlp.experts.92.down_proj", + "model.layers.76.mlp.experts.92.gate_proj", + "model.layers.76.mlp.experts.92.up_proj", + "model.layers.76.mlp.experts.93.down_proj", + "model.layers.76.mlp.experts.93.gate_proj", + "model.layers.76.mlp.experts.93.up_proj", + "model.layers.76.mlp.experts.94.down_proj", + "model.layers.76.mlp.experts.94.gate_proj", + "model.layers.76.mlp.experts.94.up_proj", + "model.layers.76.mlp.experts.95.down_proj", + "model.layers.76.mlp.experts.95.gate_proj", + "model.layers.76.mlp.experts.95.up_proj", + "model.layers.76.mlp.experts.96.down_proj", + "model.layers.76.mlp.experts.96.gate_proj", + "model.layers.76.mlp.experts.96.up_proj", + "model.layers.76.mlp.experts.97.down_proj", + "model.layers.76.mlp.experts.97.gate_proj", + "model.layers.76.mlp.experts.97.up_proj", + "model.layers.76.mlp.experts.98.down_proj", + "model.layers.76.mlp.experts.98.gate_proj", + "model.layers.76.mlp.experts.98.up_proj", + "model.layers.76.mlp.experts.99.down_proj", + "model.layers.76.mlp.experts.99.gate_proj", + "model.layers.76.mlp.experts.99.up_proj", + "model.layers.77.mlp.experts.0.down_proj", + "model.layers.77.mlp.experts.0.gate_proj", + "model.layers.77.mlp.experts.0.up_proj", + "model.layers.77.mlp.experts.1.down_proj", + "model.layers.77.mlp.experts.1.gate_proj", + "model.layers.77.mlp.experts.1.up_proj", + "model.layers.77.mlp.experts.10.down_proj", + "model.layers.77.mlp.experts.10.gate_proj", + "model.layers.77.mlp.experts.10.up_proj", + "model.layers.77.mlp.experts.100.down_proj", + "model.layers.77.mlp.experts.100.gate_proj", + "model.layers.77.mlp.experts.100.up_proj", + "model.layers.77.mlp.experts.101.down_proj", + "model.layers.77.mlp.experts.101.gate_proj", + "model.layers.77.mlp.experts.101.up_proj", + "model.layers.77.mlp.experts.102.down_proj", + "model.layers.77.mlp.experts.102.gate_proj", + "model.layers.77.mlp.experts.102.up_proj", + "model.layers.77.mlp.experts.103.down_proj", + "model.layers.77.mlp.experts.103.gate_proj", + "model.layers.77.mlp.experts.103.up_proj", + "model.layers.77.mlp.experts.104.down_proj", + "model.layers.77.mlp.experts.104.gate_proj", + "model.layers.77.mlp.experts.104.up_proj", + "model.layers.77.mlp.experts.105.down_proj", + "model.layers.77.mlp.experts.105.gate_proj", + "model.layers.77.mlp.experts.105.up_proj", + "model.layers.77.mlp.experts.106.down_proj", + "model.layers.77.mlp.experts.106.gate_proj", + "model.layers.77.mlp.experts.106.up_proj", + "model.layers.77.mlp.experts.107.down_proj", + "model.layers.77.mlp.experts.107.gate_proj", + "model.layers.77.mlp.experts.107.up_proj", + "model.layers.77.mlp.experts.108.down_proj", + "model.layers.77.mlp.experts.108.gate_proj", + "model.layers.77.mlp.experts.108.up_proj", + "model.layers.77.mlp.experts.109.down_proj", + "model.layers.77.mlp.experts.109.gate_proj", + "model.layers.77.mlp.experts.109.up_proj", + "model.layers.77.mlp.experts.11.down_proj", + "model.layers.77.mlp.experts.11.gate_proj", + "model.layers.77.mlp.experts.11.up_proj", + "model.layers.77.mlp.experts.110.down_proj", + "model.layers.77.mlp.experts.110.gate_proj", + "model.layers.77.mlp.experts.110.up_proj", + "model.layers.77.mlp.experts.111.down_proj", + "model.layers.77.mlp.experts.111.gate_proj", + "model.layers.77.mlp.experts.111.up_proj", + "model.layers.77.mlp.experts.112.down_proj", + "model.layers.77.mlp.experts.112.gate_proj", + "model.layers.77.mlp.experts.112.up_proj", + "model.layers.77.mlp.experts.113.down_proj", + "model.layers.77.mlp.experts.113.gate_proj", + "model.layers.77.mlp.experts.113.up_proj", + "model.layers.77.mlp.experts.114.down_proj", + "model.layers.77.mlp.experts.114.gate_proj", + "model.layers.77.mlp.experts.114.up_proj", + "model.layers.77.mlp.experts.115.down_proj", + "model.layers.77.mlp.experts.115.gate_proj", + "model.layers.77.mlp.experts.115.up_proj", + "model.layers.77.mlp.experts.116.down_proj", + "model.layers.77.mlp.experts.116.gate_proj", + "model.layers.77.mlp.experts.116.up_proj", + "model.layers.77.mlp.experts.117.down_proj", + "model.layers.77.mlp.experts.117.gate_proj", + "model.layers.77.mlp.experts.117.up_proj", + "model.layers.77.mlp.experts.118.down_proj", + "model.layers.77.mlp.experts.118.gate_proj", + "model.layers.77.mlp.experts.118.up_proj", + "model.layers.77.mlp.experts.119.down_proj", + "model.layers.77.mlp.experts.119.gate_proj", + "model.layers.77.mlp.experts.119.up_proj", + "model.layers.77.mlp.experts.12.down_proj", + "model.layers.77.mlp.experts.12.gate_proj", + "model.layers.77.mlp.experts.12.up_proj", + "model.layers.77.mlp.experts.120.down_proj", + "model.layers.77.mlp.experts.120.gate_proj", + "model.layers.77.mlp.experts.120.up_proj", + "model.layers.77.mlp.experts.121.down_proj", + "model.layers.77.mlp.experts.121.gate_proj", + "model.layers.77.mlp.experts.121.up_proj", + "model.layers.77.mlp.experts.122.down_proj", + "model.layers.77.mlp.experts.122.gate_proj", + "model.layers.77.mlp.experts.122.up_proj", + "model.layers.77.mlp.experts.123.down_proj", + "model.layers.77.mlp.experts.123.gate_proj", + "model.layers.77.mlp.experts.123.up_proj", + "model.layers.77.mlp.experts.124.down_proj", + "model.layers.77.mlp.experts.124.gate_proj", + "model.layers.77.mlp.experts.124.up_proj", + "model.layers.77.mlp.experts.125.down_proj", + "model.layers.77.mlp.experts.125.gate_proj", + "model.layers.77.mlp.experts.125.up_proj", + "model.layers.77.mlp.experts.126.down_proj", + "model.layers.77.mlp.experts.126.gate_proj", + "model.layers.77.mlp.experts.126.up_proj", + "model.layers.77.mlp.experts.127.down_proj", + "model.layers.77.mlp.experts.127.gate_proj", + "model.layers.77.mlp.experts.127.up_proj", + "model.layers.77.mlp.experts.128.down_proj", + "model.layers.77.mlp.experts.128.gate_proj", + "model.layers.77.mlp.experts.128.up_proj", + "model.layers.77.mlp.experts.129.down_proj", + "model.layers.77.mlp.experts.129.gate_proj", + "model.layers.77.mlp.experts.129.up_proj", + "model.layers.77.mlp.experts.13.down_proj", + "model.layers.77.mlp.experts.13.gate_proj", + "model.layers.77.mlp.experts.13.up_proj", + "model.layers.77.mlp.experts.130.down_proj", + "model.layers.77.mlp.experts.130.gate_proj", + "model.layers.77.mlp.experts.130.up_proj", + "model.layers.77.mlp.experts.131.down_proj", + "model.layers.77.mlp.experts.131.gate_proj", + "model.layers.77.mlp.experts.131.up_proj", + "model.layers.77.mlp.experts.132.down_proj", + "model.layers.77.mlp.experts.132.gate_proj", + "model.layers.77.mlp.experts.132.up_proj", + "model.layers.77.mlp.experts.133.down_proj", + "model.layers.77.mlp.experts.133.gate_proj", + "model.layers.77.mlp.experts.133.up_proj", + "model.layers.77.mlp.experts.134.down_proj", + "model.layers.77.mlp.experts.134.gate_proj", + "model.layers.77.mlp.experts.134.up_proj", + "model.layers.77.mlp.experts.135.down_proj", + "model.layers.77.mlp.experts.135.gate_proj", + "model.layers.77.mlp.experts.135.up_proj", + "model.layers.77.mlp.experts.136.down_proj", + "model.layers.77.mlp.experts.136.gate_proj", + "model.layers.77.mlp.experts.136.up_proj", + "model.layers.77.mlp.experts.137.down_proj", + "model.layers.77.mlp.experts.137.gate_proj", + "model.layers.77.mlp.experts.137.up_proj", + "model.layers.77.mlp.experts.138.down_proj", + "model.layers.77.mlp.experts.138.gate_proj", + "model.layers.77.mlp.experts.138.up_proj", + "model.layers.77.mlp.experts.139.down_proj", + "model.layers.77.mlp.experts.139.gate_proj", + "model.layers.77.mlp.experts.139.up_proj", + "model.layers.77.mlp.experts.14.down_proj", + "model.layers.77.mlp.experts.14.gate_proj", + "model.layers.77.mlp.experts.14.up_proj", + "model.layers.77.mlp.experts.140.down_proj", + "model.layers.77.mlp.experts.140.gate_proj", + "model.layers.77.mlp.experts.140.up_proj", + "model.layers.77.mlp.experts.141.down_proj", + "model.layers.77.mlp.experts.141.gate_proj", + "model.layers.77.mlp.experts.141.up_proj", + "model.layers.77.mlp.experts.142.down_proj", + "model.layers.77.mlp.experts.142.gate_proj", + "model.layers.77.mlp.experts.142.up_proj", + "model.layers.77.mlp.experts.143.down_proj", + "model.layers.77.mlp.experts.143.gate_proj", + "model.layers.77.mlp.experts.143.up_proj", + "model.layers.77.mlp.experts.144.down_proj", + "model.layers.77.mlp.experts.144.gate_proj", + "model.layers.77.mlp.experts.144.up_proj", + "model.layers.77.mlp.experts.145.down_proj", + "model.layers.77.mlp.experts.145.gate_proj", + "model.layers.77.mlp.experts.145.up_proj", + "model.layers.77.mlp.experts.146.down_proj", + "model.layers.77.mlp.experts.146.gate_proj", + "model.layers.77.mlp.experts.146.up_proj", + "model.layers.77.mlp.experts.147.down_proj", + "model.layers.77.mlp.experts.147.gate_proj", + "model.layers.77.mlp.experts.147.up_proj", + "model.layers.77.mlp.experts.148.down_proj", + "model.layers.77.mlp.experts.148.gate_proj", + "model.layers.77.mlp.experts.148.up_proj", + "model.layers.77.mlp.experts.149.down_proj", + "model.layers.77.mlp.experts.149.gate_proj", + "model.layers.77.mlp.experts.149.up_proj", + "model.layers.77.mlp.experts.15.down_proj", + "model.layers.77.mlp.experts.15.gate_proj", + "model.layers.77.mlp.experts.15.up_proj", + "model.layers.77.mlp.experts.150.down_proj", + "model.layers.77.mlp.experts.150.gate_proj", + "model.layers.77.mlp.experts.150.up_proj", + "model.layers.77.mlp.experts.151.down_proj", + "model.layers.77.mlp.experts.151.gate_proj", + "model.layers.77.mlp.experts.151.up_proj", + "model.layers.77.mlp.experts.152.down_proj", + "model.layers.77.mlp.experts.152.gate_proj", + "model.layers.77.mlp.experts.152.up_proj", + "model.layers.77.mlp.experts.153.down_proj", + "model.layers.77.mlp.experts.153.gate_proj", + "model.layers.77.mlp.experts.153.up_proj", + "model.layers.77.mlp.experts.154.down_proj", + "model.layers.77.mlp.experts.154.gate_proj", + "model.layers.77.mlp.experts.154.up_proj", + "model.layers.77.mlp.experts.155.down_proj", + "model.layers.77.mlp.experts.155.gate_proj", + "model.layers.77.mlp.experts.155.up_proj", + "model.layers.77.mlp.experts.156.down_proj", + "model.layers.77.mlp.experts.156.gate_proj", + "model.layers.77.mlp.experts.156.up_proj", + "model.layers.77.mlp.experts.157.down_proj", + "model.layers.77.mlp.experts.157.gate_proj", + "model.layers.77.mlp.experts.157.up_proj", + "model.layers.77.mlp.experts.158.down_proj", + "model.layers.77.mlp.experts.158.gate_proj", + "model.layers.77.mlp.experts.158.up_proj", + "model.layers.77.mlp.experts.159.down_proj", + "model.layers.77.mlp.experts.159.gate_proj", + "model.layers.77.mlp.experts.159.up_proj", + "model.layers.77.mlp.experts.16.down_proj", + "model.layers.77.mlp.experts.16.gate_proj", + "model.layers.77.mlp.experts.16.up_proj", + "model.layers.77.mlp.experts.160.down_proj", + "model.layers.77.mlp.experts.160.gate_proj", + "model.layers.77.mlp.experts.160.up_proj", + "model.layers.77.mlp.experts.161.down_proj", + "model.layers.77.mlp.experts.161.gate_proj", + "model.layers.77.mlp.experts.161.up_proj", + "model.layers.77.mlp.experts.162.down_proj", + "model.layers.77.mlp.experts.162.gate_proj", + "model.layers.77.mlp.experts.162.up_proj", + "model.layers.77.mlp.experts.163.down_proj", + "model.layers.77.mlp.experts.163.gate_proj", + "model.layers.77.mlp.experts.163.up_proj", + "model.layers.77.mlp.experts.164.down_proj", + "model.layers.77.mlp.experts.164.gate_proj", + "model.layers.77.mlp.experts.164.up_proj", + "model.layers.77.mlp.experts.165.down_proj", + "model.layers.77.mlp.experts.165.gate_proj", + "model.layers.77.mlp.experts.165.up_proj", + "model.layers.77.mlp.experts.166.down_proj", + "model.layers.77.mlp.experts.166.gate_proj", + "model.layers.77.mlp.experts.166.up_proj", + "model.layers.77.mlp.experts.167.down_proj", + "model.layers.77.mlp.experts.167.gate_proj", + "model.layers.77.mlp.experts.167.up_proj", + "model.layers.77.mlp.experts.168.down_proj", + "model.layers.77.mlp.experts.168.gate_proj", + "model.layers.77.mlp.experts.168.up_proj", + "model.layers.77.mlp.experts.169.down_proj", + "model.layers.77.mlp.experts.169.gate_proj", + "model.layers.77.mlp.experts.169.up_proj", + "model.layers.77.mlp.experts.17.down_proj", + "model.layers.77.mlp.experts.17.gate_proj", + "model.layers.77.mlp.experts.17.up_proj", + "model.layers.77.mlp.experts.170.down_proj", + "model.layers.77.mlp.experts.170.gate_proj", + "model.layers.77.mlp.experts.170.up_proj", + "model.layers.77.mlp.experts.171.down_proj", + "model.layers.77.mlp.experts.171.gate_proj", + "model.layers.77.mlp.experts.171.up_proj", + "model.layers.77.mlp.experts.172.down_proj", + "model.layers.77.mlp.experts.172.gate_proj", + "model.layers.77.mlp.experts.172.up_proj", + "model.layers.77.mlp.experts.173.down_proj", + "model.layers.77.mlp.experts.173.gate_proj", + "model.layers.77.mlp.experts.173.up_proj", + "model.layers.77.mlp.experts.174.down_proj", + "model.layers.77.mlp.experts.174.gate_proj", + "model.layers.77.mlp.experts.174.up_proj", + "model.layers.77.mlp.experts.175.down_proj", + "model.layers.77.mlp.experts.175.gate_proj", + "model.layers.77.mlp.experts.175.up_proj", + "model.layers.77.mlp.experts.176.down_proj", + "model.layers.77.mlp.experts.176.gate_proj", + "model.layers.77.mlp.experts.176.up_proj", + "model.layers.77.mlp.experts.177.down_proj", + "model.layers.77.mlp.experts.177.gate_proj", + "model.layers.77.mlp.experts.177.up_proj", + "model.layers.77.mlp.experts.178.down_proj", + "model.layers.77.mlp.experts.178.gate_proj", + "model.layers.77.mlp.experts.178.up_proj", + "model.layers.77.mlp.experts.179.down_proj", + "model.layers.77.mlp.experts.179.gate_proj", + "model.layers.77.mlp.experts.179.up_proj", + "model.layers.77.mlp.experts.18.down_proj", + "model.layers.77.mlp.experts.18.gate_proj", + "model.layers.77.mlp.experts.18.up_proj", + "model.layers.77.mlp.experts.180.down_proj", + "model.layers.77.mlp.experts.180.gate_proj", + "model.layers.77.mlp.experts.180.up_proj", + "model.layers.77.mlp.experts.181.down_proj", + "model.layers.77.mlp.experts.181.gate_proj", + "model.layers.77.mlp.experts.181.up_proj", + "model.layers.77.mlp.experts.182.down_proj", + "model.layers.77.mlp.experts.182.gate_proj", + "model.layers.77.mlp.experts.182.up_proj", + "model.layers.77.mlp.experts.183.down_proj", + "model.layers.77.mlp.experts.183.gate_proj", + "model.layers.77.mlp.experts.183.up_proj", + "model.layers.77.mlp.experts.184.down_proj", + "model.layers.77.mlp.experts.184.gate_proj", + "model.layers.77.mlp.experts.184.up_proj", + "model.layers.77.mlp.experts.185.down_proj", + "model.layers.77.mlp.experts.185.gate_proj", + "model.layers.77.mlp.experts.185.up_proj", + "model.layers.77.mlp.experts.186.down_proj", + "model.layers.77.mlp.experts.186.gate_proj", + "model.layers.77.mlp.experts.186.up_proj", + "model.layers.77.mlp.experts.187.down_proj", + "model.layers.77.mlp.experts.187.gate_proj", + "model.layers.77.mlp.experts.187.up_proj", + "model.layers.77.mlp.experts.188.down_proj", + "model.layers.77.mlp.experts.188.gate_proj", + "model.layers.77.mlp.experts.188.up_proj", + "model.layers.77.mlp.experts.189.down_proj", + "model.layers.77.mlp.experts.189.gate_proj", + "model.layers.77.mlp.experts.189.up_proj", + "model.layers.77.mlp.experts.19.down_proj", + "model.layers.77.mlp.experts.19.gate_proj", + "model.layers.77.mlp.experts.19.up_proj", + "model.layers.77.mlp.experts.190.down_proj", + "model.layers.77.mlp.experts.190.gate_proj", + "model.layers.77.mlp.experts.190.up_proj", + "model.layers.77.mlp.experts.191.down_proj", + "model.layers.77.mlp.experts.191.gate_proj", + "model.layers.77.mlp.experts.191.up_proj", + "model.layers.77.mlp.experts.192.down_proj", + "model.layers.77.mlp.experts.192.gate_proj", + "model.layers.77.mlp.experts.192.up_proj", + "model.layers.77.mlp.experts.193.down_proj", + "model.layers.77.mlp.experts.193.gate_proj", + "model.layers.77.mlp.experts.193.up_proj", + "model.layers.77.mlp.experts.194.down_proj", + "model.layers.77.mlp.experts.194.gate_proj", + "model.layers.77.mlp.experts.194.up_proj", + "model.layers.77.mlp.experts.195.down_proj", + "model.layers.77.mlp.experts.195.gate_proj", + "model.layers.77.mlp.experts.195.up_proj", + "model.layers.77.mlp.experts.196.down_proj", + "model.layers.77.mlp.experts.196.gate_proj", + "model.layers.77.mlp.experts.196.up_proj", + "model.layers.77.mlp.experts.197.down_proj", + "model.layers.77.mlp.experts.197.gate_proj", + "model.layers.77.mlp.experts.197.up_proj", + "model.layers.77.mlp.experts.198.down_proj", + "model.layers.77.mlp.experts.198.gate_proj", + "model.layers.77.mlp.experts.198.up_proj", + "model.layers.77.mlp.experts.199.down_proj", + "model.layers.77.mlp.experts.199.gate_proj", + "model.layers.77.mlp.experts.199.up_proj", + "model.layers.77.mlp.experts.2.down_proj", + "model.layers.77.mlp.experts.2.gate_proj", + "model.layers.77.mlp.experts.2.up_proj", + "model.layers.77.mlp.experts.20.down_proj", + "model.layers.77.mlp.experts.20.gate_proj", + "model.layers.77.mlp.experts.20.up_proj", + "model.layers.77.mlp.experts.200.down_proj", + "model.layers.77.mlp.experts.200.gate_proj", + "model.layers.77.mlp.experts.200.up_proj", + "model.layers.77.mlp.experts.201.down_proj", + "model.layers.77.mlp.experts.201.gate_proj", + "model.layers.77.mlp.experts.201.up_proj", + "model.layers.77.mlp.experts.202.down_proj", + "model.layers.77.mlp.experts.202.gate_proj", + "model.layers.77.mlp.experts.202.up_proj", + "model.layers.77.mlp.experts.203.down_proj", + "model.layers.77.mlp.experts.203.gate_proj", + "model.layers.77.mlp.experts.203.up_proj", + "model.layers.77.mlp.experts.204.down_proj", + "model.layers.77.mlp.experts.204.gate_proj", + "model.layers.77.mlp.experts.204.up_proj", + "model.layers.77.mlp.experts.205.down_proj", + "model.layers.77.mlp.experts.205.gate_proj", + "model.layers.77.mlp.experts.205.up_proj", + "model.layers.77.mlp.experts.206.down_proj", + "model.layers.77.mlp.experts.206.gate_proj", + "model.layers.77.mlp.experts.206.up_proj", + "model.layers.77.mlp.experts.207.down_proj", + "model.layers.77.mlp.experts.207.gate_proj", + "model.layers.77.mlp.experts.207.up_proj", + "model.layers.77.mlp.experts.208.down_proj", + "model.layers.77.mlp.experts.208.gate_proj", + "model.layers.77.mlp.experts.208.up_proj", + "model.layers.77.mlp.experts.209.down_proj", + "model.layers.77.mlp.experts.209.gate_proj", + "model.layers.77.mlp.experts.209.up_proj", + "model.layers.77.mlp.experts.21.down_proj", + "model.layers.77.mlp.experts.21.gate_proj", + "model.layers.77.mlp.experts.21.up_proj", + "model.layers.77.mlp.experts.210.down_proj", + "model.layers.77.mlp.experts.210.gate_proj", + "model.layers.77.mlp.experts.210.up_proj", + "model.layers.77.mlp.experts.211.down_proj", + "model.layers.77.mlp.experts.211.gate_proj", + "model.layers.77.mlp.experts.211.up_proj", + "model.layers.77.mlp.experts.212.down_proj", + "model.layers.77.mlp.experts.212.gate_proj", + "model.layers.77.mlp.experts.212.up_proj", + "model.layers.77.mlp.experts.213.down_proj", + "model.layers.77.mlp.experts.213.gate_proj", + "model.layers.77.mlp.experts.213.up_proj", + "model.layers.77.mlp.experts.214.down_proj", + "model.layers.77.mlp.experts.214.gate_proj", + "model.layers.77.mlp.experts.214.up_proj", + "model.layers.77.mlp.experts.215.down_proj", + "model.layers.77.mlp.experts.215.gate_proj", + "model.layers.77.mlp.experts.215.up_proj", + "model.layers.77.mlp.experts.216.down_proj", + "model.layers.77.mlp.experts.216.gate_proj", + "model.layers.77.mlp.experts.216.up_proj", + "model.layers.77.mlp.experts.217.down_proj", + "model.layers.77.mlp.experts.217.gate_proj", + "model.layers.77.mlp.experts.217.up_proj", + "model.layers.77.mlp.experts.218.down_proj", + "model.layers.77.mlp.experts.218.gate_proj", + "model.layers.77.mlp.experts.218.up_proj", + "model.layers.77.mlp.experts.219.down_proj", + "model.layers.77.mlp.experts.219.gate_proj", + "model.layers.77.mlp.experts.219.up_proj", + "model.layers.77.mlp.experts.22.down_proj", + "model.layers.77.mlp.experts.22.gate_proj", + "model.layers.77.mlp.experts.22.up_proj", + "model.layers.77.mlp.experts.220.down_proj", + "model.layers.77.mlp.experts.220.gate_proj", + "model.layers.77.mlp.experts.220.up_proj", + "model.layers.77.mlp.experts.221.down_proj", + "model.layers.77.mlp.experts.221.gate_proj", + "model.layers.77.mlp.experts.221.up_proj", + "model.layers.77.mlp.experts.222.down_proj", + "model.layers.77.mlp.experts.222.gate_proj", + "model.layers.77.mlp.experts.222.up_proj", + "model.layers.77.mlp.experts.223.down_proj", + "model.layers.77.mlp.experts.223.gate_proj", + "model.layers.77.mlp.experts.223.up_proj", + "model.layers.77.mlp.experts.224.down_proj", + "model.layers.77.mlp.experts.224.gate_proj", + "model.layers.77.mlp.experts.224.up_proj", + "model.layers.77.mlp.experts.225.down_proj", + "model.layers.77.mlp.experts.225.gate_proj", + "model.layers.77.mlp.experts.225.up_proj", + "model.layers.77.mlp.experts.226.down_proj", + "model.layers.77.mlp.experts.226.gate_proj", + "model.layers.77.mlp.experts.226.up_proj", + "model.layers.77.mlp.experts.227.down_proj", + "model.layers.77.mlp.experts.227.gate_proj", + "model.layers.77.mlp.experts.227.up_proj", + "model.layers.77.mlp.experts.228.down_proj", + "model.layers.77.mlp.experts.228.gate_proj", + "model.layers.77.mlp.experts.228.up_proj", + "model.layers.77.mlp.experts.229.down_proj", + "model.layers.77.mlp.experts.229.gate_proj", + "model.layers.77.mlp.experts.229.up_proj", + "model.layers.77.mlp.experts.23.down_proj", + "model.layers.77.mlp.experts.23.gate_proj", + "model.layers.77.mlp.experts.23.up_proj", + "model.layers.77.mlp.experts.230.down_proj", + "model.layers.77.mlp.experts.230.gate_proj", + "model.layers.77.mlp.experts.230.up_proj", + "model.layers.77.mlp.experts.231.down_proj", + "model.layers.77.mlp.experts.231.gate_proj", + "model.layers.77.mlp.experts.231.up_proj", + "model.layers.77.mlp.experts.232.down_proj", + "model.layers.77.mlp.experts.232.gate_proj", + "model.layers.77.mlp.experts.232.up_proj", + "model.layers.77.mlp.experts.233.down_proj", + "model.layers.77.mlp.experts.233.gate_proj", + "model.layers.77.mlp.experts.233.up_proj", + "model.layers.77.mlp.experts.234.down_proj", + "model.layers.77.mlp.experts.234.gate_proj", + "model.layers.77.mlp.experts.234.up_proj", + "model.layers.77.mlp.experts.235.down_proj", + "model.layers.77.mlp.experts.235.gate_proj", + "model.layers.77.mlp.experts.235.up_proj", + "model.layers.77.mlp.experts.236.down_proj", + "model.layers.77.mlp.experts.236.gate_proj", + "model.layers.77.mlp.experts.236.up_proj", + "model.layers.77.mlp.experts.237.down_proj", + "model.layers.77.mlp.experts.237.gate_proj", + "model.layers.77.mlp.experts.237.up_proj", + "model.layers.77.mlp.experts.238.down_proj", + "model.layers.77.mlp.experts.238.gate_proj", + "model.layers.77.mlp.experts.238.up_proj", + "model.layers.77.mlp.experts.239.down_proj", + "model.layers.77.mlp.experts.239.gate_proj", + "model.layers.77.mlp.experts.239.up_proj", + "model.layers.77.mlp.experts.24.down_proj", + "model.layers.77.mlp.experts.24.gate_proj", + "model.layers.77.mlp.experts.24.up_proj", + "model.layers.77.mlp.experts.240.down_proj", + "model.layers.77.mlp.experts.240.gate_proj", + "model.layers.77.mlp.experts.240.up_proj", + "model.layers.77.mlp.experts.241.down_proj", + "model.layers.77.mlp.experts.241.gate_proj", + "model.layers.77.mlp.experts.241.up_proj", + "model.layers.77.mlp.experts.242.down_proj", + "model.layers.77.mlp.experts.242.gate_proj", + "model.layers.77.mlp.experts.242.up_proj", + "model.layers.77.mlp.experts.243.down_proj", + "model.layers.77.mlp.experts.243.gate_proj", + "model.layers.77.mlp.experts.243.up_proj", + "model.layers.77.mlp.experts.244.down_proj", + "model.layers.77.mlp.experts.244.gate_proj", + "model.layers.77.mlp.experts.244.up_proj", + "model.layers.77.mlp.experts.245.down_proj", + "model.layers.77.mlp.experts.245.gate_proj", + "model.layers.77.mlp.experts.245.up_proj", + "model.layers.77.mlp.experts.246.down_proj", + "model.layers.77.mlp.experts.246.gate_proj", + "model.layers.77.mlp.experts.246.up_proj", + "model.layers.77.mlp.experts.247.down_proj", + "model.layers.77.mlp.experts.247.gate_proj", + "model.layers.77.mlp.experts.247.up_proj", + "model.layers.77.mlp.experts.248.down_proj", + "model.layers.77.mlp.experts.248.gate_proj", + "model.layers.77.mlp.experts.248.up_proj", + "model.layers.77.mlp.experts.249.down_proj", + "model.layers.77.mlp.experts.249.gate_proj", + "model.layers.77.mlp.experts.249.up_proj", + "model.layers.77.mlp.experts.25.down_proj", + "model.layers.77.mlp.experts.25.gate_proj", + "model.layers.77.mlp.experts.25.up_proj", + "model.layers.77.mlp.experts.250.down_proj", + "model.layers.77.mlp.experts.250.gate_proj", + "model.layers.77.mlp.experts.250.up_proj", + "model.layers.77.mlp.experts.251.down_proj", + "model.layers.77.mlp.experts.251.gate_proj", + "model.layers.77.mlp.experts.251.up_proj", + "model.layers.77.mlp.experts.252.down_proj", + "model.layers.77.mlp.experts.252.gate_proj", + "model.layers.77.mlp.experts.252.up_proj", + "model.layers.77.mlp.experts.253.down_proj", + "model.layers.77.mlp.experts.253.gate_proj", + "model.layers.77.mlp.experts.253.up_proj", + "model.layers.77.mlp.experts.254.down_proj", + "model.layers.77.mlp.experts.254.gate_proj", + "model.layers.77.mlp.experts.254.up_proj", + "model.layers.77.mlp.experts.255.down_proj", + "model.layers.77.mlp.experts.255.gate_proj", + "model.layers.77.mlp.experts.255.up_proj", + "model.layers.77.mlp.experts.26.down_proj", + "model.layers.77.mlp.experts.26.gate_proj", + "model.layers.77.mlp.experts.26.up_proj", + "model.layers.77.mlp.experts.27.down_proj", + "model.layers.77.mlp.experts.27.gate_proj", + "model.layers.77.mlp.experts.27.up_proj", + "model.layers.77.mlp.experts.28.down_proj", + "model.layers.77.mlp.experts.28.gate_proj", + "model.layers.77.mlp.experts.28.up_proj", + "model.layers.77.mlp.experts.29.down_proj", + "model.layers.77.mlp.experts.29.gate_proj", + "model.layers.77.mlp.experts.29.up_proj", + "model.layers.77.mlp.experts.3.down_proj", + "model.layers.77.mlp.experts.3.gate_proj", + "model.layers.77.mlp.experts.3.up_proj", + "model.layers.77.mlp.experts.30.down_proj", + "model.layers.77.mlp.experts.30.gate_proj", + "model.layers.77.mlp.experts.30.up_proj", + "model.layers.77.mlp.experts.31.down_proj", + "model.layers.77.mlp.experts.31.gate_proj", + "model.layers.77.mlp.experts.31.up_proj", + "model.layers.77.mlp.experts.32.down_proj", + "model.layers.77.mlp.experts.32.gate_proj", + "model.layers.77.mlp.experts.32.up_proj", + "model.layers.77.mlp.experts.33.down_proj", + "model.layers.77.mlp.experts.33.gate_proj", + "model.layers.77.mlp.experts.33.up_proj", + "model.layers.77.mlp.experts.34.down_proj", + "model.layers.77.mlp.experts.34.gate_proj", + "model.layers.77.mlp.experts.34.up_proj", + "model.layers.77.mlp.experts.35.down_proj", + "model.layers.77.mlp.experts.35.gate_proj", + "model.layers.77.mlp.experts.35.up_proj", + "model.layers.77.mlp.experts.36.down_proj", + "model.layers.77.mlp.experts.36.gate_proj", + "model.layers.77.mlp.experts.36.up_proj", + "model.layers.77.mlp.experts.37.down_proj", + "model.layers.77.mlp.experts.37.gate_proj", + "model.layers.77.mlp.experts.37.up_proj", + "model.layers.77.mlp.experts.38.down_proj", + "model.layers.77.mlp.experts.38.gate_proj", + "model.layers.77.mlp.experts.38.up_proj", + "model.layers.77.mlp.experts.39.down_proj", + "model.layers.77.mlp.experts.39.gate_proj", + "model.layers.77.mlp.experts.39.up_proj", + "model.layers.77.mlp.experts.4.down_proj", + "model.layers.77.mlp.experts.4.gate_proj", + "model.layers.77.mlp.experts.4.up_proj", + "model.layers.77.mlp.experts.40.down_proj", + "model.layers.77.mlp.experts.40.gate_proj", + "model.layers.77.mlp.experts.40.up_proj", + "model.layers.77.mlp.experts.41.down_proj", + "model.layers.77.mlp.experts.41.gate_proj", + "model.layers.77.mlp.experts.41.up_proj", + "model.layers.77.mlp.experts.42.down_proj", + "model.layers.77.mlp.experts.42.gate_proj", + "model.layers.77.mlp.experts.42.up_proj", + "model.layers.77.mlp.experts.43.down_proj", + "model.layers.77.mlp.experts.43.gate_proj", + "model.layers.77.mlp.experts.43.up_proj", + "model.layers.77.mlp.experts.44.down_proj", + "model.layers.77.mlp.experts.44.gate_proj", + "model.layers.77.mlp.experts.44.up_proj", + "model.layers.77.mlp.experts.45.down_proj", + "model.layers.77.mlp.experts.45.gate_proj", + "model.layers.77.mlp.experts.45.up_proj", + "model.layers.77.mlp.experts.46.down_proj", + "model.layers.77.mlp.experts.46.gate_proj", + "model.layers.77.mlp.experts.46.up_proj", + "model.layers.77.mlp.experts.47.down_proj", + "model.layers.77.mlp.experts.47.gate_proj", + "model.layers.77.mlp.experts.47.up_proj", + "model.layers.77.mlp.experts.48.down_proj", + "model.layers.77.mlp.experts.48.gate_proj", + "model.layers.77.mlp.experts.48.up_proj", + "model.layers.77.mlp.experts.49.down_proj", + "model.layers.77.mlp.experts.49.gate_proj", + "model.layers.77.mlp.experts.49.up_proj", + "model.layers.77.mlp.experts.5.down_proj", + "model.layers.77.mlp.experts.5.gate_proj", + "model.layers.77.mlp.experts.5.up_proj", + "model.layers.77.mlp.experts.50.down_proj", + "model.layers.77.mlp.experts.50.gate_proj", + "model.layers.77.mlp.experts.50.up_proj", + "model.layers.77.mlp.experts.51.down_proj", + "model.layers.77.mlp.experts.51.gate_proj", + "model.layers.77.mlp.experts.51.up_proj", + "model.layers.77.mlp.experts.52.down_proj", + "model.layers.77.mlp.experts.52.gate_proj", + "model.layers.77.mlp.experts.52.up_proj", + "model.layers.77.mlp.experts.53.down_proj", + "model.layers.77.mlp.experts.53.gate_proj", + "model.layers.77.mlp.experts.53.up_proj", + "model.layers.77.mlp.experts.54.down_proj", + "model.layers.77.mlp.experts.54.gate_proj", + "model.layers.77.mlp.experts.54.up_proj", + "model.layers.77.mlp.experts.55.down_proj", + "model.layers.77.mlp.experts.55.gate_proj", + "model.layers.77.mlp.experts.55.up_proj", + "model.layers.77.mlp.experts.56.down_proj", + "model.layers.77.mlp.experts.56.gate_proj", + "model.layers.77.mlp.experts.56.up_proj", + "model.layers.77.mlp.experts.57.down_proj", + "model.layers.77.mlp.experts.57.gate_proj", + "model.layers.77.mlp.experts.57.up_proj", + "model.layers.77.mlp.experts.58.down_proj", + "model.layers.77.mlp.experts.58.gate_proj", + "model.layers.77.mlp.experts.58.up_proj", + "model.layers.77.mlp.experts.59.down_proj", + "model.layers.77.mlp.experts.59.gate_proj", + "model.layers.77.mlp.experts.59.up_proj", + "model.layers.77.mlp.experts.6.down_proj", + "model.layers.77.mlp.experts.6.gate_proj", + "model.layers.77.mlp.experts.6.up_proj", + "model.layers.77.mlp.experts.60.down_proj", + "model.layers.77.mlp.experts.60.gate_proj", + "model.layers.77.mlp.experts.60.up_proj", + "model.layers.77.mlp.experts.61.down_proj", + "model.layers.77.mlp.experts.61.gate_proj", + "model.layers.77.mlp.experts.61.up_proj", + "model.layers.77.mlp.experts.62.down_proj", + "model.layers.77.mlp.experts.62.gate_proj", + "model.layers.77.mlp.experts.62.up_proj", + "model.layers.77.mlp.experts.63.down_proj", + "model.layers.77.mlp.experts.63.gate_proj", + "model.layers.77.mlp.experts.63.up_proj", + "model.layers.77.mlp.experts.64.down_proj", + "model.layers.77.mlp.experts.64.gate_proj", + "model.layers.77.mlp.experts.64.up_proj", + "model.layers.77.mlp.experts.65.down_proj", + "model.layers.77.mlp.experts.65.gate_proj", + "model.layers.77.mlp.experts.65.up_proj", + "model.layers.77.mlp.experts.66.down_proj", + "model.layers.77.mlp.experts.66.gate_proj", + "model.layers.77.mlp.experts.66.up_proj", + "model.layers.77.mlp.experts.67.down_proj", + "model.layers.77.mlp.experts.67.gate_proj", + "model.layers.77.mlp.experts.67.up_proj", + "model.layers.77.mlp.experts.68.down_proj", + "model.layers.77.mlp.experts.68.gate_proj", + "model.layers.77.mlp.experts.68.up_proj", + "model.layers.77.mlp.experts.69.down_proj", + "model.layers.77.mlp.experts.69.gate_proj", + "model.layers.77.mlp.experts.69.up_proj", + "model.layers.77.mlp.experts.7.down_proj", + "model.layers.77.mlp.experts.7.gate_proj", + "model.layers.77.mlp.experts.7.up_proj", + "model.layers.77.mlp.experts.70.down_proj", + "model.layers.77.mlp.experts.70.gate_proj", + "model.layers.77.mlp.experts.70.up_proj", + "model.layers.77.mlp.experts.71.down_proj", + "model.layers.77.mlp.experts.71.gate_proj", + "model.layers.77.mlp.experts.71.up_proj", + "model.layers.77.mlp.experts.72.down_proj", + "model.layers.77.mlp.experts.72.gate_proj", + "model.layers.77.mlp.experts.72.up_proj", + "model.layers.77.mlp.experts.73.down_proj", + "model.layers.77.mlp.experts.73.gate_proj", + "model.layers.77.mlp.experts.73.up_proj", + "model.layers.77.mlp.experts.74.down_proj", + "model.layers.77.mlp.experts.74.gate_proj", + "model.layers.77.mlp.experts.74.up_proj", + "model.layers.77.mlp.experts.75.down_proj", + "model.layers.77.mlp.experts.75.gate_proj", + "model.layers.77.mlp.experts.75.up_proj", + "model.layers.77.mlp.experts.76.down_proj", + "model.layers.77.mlp.experts.76.gate_proj", + "model.layers.77.mlp.experts.76.up_proj", + "model.layers.77.mlp.experts.77.down_proj", + "model.layers.77.mlp.experts.77.gate_proj", + "model.layers.77.mlp.experts.77.up_proj", + "model.layers.77.mlp.experts.78.down_proj", + "model.layers.77.mlp.experts.78.gate_proj", + "model.layers.77.mlp.experts.78.up_proj", + "model.layers.77.mlp.experts.79.down_proj", + "model.layers.77.mlp.experts.79.gate_proj", + "model.layers.77.mlp.experts.79.up_proj", + "model.layers.77.mlp.experts.8.down_proj", + "model.layers.77.mlp.experts.8.gate_proj", + "model.layers.77.mlp.experts.8.up_proj", + "model.layers.77.mlp.experts.80.down_proj", + "model.layers.77.mlp.experts.80.gate_proj", + "model.layers.77.mlp.experts.80.up_proj", + "model.layers.77.mlp.experts.81.down_proj", + "model.layers.77.mlp.experts.81.gate_proj", + "model.layers.77.mlp.experts.81.up_proj", + "model.layers.77.mlp.experts.82.down_proj", + "model.layers.77.mlp.experts.82.gate_proj", + "model.layers.77.mlp.experts.82.up_proj", + "model.layers.77.mlp.experts.83.down_proj", + "model.layers.77.mlp.experts.83.gate_proj", + "model.layers.77.mlp.experts.83.up_proj", + "model.layers.77.mlp.experts.84.down_proj", + "model.layers.77.mlp.experts.84.gate_proj", + "model.layers.77.mlp.experts.84.up_proj", + "model.layers.77.mlp.experts.85.down_proj", + "model.layers.77.mlp.experts.85.gate_proj", + "model.layers.77.mlp.experts.85.up_proj", + "model.layers.77.mlp.experts.86.down_proj", + "model.layers.77.mlp.experts.86.gate_proj", + "model.layers.77.mlp.experts.86.up_proj", + "model.layers.77.mlp.experts.87.down_proj", + "model.layers.77.mlp.experts.87.gate_proj", + "model.layers.77.mlp.experts.87.up_proj", + "model.layers.77.mlp.experts.88.down_proj", + "model.layers.77.mlp.experts.88.gate_proj", + "model.layers.77.mlp.experts.88.up_proj", + "model.layers.77.mlp.experts.89.down_proj", + "model.layers.77.mlp.experts.89.gate_proj", + "model.layers.77.mlp.experts.89.up_proj", + "model.layers.77.mlp.experts.9.down_proj", + "model.layers.77.mlp.experts.9.gate_proj", + "model.layers.77.mlp.experts.9.up_proj", + "model.layers.77.mlp.experts.90.down_proj", + "model.layers.77.mlp.experts.90.gate_proj", + "model.layers.77.mlp.experts.90.up_proj", + "model.layers.77.mlp.experts.91.down_proj", + "model.layers.77.mlp.experts.91.gate_proj", + "model.layers.77.mlp.experts.91.up_proj", + "model.layers.77.mlp.experts.92.down_proj", + "model.layers.77.mlp.experts.92.gate_proj", + "model.layers.77.mlp.experts.92.up_proj", + "model.layers.77.mlp.experts.93.down_proj", + "model.layers.77.mlp.experts.93.gate_proj", + "model.layers.77.mlp.experts.93.up_proj", + "model.layers.77.mlp.experts.94.down_proj", + "model.layers.77.mlp.experts.94.gate_proj", + "model.layers.77.mlp.experts.94.up_proj", + "model.layers.77.mlp.experts.95.down_proj", + "model.layers.77.mlp.experts.95.gate_proj", + "model.layers.77.mlp.experts.95.up_proj", + "model.layers.77.mlp.experts.96.down_proj", + "model.layers.77.mlp.experts.96.gate_proj", + "model.layers.77.mlp.experts.96.up_proj", + "model.layers.77.mlp.experts.97.down_proj", + "model.layers.77.mlp.experts.97.gate_proj", + "model.layers.77.mlp.experts.97.up_proj", + "model.layers.77.mlp.experts.98.down_proj", + "model.layers.77.mlp.experts.98.gate_proj", + "model.layers.77.mlp.experts.98.up_proj", + "model.layers.77.mlp.experts.99.down_proj", + "model.layers.77.mlp.experts.99.gate_proj", + "model.layers.77.mlp.experts.99.up_proj", + "model.layers.78.mlp.experts.0.down_proj", + "model.layers.78.mlp.experts.0.gate_proj", + "model.layers.78.mlp.experts.0.up_proj", + "model.layers.78.mlp.experts.1.down_proj", + "model.layers.78.mlp.experts.1.gate_proj", + "model.layers.78.mlp.experts.1.up_proj", + "model.layers.78.mlp.experts.10.down_proj", + "model.layers.78.mlp.experts.10.gate_proj", + "model.layers.78.mlp.experts.10.up_proj", + "model.layers.78.mlp.experts.100.down_proj", + "model.layers.78.mlp.experts.100.gate_proj", + "model.layers.78.mlp.experts.100.up_proj", + "model.layers.78.mlp.experts.101.down_proj", + "model.layers.78.mlp.experts.101.gate_proj", + "model.layers.78.mlp.experts.101.up_proj", + "model.layers.78.mlp.experts.102.down_proj", + "model.layers.78.mlp.experts.102.gate_proj", + "model.layers.78.mlp.experts.102.up_proj", + "model.layers.78.mlp.experts.103.down_proj", + "model.layers.78.mlp.experts.103.gate_proj", + "model.layers.78.mlp.experts.103.up_proj", + "model.layers.78.mlp.experts.104.down_proj", + "model.layers.78.mlp.experts.104.gate_proj", + "model.layers.78.mlp.experts.104.up_proj", + "model.layers.78.mlp.experts.105.down_proj", + "model.layers.78.mlp.experts.105.gate_proj", + "model.layers.78.mlp.experts.105.up_proj", + "model.layers.78.mlp.experts.106.down_proj", + "model.layers.78.mlp.experts.106.gate_proj", + "model.layers.78.mlp.experts.106.up_proj", + "model.layers.78.mlp.experts.107.down_proj", + "model.layers.78.mlp.experts.107.gate_proj", + "model.layers.78.mlp.experts.107.up_proj", + "model.layers.78.mlp.experts.108.down_proj", + "model.layers.78.mlp.experts.108.gate_proj", + "model.layers.78.mlp.experts.108.up_proj", + "model.layers.78.mlp.experts.109.down_proj", + "model.layers.78.mlp.experts.109.gate_proj", + "model.layers.78.mlp.experts.109.up_proj", + "model.layers.78.mlp.experts.11.down_proj", + "model.layers.78.mlp.experts.11.gate_proj", + "model.layers.78.mlp.experts.11.up_proj", + "model.layers.78.mlp.experts.110.down_proj", + "model.layers.78.mlp.experts.110.gate_proj", + "model.layers.78.mlp.experts.110.up_proj", + "model.layers.78.mlp.experts.111.down_proj", + "model.layers.78.mlp.experts.111.gate_proj", + "model.layers.78.mlp.experts.111.up_proj", + "model.layers.78.mlp.experts.112.down_proj", + "model.layers.78.mlp.experts.112.gate_proj", + "model.layers.78.mlp.experts.112.up_proj", + "model.layers.78.mlp.experts.113.down_proj", + "model.layers.78.mlp.experts.113.gate_proj", + "model.layers.78.mlp.experts.113.up_proj", + "model.layers.78.mlp.experts.114.down_proj", + "model.layers.78.mlp.experts.114.gate_proj", + "model.layers.78.mlp.experts.114.up_proj", + "model.layers.78.mlp.experts.115.down_proj", + "model.layers.78.mlp.experts.115.gate_proj", + "model.layers.78.mlp.experts.115.up_proj", + "model.layers.78.mlp.experts.116.down_proj", + "model.layers.78.mlp.experts.116.gate_proj", + "model.layers.78.mlp.experts.116.up_proj", + "model.layers.78.mlp.experts.117.down_proj", + "model.layers.78.mlp.experts.117.gate_proj", + "model.layers.78.mlp.experts.117.up_proj", + "model.layers.78.mlp.experts.118.down_proj", + "model.layers.78.mlp.experts.118.gate_proj", + "model.layers.78.mlp.experts.118.up_proj", + "model.layers.78.mlp.experts.119.down_proj", + "model.layers.78.mlp.experts.119.gate_proj", + "model.layers.78.mlp.experts.119.up_proj", + "model.layers.78.mlp.experts.12.down_proj", + "model.layers.78.mlp.experts.12.gate_proj", + "model.layers.78.mlp.experts.12.up_proj", + "model.layers.78.mlp.experts.120.down_proj", + "model.layers.78.mlp.experts.120.gate_proj", + "model.layers.78.mlp.experts.120.up_proj", + "model.layers.78.mlp.experts.121.down_proj", + "model.layers.78.mlp.experts.121.gate_proj", + "model.layers.78.mlp.experts.121.up_proj", + "model.layers.78.mlp.experts.122.down_proj", + "model.layers.78.mlp.experts.122.gate_proj", + "model.layers.78.mlp.experts.122.up_proj", + "model.layers.78.mlp.experts.123.down_proj", + "model.layers.78.mlp.experts.123.gate_proj", + "model.layers.78.mlp.experts.123.up_proj", + "model.layers.78.mlp.experts.124.down_proj", + "model.layers.78.mlp.experts.124.gate_proj", + "model.layers.78.mlp.experts.124.up_proj", + "model.layers.78.mlp.experts.125.down_proj", + "model.layers.78.mlp.experts.125.gate_proj", + "model.layers.78.mlp.experts.125.up_proj", + "model.layers.78.mlp.experts.126.down_proj", + "model.layers.78.mlp.experts.126.gate_proj", + "model.layers.78.mlp.experts.126.up_proj", + "model.layers.78.mlp.experts.127.down_proj", + "model.layers.78.mlp.experts.127.gate_proj", + "model.layers.78.mlp.experts.127.up_proj", + "model.layers.78.mlp.experts.128.down_proj", + "model.layers.78.mlp.experts.128.gate_proj", + "model.layers.78.mlp.experts.128.up_proj", + "model.layers.78.mlp.experts.129.down_proj", + "model.layers.78.mlp.experts.129.gate_proj", + "model.layers.78.mlp.experts.129.up_proj", + "model.layers.78.mlp.experts.13.down_proj", + "model.layers.78.mlp.experts.13.gate_proj", + "model.layers.78.mlp.experts.13.up_proj", + "model.layers.78.mlp.experts.130.down_proj", + "model.layers.78.mlp.experts.130.gate_proj", + "model.layers.78.mlp.experts.130.up_proj", + "model.layers.78.mlp.experts.131.down_proj", + "model.layers.78.mlp.experts.131.gate_proj", + "model.layers.78.mlp.experts.131.up_proj", + "model.layers.78.mlp.experts.132.down_proj", + "model.layers.78.mlp.experts.132.gate_proj", + "model.layers.78.mlp.experts.132.up_proj", + "model.layers.78.mlp.experts.133.down_proj", + "model.layers.78.mlp.experts.133.gate_proj", + "model.layers.78.mlp.experts.133.up_proj", + "model.layers.78.mlp.experts.134.down_proj", + "model.layers.78.mlp.experts.134.gate_proj", + "model.layers.78.mlp.experts.134.up_proj", + "model.layers.78.mlp.experts.135.down_proj", + "model.layers.78.mlp.experts.135.gate_proj", + "model.layers.78.mlp.experts.135.up_proj", + "model.layers.78.mlp.experts.136.down_proj", + "model.layers.78.mlp.experts.136.gate_proj", + "model.layers.78.mlp.experts.136.up_proj", + "model.layers.78.mlp.experts.137.down_proj", + "model.layers.78.mlp.experts.137.gate_proj", + "model.layers.78.mlp.experts.137.up_proj", + "model.layers.78.mlp.experts.138.down_proj", + "model.layers.78.mlp.experts.138.gate_proj", + "model.layers.78.mlp.experts.138.up_proj", + "model.layers.78.mlp.experts.139.down_proj", + "model.layers.78.mlp.experts.139.gate_proj", + "model.layers.78.mlp.experts.139.up_proj", + "model.layers.78.mlp.experts.14.down_proj", + "model.layers.78.mlp.experts.14.gate_proj", + "model.layers.78.mlp.experts.14.up_proj", + "model.layers.78.mlp.experts.140.down_proj", + "model.layers.78.mlp.experts.140.gate_proj", + "model.layers.78.mlp.experts.140.up_proj", + "model.layers.78.mlp.experts.141.down_proj", + "model.layers.78.mlp.experts.141.gate_proj", + "model.layers.78.mlp.experts.141.up_proj", + "model.layers.78.mlp.experts.142.down_proj", + "model.layers.78.mlp.experts.142.gate_proj", + "model.layers.78.mlp.experts.142.up_proj", + "model.layers.78.mlp.experts.143.down_proj", + "model.layers.78.mlp.experts.143.gate_proj", + "model.layers.78.mlp.experts.143.up_proj", + "model.layers.78.mlp.experts.144.down_proj", + "model.layers.78.mlp.experts.144.gate_proj", + "model.layers.78.mlp.experts.144.up_proj", + "model.layers.78.mlp.experts.145.down_proj", + "model.layers.78.mlp.experts.145.gate_proj", + "model.layers.78.mlp.experts.145.up_proj", + "model.layers.78.mlp.experts.146.down_proj", + "model.layers.78.mlp.experts.146.gate_proj", + "model.layers.78.mlp.experts.146.up_proj", + "model.layers.78.mlp.experts.147.down_proj", + "model.layers.78.mlp.experts.147.gate_proj", + "model.layers.78.mlp.experts.147.up_proj", + "model.layers.78.mlp.experts.148.down_proj", + "model.layers.78.mlp.experts.148.gate_proj", + "model.layers.78.mlp.experts.148.up_proj", + "model.layers.78.mlp.experts.149.down_proj", + "model.layers.78.mlp.experts.149.gate_proj", + "model.layers.78.mlp.experts.149.up_proj", + "model.layers.78.mlp.experts.15.down_proj", + "model.layers.78.mlp.experts.15.gate_proj", + "model.layers.78.mlp.experts.15.up_proj", + "model.layers.78.mlp.experts.150.down_proj", + "model.layers.78.mlp.experts.150.gate_proj", + "model.layers.78.mlp.experts.150.up_proj", + "model.layers.78.mlp.experts.151.down_proj", + "model.layers.78.mlp.experts.151.gate_proj", + "model.layers.78.mlp.experts.151.up_proj", + "model.layers.78.mlp.experts.152.down_proj", + "model.layers.78.mlp.experts.152.gate_proj", + "model.layers.78.mlp.experts.152.up_proj", + "model.layers.78.mlp.experts.153.down_proj", + "model.layers.78.mlp.experts.153.gate_proj", + "model.layers.78.mlp.experts.153.up_proj", + "model.layers.78.mlp.experts.154.down_proj", + "model.layers.78.mlp.experts.154.gate_proj", + "model.layers.78.mlp.experts.154.up_proj", + "model.layers.78.mlp.experts.155.down_proj", + "model.layers.78.mlp.experts.155.gate_proj", + "model.layers.78.mlp.experts.155.up_proj", + "model.layers.78.mlp.experts.156.down_proj", + "model.layers.78.mlp.experts.156.gate_proj", + "model.layers.78.mlp.experts.156.up_proj", + "model.layers.78.mlp.experts.157.down_proj", + "model.layers.78.mlp.experts.157.gate_proj", + "model.layers.78.mlp.experts.157.up_proj", + "model.layers.78.mlp.experts.158.down_proj", + "model.layers.78.mlp.experts.158.gate_proj", + "model.layers.78.mlp.experts.158.up_proj", + "model.layers.78.mlp.experts.159.down_proj", + "model.layers.78.mlp.experts.159.gate_proj", + "model.layers.78.mlp.experts.159.up_proj", + "model.layers.78.mlp.experts.16.down_proj", + "model.layers.78.mlp.experts.16.gate_proj", + "model.layers.78.mlp.experts.16.up_proj", + "model.layers.78.mlp.experts.160.down_proj", + "model.layers.78.mlp.experts.160.gate_proj", + "model.layers.78.mlp.experts.160.up_proj", + "model.layers.78.mlp.experts.161.down_proj", + "model.layers.78.mlp.experts.161.gate_proj", + "model.layers.78.mlp.experts.161.up_proj", + "model.layers.78.mlp.experts.162.down_proj", + "model.layers.78.mlp.experts.162.gate_proj", + "model.layers.78.mlp.experts.162.up_proj", + "model.layers.78.mlp.experts.163.down_proj", + "model.layers.78.mlp.experts.163.gate_proj", + "model.layers.78.mlp.experts.163.up_proj", + "model.layers.78.mlp.experts.164.down_proj", + "model.layers.78.mlp.experts.164.gate_proj", + "model.layers.78.mlp.experts.164.up_proj", + "model.layers.78.mlp.experts.165.down_proj", + "model.layers.78.mlp.experts.165.gate_proj", + "model.layers.78.mlp.experts.165.up_proj", + "model.layers.78.mlp.experts.166.down_proj", + "model.layers.78.mlp.experts.166.gate_proj", + "model.layers.78.mlp.experts.166.up_proj", + "model.layers.78.mlp.experts.167.down_proj", + "model.layers.78.mlp.experts.167.gate_proj", + "model.layers.78.mlp.experts.167.up_proj", + "model.layers.78.mlp.experts.168.down_proj", + "model.layers.78.mlp.experts.168.gate_proj", + "model.layers.78.mlp.experts.168.up_proj", + "model.layers.78.mlp.experts.169.down_proj", + "model.layers.78.mlp.experts.169.gate_proj", + "model.layers.78.mlp.experts.169.up_proj", + "model.layers.78.mlp.experts.17.down_proj", + "model.layers.78.mlp.experts.17.gate_proj", + "model.layers.78.mlp.experts.17.up_proj", + "model.layers.78.mlp.experts.170.down_proj", + "model.layers.78.mlp.experts.170.gate_proj", + "model.layers.78.mlp.experts.170.up_proj", + "model.layers.78.mlp.experts.171.down_proj", + "model.layers.78.mlp.experts.171.gate_proj", + "model.layers.78.mlp.experts.171.up_proj", + "model.layers.78.mlp.experts.172.down_proj", + "model.layers.78.mlp.experts.172.gate_proj", + "model.layers.78.mlp.experts.172.up_proj", + "model.layers.78.mlp.experts.173.down_proj", + "model.layers.78.mlp.experts.173.gate_proj", + "model.layers.78.mlp.experts.173.up_proj", + "model.layers.78.mlp.experts.174.down_proj", + "model.layers.78.mlp.experts.174.gate_proj", + "model.layers.78.mlp.experts.174.up_proj", + "model.layers.78.mlp.experts.175.down_proj", + "model.layers.78.mlp.experts.175.gate_proj", + "model.layers.78.mlp.experts.175.up_proj", + "model.layers.78.mlp.experts.176.down_proj", + "model.layers.78.mlp.experts.176.gate_proj", + "model.layers.78.mlp.experts.176.up_proj", + "model.layers.78.mlp.experts.177.down_proj", + "model.layers.78.mlp.experts.177.gate_proj", + "model.layers.78.mlp.experts.177.up_proj", + "model.layers.78.mlp.experts.178.down_proj", + "model.layers.78.mlp.experts.178.gate_proj", + "model.layers.78.mlp.experts.178.up_proj", + "model.layers.78.mlp.experts.179.down_proj", + "model.layers.78.mlp.experts.179.gate_proj", + "model.layers.78.mlp.experts.179.up_proj", + "model.layers.78.mlp.experts.18.down_proj", + "model.layers.78.mlp.experts.18.gate_proj", + "model.layers.78.mlp.experts.18.up_proj", + "model.layers.78.mlp.experts.180.down_proj", + "model.layers.78.mlp.experts.180.gate_proj", + "model.layers.78.mlp.experts.180.up_proj", + "model.layers.78.mlp.experts.181.down_proj", + "model.layers.78.mlp.experts.181.gate_proj", + "model.layers.78.mlp.experts.181.up_proj", + "model.layers.78.mlp.experts.182.down_proj", + "model.layers.78.mlp.experts.182.gate_proj", + "model.layers.78.mlp.experts.182.up_proj", + "model.layers.78.mlp.experts.183.down_proj", + "model.layers.78.mlp.experts.183.gate_proj", + "model.layers.78.mlp.experts.183.up_proj", + "model.layers.78.mlp.experts.184.down_proj", + "model.layers.78.mlp.experts.184.gate_proj", + "model.layers.78.mlp.experts.184.up_proj", + "model.layers.78.mlp.experts.185.down_proj", + "model.layers.78.mlp.experts.185.gate_proj", + "model.layers.78.mlp.experts.185.up_proj", + "model.layers.78.mlp.experts.186.down_proj", + "model.layers.78.mlp.experts.186.gate_proj", + "model.layers.78.mlp.experts.186.up_proj", + "model.layers.78.mlp.experts.187.down_proj", + "model.layers.78.mlp.experts.187.gate_proj", + "model.layers.78.mlp.experts.187.up_proj", + "model.layers.78.mlp.experts.188.down_proj", + "model.layers.78.mlp.experts.188.gate_proj", + "model.layers.78.mlp.experts.188.up_proj", + "model.layers.78.mlp.experts.189.down_proj", + "model.layers.78.mlp.experts.189.gate_proj", + "model.layers.78.mlp.experts.189.up_proj", + "model.layers.78.mlp.experts.19.down_proj", + "model.layers.78.mlp.experts.19.gate_proj", + "model.layers.78.mlp.experts.19.up_proj", + "model.layers.78.mlp.experts.190.down_proj", + "model.layers.78.mlp.experts.190.gate_proj", + "model.layers.78.mlp.experts.190.up_proj", + "model.layers.78.mlp.experts.191.down_proj", + "model.layers.78.mlp.experts.191.gate_proj", + "model.layers.78.mlp.experts.191.up_proj", + "model.layers.78.mlp.experts.192.down_proj", + "model.layers.78.mlp.experts.192.gate_proj", + "model.layers.78.mlp.experts.192.up_proj", + "model.layers.78.mlp.experts.193.down_proj", + "model.layers.78.mlp.experts.193.gate_proj", + "model.layers.78.mlp.experts.193.up_proj", + "model.layers.78.mlp.experts.194.down_proj", + "model.layers.78.mlp.experts.194.gate_proj", + "model.layers.78.mlp.experts.194.up_proj", + "model.layers.78.mlp.experts.195.down_proj", + "model.layers.78.mlp.experts.195.gate_proj", + "model.layers.78.mlp.experts.195.up_proj", + "model.layers.78.mlp.experts.196.down_proj", + "model.layers.78.mlp.experts.196.gate_proj", + "model.layers.78.mlp.experts.196.up_proj", + "model.layers.78.mlp.experts.197.down_proj", + "model.layers.78.mlp.experts.197.gate_proj", + "model.layers.78.mlp.experts.197.up_proj", + "model.layers.78.mlp.experts.198.down_proj", + "model.layers.78.mlp.experts.198.gate_proj", + "model.layers.78.mlp.experts.198.up_proj", + "model.layers.78.mlp.experts.199.down_proj", + "model.layers.78.mlp.experts.199.gate_proj", + "model.layers.78.mlp.experts.199.up_proj", + "model.layers.78.mlp.experts.2.down_proj", + "model.layers.78.mlp.experts.2.gate_proj", + "model.layers.78.mlp.experts.2.up_proj", + "model.layers.78.mlp.experts.20.down_proj", + "model.layers.78.mlp.experts.20.gate_proj", + "model.layers.78.mlp.experts.20.up_proj", + "model.layers.78.mlp.experts.200.down_proj", + "model.layers.78.mlp.experts.200.gate_proj", + "model.layers.78.mlp.experts.200.up_proj", + "model.layers.78.mlp.experts.201.down_proj", + "model.layers.78.mlp.experts.201.gate_proj", + "model.layers.78.mlp.experts.201.up_proj", + "model.layers.78.mlp.experts.202.down_proj", + "model.layers.78.mlp.experts.202.gate_proj", + "model.layers.78.mlp.experts.202.up_proj", + "model.layers.78.mlp.experts.203.down_proj", + "model.layers.78.mlp.experts.203.gate_proj", + "model.layers.78.mlp.experts.203.up_proj", + "model.layers.78.mlp.experts.204.down_proj", + "model.layers.78.mlp.experts.204.gate_proj", + "model.layers.78.mlp.experts.204.up_proj", + "model.layers.78.mlp.experts.205.down_proj", + "model.layers.78.mlp.experts.205.gate_proj", + "model.layers.78.mlp.experts.205.up_proj", + "model.layers.78.mlp.experts.206.down_proj", + "model.layers.78.mlp.experts.206.gate_proj", + "model.layers.78.mlp.experts.206.up_proj", + "model.layers.78.mlp.experts.207.down_proj", + "model.layers.78.mlp.experts.207.gate_proj", + "model.layers.78.mlp.experts.207.up_proj", + "model.layers.78.mlp.experts.208.down_proj", + "model.layers.78.mlp.experts.208.gate_proj", + "model.layers.78.mlp.experts.208.up_proj", + "model.layers.78.mlp.experts.209.down_proj", + "model.layers.78.mlp.experts.209.gate_proj", + "model.layers.78.mlp.experts.209.up_proj", + "model.layers.78.mlp.experts.21.down_proj", + "model.layers.78.mlp.experts.21.gate_proj", + "model.layers.78.mlp.experts.21.up_proj", + "model.layers.78.mlp.experts.210.down_proj", + "model.layers.78.mlp.experts.210.gate_proj", + "model.layers.78.mlp.experts.210.up_proj", + "model.layers.78.mlp.experts.211.down_proj", + "model.layers.78.mlp.experts.211.gate_proj", + "model.layers.78.mlp.experts.211.up_proj", + "model.layers.78.mlp.experts.212.down_proj", + "model.layers.78.mlp.experts.212.gate_proj", + "model.layers.78.mlp.experts.212.up_proj", + "model.layers.78.mlp.experts.213.down_proj", + "model.layers.78.mlp.experts.213.gate_proj", + "model.layers.78.mlp.experts.213.up_proj", + "model.layers.78.mlp.experts.214.down_proj", + "model.layers.78.mlp.experts.214.gate_proj", + "model.layers.78.mlp.experts.214.up_proj", + "model.layers.78.mlp.experts.215.down_proj", + "model.layers.78.mlp.experts.215.gate_proj", + "model.layers.78.mlp.experts.215.up_proj", + "model.layers.78.mlp.experts.216.down_proj", + "model.layers.78.mlp.experts.216.gate_proj", + "model.layers.78.mlp.experts.216.up_proj", + "model.layers.78.mlp.experts.217.down_proj", + "model.layers.78.mlp.experts.217.gate_proj", + "model.layers.78.mlp.experts.217.up_proj", + "model.layers.78.mlp.experts.218.down_proj", + "model.layers.78.mlp.experts.218.gate_proj", + "model.layers.78.mlp.experts.218.up_proj", + "model.layers.78.mlp.experts.219.down_proj", + "model.layers.78.mlp.experts.219.gate_proj", + "model.layers.78.mlp.experts.219.up_proj", + "model.layers.78.mlp.experts.22.down_proj", + "model.layers.78.mlp.experts.22.gate_proj", + "model.layers.78.mlp.experts.22.up_proj", + "model.layers.78.mlp.experts.220.down_proj", + "model.layers.78.mlp.experts.220.gate_proj", + "model.layers.78.mlp.experts.220.up_proj", + "model.layers.78.mlp.experts.221.down_proj", + "model.layers.78.mlp.experts.221.gate_proj", + "model.layers.78.mlp.experts.221.up_proj", + "model.layers.78.mlp.experts.222.down_proj", + "model.layers.78.mlp.experts.222.gate_proj", + "model.layers.78.mlp.experts.222.up_proj", + "model.layers.78.mlp.experts.223.down_proj", + "model.layers.78.mlp.experts.223.gate_proj", + "model.layers.78.mlp.experts.223.up_proj", + "model.layers.78.mlp.experts.224.down_proj", + "model.layers.78.mlp.experts.224.gate_proj", + "model.layers.78.mlp.experts.224.up_proj", + "model.layers.78.mlp.experts.225.down_proj", + "model.layers.78.mlp.experts.225.gate_proj", + "model.layers.78.mlp.experts.225.up_proj", + "model.layers.78.mlp.experts.226.down_proj", + "model.layers.78.mlp.experts.226.gate_proj", + "model.layers.78.mlp.experts.226.up_proj", + "model.layers.78.mlp.experts.227.down_proj", + "model.layers.78.mlp.experts.227.gate_proj", + "model.layers.78.mlp.experts.227.up_proj", + "model.layers.78.mlp.experts.228.down_proj", + "model.layers.78.mlp.experts.228.gate_proj", + "model.layers.78.mlp.experts.228.up_proj", + "model.layers.78.mlp.experts.229.down_proj", + "model.layers.78.mlp.experts.229.gate_proj", + "model.layers.78.mlp.experts.229.up_proj", + "model.layers.78.mlp.experts.23.down_proj", + "model.layers.78.mlp.experts.23.gate_proj", + "model.layers.78.mlp.experts.23.up_proj", + "model.layers.78.mlp.experts.230.down_proj", + "model.layers.78.mlp.experts.230.gate_proj", + "model.layers.78.mlp.experts.230.up_proj", + "model.layers.78.mlp.experts.231.down_proj", + "model.layers.78.mlp.experts.231.gate_proj", + "model.layers.78.mlp.experts.231.up_proj", + "model.layers.78.mlp.experts.232.down_proj", + "model.layers.78.mlp.experts.232.gate_proj", + "model.layers.78.mlp.experts.232.up_proj", + "model.layers.78.mlp.experts.233.down_proj", + "model.layers.78.mlp.experts.233.gate_proj", + "model.layers.78.mlp.experts.233.up_proj", + "model.layers.78.mlp.experts.234.down_proj", + "model.layers.78.mlp.experts.234.gate_proj", + "model.layers.78.mlp.experts.234.up_proj", + "model.layers.78.mlp.experts.235.down_proj", + "model.layers.78.mlp.experts.235.gate_proj", + "model.layers.78.mlp.experts.235.up_proj", + "model.layers.78.mlp.experts.236.down_proj", + "model.layers.78.mlp.experts.236.gate_proj", + "model.layers.78.mlp.experts.236.up_proj", + "model.layers.78.mlp.experts.237.down_proj", + "model.layers.78.mlp.experts.237.gate_proj", + "model.layers.78.mlp.experts.237.up_proj", + "model.layers.78.mlp.experts.238.down_proj", + "model.layers.78.mlp.experts.238.gate_proj", + "model.layers.78.mlp.experts.238.up_proj", + "model.layers.78.mlp.experts.239.down_proj", + "model.layers.78.mlp.experts.239.gate_proj", + "model.layers.78.mlp.experts.239.up_proj", + "model.layers.78.mlp.experts.24.down_proj", + "model.layers.78.mlp.experts.24.gate_proj", + "model.layers.78.mlp.experts.24.up_proj", + "model.layers.78.mlp.experts.240.down_proj", + "model.layers.78.mlp.experts.240.gate_proj", + "model.layers.78.mlp.experts.240.up_proj", + "model.layers.78.mlp.experts.241.down_proj", + "model.layers.78.mlp.experts.241.gate_proj", + "model.layers.78.mlp.experts.241.up_proj", + "model.layers.78.mlp.experts.242.down_proj", + "model.layers.78.mlp.experts.242.gate_proj", + "model.layers.78.mlp.experts.242.up_proj", + "model.layers.78.mlp.experts.243.down_proj", + "model.layers.78.mlp.experts.243.gate_proj", + "model.layers.78.mlp.experts.243.up_proj", + "model.layers.78.mlp.experts.244.down_proj", + "model.layers.78.mlp.experts.244.gate_proj", + "model.layers.78.mlp.experts.244.up_proj", + "model.layers.78.mlp.experts.245.down_proj", + "model.layers.78.mlp.experts.245.gate_proj", + "model.layers.78.mlp.experts.245.up_proj", + "model.layers.78.mlp.experts.246.down_proj", + "model.layers.78.mlp.experts.246.gate_proj", + "model.layers.78.mlp.experts.246.up_proj", + "model.layers.78.mlp.experts.247.down_proj", + "model.layers.78.mlp.experts.247.gate_proj", + "model.layers.78.mlp.experts.247.up_proj", + "model.layers.78.mlp.experts.248.down_proj", + "model.layers.78.mlp.experts.248.gate_proj", + "model.layers.78.mlp.experts.248.up_proj", + "model.layers.78.mlp.experts.249.down_proj", + "model.layers.78.mlp.experts.249.gate_proj", + "model.layers.78.mlp.experts.249.up_proj", + "model.layers.78.mlp.experts.25.down_proj", + "model.layers.78.mlp.experts.25.gate_proj", + "model.layers.78.mlp.experts.25.up_proj", + "model.layers.78.mlp.experts.250.down_proj", + "model.layers.78.mlp.experts.250.gate_proj", + "model.layers.78.mlp.experts.250.up_proj", + "model.layers.78.mlp.experts.251.down_proj", + "model.layers.78.mlp.experts.251.gate_proj", + "model.layers.78.mlp.experts.251.up_proj", + "model.layers.78.mlp.experts.252.down_proj", + "model.layers.78.mlp.experts.252.gate_proj", + "model.layers.78.mlp.experts.252.up_proj", + "model.layers.78.mlp.experts.253.down_proj", + "model.layers.78.mlp.experts.253.gate_proj", + "model.layers.78.mlp.experts.253.up_proj", + "model.layers.78.mlp.experts.254.down_proj", + "model.layers.78.mlp.experts.254.gate_proj", + "model.layers.78.mlp.experts.254.up_proj", + "model.layers.78.mlp.experts.255.down_proj", + "model.layers.78.mlp.experts.255.gate_proj", + "model.layers.78.mlp.experts.255.up_proj", + "model.layers.78.mlp.experts.26.down_proj", + "model.layers.78.mlp.experts.26.gate_proj", + "model.layers.78.mlp.experts.26.up_proj", + "model.layers.78.mlp.experts.27.down_proj", + "model.layers.78.mlp.experts.27.gate_proj", + "model.layers.78.mlp.experts.27.up_proj", + "model.layers.78.mlp.experts.28.down_proj", + "model.layers.78.mlp.experts.28.gate_proj", + "model.layers.78.mlp.experts.28.up_proj", + "model.layers.78.mlp.experts.29.down_proj", + "model.layers.78.mlp.experts.29.gate_proj", + "model.layers.78.mlp.experts.29.up_proj", + "model.layers.78.mlp.experts.3.down_proj", + "model.layers.78.mlp.experts.3.gate_proj", + "model.layers.78.mlp.experts.3.up_proj", + "model.layers.78.mlp.experts.30.down_proj", + "model.layers.78.mlp.experts.30.gate_proj", + "model.layers.78.mlp.experts.30.up_proj", + "model.layers.78.mlp.experts.31.down_proj", + "model.layers.78.mlp.experts.31.gate_proj", + "model.layers.78.mlp.experts.31.up_proj", + "model.layers.78.mlp.experts.32.down_proj", + "model.layers.78.mlp.experts.32.gate_proj", + "model.layers.78.mlp.experts.32.up_proj", + "model.layers.78.mlp.experts.33.down_proj", + "model.layers.78.mlp.experts.33.gate_proj", + "model.layers.78.mlp.experts.33.up_proj", + "model.layers.78.mlp.experts.34.down_proj", + "model.layers.78.mlp.experts.34.gate_proj", + "model.layers.78.mlp.experts.34.up_proj", + "model.layers.78.mlp.experts.35.down_proj", + "model.layers.78.mlp.experts.35.gate_proj", + "model.layers.78.mlp.experts.35.up_proj", + "model.layers.78.mlp.experts.36.down_proj", + "model.layers.78.mlp.experts.36.gate_proj", + "model.layers.78.mlp.experts.36.up_proj", + "model.layers.78.mlp.experts.37.down_proj", + "model.layers.78.mlp.experts.37.gate_proj", + "model.layers.78.mlp.experts.37.up_proj", + "model.layers.78.mlp.experts.38.down_proj", + "model.layers.78.mlp.experts.38.gate_proj", + "model.layers.78.mlp.experts.38.up_proj", + "model.layers.78.mlp.experts.39.down_proj", + "model.layers.78.mlp.experts.39.gate_proj", + "model.layers.78.mlp.experts.39.up_proj", + "model.layers.78.mlp.experts.4.down_proj", + "model.layers.78.mlp.experts.4.gate_proj", + "model.layers.78.mlp.experts.4.up_proj", + "model.layers.78.mlp.experts.40.down_proj", + "model.layers.78.mlp.experts.40.gate_proj", + "model.layers.78.mlp.experts.40.up_proj", + "model.layers.78.mlp.experts.41.down_proj", + "model.layers.78.mlp.experts.41.gate_proj", + "model.layers.78.mlp.experts.41.up_proj", + "model.layers.78.mlp.experts.42.down_proj", + "model.layers.78.mlp.experts.42.gate_proj", + "model.layers.78.mlp.experts.42.up_proj", + "model.layers.78.mlp.experts.43.down_proj", + "model.layers.78.mlp.experts.43.gate_proj", + "model.layers.78.mlp.experts.43.up_proj", + "model.layers.78.mlp.experts.44.down_proj", + "model.layers.78.mlp.experts.44.gate_proj", + "model.layers.78.mlp.experts.44.up_proj", + "model.layers.78.mlp.experts.45.down_proj", + "model.layers.78.mlp.experts.45.gate_proj", + "model.layers.78.mlp.experts.45.up_proj", + "model.layers.78.mlp.experts.46.down_proj", + "model.layers.78.mlp.experts.46.gate_proj", + "model.layers.78.mlp.experts.46.up_proj", + "model.layers.78.mlp.experts.47.down_proj", + "model.layers.78.mlp.experts.47.gate_proj", + "model.layers.78.mlp.experts.47.up_proj", + "model.layers.78.mlp.experts.48.down_proj", + "model.layers.78.mlp.experts.48.gate_proj", + "model.layers.78.mlp.experts.48.up_proj", + "model.layers.78.mlp.experts.49.down_proj", + "model.layers.78.mlp.experts.49.gate_proj", + "model.layers.78.mlp.experts.49.up_proj", + "model.layers.78.mlp.experts.5.down_proj", + "model.layers.78.mlp.experts.5.gate_proj", + "model.layers.78.mlp.experts.5.up_proj", + "model.layers.78.mlp.experts.50.down_proj", + "model.layers.78.mlp.experts.50.gate_proj", + "model.layers.78.mlp.experts.50.up_proj", + "model.layers.78.mlp.experts.51.down_proj", + "model.layers.78.mlp.experts.51.gate_proj", + "model.layers.78.mlp.experts.51.up_proj", + "model.layers.78.mlp.experts.52.down_proj", + "model.layers.78.mlp.experts.52.gate_proj", + "model.layers.78.mlp.experts.52.up_proj", + "model.layers.78.mlp.experts.53.down_proj", + "model.layers.78.mlp.experts.53.gate_proj", + "model.layers.78.mlp.experts.53.up_proj", + "model.layers.78.mlp.experts.54.down_proj", + "model.layers.78.mlp.experts.54.gate_proj", + "model.layers.78.mlp.experts.54.up_proj", + "model.layers.78.mlp.experts.55.down_proj", + "model.layers.78.mlp.experts.55.gate_proj", + "model.layers.78.mlp.experts.55.up_proj", + "model.layers.78.mlp.experts.56.down_proj", + "model.layers.78.mlp.experts.56.gate_proj", + "model.layers.78.mlp.experts.56.up_proj", + "model.layers.78.mlp.experts.57.down_proj", + "model.layers.78.mlp.experts.57.gate_proj", + "model.layers.78.mlp.experts.57.up_proj", + "model.layers.78.mlp.experts.58.down_proj", + "model.layers.78.mlp.experts.58.gate_proj", + "model.layers.78.mlp.experts.58.up_proj", + "model.layers.78.mlp.experts.59.down_proj", + "model.layers.78.mlp.experts.59.gate_proj", + "model.layers.78.mlp.experts.59.up_proj", + "model.layers.78.mlp.experts.6.down_proj", + "model.layers.78.mlp.experts.6.gate_proj", + "model.layers.78.mlp.experts.6.up_proj", + "model.layers.78.mlp.experts.60.down_proj", + "model.layers.78.mlp.experts.60.gate_proj", + "model.layers.78.mlp.experts.60.up_proj", + "model.layers.78.mlp.experts.61.down_proj", + "model.layers.78.mlp.experts.61.gate_proj", + "model.layers.78.mlp.experts.61.up_proj", + "model.layers.78.mlp.experts.62.down_proj", + "model.layers.78.mlp.experts.62.gate_proj", + "model.layers.78.mlp.experts.62.up_proj", + "model.layers.78.mlp.experts.63.down_proj", + "model.layers.78.mlp.experts.63.gate_proj", + "model.layers.78.mlp.experts.63.up_proj", + "model.layers.78.mlp.experts.64.down_proj", + "model.layers.78.mlp.experts.64.gate_proj", + "model.layers.78.mlp.experts.64.up_proj", + "model.layers.78.mlp.experts.65.down_proj", + "model.layers.78.mlp.experts.65.gate_proj", + "model.layers.78.mlp.experts.65.up_proj", + "model.layers.78.mlp.experts.66.down_proj", + "model.layers.78.mlp.experts.66.gate_proj", + "model.layers.78.mlp.experts.66.up_proj", + "model.layers.78.mlp.experts.67.down_proj", + "model.layers.78.mlp.experts.67.gate_proj", + "model.layers.78.mlp.experts.67.up_proj", + "model.layers.78.mlp.experts.68.down_proj", + "model.layers.78.mlp.experts.68.gate_proj", + "model.layers.78.mlp.experts.68.up_proj", + "model.layers.78.mlp.experts.69.down_proj", + "model.layers.78.mlp.experts.69.gate_proj", + "model.layers.78.mlp.experts.69.up_proj", + "model.layers.78.mlp.experts.7.down_proj", + "model.layers.78.mlp.experts.7.gate_proj", + "model.layers.78.mlp.experts.7.up_proj", + "model.layers.78.mlp.experts.70.down_proj", + "model.layers.78.mlp.experts.70.gate_proj", + "model.layers.78.mlp.experts.70.up_proj", + "model.layers.78.mlp.experts.71.down_proj", + "model.layers.78.mlp.experts.71.gate_proj", + "model.layers.78.mlp.experts.71.up_proj", + "model.layers.78.mlp.experts.72.down_proj", + "model.layers.78.mlp.experts.72.gate_proj", + "model.layers.78.mlp.experts.72.up_proj", + "model.layers.78.mlp.experts.73.down_proj", + "model.layers.78.mlp.experts.73.gate_proj", + "model.layers.78.mlp.experts.73.up_proj", + "model.layers.78.mlp.experts.74.down_proj", + "model.layers.78.mlp.experts.74.gate_proj", + "model.layers.78.mlp.experts.74.up_proj", + "model.layers.78.mlp.experts.75.down_proj", + "model.layers.78.mlp.experts.75.gate_proj", + "model.layers.78.mlp.experts.75.up_proj", + "model.layers.78.mlp.experts.76.down_proj", + "model.layers.78.mlp.experts.76.gate_proj", + "model.layers.78.mlp.experts.76.up_proj", + "model.layers.78.mlp.experts.77.down_proj", + "model.layers.78.mlp.experts.77.gate_proj", + "model.layers.78.mlp.experts.77.up_proj", + "model.layers.78.mlp.experts.78.down_proj", + "model.layers.78.mlp.experts.78.gate_proj", + "model.layers.78.mlp.experts.78.up_proj", + "model.layers.78.mlp.experts.79.down_proj", + "model.layers.78.mlp.experts.79.gate_proj", + "model.layers.78.mlp.experts.79.up_proj", + "model.layers.78.mlp.experts.8.down_proj", + "model.layers.78.mlp.experts.8.gate_proj", + "model.layers.78.mlp.experts.8.up_proj", + "model.layers.78.mlp.experts.80.down_proj", + "model.layers.78.mlp.experts.80.gate_proj", + "model.layers.78.mlp.experts.80.up_proj", + "model.layers.78.mlp.experts.81.down_proj", + "model.layers.78.mlp.experts.81.gate_proj", + "model.layers.78.mlp.experts.81.up_proj", + "model.layers.78.mlp.experts.82.down_proj", + "model.layers.78.mlp.experts.82.gate_proj", + "model.layers.78.mlp.experts.82.up_proj", + "model.layers.78.mlp.experts.83.down_proj", + "model.layers.78.mlp.experts.83.gate_proj", + "model.layers.78.mlp.experts.83.up_proj", + "model.layers.78.mlp.experts.84.down_proj", + "model.layers.78.mlp.experts.84.gate_proj", + "model.layers.78.mlp.experts.84.up_proj", + "model.layers.78.mlp.experts.85.down_proj", + "model.layers.78.mlp.experts.85.gate_proj", + "model.layers.78.mlp.experts.85.up_proj", + "model.layers.78.mlp.experts.86.down_proj", + "model.layers.78.mlp.experts.86.gate_proj", + "model.layers.78.mlp.experts.86.up_proj", + "model.layers.78.mlp.experts.87.down_proj", + "model.layers.78.mlp.experts.87.gate_proj", + "model.layers.78.mlp.experts.87.up_proj", + "model.layers.78.mlp.experts.88.down_proj", + "model.layers.78.mlp.experts.88.gate_proj", + "model.layers.78.mlp.experts.88.up_proj", + "model.layers.78.mlp.experts.89.down_proj", + "model.layers.78.mlp.experts.89.gate_proj", + "model.layers.78.mlp.experts.89.up_proj", + "model.layers.78.mlp.experts.9.down_proj", + "model.layers.78.mlp.experts.9.gate_proj", + "model.layers.78.mlp.experts.9.up_proj", + "model.layers.78.mlp.experts.90.down_proj", + "model.layers.78.mlp.experts.90.gate_proj", + "model.layers.78.mlp.experts.90.up_proj", + "model.layers.78.mlp.experts.91.down_proj", + "model.layers.78.mlp.experts.91.gate_proj", + "model.layers.78.mlp.experts.91.up_proj", + "model.layers.78.mlp.experts.92.down_proj", + "model.layers.78.mlp.experts.92.gate_proj", + "model.layers.78.mlp.experts.92.up_proj", + "model.layers.78.mlp.experts.93.down_proj", + "model.layers.78.mlp.experts.93.gate_proj", + "model.layers.78.mlp.experts.93.up_proj", + "model.layers.78.mlp.experts.94.down_proj", + "model.layers.78.mlp.experts.94.gate_proj", + "model.layers.78.mlp.experts.94.up_proj", + "model.layers.78.mlp.experts.95.down_proj", + "model.layers.78.mlp.experts.95.gate_proj", + "model.layers.78.mlp.experts.95.up_proj", + "model.layers.78.mlp.experts.96.down_proj", + "model.layers.78.mlp.experts.96.gate_proj", + "model.layers.78.mlp.experts.96.up_proj", + "model.layers.78.mlp.experts.97.down_proj", + "model.layers.78.mlp.experts.97.gate_proj", + "model.layers.78.mlp.experts.97.up_proj", + "model.layers.78.mlp.experts.98.down_proj", + "model.layers.78.mlp.experts.98.gate_proj", + "model.layers.78.mlp.experts.98.up_proj", + "model.layers.78.mlp.experts.99.down_proj", + "model.layers.78.mlp.experts.99.gate_proj", + "model.layers.78.mlp.experts.99.up_proj", + "model.layers.8.mlp.experts.0.down_proj", + "model.layers.8.mlp.experts.0.gate_proj", + "model.layers.8.mlp.experts.0.up_proj", + "model.layers.8.mlp.experts.1.down_proj", + "model.layers.8.mlp.experts.1.gate_proj", + "model.layers.8.mlp.experts.1.up_proj", + "model.layers.8.mlp.experts.10.down_proj", + "model.layers.8.mlp.experts.10.gate_proj", + "model.layers.8.mlp.experts.10.up_proj", + "model.layers.8.mlp.experts.100.down_proj", + "model.layers.8.mlp.experts.100.gate_proj", + "model.layers.8.mlp.experts.100.up_proj", + "model.layers.8.mlp.experts.101.down_proj", + "model.layers.8.mlp.experts.101.gate_proj", + "model.layers.8.mlp.experts.101.up_proj", + "model.layers.8.mlp.experts.102.down_proj", + "model.layers.8.mlp.experts.102.gate_proj", + "model.layers.8.mlp.experts.102.up_proj", + "model.layers.8.mlp.experts.103.down_proj", + "model.layers.8.mlp.experts.103.gate_proj", + "model.layers.8.mlp.experts.103.up_proj", + "model.layers.8.mlp.experts.104.down_proj", + "model.layers.8.mlp.experts.104.gate_proj", + "model.layers.8.mlp.experts.104.up_proj", + "model.layers.8.mlp.experts.105.down_proj", + "model.layers.8.mlp.experts.105.gate_proj", + "model.layers.8.mlp.experts.105.up_proj", + "model.layers.8.mlp.experts.106.down_proj", + "model.layers.8.mlp.experts.106.gate_proj", + "model.layers.8.mlp.experts.106.up_proj", + "model.layers.8.mlp.experts.107.down_proj", + "model.layers.8.mlp.experts.107.gate_proj", + "model.layers.8.mlp.experts.107.up_proj", + "model.layers.8.mlp.experts.108.down_proj", + "model.layers.8.mlp.experts.108.gate_proj", + "model.layers.8.mlp.experts.108.up_proj", + "model.layers.8.mlp.experts.109.down_proj", + "model.layers.8.mlp.experts.109.gate_proj", + "model.layers.8.mlp.experts.109.up_proj", + "model.layers.8.mlp.experts.11.down_proj", + "model.layers.8.mlp.experts.11.gate_proj", + "model.layers.8.mlp.experts.11.up_proj", + "model.layers.8.mlp.experts.110.down_proj", + "model.layers.8.mlp.experts.110.gate_proj", + "model.layers.8.mlp.experts.110.up_proj", + "model.layers.8.mlp.experts.111.down_proj", + "model.layers.8.mlp.experts.111.gate_proj", + "model.layers.8.mlp.experts.111.up_proj", + "model.layers.8.mlp.experts.112.down_proj", + "model.layers.8.mlp.experts.112.gate_proj", + "model.layers.8.mlp.experts.112.up_proj", + "model.layers.8.mlp.experts.113.down_proj", + "model.layers.8.mlp.experts.113.gate_proj", + "model.layers.8.mlp.experts.113.up_proj", + "model.layers.8.mlp.experts.114.down_proj", + "model.layers.8.mlp.experts.114.gate_proj", + "model.layers.8.mlp.experts.114.up_proj", + "model.layers.8.mlp.experts.115.down_proj", + "model.layers.8.mlp.experts.115.gate_proj", + "model.layers.8.mlp.experts.115.up_proj", + "model.layers.8.mlp.experts.116.down_proj", + "model.layers.8.mlp.experts.116.gate_proj", + "model.layers.8.mlp.experts.116.up_proj", + "model.layers.8.mlp.experts.117.down_proj", + "model.layers.8.mlp.experts.117.gate_proj", + "model.layers.8.mlp.experts.117.up_proj", + "model.layers.8.mlp.experts.118.down_proj", + "model.layers.8.mlp.experts.118.gate_proj", + "model.layers.8.mlp.experts.118.up_proj", + "model.layers.8.mlp.experts.119.down_proj", + "model.layers.8.mlp.experts.119.gate_proj", + "model.layers.8.mlp.experts.119.up_proj", + "model.layers.8.mlp.experts.12.down_proj", + "model.layers.8.mlp.experts.12.gate_proj", + "model.layers.8.mlp.experts.12.up_proj", + "model.layers.8.mlp.experts.120.down_proj", + "model.layers.8.mlp.experts.120.gate_proj", + "model.layers.8.mlp.experts.120.up_proj", + "model.layers.8.mlp.experts.121.down_proj", + "model.layers.8.mlp.experts.121.gate_proj", + "model.layers.8.mlp.experts.121.up_proj", + "model.layers.8.mlp.experts.122.down_proj", + "model.layers.8.mlp.experts.122.gate_proj", + "model.layers.8.mlp.experts.122.up_proj", + "model.layers.8.mlp.experts.123.down_proj", + "model.layers.8.mlp.experts.123.gate_proj", + "model.layers.8.mlp.experts.123.up_proj", + "model.layers.8.mlp.experts.124.down_proj", + "model.layers.8.mlp.experts.124.gate_proj", + "model.layers.8.mlp.experts.124.up_proj", + "model.layers.8.mlp.experts.125.down_proj", + "model.layers.8.mlp.experts.125.gate_proj", + "model.layers.8.mlp.experts.125.up_proj", + "model.layers.8.mlp.experts.126.down_proj", + "model.layers.8.mlp.experts.126.gate_proj", + "model.layers.8.mlp.experts.126.up_proj", + "model.layers.8.mlp.experts.127.down_proj", + "model.layers.8.mlp.experts.127.gate_proj", + "model.layers.8.mlp.experts.127.up_proj", + "model.layers.8.mlp.experts.128.down_proj", + "model.layers.8.mlp.experts.128.gate_proj", + "model.layers.8.mlp.experts.128.up_proj", + "model.layers.8.mlp.experts.129.down_proj", + "model.layers.8.mlp.experts.129.gate_proj", + "model.layers.8.mlp.experts.129.up_proj", + "model.layers.8.mlp.experts.13.down_proj", + "model.layers.8.mlp.experts.13.gate_proj", + "model.layers.8.mlp.experts.13.up_proj", + "model.layers.8.mlp.experts.130.down_proj", + "model.layers.8.mlp.experts.130.gate_proj", + "model.layers.8.mlp.experts.130.up_proj", + "model.layers.8.mlp.experts.131.down_proj", + "model.layers.8.mlp.experts.131.gate_proj", + "model.layers.8.mlp.experts.131.up_proj", + "model.layers.8.mlp.experts.132.down_proj", + "model.layers.8.mlp.experts.132.gate_proj", + "model.layers.8.mlp.experts.132.up_proj", + "model.layers.8.mlp.experts.133.down_proj", + "model.layers.8.mlp.experts.133.gate_proj", + "model.layers.8.mlp.experts.133.up_proj", + "model.layers.8.mlp.experts.134.down_proj", + "model.layers.8.mlp.experts.134.gate_proj", + "model.layers.8.mlp.experts.134.up_proj", + "model.layers.8.mlp.experts.135.down_proj", + "model.layers.8.mlp.experts.135.gate_proj", + "model.layers.8.mlp.experts.135.up_proj", + "model.layers.8.mlp.experts.136.down_proj", + "model.layers.8.mlp.experts.136.gate_proj", + "model.layers.8.mlp.experts.136.up_proj", + "model.layers.8.mlp.experts.137.down_proj", + "model.layers.8.mlp.experts.137.gate_proj", + "model.layers.8.mlp.experts.137.up_proj", + "model.layers.8.mlp.experts.138.down_proj", + "model.layers.8.mlp.experts.138.gate_proj", + "model.layers.8.mlp.experts.138.up_proj", + "model.layers.8.mlp.experts.139.down_proj", + "model.layers.8.mlp.experts.139.gate_proj", + "model.layers.8.mlp.experts.139.up_proj", + "model.layers.8.mlp.experts.14.down_proj", + "model.layers.8.mlp.experts.14.gate_proj", + "model.layers.8.mlp.experts.14.up_proj", + "model.layers.8.mlp.experts.140.down_proj", + "model.layers.8.mlp.experts.140.gate_proj", + "model.layers.8.mlp.experts.140.up_proj", + "model.layers.8.mlp.experts.141.down_proj", + "model.layers.8.mlp.experts.141.gate_proj", + "model.layers.8.mlp.experts.141.up_proj", + "model.layers.8.mlp.experts.142.down_proj", + "model.layers.8.mlp.experts.142.gate_proj", + "model.layers.8.mlp.experts.142.up_proj", + "model.layers.8.mlp.experts.143.down_proj", + "model.layers.8.mlp.experts.143.gate_proj", + "model.layers.8.mlp.experts.143.up_proj", + "model.layers.8.mlp.experts.144.down_proj", + "model.layers.8.mlp.experts.144.gate_proj", + "model.layers.8.mlp.experts.144.up_proj", + "model.layers.8.mlp.experts.145.down_proj", + "model.layers.8.mlp.experts.145.gate_proj", + "model.layers.8.mlp.experts.145.up_proj", + "model.layers.8.mlp.experts.146.down_proj", + "model.layers.8.mlp.experts.146.gate_proj", + "model.layers.8.mlp.experts.146.up_proj", + "model.layers.8.mlp.experts.147.down_proj", + "model.layers.8.mlp.experts.147.gate_proj", + "model.layers.8.mlp.experts.147.up_proj", + "model.layers.8.mlp.experts.148.down_proj", + "model.layers.8.mlp.experts.148.gate_proj", + "model.layers.8.mlp.experts.148.up_proj", + "model.layers.8.mlp.experts.149.down_proj", + "model.layers.8.mlp.experts.149.gate_proj", + "model.layers.8.mlp.experts.149.up_proj", + "model.layers.8.mlp.experts.15.down_proj", + "model.layers.8.mlp.experts.15.gate_proj", + "model.layers.8.mlp.experts.15.up_proj", + "model.layers.8.mlp.experts.150.down_proj", + "model.layers.8.mlp.experts.150.gate_proj", + "model.layers.8.mlp.experts.150.up_proj", + "model.layers.8.mlp.experts.151.down_proj", + "model.layers.8.mlp.experts.151.gate_proj", + "model.layers.8.mlp.experts.151.up_proj", + "model.layers.8.mlp.experts.152.down_proj", + "model.layers.8.mlp.experts.152.gate_proj", + "model.layers.8.mlp.experts.152.up_proj", + "model.layers.8.mlp.experts.153.down_proj", + "model.layers.8.mlp.experts.153.gate_proj", + "model.layers.8.mlp.experts.153.up_proj", + "model.layers.8.mlp.experts.154.down_proj", + "model.layers.8.mlp.experts.154.gate_proj", + "model.layers.8.mlp.experts.154.up_proj", + "model.layers.8.mlp.experts.155.down_proj", + "model.layers.8.mlp.experts.155.gate_proj", + "model.layers.8.mlp.experts.155.up_proj", + "model.layers.8.mlp.experts.156.down_proj", + "model.layers.8.mlp.experts.156.gate_proj", + "model.layers.8.mlp.experts.156.up_proj", + "model.layers.8.mlp.experts.157.down_proj", + "model.layers.8.mlp.experts.157.gate_proj", + "model.layers.8.mlp.experts.157.up_proj", + "model.layers.8.mlp.experts.158.down_proj", + "model.layers.8.mlp.experts.158.gate_proj", + "model.layers.8.mlp.experts.158.up_proj", + "model.layers.8.mlp.experts.159.down_proj", + "model.layers.8.mlp.experts.159.gate_proj", + "model.layers.8.mlp.experts.159.up_proj", + "model.layers.8.mlp.experts.16.down_proj", + "model.layers.8.mlp.experts.16.gate_proj", + "model.layers.8.mlp.experts.16.up_proj", + "model.layers.8.mlp.experts.160.down_proj", + "model.layers.8.mlp.experts.160.gate_proj", + "model.layers.8.mlp.experts.160.up_proj", + "model.layers.8.mlp.experts.161.down_proj", + "model.layers.8.mlp.experts.161.gate_proj", + "model.layers.8.mlp.experts.161.up_proj", + "model.layers.8.mlp.experts.162.down_proj", + "model.layers.8.mlp.experts.162.gate_proj", + "model.layers.8.mlp.experts.162.up_proj", + "model.layers.8.mlp.experts.163.down_proj", + "model.layers.8.mlp.experts.163.gate_proj", + "model.layers.8.mlp.experts.163.up_proj", + "model.layers.8.mlp.experts.164.down_proj", + "model.layers.8.mlp.experts.164.gate_proj", + "model.layers.8.mlp.experts.164.up_proj", + "model.layers.8.mlp.experts.165.down_proj", + "model.layers.8.mlp.experts.165.gate_proj", + "model.layers.8.mlp.experts.165.up_proj", + "model.layers.8.mlp.experts.166.down_proj", + "model.layers.8.mlp.experts.166.gate_proj", + "model.layers.8.mlp.experts.166.up_proj", + "model.layers.8.mlp.experts.167.down_proj", + "model.layers.8.mlp.experts.167.gate_proj", + "model.layers.8.mlp.experts.167.up_proj", + "model.layers.8.mlp.experts.168.down_proj", + "model.layers.8.mlp.experts.168.gate_proj", + "model.layers.8.mlp.experts.168.up_proj", + "model.layers.8.mlp.experts.169.down_proj", + "model.layers.8.mlp.experts.169.gate_proj", + "model.layers.8.mlp.experts.169.up_proj", + "model.layers.8.mlp.experts.17.down_proj", + "model.layers.8.mlp.experts.17.gate_proj", + "model.layers.8.mlp.experts.17.up_proj", + "model.layers.8.mlp.experts.170.down_proj", + "model.layers.8.mlp.experts.170.gate_proj", + "model.layers.8.mlp.experts.170.up_proj", + "model.layers.8.mlp.experts.171.down_proj", + "model.layers.8.mlp.experts.171.gate_proj", + "model.layers.8.mlp.experts.171.up_proj", + "model.layers.8.mlp.experts.172.down_proj", + "model.layers.8.mlp.experts.172.gate_proj", + "model.layers.8.mlp.experts.172.up_proj", + "model.layers.8.mlp.experts.173.down_proj", + "model.layers.8.mlp.experts.173.gate_proj", + "model.layers.8.mlp.experts.173.up_proj", + "model.layers.8.mlp.experts.174.down_proj", + "model.layers.8.mlp.experts.174.gate_proj", + "model.layers.8.mlp.experts.174.up_proj", + "model.layers.8.mlp.experts.175.down_proj", + "model.layers.8.mlp.experts.175.gate_proj", + "model.layers.8.mlp.experts.175.up_proj", + "model.layers.8.mlp.experts.176.down_proj", + "model.layers.8.mlp.experts.176.gate_proj", + "model.layers.8.mlp.experts.176.up_proj", + "model.layers.8.mlp.experts.177.down_proj", + "model.layers.8.mlp.experts.177.gate_proj", + "model.layers.8.mlp.experts.177.up_proj", + "model.layers.8.mlp.experts.178.down_proj", + "model.layers.8.mlp.experts.178.gate_proj", + "model.layers.8.mlp.experts.178.up_proj", + "model.layers.8.mlp.experts.179.down_proj", + "model.layers.8.mlp.experts.179.gate_proj", + "model.layers.8.mlp.experts.179.up_proj", + "model.layers.8.mlp.experts.18.down_proj", + "model.layers.8.mlp.experts.18.gate_proj", + "model.layers.8.mlp.experts.18.up_proj", + "model.layers.8.mlp.experts.180.down_proj", + "model.layers.8.mlp.experts.180.gate_proj", + "model.layers.8.mlp.experts.180.up_proj", + "model.layers.8.mlp.experts.181.down_proj", + "model.layers.8.mlp.experts.181.gate_proj", + "model.layers.8.mlp.experts.181.up_proj", + "model.layers.8.mlp.experts.182.down_proj", + "model.layers.8.mlp.experts.182.gate_proj", + "model.layers.8.mlp.experts.182.up_proj", + "model.layers.8.mlp.experts.183.down_proj", + "model.layers.8.mlp.experts.183.gate_proj", + "model.layers.8.mlp.experts.183.up_proj", + "model.layers.8.mlp.experts.184.down_proj", + "model.layers.8.mlp.experts.184.gate_proj", + "model.layers.8.mlp.experts.184.up_proj", + "model.layers.8.mlp.experts.185.down_proj", + "model.layers.8.mlp.experts.185.gate_proj", + "model.layers.8.mlp.experts.185.up_proj", + "model.layers.8.mlp.experts.186.down_proj", + "model.layers.8.mlp.experts.186.gate_proj", + "model.layers.8.mlp.experts.186.up_proj", + "model.layers.8.mlp.experts.187.down_proj", + "model.layers.8.mlp.experts.187.gate_proj", + "model.layers.8.mlp.experts.187.up_proj", + "model.layers.8.mlp.experts.188.down_proj", + "model.layers.8.mlp.experts.188.gate_proj", + "model.layers.8.mlp.experts.188.up_proj", + "model.layers.8.mlp.experts.189.down_proj", + "model.layers.8.mlp.experts.189.gate_proj", + "model.layers.8.mlp.experts.189.up_proj", + "model.layers.8.mlp.experts.19.down_proj", + "model.layers.8.mlp.experts.19.gate_proj", + "model.layers.8.mlp.experts.19.up_proj", + "model.layers.8.mlp.experts.190.down_proj", + "model.layers.8.mlp.experts.190.gate_proj", + "model.layers.8.mlp.experts.190.up_proj", + "model.layers.8.mlp.experts.191.down_proj", + "model.layers.8.mlp.experts.191.gate_proj", + "model.layers.8.mlp.experts.191.up_proj", + "model.layers.8.mlp.experts.192.down_proj", + "model.layers.8.mlp.experts.192.gate_proj", + "model.layers.8.mlp.experts.192.up_proj", + "model.layers.8.mlp.experts.193.down_proj", + "model.layers.8.mlp.experts.193.gate_proj", + "model.layers.8.mlp.experts.193.up_proj", + "model.layers.8.mlp.experts.194.down_proj", + "model.layers.8.mlp.experts.194.gate_proj", + "model.layers.8.mlp.experts.194.up_proj", + "model.layers.8.mlp.experts.195.down_proj", + "model.layers.8.mlp.experts.195.gate_proj", + "model.layers.8.mlp.experts.195.up_proj", + "model.layers.8.mlp.experts.196.down_proj", + "model.layers.8.mlp.experts.196.gate_proj", + "model.layers.8.mlp.experts.196.up_proj", + "model.layers.8.mlp.experts.197.down_proj", + "model.layers.8.mlp.experts.197.gate_proj", + "model.layers.8.mlp.experts.197.up_proj", + "model.layers.8.mlp.experts.198.down_proj", + "model.layers.8.mlp.experts.198.gate_proj", + "model.layers.8.mlp.experts.198.up_proj", + "model.layers.8.mlp.experts.199.down_proj", + "model.layers.8.mlp.experts.199.gate_proj", + "model.layers.8.mlp.experts.199.up_proj", + "model.layers.8.mlp.experts.2.down_proj", + "model.layers.8.mlp.experts.2.gate_proj", + "model.layers.8.mlp.experts.2.up_proj", + "model.layers.8.mlp.experts.20.down_proj", + "model.layers.8.mlp.experts.20.gate_proj", + "model.layers.8.mlp.experts.20.up_proj", + "model.layers.8.mlp.experts.200.down_proj", + "model.layers.8.mlp.experts.200.gate_proj", + "model.layers.8.mlp.experts.200.up_proj", + "model.layers.8.mlp.experts.201.down_proj", + "model.layers.8.mlp.experts.201.gate_proj", + "model.layers.8.mlp.experts.201.up_proj", + "model.layers.8.mlp.experts.202.down_proj", + "model.layers.8.mlp.experts.202.gate_proj", + "model.layers.8.mlp.experts.202.up_proj", + "model.layers.8.mlp.experts.203.down_proj", + "model.layers.8.mlp.experts.203.gate_proj", + "model.layers.8.mlp.experts.203.up_proj", + "model.layers.8.mlp.experts.204.down_proj", + "model.layers.8.mlp.experts.204.gate_proj", + "model.layers.8.mlp.experts.204.up_proj", + "model.layers.8.mlp.experts.205.down_proj", + "model.layers.8.mlp.experts.205.gate_proj", + "model.layers.8.mlp.experts.205.up_proj", + "model.layers.8.mlp.experts.206.down_proj", + "model.layers.8.mlp.experts.206.gate_proj", + "model.layers.8.mlp.experts.206.up_proj", + "model.layers.8.mlp.experts.207.down_proj", + "model.layers.8.mlp.experts.207.gate_proj", + "model.layers.8.mlp.experts.207.up_proj", + "model.layers.8.mlp.experts.208.down_proj", + "model.layers.8.mlp.experts.208.gate_proj", + "model.layers.8.mlp.experts.208.up_proj", + "model.layers.8.mlp.experts.209.down_proj", + "model.layers.8.mlp.experts.209.gate_proj", + "model.layers.8.mlp.experts.209.up_proj", + "model.layers.8.mlp.experts.21.down_proj", + "model.layers.8.mlp.experts.21.gate_proj", + "model.layers.8.mlp.experts.21.up_proj", + "model.layers.8.mlp.experts.210.down_proj", + "model.layers.8.mlp.experts.210.gate_proj", + "model.layers.8.mlp.experts.210.up_proj", + "model.layers.8.mlp.experts.211.down_proj", + "model.layers.8.mlp.experts.211.gate_proj", + "model.layers.8.mlp.experts.211.up_proj", + "model.layers.8.mlp.experts.212.down_proj", + "model.layers.8.mlp.experts.212.gate_proj", + "model.layers.8.mlp.experts.212.up_proj", + "model.layers.8.mlp.experts.213.down_proj", + "model.layers.8.mlp.experts.213.gate_proj", + "model.layers.8.mlp.experts.213.up_proj", + "model.layers.8.mlp.experts.214.down_proj", + "model.layers.8.mlp.experts.214.gate_proj", + "model.layers.8.mlp.experts.214.up_proj", + "model.layers.8.mlp.experts.215.down_proj", + "model.layers.8.mlp.experts.215.gate_proj", + "model.layers.8.mlp.experts.215.up_proj", + "model.layers.8.mlp.experts.216.down_proj", + "model.layers.8.mlp.experts.216.gate_proj", + "model.layers.8.mlp.experts.216.up_proj", + "model.layers.8.mlp.experts.217.down_proj", + "model.layers.8.mlp.experts.217.gate_proj", + "model.layers.8.mlp.experts.217.up_proj", + "model.layers.8.mlp.experts.218.down_proj", + "model.layers.8.mlp.experts.218.gate_proj", + "model.layers.8.mlp.experts.218.up_proj", + "model.layers.8.mlp.experts.219.down_proj", + "model.layers.8.mlp.experts.219.gate_proj", + "model.layers.8.mlp.experts.219.up_proj", + "model.layers.8.mlp.experts.22.down_proj", + "model.layers.8.mlp.experts.22.gate_proj", + "model.layers.8.mlp.experts.22.up_proj", + "model.layers.8.mlp.experts.220.down_proj", + "model.layers.8.mlp.experts.220.gate_proj", + "model.layers.8.mlp.experts.220.up_proj", + "model.layers.8.mlp.experts.221.down_proj", + "model.layers.8.mlp.experts.221.gate_proj", + "model.layers.8.mlp.experts.221.up_proj", + "model.layers.8.mlp.experts.222.down_proj", + "model.layers.8.mlp.experts.222.gate_proj", + "model.layers.8.mlp.experts.222.up_proj", + "model.layers.8.mlp.experts.223.down_proj", + "model.layers.8.mlp.experts.223.gate_proj", + "model.layers.8.mlp.experts.223.up_proj", + "model.layers.8.mlp.experts.224.down_proj", + "model.layers.8.mlp.experts.224.gate_proj", + "model.layers.8.mlp.experts.224.up_proj", + "model.layers.8.mlp.experts.225.down_proj", + "model.layers.8.mlp.experts.225.gate_proj", + "model.layers.8.mlp.experts.225.up_proj", + "model.layers.8.mlp.experts.226.down_proj", + "model.layers.8.mlp.experts.226.gate_proj", + "model.layers.8.mlp.experts.226.up_proj", + "model.layers.8.mlp.experts.227.down_proj", + "model.layers.8.mlp.experts.227.gate_proj", + "model.layers.8.mlp.experts.227.up_proj", + "model.layers.8.mlp.experts.228.down_proj", + "model.layers.8.mlp.experts.228.gate_proj", + "model.layers.8.mlp.experts.228.up_proj", + "model.layers.8.mlp.experts.229.down_proj", + "model.layers.8.mlp.experts.229.gate_proj", + "model.layers.8.mlp.experts.229.up_proj", + "model.layers.8.mlp.experts.23.down_proj", + "model.layers.8.mlp.experts.23.gate_proj", + "model.layers.8.mlp.experts.23.up_proj", + "model.layers.8.mlp.experts.230.down_proj", + "model.layers.8.mlp.experts.230.gate_proj", + "model.layers.8.mlp.experts.230.up_proj", + "model.layers.8.mlp.experts.231.down_proj", + "model.layers.8.mlp.experts.231.gate_proj", + "model.layers.8.mlp.experts.231.up_proj", + "model.layers.8.mlp.experts.232.down_proj", + "model.layers.8.mlp.experts.232.gate_proj", + "model.layers.8.mlp.experts.232.up_proj", + "model.layers.8.mlp.experts.233.down_proj", + "model.layers.8.mlp.experts.233.gate_proj", + "model.layers.8.mlp.experts.233.up_proj", + "model.layers.8.mlp.experts.234.down_proj", + "model.layers.8.mlp.experts.234.gate_proj", + "model.layers.8.mlp.experts.234.up_proj", + "model.layers.8.mlp.experts.235.down_proj", + "model.layers.8.mlp.experts.235.gate_proj", + "model.layers.8.mlp.experts.235.up_proj", + "model.layers.8.mlp.experts.236.down_proj", + "model.layers.8.mlp.experts.236.gate_proj", + "model.layers.8.mlp.experts.236.up_proj", + "model.layers.8.mlp.experts.237.down_proj", + "model.layers.8.mlp.experts.237.gate_proj", + "model.layers.8.mlp.experts.237.up_proj", + "model.layers.8.mlp.experts.238.down_proj", + "model.layers.8.mlp.experts.238.gate_proj", + "model.layers.8.mlp.experts.238.up_proj", + "model.layers.8.mlp.experts.239.down_proj", + "model.layers.8.mlp.experts.239.gate_proj", + "model.layers.8.mlp.experts.239.up_proj", + "model.layers.8.mlp.experts.24.down_proj", + "model.layers.8.mlp.experts.24.gate_proj", + "model.layers.8.mlp.experts.24.up_proj", + "model.layers.8.mlp.experts.240.down_proj", + "model.layers.8.mlp.experts.240.gate_proj", + "model.layers.8.mlp.experts.240.up_proj", + "model.layers.8.mlp.experts.241.down_proj", + "model.layers.8.mlp.experts.241.gate_proj", + "model.layers.8.mlp.experts.241.up_proj", + "model.layers.8.mlp.experts.242.down_proj", + "model.layers.8.mlp.experts.242.gate_proj", + "model.layers.8.mlp.experts.242.up_proj", + "model.layers.8.mlp.experts.243.down_proj", + "model.layers.8.mlp.experts.243.gate_proj", + "model.layers.8.mlp.experts.243.up_proj", + "model.layers.8.mlp.experts.244.down_proj", + "model.layers.8.mlp.experts.244.gate_proj", + "model.layers.8.mlp.experts.244.up_proj", + "model.layers.8.mlp.experts.245.down_proj", + "model.layers.8.mlp.experts.245.gate_proj", + "model.layers.8.mlp.experts.245.up_proj", + "model.layers.8.mlp.experts.246.down_proj", + "model.layers.8.mlp.experts.246.gate_proj", + "model.layers.8.mlp.experts.246.up_proj", + "model.layers.8.mlp.experts.247.down_proj", + "model.layers.8.mlp.experts.247.gate_proj", + "model.layers.8.mlp.experts.247.up_proj", + "model.layers.8.mlp.experts.248.down_proj", + "model.layers.8.mlp.experts.248.gate_proj", + "model.layers.8.mlp.experts.248.up_proj", + "model.layers.8.mlp.experts.249.down_proj", + "model.layers.8.mlp.experts.249.gate_proj", + "model.layers.8.mlp.experts.249.up_proj", + "model.layers.8.mlp.experts.25.down_proj", + "model.layers.8.mlp.experts.25.gate_proj", + "model.layers.8.mlp.experts.25.up_proj", + "model.layers.8.mlp.experts.250.down_proj", + "model.layers.8.mlp.experts.250.gate_proj", + "model.layers.8.mlp.experts.250.up_proj", + "model.layers.8.mlp.experts.251.down_proj", + "model.layers.8.mlp.experts.251.gate_proj", + "model.layers.8.mlp.experts.251.up_proj", + "model.layers.8.mlp.experts.252.down_proj", + "model.layers.8.mlp.experts.252.gate_proj", + "model.layers.8.mlp.experts.252.up_proj", + "model.layers.8.mlp.experts.253.down_proj", + "model.layers.8.mlp.experts.253.gate_proj", + "model.layers.8.mlp.experts.253.up_proj", + "model.layers.8.mlp.experts.254.down_proj", + "model.layers.8.mlp.experts.254.gate_proj", + "model.layers.8.mlp.experts.254.up_proj", + "model.layers.8.mlp.experts.255.down_proj", + "model.layers.8.mlp.experts.255.gate_proj", + "model.layers.8.mlp.experts.255.up_proj", + "model.layers.8.mlp.experts.26.down_proj", + "model.layers.8.mlp.experts.26.gate_proj", + "model.layers.8.mlp.experts.26.up_proj", + "model.layers.8.mlp.experts.27.down_proj", + "model.layers.8.mlp.experts.27.gate_proj", + "model.layers.8.mlp.experts.27.up_proj", + "model.layers.8.mlp.experts.28.down_proj", + "model.layers.8.mlp.experts.28.gate_proj", + "model.layers.8.mlp.experts.28.up_proj", + "model.layers.8.mlp.experts.29.down_proj", + "model.layers.8.mlp.experts.29.gate_proj", + "model.layers.8.mlp.experts.29.up_proj", + "model.layers.8.mlp.experts.3.down_proj", + "model.layers.8.mlp.experts.3.gate_proj", + "model.layers.8.mlp.experts.3.up_proj", + "model.layers.8.mlp.experts.30.down_proj", + "model.layers.8.mlp.experts.30.gate_proj", + "model.layers.8.mlp.experts.30.up_proj", + "model.layers.8.mlp.experts.31.down_proj", + "model.layers.8.mlp.experts.31.gate_proj", + "model.layers.8.mlp.experts.31.up_proj", + "model.layers.8.mlp.experts.32.down_proj", + "model.layers.8.mlp.experts.32.gate_proj", + "model.layers.8.mlp.experts.32.up_proj", + "model.layers.8.mlp.experts.33.down_proj", + "model.layers.8.mlp.experts.33.gate_proj", + "model.layers.8.mlp.experts.33.up_proj", + "model.layers.8.mlp.experts.34.down_proj", + "model.layers.8.mlp.experts.34.gate_proj", + "model.layers.8.mlp.experts.34.up_proj", + "model.layers.8.mlp.experts.35.down_proj", + "model.layers.8.mlp.experts.35.gate_proj", + "model.layers.8.mlp.experts.35.up_proj", + "model.layers.8.mlp.experts.36.down_proj", + "model.layers.8.mlp.experts.36.gate_proj", + "model.layers.8.mlp.experts.36.up_proj", + "model.layers.8.mlp.experts.37.down_proj", + "model.layers.8.mlp.experts.37.gate_proj", + "model.layers.8.mlp.experts.37.up_proj", + "model.layers.8.mlp.experts.38.down_proj", + "model.layers.8.mlp.experts.38.gate_proj", + "model.layers.8.mlp.experts.38.up_proj", + "model.layers.8.mlp.experts.39.down_proj", + "model.layers.8.mlp.experts.39.gate_proj", + "model.layers.8.mlp.experts.39.up_proj", + "model.layers.8.mlp.experts.4.down_proj", + "model.layers.8.mlp.experts.4.gate_proj", + "model.layers.8.mlp.experts.4.up_proj", + "model.layers.8.mlp.experts.40.down_proj", + "model.layers.8.mlp.experts.40.gate_proj", + "model.layers.8.mlp.experts.40.up_proj", + "model.layers.8.mlp.experts.41.down_proj", + "model.layers.8.mlp.experts.41.gate_proj", + "model.layers.8.mlp.experts.41.up_proj", + "model.layers.8.mlp.experts.42.down_proj", + "model.layers.8.mlp.experts.42.gate_proj", + "model.layers.8.mlp.experts.42.up_proj", + "model.layers.8.mlp.experts.43.down_proj", + "model.layers.8.mlp.experts.43.gate_proj", + "model.layers.8.mlp.experts.43.up_proj", + "model.layers.8.mlp.experts.44.down_proj", + "model.layers.8.mlp.experts.44.gate_proj", + "model.layers.8.mlp.experts.44.up_proj", + "model.layers.8.mlp.experts.45.down_proj", + "model.layers.8.mlp.experts.45.gate_proj", + "model.layers.8.mlp.experts.45.up_proj", + "model.layers.8.mlp.experts.46.down_proj", + "model.layers.8.mlp.experts.46.gate_proj", + "model.layers.8.mlp.experts.46.up_proj", + "model.layers.8.mlp.experts.47.down_proj", + "model.layers.8.mlp.experts.47.gate_proj", + "model.layers.8.mlp.experts.47.up_proj", + "model.layers.8.mlp.experts.48.down_proj", + "model.layers.8.mlp.experts.48.gate_proj", + "model.layers.8.mlp.experts.48.up_proj", + "model.layers.8.mlp.experts.49.down_proj", + "model.layers.8.mlp.experts.49.gate_proj", + "model.layers.8.mlp.experts.49.up_proj", + "model.layers.8.mlp.experts.5.down_proj", + "model.layers.8.mlp.experts.5.gate_proj", + "model.layers.8.mlp.experts.5.up_proj", + "model.layers.8.mlp.experts.50.down_proj", + "model.layers.8.mlp.experts.50.gate_proj", + "model.layers.8.mlp.experts.50.up_proj", + "model.layers.8.mlp.experts.51.down_proj", + "model.layers.8.mlp.experts.51.gate_proj", + "model.layers.8.mlp.experts.51.up_proj", + "model.layers.8.mlp.experts.52.down_proj", + "model.layers.8.mlp.experts.52.gate_proj", + "model.layers.8.mlp.experts.52.up_proj", + "model.layers.8.mlp.experts.53.down_proj", + "model.layers.8.mlp.experts.53.gate_proj", + "model.layers.8.mlp.experts.53.up_proj", + "model.layers.8.mlp.experts.54.down_proj", + "model.layers.8.mlp.experts.54.gate_proj", + "model.layers.8.mlp.experts.54.up_proj", + "model.layers.8.mlp.experts.55.down_proj", + "model.layers.8.mlp.experts.55.gate_proj", + "model.layers.8.mlp.experts.55.up_proj", + "model.layers.8.mlp.experts.56.down_proj", + "model.layers.8.mlp.experts.56.gate_proj", + "model.layers.8.mlp.experts.56.up_proj", + "model.layers.8.mlp.experts.57.down_proj", + "model.layers.8.mlp.experts.57.gate_proj", + "model.layers.8.mlp.experts.57.up_proj", + "model.layers.8.mlp.experts.58.down_proj", + "model.layers.8.mlp.experts.58.gate_proj", + "model.layers.8.mlp.experts.58.up_proj", + "model.layers.8.mlp.experts.59.down_proj", + "model.layers.8.mlp.experts.59.gate_proj", + "model.layers.8.mlp.experts.59.up_proj", + "model.layers.8.mlp.experts.6.down_proj", + "model.layers.8.mlp.experts.6.gate_proj", + "model.layers.8.mlp.experts.6.up_proj", + "model.layers.8.mlp.experts.60.down_proj", + "model.layers.8.mlp.experts.60.gate_proj", + "model.layers.8.mlp.experts.60.up_proj", + "model.layers.8.mlp.experts.61.down_proj", + "model.layers.8.mlp.experts.61.gate_proj", + "model.layers.8.mlp.experts.61.up_proj", + "model.layers.8.mlp.experts.62.down_proj", + "model.layers.8.mlp.experts.62.gate_proj", + "model.layers.8.mlp.experts.62.up_proj", + "model.layers.8.mlp.experts.63.down_proj", + "model.layers.8.mlp.experts.63.gate_proj", + "model.layers.8.mlp.experts.63.up_proj", + "model.layers.8.mlp.experts.64.down_proj", + "model.layers.8.mlp.experts.64.gate_proj", + "model.layers.8.mlp.experts.64.up_proj", + "model.layers.8.mlp.experts.65.down_proj", + "model.layers.8.mlp.experts.65.gate_proj", + "model.layers.8.mlp.experts.65.up_proj", + "model.layers.8.mlp.experts.66.down_proj", + "model.layers.8.mlp.experts.66.gate_proj", + "model.layers.8.mlp.experts.66.up_proj", + "model.layers.8.mlp.experts.67.down_proj", + "model.layers.8.mlp.experts.67.gate_proj", + "model.layers.8.mlp.experts.67.up_proj", + "model.layers.8.mlp.experts.68.down_proj", + "model.layers.8.mlp.experts.68.gate_proj", + "model.layers.8.mlp.experts.68.up_proj", + "model.layers.8.mlp.experts.69.down_proj", + "model.layers.8.mlp.experts.69.gate_proj", + "model.layers.8.mlp.experts.69.up_proj", + "model.layers.8.mlp.experts.7.down_proj", + "model.layers.8.mlp.experts.7.gate_proj", + "model.layers.8.mlp.experts.7.up_proj", + "model.layers.8.mlp.experts.70.down_proj", + "model.layers.8.mlp.experts.70.gate_proj", + "model.layers.8.mlp.experts.70.up_proj", + "model.layers.8.mlp.experts.71.down_proj", + "model.layers.8.mlp.experts.71.gate_proj", + "model.layers.8.mlp.experts.71.up_proj", + "model.layers.8.mlp.experts.72.down_proj", + "model.layers.8.mlp.experts.72.gate_proj", + "model.layers.8.mlp.experts.72.up_proj", + "model.layers.8.mlp.experts.73.down_proj", + "model.layers.8.mlp.experts.73.gate_proj", + "model.layers.8.mlp.experts.73.up_proj", + "model.layers.8.mlp.experts.74.down_proj", + "model.layers.8.mlp.experts.74.gate_proj", + "model.layers.8.mlp.experts.74.up_proj", + "model.layers.8.mlp.experts.75.down_proj", + "model.layers.8.mlp.experts.75.gate_proj", + "model.layers.8.mlp.experts.75.up_proj", + "model.layers.8.mlp.experts.76.down_proj", + "model.layers.8.mlp.experts.76.gate_proj", + "model.layers.8.mlp.experts.76.up_proj", + "model.layers.8.mlp.experts.77.down_proj", + "model.layers.8.mlp.experts.77.gate_proj", + "model.layers.8.mlp.experts.77.up_proj", + "model.layers.8.mlp.experts.78.down_proj", + "model.layers.8.mlp.experts.78.gate_proj", + "model.layers.8.mlp.experts.78.up_proj", + "model.layers.8.mlp.experts.79.down_proj", + "model.layers.8.mlp.experts.79.gate_proj", + "model.layers.8.mlp.experts.79.up_proj", + "model.layers.8.mlp.experts.8.down_proj", + "model.layers.8.mlp.experts.8.gate_proj", + "model.layers.8.mlp.experts.8.up_proj", + "model.layers.8.mlp.experts.80.down_proj", + "model.layers.8.mlp.experts.80.gate_proj", + "model.layers.8.mlp.experts.80.up_proj", + "model.layers.8.mlp.experts.81.down_proj", + "model.layers.8.mlp.experts.81.gate_proj", + "model.layers.8.mlp.experts.81.up_proj", + "model.layers.8.mlp.experts.82.down_proj", + "model.layers.8.mlp.experts.82.gate_proj", + "model.layers.8.mlp.experts.82.up_proj", + "model.layers.8.mlp.experts.83.down_proj", + "model.layers.8.mlp.experts.83.gate_proj", + "model.layers.8.mlp.experts.83.up_proj", + "model.layers.8.mlp.experts.84.down_proj", + "model.layers.8.mlp.experts.84.gate_proj", + "model.layers.8.mlp.experts.84.up_proj", + "model.layers.8.mlp.experts.85.down_proj", + "model.layers.8.mlp.experts.85.gate_proj", + "model.layers.8.mlp.experts.85.up_proj", + "model.layers.8.mlp.experts.86.down_proj", + "model.layers.8.mlp.experts.86.gate_proj", + "model.layers.8.mlp.experts.86.up_proj", + "model.layers.8.mlp.experts.87.down_proj", + "model.layers.8.mlp.experts.87.gate_proj", + "model.layers.8.mlp.experts.87.up_proj", + "model.layers.8.mlp.experts.88.down_proj", + "model.layers.8.mlp.experts.88.gate_proj", + "model.layers.8.mlp.experts.88.up_proj", + "model.layers.8.mlp.experts.89.down_proj", + "model.layers.8.mlp.experts.89.gate_proj", + "model.layers.8.mlp.experts.89.up_proj", + "model.layers.8.mlp.experts.9.down_proj", + "model.layers.8.mlp.experts.9.gate_proj", + "model.layers.8.mlp.experts.9.up_proj", + "model.layers.8.mlp.experts.90.down_proj", + "model.layers.8.mlp.experts.90.gate_proj", + "model.layers.8.mlp.experts.90.up_proj", + "model.layers.8.mlp.experts.91.down_proj", + "model.layers.8.mlp.experts.91.gate_proj", + "model.layers.8.mlp.experts.91.up_proj", + "model.layers.8.mlp.experts.92.down_proj", + "model.layers.8.mlp.experts.92.gate_proj", + "model.layers.8.mlp.experts.92.up_proj", + "model.layers.8.mlp.experts.93.down_proj", + "model.layers.8.mlp.experts.93.gate_proj", + "model.layers.8.mlp.experts.93.up_proj", + "model.layers.8.mlp.experts.94.down_proj", + "model.layers.8.mlp.experts.94.gate_proj", + "model.layers.8.mlp.experts.94.up_proj", + "model.layers.8.mlp.experts.95.down_proj", + "model.layers.8.mlp.experts.95.gate_proj", + "model.layers.8.mlp.experts.95.up_proj", + "model.layers.8.mlp.experts.96.down_proj", + "model.layers.8.mlp.experts.96.gate_proj", + "model.layers.8.mlp.experts.96.up_proj", + "model.layers.8.mlp.experts.97.down_proj", + "model.layers.8.mlp.experts.97.gate_proj", + "model.layers.8.mlp.experts.97.up_proj", + "model.layers.8.mlp.experts.98.down_proj", + "model.layers.8.mlp.experts.98.gate_proj", + "model.layers.8.mlp.experts.98.up_proj", + "model.layers.8.mlp.experts.99.down_proj", + "model.layers.8.mlp.experts.99.gate_proj", + "model.layers.8.mlp.experts.99.up_proj", + "model.layers.9.mlp.experts.0.down_proj", + "model.layers.9.mlp.experts.0.gate_proj", + "model.layers.9.mlp.experts.0.up_proj", + "model.layers.9.mlp.experts.1.down_proj", + "model.layers.9.mlp.experts.1.gate_proj", + "model.layers.9.mlp.experts.1.up_proj", + "model.layers.9.mlp.experts.10.down_proj", + "model.layers.9.mlp.experts.10.gate_proj", + "model.layers.9.mlp.experts.10.up_proj", + "model.layers.9.mlp.experts.100.down_proj", + "model.layers.9.mlp.experts.100.gate_proj", + "model.layers.9.mlp.experts.100.up_proj", + "model.layers.9.mlp.experts.101.down_proj", + "model.layers.9.mlp.experts.101.gate_proj", + "model.layers.9.mlp.experts.101.up_proj", + "model.layers.9.mlp.experts.102.down_proj", + "model.layers.9.mlp.experts.102.gate_proj", + "model.layers.9.mlp.experts.102.up_proj", + "model.layers.9.mlp.experts.103.down_proj", + "model.layers.9.mlp.experts.103.gate_proj", + "model.layers.9.mlp.experts.103.up_proj", + "model.layers.9.mlp.experts.104.down_proj", + "model.layers.9.mlp.experts.104.gate_proj", + "model.layers.9.mlp.experts.104.up_proj", + "model.layers.9.mlp.experts.105.down_proj", + "model.layers.9.mlp.experts.105.gate_proj", + "model.layers.9.mlp.experts.105.up_proj", + "model.layers.9.mlp.experts.106.down_proj", + "model.layers.9.mlp.experts.106.gate_proj", + "model.layers.9.mlp.experts.106.up_proj", + "model.layers.9.mlp.experts.107.down_proj", + "model.layers.9.mlp.experts.107.gate_proj", + "model.layers.9.mlp.experts.107.up_proj", + "model.layers.9.mlp.experts.108.down_proj", + "model.layers.9.mlp.experts.108.gate_proj", + "model.layers.9.mlp.experts.108.up_proj", + "model.layers.9.mlp.experts.109.down_proj", + "model.layers.9.mlp.experts.109.gate_proj", + "model.layers.9.mlp.experts.109.up_proj", + "model.layers.9.mlp.experts.11.down_proj", + "model.layers.9.mlp.experts.11.gate_proj", + "model.layers.9.mlp.experts.11.up_proj", + "model.layers.9.mlp.experts.110.down_proj", + "model.layers.9.mlp.experts.110.gate_proj", + "model.layers.9.mlp.experts.110.up_proj", + "model.layers.9.mlp.experts.111.down_proj", + "model.layers.9.mlp.experts.111.gate_proj", + "model.layers.9.mlp.experts.111.up_proj", + "model.layers.9.mlp.experts.112.down_proj", + "model.layers.9.mlp.experts.112.gate_proj", + "model.layers.9.mlp.experts.112.up_proj", + "model.layers.9.mlp.experts.113.down_proj", + "model.layers.9.mlp.experts.113.gate_proj", + "model.layers.9.mlp.experts.113.up_proj", + "model.layers.9.mlp.experts.114.down_proj", + "model.layers.9.mlp.experts.114.gate_proj", + "model.layers.9.mlp.experts.114.up_proj", + "model.layers.9.mlp.experts.115.down_proj", + "model.layers.9.mlp.experts.115.gate_proj", + "model.layers.9.mlp.experts.115.up_proj", + "model.layers.9.mlp.experts.116.down_proj", + "model.layers.9.mlp.experts.116.gate_proj", + "model.layers.9.mlp.experts.116.up_proj", + "model.layers.9.mlp.experts.117.down_proj", + "model.layers.9.mlp.experts.117.gate_proj", + "model.layers.9.mlp.experts.117.up_proj", + "model.layers.9.mlp.experts.118.down_proj", + "model.layers.9.mlp.experts.118.gate_proj", + "model.layers.9.mlp.experts.118.up_proj", + "model.layers.9.mlp.experts.119.down_proj", + "model.layers.9.mlp.experts.119.gate_proj", + "model.layers.9.mlp.experts.119.up_proj", + "model.layers.9.mlp.experts.12.down_proj", + "model.layers.9.mlp.experts.12.gate_proj", + "model.layers.9.mlp.experts.12.up_proj", + "model.layers.9.mlp.experts.120.down_proj", + "model.layers.9.mlp.experts.120.gate_proj", + "model.layers.9.mlp.experts.120.up_proj", + "model.layers.9.mlp.experts.121.down_proj", + "model.layers.9.mlp.experts.121.gate_proj", + "model.layers.9.mlp.experts.121.up_proj", + "model.layers.9.mlp.experts.122.down_proj", + "model.layers.9.mlp.experts.122.gate_proj", + "model.layers.9.mlp.experts.122.up_proj", + "model.layers.9.mlp.experts.123.down_proj", + "model.layers.9.mlp.experts.123.gate_proj", + "model.layers.9.mlp.experts.123.up_proj", + "model.layers.9.mlp.experts.124.down_proj", + "model.layers.9.mlp.experts.124.gate_proj", + "model.layers.9.mlp.experts.124.up_proj", + "model.layers.9.mlp.experts.125.down_proj", + "model.layers.9.mlp.experts.125.gate_proj", + "model.layers.9.mlp.experts.125.up_proj", + "model.layers.9.mlp.experts.126.down_proj", + "model.layers.9.mlp.experts.126.gate_proj", + "model.layers.9.mlp.experts.126.up_proj", + "model.layers.9.mlp.experts.127.down_proj", + "model.layers.9.mlp.experts.127.gate_proj", + "model.layers.9.mlp.experts.127.up_proj", + "model.layers.9.mlp.experts.128.down_proj", + "model.layers.9.mlp.experts.128.gate_proj", + "model.layers.9.mlp.experts.128.up_proj", + "model.layers.9.mlp.experts.129.down_proj", + "model.layers.9.mlp.experts.129.gate_proj", + "model.layers.9.mlp.experts.129.up_proj", + "model.layers.9.mlp.experts.13.down_proj", + "model.layers.9.mlp.experts.13.gate_proj", + "model.layers.9.mlp.experts.13.up_proj", + "model.layers.9.mlp.experts.130.down_proj", + "model.layers.9.mlp.experts.130.gate_proj", + "model.layers.9.mlp.experts.130.up_proj", + "model.layers.9.mlp.experts.131.down_proj", + "model.layers.9.mlp.experts.131.gate_proj", + "model.layers.9.mlp.experts.131.up_proj", + "model.layers.9.mlp.experts.132.down_proj", + "model.layers.9.mlp.experts.132.gate_proj", + "model.layers.9.mlp.experts.132.up_proj", + "model.layers.9.mlp.experts.133.down_proj", + "model.layers.9.mlp.experts.133.gate_proj", + "model.layers.9.mlp.experts.133.up_proj", + "model.layers.9.mlp.experts.134.down_proj", + "model.layers.9.mlp.experts.134.gate_proj", + "model.layers.9.mlp.experts.134.up_proj", + "model.layers.9.mlp.experts.135.down_proj", + "model.layers.9.mlp.experts.135.gate_proj", + "model.layers.9.mlp.experts.135.up_proj", + "model.layers.9.mlp.experts.136.down_proj", + "model.layers.9.mlp.experts.136.gate_proj", + "model.layers.9.mlp.experts.136.up_proj", + "model.layers.9.mlp.experts.137.down_proj", + "model.layers.9.mlp.experts.137.gate_proj", + "model.layers.9.mlp.experts.137.up_proj", + "model.layers.9.mlp.experts.138.down_proj", + "model.layers.9.mlp.experts.138.gate_proj", + "model.layers.9.mlp.experts.138.up_proj", + "model.layers.9.mlp.experts.139.down_proj", + "model.layers.9.mlp.experts.139.gate_proj", + "model.layers.9.mlp.experts.139.up_proj", + "model.layers.9.mlp.experts.14.down_proj", + "model.layers.9.mlp.experts.14.gate_proj", + "model.layers.9.mlp.experts.14.up_proj", + "model.layers.9.mlp.experts.140.down_proj", + "model.layers.9.mlp.experts.140.gate_proj", + "model.layers.9.mlp.experts.140.up_proj", + "model.layers.9.mlp.experts.141.down_proj", + "model.layers.9.mlp.experts.141.gate_proj", + "model.layers.9.mlp.experts.141.up_proj", + "model.layers.9.mlp.experts.142.down_proj", + "model.layers.9.mlp.experts.142.gate_proj", + "model.layers.9.mlp.experts.142.up_proj", + "model.layers.9.mlp.experts.143.down_proj", + "model.layers.9.mlp.experts.143.gate_proj", + "model.layers.9.mlp.experts.143.up_proj", + "model.layers.9.mlp.experts.144.down_proj", + "model.layers.9.mlp.experts.144.gate_proj", + "model.layers.9.mlp.experts.144.up_proj", + "model.layers.9.mlp.experts.145.down_proj", + "model.layers.9.mlp.experts.145.gate_proj", + "model.layers.9.mlp.experts.145.up_proj", + "model.layers.9.mlp.experts.146.down_proj", + "model.layers.9.mlp.experts.146.gate_proj", + "model.layers.9.mlp.experts.146.up_proj", + "model.layers.9.mlp.experts.147.down_proj", + "model.layers.9.mlp.experts.147.gate_proj", + "model.layers.9.mlp.experts.147.up_proj", + "model.layers.9.mlp.experts.148.down_proj", + "model.layers.9.mlp.experts.148.gate_proj", + "model.layers.9.mlp.experts.148.up_proj", + "model.layers.9.mlp.experts.149.down_proj", + "model.layers.9.mlp.experts.149.gate_proj", + "model.layers.9.mlp.experts.149.up_proj", + "model.layers.9.mlp.experts.15.down_proj", + "model.layers.9.mlp.experts.15.gate_proj", + "model.layers.9.mlp.experts.15.up_proj", + "model.layers.9.mlp.experts.150.down_proj", + "model.layers.9.mlp.experts.150.gate_proj", + "model.layers.9.mlp.experts.150.up_proj", + "model.layers.9.mlp.experts.151.down_proj", + "model.layers.9.mlp.experts.151.gate_proj", + "model.layers.9.mlp.experts.151.up_proj", + "model.layers.9.mlp.experts.152.down_proj", + "model.layers.9.mlp.experts.152.gate_proj", + "model.layers.9.mlp.experts.152.up_proj", + "model.layers.9.mlp.experts.153.down_proj", + "model.layers.9.mlp.experts.153.gate_proj", + "model.layers.9.mlp.experts.153.up_proj", + "model.layers.9.mlp.experts.154.down_proj", + "model.layers.9.mlp.experts.154.gate_proj", + "model.layers.9.mlp.experts.154.up_proj", + "model.layers.9.mlp.experts.155.down_proj", + "model.layers.9.mlp.experts.155.gate_proj", + "model.layers.9.mlp.experts.155.up_proj", + "model.layers.9.mlp.experts.156.down_proj", + "model.layers.9.mlp.experts.156.gate_proj", + "model.layers.9.mlp.experts.156.up_proj", + "model.layers.9.mlp.experts.157.down_proj", + "model.layers.9.mlp.experts.157.gate_proj", + "model.layers.9.mlp.experts.157.up_proj", + "model.layers.9.mlp.experts.158.down_proj", + "model.layers.9.mlp.experts.158.gate_proj", + "model.layers.9.mlp.experts.158.up_proj", + "model.layers.9.mlp.experts.159.down_proj", + "model.layers.9.mlp.experts.159.gate_proj", + "model.layers.9.mlp.experts.159.up_proj", + "model.layers.9.mlp.experts.16.down_proj", + "model.layers.9.mlp.experts.16.gate_proj", + "model.layers.9.mlp.experts.16.up_proj", + "model.layers.9.mlp.experts.160.down_proj", + "model.layers.9.mlp.experts.160.gate_proj", + "model.layers.9.mlp.experts.160.up_proj", + "model.layers.9.mlp.experts.161.down_proj", + "model.layers.9.mlp.experts.161.gate_proj", + "model.layers.9.mlp.experts.161.up_proj", + "model.layers.9.mlp.experts.162.down_proj", + "model.layers.9.mlp.experts.162.gate_proj", + "model.layers.9.mlp.experts.162.up_proj", + "model.layers.9.mlp.experts.163.down_proj", + "model.layers.9.mlp.experts.163.gate_proj", + "model.layers.9.mlp.experts.163.up_proj", + "model.layers.9.mlp.experts.164.down_proj", + "model.layers.9.mlp.experts.164.gate_proj", + "model.layers.9.mlp.experts.164.up_proj", + "model.layers.9.mlp.experts.165.down_proj", + "model.layers.9.mlp.experts.165.gate_proj", + "model.layers.9.mlp.experts.165.up_proj", + "model.layers.9.mlp.experts.166.down_proj", + "model.layers.9.mlp.experts.166.gate_proj", + "model.layers.9.mlp.experts.166.up_proj", + "model.layers.9.mlp.experts.167.down_proj", + "model.layers.9.mlp.experts.167.gate_proj", + "model.layers.9.mlp.experts.167.up_proj", + "model.layers.9.mlp.experts.168.down_proj", + "model.layers.9.mlp.experts.168.gate_proj", + "model.layers.9.mlp.experts.168.up_proj", + "model.layers.9.mlp.experts.169.down_proj", + "model.layers.9.mlp.experts.169.gate_proj", + "model.layers.9.mlp.experts.169.up_proj", + "model.layers.9.mlp.experts.17.down_proj", + "model.layers.9.mlp.experts.17.gate_proj", + "model.layers.9.mlp.experts.17.up_proj", + "model.layers.9.mlp.experts.170.down_proj", + "model.layers.9.mlp.experts.170.gate_proj", + "model.layers.9.mlp.experts.170.up_proj", + "model.layers.9.mlp.experts.171.down_proj", + "model.layers.9.mlp.experts.171.gate_proj", + "model.layers.9.mlp.experts.171.up_proj", + "model.layers.9.mlp.experts.172.down_proj", + "model.layers.9.mlp.experts.172.gate_proj", + "model.layers.9.mlp.experts.172.up_proj", + "model.layers.9.mlp.experts.173.down_proj", + "model.layers.9.mlp.experts.173.gate_proj", + "model.layers.9.mlp.experts.173.up_proj", + "model.layers.9.mlp.experts.174.down_proj", + "model.layers.9.mlp.experts.174.gate_proj", + "model.layers.9.mlp.experts.174.up_proj", + "model.layers.9.mlp.experts.175.down_proj", + "model.layers.9.mlp.experts.175.gate_proj", + "model.layers.9.mlp.experts.175.up_proj", + "model.layers.9.mlp.experts.176.down_proj", + "model.layers.9.mlp.experts.176.gate_proj", + "model.layers.9.mlp.experts.176.up_proj", + "model.layers.9.mlp.experts.177.down_proj", + "model.layers.9.mlp.experts.177.gate_proj", + "model.layers.9.mlp.experts.177.up_proj", + "model.layers.9.mlp.experts.178.down_proj", + "model.layers.9.mlp.experts.178.gate_proj", + "model.layers.9.mlp.experts.178.up_proj", + "model.layers.9.mlp.experts.179.down_proj", + "model.layers.9.mlp.experts.179.gate_proj", + "model.layers.9.mlp.experts.179.up_proj", + "model.layers.9.mlp.experts.18.down_proj", + "model.layers.9.mlp.experts.18.gate_proj", + "model.layers.9.mlp.experts.18.up_proj", + "model.layers.9.mlp.experts.180.down_proj", + "model.layers.9.mlp.experts.180.gate_proj", + "model.layers.9.mlp.experts.180.up_proj", + "model.layers.9.mlp.experts.181.down_proj", + "model.layers.9.mlp.experts.181.gate_proj", + "model.layers.9.mlp.experts.181.up_proj", + "model.layers.9.mlp.experts.182.down_proj", + "model.layers.9.mlp.experts.182.gate_proj", + "model.layers.9.mlp.experts.182.up_proj", + "model.layers.9.mlp.experts.183.down_proj", + "model.layers.9.mlp.experts.183.gate_proj", + "model.layers.9.mlp.experts.183.up_proj", + "model.layers.9.mlp.experts.184.down_proj", + "model.layers.9.mlp.experts.184.gate_proj", + "model.layers.9.mlp.experts.184.up_proj", + "model.layers.9.mlp.experts.185.down_proj", + "model.layers.9.mlp.experts.185.gate_proj", + "model.layers.9.mlp.experts.185.up_proj", + "model.layers.9.mlp.experts.186.down_proj", + "model.layers.9.mlp.experts.186.gate_proj", + "model.layers.9.mlp.experts.186.up_proj", + "model.layers.9.mlp.experts.187.down_proj", + "model.layers.9.mlp.experts.187.gate_proj", + "model.layers.9.mlp.experts.187.up_proj", + "model.layers.9.mlp.experts.188.down_proj", + "model.layers.9.mlp.experts.188.gate_proj", + "model.layers.9.mlp.experts.188.up_proj", + "model.layers.9.mlp.experts.189.down_proj", + "model.layers.9.mlp.experts.189.gate_proj", + "model.layers.9.mlp.experts.189.up_proj", + "model.layers.9.mlp.experts.19.down_proj", + "model.layers.9.mlp.experts.19.gate_proj", + "model.layers.9.mlp.experts.19.up_proj", + "model.layers.9.mlp.experts.190.down_proj", + "model.layers.9.mlp.experts.190.gate_proj", + "model.layers.9.mlp.experts.190.up_proj", + "model.layers.9.mlp.experts.191.down_proj", + "model.layers.9.mlp.experts.191.gate_proj", + "model.layers.9.mlp.experts.191.up_proj", + "model.layers.9.mlp.experts.192.down_proj", + "model.layers.9.mlp.experts.192.gate_proj", + "model.layers.9.mlp.experts.192.up_proj", + "model.layers.9.mlp.experts.193.down_proj", + "model.layers.9.mlp.experts.193.gate_proj", + "model.layers.9.mlp.experts.193.up_proj", + "model.layers.9.mlp.experts.194.down_proj", + "model.layers.9.mlp.experts.194.gate_proj", + "model.layers.9.mlp.experts.194.up_proj", + "model.layers.9.mlp.experts.195.down_proj", + "model.layers.9.mlp.experts.195.gate_proj", + "model.layers.9.mlp.experts.195.up_proj", + "model.layers.9.mlp.experts.196.down_proj", + "model.layers.9.mlp.experts.196.gate_proj", + "model.layers.9.mlp.experts.196.up_proj", + "model.layers.9.mlp.experts.197.down_proj", + "model.layers.9.mlp.experts.197.gate_proj", + "model.layers.9.mlp.experts.197.up_proj", + "model.layers.9.mlp.experts.198.down_proj", + "model.layers.9.mlp.experts.198.gate_proj", + "model.layers.9.mlp.experts.198.up_proj", + "model.layers.9.mlp.experts.199.down_proj", + "model.layers.9.mlp.experts.199.gate_proj", + "model.layers.9.mlp.experts.199.up_proj", + "model.layers.9.mlp.experts.2.down_proj", + "model.layers.9.mlp.experts.2.gate_proj", + "model.layers.9.mlp.experts.2.up_proj", + "model.layers.9.mlp.experts.20.down_proj", + "model.layers.9.mlp.experts.20.gate_proj", + "model.layers.9.mlp.experts.20.up_proj", + "model.layers.9.mlp.experts.200.down_proj", + "model.layers.9.mlp.experts.200.gate_proj", + "model.layers.9.mlp.experts.200.up_proj", + "model.layers.9.mlp.experts.201.down_proj", + "model.layers.9.mlp.experts.201.gate_proj", + "model.layers.9.mlp.experts.201.up_proj", + "model.layers.9.mlp.experts.202.down_proj", + "model.layers.9.mlp.experts.202.gate_proj", + "model.layers.9.mlp.experts.202.up_proj", + "model.layers.9.mlp.experts.203.down_proj", + "model.layers.9.mlp.experts.203.gate_proj", + "model.layers.9.mlp.experts.203.up_proj", + "model.layers.9.mlp.experts.204.down_proj", + "model.layers.9.mlp.experts.204.gate_proj", + "model.layers.9.mlp.experts.204.up_proj", + "model.layers.9.mlp.experts.205.down_proj", + "model.layers.9.mlp.experts.205.gate_proj", + "model.layers.9.mlp.experts.205.up_proj", + "model.layers.9.mlp.experts.206.down_proj", + "model.layers.9.mlp.experts.206.gate_proj", + "model.layers.9.mlp.experts.206.up_proj", + "model.layers.9.mlp.experts.207.down_proj", + "model.layers.9.mlp.experts.207.gate_proj", + "model.layers.9.mlp.experts.207.up_proj", + "model.layers.9.mlp.experts.208.down_proj", + "model.layers.9.mlp.experts.208.gate_proj", + "model.layers.9.mlp.experts.208.up_proj", + "model.layers.9.mlp.experts.209.down_proj", + "model.layers.9.mlp.experts.209.gate_proj", + "model.layers.9.mlp.experts.209.up_proj", + "model.layers.9.mlp.experts.21.down_proj", + "model.layers.9.mlp.experts.21.gate_proj", + "model.layers.9.mlp.experts.21.up_proj", + "model.layers.9.mlp.experts.210.down_proj", + "model.layers.9.mlp.experts.210.gate_proj", + "model.layers.9.mlp.experts.210.up_proj", + "model.layers.9.mlp.experts.211.down_proj", + "model.layers.9.mlp.experts.211.gate_proj", + "model.layers.9.mlp.experts.211.up_proj", + "model.layers.9.mlp.experts.212.down_proj", + "model.layers.9.mlp.experts.212.gate_proj", + "model.layers.9.mlp.experts.212.up_proj", + "model.layers.9.mlp.experts.213.down_proj", + "model.layers.9.mlp.experts.213.gate_proj", + "model.layers.9.mlp.experts.213.up_proj", + "model.layers.9.mlp.experts.214.down_proj", + "model.layers.9.mlp.experts.214.gate_proj", + "model.layers.9.mlp.experts.214.up_proj", + "model.layers.9.mlp.experts.215.down_proj", + "model.layers.9.mlp.experts.215.gate_proj", + "model.layers.9.mlp.experts.215.up_proj", + "model.layers.9.mlp.experts.216.down_proj", + "model.layers.9.mlp.experts.216.gate_proj", + "model.layers.9.mlp.experts.216.up_proj", + "model.layers.9.mlp.experts.217.down_proj", + "model.layers.9.mlp.experts.217.gate_proj", + "model.layers.9.mlp.experts.217.up_proj", + "model.layers.9.mlp.experts.218.down_proj", + "model.layers.9.mlp.experts.218.gate_proj", + "model.layers.9.mlp.experts.218.up_proj", + "model.layers.9.mlp.experts.219.down_proj", + "model.layers.9.mlp.experts.219.gate_proj", + "model.layers.9.mlp.experts.219.up_proj", + "model.layers.9.mlp.experts.22.down_proj", + "model.layers.9.mlp.experts.22.gate_proj", + "model.layers.9.mlp.experts.22.up_proj", + "model.layers.9.mlp.experts.220.down_proj", + "model.layers.9.mlp.experts.220.gate_proj", + "model.layers.9.mlp.experts.220.up_proj", + "model.layers.9.mlp.experts.221.down_proj", + "model.layers.9.mlp.experts.221.gate_proj", + "model.layers.9.mlp.experts.221.up_proj", + "model.layers.9.mlp.experts.222.down_proj", + "model.layers.9.mlp.experts.222.gate_proj", + "model.layers.9.mlp.experts.222.up_proj", + "model.layers.9.mlp.experts.223.down_proj", + "model.layers.9.mlp.experts.223.gate_proj", + "model.layers.9.mlp.experts.223.up_proj", + "model.layers.9.mlp.experts.224.down_proj", + "model.layers.9.mlp.experts.224.gate_proj", + "model.layers.9.mlp.experts.224.up_proj", + "model.layers.9.mlp.experts.225.down_proj", + "model.layers.9.mlp.experts.225.gate_proj", + "model.layers.9.mlp.experts.225.up_proj", + "model.layers.9.mlp.experts.226.down_proj", + "model.layers.9.mlp.experts.226.gate_proj", + "model.layers.9.mlp.experts.226.up_proj", + "model.layers.9.mlp.experts.227.down_proj", + "model.layers.9.mlp.experts.227.gate_proj", + "model.layers.9.mlp.experts.227.up_proj", + "model.layers.9.mlp.experts.228.down_proj", + "model.layers.9.mlp.experts.228.gate_proj", + "model.layers.9.mlp.experts.228.up_proj", + "model.layers.9.mlp.experts.229.down_proj", + "model.layers.9.mlp.experts.229.gate_proj", + "model.layers.9.mlp.experts.229.up_proj", + "model.layers.9.mlp.experts.23.down_proj", + "model.layers.9.mlp.experts.23.gate_proj", + "model.layers.9.mlp.experts.23.up_proj", + "model.layers.9.mlp.experts.230.down_proj", + "model.layers.9.mlp.experts.230.gate_proj", + "model.layers.9.mlp.experts.230.up_proj", + "model.layers.9.mlp.experts.231.down_proj", + "model.layers.9.mlp.experts.231.gate_proj", + "model.layers.9.mlp.experts.231.up_proj", + "model.layers.9.mlp.experts.232.down_proj", + "model.layers.9.mlp.experts.232.gate_proj", + "model.layers.9.mlp.experts.232.up_proj", + "model.layers.9.mlp.experts.233.down_proj", + "model.layers.9.mlp.experts.233.gate_proj", + "model.layers.9.mlp.experts.233.up_proj", + "model.layers.9.mlp.experts.234.down_proj", + "model.layers.9.mlp.experts.234.gate_proj", + "model.layers.9.mlp.experts.234.up_proj", + "model.layers.9.mlp.experts.235.down_proj", + "model.layers.9.mlp.experts.235.gate_proj", + "model.layers.9.mlp.experts.235.up_proj", + "model.layers.9.mlp.experts.236.down_proj", + "model.layers.9.mlp.experts.236.gate_proj", + "model.layers.9.mlp.experts.236.up_proj", + "model.layers.9.mlp.experts.237.down_proj", + "model.layers.9.mlp.experts.237.gate_proj", + "model.layers.9.mlp.experts.237.up_proj", + "model.layers.9.mlp.experts.238.down_proj", + "model.layers.9.mlp.experts.238.gate_proj", + "model.layers.9.mlp.experts.238.up_proj", + "model.layers.9.mlp.experts.239.down_proj", + "model.layers.9.mlp.experts.239.gate_proj", + "model.layers.9.mlp.experts.239.up_proj", + "model.layers.9.mlp.experts.24.down_proj", + "model.layers.9.mlp.experts.24.gate_proj", + "model.layers.9.mlp.experts.24.up_proj", + "model.layers.9.mlp.experts.240.down_proj", + "model.layers.9.mlp.experts.240.gate_proj", + "model.layers.9.mlp.experts.240.up_proj", + "model.layers.9.mlp.experts.241.down_proj", + "model.layers.9.mlp.experts.241.gate_proj", + "model.layers.9.mlp.experts.241.up_proj", + "model.layers.9.mlp.experts.242.down_proj", + "model.layers.9.mlp.experts.242.gate_proj", + "model.layers.9.mlp.experts.242.up_proj", + "model.layers.9.mlp.experts.243.down_proj", + "model.layers.9.mlp.experts.243.gate_proj", + "model.layers.9.mlp.experts.243.up_proj", + "model.layers.9.mlp.experts.244.down_proj", + "model.layers.9.mlp.experts.244.gate_proj", + "model.layers.9.mlp.experts.244.up_proj", + "model.layers.9.mlp.experts.245.down_proj", + "model.layers.9.mlp.experts.245.gate_proj", + "model.layers.9.mlp.experts.245.up_proj", + "model.layers.9.mlp.experts.246.down_proj", + "model.layers.9.mlp.experts.246.gate_proj", + "model.layers.9.mlp.experts.246.up_proj", + "model.layers.9.mlp.experts.247.down_proj", + "model.layers.9.mlp.experts.247.gate_proj", + "model.layers.9.mlp.experts.247.up_proj", + "model.layers.9.mlp.experts.248.down_proj", + "model.layers.9.mlp.experts.248.gate_proj", + "model.layers.9.mlp.experts.248.up_proj", + "model.layers.9.mlp.experts.249.down_proj", + "model.layers.9.mlp.experts.249.gate_proj", + "model.layers.9.mlp.experts.249.up_proj", + "model.layers.9.mlp.experts.25.down_proj", + "model.layers.9.mlp.experts.25.gate_proj", + "model.layers.9.mlp.experts.25.up_proj", + "model.layers.9.mlp.experts.250.down_proj", + "model.layers.9.mlp.experts.250.gate_proj", + "model.layers.9.mlp.experts.250.up_proj", + "model.layers.9.mlp.experts.251.down_proj", + "model.layers.9.mlp.experts.251.gate_proj", + "model.layers.9.mlp.experts.251.up_proj", + "model.layers.9.mlp.experts.252.down_proj", + "model.layers.9.mlp.experts.252.gate_proj", + "model.layers.9.mlp.experts.252.up_proj", + "model.layers.9.mlp.experts.253.down_proj", + "model.layers.9.mlp.experts.253.gate_proj", + "model.layers.9.mlp.experts.253.up_proj", + "model.layers.9.mlp.experts.254.down_proj", + "model.layers.9.mlp.experts.254.gate_proj", + "model.layers.9.mlp.experts.254.up_proj", + "model.layers.9.mlp.experts.255.down_proj", + "model.layers.9.mlp.experts.255.gate_proj", + "model.layers.9.mlp.experts.255.up_proj", + "model.layers.9.mlp.experts.26.down_proj", + "model.layers.9.mlp.experts.26.gate_proj", + "model.layers.9.mlp.experts.26.up_proj", + "model.layers.9.mlp.experts.27.down_proj", + "model.layers.9.mlp.experts.27.gate_proj", + "model.layers.9.mlp.experts.27.up_proj", + "model.layers.9.mlp.experts.28.down_proj", + "model.layers.9.mlp.experts.28.gate_proj", + "model.layers.9.mlp.experts.28.up_proj", + "model.layers.9.mlp.experts.29.down_proj", + "model.layers.9.mlp.experts.29.gate_proj", + "model.layers.9.mlp.experts.29.up_proj", + "model.layers.9.mlp.experts.3.down_proj", + "model.layers.9.mlp.experts.3.gate_proj", + "model.layers.9.mlp.experts.3.up_proj", + "model.layers.9.mlp.experts.30.down_proj", + "model.layers.9.mlp.experts.30.gate_proj", + "model.layers.9.mlp.experts.30.up_proj", + "model.layers.9.mlp.experts.31.down_proj", + "model.layers.9.mlp.experts.31.gate_proj", + "model.layers.9.mlp.experts.31.up_proj", + "model.layers.9.mlp.experts.32.down_proj", + "model.layers.9.mlp.experts.32.gate_proj", + "model.layers.9.mlp.experts.32.up_proj", + "model.layers.9.mlp.experts.33.down_proj", + "model.layers.9.mlp.experts.33.gate_proj", + "model.layers.9.mlp.experts.33.up_proj", + "model.layers.9.mlp.experts.34.down_proj", + "model.layers.9.mlp.experts.34.gate_proj", + "model.layers.9.mlp.experts.34.up_proj", + "model.layers.9.mlp.experts.35.down_proj", + "model.layers.9.mlp.experts.35.gate_proj", + "model.layers.9.mlp.experts.35.up_proj", + "model.layers.9.mlp.experts.36.down_proj", + "model.layers.9.mlp.experts.36.gate_proj", + "model.layers.9.mlp.experts.36.up_proj", + "model.layers.9.mlp.experts.37.down_proj", + "model.layers.9.mlp.experts.37.gate_proj", + "model.layers.9.mlp.experts.37.up_proj", + "model.layers.9.mlp.experts.38.down_proj", + "model.layers.9.mlp.experts.38.gate_proj", + "model.layers.9.mlp.experts.38.up_proj", + "model.layers.9.mlp.experts.39.down_proj", + "model.layers.9.mlp.experts.39.gate_proj", + "model.layers.9.mlp.experts.39.up_proj", + "model.layers.9.mlp.experts.4.down_proj", + "model.layers.9.mlp.experts.4.gate_proj", + "model.layers.9.mlp.experts.4.up_proj", + "model.layers.9.mlp.experts.40.down_proj", + "model.layers.9.mlp.experts.40.gate_proj", + "model.layers.9.mlp.experts.40.up_proj", + "model.layers.9.mlp.experts.41.down_proj", + "model.layers.9.mlp.experts.41.gate_proj", + "model.layers.9.mlp.experts.41.up_proj", + "model.layers.9.mlp.experts.42.down_proj", + "model.layers.9.mlp.experts.42.gate_proj", + "model.layers.9.mlp.experts.42.up_proj", + "model.layers.9.mlp.experts.43.down_proj", + "model.layers.9.mlp.experts.43.gate_proj", + "model.layers.9.mlp.experts.43.up_proj", + "model.layers.9.mlp.experts.44.down_proj", + "model.layers.9.mlp.experts.44.gate_proj", + "model.layers.9.mlp.experts.44.up_proj", + "model.layers.9.mlp.experts.45.down_proj", + "model.layers.9.mlp.experts.45.gate_proj", + "model.layers.9.mlp.experts.45.up_proj", + "model.layers.9.mlp.experts.46.down_proj", + "model.layers.9.mlp.experts.46.gate_proj", + "model.layers.9.mlp.experts.46.up_proj", + "model.layers.9.mlp.experts.47.down_proj", + "model.layers.9.mlp.experts.47.gate_proj", + "model.layers.9.mlp.experts.47.up_proj", + "model.layers.9.mlp.experts.48.down_proj", + "model.layers.9.mlp.experts.48.gate_proj", + "model.layers.9.mlp.experts.48.up_proj", + "model.layers.9.mlp.experts.49.down_proj", + "model.layers.9.mlp.experts.49.gate_proj", + "model.layers.9.mlp.experts.49.up_proj", + "model.layers.9.mlp.experts.5.down_proj", + "model.layers.9.mlp.experts.5.gate_proj", + "model.layers.9.mlp.experts.5.up_proj", + "model.layers.9.mlp.experts.50.down_proj", + "model.layers.9.mlp.experts.50.gate_proj", + "model.layers.9.mlp.experts.50.up_proj", + "model.layers.9.mlp.experts.51.down_proj", + "model.layers.9.mlp.experts.51.gate_proj", + "model.layers.9.mlp.experts.51.up_proj", + "model.layers.9.mlp.experts.52.down_proj", + "model.layers.9.mlp.experts.52.gate_proj", + "model.layers.9.mlp.experts.52.up_proj", + "model.layers.9.mlp.experts.53.down_proj", + "model.layers.9.mlp.experts.53.gate_proj", + "model.layers.9.mlp.experts.53.up_proj", + "model.layers.9.mlp.experts.54.down_proj", + "model.layers.9.mlp.experts.54.gate_proj", + "model.layers.9.mlp.experts.54.up_proj", + "model.layers.9.mlp.experts.55.down_proj", + "model.layers.9.mlp.experts.55.gate_proj", + "model.layers.9.mlp.experts.55.up_proj", + "model.layers.9.mlp.experts.56.down_proj", + "model.layers.9.mlp.experts.56.gate_proj", + "model.layers.9.mlp.experts.56.up_proj", + "model.layers.9.mlp.experts.57.down_proj", + "model.layers.9.mlp.experts.57.gate_proj", + "model.layers.9.mlp.experts.57.up_proj", + "model.layers.9.mlp.experts.58.down_proj", + "model.layers.9.mlp.experts.58.gate_proj", + "model.layers.9.mlp.experts.58.up_proj", + "model.layers.9.mlp.experts.59.down_proj", + "model.layers.9.mlp.experts.59.gate_proj", + "model.layers.9.mlp.experts.59.up_proj", + "model.layers.9.mlp.experts.6.down_proj", + "model.layers.9.mlp.experts.6.gate_proj", + "model.layers.9.mlp.experts.6.up_proj", + "model.layers.9.mlp.experts.60.down_proj", + "model.layers.9.mlp.experts.60.gate_proj", + "model.layers.9.mlp.experts.60.up_proj", + "model.layers.9.mlp.experts.61.down_proj", + "model.layers.9.mlp.experts.61.gate_proj", + "model.layers.9.mlp.experts.61.up_proj", + "model.layers.9.mlp.experts.62.down_proj", + "model.layers.9.mlp.experts.62.gate_proj", + "model.layers.9.mlp.experts.62.up_proj", + "model.layers.9.mlp.experts.63.down_proj", + "model.layers.9.mlp.experts.63.gate_proj", + "model.layers.9.mlp.experts.63.up_proj", + "model.layers.9.mlp.experts.64.down_proj", + "model.layers.9.mlp.experts.64.gate_proj", + "model.layers.9.mlp.experts.64.up_proj", + "model.layers.9.mlp.experts.65.down_proj", + "model.layers.9.mlp.experts.65.gate_proj", + "model.layers.9.mlp.experts.65.up_proj", + "model.layers.9.mlp.experts.66.down_proj", + "model.layers.9.mlp.experts.66.gate_proj", + "model.layers.9.mlp.experts.66.up_proj", + "model.layers.9.mlp.experts.67.down_proj", + "model.layers.9.mlp.experts.67.gate_proj", + "model.layers.9.mlp.experts.67.up_proj", + "model.layers.9.mlp.experts.68.down_proj", + "model.layers.9.mlp.experts.68.gate_proj", + "model.layers.9.mlp.experts.68.up_proj", + "model.layers.9.mlp.experts.69.down_proj", + "model.layers.9.mlp.experts.69.gate_proj", + "model.layers.9.mlp.experts.69.up_proj", + "model.layers.9.mlp.experts.7.down_proj", + "model.layers.9.mlp.experts.7.gate_proj", + "model.layers.9.mlp.experts.7.up_proj", + "model.layers.9.mlp.experts.70.down_proj", + "model.layers.9.mlp.experts.70.gate_proj", + "model.layers.9.mlp.experts.70.up_proj", + "model.layers.9.mlp.experts.71.down_proj", + "model.layers.9.mlp.experts.71.gate_proj", + "model.layers.9.mlp.experts.71.up_proj", + "model.layers.9.mlp.experts.72.down_proj", + "model.layers.9.mlp.experts.72.gate_proj", + "model.layers.9.mlp.experts.72.up_proj", + "model.layers.9.mlp.experts.73.down_proj", + "model.layers.9.mlp.experts.73.gate_proj", + "model.layers.9.mlp.experts.73.up_proj", + "model.layers.9.mlp.experts.74.down_proj", + "model.layers.9.mlp.experts.74.gate_proj", + "model.layers.9.mlp.experts.74.up_proj", + "model.layers.9.mlp.experts.75.down_proj", + "model.layers.9.mlp.experts.75.gate_proj", + "model.layers.9.mlp.experts.75.up_proj", + "model.layers.9.mlp.experts.76.down_proj", + "model.layers.9.mlp.experts.76.gate_proj", + "model.layers.9.mlp.experts.76.up_proj", + "model.layers.9.mlp.experts.77.down_proj", + "model.layers.9.mlp.experts.77.gate_proj", + "model.layers.9.mlp.experts.77.up_proj", + "model.layers.9.mlp.experts.78.down_proj", + "model.layers.9.mlp.experts.78.gate_proj", + "model.layers.9.mlp.experts.78.up_proj", + "model.layers.9.mlp.experts.79.down_proj", + "model.layers.9.mlp.experts.79.gate_proj", + "model.layers.9.mlp.experts.79.up_proj", + "model.layers.9.mlp.experts.8.down_proj", + "model.layers.9.mlp.experts.8.gate_proj", + "model.layers.9.mlp.experts.8.up_proj", + "model.layers.9.mlp.experts.80.down_proj", + "model.layers.9.mlp.experts.80.gate_proj", + "model.layers.9.mlp.experts.80.up_proj", + "model.layers.9.mlp.experts.81.down_proj", + "model.layers.9.mlp.experts.81.gate_proj", + "model.layers.9.mlp.experts.81.up_proj", + "model.layers.9.mlp.experts.82.down_proj", + "model.layers.9.mlp.experts.82.gate_proj", + "model.layers.9.mlp.experts.82.up_proj", + "model.layers.9.mlp.experts.83.down_proj", + "model.layers.9.mlp.experts.83.gate_proj", + "model.layers.9.mlp.experts.83.up_proj", + "model.layers.9.mlp.experts.84.down_proj", + "model.layers.9.mlp.experts.84.gate_proj", + "model.layers.9.mlp.experts.84.up_proj", + "model.layers.9.mlp.experts.85.down_proj", + "model.layers.9.mlp.experts.85.gate_proj", + "model.layers.9.mlp.experts.85.up_proj", + "model.layers.9.mlp.experts.86.down_proj", + "model.layers.9.mlp.experts.86.gate_proj", + "model.layers.9.mlp.experts.86.up_proj", + "model.layers.9.mlp.experts.87.down_proj", + "model.layers.9.mlp.experts.87.gate_proj", + "model.layers.9.mlp.experts.87.up_proj", + "model.layers.9.mlp.experts.88.down_proj", + "model.layers.9.mlp.experts.88.gate_proj", + "model.layers.9.mlp.experts.88.up_proj", + "model.layers.9.mlp.experts.89.down_proj", + "model.layers.9.mlp.experts.89.gate_proj", + "model.layers.9.mlp.experts.89.up_proj", + "model.layers.9.mlp.experts.9.down_proj", + "model.layers.9.mlp.experts.9.gate_proj", + "model.layers.9.mlp.experts.9.up_proj", + "model.layers.9.mlp.experts.90.down_proj", + "model.layers.9.mlp.experts.90.gate_proj", + "model.layers.9.mlp.experts.90.up_proj", + "model.layers.9.mlp.experts.91.down_proj", + "model.layers.9.mlp.experts.91.gate_proj", + "model.layers.9.mlp.experts.91.up_proj", + "model.layers.9.mlp.experts.92.down_proj", + "model.layers.9.mlp.experts.92.gate_proj", + "model.layers.9.mlp.experts.92.up_proj", + "model.layers.9.mlp.experts.93.down_proj", + "model.layers.9.mlp.experts.93.gate_proj", + "model.layers.9.mlp.experts.93.up_proj", + "model.layers.9.mlp.experts.94.down_proj", + "model.layers.9.mlp.experts.94.gate_proj", + "model.layers.9.mlp.experts.94.up_proj", + "model.layers.9.mlp.experts.95.down_proj", + "model.layers.9.mlp.experts.95.gate_proj", + "model.layers.9.mlp.experts.95.up_proj", + "model.layers.9.mlp.experts.96.down_proj", + "model.layers.9.mlp.experts.96.gate_proj", + "model.layers.9.mlp.experts.96.up_proj", + "model.layers.9.mlp.experts.97.down_proj", + "model.layers.9.mlp.experts.97.gate_proj", + "model.layers.9.mlp.experts.97.up_proj", + "model.layers.9.mlp.experts.98.down_proj", + "model.layers.9.mlp.experts.98.gate_proj", + "model.layers.9.mlp.experts.98.up_proj", + "model.layers.9.mlp.experts.99.down_proj", + "model.layers.9.mlp.experts.99.gate_proj", + "model.layers.9.mlp.experts.99.up_proj" + ], + "weights": { + "num_bits": 4, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": false, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": "memoryless_minmax", + "observer_kwargs": {} + }, + "input_activations": { + "num_bits": 4, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": true, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": null, + "observer_kwargs": {} + }, + "output_activations": null, + "format": "mxfp4-pack-quantized" + }, + "group_1": { + "targets": [ + "Linear" + ], + "weights": { + "num_bits": 8, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": false, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": "memoryless_minmax", + "observer_kwargs": {} + }, + "input_activations": { + "num_bits": 8, + "type": "float", + "symmetric": true, + "group_size": 32, + "strategy": "group", + "block_structure": null, + "dynamic": true, + "actorder": null, + "scale_dtype": "torch.uint8", + "zp_dtype": null, + "observer": null, + "observer_kwargs": {} + }, + "output_activations": null, + "format": "mxfp8-quantized" + } + }, + "quant_method": "compressed-tensors", + "kv_cache_scheme": null, + "format": "mixed-precision", + "quantization_status": "compressed", + "global_compression_ratio": null, + "ignore": [ + "lm_head", + "model.embed_tokens", + "model.layers.0.mlp.down_proj", + "model.layers.0.mlp.gate_proj", + "model.layers.0.mlp.up_proj", + "model.layers.0.self_attn.indexer.weights_proj", + "model.layers.1.mlp.down_proj", + "model.layers.1.mlp.gate_proj", + "model.layers.1.mlp.up_proj", + "model.layers.1.self_attn.indexer.weights_proj", + "model.layers.10.mlp.gate", + "model.layers.10.self_attn.indexer.weights_proj", + "model.layers.11.mlp.gate", + "model.layers.12.mlp.gate", + "model.layers.13.mlp.gate", + "model.layers.14.mlp.gate", + "model.layers.14.self_attn.indexer.weights_proj", + "model.layers.15.mlp.gate", + "model.layers.16.mlp.gate", + "model.layers.17.mlp.gate", + "model.layers.18.mlp.gate", + "model.layers.18.self_attn.indexer.weights_proj", + "model.layers.19.mlp.gate", + "model.layers.2.mlp.down_proj", + "model.layers.2.mlp.gate_proj", + "model.layers.2.mlp.up_proj", + "model.layers.2.self_attn.indexer.weights_proj", + "model.layers.20.mlp.gate", + "model.layers.21.mlp.gate", + "model.layers.22.mlp.gate", + "model.layers.22.self_attn.indexer.weights_proj", + "model.layers.23.mlp.gate", + "model.layers.24.mlp.gate", + "model.layers.25.mlp.gate", + "model.layers.26.mlp.gate", + "model.layers.26.self_attn.indexer.weights_proj", + "model.layers.27.mlp.gate", + "model.layers.28.mlp.gate", + "model.layers.29.mlp.gate", + "model.layers.3.mlp.gate", + "model.layers.30.mlp.gate", + "model.layers.30.self_attn.indexer.weights_proj", + "model.layers.31.mlp.gate", + "model.layers.32.mlp.gate", + "model.layers.33.mlp.gate", + "model.layers.34.mlp.gate", + "model.layers.34.self_attn.indexer.weights_proj", + "model.layers.35.mlp.gate", + "model.layers.36.mlp.gate", + "model.layers.37.mlp.gate", + "model.layers.38.mlp.gate", + "model.layers.38.self_attn.indexer.weights_proj", + "model.layers.39.mlp.gate", + "model.layers.4.mlp.gate", + "model.layers.40.mlp.gate", + "model.layers.41.mlp.gate", + "model.layers.42.mlp.gate", + "model.layers.42.self_attn.indexer.weights_proj", + "model.layers.43.mlp.gate", + "model.layers.44.mlp.gate", + "model.layers.45.mlp.gate", + "model.layers.46.mlp.gate", + "model.layers.46.self_attn.indexer.weights_proj", + "model.layers.47.mlp.gate", + "model.layers.48.mlp.gate", + "model.layers.49.mlp.gate", + "model.layers.5.mlp.gate", + "model.layers.50.mlp.gate", + "model.layers.50.self_attn.indexer.weights_proj", + "model.layers.51.mlp.gate", + "model.layers.52.mlp.gate", + "model.layers.53.mlp.gate", + "model.layers.54.mlp.gate", + "model.layers.54.self_attn.indexer.weights_proj", + "model.layers.55.mlp.gate", + "model.layers.56.mlp.gate", + "model.layers.57.mlp.gate", + "model.layers.58.mlp.gate", + "model.layers.58.self_attn.indexer.weights_proj", + "model.layers.59.mlp.gate", + "model.layers.6.mlp.gate", + "model.layers.6.self_attn.indexer.weights_proj", + "model.layers.60.mlp.gate", + "model.layers.61.mlp.gate", + "model.layers.62.mlp.gate", + "model.layers.62.self_attn.indexer.weights_proj", + "model.layers.63.mlp.gate", + "model.layers.64.mlp.gate", + "model.layers.65.mlp.gate", + "model.layers.66.mlp.gate", + "model.layers.66.self_attn.indexer.weights_proj", + "model.layers.67.mlp.gate", + "model.layers.68.mlp.gate", + "model.layers.69.mlp.gate", + "model.layers.7.mlp.gate", + "model.layers.70.mlp.gate", + "model.layers.70.self_attn.indexer.weights_proj", + "model.layers.71.mlp.gate", + "model.layers.72.mlp.gate", + "model.layers.73.mlp.gate", + "model.layers.74.mlp.gate", + "model.layers.74.self_attn.indexer.weights_proj", + "model.layers.75.mlp.gate", + "model.layers.76.mlp.gate", + "model.layers.77.mlp.gate", + "model.layers.78.mlp.gate", + "model.layers.78.self_attn.indexer.weights_proj", + "model.layers.8.mlp.gate", + "model.layers.9.mlp.gate" + ], + "provider": "auto-round" +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..aba40197a4cdb5607f4ab7a05fb0a4ee8054fd6d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d +size 20217442 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e375fa0a4e16660ce0e2026e39374d35dad4c079 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,33 @@ +{ + "backend": "tokenizers", + "clean_up_tokenization_spaces": false, + "do_lower_case": false, + "eos_token": "<|endoftext|>", + "extra_special_tokens": [ + "<|endoftext|>", + "[MASK]", + "[gMASK]", + "[sMASK]", + "", + "", + "<|system|>", + "<|user|>", + "<|assistant|>", + "<|observation|>", + "<|begin_of_image|>", + "<|end_of_image|>", + "<|begin_of_video|>", + "<|end_of_video|>", + "<|begin_of_audio|>", + "<|end_of_audio|>", + "<|begin_of_transcription|>", + "<|end_of_transcription|>" + ], + "is_local": true, + "model_max_length": 1048576, + "model_specific_special_tokens": {}, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "remove_space": false, + "tokenizer_class": "TokenizersBackend" +}