diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-boost.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..9679c7c5633c60566d48164fc79864068cc64440 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-boost.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +EXP_BASE_NAME="sensi-boost" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-directive +layers="31 1" +ATTEMPT="${weight_algo}" +EXP_NAME="${ATTEMPT}" +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +echo "=========Run perplexity evaluation on batch ${EXP_NAME}=========" +python ../src/cli.py llm \ + --task eval_ppl \ + --model 0 \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-layer $layers \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi +echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + +echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +python ../src/cli.py dump \ + --type quant_config \ + --model 0 \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=mxq +model_ids="0" +for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done +echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv +$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv +$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv +pdflatex table.tex + +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf" +BGS="4.13 4.25 4.51" +for model in $MODELS; do + for budget in $BGS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-kurt-boost.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-kurt-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..c07526d6be4a6fbcaaea4bc7b0c029a8ba98525f --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-kurt-boost.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=kurt-boost +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +# BOOST_STOPS="2 3" +# BOOST_TOP_MS="1 2 3 0" +BOOST_STOPS="2" +BOOST_TOP_MS="1" + + +for BOOST_STOP in $BOOST_STOPS; do + for BOOST_TOP_M in $BOOST_TOP_MS; do + ATTEMPT="kurt-boost-${BOOST_STOP}-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-stop $BOOST_STOP \ + --top-m-layer $BOOST_TOP_M \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + pdflatex table.tex + + # plot configuration allocations for 3 * 12 MXQ combinations + for model in $MODEL_NAMES; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR + done +done + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-ablation.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-ablation.sh new file mode 100644 index 0000000000000000000000000000000000000000..648583c9743e6070aa19f60e39d730edb25ca6b6 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-ablation.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-boost +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +# BOOST_STOPS="2 3" +# BOOST_TOP_MS="1 2 3 0" + +BOOST_STOPS="3" +BOOST_TOP_MS="3 0" + +for BOOST_STOP in $BOOST_STOPS; do + for BOOST_TOP_M in $BOOST_TOP_MS; do + ATTEMPT="sensi-abl-${BOOST_STOP}-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-stop $BOOST_STOP \ + --top-m-layer $BOOST_TOP_M \ + --ablation \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + pdflatex table.tex + + # plot configuration allocations for 3 * 12 MXQ combinations + for model in $MODEL_NAMES; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR + done +done + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..b4374934578516b03a10449ba6d7a5e8994e4a86 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-boost +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +# BOOST_STOPS="2 3" +BOOST_STOPS="3" +BOOST_TOP_MS="0" + +for BOOST_STOP in $BOOST_STOPS; do + for BOOST_TOP_M in $BOOST_TOP_MS; do + if [[ $BOOST_STOP -eq 2 && $BOOST_TOP_M -eq 1 ]]; then + continue + fi + ATTEMPT="sensi-boost-${BOOST_STOP}-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-stop $BOOST_STOP \ + --top-m-layer $BOOST_TOP_M \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + pdflatex table.tex + + # plot configuration allocations for 3 * 12 MXQ combinations + for model in $MODEL_NAMES; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR + done +done + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..6b7d87eb5c0f4e18430f217ef4a000bca3ceef90 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +ATTEMPT="llama-sensi-milp-2x" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-milp + +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +MODELS="0 2" +EXP_NAME="${ATTEMPT}" +echo "=========Run perplexity evaluation on Llama-2-7b=========" +python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi +echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + +echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=mxq +model_ids=$MODELS +for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done +echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv +$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv +$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv +pdflatex table.tex + +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf Meta-Llama-3-8B" +BGS="4.13 4.25 4.51" +for model in $MODELS; do + for budget in $BGS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh new file mode 100644 index 0000000000000000000000000000000000000000..633119aa50095b4ef62e9d82c32be4db613b5a92 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-milp +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +# for SensiMiLP ablation test, all topm values are equivalent + +ATTEMPT="sensi-milp-abl" +EXP_BASE_NAME=$ATTEMPT +EXP_NAME="${ATTEMPT}" + +log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +echo "=========Run perplexity evaluation=========" +python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --ablation \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi +echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + +echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=mxq +model_ids=$MODELS +for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done +echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs ]; then + mkdir pdfs +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT +cd pdfs +pdflatex table.tex +cd .. + +cd $OLD_DIR diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..9ed1242ada4bd171a623caa82e881aca111516dd --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +MODELS="1" +ATTEMPT="Llama2-13B-boost" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-directive +boost_layers="3 39" +EXP_NAME="${ATTEMPT}" +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +echo "=========Run perplexity evaluation on batch ${EXP_NAME}=========" +python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-layer $boost_layers \ + --boost-stop 2 \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi +echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + +echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=mxq +model_ids=$MODELS +for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done +echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv +$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv +$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv +pdflatex table.tex + +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-13b-hf" +BGS="4.13 4.25 4.51" +for model in $MODELS; do + for budget in $BGS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh new file mode 100644 index 0000000000000000000000000000000000000000..0808bd6a9a306656b1382b4ec13502559678fd76 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh @@ -0,0 +1,128 @@ +#!/bin/bash + + +# 100%, 99%, 98%, 97%, 96%, 95% of [3.51, 4.25] +# MXQ2=(minibatch 3.51 3.47 3.44 3.40 3.37 3.33 4.25 4.21 4.17 4.12 4.08 4.04) +MXQ2=(minibatch 6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07) +MXQ_BATCHES=(MXQ2) +declare -n MXQ_BATCH + +ATTEMPT="sensi-milp-mini2" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" +EXP_BASE_NAME="${ATTEMPT}" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" +# Setup data files directories for reporting +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + +weight_algo=sensi-milp + +for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do + batch_name=${MXQ_BATCH[@]:0:1} + EXP_NAME="${EXP_BASE_NAME}-${batch_name}" + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/${EXP_NAME}_stor + log_file="logs/bench-${EXP_NAME}-$(date +%Y%m%d%H%M%S).log" + + echo "=========Run perplexity evaluation on batch ${batch_name}=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model 0 1 2 \ + --algo mxq \ + --weight-algo ${weight_algo} \ + --config ${MXQ_BATCH[@]:1} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${batch_name}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${batch_name}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model 0 1 2 \ + --budget ${MXQ_BATCH[@]:1} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${batch_name}=========" + algo=mxq + model_ids="0 1 2" + for m in $model_ids; do + for cfg in "${MXQ_BATCH[@]:1}"; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${batch_name}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs ]; then + mkdir pdfs + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + cd pdfs + pdflatex table.tex + cd .. + + # plot configuration allocations for 3 * 12 MXQ combinations + MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + BUDGETS=${MXQ_BATCH[@]:1} + for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --attempt1 mxq1 \ + --attempt2 $ATTEMPT \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-tail-reduction.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-tail-reduction.sh new file mode 100644 index 0000000000000000000000000000000000000000..de284c9e9fb30f4b6b00d31860ac69d62d4d2925 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-tail-reduction.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +EXP_BASE_NAME="tail-reduction" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=tail_boost +ATTEMPT="${EXP_BASE_NAME}" +EXP_NAME="${ATTEMPT}" +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +echo "=========Run perplexity evaluation on batch ${EXP_NAME}=========" +python ../src/cli.py llm \ + --task eval_ppl \ + --model 0 1 2 \ + --algo mxq \ + --weight-algo $weight_algo \ + --factor -1 \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi +echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" +# find $RESULT_DIR/${EXP_NAME}_ppl \ +# -name "result-*.csv" \ +# -printf '%T@ %p\n' \ +# | sort -n \ +# | tail -1 \ +# | cut -d' ' -f2 \ +# | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT +# +# echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +# mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +# python ../src/cli.py dump \ +# --type quant_config \ +# --model 0 1 2 \ +# --budget ${BUDGETS} \ +# --attempt $ATTEMPT \ +# --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ +# --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ +# 2>&1 \ +# | tee -a $log_file +# +# echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +# algo=mxq +# model_ids="0 1 2" +# for m in $model_ids; do +# for cfg in ${BUDGETS}; do +# python ../src/cli.py llm \ +# --model $m \ +# --algo ${algo} \ +# --config ${cfg} \ +# --task eval_model_storage \ +# --experiment-name "${EXP_NAME}_stor" \ +# --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ +# --result-dir=$RESULT_DIR \ +# 2>&1 \ +# | tee -a $log_file +# done +# done +# echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +# find $RESULT_DIR/${EXP_NAME}_stor \ +# -name "result-*.csv" \ +# -printf '%T@ %p\n' \ +# | sort -n \ +# | tail -1 \ +# | cut -d' ' -f2 \ +# | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv +$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv +$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv +pdflatex table.tex + +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" +BUDGETS="4.13 4.25 4.51" +for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 tail-reduction \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..2fc64436870007de2ca14d039a1543808fa1d039 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ ! -d logs ]; then + mkdir logs +fi + +# python ../src/cli.py vit \ +# --task eval_zeroshot_cls \ +# --model 0 1 \ +# --config 4.51 4.25 4.13 3.51 3.25 3.13\ +# --algo mxq \ +# --weight-algo kurt-boost \ +# --boost-stop 2 \ +# --top-m-layer 1 \ +# --experiment-name eval_zs_BH_mxq_kurt_boost \ +# --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ +# --result-dir="/fdata/llm/mxq/results" \ +# 2>&1 \ +# | tee logs/bench-vit-$(date +%Y%m%d%H%M%S).log + +python ../src/cli.py vit \ + --task eval_zeroshot_cls \ + --model 0 1 \ + --config 4.51 4.25 4.13 3.51 3.25 3.13\ + --algo mxq \ + --weight-algo kurt-boost \ + --boost-stop 2 \ + --top-m-layer 2 \ + --experiment-name eval_zs_BH_mxq_kurt_boost_22 \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-vit-$(date +%Y%m%d%H%M%S).log + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..8720c0ead1a768afdeba65d7d2ad9e0f99932e45 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +BUDGETS="3.25" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-boost +MODELS="0" +MODEL_NAMES="Llama-2-7b-hf" + + +# BOOST_STOPS="2 3" +BOOST_STOPS="3" +BOOST_TOP_MS="3" + +for BOOST_STOP in $BOOST_STOPS; do + for BOOST_TOP_M in $BOOST_TOP_MS; do + if [[ $BOOST_STOP -eq 2 && $BOOST_TOP_M -eq 1 ]]; then + continue + fi + ATTEMPT="sb-3257_2-fix-${BOOST_STOP}-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-stop $BOOST_STOP \ + --top-m-layer $BOOST_TOP_M \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + + cd $OLD_DIR + done +done + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f5ceabba3a040ef0565960c073057726973d2e0 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=kurt-milp +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +# for SensiMiLP ablation test, all topm values are equivalent + +ATTEMPT="kurt-milp-abl" +EXP_BASE_NAME=$ATTEMPT +EXP_NAME="${ATTEMPT}" + +log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=mxq +model_ids=$MODELS +for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done +echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs ]; then + mkdir pdfs +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT +cd pdfs +pdflatex table.tex +cd .. + +cd $OLD_DIR diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh new file mode 100644 index 0000000000000000000000000000000000000000..b6bbaac99fe00c5ffe4b6d70bb4bf72f23208a3e --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-milp +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +BOOST_TOP_MS="3" + +for BOOST_TOP_M in $BOOST_TOP_MS; do + ATTEMPT="sensi-milp-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --top-m-layer $BOOST_TOP_M \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs ]; then + mkdir pdfs + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + cd pdfs + pdflatex table.tex + cd .. + + # plot configuration allocations for 3 * 12 MXQ combinations + MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + BUDGETS=${MXQ_BATCH[@]:1} + for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --attempt1 mxq1 \ + --attempt2 $ATTEMPT \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR + +done + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh new file mode 100644 index 0000000000000000000000000000000000000000..dd2f4738e3a0a9c9af40f2d827e4488c55ffe1a2 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=kurt-milp +MODELS="0 1 2" +MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + + +BOOST_TOP_MS="1" + +for BOOST_TOP_M in $BOOST_TOP_MS; do + ATTEMPT="kurt-milp-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs ]; then + mkdir pdfs + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + cd pdfs + pdflatex table.tex + cd .. + + cd $OLD_DIR + +done + diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh new file mode 100644 index 0000000000000000000000000000000000000000..f0bcb1fadd3e8d4699d72269e210a137c2b941c2 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# MXQ1=(2bit 2.13 2.15 2.17 2.19 2.21 2.23 2.25 2.27 2.29 2.31 2.45 2.47 2.49 2.51 2.53 2.55 2.57) +# MXQ2=(3bit 3.07 3.09 3.11 3.13 3.15 3.17 3.19 3.21 3.23 3.25 3.27 3.29 3.31 3.33 3.35 3.37 3.39 3.41 3.42 3.43 3.45 3.47 3.49 3.51 3.53 3.55 3.57 3.59 3.61 3.63 3.65 3.67 3.69 3.71 3.73 3.75 3.77 3.79 3.81 3.83 3.85 3.87 3.89 3.91 3.93 3.95 3.97 3.99) +# Removed 4.99 which is unsolvable with kurt-scaled scheme +MXQ4=(4bit 4.01 4.03 4.05 4.07 4.09 4.11 4.13 4.15 4.17 4.19 4.21 4.23 4.25 4.27 4.29 4.31 4.33 4.35 4.37 4.39 4.41 4.43 4.45 4.47 4.49 4.51 4.53 4.55 4.57 4.59 4.61 4.63 4.65 4.67 4.69 4.71 4.73 4.75 4.77 4.79 4.81 4.83 4.85 4.87 4.89 4.91 4.93 4.95 4.97) +# Remove 5.01 5.02, 5.04 5.10 5.32 which is unsolvable with kurt-scaled scheme +#MXQ5=(5bit 5.00 5.03 5.06 5.08 5.09 5.12 5.14 5.16 5.18 5.20 5.22 5.24 5.26 5.28 5.30 5.33 5.34 5.36 5.38 5.40 5.42 5.44 5.46 5.48 5.50 5.52 5.54 5.56 5.58 5.60 5.62 5.64 5.66 5.68 5.70 5.72 5.74 5.76 5.78 5.80 5.82 5.84 5.86 5.88 5.90 5.92 5.94 5.96 5.98) +#MXQ6=(6bit 6.00 6.02 6.03 6.04 6.05 6.06 6.07 6.09 6.11 6.13 6.15 6.17 6.19 6.21 6.23 6.25 6.27 6.29 6.31 6.33 6.35 6.37 6.39 6.41 6.43 6.45 6.47 6.49 6.51 6.53 6.55 6.57 6.59 6.61 6.63 6.65 6.68 6.69 6.71 6.72 6.75 6.77 6.78 6.81 6.83 6.86 6.87 6.89 6.92 6.95 6.96 6.98) +# MXQ7=(78bit 7.01 7.02 7.04 7.06 7.08 7.10 7.13 7.14 7.16 7.19 7.20 7.22 7.24 7.26 7.29 7.30 7.32 7.34 7.36 7.38 7.41 7.42 7.44 7.47 7.48 7.50 7.53 7.54 7.56 7.57 7.60 7.62 7.63 7.66 7.68 7.70 7.72 7.74 7.76 8.13 8.25 8.51) +MXQ_BATCHES=(MXQ4) +declare -n MXQ_BATCH + +ATTEMPT="kurt-scaled" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" +EXP_BASE_NAME="${ATTEMPT}-dense" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" +# Setup data files directories for reporting +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs ]; then + mkdir pdfs +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + +# plot configuration allocations for 3 * 12 MXQ combinations +# MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" +MODELS="Llama-2-7b-hf" +BUDGETS="4.13 4.25 4.51" +for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R -m $model -b $budget \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --attempt1 mxq1 \ + --attempt2 $ATTEMPT \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-llama-sensi.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-llama-sensi.sh new file mode 100644 index 0000000000000000000000000000000000000000..22b13942f59544ca88b8057498d3b9665218dc53 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-llama-sensi.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +ATTEMPT="llama-sensi" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-directive + +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi + +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf" +BGS="4.13 4.25 4.51" +for model in $MODELS; do + for budget in $BGS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --fnorm FALSE \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-milp-low-bit.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-milp-low-bit.sh new file mode 100644 index 0000000000000000000000000000000000000000..41daba614432618fc21a16d26a0d9e9e24df3039 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-milp-low-bit.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +BUDGETS="4.25" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +FACTOR=2 +ATTEMPT="sensi-milp-debug-lowbits9" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-milp + +log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + +# MODELS="0 1 2" +MODELS="0" +EXP_NAME="${ATTEMPT}" + +echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + +echo "=========Dump quantization configs on batch ${EXP_NAME}=========" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT +python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=mxq +model_ids=$MODELS +for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done +echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" +find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + +# echo "=========Delete quantized models of batch ${batch_name}=========" +# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv +$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv +$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv +$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT +pdflatex table.tex + +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf" +for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh new file mode 100644 index 0000000000000000000000000000000000000000..78efdf20e3924b0539d4d4e611f1de5631406476 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +RESULT_BASE_DIR="/fdata/llm/mxq/results" +CALIB_DATASETS="bos pileval wikitext c4" +CONFIGS="b2g128 b2g64 b2g32 b3g128 b3g64 b3g32 b4g128 b4g64 b4g32 b8g128 b8g64 b8g32" +MODELS="Qwen/Qwen2.5-7B Qwen/Qwen2.5-Coder-7B Qwen/Qwen2.5-Coder-7B-Instruct Qwen/Qwen2.5-Math-7B" + +EXP_NAME=sensi_qwen25 +RESULT_DIR=$RESULT_BASE_DIR/$EXP_NAME +mkdir -p $RESULT_DIR/data + +OLD_DIR=$(pwd) +cd $RESULT_DIR +if [ ! -d pdfs ]; then + mkdir -p pdfs +fi +$OLD_DIR/../data-vis/plot-variant-sensi.R data/ diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh new file mode 100644 index 0000000000000000000000000000000000000000..92d98d1efb0dd0232846f324d435bdf7f2ec9ea0 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh @@ -0,0 +1,53 @@ +#!/bin/bash + + +# 100%, 99%, 98%, 97%, 96%, 95% of [3.51, 4.25] +# MXQ2=(minibatch 3.51 3.47 3.44 3.40 3.37 3.33 4.25 4.21 4.17 4.12 4.08 4.04) +MXQ2=(minibatch 6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07) +MXQ_BATCHES=(MXQ2) +declare -n MXQ_BATCH + +ATTEMPT="sensi-milp-mini2" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" +EXP_BASE_NAME="${ATTEMPT}" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" +# Setup data files directories for reporting +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + +weight_algo=sensi-milp + +for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do + batch_name=${MXQ_BATCH[@]:0:1} + EXP_NAME="${EXP_BASE_NAME}-${batch_name}" + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/${EXP_NAME}_stor + log_file="logs/bench-${EXP_NAME}-$(date +%Y%m%d%H%M%S).log" + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs ]; then + mkdir pdfs + fi + + # plot configuration allocations for 3 * 12 MXQ combinations + MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + BUDGETS=${MXQ_BATCH[@]:1} + for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --attempt1 mxq1 \ + --attempt2 $ATTEMPT \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh new file mode 100644 index 0000000000000000000000000000000000000000..59f43c68579e863a5fda735612778bb87f206686 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 1 \ + --algo awq \ + --config b4g32 b4g64 b4g128 \ + --experiment-name quant_llm_13B-awq2 \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..cec2fe1abcddb33f0ac5a195b14a4039056aa05a --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --config 2.57 2.55 2.53 2.51 2.49 2.47 2.45 2.31 2.29 2.27 2.25 2.23 2.21 2.19 2.17 2.15 2.13 \ + --experiment-name quant_llm_2bit_dense1-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..2b6838194b54d95a8f1b99a21e2bc0e06fdaf5f6 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --config 3.57 3.55 3.53 3.51 3.49 3.47 3.45 3.31 3.29 3.27 3.25 3.23 3.21 3.19 3.17 3.15 3.13 3.11 3.09 3.07 \ + --experiment-name quant_llm_3bit_dense1-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..f2832ed6e1921dfa08fd5c28d641d5ed8bdc9625 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 \ + --algo mxq \ + --config 4.51 \ + --experiment-name quant_llm_4_51-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..fe450dbfde5f7378fba45e63183d9d6986a1e4b9 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --config 4.61 4.59 4.57 4.55 4.53 4.51 4.49 4.47 4.45 4.43 4.41 4.35 4.33 4.31 4.29 4.27 4.25 4.23 4.21 4.19 4.17 4.15 4.13 4.11 4.09 4.07 4.05 4.03 \ + --experiment-name quant_llm_4bit_dense1-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh new file mode 100644 index 0000000000000000000000000000000000000000..f32305f7a7b2e50eb0ff0c53c9263a1584c562a6 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo gptq \ + --config b8g32 b8g64 b8g128 \ + --experiment-name quant_llm-8bit-gptq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-awq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-awq.sh new file mode 100644 index 0000000000000000000000000000000000000000..2c378867af209dd6467618c56fb1e22dbdda2204 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-awq.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 1 \ + --algo awq \ + --config b4g32 b4g64 b4g128 \ + --experiment-name quant_llm_13B-awq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..dcd7dd749d655914857c6af8179f6d2aa85f3be2 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --config 4.51 4.25 4.13 \ + --experiment-name quant_llm_b4-mxq2 \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-gptq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-gptq.sh new file mode 100644 index 0000000000000000000000000000000000000000..7f87069f1e6d8c72f6d4c054e72f713d23f9c7c3 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-gptq.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/lm-quant-toolkit/src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo gptq \ + --experiment-name quant_llm-gptq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-hqq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-hqq.sh new file mode 100644 index 0000000000000000000000000000000000000000..7cca1ed873d34d01908446e11a129a49079aa29e --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-hqq.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo hqq \ + --experiment-name quant_llm-hqq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd3b388fefb627d31153a7d41b75e96ca06cb5dd --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 1 \ + --algo mxq \ + --config 4.51 4.25 4.13 \ + --experiment-name quant_llm-weighted-113b-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots-kurt" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-mxq.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..8dc71db02c8bb95d74827dbff6f7e51bb8e685c9 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-mxq.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --experiment-name quant_llm-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-allot.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-allot.sh new file mode 100644 index 0000000000000000000000000000000000000000..ef109cae40e5c37ee5ff7bdc122f0eb04b457eaf --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-allot.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" + +TAILS=(tail-prioritized 2.00 4.00) +# MXQ_BATCHES=(HEADS TAILS) +MXQ_BATCHES=(TAILS) +declare -n MXQ_BATCH + +EXP_BASE_NAME="tail-factor-search" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME + +for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do + weight_algo=${MXQ_BATCH[@]:0:1} + for factor in ${MXQ_BATCH[@]:1}; do + ATTEMPT="${weight_algo}_${factor/./_}" + EXP_NAME="${ATTEMPT}" + log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + + # python -m pdb ../src/cli.py dump \ + python ../src/cli.py dump \ + --type quant_config_sim \ + --model 0 1 2 \ + --weight-algo $weight_algo \ + --factor $factor \ + --budget ${BUDGETS} \ + --output-file="$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/${EXP_NAME}.csv"\ + 2>&1 \ + | tee -a $log_file + + done + +done + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" +for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 tail-prioritized_2_00 \ + --attempt2 tail-prioritized_4_00 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-milp.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-milp.sh new file mode 100644 index 0000000000000000000000000000000000000000..14b11a6ec487ac6599f8d73dcdd73f09b28a7dc5 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-milp.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# python ../src/lm_quant_toolkit/eval/common.py > 2-8-quant-bits.txt +python ../src/lm_quant_toolkit/eval/common.py > 2-8-quant-bits-sensi.txt diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh new file mode 100644 index 0000000000000000000000000000000000000000..8edd3705bdddab5e99323d5f82c64549b3deb968 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +weight_algo="tail_boost" +EXP_BASE_NAME="$weight_algo" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME + +ATTEMPT="$weight_algo" +EXP_NAME="${ATTEMPT}" +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + +# python -m pdb ../src/cli.py dump \ +python ../src/cli.py dump \ + --type quant_config_sim \ + --model 0 1 2 \ + --weight-algo $weight_algo \ + --budget ${BUDGETS} \ + --output-file="$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/${EXP_NAME}.csv"\ + 2>&1 \ + | tee -a $log_file + + +OLD_DIR=$(pwd) +cd $RESULT_DIR/$EXP_BASE_NAME +if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot +fi +$OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data +# plot configuration allocations for 3 * 12 MXQ combinations +MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" +for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $weight_algo \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done +done diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-CLIP-ViT-B-32-laion2B-s34B-b79K.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-CLIP-ViT-B-32-laion2B-s34B-b79K.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff2f5c6ad3d955758de1c09fd3522aff9379e6fe --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-CLIP-ViT-B-32-laion2B-s34B-b79K.csv @@ -0,0 +1,577 @@ +layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params,kurtosis +0,vision.mlp.c_fc,2,32,8,128,9.128504753112791,0.705322265625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,2,64,8,128,10.76142692565918,0.6339111328125,2359296,7.5647333557116845 +0,vision.mlp.c_fc,2,128,8,128,12.15142059326172,0.59820556640625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,3,32,8,128,4.122710227966309,0.986572265625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,3,64,8,128,5.038382530212402,0.9151611328125,2359296,7.5647333557116845 +0,vision.mlp.c_fc,3,128,8,128,5.910652160644531,0.87945556640625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,4,32,8,128,1.938106060028076,1.267822265625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,4,64,8,128,2.3859243392944336,1.1964111328125,2359296,7.5647333557116845 +0,vision.mlp.c_fc,4,128,8,128,2.824324131011963,1.16070556640625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,8,32,8,128,0.11737483739852904,2.392822265625,2359296,7.5647333557116845 +0,vision.mlp.c_fc,8,64,8,128,0.14461086690425873,2.3214111328125,2359296,7.5647333557116845 +0,vision.mlp.c_fc,8,128,8,128,0.171279177069664,2.28570556640625,2359296,7.5647333557116845 +0,vision.mlp.c_proj,2,32,8,128,6.839951992034912,0.705322265625,2359296,8.024042788345657 +0,vision.mlp.c_proj,2,64,8,128,8.144723892211914,0.6339111328125,2359296,8.024042788345657 +0,vision.mlp.c_proj,2,128,8,128,9.282066345214844,0.59820556640625,2359296,8.024042788345657 +0,vision.mlp.c_proj,3,32,8,128,2.976739406585694,0.986572265625,2359296,8.024042788345657 +0,vision.mlp.c_proj,3,64,8,128,3.607946395874023,0.9151611328125,2359296,8.024042788345657 +0,vision.mlp.c_proj,3,128,8,128,4.226345539093018,0.87945556640625,2359296,8.024042788345657 +0,vision.mlp.c_proj,4,32,8,128,1.3918942213058472,1.267822265625,2359296,8.024042788345657 +0,vision.mlp.c_proj,4,64,8,128,1.6903578042984009,1.1964111328125,2359296,8.024042788345657 +0,vision.mlp.c_proj,4,128,8,128,1.9883592128753664,1.16070556640625,2359296,8.024042788345657 +0,vision.mlp.c_proj,8,32,8,128,0.0842554122209549,2.392822265625,2359296,8.024042788345657 +0,vision.mlp.c_proj,8,64,8,128,0.10231027752161026,2.3214111328125,2359296,8.024042788345657 +0,vision.mlp.c_proj,8,128,8,128,0.11994849145412444,2.28570556640625,2359296,8.024042788345657 +1,vision.mlp.c_fc,2,32,8,128,9.103703498840332,0.705322265625,2359296,5.556482388585281 +1,vision.mlp.c_fc,2,64,8,128,10.854814529418944,0.6339111328125,2359296,5.556482388585281 +1,vision.mlp.c_fc,2,128,8,128,12.433868408203123,0.59820556640625,2359296,5.556482388585281 +1,vision.mlp.c_fc,3,32,8,128,4.004519939422607,0.986572265625,2359296,5.556482388585281 +1,vision.mlp.c_fc,3,64,8,128,4.869235992431641,0.9151611328125,2359296,5.556482388585281 +1,vision.mlp.c_fc,3,128,8,128,5.696018218994141,0.87945556640625,2359296,5.556482388585281 +1,vision.mlp.c_fc,4,32,8,128,1.875130295753479,1.267822265625,2359296,5.556482388585281 +1,vision.mlp.c_fc,4,64,8,128,2.288634777069092,1.1964111328125,2359296,5.556482388585281 +1,vision.mlp.c_fc,4,128,8,128,2.692777395248413,1.16070556640625,2359296,5.556482388585281 +1,vision.mlp.c_fc,8,32,8,128,0.1136341467499733,2.392822265625,2359296,5.556482388585281 +1,vision.mlp.c_fc,8,64,8,128,0.13860879838466644,2.3214111328125,2359296,5.556482388585281 +1,vision.mlp.c_fc,8,128,8,128,0.16262513399124146,2.28570556640625,2359296,5.556482388585281 +1,vision.mlp.c_proj,2,32,8,128,6.911839962005615,0.705322265625,2359296,10.127617353490804 +1,vision.mlp.c_proj,2,64,8,128,8.261187553405762,0.6339111328125,2359296,10.127617353490804 +1,vision.mlp.c_proj,2,128,8,128,9.480621337890623,0.59820556640625,2359296,10.127617353490804 +1,vision.mlp.c_proj,3,32,8,128,2.991220712661743,0.986572265625,2359296,10.127617353490804 +1,vision.mlp.c_proj,3,64,8,128,3.6111984252929688,0.9151611328125,2359296,10.127617353490804 +1,vision.mlp.c_proj,3,128,8,128,4.218574523925781,0.87945556640625,2359296,10.127617353490804 +1,vision.mlp.c_proj,4,32,8,128,1.3967558145523071,1.267822265625,2359296,10.127617353490804 +1,vision.mlp.c_proj,4,64,8,128,1.6911554336547852,1.1964111328125,2359296,10.127617353490804 +1,vision.mlp.c_proj,4,128,8,128,1.983748197555542,1.16070556640625,2359296,10.127617353490804 +1,vision.mlp.c_proj,8,32,8,128,0.08465902507305145,2.392822265625,2359296,10.127617353490804 +1,vision.mlp.c_proj,8,64,8,128,0.10233005881309508,2.3214111328125,2359296,10.127617353490804 +1,vision.mlp.c_proj,8,128,8,128,0.11965607851743698,2.28570556640625,2359296,10.127617353490804 +2,vision.mlp.c_fc,2,32,8,128,8.82172966003418,0.705322265625,2359296,4.752866879979445 +2,vision.mlp.c_fc,2,64,8,128,10.50925350189209,0.6339111328125,2359296,4.752866879979445 +2,vision.mlp.c_fc,2,128,8,128,12.03489589691162,0.59820556640625,2359296,4.752866879979445 +2,vision.mlp.c_fc,3,32,8,128,3.823518753051758,0.986572265625,2359296,4.752866879979445 +2,vision.mlp.c_fc,3,64,8,128,4.6140289306640625,0.9151611328125,2359296,4.752866879979445 +2,vision.mlp.c_fc,3,128,8,128,5.374442100524902,0.87945556640625,2359296,4.752866879979445 +2,vision.mlp.c_fc,4,32,8,128,1.785937786102295,1.267822265625,2359296,4.752866879979445 +2,vision.mlp.c_fc,4,64,8,128,2.158376693725586,1.1964111328125,2359296,4.752866879979445 +2,vision.mlp.c_fc,4,128,8,128,2.523719310760498,1.16070556640625,2359296,4.752866879979445 +2,vision.mlp.c_fc,8,32,8,128,0.10810095071792604,2.392822265625,2359296,4.752866879979445 +2,vision.mlp.c_fc,8,64,8,128,0.13063114881515503,2.3214111328125,2359296,4.752866879979445 +2,vision.mlp.c_fc,8,128,8,128,0.15222015976905823,2.28570556640625,2359296,4.752866879979445 +2,vision.mlp.c_proj,2,32,8,128,6.639797687530518,0.705322265625,2359296,5.964957611759415 +2,vision.mlp.c_proj,2,64,8,128,7.880718231201172,0.6339111328125,2359296,5.964957611759415 +2,vision.mlp.c_proj,2,128,8,128,9.019220352172852,0.59820556640625,2359296,5.964957611759415 +2,vision.mlp.c_proj,3,32,8,128,2.859175443649292,0.986572265625,2359296,5.964957611759415 +2,vision.mlp.c_proj,3,64,8,128,3.413665294647217,0.9151611328125,2359296,5.964957611759415 +2,vision.mlp.c_proj,3,128,8,128,3.95273756980896,0.87945556640625,2359296,5.964957611759415 +2,vision.mlp.c_proj,4,32,8,128,1.333265781402588,1.267822265625,2359296,5.964957611759415 +2,vision.mlp.c_proj,4,64,8,128,1.5933430194854736,1.1964111328125,2359296,5.964957611759415 +2,vision.mlp.c_proj,4,128,8,128,1.8488850593566897,1.16070556640625,2359296,5.964957611759415 +2,vision.mlp.c_proj,8,32,8,128,0.08075807988643646,2.392822265625,2359296,5.964957611759415 +2,vision.mlp.c_proj,8,64,8,128,0.09648673981428146,2.3214111328125,2359296,5.964957611759415 +2,vision.mlp.c_proj,8,128,8,128,0.11135073751211166,2.28570556640625,2359296,5.964957611759415 +3,vision.mlp.c_fc,2,32,8,128,8.47494125366211,0.705322265625,2359296,4.378746321591061 +3,vision.mlp.c_fc,2,64,8,128,10.082805633544922,0.6339111328125,2359296,4.378746321591061 +3,vision.mlp.c_fc,2,128,8,128,11.549962043762209,0.59820556640625,2359296,4.378746321591061 +3,vision.mlp.c_fc,3,32,8,128,3.6546263694763184,0.986572265625,2359296,4.378746321591061 +3,vision.mlp.c_fc,3,64,8,128,4.391681671142578,0.9151611328125,2359296,4.378746321591061 +3,vision.mlp.c_fc,3,128,8,128,5.095606327056885,0.87945556640625,2359296,4.378746321591061 +3,vision.mlp.c_fc,4,32,8,128,1.706535577774048,1.267822265625,2359296,4.378746321591061 +3,vision.mlp.c_fc,4,64,8,128,2.050750970840454,1.1964111328125,2359296,4.378746321591061 +3,vision.mlp.c_fc,4,128,8,128,2.3856451511383057,1.16070556640625,2359296,4.378746321591061 +3,vision.mlp.c_fc,8,32,8,128,0.10344040393829346,2.392822265625,2359296,4.378746321591061 +3,vision.mlp.c_fc,8,64,8,128,0.12399496138095856,2.3214111328125,2359296,4.378746321591061 +3,vision.mlp.c_fc,8,128,8,128,0.1439141482114792,2.28570556640625,2359296,4.378746321591061 +3,vision.mlp.c_proj,2,32,8,128,6.277030944824219,0.705322265625,2359296,5.006922011163274 +3,vision.mlp.c_proj,2,64,8,128,7.414824485778809,0.6339111328125,2359296,5.006922011163274 +3,vision.mlp.c_proj,2,128,8,128,8.46260929107666,0.59820556640625,2359296,5.006922011163274 +3,vision.mlp.c_proj,3,32,8,128,2.693903923034668,0.986572265625,2359296,5.006922011163274 +3,vision.mlp.c_proj,3,64,8,128,3.1912341117858887,0.9151611328125,2359296,5.006922011163274 +3,vision.mlp.c_proj,3,128,8,128,3.6711747646331783,0.87945556640625,2359296,5.006922011163274 +3,vision.mlp.c_proj,4,32,8,128,1.256372094154358,1.267822265625,2359296,5.006922011163274 +3,vision.mlp.c_proj,4,64,8,128,1.489571452140808,1.1964111328125,2359296,5.006922011163274 +3,vision.mlp.c_proj,4,128,8,128,1.7132800817489624,1.16070556640625,2359296,5.006922011163274 +3,vision.mlp.c_proj,8,32,8,128,0.0760984867811203,2.392822265625,2359296,5.006922011163274 +3,vision.mlp.c_proj,8,64,8,128,0.09016053378582,2.3214111328125,2359296,5.006922011163274 +3,vision.mlp.c_proj,8,128,8,128,0.10330581665039062,2.28570556640625,2359296,5.006922011163274 +4,vision.mlp.c_fc,2,32,8,128,8.274068832397461,0.705322265625,2359296,3.918504223899523 +4,vision.mlp.c_fc,2,64,8,128,9.78947639465332,0.6339111328125,2359296,3.918504223899523 +4,vision.mlp.c_fc,2,128,8,128,11.202027320861816,0.59820556640625,2359296,3.918504223899523 +4,vision.mlp.c_fc,3,32,8,128,3.5570082664489746,0.986572265625,2359296,3.918504223899523 +4,vision.mlp.c_fc,3,64,8,128,4.2356367111206055,0.9151611328125,2359296,3.918504223899523 +4,vision.mlp.c_fc,3,128,8,128,4.884209632873535,0.87945556640625,2359296,3.918504223899523 +4,vision.mlp.c_fc,4,32,8,128,1.6616051197052002,1.267822265625,2359296,3.918504223899523 +4,vision.mlp.c_fc,4,64,8,128,1.979529857635498,1.1964111328125,2359296,3.918504223899523 +4,vision.mlp.c_fc,4,128,8,128,2.284747838973999,1.16070556640625,2359296,3.918504223899523 +4,vision.mlp.c_fc,8,32,8,128,0.10057099163532256,2.392822265625,2359296,3.918504223899523 +4,vision.mlp.c_fc,8,64,8,128,0.11972232908010484,2.3214111328125,2359296,3.918504223899523 +4,vision.mlp.c_fc,8,128,8,128,0.13779735565185547,2.28570556640625,2359296,3.918504223899523 +4,vision.mlp.c_proj,2,32,8,128,6.439798831939697,0.705322265625,2359296,4.362240481953023 +4,vision.mlp.c_proj,2,64,8,128,7.564638137817383,0.6339111328125,2359296,4.362240481953023 +4,vision.mlp.c_proj,2,128,8,128,8.594568252563477,0.59820556640625,2359296,4.362240481953023 +4,vision.mlp.c_proj,3,32,8,128,2.7584915161132812,0.986572265625,2359296,4.362240481953023 +4,vision.mlp.c_proj,3,64,8,128,3.251418113708496,0.9151611328125,2359296,4.362240481953023 +4,vision.mlp.c_proj,3,128,8,128,3.716466665267944,0.87945556640625,2359296,4.362240481953023 +4,vision.mlp.c_proj,4,32,8,128,1.2862253189086914,1.267822265625,2359296,4.362240481953023 +4,vision.mlp.c_proj,4,64,8,128,1.516956448554993,1.1964111328125,2359296,4.362240481953023 +4,vision.mlp.c_proj,4,128,8,128,1.7344794273376465,1.16070556640625,2359296,4.362240481953023 +4,vision.mlp.c_proj,8,32,8,128,0.07793856412172318,2.392822265625,2359296,4.362240481953023 +4,vision.mlp.c_proj,8,64,8,128,0.0918104276061058,2.3214111328125,2359296,4.362240481953023 +4,vision.mlp.c_proj,8,128,8,128,0.10458982735872269,2.28570556640625,2359296,4.362240481953023 +5,vision.mlp.c_fc,2,32,8,128,8.392443656921387,0.705322265625,2359296,3.714197056179566 +5,vision.mlp.c_fc,2,64,8,128,9.947240829467772,0.6339111328125,2359296,3.714197056179566 +5,vision.mlp.c_fc,2,128,8,128,11.396702766418455,0.59820556640625,2359296,3.714197056179566 +5,vision.mlp.c_fc,3,32,8,128,3.599330425262451,0.986572265625,2359296,3.714197056179566 +5,vision.mlp.c_fc,3,64,8,128,4.279275417327881,0.9151611328125,2359296,3.714197056179566 +5,vision.mlp.c_fc,3,128,8,128,4.932392597198486,0.87945556640625,2359296,3.714197056179566 +5,vision.mlp.c_fc,4,32,8,128,1.6796408891677856,1.267822265625,2359296,3.714197056179566 +5,vision.mlp.c_fc,4,64,8,128,1.997564435005188,1.1964111328125,2359296,3.714197056179566 +5,vision.mlp.c_fc,4,128,8,128,2.30387020111084,1.16070556640625,2359296,3.714197056179566 +5,vision.mlp.c_fc,8,32,8,128,0.10184947401285172,2.392822265625,2359296,3.714197056179566 +5,vision.mlp.c_fc,8,64,8,128,0.12093502283096312,2.3214111328125,2359296,3.714197056179566 +5,vision.mlp.c_fc,8,128,8,128,0.13896310329437256,2.28570556640625,2359296,3.714197056179566 +5,vision.mlp.c_proj,2,32,8,128,6.786053657531738,0.705322265625,2359296,4.504410342372665 +5,vision.mlp.c_proj,2,64,8,128,7.957103252410889,0.6339111328125,2359296,4.504410342372665 +5,vision.mlp.c_proj,2,128,8,128,9.033921241760254,0.59820556640625,2359296,4.504410342372665 +5,vision.mlp.c_proj,3,32,8,128,2.905226469039917,0.986572265625,2359296,4.504410342372665 +5,vision.mlp.c_proj,3,64,8,128,3.4214489459991455,0.9151611328125,2359296,4.504410342372665 +5,vision.mlp.c_proj,3,128,8,128,3.9075539112091064,0.87945556640625,2359296,4.504410342372665 +5,vision.mlp.c_proj,4,32,8,128,1.355237364768982,1.267822265625,2359296,4.504410342372665 +5,vision.mlp.c_proj,4,64,8,128,1.5957733392715454,1.1964111328125,2359296,4.504410342372665 +5,vision.mlp.c_proj,4,128,8,128,1.8257287740707395,1.16070556640625,2359296,4.504410342372665 +5,vision.mlp.c_proj,8,32,8,128,0.0820372924208641,2.392822265625,2359296,4.504410342372665 +5,vision.mlp.c_proj,8,64,8,128,0.09651072323322296,2.3214111328125,2359296,4.504410342372665 +5,vision.mlp.c_proj,8,128,8,128,0.10990241914987564,2.28570556640625,2359296,4.504410342372665 +6,vision.mlp.c_fc,2,32,8,128,8.521951675415039,0.705322265625,2359296,3.614600960861904 +6,vision.mlp.c_fc,2,64,8,128,10.0691499710083,0.6339111328125,2359296,3.614600960861904 +6,vision.mlp.c_fc,2,128,8,128,11.54863739013672,0.59820556640625,2359296,3.614600960861904 +6,vision.mlp.c_fc,3,32,8,128,3.649571895599365,0.986572265625,2359296,3.614600960861904 +6,vision.mlp.c_fc,3,64,8,128,4.324981689453125,0.9151611328125,2359296,3.614600960861904 +6,vision.mlp.c_fc,3,128,8,128,4.97314977645874,0.87945556640625,2359296,3.614600960861904 +6,vision.mlp.c_fc,4,32,8,128,1.7034014463424685,1.267822265625,2359296,3.614600960861904 +6,vision.mlp.c_fc,4,64,8,128,2.0181891918182373,1.1964111328125,2359296,3.614600960861904 +6,vision.mlp.c_fc,4,128,8,128,2.3228890895843506,1.16070556640625,2359296,3.614600960861904 +6,vision.mlp.c_fc,8,32,8,128,0.10322079807519911,2.392822265625,2359296,3.614600960861904 +6,vision.mlp.c_fc,8,64,8,128,0.12220923602581024,2.3214111328125,2359296,3.614600960861904 +6,vision.mlp.c_fc,8,128,8,128,0.14006850123405457,2.28570556640625,2359296,3.614600960861904 +6,vision.mlp.c_proj,2,32,8,128,7.034173488616943,0.705322265625,2359296,3.703802696792269 +6,vision.mlp.c_proj,2,64,8,128,8.234294891357422,0.6339111328125,2359296,3.703802696792269 +6,vision.mlp.c_proj,2,128,8,128,9.333321571350098,0.59820556640625,2359296,3.703802696792269 +6,vision.mlp.c_proj,3,32,8,128,3.009636878967285,0.986572265625,2359296,3.703802696792269 +6,vision.mlp.c_proj,3,64,8,128,3.5273571014404297,0.9151611328125,2359296,3.703802696792269 +6,vision.mlp.c_proj,3,128,8,128,4.010415077209473,0.87945556640625,2359296,3.703802696792269 +6,vision.mlp.c_proj,4,32,8,128,1.404062271118164,1.267822265625,2359296,3.703802696792269 +6,vision.mlp.c_proj,4,64,8,128,1.6456388235092163,1.1964111328125,2359296,3.703802696792269 +6,vision.mlp.c_proj,4,128,8,128,1.8724839687347412,1.16070556640625,2359296,3.703802696792269 +6,vision.mlp.c_proj,8,32,8,128,0.08510661125183105,2.392822265625,2359296,3.703802696792269 +6,vision.mlp.c_proj,8,64,8,128,0.09971028566360474,2.3214111328125,2359296,3.703802696792269 +6,vision.mlp.c_proj,8,128,8,128,0.1129401922225952,2.28570556640625,2359296,3.703802696792269 +7,vision.mlp.c_fc,2,32,8,128,8.388879776000977,0.705322265625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,2,64,8,128,9.876380920410156,0.6339111328125,2359296,3.4103181441120247 +7,vision.mlp.c_fc,2,128,8,128,11.25124740600586,0.59820556640625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,3,32,8,128,3.5898404121398926,0.986572265625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,3,64,8,128,4.232213020324707,0.9151611328125,2359296,3.4103181441120247 +7,vision.mlp.c_fc,3,128,8,128,4.836949348449707,0.87945556640625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,4,32,8,128,1.674543380737305,1.267822265625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,4,64,8,128,1.9748677015304563,1.1964111328125,2359296,3.4103181441120247 +7,vision.mlp.c_fc,4,128,8,128,2.2568445205688477,1.16070556640625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,8,32,8,128,0.10150232911109924,2.392822265625,2359296,3.4103181441120247 +7,vision.mlp.c_fc,8,64,8,128,0.11956389248371124,2.3214111328125,2359296,3.4103181441120247 +7,vision.mlp.c_fc,8,128,8,128,0.1361236423254013,2.28570556640625,2359296,3.4103181441120247 +7,vision.mlp.c_proj,2,32,8,128,7.932621479034424,0.705322265625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,2,64,8,128,9.285072326660156,0.6339111328125,2359296,3.7265741474392886 +7,vision.mlp.c_proj,2,128,8,128,10.539323806762695,0.59820556640625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,3,32,8,128,3.394057273864746,0.986572265625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,3,64,8,128,3.9794647693634033,0.9151611328125,2359296,3.7265741474392886 +7,vision.mlp.c_proj,3,128,8,128,4.525967597961426,0.87945556640625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,4,32,8,128,1.583614945411682,1.267822265625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,4,64,8,128,1.8572959899902344,1.1964111328125,2359296,3.7265741474392886 +7,vision.mlp.c_proj,4,128,8,128,2.113091468811035,1.16070556640625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,8,32,8,128,0.09593687206506728,2.392822265625,2359296,3.7265741474392886 +7,vision.mlp.c_proj,8,64,8,128,0.11239387094974518,2.3214111328125,2359296,3.7265741474392886 +7,vision.mlp.c_proj,8,128,8,128,0.12745003402233124,2.28570556640625,2359296,3.7265741474392886 +8,vision.mlp.c_fc,2,32,8,128,8.285651206970215,0.705322265625,2359296,3.734153132112317 +8,vision.mlp.c_fc,2,64,8,128,9.778728485107422,0.6339111328125,2359296,3.734153132112317 +8,vision.mlp.c_fc,2,128,8,128,11.152859687805176,0.59820556640625,2359296,3.734153132112317 +8,vision.mlp.c_fc,3,32,8,128,3.547513008117676,0.986572265625,2359296,3.734153132112317 +8,vision.mlp.c_fc,3,64,8,128,4.191949367523193,0.9151611328125,2359296,3.734153132112317 +8,vision.mlp.c_fc,3,128,8,128,4.793965816497803,0.87945556640625,2359296,3.734153132112317 +8,vision.mlp.c_fc,4,32,8,128,1.6551411151885986,1.267822265625,2359296,3.734153132112317 +8,vision.mlp.c_fc,4,64,8,128,1.956448316574097,1.1964111328125,2359296,3.734153132112317 +8,vision.mlp.c_fc,4,128,8,128,2.2382309436798096,1.16070556640625,2359296,3.734153132112317 +8,vision.mlp.c_fc,8,32,8,128,0.100438229739666,2.392822265625,2359296,3.734153132112317 +8,vision.mlp.c_fc,8,64,8,128,0.11862816661596298,2.3214111328125,2359296,3.734153132112317 +8,vision.mlp.c_fc,8,128,8,128,0.13508424162864685,2.28570556640625,2359296,3.734153132112317 +8,vision.mlp.c_proj,2,32,8,128,8.032468795776367,0.705322265625,2359296,3.770444941283215 +8,vision.mlp.c_proj,2,64,8,128,9.38332176208496,0.6339111328125,2359296,3.770444941283215 +8,vision.mlp.c_proj,2,128,8,128,10.626753807067873,0.59820556640625,2359296,3.770444941283215 +8,vision.mlp.c_proj,3,32,8,128,3.433583498001098,0.986572265625,2359296,3.770444941283215 +8,vision.mlp.c_proj,3,64,8,128,4.0180745124816895,0.9151611328125,2359296,3.770444941283215 +8,vision.mlp.c_proj,3,128,8,128,4.556089878082275,0.87945556640625,2359296,3.770444941283215 +8,vision.mlp.c_proj,4,32,8,128,1.602971076965332,1.267822265625,2359296,3.770444941283215 +8,vision.mlp.c_proj,4,64,8,128,1.875213384628296,1.1964111328125,2359296,3.770444941283215 +8,vision.mlp.c_proj,4,128,8,128,2.125036239624023,1.16070556640625,2359296,3.770444941283215 +8,vision.mlp.c_proj,8,32,8,128,0.09719990193843842,2.392822265625,2359296,3.770444941283215 +8,vision.mlp.c_proj,8,64,8,128,0.1135389283299446,2.3214111328125,2359296,3.770444941283215 +8,vision.mlp.c_proj,8,128,8,128,0.1282404214143753,2.28570556640625,2359296,3.770444941283215 +9,vision.mlp.c_fc,2,32,8,128,8.818232536315918,0.705322265625,2359296,3.406923087940011 +9,vision.mlp.c_fc,2,64,8,128,10.373499870300291,0.6339111328125,2359296,3.406923087940011 +9,vision.mlp.c_fc,2,128,8,128,11.801898002624512,0.59820556640625,2359296,3.406923087940011 +9,vision.mlp.c_fc,3,32,8,128,3.7690412998199463,0.986572265625,2359296,3.406923087940011 +9,vision.mlp.c_fc,3,64,8,128,4.43658447265625,0.9151611328125,2359296,3.406923087940011 +9,vision.mlp.c_fc,3,128,8,128,5.0595269203186035,0.87945556640625,2359296,3.406923087940011 +9,vision.mlp.c_fc,4,32,8,128,1.759302854537964,1.267822265625,2359296,3.406923087940011 +9,vision.mlp.c_fc,4,64,8,128,2.071089506149292,1.1964111328125,2359296,3.406923087940011 +9,vision.mlp.c_fc,4,128,8,128,2.3599820137023926,1.16070556640625,2359296,3.406923087940011 +9,vision.mlp.c_fc,8,32,8,128,0.10690240561962128,2.392822265625,2359296,3.406923087940011 +9,vision.mlp.c_fc,8,64,8,128,0.1255631297826767,2.3214111328125,2359296,3.406923087940011 +9,vision.mlp.c_fc,8,128,8,128,0.14248855412006378,2.28570556640625,2359296,3.406923087940011 +9,vision.mlp.c_proj,2,32,8,128,9.092228889465332,0.705322265625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,2,64,8,128,10.604211807250977,0.6339111328125,2359296,3.7473560752962425 +9,vision.mlp.c_proj,2,128,8,128,11.983688354492188,0.59820556640625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,3,32,8,128,3.8891754150390625,0.986572265625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,3,64,8,128,4.537405967712402,0.9151611328125,2359296,3.7473560752962425 +9,vision.mlp.c_proj,3,128,8,128,5.127986431121826,0.87945556640625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,4,32,8,128,1.8149250745773315,1.267822265625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,4,64,8,128,2.1180083751678467,1.1964111328125,2359296,3.7473560752962425 +9,vision.mlp.c_proj,4,128,8,128,2.3949661254882812,1.16070556640625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,8,32,8,128,0.10990245640277864,2.392822265625,2359296,3.7473560752962425 +9,vision.mlp.c_proj,8,64,8,128,0.12820811569690704,2.3214111328125,2359296,3.7473560752962425 +9,vision.mlp.c_proj,8,128,8,128,0.14443348348140717,2.28570556640625,2359296,3.7473560752962425 +10,vision.mlp.c_fc,2,32,8,128,9.356571197509766,0.705322265625,2359296,3.506843786702064 +10,vision.mlp.c_fc,2,64,8,128,10.991547584533691,0.6339111328125,2359296,3.506843786702064 +10,vision.mlp.c_fc,2,128,8,128,12.464940071105955,0.59820556640625,2359296,3.506843786702064 +10,vision.mlp.c_fc,3,32,8,128,3.9996769428253174,0.986572265625,2359296,3.506843786702064 +10,vision.mlp.c_fc,3,64,8,128,4.703681468963623,0.9151611328125,2359296,3.506843786702064 +10,vision.mlp.c_fc,3,128,8,128,5.345017910003662,0.87945556640625,2359296,3.506843786702064 +10,vision.mlp.c_fc,4,32,8,128,1.865874171257019,1.267822265625,2359296,3.506843786702064 +10,vision.mlp.c_fc,4,64,8,128,2.195168972015381,1.1964111328125,2359296,3.506843786702064 +10,vision.mlp.c_fc,4,128,8,128,2.495924472808838,1.16070556640625,2359296,3.506843786702064 +10,vision.mlp.c_fc,8,32,8,128,0.11316034197807312,2.392822265625,2359296,3.506843786702064 +10,vision.mlp.c_fc,8,64,8,128,0.13293932378292084,2.3214111328125,2359296,3.506843786702064 +10,vision.mlp.c_fc,8,128,8,128,0.15064848959445953,2.28570556640625,2359296,3.506843786702064 +10,vision.mlp.c_proj,2,32,8,128,9.723986625671388,0.705322265625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,2,64,8,128,11.35032558441162,0.6339111328125,2359296,3.7916828549200345 +10,vision.mlp.c_proj,2,128,8,128,12.832988739013672,0.59820556640625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,3,32,8,128,4.1578369140625,0.986572265625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,3,64,8,128,4.862079620361328,0.9151611328125,2359296,3.7916828549200345 +10,vision.mlp.c_proj,3,128,8,128,5.495484352111816,0.87945556640625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,4,32,8,128,1.9391449689865112,1.267822265625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,4,64,8,128,2.2657644748687744,1.1964111328125,2359296,3.7916828549200345 +10,vision.mlp.c_proj,4,128,8,128,2.5652666091918945,1.16070556640625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,8,32,8,128,0.11764831840991974,2.392822265625,2359296,3.7916828549200345 +10,vision.mlp.c_proj,8,64,8,128,0.13722199201583862,2.3214111328125,2359296,3.7916828549200345 +10,vision.mlp.c_proj,8,128,8,128,0.1546991765499115,2.28570556640625,2359296,3.7916828549200345 +11,vision.mlp.c_fc,2,32,8,128,8.082159042358398,0.705322265625,2359296,3.336296405622262 +11,vision.mlp.c_fc,2,64,8,128,9.489198684692385,0.6339111328125,2359296,3.336296405622262 +11,vision.mlp.c_fc,2,128,8,128,10.802061080932615,0.59820556640625,2359296,3.336296405622262 +11,vision.mlp.c_fc,3,32,8,128,3.4594030380249023,0.986572265625,2359296,3.336296405622262 +11,vision.mlp.c_fc,3,64,8,128,4.068918704986572,0.9151611328125,2359296,3.336296405622262 +11,vision.mlp.c_fc,3,128,8,128,4.641410827636719,0.87945556640625,2359296,3.336296405622262 +11,vision.mlp.c_fc,4,32,8,128,1.6126223802566528,1.267822265625,2359296,3.336296405622262 +11,vision.mlp.c_fc,4,64,8,128,1.8982433080673216,1.1964111328125,2359296,3.336296405622262 +11,vision.mlp.c_fc,4,128,8,128,2.1648974418640137,1.16070556640625,2359296,3.336296405622262 +11,vision.mlp.c_fc,8,32,8,128,0.09783997386693954,2.392822265625,2359296,3.336296405622262 +11,vision.mlp.c_fc,8,64,8,128,0.11497506499290466,2.3214111328125,2359296,3.336296405622262 +11,vision.mlp.c_fc,8,128,8,128,0.1306662559509277,2.28570556640625,2359296,3.336296405622262 +11,vision.mlp.c_proj,2,32,8,128,7.7494797706604,0.705322265625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,2,64,8,128,9.04317855834961,0.6339111328125,2359296,3.5289485578856628 +11,vision.mlp.c_proj,2,128,8,128,10.23215389251709,0.59820556640625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,3,32,8,128,3.3150153160095215,0.986572265625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,3,64,8,128,3.8718862533569336,0.9151611328125,2359296,3.5289485578856628 +11,vision.mlp.c_proj,3,128,8,128,4.384607315063477,0.87945556640625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,4,32,8,128,1.545481562614441,1.267822265625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,4,64,8,128,1.806149125099182,1.1964111328125,2359296,3.5289485578856628 +11,vision.mlp.c_proj,4,128,8,128,2.0433690547943115,1.16070556640625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,8,32,8,128,0.0937107503414154,2.392822265625,2359296,3.5289485578856628 +11,vision.mlp.c_proj,8,64,8,128,0.10938263684511185,2.3214111328125,2359296,3.5289485578856628 +11,vision.mlp.c_proj,8,128,8,128,0.1232990100979805,2.28570556640625,2359296,3.5289485578856628 +0,text.mlp.c_fc,2,32,8,128,6.464750289916992,0.3134765625,1048576,18.89034464279892 +0,text.mlp.c_fc,2,64,8,128,7.523227214813232,0.28173828125,1048576,18.89034464279892 +0,text.mlp.c_fc,2,128,8,128,8.498074531555176,0.265869140625,1048576,18.89034464279892 +0,text.mlp.c_fc,3,32,8,128,2.7694246768951416,0.4384765625,1048576,18.89034464279892 +0,text.mlp.c_fc,3,64,8,128,3.2232933044433594,0.40673828125,1048576,18.89034464279892 +0,text.mlp.c_fc,3,128,8,128,3.6492912769317623,0.390869140625,1048576,18.89034464279892 +0,text.mlp.c_fc,4,32,8,128,1.2936551570892334,0.5634765625,1048576,18.89034464279892 +0,text.mlp.c_fc,4,64,8,128,1.5072178840637207,0.53173828125,1048576,18.89034464279892 +0,text.mlp.c_fc,4,128,8,128,1.7056944370269775,0.515869140625,1048576,18.89034464279892 +0,text.mlp.c_fc,8,32,8,128,0.07893156260251999,1.0634765625,1048576,18.89034464279892 +0,text.mlp.c_fc,8,64,8,128,0.09200920909643172,1.03173828125,1048576,18.89034464279892 +0,text.mlp.c_fc,8,128,8,128,0.10402266681194304,1.015869140625,1048576,18.89034464279892 +0,text.mlp.c_proj,2,32,8,128,4.214115142822266,0.3134765625,1048576,8.257733339213775 +0,text.mlp.c_proj,2,64,8,128,4.919820785522461,0.28173828125,1048576,8.257733339213775 +0,text.mlp.c_proj,2,128,8,128,5.572372913360596,0.265869140625,1048576,8.257733339213775 +0,text.mlp.c_proj,3,32,8,128,1.801061511039734,0.4384765625,1048576,8.257733339213775 +0,text.mlp.c_proj,3,64,8,128,2.1083173751831055,0.40673828125,1048576,8.257733339213775 +0,text.mlp.c_proj,3,128,8,128,2.391723871231079,0.390869140625,1048576,8.257733339213775 +0,text.mlp.c_proj,4,32,8,128,0.8409978151321411,0.5634765625,1048576,8.257733339213775 +0,text.mlp.c_proj,4,64,8,128,0.9834749698638916,0.53173828125,1048576,8.257733339213775 +0,text.mlp.c_proj,4,128,8,128,1.1154149770736694,0.515869140625,1048576,8.257733339213775 +0,text.mlp.c_proj,8,32,8,128,0.05123524367809296,1.0634765625,1048576,8.257733339213775 +0,text.mlp.c_proj,8,64,8,128,0.05990749597549438,1.03173828125,1048576,8.257733339213775 +0,text.mlp.c_proj,8,128,8,128,0.0680227056145668,1.015869140625,1048576,8.257733339213775 +1,text.mlp.c_fc,2,32,8,128,6.351968765258789,0.3134765625,1048576,3.0800089847603167 +1,text.mlp.c_fc,2,64,8,128,7.389888286590576,0.28173828125,1048576,3.0800089847603167 +1,text.mlp.c_fc,2,128,8,128,8.325387001037598,0.265869140625,1048576,3.0800089847603167 +1,text.mlp.c_fc,3,32,8,128,2.71328067779541,0.4384765625,1048576,3.0800089847603167 +1,text.mlp.c_fc,3,64,8,128,3.1635944843292236,0.40673828125,1048576,3.0800089847603167 +1,text.mlp.c_fc,3,128,8,128,3.561861991882324,0.390869140625,1048576,3.0800089847603167 +1,text.mlp.c_fc,4,32,8,128,1.266817927360535,0.5634765625,1048576,3.0800089847603167 +1,text.mlp.c_fc,4,64,8,128,1.4758033752441406,0.53173828125,1048576,3.0800089847603167 +1,text.mlp.c_fc,4,128,8,128,1.664614200592041,0.515869140625,1048576,3.0800089847603167 +1,text.mlp.c_fc,8,32,8,128,0.07674290984869003,1.0634765625,1048576,3.0800089847603167 +1,text.mlp.c_fc,8,64,8,128,0.08936163038015366,1.03173828125,1048576,3.0800089847603167 +1,text.mlp.c_fc,8,128,8,128,0.10043232887983322,1.015869140625,1048576,3.0800089847603167 +1,text.mlp.c_proj,2,32,8,128,4.492769718170166,0.3134765625,1048576,3.2167251445240903 +1,text.mlp.c_proj,2,64,8,128,5.24850606918335,0.28173828125,1048576,3.2167251445240903 +1,text.mlp.c_proj,2,128,8,128,5.924077033996582,0.265869140625,1048576,3.2167251445240903 +1,text.mlp.c_proj,3,32,8,128,1.9200174808502195,0.4384765625,1048576,3.2167251445240903 +1,text.mlp.c_proj,3,64,8,128,2.2457592487335205,0.40673828125,1048576,3.2167251445240903 +1,text.mlp.c_proj,3,128,8,128,2.539240598678589,0.390869140625,1048576,3.2167251445240903 +1,text.mlp.c_proj,4,32,8,128,0.8956782817840576,0.5634765625,1048576,3.2167251445240903 +1,text.mlp.c_proj,4,64,8,128,1.047521471977234,0.53173828125,1048576,3.2167251445240903 +1,text.mlp.c_proj,4,128,8,128,1.18568217754364,0.515869140625,1048576,3.2167251445240903 +1,text.mlp.c_proj,8,32,8,128,0.0543127991259098,1.0634765625,1048576,3.2167251445240903 +1,text.mlp.c_proj,8,64,8,128,0.06340169906616211,1.03173828125,1048576,3.2167251445240903 +1,text.mlp.c_proj,8,128,8,128,0.0714043602347374,1.015869140625,1048576,3.2167251445240903 +2,text.mlp.c_fc,2,32,8,128,6.2713623046875,0.3134765625,1048576,3.112510371474206 +2,text.mlp.c_fc,2,64,8,128,7.308136940002441,0.28173828125,1048576,3.112510371474206 +2,text.mlp.c_fc,2,128,8,128,8.229869842529297,0.265869140625,1048576,3.112510371474206 +2,text.mlp.c_fc,3,32,8,128,2.6817820072174072,0.4384765625,1048576,3.112510371474206 +2,text.mlp.c_fc,3,64,8,128,3.1245784759521484,0.40673828125,1048576,3.112510371474206 +2,text.mlp.c_fc,3,128,8,128,3.5175602436065674,0.390869140625,1048576,3.112510371474206 +2,text.mlp.c_fc,4,32,8,128,1.2500064373016355,0.5634765625,1048576,3.112510371474206 +2,text.mlp.c_fc,4,64,8,128,1.4578500986099243,0.53173828125,1048576,3.112510371474206 +2,text.mlp.c_fc,4,128,8,128,1.6433409452438354,0.515869140625,1048576,3.112510371474206 +2,text.mlp.c_fc,8,32,8,128,0.07576245814561844,1.0634765625,1048576,3.112510371474206 +2,text.mlp.c_fc,8,64,8,128,0.08829101175069809,1.03173828125,1048576,3.112510371474206 +2,text.mlp.c_fc,8,128,8,128,0.0992748960852623,1.015869140625,1048576,3.112510371474206 +2,text.mlp.c_proj,2,32,8,128,4.444961071014404,0.3134765625,1048576,3.292999208834084 +2,text.mlp.c_proj,2,64,8,128,5.180168628692627,0.28173828125,1048576,3.292999208834084 +2,text.mlp.c_proj,2,128,8,128,5.851198196411133,0.265869140625,1048576,3.292999208834084 +2,text.mlp.c_proj,3,32,8,128,1.899754285812378,0.4384765625,1048576,3.292999208834084 +2,text.mlp.c_proj,3,64,8,128,2.2183635234832764,0.40673828125,1048576,3.292999208834084 +2,text.mlp.c_proj,3,128,8,128,2.505136013031006,0.390869140625,1048576,3.292999208834084 +2,text.mlp.c_proj,4,32,8,128,0.8862735033035278,0.5634765625,1048576,3.292999208834084 +2,text.mlp.c_proj,4,64,8,128,1.0349557399749756,0.53173828125,1048576,3.292999208834084 +2,text.mlp.c_proj,4,128,8,128,1.1686416864395142,0.515869140625,1048576,3.292999208834084 +2,text.mlp.c_proj,8,32,8,128,0.05371652543544769,1.0634765625,1048576,3.292999208834084 +2,text.mlp.c_proj,8,64,8,128,0.06263848394155502,1.03173828125,1048576,3.292999208834084 +2,text.mlp.c_proj,8,128,8,128,0.07050774246454239,1.015869140625,1048576,3.292999208834084 +3,text.mlp.c_fc,2,32,8,128,6.199350357055664,0.3134765625,1048576,3.124703500598623 +3,text.mlp.c_fc,2,64,8,128,7.228718757629395,0.28173828125,1048576,3.124703500598623 +3,text.mlp.c_fc,2,128,8,128,8.150893211364746,0.265869140625,1048576,3.124703500598623 +3,text.mlp.c_fc,3,32,8,128,2.648998737335205,0.4384765625,1048576,3.124703500598623 +3,text.mlp.c_fc,3,64,8,128,3.0911576747894287,0.40673828125,1048576,3.124703500598623 +3,text.mlp.c_fc,3,128,8,128,3.488417863845825,0.390869140625,1048576,3.124703500598623 +3,text.mlp.c_fc,4,32,8,128,1.2359066009521484,0.5634765625,1048576,3.124703500598623 +3,text.mlp.c_fc,4,64,8,128,1.441812515258789,0.53173828125,1048576,3.124703500598623 +3,text.mlp.c_fc,4,128,8,128,1.6281919479370115,0.515869140625,1048576,3.124703500598623 +3,text.mlp.c_fc,8,32,8,128,0.07503717392683029,1.0634765625,1048576,3.124703500598623 +3,text.mlp.c_fc,8,64,8,128,0.0874534398317337,1.03173828125,1048576,3.124703500598623 +3,text.mlp.c_fc,8,128,8,128,0.09830143302679062,1.015869140625,1048576,3.124703500598623 +3,text.mlp.c_proj,2,32,8,128,4.45910120010376,0.3134765625,1048576,3.365812129846043 +3,text.mlp.c_proj,2,64,8,128,5.196355819702148,0.28173828125,1048576,3.365812129846043 +3,text.mlp.c_proj,2,128,8,128,5.872509479522705,0.265869140625,1048576,3.365812129846043 +3,text.mlp.c_proj,3,32,8,128,1.904448628425598,0.4384765625,1048576,3.365812129846043 +3,text.mlp.c_proj,3,64,8,128,2.2250046730041504,0.40673828125,1048576,3.365812129846043 +3,text.mlp.c_proj,3,128,8,128,2.5161800384521484,0.390869140625,1048576,3.365812129846043 +3,text.mlp.c_proj,4,32,8,128,0.8890355229377747,0.5634765625,1048576,3.365812129846043 +3,text.mlp.c_proj,4,64,8,128,1.0387694835662842,0.53173828125,1048576,3.365812129846043 +3,text.mlp.c_proj,4,128,8,128,1.1744872331619265,0.515869140625,1048576,3.365812129846043 +3,text.mlp.c_proj,8,32,8,128,0.05386906117200851,1.0634765625,1048576,3.365812129846043 +3,text.mlp.c_proj,8,64,8,128,0.06280947476625443,1.03173828125,1048576,3.365812129846043 +3,text.mlp.c_proj,8,128,8,128,0.07080750912427902,1.015869140625,1048576,3.365812129846043 +4,text.mlp.c_fc,2,32,8,128,6.147647857666016,0.3134765625,1048576,3.1205782672974403 +4,text.mlp.c_fc,2,64,8,128,7.176209449768066,0.28173828125,1048576,3.1205782672974403 +4,text.mlp.c_fc,2,128,8,128,8.08156967163086,0.265869140625,1048576,3.1205782672974403 +4,text.mlp.c_fc,3,32,8,128,2.6306636333465576,0.4384765625,1048576,3.1205782672974403 +4,text.mlp.c_fc,3,64,8,128,3.0671095848083496,0.40673828125,1048576,3.1205782672974403 +4,text.mlp.c_fc,3,128,8,128,3.458563804626465,0.390869140625,1048576,3.1205782672974403 +4,text.mlp.c_fc,4,32,8,128,1.2271418571472168,0.5634765625,1048576,3.1205782672974403 +4,text.mlp.c_fc,4,64,8,128,1.4321166276931765,0.53173828125,1048576,3.1205782672974403 +4,text.mlp.c_fc,4,128,8,128,1.6170594692230225,0.515869140625,1048576,3.1205782672974403 +4,text.mlp.c_fc,8,32,8,128,0.07444850355386734,1.0634765625,1048576,3.1205782672974403 +4,text.mlp.c_fc,8,64,8,128,0.08672907203435898,1.03173828125,1048576,3.1205782672974403 +4,text.mlp.c_fc,8,128,8,128,0.09753215312957764,1.015869140625,1048576,3.1205782672974403 +4,text.mlp.c_proj,2,32,8,128,4.5684428215026855,0.3134765625,1048576,3.3007157471391952 +4,text.mlp.c_proj,2,64,8,128,5.32727575302124,0.28173828125,1048576,3.3007157471391952 +4,text.mlp.c_proj,2,128,8,128,6.010774612426758,0.265869140625,1048576,3.3007157471391952 +4,text.mlp.c_proj,3,32,8,128,1.953374981880188,0.4384765625,1048576,3.3007157471391952 +4,text.mlp.c_proj,3,64,8,128,2.280749797821045,0.40673828125,1048576,3.3007157471391952 +4,text.mlp.c_proj,3,128,8,128,2.575989007949829,0.390869140625,1048576,3.3007157471391952 +4,text.mlp.c_proj,4,32,8,128,0.9109432697296144,0.5634765625,1048576,3.3007157471391952 +4,text.mlp.c_proj,4,64,8,128,1.0641778707504272,0.53173828125,1048576,3.3007157471391952 +4,text.mlp.c_proj,4,128,8,128,1.2020576000213623,0.515869140625,1048576,3.3007157471391952 +4,text.mlp.c_proj,8,32,8,128,0.05520271509885788,1.0634765625,1048576,3.3007157471391952 +4,text.mlp.c_proj,8,64,8,128,0.06432124227285385,1.03173828125,1048576,3.3007157471391952 +4,text.mlp.c_proj,8,128,8,128,0.07244100421667099,1.015869140625,1048576,3.3007157471391952 +5,text.mlp.c_fc,2,32,8,128,6.13761568069458,0.3134765625,1048576,3.12209161166918 +5,text.mlp.c_fc,2,64,8,128,7.154106140136719,0.28173828125,1048576,3.12209161166918 +5,text.mlp.c_fc,2,128,8,128,8.066065788269043,0.265869140625,1048576,3.12209161166918 +5,text.mlp.c_fc,3,32,8,128,2.622965097427368,0.4384765625,1048576,3.12209161166918 +5,text.mlp.c_fc,3,64,8,128,3.059314012527466,0.40673828125,1048576,3.12209161166918 +5,text.mlp.c_fc,3,128,8,128,3.4494223594665527,0.390869140625,1048576,3.12209161166918 +5,text.mlp.c_fc,4,32,8,128,1.2235389947891235,0.5634765625,1048576,3.12209161166918 +5,text.mlp.c_fc,4,64,8,128,1.4277613162994385,0.53173828125,1048576,3.12209161166918 +5,text.mlp.c_fc,4,128,8,128,1.6116138696670532,0.515869140625,1048576,3.12209161166918 +5,text.mlp.c_fc,8,32,8,128,0.0742468386888504,1.0634765625,1048576,3.12209161166918 +5,text.mlp.c_fc,8,64,8,128,0.08646093308925629,1.03173828125,1048576,3.12209161166918 +5,text.mlp.c_fc,8,128,8,128,0.09725197404623032,1.015869140625,1048576,3.12209161166918 +5,text.mlp.c_proj,2,32,8,128,4.701293468475342,0.3134765625,1048576,3.399940581104202 +5,text.mlp.c_proj,2,64,8,128,5.488790988922119,0.28173828125,1048576,3.399940581104202 +5,text.mlp.c_proj,2,128,8,128,6.199483871459961,0.265869140625,1048576,3.399940581104202 +5,text.mlp.c_proj,3,32,8,128,2.0124988555908203,0.4384765625,1048576,3.399940581104202 +5,text.mlp.c_proj,3,64,8,128,2.3516228199005127,0.40673828125,1048576,3.399940581104202 +5,text.mlp.c_proj,3,128,8,128,2.659825086593628,0.390869140625,1048576,3.399940581104202 +5,text.mlp.c_proj,4,32,8,128,0.9377811551094056,0.5634765625,1048576,3.399940581104202 +5,text.mlp.c_proj,4,64,8,128,1.0970691442489624,0.53173828125,1048576,3.399940581104202 +5,text.mlp.c_proj,4,128,8,128,1.2429126501083374,0.515869140625,1048576,3.399940581104202 +5,text.mlp.c_proj,8,32,8,128,0.05688211694359779,1.0634765625,1048576,3.399940581104202 +5,text.mlp.c_proj,8,64,8,128,0.06637991219758987,1.03173828125,1048576,3.399940581104202 +5,text.mlp.c_proj,8,128,8,128,0.0749565064907074,1.015869140625,1048576,3.399940581104202 +6,text.mlp.c_fc,2,32,8,128,6.08525276184082,0.3134765625,1048576,3.104312609554148 +6,text.mlp.c_fc,2,64,8,128,7.091857433319092,0.28173828125,1048576,3.104312609554148 +6,text.mlp.c_fc,2,128,8,128,8.011824607849121,0.265869140625,1048576,3.104312609554148 +6,text.mlp.c_fc,3,32,8,128,2.600118398666382,0.4384765625,1048576,3.104312609554148 +6,text.mlp.c_fc,3,64,8,128,3.0300920009613037,0.40673828125,1048576,3.104312609554148 +6,text.mlp.c_fc,3,128,8,128,3.4273650646209717,0.390869140625,1048576,3.104312609554148 +6,text.mlp.c_fc,4,32,8,128,1.2133545875549316,0.5634765625,1048576,3.104312609554148 +6,text.mlp.c_fc,4,64,8,128,1.4163140058517456,0.53173828125,1048576,3.104312609554148 +6,text.mlp.c_fc,4,128,8,128,1.5989820957183838,0.515869140625,1048576,3.104312609554148 +6,text.mlp.c_fc,8,32,8,128,0.07371968030929565,1.0634765625,1048576,3.104312609554148 +6,text.mlp.c_fc,8,64,8,128,0.08582986146211624,1.03173828125,1048576,3.104312609554148 +6,text.mlp.c_fc,8,128,8,128,0.09657172113656998,1.015869140625,1048576,3.104312609554148 +6,text.mlp.c_proj,2,32,8,128,4.809174060821533,0.3134765625,1048576,3.418852076553872 +6,text.mlp.c_proj,2,64,8,128,5.619462966918945,0.28173828125,1048576,3.418852076553872 +6,text.mlp.c_proj,2,128,8,128,6.347733974456787,0.265869140625,1048576,3.418852076553872 +6,text.mlp.c_proj,3,32,8,128,2.0561914443969727,0.4384765625,1048576,3.418852076553872 +6,text.mlp.c_proj,3,64,8,128,2.4042656421661377,0.40673828125,1048576,3.418852076553872 +6,text.mlp.c_proj,3,128,8,128,2.7215018272399902,0.390869140625,1048576,3.418852076553872 +6,text.mlp.c_proj,4,32,8,128,0.9595969915390016,0.5634765625,1048576,3.418852076553872 +6,text.mlp.c_proj,4,64,8,128,1.1221243143081665,0.53173828125,1048576,3.418852076553872 +6,text.mlp.c_proj,4,128,8,128,1.2688665390014648,0.515869140625,1048576,3.418852076553872 +6,text.mlp.c_proj,8,32,8,128,0.05815136805176735,1.0634765625,1048576,3.418852076553872 +6,text.mlp.c_proj,8,64,8,128,0.0679142028093338,1.03173828125,1048576,3.418852076553872 +6,text.mlp.c_proj,8,128,8,128,0.07667670398950577,1.015869140625,1048576,3.418852076553872 +7,text.mlp.c_fc,2,32,8,128,6.016368865966797,0.3134765625,1048576,3.1152966389066306 +7,text.mlp.c_fc,2,64,8,128,7.0210137367248535,0.28173828125,1048576,3.1152966389066306 +7,text.mlp.c_fc,2,128,8,128,7.924512386322021,0.265869140625,1048576,3.1152966389066306 +7,text.mlp.c_fc,3,32,8,128,2.573953628540039,0.4384765625,1048576,3.1152966389066306 +7,text.mlp.c_fc,3,64,8,128,3.004284620285034,0.40673828125,1048576,3.1152966389066306 +7,text.mlp.c_fc,3,128,8,128,3.3958704471588135,0.390869140625,1048576,3.1152966389066306 +7,text.mlp.c_fc,4,32,8,128,1.2008792161941528,0.5634765625,1048576,3.1152966389066306 +7,text.mlp.c_fc,4,64,8,128,1.401782751083374,0.53173828125,1048576,3.1152966389066306 +7,text.mlp.c_fc,4,128,8,128,1.5841267108917236,0.515869140625,1048576,3.1152966389066306 +7,text.mlp.c_fc,8,32,8,128,0.07277538627386093,1.0634765625,1048576,3.1152966389066306 +7,text.mlp.c_fc,8,64,8,128,0.08490202575922012,1.03173828125,1048576,3.1152966389066306 +7,text.mlp.c_fc,8,128,8,128,0.09555526822805405,1.015869140625,1048576,3.1152966389066306 +7,text.mlp.c_proj,2,32,8,128,4.9375319480896,0.3134765625,1048576,3.563658370504532 +7,text.mlp.c_proj,2,64,8,128,5.766933441162109,0.28173828125,1048576,3.563658370504532 +7,text.mlp.c_proj,2,128,8,128,6.5257158279418945,0.265869140625,1048576,3.563658370504532 +7,text.mlp.c_proj,3,32,8,128,2.1138291358947754,0.4384765625,1048576,3.563658370504532 +7,text.mlp.c_proj,3,64,8,128,2.4740889072418213,0.40673828125,1048576,3.563658370504532 +7,text.mlp.c_proj,3,128,8,128,2.8029916286468506,0.390869140625,1048576,3.563658370504532 +7,text.mlp.c_proj,4,32,8,128,0.985521137714386,0.5634765625,1048576,3.563658370504532 +7,text.mlp.c_proj,4,64,8,128,1.152884602546692,0.53173828125,1048576,3.563658370504532 +7,text.mlp.c_proj,4,128,8,128,1.3091634511947632,0.515869140625,1048576,3.563658370504532 +7,text.mlp.c_proj,8,32,8,128,0.05971698090434074,1.0634765625,1048576,3.563658370504532 +7,text.mlp.c_proj,8,64,8,128,0.06980670988559723,1.03173828125,1048576,3.563658370504532 +7,text.mlp.c_proj,8,128,8,128,0.07889241725206375,1.015869140625,1048576,3.563658370504532 +8,text.mlp.c_fc,2,32,8,128,5.901716709136963,0.3134765625,1048576,3.1079712489156552 +8,text.mlp.c_fc,2,64,8,128,6.884827136993408,0.28173828125,1048576,3.1079712489156552 +8,text.mlp.c_fc,2,128,8,128,7.761914730072021,0.265869140625,1048576,3.1079712489156552 +8,text.mlp.c_fc,3,32,8,128,2.520334243774414,0.4384765625,1048576,3.1079712489156552 +8,text.mlp.c_fc,3,64,8,128,2.9441771507263184,0.40673828125,1048576,3.1079712489156552 +8,text.mlp.c_fc,3,128,8,128,3.32405948638916,0.390869140625,1048576,3.1079712489156552 +8,text.mlp.c_fc,4,32,8,128,1.1775058507919312,0.5634765625,1048576,3.1079712489156552 +8,text.mlp.c_fc,4,64,8,128,1.37420654296875,0.53173828125,1048576,3.1079712489156552 +8,text.mlp.c_fc,4,128,8,128,1.5516901016235352,0.515869140625,1048576,3.1079712489156552 +8,text.mlp.c_fc,8,32,8,128,0.07130877673625946,1.0634765625,1048576,3.1079712489156552 +8,text.mlp.c_fc,8,64,8,128,0.0831245630979538,1.03173828125,1048576,3.1079712489156552 +8,text.mlp.c_fc,8,128,8,128,0.09355318546295166,1.015869140625,1048576,3.1079712489156552 +8,text.mlp.c_proj,2,32,8,128,5.102689266204834,0.3134765625,1048576,3.6947691166226 +8,text.mlp.c_proj,2,64,8,128,5.964451789855957,0.28173828125,1048576,3.6947691166226 +8,text.mlp.c_proj,2,128,8,128,6.735776901245117,0.265869140625,1048576,3.6947691166226 +8,text.mlp.c_proj,3,32,8,128,2.18440318107605,0.4384765625,1048576,3.6947691166226 +8,text.mlp.c_proj,3,64,8,128,2.556516647338867,0.40673828125,1048576,3.6947691166226 +8,text.mlp.c_proj,3,128,8,128,2.8903772830963135,0.390869140625,1048576,3.6947691166226 +8,text.mlp.c_proj,4,32,8,128,1.018601417541504,0.5634765625,1048576,3.6947691166226 +8,text.mlp.c_proj,4,64,8,128,1.192069172859192,0.53173828125,1048576,3.6947691166226 +8,text.mlp.c_proj,4,128,8,128,1.349886178970337,0.515869140625,1048576,3.6947691166226 +8,text.mlp.c_proj,8,32,8,128,0.0618121437728405,1.0634765625,1048576,3.6947691166226 +8,text.mlp.c_proj,8,64,8,128,0.07216800004243851,1.03173828125,1048576,3.6947691166226 +8,text.mlp.c_proj,8,128,8,128,0.08146006613969803,1.015869140625,1048576,3.6947691166226 +9,text.mlp.c_fc,2,32,8,128,5.788586139678955,0.3134765625,1048576,3.1305828189206824 +9,text.mlp.c_fc,2,64,8,128,6.742534637451172,0.28173828125,1048576,3.1305828189206824 +9,text.mlp.c_fc,2,128,8,128,7.599257946014404,0.265869140625,1048576,3.1305828189206824 +9,text.mlp.c_fc,3,32,8,128,2.47590970993042,0.4384765625,1048576,3.1305828189206824 +9,text.mlp.c_fc,3,64,8,128,2.887095212936402,0.40673828125,1048576,3.1305828189206824 +9,text.mlp.c_fc,3,128,8,128,3.255934953689575,0.390869140625,1048576,3.1305828189206824 +9,text.mlp.c_fc,4,32,8,128,1.1548055410385132,0.5634765625,1048576,3.1305828189206824 +9,text.mlp.c_fc,4,64,8,128,1.346752047538757,0.53173828125,1048576,3.1305828189206824 +9,text.mlp.c_fc,4,128,8,128,1.517864465713501,0.515869140625,1048576,3.1305828189206824 +9,text.mlp.c_fc,8,32,8,128,0.06999316066503525,1.0634765625,1048576,3.1305828189206824 +9,text.mlp.c_fc,8,64,8,128,0.0815470740199089,1.03173828125,1048576,3.1305828189206824 +9,text.mlp.c_fc,8,128,8,128,0.09159331023693083,1.015869140625,1048576,3.1305828189206824 +9,text.mlp.c_proj,2,32,8,128,5.207601547241211,0.3134765625,1048576,3.495399170438296 +9,text.mlp.c_proj,2,64,8,128,6.071842193603516,0.28173828125,1048576,3.495399170438296 +9,text.mlp.c_proj,2,128,8,128,6.8601274490356445,0.265869140625,1048576,3.495399170438296 +9,text.mlp.c_proj,3,32,8,128,2.2280025482177734,0.4384765625,1048576,3.495399170438296 +9,text.mlp.c_proj,3,64,8,128,2.5987324714660645,0.40673828125,1048576,3.495399170438296 +9,text.mlp.c_proj,3,128,8,128,2.939124822616577,0.390869140625,1048576,3.495399170438296 +9,text.mlp.c_proj,4,32,8,128,1.0387369394302368,0.5634765625,1048576,3.495399170438296 +9,text.mlp.c_proj,4,64,8,128,1.2127223014831543,0.53173828125,1048576,3.495399170438296 +9,text.mlp.c_proj,4,128,8,128,1.3731480836868286,0.515869140625,1048576,3.495399170438296 +9,text.mlp.c_proj,8,32,8,128,0.06300804764032364,1.0634765625,1048576,3.495399170438296 +9,text.mlp.c_proj,8,64,8,128,0.07346633821725845,1.03173828125,1048576,3.495399170438296 +9,text.mlp.c_proj,8,128,8,128,0.08280103653669357,1.015869140625,1048576,3.495399170438296 +10,text.mlp.c_fc,2,32,8,128,5.7948408126831055,0.3134765625,1048576,3.144641316115429 +10,text.mlp.c_fc,2,64,8,128,6.760894298553467,0.28173828125,1048576,3.144641316115429 +10,text.mlp.c_fc,2,128,8,128,7.649384498596191,0.265869140625,1048576,3.144641316115429 +10,text.mlp.c_fc,3,32,8,128,2.475472211837769,0.4384765625,1048576,3.144641316115429 +10,text.mlp.c_fc,3,64,8,128,2.8930392265319824,0.40673828125,1048576,3.144641316115429 +10,text.mlp.c_fc,3,128,8,128,3.2758681774139404,0.390869140625,1048576,3.144641316115429 +10,text.mlp.c_fc,4,32,8,128,1.1556246280670166,0.5634765625,1048576,3.144641316115429 +10,text.mlp.c_fc,4,64,8,128,1.3502486944198608,0.53173828125,1048576,3.144641316115429 +10,text.mlp.c_fc,4,128,8,128,1.5289539098739624,0.515869140625,1048576,3.144641316115429 +10,text.mlp.c_fc,8,32,8,128,0.0699719563126564,1.0634765625,1048576,3.144641316115429 +10,text.mlp.c_fc,8,64,8,128,0.08176594972610474,1.03173828125,1048576,3.144641316115429 +10,text.mlp.c_fc,8,128,8,128,0.09224754571914671,1.015869140625,1048576,3.144641316115429 +10,text.mlp.c_proj,2,32,8,128,5.407021522521973,0.3134765625,1048576,3.655921840018199 +10,text.mlp.c_proj,2,64,8,128,6.310086727142334,0.28173828125,1048576,3.655921840018199 +10,text.mlp.c_proj,2,128,8,128,7.1317644119262695,0.265869140625,1048576,3.655921840018199 +10,text.mlp.c_proj,3,32,8,128,2.315161943435669,0.4384765625,1048576,3.655921840018199 +10,text.mlp.c_proj,3,64,8,128,2.706345558166504,0.40673828125,1048576,3.655921840018199 +10,text.mlp.c_proj,3,128,8,128,3.065629720687866,0.390869140625,1048576,3.655921840018199 +10,text.mlp.c_proj,4,32,8,128,1.0800068378448486,0.5634765625,1048576,3.655921840018199 +10,text.mlp.c_proj,4,64,8,128,1.2637077569961548,0.53173828125,1048576,3.655921840018199 +10,text.mlp.c_proj,4,128,8,128,1.432173252105713,0.515869140625,1048576,3.655921840018199 +10,text.mlp.c_proj,8,32,8,128,0.06542721390724182,1.0634765625,1048576,3.655921840018199 +10,text.mlp.c_proj,8,64,8,128,0.0764131173491478,1.03173828125,1048576,3.655921840018199 +10,text.mlp.c_proj,8,128,8,128,0.08641009777784348,1.015869140625,1048576,3.655921840018199 +11,text.mlp.c_fc,2,32,8,128,6.029895782470703,0.3134765625,1048576,3.227518868269559 +11,text.mlp.c_fc,2,64,8,128,7.065587997436523,0.28173828125,1048576,3.227518868269559 +11,text.mlp.c_fc,2,128,8,128,8.00993537902832,0.265869140625,1048576,3.227518868269559 +11,text.mlp.c_fc,3,32,8,128,2.577655553817749,0.4384765625,1048576,3.227518868269559 +11,text.mlp.c_fc,3,64,8,128,3.0226359367370605,0.40673828125,1048576,3.227518868269559 +11,text.mlp.c_fc,3,128,8,128,3.4333667755126953,0.390869140625,1048576,3.227518868269559 +11,text.mlp.c_fc,4,32,8,128,1.201055645942688,0.5634765625,1048576,3.227518868269559 +11,text.mlp.c_fc,4,64,8,128,1.4092037677764893,0.53173828125,1048576,3.227518868269559 +11,text.mlp.c_fc,4,128,8,128,1.6002758741378784,0.515869140625,1048576,3.227518868269559 +11,text.mlp.c_fc,8,32,8,128,0.07296200841665268,1.0634765625,1048576,3.227518868269559 +11,text.mlp.c_fc,8,64,8,128,0.08532291650772095,1.03173828125,1048576,3.227518868269559 +11,text.mlp.c_fc,8,128,8,128,0.09650221467018127,1.015869140625,1048576,3.227518868269559 +11,text.mlp.c_proj,2,32,8,128,5.306176662445068,0.3134765625,1048576,3.351559119010831 +11,text.mlp.c_proj,2,64,8,128,6.189615726470947,0.28173828125,1048576,3.351559119010831 +11,text.mlp.c_proj,2,128,8,128,6.987279891967773,0.265869140625,1048576,3.351559119010831 +11,text.mlp.c_proj,3,32,8,128,2.267962694168091,0.4384765625,1048576,3.351559119010831 +11,text.mlp.c_proj,3,64,8,128,2.647044897079468,0.40673828125,1048576,3.351559119010831 +11,text.mlp.c_proj,3,128,8,128,2.989635467529297,0.390869140625,1048576,3.351559119010831 +11,text.mlp.c_proj,4,32,8,128,1.0588033199310305,0.5634765625,1048576,3.351559119010831 +11,text.mlp.c_proj,4,64,8,128,1.2367676496505735,0.53173828125,1048576,3.351559119010831 +11,text.mlp.c_proj,4,128,8,128,1.3962570428848269,0.515869140625,1048576,3.351559119010831 +11,text.mlp.c_proj,8,32,8,128,0.06412201374769211,1.0634765625,1048576,3.351559119010831 +11,text.mlp.c_proj,8,64,8,128,0.07475418597459793,1.03173828125,1048576,3.351559119010831 +11,text.mlp.c_proj,8,128,8,128,0.0842188373208046,1.015869140625,1048576,3.351559119010831 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-2-70b-hf.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-2-70b-hf.csv new file mode 100644 index 0000000000000000000000000000000000000000..f8f588393b83c8cabc8bd2d93273fa07b896c51c --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-2-70b-hf.csv @@ -0,0 +1,6721 @@ +layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params +0,self_attn.q_proj,2,32,8,128,33.875,20.0625,67108864 +0,self_attn.q_proj,2,64,8,128,40.5625,18.03125,67108864 +0,self_attn.q_proj,2,128,8,128,46.625,17.015625,67108864 +0,self_attn.q_proj,3,32,8,128,14.78125,28.0625,67108864 +0,self_attn.q_proj,3,64,8,128,18.09375,26.03125,67108864 +0,self_attn.q_proj,3,128,8,128,21.5,25.015625,67108864 +0,self_attn.q_proj,4,32,8,128,6.90234375,36.0625,67108864 +0,self_attn.q_proj,4,64,8,128,8.453125,34.03125,67108864 +0,self_attn.q_proj,4,128,8,128,10.09375,33.015625,67108864 +0,self_attn.q_proj,8,32,8,128,0.41796875,68.0625,67108864 +0,self_attn.q_proj,8,64,8,128,0.51171875,66.03125,67108864 +0,self_attn.q_proj,8,128,8,128,0.609375,65.015625,67108864 +0,self_attn.v_proj,2,32,8,128,9.8515625,2.5078125,8388608 +0,self_attn.v_proj,2,64,8,128,11.765625,2.25390625,8388608 +0,self_attn.v_proj,2,128,8,128,13.578125,2.126953125,8388608 +0,self_attn.v_proj,3,32,8,128,4.21484375,3.5078125,8388608 +0,self_attn.v_proj,3,64,8,128,5.05078125,3.25390625,8388608 +0,self_attn.v_proj,3,128,8,128,5.8515625,3.126953125,8388608 +0,self_attn.v_proj,4,32,8,128,1.9677734375,4.5078125,8388608 +0,self_attn.v_proj,4,64,8,128,2.357421875,4.25390625,8388608 +0,self_attn.v_proj,4,128,8,128,2.73046875,4.126953125,8388608 +0,self_attn.v_proj,8,32,8,128,0.119140625,8.5078125,8388608 +0,self_attn.v_proj,8,64,8,128,0.142578125,8.25390625,8388608 +0,self_attn.v_proj,8,128,8,128,0.16455078125,8.126953125,8388608 +0,self_attn.k_proj,2,32,8,128,27.875,2.5078125,8388608 +0,self_attn.k_proj,2,64,8,128,33.96875,2.25390625,8388608 +0,self_attn.k_proj,2,128,8,128,38.625,2.126953125,8388608 +0,self_attn.k_proj,3,32,8,128,12.40625,3.5078125,8388608 +0,self_attn.k_proj,3,64,8,128,16.21875,3.25390625,8388608 +0,self_attn.k_proj,3,128,8,128,19.703125,3.126953125,8388608 +0,self_attn.k_proj,4,32,8,128,5.79296875,4.5078125,8388608 +0,self_attn.k_proj,4,64,8,128,7.6015625,4.25390625,8388608 +0,self_attn.k_proj,4,128,8,128,9.3125,4.126953125,8388608 +0,self_attn.k_proj,8,32,8,128,0.350830078125,8.5078125,8388608 +0,self_attn.k_proj,8,64,8,128,0.459228515625,8.25390625,8388608 +0,self_attn.k_proj,8,128,8,128,0.56103515625,8.126953125,8388608 +0,self_attn.o_proj,2,32,8,128,22.90625,20.0625,67108864 +0,self_attn.o_proj,2,64,8,128,26.890625,18.03125,67108864 +0,self_attn.o_proj,2,128,8,128,30.609375,17.015625,67108864 +0,self_attn.o_proj,3,32,8,128,9.8671875,28.0625,67108864 +0,self_attn.o_proj,3,64,8,128,11.6484375,26.03125,67108864 +0,self_attn.o_proj,3,128,8,128,13.40625,25.015625,67108864 +0,self_attn.o_proj,4,32,8,128,4.62109375,36.0625,67108864 +0,self_attn.o_proj,4,64,8,128,5.47265625,34.03125,67108864 +0,self_attn.o_proj,4,128,8,128,6.328125,33.015625,67108864 +0,self_attn.o_proj,8,32,8,128,0.2802734375,68.0625,67108864 +0,self_attn.o_proj,8,64,8,128,0.33154296875,66.03125,67108864 +0,self_attn.o_proj,8,128,8,128,0.3828125,65.015625,67108864 +0,mlp.gate_proj,2,32,8,128,56.25,70.21875,234881024 +0,mlp.gate_proj,2,64,8,128,66.6875,63.109375,234881024 +0,mlp.gate_proj,2,128,8,128,76.875,59.5546875,234881024 +0,mlp.gate_proj,3,32,8,128,24.09375,98.21875,234881024 +0,mlp.gate_proj,3,64,8,128,28.734375,91.109375,234881024 +0,mlp.gate_proj,3,128,8,128,33.28125,87.5546875,234881024 +0,mlp.gate_proj,4,32,8,128,11.2421875,126.21875,234881024 +0,mlp.gate_proj,4,64,8,128,13.421875,119.109375,234881024 +0,mlp.gate_proj,4,128,8,128,15.5390625,115.5546875,234881024 +0,mlp.gate_proj,8,32,8,128,0.68408203125,238.21875,234881024 +0,mlp.gate_proj,8,64,8,128,0.8154296875,231.109375,234881024 +0,mlp.gate_proj,8,128,8,128,0.94091796875,227.5546875,234881024 +0,mlp.up_proj,2,32,8,128,54.375,70.21875,234881024 +0,mlp.up_proj,2,64,8,128,64.0625,63.109375,234881024 +0,mlp.up_proj,2,128,8,128,73.25,59.5546875,234881024 +0,mlp.up_proj,3,32,8,128,23.234375,98.21875,234881024 +0,mlp.up_proj,3,64,8,128,27.46875,91.109375,234881024 +0,mlp.up_proj,3,128,8,128,31.375,87.5546875,234881024 +0,mlp.up_proj,4,32,8,128,10.8359375,126.21875,234881024 +0,mlp.up_proj,4,64,8,128,12.8125,119.109375,234881024 +0,mlp.up_proj,4,128,8,128,14.6484375,115.5546875,234881024 +0,mlp.up_proj,8,32,8,128,0.65966796875,238.21875,234881024 +0,mlp.up_proj,8,64,8,128,0.77880859375,231.109375,234881024 +0,mlp.up_proj,8,128,8,128,0.88720703125,227.5546875,234881024 +0,mlp.down_proj,2,32,8,128,65.375,70.21875,234881024 +0,mlp.down_proj,2,64,8,128,76.5625,63.109375,234881024 +0,mlp.down_proj,2,128,8,128,86.6875,59.5546875,234881024 +0,mlp.down_proj,3,32,8,128,28.0625,98.21875,234881024 +0,mlp.down_proj,3,64,8,128,32.875,91.109375,234881024 +0,mlp.down_proj,3,128,8,128,37.46875,87.5546875,234881024 +0,mlp.down_proj,4,32,8,128,13.1015625,126.21875,234881024 +0,mlp.down_proj,4,64,8,128,15.3671875,119.109375,234881024 +0,mlp.down_proj,4,128,8,128,17.515625,115.5546875,234881024 +0,mlp.down_proj,8,32,8,128,0.7958984375,238.21875,234881024 +0,mlp.down_proj,8,64,8,128,0.93359375,231.109375,234881024 +0,mlp.down_proj,8,128,8,128,1.0615234375,227.5546875,234881024 +1,self_attn.q_proj,2,32,8,128,47.625,20.0625,67108864 +1,self_attn.q_proj,2,64,8,128,56.15625,18.03125,67108864 +1,self_attn.q_proj,2,128,8,128,64.0625,17.015625,67108864 +1,self_attn.q_proj,3,32,8,128,21.5,28.0625,67108864 +1,self_attn.q_proj,3,64,8,128,26.359375,26.03125,67108864 +1,self_attn.q_proj,3,128,8,128,31.578125,25.015625,67108864 +1,self_attn.q_proj,4,32,8,128,10.0546875,36.0625,67108864 +1,self_attn.q_proj,4,64,8,128,12.359375,34.03125,67108864 +1,self_attn.q_proj,4,128,8,128,14.8828125,33.015625,67108864 +1,self_attn.q_proj,8,32,8,128,0.60888671875,68.0625,67108864 +1,self_attn.q_proj,8,64,8,128,0.74755859375,66.03125,67108864 +1,self_attn.q_proj,8,128,8,128,0.8974609375,65.015625,67108864 +1,self_attn.v_proj,2,32,8,128,9.65625,2.5078125,8388608 +1,self_attn.v_proj,2,64,8,128,11.4765625,2.25390625,8388608 +1,self_attn.v_proj,2,128,8,128,13.1328125,2.126953125,8388608 +1,self_attn.v_proj,3,32,8,128,4.1328125,3.5078125,8388608 +1,self_attn.v_proj,3,64,8,128,4.921875,3.25390625,8388608 +1,self_attn.v_proj,3,128,8,128,5.65625,3.126953125,8388608 +1,self_attn.v_proj,4,32,8,128,1.927734375,4.5078125,8388608 +1,self_attn.v_proj,4,64,8,128,2.296875,4.25390625,8388608 +1,self_attn.v_proj,4,128,8,128,2.640625,4.126953125,8388608 +1,self_attn.v_proj,8,32,8,128,0.11676025390625,8.5078125,8388608 +1,self_attn.v_proj,8,64,8,128,0.138916015625,8.25390625,8388608 +1,self_attn.v_proj,8,128,8,128,0.1591796875,8.126953125,8388608 +1,self_attn.k_proj,2,32,8,128,32.03125,2.5078125,8388608 +1,self_attn.k_proj,2,64,8,128,38.46875,2.25390625,8388608 +1,self_attn.k_proj,2,128,8,128,43.8125,2.126953125,8388608 +1,self_attn.k_proj,3,32,8,128,14.2734375,3.5078125,8388608 +1,self_attn.k_proj,3,64,8,128,17.796875,3.25390625,8388608 +1,self_attn.k_proj,3,128,8,128,21.3125,3.126953125,8388608 +1,self_attn.k_proj,4,32,8,128,6.671875,4.5078125,8388608 +1,self_attn.k_proj,4,64,8,128,8.34375,4.25390625,8388608 +1,self_attn.k_proj,4,128,8,128,10.0234375,4.126953125,8388608 +1,self_attn.k_proj,8,32,8,128,0.403076171875,8.5078125,8388608 +1,self_attn.k_proj,8,64,8,128,0.50390625,8.25390625,8388608 +1,self_attn.k_proj,8,128,8,128,0.603515625,8.126953125,8388608 +1,self_attn.o_proj,2,32,8,128,27.6875,20.0625,67108864 +1,self_attn.o_proj,2,64,8,128,32.53125,18.03125,67108864 +1,self_attn.o_proj,2,128,8,128,37.15625,17.015625,67108864 +1,self_attn.o_proj,3,32,8,128,11.90625,28.0625,67108864 +1,self_attn.o_proj,3,64,8,128,14.0859375,26.03125,67108864 +1,self_attn.o_proj,3,128,8,128,16.234375,25.015625,67108864 +1,self_attn.o_proj,4,32,8,128,5.5625,36.0625,67108864 +1,self_attn.o_proj,4,64,8,128,6.58984375,34.03125,67108864 +1,self_attn.o_proj,4,128,8,128,7.62109375,33.015625,67108864 +1,self_attn.o_proj,8,32,8,128,0.3369140625,68.0625,67108864 +1,self_attn.o_proj,8,64,8,128,0.398681640625,66.03125,67108864 +1,self_attn.o_proj,8,128,8,128,0.459716796875,65.015625,67108864 +1,mlp.gate_proj,2,32,8,128,73.3125,70.21875,234881024 +1,mlp.gate_proj,2,64,8,128,85.875,63.109375,234881024 +1,mlp.gate_proj,2,128,8,128,97.5625,59.5546875,234881024 +1,mlp.gate_proj,3,32,8,128,31.359375,98.21875,234881024 +1,mlp.gate_proj,3,64,8,128,36.84375,91.109375,234881024 +1,mlp.gate_proj,3,128,8,128,41.96875,87.5546875,234881024 +1,mlp.gate_proj,4,32,8,128,14.6328125,126.21875,234881024 +1,mlp.gate_proj,4,64,8,128,17.1875,119.109375,234881024 +1,mlp.gate_proj,4,128,8,128,19.625,115.5546875,234881024 +1,mlp.gate_proj,8,32,8,128,0.88916015625,238.21875,234881024 +1,mlp.gate_proj,8,64,8,128,1.044921875,231.109375,234881024 +1,mlp.gate_proj,8,128,8,128,1.189453125,227.5546875,234881024 +1,mlp.up_proj,2,32,8,128,71.3125,70.21875,234881024 +1,mlp.up_proj,2,64,8,128,83.1875,63.109375,234881024 +1,mlp.up_proj,2,128,8,128,93.8125,59.5546875,234881024 +1,mlp.up_proj,3,32,8,128,30.484375,98.21875,234881024 +1,mlp.up_proj,3,64,8,128,35.625,91.109375,234881024 +1,mlp.up_proj,3,128,8,128,40.25,87.5546875,234881024 +1,mlp.up_proj,4,32,8,128,14.234375,126.21875,234881024 +1,mlp.up_proj,4,64,8,128,16.609375,119.109375,234881024 +1,mlp.up_proj,4,128,8,128,18.78125,115.5546875,234881024 +1,mlp.up_proj,8,32,8,128,0.86474609375,238.21875,234881024 +1,mlp.up_proj,8,64,8,128,1.009765625,231.109375,234881024 +1,mlp.up_proj,8,128,8,128,1.138671875,227.5546875,234881024 +1,mlp.down_proj,2,32,8,128,76.75,70.21875,234881024 +1,mlp.down_proj,2,64,8,128,89.375,63.109375,234881024 +1,mlp.down_proj,2,128,8,128,100.9375,59.5546875,234881024 +1,mlp.down_proj,3,32,8,128,32.8125,98.21875,234881024 +1,mlp.down_proj,3,64,8,128,38.34375,91.109375,234881024 +1,mlp.down_proj,3,128,8,128,43.3125,87.5546875,234881024 +1,mlp.down_proj,4,32,8,128,15.3046875,126.21875,234881024 +1,mlp.down_proj,4,64,8,128,17.890625,119.109375,234881024 +1,mlp.down_proj,4,128,8,128,20.234375,115.5546875,234881024 +1,mlp.down_proj,8,32,8,128,0.93017578125,238.21875,234881024 +1,mlp.down_proj,8,64,8,128,1.0869140625,231.109375,234881024 +1,mlp.down_proj,8,128,8,128,1.2275390625,227.5546875,234881024 +2,self_attn.q_proj,2,32,8,128,57.03125,20.0625,67108864 +2,self_attn.q_proj,2,64,8,128,67.4375,18.03125,67108864 +2,self_attn.q_proj,2,128,8,128,78.5625,17.015625,67108864 +2,self_attn.q_proj,3,32,8,128,24.5,28.0625,67108864 +2,self_attn.q_proj,3,64,8,128,29.15625,26.03125,67108864 +2,self_attn.q_proj,3,128,8,128,34.3125,25.015625,67108864 +2,self_attn.q_proj,4,32,8,128,11.4296875,36.0625,67108864 +2,self_attn.q_proj,4,64,8,128,13.609375,34.03125,67108864 +2,self_attn.q_proj,4,128,8,128,16.015625,33.015625,67108864 +2,self_attn.q_proj,8,32,8,128,0.69189453125,68.0625,67108864 +2,self_attn.q_proj,8,64,8,128,0.82275390625,66.03125,67108864 +2,self_attn.q_proj,8,128,8,128,0.96630859375,65.015625,67108864 +2,self_attn.v_proj,2,32,8,128,10.03125,2.5078125,8388608 +2,self_attn.v_proj,2,64,8,128,11.75,2.25390625,8388608 +2,self_attn.v_proj,2,128,8,128,13.3125,2.126953125,8388608 +2,self_attn.v_proj,3,32,8,128,4.2890625,3.5078125,8388608 +2,self_attn.v_proj,3,64,8,128,5.02734375,3.25390625,8388608 +2,self_attn.v_proj,3,128,8,128,5.6953125,3.126953125,8388608 +2,self_attn.v_proj,4,32,8,128,2.0,4.5078125,8388608 +2,self_attn.v_proj,4,64,8,128,2.345703125,4.25390625,8388608 +2,self_attn.v_proj,4,128,8,128,2.65625,4.126953125,8388608 +2,self_attn.v_proj,8,32,8,128,0.1212158203125,8.5078125,8388608 +2,self_attn.v_proj,8,64,8,128,0.141845703125,8.25390625,8388608 +2,self_attn.v_proj,8,128,8,128,0.1602783203125,8.126953125,8388608 +2,self_attn.k_proj,2,32,8,128,36.46875,2.5078125,8388608 +2,self_attn.k_proj,2,64,8,128,43.71875,2.25390625,8388608 +2,self_attn.k_proj,2,128,8,128,49.75,2.126953125,8388608 +2,self_attn.k_proj,3,32,8,128,15.796875,3.5078125,8388608 +2,self_attn.k_proj,3,64,8,128,19.1875,3.25390625,8388608 +2,self_attn.k_proj,3,128,8,128,22.40625,3.126953125,8388608 +2,self_attn.k_proj,4,32,8,128,7.37109375,4.5078125,8388608 +2,self_attn.k_proj,4,64,8,128,8.953125,4.25390625,8388608 +2,self_attn.k_proj,4,128,8,128,10.4609375,4.126953125,8388608 +2,self_attn.k_proj,8,32,8,128,0.44580078125,8.5078125,8388608 +2,self_attn.k_proj,8,64,8,128,0.54150390625,8.25390625,8388608 +2,self_attn.k_proj,8,128,8,128,0.63037109375,8.126953125,8388608 +2,self_attn.o_proj,2,32,8,128,34.6875,20.0625,67108864 +2,self_attn.o_proj,2,64,8,128,40.65625,18.03125,67108864 +2,self_attn.o_proj,2,128,8,128,46.28125,17.015625,67108864 +2,self_attn.o_proj,3,32,8,128,14.8828125,28.0625,67108864 +2,self_attn.o_proj,3,64,8,128,17.515625,26.03125,67108864 +2,self_attn.o_proj,3,128,8,128,20.09375,25.015625,67108864 +2,self_attn.o_proj,4,32,8,128,6.94921875,36.0625,67108864 +2,self_attn.o_proj,4,64,8,128,8.1796875,34.03125,67108864 +2,self_attn.o_proj,4,128,8,128,9.3984375,33.015625,67108864 +2,self_attn.o_proj,8,32,8,128,0.420166015625,68.0625,67108864 +2,self_attn.o_proj,8,64,8,128,0.4951171875,66.03125,67108864 +2,self_attn.o_proj,8,128,8,128,0.56640625,65.015625,67108864 +2,mlp.gate_proj,2,32,8,128,80.1875,70.21875,234881024 +2,mlp.gate_proj,2,64,8,128,93.625,63.109375,234881024 +2,mlp.gate_proj,2,128,8,128,106.0625,59.5546875,234881024 +2,mlp.gate_proj,3,32,8,128,34.28125,98.21875,234881024 +2,mlp.gate_proj,3,64,8,128,40.15625,91.109375,234881024 +2,mlp.gate_proj,3,128,8,128,45.5,87.5546875,234881024 +2,mlp.gate_proj,4,32,8,128,15.9921875,126.21875,234881024 +2,mlp.gate_proj,4,64,8,128,18.734375,119.109375,234881024 +2,mlp.gate_proj,4,128,8,128,21.25,115.5546875,234881024 +2,mlp.gate_proj,8,32,8,128,0.97265625,238.21875,234881024 +2,mlp.gate_proj,8,64,8,128,1.1396484375,231.109375,234881024 +2,mlp.gate_proj,8,128,8,128,1.2890625,227.5546875,234881024 +2,mlp.up_proj,2,32,8,128,76.375,70.21875,234881024 +2,mlp.up_proj,2,64,8,128,88.875,63.109375,234881024 +2,mlp.up_proj,2,128,8,128,100.25,59.5546875,234881024 +2,mlp.up_proj,3,32,8,128,32.625,98.21875,234881024 +2,mlp.up_proj,3,64,8,128,38.09375,91.109375,234881024 +2,mlp.up_proj,3,128,8,128,42.9375,87.5546875,234881024 +2,mlp.up_proj,4,32,8,128,15.2265625,126.21875,234881024 +2,mlp.up_proj,4,64,8,128,17.765625,119.109375,234881024 +2,mlp.up_proj,4,128,8,128,20.046875,115.5546875,234881024 +2,mlp.up_proj,8,32,8,128,0.92578125,238.21875,234881024 +2,mlp.up_proj,8,64,8,128,1.080078125,231.109375,234881024 +2,mlp.up_proj,8,128,8,128,1.216796875,227.5546875,234881024 +2,mlp.down_proj,2,32,8,128,79.5,70.21875,234881024 +2,mlp.down_proj,2,64,8,128,92.5,63.109375,234881024 +2,mlp.down_proj,2,128,8,128,104.3125,59.5546875,234881024 +2,mlp.down_proj,3,32,8,128,33.96875,98.21875,234881024 +2,mlp.down_proj,3,64,8,128,39.625,91.109375,234881024 +2,mlp.down_proj,3,128,8,128,44.6875,87.5546875,234881024 +2,mlp.down_proj,4,32,8,128,15.84375,126.21875,234881024 +2,mlp.down_proj,4,64,8,128,18.484375,119.109375,234881024 +2,mlp.down_proj,4,128,8,128,20.859375,115.5546875,234881024 +2,mlp.down_proj,8,32,8,128,0.9638671875,238.21875,234881024 +2,mlp.down_proj,8,64,8,128,1.125,231.109375,234881024 +2,mlp.down_proj,8,128,8,128,1.2666015625,227.5546875,234881024 +3,self_attn.q_proj,2,32,8,128,51.6875,20.0625,67108864 +3,self_attn.q_proj,2,64,8,128,61.3125,18.03125,67108864 +3,self_attn.q_proj,2,128,8,128,70.9375,17.015625,67108864 +3,self_attn.q_proj,3,32,8,128,22.140625,28.0625,67108864 +3,self_attn.q_proj,3,64,8,128,26.328125,26.03125,67108864 +3,self_attn.q_proj,3,128,8,128,30.609375,25.015625,67108864 +3,self_attn.q_proj,4,32,8,128,10.328125,36.0625,67108864 +3,self_attn.q_proj,4,64,8,128,12.28125,34.03125,67108864 +3,self_attn.q_proj,4,128,8,128,14.28125,33.015625,67108864 +3,self_attn.q_proj,8,32,8,128,0.62548828125,68.0625,67108864 +3,self_attn.q_proj,8,64,8,128,0.74267578125,66.03125,67108864 +3,self_attn.q_proj,8,128,8,128,0.86083984375,65.015625,67108864 +3,self_attn.v_proj,2,32,8,128,11.6484375,2.5078125,8388608 +3,self_attn.v_proj,2,64,8,128,13.6328125,2.25390625,8388608 +3,self_attn.v_proj,2,128,8,128,15.4140625,2.126953125,8388608 +3,self_attn.v_proj,3,32,8,128,4.98046875,3.5078125,8388608 +3,self_attn.v_proj,3,64,8,128,5.828125,3.25390625,8388608 +3,self_attn.v_proj,3,128,8,128,6.58984375,3.126953125,8388608 +3,self_attn.v_proj,4,32,8,128,2.32421875,4.5078125,8388608 +3,self_attn.v_proj,4,64,8,128,2.71875,4.25390625,8388608 +3,self_attn.v_proj,4,128,8,128,3.076171875,4.126953125,8388608 +3,self_attn.v_proj,8,32,8,128,0.140869140625,8.5078125,8388608 +3,self_attn.v_proj,8,64,8,128,0.1644287109375,8.25390625,8388608 +3,self_attn.v_proj,8,128,8,128,0.1854248046875,8.126953125,8388608 +3,self_attn.k_proj,2,32,8,128,31.90625,2.5078125,8388608 +3,self_attn.k_proj,2,64,8,128,38.375,2.25390625,8388608 +3,self_attn.k_proj,2,128,8,128,43.78125,2.126953125,8388608 +3,self_attn.k_proj,3,32,8,128,13.875,3.5078125,8388608 +3,self_attn.k_proj,3,64,8,128,16.84375,3.25390625,8388608 +3,self_attn.k_proj,3,128,8,128,19.703125,3.126953125,8388608 +3,self_attn.k_proj,4,32,8,128,6.47265625,4.5078125,8388608 +3,self_attn.k_proj,4,64,8,128,7.86328125,4.25390625,8388608 +3,self_attn.k_proj,4,128,8,128,9.203125,4.126953125,8388608 +3,self_attn.k_proj,8,32,8,128,0.3916015625,8.5078125,8388608 +3,self_attn.k_proj,8,64,8,128,0.4755859375,8.25390625,8388608 +3,self_attn.k_proj,8,128,8,128,0.5546875,8.126953125,8388608 +3,self_attn.o_proj,2,32,8,128,38.125,20.0625,67108864 +3,self_attn.o_proj,2,64,8,128,44.53125,18.03125,67108864 +3,self_attn.o_proj,2,128,8,128,50.40625,17.015625,67108864 +3,self_attn.o_proj,3,32,8,128,16.328125,28.0625,67108864 +3,self_attn.o_proj,3,64,8,128,19.109375,26.03125,67108864 +3,self_attn.o_proj,3,128,8,128,21.703125,25.015625,67108864 +3,self_attn.o_proj,4,32,8,128,7.6171875,36.0625,67108864 +3,self_attn.o_proj,4,64,8,128,8.921875,34.03125,67108864 +3,self_attn.o_proj,4,128,8,128,10.1328125,33.015625,67108864 +3,self_attn.o_proj,8,32,8,128,0.461181640625,68.0625,67108864 +3,self_attn.o_proj,8,64,8,128,0.53955078125,66.03125,67108864 +3,self_attn.o_proj,8,128,8,128,0.61083984375,65.015625,67108864 +3,mlp.gate_proj,2,32,8,128,82.0625,70.21875,234881024 +3,mlp.gate_proj,2,64,8,128,95.875,63.109375,234881024 +3,mlp.gate_proj,2,128,8,128,108.625,59.5546875,234881024 +3,mlp.gate_proj,3,32,8,128,35.09375,98.21875,234881024 +3,mlp.gate_proj,3,64,8,128,41.0625,91.109375,234881024 +3,mlp.gate_proj,3,128,8,128,46.53125,87.5546875,234881024 +3,mlp.gate_proj,4,32,8,128,16.359375,126.21875,234881024 +3,mlp.gate_proj,4,64,8,128,19.171875,119.109375,234881024 +3,mlp.gate_proj,4,128,8,128,21.71875,115.5546875,234881024 +3,mlp.gate_proj,8,32,8,128,0.9951171875,238.21875,234881024 +3,mlp.gate_proj,8,64,8,128,1.166015625,231.109375,234881024 +3,mlp.gate_proj,8,128,8,128,1.3173828125,227.5546875,234881024 +3,mlp.up_proj,2,32,8,128,78.4375,70.21875,234881024 +3,mlp.up_proj,2,64,8,128,91.25,63.109375,234881024 +3,mlp.up_proj,2,128,8,128,103.0,59.5546875,234881024 +3,mlp.up_proj,3,32,8,128,33.5,98.21875,234881024 +3,mlp.up_proj,3,64,8,128,39.09375,91.109375,234881024 +3,mlp.up_proj,3,128,8,128,44.09375,87.5546875,234881024 +3,mlp.up_proj,4,32,8,128,15.6171875,126.21875,234881024 +3,mlp.up_proj,4,64,8,128,18.234375,119.109375,234881024 +3,mlp.up_proj,4,128,8,128,20.59375,115.5546875,234881024 +3,mlp.up_proj,8,32,8,128,0.94970703125,238.21875,234881024 +3,mlp.up_proj,8,64,8,128,1.109375,231.109375,234881024 +3,mlp.up_proj,8,128,8,128,1.2490234375,227.5546875,234881024 +3,mlp.down_proj,2,32,8,128,80.0,70.21875,234881024 +3,mlp.down_proj,2,64,8,128,93.125,63.109375,234881024 +3,mlp.down_proj,2,128,8,128,105.0625,59.5546875,234881024 +3,mlp.down_proj,3,32,8,128,34.1875,98.21875,234881024 +3,mlp.down_proj,3,64,8,128,39.875,91.109375,234881024 +3,mlp.down_proj,3,128,8,128,44.96875,87.5546875,234881024 +3,mlp.down_proj,4,32,8,128,15.9453125,126.21875,234881024 +3,mlp.down_proj,4,64,8,128,18.609375,119.109375,234881024 +3,mlp.down_proj,4,128,8,128,21.0,115.5546875,234881024 +3,mlp.down_proj,8,32,8,128,0.9697265625,238.21875,234881024 +3,mlp.down_proj,8,64,8,128,1.1318359375,231.109375,234881024 +3,mlp.down_proj,8,128,8,128,1.2734375,227.5546875,234881024 +4,self_attn.q_proj,2,32,8,128,48.125,20.0625,67108864 +4,self_attn.q_proj,2,64,8,128,56.875,18.03125,67108864 +4,self_attn.q_proj,2,128,8,128,65.5625,17.015625,67108864 +4,self_attn.q_proj,3,32,8,128,20.59375,28.0625,67108864 +4,self_attn.q_proj,3,64,8,128,24.40625,26.03125,67108864 +4,self_attn.q_proj,3,128,8,128,28.21875,25.015625,67108864 +4,self_attn.q_proj,4,32,8,128,9.609375,36.0625,67108864 +4,self_attn.q_proj,4,64,8,128,11.390625,34.03125,67108864 +4,self_attn.q_proj,4,128,8,128,13.171875,33.015625,67108864 +4,self_attn.q_proj,8,32,8,128,0.58203125,68.0625,67108864 +4,self_attn.q_proj,8,64,8,128,0.68896484375,66.03125,67108864 +4,self_attn.q_proj,8,128,8,128,0.7939453125,65.015625,67108864 +4,self_attn.v_proj,2,32,8,128,11.40625,2.5078125,8388608 +4,self_attn.v_proj,2,64,8,128,13.40625,2.25390625,8388608 +4,self_attn.v_proj,2,128,8,128,15.21875,2.126953125,8388608 +4,self_attn.v_proj,3,32,8,128,4.87890625,3.5078125,8388608 +4,self_attn.v_proj,3,64,8,128,5.73046875,3.25390625,8388608 +4,self_attn.v_proj,3,128,8,128,6.51171875,3.126953125,8388608 +4,self_attn.v_proj,4,32,8,128,2.275390625,4.5078125,8388608 +4,self_attn.v_proj,4,64,8,128,2.67578125,4.25390625,8388608 +4,self_attn.v_proj,4,128,8,128,3.037109375,4.126953125,8388608 +4,self_attn.v_proj,8,32,8,128,0.137939453125,8.5078125,8388608 +4,self_attn.v_proj,8,64,8,128,0.161865234375,8.25390625,8388608 +4,self_attn.v_proj,8,128,8,128,0.18310546875,8.126953125,8388608 +4,self_attn.k_proj,2,32,8,128,27.859375,2.5078125,8388608 +4,self_attn.k_proj,2,64,8,128,33.59375,2.25390625,8388608 +4,self_attn.k_proj,2,128,8,128,38.3125,2.126953125,8388608 +4,self_attn.k_proj,3,32,8,128,12.1484375,3.5078125,8388608 +4,self_attn.k_proj,3,64,8,128,14.90625,3.25390625,8388608 +4,self_attn.k_proj,3,128,8,128,17.546875,3.126953125,8388608 +4,self_attn.k_proj,4,32,8,128,5.66796875,4.5078125,8388608 +4,self_attn.k_proj,4,64,8,128,6.9609375,4.25390625,8388608 +4,self_attn.k_proj,4,128,8,128,8.2109375,4.126953125,8388608 +4,self_attn.k_proj,8,32,8,128,0.343017578125,8.5078125,8388608 +4,self_attn.k_proj,8,64,8,128,0.4208984375,8.25390625,8388608 +4,self_attn.k_proj,8,128,8,128,0.494384765625,8.126953125,8388608 +4,self_attn.o_proj,2,32,8,128,36.09375,20.0625,67108864 +4,self_attn.o_proj,2,64,8,128,42.125,18.03125,67108864 +4,self_attn.o_proj,2,128,8,128,47.6875,17.015625,67108864 +4,self_attn.o_proj,3,32,8,128,15.4453125,28.0625,67108864 +4,self_attn.o_proj,3,64,8,128,18.0625,26.03125,67108864 +4,self_attn.o_proj,3,128,8,128,20.5,25.015625,67108864 +4,self_attn.o_proj,4,32,8,128,7.2109375,36.0625,67108864 +4,self_attn.o_proj,4,64,8,128,8.4375,34.03125,67108864 +4,self_attn.o_proj,4,128,8,128,9.578125,33.015625,67108864 +4,self_attn.o_proj,8,32,8,128,0.436279296875,68.0625,67108864 +4,self_attn.o_proj,8,64,8,128,0.51025390625,66.03125,67108864 +4,self_attn.o_proj,8,128,8,128,0.5771484375,65.015625,67108864 +4,mlp.gate_proj,2,32,8,128,82.8125,70.21875,234881024 +4,mlp.gate_proj,2,64,8,128,96.8125,63.109375,234881024 +4,mlp.gate_proj,2,128,8,128,109.875,59.5546875,234881024 +4,mlp.gate_proj,3,32,8,128,35.40625,98.21875,234881024 +4,mlp.gate_proj,3,64,8,128,41.46875,91.109375,234881024 +4,mlp.gate_proj,3,128,8,128,47.03125,87.5546875,234881024 +4,mlp.gate_proj,4,32,8,128,16.515625,126.21875,234881024 +4,mlp.gate_proj,4,64,8,128,19.359375,119.109375,234881024 +4,mlp.gate_proj,4,128,8,128,21.953125,115.5546875,234881024 +4,mlp.gate_proj,8,32,8,128,1.0048828125,238.21875,234881024 +4,mlp.gate_proj,8,64,8,128,1.177734375,231.109375,234881024 +4,mlp.gate_proj,8,128,8,128,1.33203125,227.5546875,234881024 +4,mlp.up_proj,2,32,8,128,78.75,70.21875,234881024 +4,mlp.up_proj,2,64,8,128,91.625,63.109375,234881024 +4,mlp.up_proj,2,128,8,128,103.375,59.5546875,234881024 +4,mlp.up_proj,3,32,8,128,33.625,98.21875,234881024 +4,mlp.up_proj,3,64,8,128,39.25,91.109375,234881024 +4,mlp.up_proj,3,128,8,128,44.28125,87.5546875,234881024 +4,mlp.up_proj,4,32,8,128,15.6875,126.21875,234881024 +4,mlp.up_proj,4,64,8,128,18.3125,119.109375,234881024 +4,mlp.up_proj,4,128,8,128,20.671875,115.5546875,234881024 +4,mlp.up_proj,8,32,8,128,0.95361328125,238.21875,234881024 +4,mlp.up_proj,8,64,8,128,1.11328125,231.109375,234881024 +4,mlp.up_proj,8,128,8,128,1.25390625,227.5546875,234881024 +4,mlp.down_proj,2,32,8,128,80.0,70.21875,234881024 +4,mlp.down_proj,2,64,8,128,93.0625,63.109375,234881024 +4,mlp.down_proj,2,128,8,128,104.9375,59.5546875,234881024 +4,mlp.down_proj,3,32,8,128,34.1875,98.21875,234881024 +4,mlp.down_proj,3,64,8,128,39.84375,91.109375,234881024 +4,mlp.down_proj,3,128,8,128,44.875,87.5546875,234881024 +4,mlp.down_proj,4,32,8,128,15.9375,126.21875,234881024 +4,mlp.down_proj,4,64,8,128,18.59375,119.109375,234881024 +4,mlp.down_proj,4,128,8,128,20.953125,115.5546875,234881024 +4,mlp.down_proj,8,32,8,128,0.96923828125,238.21875,234881024 +4,mlp.down_proj,8,64,8,128,1.130859375,231.109375,234881024 +4,mlp.down_proj,8,128,8,128,1.271484375,227.5546875,234881024 +5,self_attn.q_proj,2,32,8,128,50.375,20.0625,67108864 +5,self_attn.q_proj,2,64,8,128,59.25,18.03125,67108864 +5,self_attn.q_proj,2,128,8,128,68.1875,17.015625,67108864 +5,self_attn.q_proj,3,32,8,128,21.5625,28.0625,67108864 +5,self_attn.q_proj,3,64,8,128,25.390625,26.03125,67108864 +5,self_attn.q_proj,3,128,8,128,29.28125,25.015625,67108864 +5,self_attn.q_proj,4,32,8,128,10.0625,36.0625,67108864 +5,self_attn.q_proj,4,64,8,128,11.8515625,34.03125,67108864 +5,self_attn.q_proj,4,128,8,128,13.671875,33.015625,67108864 +5,self_attn.q_proj,8,32,8,128,0.609375,68.0625,67108864 +5,self_attn.q_proj,8,64,8,128,0.716796875,66.03125,67108864 +5,self_attn.q_proj,8,128,8,128,0.82421875,65.015625,67108864 +5,self_attn.v_proj,2,32,8,128,11.46875,2.5078125,8388608 +5,self_attn.v_proj,2,64,8,128,13.40625,2.25390625,8388608 +5,self_attn.v_proj,2,128,8,128,15.1640625,2.126953125,8388608 +5,self_attn.v_proj,3,32,8,128,4.90234375,3.5078125,8388608 +5,self_attn.v_proj,3,64,8,128,5.73828125,3.25390625,8388608 +5,self_attn.v_proj,3,128,8,128,6.484375,3.126953125,8388608 +5,self_attn.v_proj,4,32,8,128,2.287109375,4.5078125,8388608 +5,self_attn.v_proj,4,64,8,128,2.67578125,4.25390625,8388608 +5,self_attn.v_proj,4,128,8,128,3.025390625,4.126953125,8388608 +5,self_attn.v_proj,8,32,8,128,0.1385498046875,8.5078125,8388608 +5,self_attn.v_proj,8,64,8,128,0.161865234375,8.25390625,8388608 +5,self_attn.v_proj,8,128,8,128,0.1824951171875,8.126953125,8388608 +5,self_attn.k_proj,2,32,8,128,28.84375,2.5078125,8388608 +5,self_attn.k_proj,2,64,8,128,34.8125,2.25390625,8388608 +5,self_attn.k_proj,2,128,8,128,39.9375,2.126953125,8388608 +5,self_attn.k_proj,3,32,8,128,12.5078125,3.5078125,8388608 +5,self_attn.k_proj,3,64,8,128,15.2890625,3.25390625,8388608 +5,self_attn.k_proj,3,128,8,128,18.0,3.126953125,8388608 +5,self_attn.k_proj,4,32,8,128,5.8359375,4.5078125,8388608 +5,self_attn.k_proj,4,64,8,128,7.140625,4.25390625,8388608 +5,self_attn.k_proj,4,128,8,128,8.40625,4.126953125,8388608 +5,self_attn.k_proj,8,32,8,128,0.353271484375,8.5078125,8388608 +5,self_attn.k_proj,8,64,8,128,0.431640625,8.25390625,8388608 +5,self_attn.k_proj,8,128,8,128,0.5068359375,8.126953125,8388608 +5,self_attn.o_proj,2,32,8,128,37.5,20.0625,67108864 +5,self_attn.o_proj,2,64,8,128,43.71875,18.03125,67108864 +5,self_attn.o_proj,2,128,8,128,49.40625,17.015625,67108864 +5,self_attn.o_proj,3,32,8,128,16.03125,28.0625,67108864 +5,self_attn.o_proj,3,64,8,128,18.71875,26.03125,67108864 +5,self_attn.o_proj,3,128,8,128,21.1875,25.015625,67108864 +5,self_attn.o_proj,4,32,8,128,7.48046875,36.0625,67108864 +5,self_attn.o_proj,4,64,8,128,8.734375,34.03125,67108864 +5,self_attn.o_proj,4,128,8,128,9.890625,33.015625,67108864 +5,self_attn.o_proj,8,32,8,128,0.452880859375,68.0625,67108864 +5,self_attn.o_proj,8,64,8,128,0.5283203125,66.03125,67108864 +5,self_attn.o_proj,8,128,8,128,0.59619140625,65.015625,67108864 +5,mlp.gate_proj,2,32,8,128,83.1875,70.21875,234881024 +5,mlp.gate_proj,2,64,8,128,97.3125,63.109375,234881024 +5,mlp.gate_proj,2,128,8,128,110.3125,59.5546875,234881024 +5,mlp.gate_proj,3,32,8,128,35.59375,98.21875,234881024 +5,mlp.gate_proj,3,64,8,128,41.65625,91.109375,234881024 +5,mlp.gate_proj,3,128,8,128,47.25,87.5546875,234881024 +5,mlp.gate_proj,4,32,8,128,16.59375,126.21875,234881024 +5,mlp.gate_proj,4,64,8,128,19.453125,119.109375,234881024 +5,mlp.gate_proj,4,128,8,128,22.046875,115.5546875,234881024 +5,mlp.gate_proj,8,32,8,128,1.009765625,238.21875,234881024 +5,mlp.gate_proj,8,64,8,128,1.18359375,231.109375,234881024 +5,mlp.gate_proj,8,128,8,128,1.337890625,227.5546875,234881024 +5,mlp.up_proj,2,32,8,128,79.1875,70.21875,234881024 +5,mlp.up_proj,2,64,8,128,92.1875,63.109375,234881024 +5,mlp.up_proj,2,128,8,128,104.0625,59.5546875,234881024 +5,mlp.up_proj,3,32,8,128,33.8125,98.21875,234881024 +5,mlp.up_proj,3,64,8,128,39.5,91.109375,234881024 +5,mlp.up_proj,3,128,8,128,44.53125,87.5546875,234881024 +5,mlp.up_proj,4,32,8,128,15.7734375,126.21875,234881024 +5,mlp.up_proj,4,64,8,128,18.421875,119.109375,234881024 +5,mlp.up_proj,4,128,8,128,20.796875,115.5546875,234881024 +5,mlp.up_proj,8,32,8,128,0.958984375,238.21875,234881024 +5,mlp.up_proj,8,64,8,128,1.1201171875,231.109375,234881024 +5,mlp.up_proj,8,128,8,128,1.26171875,227.5546875,234881024 +5,mlp.down_proj,2,32,8,128,80.0,70.21875,234881024 +5,mlp.down_proj,2,64,8,128,93.0625,63.109375,234881024 +5,mlp.down_proj,2,128,8,128,104.9375,59.5546875,234881024 +5,mlp.down_proj,3,32,8,128,34.1875,98.21875,234881024 +5,mlp.down_proj,3,64,8,128,39.875,91.109375,234881024 +5,mlp.down_proj,3,128,8,128,44.9375,87.5546875,234881024 +5,mlp.down_proj,4,32,8,128,15.9453125,126.21875,234881024 +5,mlp.down_proj,4,64,8,128,18.609375,119.109375,234881024 +5,mlp.down_proj,4,128,8,128,20.96875,115.5546875,234881024 +5,mlp.down_proj,8,32,8,128,0.9697265625,238.21875,234881024 +5,mlp.down_proj,8,64,8,128,1.130859375,231.109375,234881024 +5,mlp.down_proj,8,128,8,128,1.2724609375,227.5546875,234881024 +6,self_attn.q_proj,2,32,8,128,52.15625,20.0625,67108864 +6,self_attn.q_proj,2,64,8,128,61.40625,18.03125,67108864 +6,self_attn.q_proj,2,128,8,128,71.0,17.015625,67108864 +6,self_attn.q_proj,3,32,8,128,22.34375,28.0625,67108864 +6,self_attn.q_proj,3,64,8,128,26.375,26.03125,67108864 +6,self_attn.q_proj,3,128,8,128,30.625,25.015625,67108864 +6,self_attn.q_proj,4,32,8,128,10.4296875,36.0625,67108864 +6,self_attn.q_proj,4,64,8,128,12.3046875,34.03125,67108864 +6,self_attn.q_proj,4,128,8,128,14.3046875,33.015625,67108864 +6,self_attn.q_proj,8,32,8,128,0.63134765625,68.0625,67108864 +6,self_attn.q_proj,8,64,8,128,0.74462890625,66.03125,67108864 +6,self_attn.q_proj,8,128,8,128,0.86181640625,65.015625,67108864 +6,self_attn.v_proj,2,32,8,128,11.5546875,2.5078125,8388608 +6,self_attn.v_proj,2,64,8,128,13.484375,2.25390625,8388608 +6,self_attn.v_proj,2,128,8,128,15.2421875,2.126953125,8388608 +6,self_attn.v_proj,3,32,8,128,4.9375,3.5078125,8388608 +6,self_attn.v_proj,3,64,8,128,5.765625,3.25390625,8388608 +6,self_attn.v_proj,3,128,8,128,6.51953125,3.126953125,8388608 +6,self_attn.v_proj,4,32,8,128,2.3046875,4.5078125,8388608 +6,self_attn.v_proj,4,64,8,128,2.69140625,4.25390625,8388608 +6,self_attn.v_proj,4,128,8,128,3.041015625,4.126953125,8388608 +6,self_attn.v_proj,8,32,8,128,0.1396484375,8.5078125,8388608 +6,self_attn.v_proj,8,64,8,128,0.1627197265625,8.25390625,8388608 +6,self_attn.v_proj,8,128,8,128,0.183349609375,8.126953125,8388608 +6,self_attn.k_proj,2,32,8,128,28.609375,2.5078125,8388608 +6,self_attn.k_proj,2,64,8,128,34.5,2.25390625,8388608 +6,self_attn.k_proj,2,128,8,128,39.5,2.126953125,8388608 +6,self_attn.k_proj,3,32,8,128,12.3671875,3.5078125,8388608 +6,self_attn.k_proj,3,64,8,128,15.1015625,3.25390625,8388608 +6,self_attn.k_proj,3,128,8,128,17.65625,3.126953125,8388608 +6,self_attn.k_proj,4,32,8,128,5.76953125,4.5078125,8388608 +6,self_attn.k_proj,4,64,8,128,7.046875,4.25390625,8388608 +6,self_attn.k_proj,4,128,8,128,8.25,4.126953125,8388608 +6,self_attn.k_proj,8,32,8,128,0.349365234375,8.5078125,8388608 +6,self_attn.k_proj,8,64,8,128,0.42626953125,8.25390625,8388608 +6,self_attn.k_proj,8,128,8,128,0.4970703125,8.126953125,8388608 +6,self_attn.o_proj,2,32,8,128,38.1875,20.0625,67108864 +6,self_attn.o_proj,2,64,8,128,44.5,18.03125,67108864 +6,self_attn.o_proj,2,128,8,128,50.25,17.015625,67108864 +6,self_attn.o_proj,3,32,8,128,16.34375,28.0625,67108864 +6,self_attn.o_proj,3,64,8,128,19.0625,26.03125,67108864 +6,self_attn.o_proj,3,128,8,128,21.546875,25.015625,67108864 +6,self_attn.o_proj,4,32,8,128,7.625,36.0625,67108864 +6,self_attn.o_proj,4,64,8,128,8.8984375,34.03125,67108864 +6,self_attn.o_proj,4,128,8,128,10.0625,33.015625,67108864 +6,self_attn.o_proj,8,32,8,128,0.46142578125,68.0625,67108864 +6,self_attn.o_proj,8,64,8,128,0.53857421875,66.03125,67108864 +6,self_attn.o_proj,8,128,8,128,0.60693359375,65.015625,67108864 +6,mlp.gate_proj,2,32,8,128,83.625,70.21875,234881024 +6,mlp.gate_proj,2,64,8,128,97.75,63.109375,234881024 +6,mlp.gate_proj,2,128,8,128,110.75,59.5546875,234881024 +6,mlp.gate_proj,3,32,8,128,35.78125,98.21875,234881024 +6,mlp.gate_proj,3,64,8,128,41.84375,91.109375,234881024 +6,mlp.gate_proj,3,128,8,128,47.40625,87.5546875,234881024 +6,mlp.gate_proj,4,32,8,128,16.6875,126.21875,234881024 +6,mlp.gate_proj,4,64,8,128,19.546875,119.109375,234881024 +6,mlp.gate_proj,4,128,8,128,22.125,115.5546875,234881024 +6,mlp.gate_proj,8,32,8,128,1.0146484375,238.21875,234881024 +6,mlp.gate_proj,8,64,8,128,1.1884765625,231.109375,234881024 +6,mlp.gate_proj,8,128,8,128,1.3427734375,227.5546875,234881024 +6,mlp.up_proj,2,32,8,128,79.5625,70.21875,234881024 +6,mlp.up_proj,2,64,8,128,92.6875,63.109375,234881024 +6,mlp.up_proj,2,128,8,128,104.5625,59.5546875,234881024 +6,mlp.up_proj,3,32,8,128,34.0,98.21875,234881024 +6,mlp.up_proj,3,64,8,128,39.6875,91.109375,234881024 +6,mlp.up_proj,3,128,8,128,44.75,87.5546875,234881024 +6,mlp.up_proj,4,32,8,128,15.859375,126.21875,234881024 +6,mlp.up_proj,4,64,8,128,18.515625,119.109375,234881024 +6,mlp.up_proj,4,128,8,128,20.890625,115.5546875,234881024 +6,mlp.up_proj,8,32,8,128,0.96435546875,238.21875,234881024 +6,mlp.up_proj,8,64,8,128,1.1259765625,231.109375,234881024 +6,mlp.up_proj,8,128,8,128,1.267578125,227.5546875,234881024 +6,mlp.down_proj,2,32,8,128,80.375,70.21875,234881024 +6,mlp.down_proj,2,64,8,128,93.5625,63.109375,234881024 +6,mlp.down_proj,2,128,8,128,105.5,59.5546875,234881024 +6,mlp.down_proj,3,32,8,128,34.34375,98.21875,234881024 +6,mlp.down_proj,3,64,8,128,40.0625,91.109375,234881024 +6,mlp.down_proj,3,128,8,128,45.15625,87.5546875,234881024 +6,mlp.down_proj,4,32,8,128,16.015625,126.21875,234881024 +6,mlp.down_proj,4,64,8,128,18.6875,119.109375,234881024 +6,mlp.down_proj,4,128,8,128,21.078125,115.5546875,234881024 +6,mlp.down_proj,8,32,8,128,0.97412109375,238.21875,234881024 +6,mlp.down_proj,8,64,8,128,1.13671875,231.109375,234881024 +6,mlp.down_proj,8,128,8,128,1.279296875,227.5546875,234881024 +7,self_attn.q_proj,2,32,8,128,51.46875,20.0625,67108864 +7,self_attn.q_proj,2,64,8,128,60.59375,18.03125,67108864 +7,self_attn.q_proj,2,128,8,128,70.0,17.015625,67108864 +7,self_attn.q_proj,3,32,8,128,22.03125,28.0625,67108864 +7,self_attn.q_proj,3,64,8,128,26.0,26.03125,67108864 +7,self_attn.q_proj,3,128,8,128,30.109375,25.015625,67108864 +7,self_attn.q_proj,4,32,8,128,10.28125,36.0625,67108864 +7,self_attn.q_proj,4,64,8,128,12.1328125,34.03125,67108864 +7,self_attn.q_proj,4,128,8,128,14.0546875,33.015625,67108864 +7,self_attn.q_proj,8,32,8,128,0.62255859375,68.0625,67108864 +7,self_attn.q_proj,8,64,8,128,0.7333984375,66.03125,67108864 +7,self_attn.q_proj,8,128,8,128,0.84716796875,65.015625,67108864 +7,self_attn.v_proj,2,32,8,128,11.3046875,2.5078125,8388608 +7,self_attn.v_proj,2,64,8,128,13.25,2.25390625,8388608 +7,self_attn.v_proj,2,128,8,128,15.0390625,2.126953125,8388608 +7,self_attn.v_proj,3,32,8,128,4.8359375,3.5078125,8388608 +7,self_attn.v_proj,3,64,8,128,5.66796875,3.25390625,8388608 +7,self_attn.v_proj,3,128,8,128,6.4296875,3.126953125,8388608 +7,self_attn.v_proj,4,32,8,128,2.255859375,4.5078125,8388608 +7,self_attn.v_proj,4,64,8,128,2.64453125,4.25390625,8388608 +7,self_attn.v_proj,4,128,8,128,3.001953125,4.126953125,8388608 +7,self_attn.v_proj,8,32,8,128,0.13671875,8.5078125,8388608 +7,self_attn.v_proj,8,64,8,128,0.1600341796875,8.25390625,8388608 +7,self_attn.v_proj,8,128,8,128,0.1810302734375,8.126953125,8388608 +7,self_attn.k_proj,2,32,8,128,28.65625,2.5078125,8388608 +7,self_attn.k_proj,2,64,8,128,34.6875,2.25390625,8388608 +7,self_attn.k_proj,2,128,8,128,39.75,2.126953125,8388608 +7,self_attn.k_proj,3,32,8,128,12.390625,3.5078125,8388608 +7,self_attn.k_proj,3,64,8,128,15.2421875,3.25390625,8388608 +7,self_attn.k_proj,3,128,8,128,17.953125,3.126953125,8388608 +7,self_attn.k_proj,4,32,8,128,5.78125,4.5078125,8388608 +7,self_attn.k_proj,4,64,8,128,7.11328125,4.25390625,8388608 +7,self_attn.k_proj,4,128,8,128,8.3828125,4.126953125,8388608 +7,self_attn.k_proj,8,32,8,128,0.349853515625,8.5078125,8388608 +7,self_attn.k_proj,8,64,8,128,0.43017578125,8.25390625,8388608 +7,self_attn.k_proj,8,128,8,128,0.5048828125,8.126953125,8388608 +7,self_attn.o_proj,2,32,8,128,38.15625,20.0625,67108864 +7,self_attn.o_proj,2,64,8,128,44.46875,18.03125,67108864 +7,self_attn.o_proj,2,128,8,128,50.1875,17.015625,67108864 +7,self_attn.o_proj,3,32,8,128,16.3125,28.0625,67108864 +7,self_attn.o_proj,3,64,8,128,19.03125,26.03125,67108864 +7,self_attn.o_proj,3,128,8,128,21.5,25.015625,67108864 +7,self_attn.o_proj,4,32,8,128,7.61328125,36.0625,67108864 +7,self_attn.o_proj,4,64,8,128,8.8828125,34.03125,67108864 +7,self_attn.o_proj,4,128,8,128,10.0390625,33.015625,67108864 +7,self_attn.o_proj,8,32,8,128,0.4609375,68.0625,67108864 +7,self_attn.o_proj,8,64,8,128,0.537109375,66.03125,67108864 +7,self_attn.o_proj,8,128,8,128,0.60498046875,65.015625,67108864 +7,mlp.gate_proj,2,32,8,128,84.0,70.21875,234881024 +7,mlp.gate_proj,2,64,8,128,98.1875,63.109375,234881024 +7,mlp.gate_proj,2,128,8,128,111.1875,59.5546875,234881024 +7,mlp.gate_proj,3,32,8,128,35.9375,98.21875,234881024 +7,mlp.gate_proj,3,64,8,128,42.03125,91.109375,234881024 +7,mlp.gate_proj,3,128,8,128,47.625,87.5546875,234881024 +7,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024 +7,mlp.gate_proj,4,64,8,128,19.625,119.109375,234881024 +7,mlp.gate_proj,4,128,8,128,22.234375,115.5546875,234881024 +7,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024 +7,mlp.gate_proj,8,64,8,128,1.1943359375,231.109375,234881024 +7,mlp.gate_proj,8,128,8,128,1.3486328125,227.5546875,234881024 +7,mlp.up_proj,2,32,8,128,79.9375,70.21875,234881024 +7,mlp.up_proj,2,64,8,128,93.125,63.109375,234881024 +7,mlp.up_proj,2,128,8,128,105.0,59.5546875,234881024 +7,mlp.up_proj,3,32,8,128,34.15625,98.21875,234881024 +7,mlp.up_proj,3,64,8,128,39.875,91.109375,234881024 +7,mlp.up_proj,3,128,8,128,44.9375,87.5546875,234881024 +7,mlp.up_proj,4,32,8,128,15.9375,126.21875,234881024 +7,mlp.up_proj,4,64,8,128,18.59375,119.109375,234881024 +7,mlp.up_proj,4,128,8,128,20.984375,115.5546875,234881024 +7,mlp.up_proj,8,32,8,128,0.96923828125,238.21875,234881024 +7,mlp.up_proj,8,64,8,128,1.130859375,231.109375,234881024 +7,mlp.up_proj,8,128,8,128,1.2724609375,227.5546875,234881024 +7,mlp.down_proj,2,32,8,128,80.75,70.21875,234881024 +7,mlp.down_proj,2,64,8,128,94.0,63.109375,234881024 +7,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024 +7,mlp.down_proj,3,32,8,128,34.53125,98.21875,234881024 +7,mlp.down_proj,3,64,8,128,40.25,91.109375,234881024 +7,mlp.down_proj,3,128,8,128,45.34375,87.5546875,234881024 +7,mlp.down_proj,4,32,8,128,16.09375,126.21875,234881024 +7,mlp.down_proj,4,64,8,128,18.78125,119.109375,234881024 +7,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024 +7,mlp.down_proj,8,32,8,128,0.97900390625,238.21875,234881024 +7,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024 +7,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024 +8,self_attn.q_proj,2,32,8,128,52.15625,20.0625,67108864 +8,self_attn.q_proj,2,64,8,128,61.28125,18.03125,67108864 +8,self_attn.q_proj,2,128,8,128,70.625,17.015625,67108864 +8,self_attn.q_proj,3,32,8,128,22.328125,28.0625,67108864 +8,self_attn.q_proj,3,64,8,128,26.28125,26.03125,67108864 +8,self_attn.q_proj,3,128,8,128,30.390625,25.015625,67108864 +8,self_attn.q_proj,4,32,8,128,10.421875,36.0625,67108864 +8,self_attn.q_proj,4,64,8,128,12.2578125,34.03125,67108864 +8,self_attn.q_proj,4,128,8,128,14.1796875,33.015625,67108864 +8,self_attn.q_proj,8,32,8,128,0.630859375,68.0625,67108864 +8,self_attn.q_proj,8,64,8,128,0.74169921875,66.03125,67108864 +8,self_attn.q_proj,8,128,8,128,0.85498046875,65.015625,67108864 +8,self_attn.v_proj,2,32,8,128,11.4921875,2.5078125,8388608 +8,self_attn.v_proj,2,64,8,128,13.453125,2.25390625,8388608 +8,self_attn.v_proj,2,128,8,128,15.25,2.126953125,8388608 +8,self_attn.v_proj,3,32,8,128,4.9140625,3.5078125,8388608 +8,self_attn.v_proj,3,64,8,128,5.7578125,3.25390625,8388608 +8,self_attn.v_proj,3,128,8,128,6.52734375,3.126953125,8388608 +8,self_attn.v_proj,4,32,8,128,2.29296875,4.5078125,8388608 +8,self_attn.v_proj,4,64,8,128,2.685546875,4.25390625,8388608 +8,self_attn.v_proj,4,128,8,128,3.046875,4.126953125,8388608 +8,self_attn.v_proj,8,32,8,128,0.1387939453125,8.5078125,8388608 +8,self_attn.v_proj,8,64,8,128,0.1624755859375,8.25390625,8388608 +8,self_attn.v_proj,8,128,8,128,0.18359375,8.126953125,8388608 +8,self_attn.k_proj,2,32,8,128,30.90625,2.5078125,8388608 +8,self_attn.k_proj,2,64,8,128,37.3125,2.25390625,8388608 +8,self_attn.k_proj,2,128,8,128,42.625,2.126953125,8388608 +8,self_attn.k_proj,3,32,8,128,13.3671875,3.5078125,8388608 +8,self_attn.k_proj,3,64,8,128,16.359375,3.25390625,8388608 +8,self_attn.k_proj,3,128,8,128,19.21875,3.126953125,8388608 +8,self_attn.k_proj,4,32,8,128,6.2421875,4.5078125,8388608 +8,self_attn.k_proj,4,64,8,128,7.63671875,4.25390625,8388608 +8,self_attn.k_proj,4,128,8,128,8.96875,4.126953125,8388608 +8,self_attn.k_proj,8,32,8,128,0.377685546875,8.5078125,8388608 +8,self_attn.k_proj,8,64,8,128,0.4619140625,8.25390625,8388608 +8,self_attn.k_proj,8,128,8,128,0.54052734375,8.126953125,8388608 +8,self_attn.o_proj,2,32,8,128,38.6875,20.0625,67108864 +8,self_attn.o_proj,2,64,8,128,45.1875,18.03125,67108864 +8,self_attn.o_proj,2,128,8,128,51.125,17.015625,67108864 +8,self_attn.o_proj,3,32,8,128,16.578125,28.0625,67108864 +8,self_attn.o_proj,3,64,8,128,19.40625,26.03125,67108864 +8,self_attn.o_proj,3,128,8,128,22.078125,25.015625,67108864 +8,self_attn.o_proj,4,32,8,128,7.75,36.0625,67108864 +8,self_attn.o_proj,4,64,8,128,9.0859375,34.03125,67108864 +8,self_attn.o_proj,4,128,8,128,10.3515625,33.015625,67108864 +8,self_attn.o_proj,8,32,8,128,0.46923828125,68.0625,67108864 +8,self_attn.o_proj,8,64,8,128,0.55029296875,66.03125,67108864 +8,self_attn.o_proj,8,128,8,128,0.62548828125,65.015625,67108864 +8,mlp.gate_proj,2,32,8,128,85.9375,70.21875,234881024 +8,mlp.gate_proj,2,64,8,128,100.8125,63.109375,234881024 +8,mlp.gate_proj,2,128,8,128,114.625,59.5546875,234881024 +8,mlp.gate_proj,3,32,8,128,36.84375,98.21875,234881024 +8,mlp.gate_proj,3,64,8,128,43.3125,91.109375,234881024 +8,mlp.gate_proj,3,128,8,128,49.46875,87.5546875,234881024 +8,mlp.gate_proj,4,32,8,128,17.171875,126.21875,234881024 +8,mlp.gate_proj,4,64,8,128,20.234375,119.109375,234881024 +8,mlp.gate_proj,4,128,8,128,23.125,115.5546875,234881024 +8,mlp.gate_proj,8,32,8,128,1.044921875,238.21875,234881024 +8,mlp.gate_proj,8,64,8,128,1.2314453125,231.109375,234881024 +8,mlp.gate_proj,8,128,8,128,1.40234375,227.5546875,234881024 +8,mlp.up_proj,2,32,8,128,80.1875,70.21875,234881024 +8,mlp.up_proj,2,64,8,128,93.4375,63.109375,234881024 +8,mlp.up_proj,2,128,8,128,105.625,59.5546875,234881024 +8,mlp.up_proj,3,32,8,128,34.28125,98.21875,234881024 +8,mlp.up_proj,3,64,8,128,40.0625,91.109375,234881024 +8,mlp.up_proj,3,128,8,128,45.3125,87.5546875,234881024 +8,mlp.up_proj,4,32,8,128,15.984375,126.21875,234881024 +8,mlp.up_proj,4,64,8,128,18.6875,119.109375,234881024 +8,mlp.up_proj,4,128,8,128,21.140625,115.5546875,234881024 +8,mlp.up_proj,8,32,8,128,0.97216796875,238.21875,234881024 +8,mlp.up_proj,8,64,8,128,1.13671875,231.109375,234881024 +8,mlp.up_proj,8,128,8,128,1.2822265625,227.5546875,234881024 +8,mlp.down_proj,2,32,8,128,80.9375,70.21875,234881024 +8,mlp.down_proj,2,64,8,128,94.1875,63.109375,234881024 +8,mlp.down_proj,2,128,8,128,106.1875,59.5546875,234881024 +8,mlp.down_proj,3,32,8,128,34.59375,98.21875,234881024 +8,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024 +8,mlp.down_proj,3,128,8,128,45.46875,87.5546875,234881024 +8,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024 +8,mlp.down_proj,4,64,8,128,18.84375,119.109375,234881024 +8,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024 +8,mlp.down_proj,8,32,8,128,0.982421875,238.21875,234881024 +8,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024 +8,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +9,self_attn.q_proj,2,32,8,128,47.28125,20.0625,67108864 +9,self_attn.q_proj,2,64,8,128,55.5625,18.03125,67108864 +9,self_attn.q_proj,2,128,8,128,63.9375,17.015625,67108864 +9,self_attn.q_proj,3,32,8,128,20.234375,28.0625,67108864 +9,self_attn.q_proj,3,64,8,128,23.8125,26.03125,67108864 +9,self_attn.q_proj,3,128,8,128,27.46875,25.015625,67108864 +9,self_attn.q_proj,4,32,8,128,9.4453125,36.0625,67108864 +9,self_attn.q_proj,4,64,8,128,11.109375,34.03125,67108864 +9,self_attn.q_proj,4,128,8,128,12.8125,33.015625,67108864 +9,self_attn.q_proj,8,32,8,128,0.57177734375,68.0625,67108864 +9,self_attn.q_proj,8,64,8,128,0.67236328125,66.03125,67108864 +9,self_attn.q_proj,8,128,8,128,0.7724609375,65.015625,67108864 +9,self_attn.v_proj,2,32,8,128,12.7734375,2.5078125,8388608 +9,self_attn.v_proj,2,64,8,128,14.9453125,2.25390625,8388608 +9,self_attn.v_proj,2,128,8,128,16.921875,2.126953125,8388608 +9,self_attn.v_proj,3,32,8,128,5.4609375,3.5078125,8388608 +9,self_attn.v_proj,3,64,8,128,6.390625,3.25390625,8388608 +9,self_attn.v_proj,3,128,8,128,7.23828125,3.126953125,8388608 +9,self_attn.v_proj,4,32,8,128,2.546875,4.5078125,8388608 +9,self_attn.v_proj,4,64,8,128,2.982421875,4.25390625,8388608 +9,self_attn.v_proj,4,128,8,128,3.37890625,4.126953125,8388608 +9,self_attn.v_proj,8,32,8,128,0.154296875,8.5078125,8388608 +9,self_attn.v_proj,8,64,8,128,0.180419921875,8.25390625,8388608 +9,self_attn.v_proj,8,128,8,128,0.20361328125,8.126953125,8388608 +9,self_attn.k_proj,2,32,8,128,28.15625,2.5078125,8388608 +9,self_attn.k_proj,2,64,8,128,34.0625,2.25390625,8388608 +9,self_attn.k_proj,2,128,8,128,39.03125,2.126953125,8388608 +9,self_attn.k_proj,3,32,8,128,12.1640625,3.5078125,8388608 +9,self_attn.k_proj,3,64,8,128,14.953125,3.25390625,8388608 +9,self_attn.k_proj,3,128,8,128,17.640625,3.126953125,8388608 +9,self_attn.k_proj,4,32,8,128,5.6796875,4.5078125,8388608 +9,self_attn.k_proj,4,64,8,128,6.98046875,4.25390625,8388608 +9,self_attn.k_proj,4,128,8,128,8.234375,4.126953125,8388608 +9,self_attn.k_proj,8,32,8,128,0.34375,8.5078125,8388608 +9,self_attn.k_proj,8,64,8,128,0.421875,8.25390625,8388608 +9,self_attn.k_proj,8,128,8,128,0.495849609375,8.126953125,8388608 +9,self_attn.o_proj,2,32,8,128,39.5,20.0625,67108864 +9,self_attn.o_proj,2,64,8,128,46.0,18.03125,67108864 +9,self_attn.o_proj,2,128,8,128,51.875,17.015625,67108864 +9,self_attn.o_proj,3,32,8,128,16.890625,28.0625,67108864 +9,self_attn.o_proj,3,64,8,128,19.703125,26.03125,67108864 +9,self_attn.o_proj,3,128,8,128,22.265625,25.015625,67108864 +9,self_attn.o_proj,4,32,8,128,7.8828125,36.0625,67108864 +9,self_attn.o_proj,4,64,8,128,9.1953125,34.03125,67108864 +9,self_attn.o_proj,4,128,8,128,10.390625,33.015625,67108864 +9,self_attn.o_proj,8,32,8,128,0.477294921875,68.0625,67108864 +9,self_attn.o_proj,8,64,8,128,0.55615234375,66.03125,67108864 +9,self_attn.o_proj,8,128,8,128,0.62646484375,65.015625,67108864 +9,mlp.gate_proj,2,32,8,128,86.0625,70.21875,234881024 +9,mlp.gate_proj,2,64,8,128,100.75,63.109375,234881024 +9,mlp.gate_proj,2,128,8,128,114.125,59.5546875,234881024 +9,mlp.gate_proj,3,32,8,128,36.875,98.21875,234881024 +9,mlp.gate_proj,3,64,8,128,43.15625,91.109375,234881024 +9,mlp.gate_proj,3,128,8,128,48.96875,87.5546875,234881024 +9,mlp.gate_proj,4,32,8,128,17.1875,126.21875,234881024 +9,mlp.gate_proj,4,64,8,128,20.15625,119.109375,234881024 +9,mlp.gate_proj,4,128,8,128,22.890625,115.5546875,234881024 +9,mlp.gate_proj,8,32,8,128,1.0458984375,238.21875,234881024 +9,mlp.gate_proj,8,64,8,128,1.2265625,231.109375,234881024 +9,mlp.gate_proj,8,128,8,128,1.3876953125,227.5546875,234881024 +9,mlp.up_proj,2,32,8,128,80.4375,70.21875,234881024 +9,mlp.up_proj,2,64,8,128,93.5625,63.109375,234881024 +9,mlp.up_proj,2,128,8,128,105.5625,59.5546875,234881024 +9,mlp.up_proj,3,32,8,128,34.375,98.21875,234881024 +9,mlp.up_proj,3,64,8,128,40.0625,91.109375,234881024 +9,mlp.up_proj,3,128,8,128,45.125,87.5546875,234881024 +9,mlp.up_proj,4,32,8,128,16.03125,126.21875,234881024 +9,mlp.up_proj,4,64,8,128,18.703125,119.109375,234881024 +9,mlp.up_proj,4,128,8,128,21.0625,115.5546875,234881024 +9,mlp.up_proj,8,32,8,128,0.974609375,238.21875,234881024 +9,mlp.up_proj,8,64,8,128,1.13671875,231.109375,234881024 +9,mlp.up_proj,8,128,8,128,1.2783203125,227.5546875,234881024 +9,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024 +9,mlp.down_proj,2,64,8,128,94.5,63.109375,234881024 +9,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024 +9,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024 +9,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024 +9,mlp.down_proj,3,128,8,128,45.5625,87.5546875,234881024 +9,mlp.down_proj,4,32,8,128,16.1875,126.21875,234881024 +9,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024 +9,mlp.down_proj,4,128,8,128,21.265625,115.5546875,234881024 +9,mlp.down_proj,8,32,8,128,0.98486328125,238.21875,234881024 +9,mlp.down_proj,8,64,8,128,1.1484375,231.109375,234881024 +9,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +10,self_attn.q_proj,2,32,8,128,45.96875,20.0625,67108864 +10,self_attn.q_proj,2,64,8,128,54.625,18.03125,67108864 +10,self_attn.q_proj,2,128,8,128,62.8125,17.015625,67108864 +10,self_attn.q_proj,3,32,8,128,19.71875,28.0625,67108864 +10,self_attn.q_proj,3,64,8,128,23.546875,26.03125,67108864 +10,self_attn.q_proj,3,128,8,128,27.34375,25.015625,67108864 +10,self_attn.q_proj,4,32,8,128,9.203125,36.0625,67108864 +10,self_attn.q_proj,4,64,8,128,10.984375,34.03125,67108864 +10,self_attn.q_proj,4,128,8,128,12.7578125,33.015625,67108864 +10,self_attn.q_proj,8,32,8,128,0.55712890625,68.0625,67108864 +10,self_attn.q_proj,8,64,8,128,0.6650390625,66.03125,67108864 +10,self_attn.q_proj,8,128,8,128,0.76904296875,65.015625,67108864 +10,self_attn.v_proj,2,32,8,128,13.546875,2.5078125,8388608 +10,self_attn.v_proj,2,64,8,128,15.7734375,2.25390625,8388608 +10,self_attn.v_proj,2,128,8,128,17.78125,2.126953125,8388608 +10,self_attn.v_proj,3,32,8,128,5.79296875,3.5078125,8388608 +10,self_attn.v_proj,3,64,8,128,6.75,3.25390625,8388608 +10,self_attn.v_proj,3,128,8,128,7.60546875,3.126953125,8388608 +10,self_attn.v_proj,4,32,8,128,2.703125,4.5078125,8388608 +10,self_attn.v_proj,4,64,8,128,3.1484375,4.25390625,8388608 +10,self_attn.v_proj,4,128,8,128,3.55078125,4.126953125,8388608 +10,self_attn.v_proj,8,32,8,128,0.1636962890625,8.5078125,8388608 +10,self_attn.v_proj,8,64,8,128,0.1905517578125,8.25390625,8388608 +10,self_attn.v_proj,8,128,8,128,0.2139892578125,8.126953125,8388608 +10,self_attn.k_proj,2,32,8,128,26.1875,2.5078125,8388608 +10,self_attn.k_proj,2,64,8,128,31.78125,2.25390625,8388608 +10,self_attn.k_proj,2,128,8,128,36.625,2.126953125,8388608 +10,self_attn.k_proj,3,32,8,128,11.3515625,3.5078125,8388608 +10,self_attn.k_proj,3,64,8,128,14.0390625,3.25390625,8388608 +10,self_attn.k_proj,3,128,8,128,16.703125,3.126953125,8388608 +10,self_attn.k_proj,4,32,8,128,5.296875,4.5078125,8388608 +10,self_attn.k_proj,4,64,8,128,6.5546875,4.25390625,8388608 +10,self_attn.k_proj,4,128,8,128,7.80078125,4.126953125,8388608 +10,self_attn.k_proj,8,32,8,128,0.320556640625,8.5078125,8388608 +10,self_attn.k_proj,8,64,8,128,0.396240234375,8.25390625,8388608 +10,self_attn.k_proj,8,128,8,128,0.47021484375,8.126953125,8388608 +10,self_attn.o_proj,2,32,8,128,38.625,20.0625,67108864 +10,self_attn.o_proj,2,64,8,128,45.03125,18.03125,67108864 +10,self_attn.o_proj,2,128,8,128,50.90625,17.015625,67108864 +10,self_attn.o_proj,3,32,8,128,16.546875,28.0625,67108864 +10,self_attn.o_proj,3,64,8,128,19.359375,26.03125,67108864 +10,self_attn.o_proj,3,128,8,128,21.96875,25.015625,67108864 +10,self_attn.o_proj,4,32,8,128,7.73046875,36.0625,67108864 +10,self_attn.o_proj,4,64,8,128,9.046875,34.03125,67108864 +10,self_attn.o_proj,4,128,8,128,10.2890625,33.015625,67108864 +10,self_attn.o_proj,8,32,8,128,0.468017578125,68.0625,67108864 +10,self_attn.o_proj,8,64,8,128,0.5478515625,66.03125,67108864 +10,self_attn.o_proj,8,128,8,128,0.62060546875,65.015625,67108864 +10,mlp.gate_proj,2,32,8,128,86.5,70.21875,234881024 +10,mlp.gate_proj,2,64,8,128,101.1875,63.109375,234881024 +10,mlp.gate_proj,2,128,8,128,114.5625,59.5546875,234881024 +10,mlp.gate_proj,3,32,8,128,37.03125,98.21875,234881024 +10,mlp.gate_proj,3,64,8,128,43.34375,91.109375,234881024 +10,mlp.gate_proj,3,128,8,128,49.15625,87.5546875,234881024 +10,mlp.gate_proj,4,32,8,128,17.28125,126.21875,234881024 +10,mlp.gate_proj,4,64,8,128,20.234375,119.109375,234881024 +10,mlp.gate_proj,4,128,8,128,22.953125,115.5546875,234881024 +10,mlp.gate_proj,8,32,8,128,1.05078125,238.21875,234881024 +10,mlp.gate_proj,8,64,8,128,1.232421875,231.109375,234881024 +10,mlp.gate_proj,8,128,8,128,1.392578125,227.5546875,234881024 +10,mlp.up_proj,2,32,8,128,80.8125,70.21875,234881024 +10,mlp.up_proj,2,64,8,128,94.0,63.109375,234881024 +10,mlp.up_proj,2,128,8,128,106.0,59.5546875,234881024 +10,mlp.up_proj,3,32,8,128,34.53125,98.21875,234881024 +10,mlp.up_proj,3,64,8,128,40.28125,91.109375,234881024 +10,mlp.up_proj,3,128,8,128,45.375,87.5546875,234881024 +10,mlp.up_proj,4,32,8,128,16.109375,126.21875,234881024 +10,mlp.up_proj,4,64,8,128,18.796875,119.109375,234881024 +10,mlp.up_proj,4,128,8,128,21.171875,115.5546875,234881024 +10,mlp.up_proj,8,32,8,128,0.97900390625,238.21875,234881024 +10,mlp.up_proj,8,64,8,128,1.142578125,231.109375,234881024 +10,mlp.up_proj,8,128,8,128,1.2841796875,227.5546875,234881024 +10,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024 +10,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024 +10,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024 +10,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024 +10,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024 +10,mlp.down_proj,3,128,8,128,45.5625,87.5546875,234881024 +10,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024 +10,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024 +10,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +10,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024 +10,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +10,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +11,self_attn.q_proj,2,32,8,128,47.5,20.0625,67108864 +11,self_attn.q_proj,2,64,8,128,56.25,18.03125,67108864 +11,self_attn.q_proj,2,128,8,128,64.4375,17.015625,67108864 +11,self_attn.q_proj,3,32,8,128,20.359375,28.0625,67108864 +11,self_attn.q_proj,3,64,8,128,24.1875,26.03125,67108864 +11,self_attn.q_proj,3,128,8,128,27.9375,25.015625,67108864 +11,self_attn.q_proj,4,32,8,128,9.5078125,36.0625,67108864 +11,self_attn.q_proj,4,64,8,128,11.28125,34.03125,67108864 +11,self_attn.q_proj,4,128,8,128,13.03125,33.015625,67108864 +11,self_attn.q_proj,8,32,8,128,0.57568359375,68.0625,67108864 +11,self_attn.q_proj,8,64,8,128,0.6826171875,66.03125,67108864 +11,self_attn.q_proj,8,128,8,128,0.78564453125,65.015625,67108864 +11,self_attn.v_proj,2,32,8,128,12.765625,2.5078125,8388608 +11,self_attn.v_proj,2,64,8,128,14.8828125,2.25390625,8388608 +11,self_attn.v_proj,2,128,8,128,16.796875,2.126953125,8388608 +11,self_attn.v_proj,3,32,8,128,5.4609375,3.5078125,8388608 +11,self_attn.v_proj,3,64,8,128,6.3671875,3.25390625,8388608 +11,self_attn.v_proj,3,128,8,128,7.1875,3.126953125,8388608 +11,self_attn.v_proj,4,32,8,128,2.546875,4.5078125,8388608 +11,self_attn.v_proj,4,64,8,128,2.970703125,4.25390625,8388608 +11,self_attn.v_proj,4,128,8,128,3.353515625,4.126953125,8388608 +11,self_attn.v_proj,8,32,8,128,0.154296875,8.5078125,8388608 +11,self_attn.v_proj,8,64,8,128,0.1796875,8.25390625,8388608 +11,self_attn.v_proj,8,128,8,128,0.2021484375,8.126953125,8388608 +11,self_attn.k_proj,2,32,8,128,27.3125,2.5078125,8388608 +11,self_attn.k_proj,2,64,8,128,32.78125,2.25390625,8388608 +11,self_attn.k_proj,2,128,8,128,37.375,2.126953125,8388608 +11,self_attn.k_proj,3,32,8,128,11.765625,3.5078125,8388608 +11,self_attn.k_proj,3,64,8,128,14.2578125,3.25390625,8388608 +11,self_attn.k_proj,3,128,8,128,16.609375,3.126953125,8388608 +11,self_attn.k_proj,4,32,8,128,5.4921875,4.5078125,8388608 +11,self_attn.k_proj,4,64,8,128,6.65625,4.25390625,8388608 +11,self_attn.k_proj,4,128,8,128,7.75390625,4.126953125,8388608 +11,self_attn.k_proj,8,32,8,128,0.332275390625,8.5078125,8388608 +11,self_attn.k_proj,8,64,8,128,0.402587890625,8.25390625,8388608 +11,self_attn.k_proj,8,128,8,128,0.467041015625,8.126953125,8388608 +11,self_attn.o_proj,2,32,8,128,38.1875,20.0625,67108864 +11,self_attn.o_proj,2,64,8,128,44.53125,18.03125,67108864 +11,self_attn.o_proj,2,128,8,128,50.28125,17.015625,67108864 +11,self_attn.o_proj,3,32,8,128,16.34375,28.0625,67108864 +11,self_attn.o_proj,3,64,8,128,19.109375,26.03125,67108864 +11,self_attn.o_proj,3,128,8,128,21.625,25.015625,67108864 +11,self_attn.o_proj,4,32,8,128,7.6328125,36.0625,67108864 +11,self_attn.o_proj,4,64,8,128,8.921875,34.03125,67108864 +11,self_attn.o_proj,4,128,8,128,10.109375,33.015625,67108864 +11,self_attn.o_proj,8,32,8,128,0.4619140625,68.0625,67108864 +11,self_attn.o_proj,8,64,8,128,0.53955078125,66.03125,67108864 +11,self_attn.o_proj,8,128,8,128,0.60986328125,65.015625,67108864 +11,mlp.gate_proj,2,32,8,128,86.375,70.21875,234881024 +11,mlp.gate_proj,2,64,8,128,101.0,63.109375,234881024 +11,mlp.gate_proj,2,128,8,128,114.3125,59.5546875,234881024 +11,mlp.gate_proj,3,32,8,128,36.96875,98.21875,234881024 +11,mlp.gate_proj,3,64,8,128,43.28125,91.109375,234881024 +11,mlp.gate_proj,3,128,8,128,49.0625,87.5546875,234881024 +11,mlp.gate_proj,4,32,8,128,17.25,126.21875,234881024 +11,mlp.gate_proj,4,64,8,128,20.203125,119.109375,234881024 +11,mlp.gate_proj,4,128,8,128,22.90625,115.5546875,234881024 +11,mlp.gate_proj,8,32,8,128,1.048828125,238.21875,234881024 +11,mlp.gate_proj,8,64,8,128,1.2294921875,231.109375,234881024 +11,mlp.gate_proj,8,128,8,128,1.3896484375,227.5546875,234881024 +11,mlp.up_proj,2,32,8,128,81.0625,70.21875,234881024 +11,mlp.up_proj,2,64,8,128,94.3125,63.109375,234881024 +11,mlp.up_proj,2,128,8,128,106.375,59.5546875,234881024 +11,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024 +11,mlp.up_proj,3,64,8,128,40.40625,91.109375,234881024 +11,mlp.up_proj,3,128,8,128,45.53125,87.5546875,234881024 +11,mlp.up_proj,4,32,8,128,16.15625,126.21875,234881024 +11,mlp.up_proj,4,64,8,128,18.84375,119.109375,234881024 +11,mlp.up_proj,4,128,8,128,21.25,115.5546875,234881024 +11,mlp.up_proj,8,32,8,128,0.982421875,238.21875,234881024 +11,mlp.up_proj,8,64,8,128,1.146484375,231.109375,234881024 +11,mlp.up_proj,8,128,8,128,1.2890625,227.5546875,234881024 +11,mlp.down_proj,2,32,8,128,81.1875,70.21875,234881024 +11,mlp.down_proj,2,64,8,128,94.4375,63.109375,234881024 +11,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024 +11,mlp.down_proj,3,32,8,128,34.6875,98.21875,234881024 +11,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024 +11,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024 +11,mlp.down_proj,4,32,8,128,16.171875,126.21875,234881024 +11,mlp.down_proj,4,64,8,128,18.859375,119.109375,234881024 +11,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024 +11,mlp.down_proj,8,32,8,128,0.98388671875,238.21875,234881024 +11,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024 +11,mlp.down_proj,8,128,8,128,1.2900390625,227.5546875,234881024 +12,self_attn.q_proj,2,32,8,128,49.1875,20.0625,67108864 +12,self_attn.q_proj,2,64,8,128,58.25,18.03125,67108864 +12,self_attn.q_proj,2,128,8,128,67.0,17.015625,67108864 +12,self_attn.q_proj,3,32,8,128,21.046875,28.0625,67108864 +12,self_attn.q_proj,3,64,8,128,24.984375,26.03125,67108864 +12,self_attn.q_proj,3,128,8,128,28.828125,25.015625,67108864 +12,self_attn.q_proj,4,32,8,128,9.8203125,36.0625,67108864 +12,self_attn.q_proj,4,64,8,128,11.65625,34.03125,67108864 +12,self_attn.q_proj,4,128,8,128,13.4609375,33.015625,67108864 +12,self_attn.q_proj,8,32,8,128,0.5947265625,68.0625,67108864 +12,self_attn.q_proj,8,64,8,128,0.705078125,66.03125,67108864 +12,self_attn.q_proj,8,128,8,128,0.81103515625,65.015625,67108864 +12,self_attn.v_proj,2,32,8,128,13.3203125,2.5078125,8388608 +12,self_attn.v_proj,2,64,8,128,15.5390625,2.25390625,8388608 +12,self_attn.v_proj,2,128,8,128,17.53125,2.126953125,8388608 +12,self_attn.v_proj,3,32,8,128,5.69140625,3.5078125,8388608 +12,self_attn.v_proj,3,64,8,128,6.640625,3.25390625,8388608 +12,self_attn.v_proj,3,128,8,128,7.5,3.126953125,8388608 +12,self_attn.v_proj,4,32,8,128,2.65625,4.5078125,8388608 +12,self_attn.v_proj,4,64,8,128,3.099609375,4.25390625,8388608 +12,self_attn.v_proj,4,128,8,128,3.5,4.126953125,8388608 +12,self_attn.v_proj,8,32,8,128,0.160888671875,8.5078125,8388608 +12,self_attn.v_proj,8,64,8,128,0.1875,8.25390625,8388608 +12,self_attn.v_proj,8,128,8,128,0.2109375,8.126953125,8388608 +12,self_attn.k_proj,2,32,8,128,28.5625,2.5078125,8388608 +12,self_attn.k_proj,2,64,8,128,34.3125,2.25390625,8388608 +12,self_attn.k_proj,2,128,8,128,39.375,2.126953125,8388608 +12,self_attn.k_proj,3,32,8,128,12.28125,3.5078125,8388608 +12,self_attn.k_proj,3,64,8,128,14.828125,3.25390625,8388608 +12,self_attn.k_proj,3,128,8,128,17.28125,3.126953125,8388608 +12,self_attn.k_proj,4,32,8,128,5.734375,4.5078125,8388608 +12,self_attn.k_proj,4,64,8,128,6.921875,4.25390625,8388608 +12,self_attn.k_proj,4,128,8,128,8.0625,4.126953125,8388608 +12,self_attn.k_proj,8,32,8,128,0.346923828125,8.5078125,8388608 +12,self_attn.k_proj,8,64,8,128,0.418701171875,8.25390625,8388608 +12,self_attn.k_proj,8,128,8,128,0.48583984375,8.126953125,8388608 +12,self_attn.o_proj,2,32,8,128,39.21875,20.0625,67108864 +12,self_attn.o_proj,2,64,8,128,45.78125,18.03125,67108864 +12,self_attn.o_proj,2,128,8,128,51.8125,17.015625,67108864 +12,self_attn.o_proj,3,32,8,128,16.78125,28.0625,67108864 +12,self_attn.o_proj,3,64,8,128,19.625,26.03125,67108864 +12,self_attn.o_proj,3,128,8,128,22.28125,25.015625,67108864 +12,self_attn.o_proj,4,32,8,128,7.828125,36.0625,67108864 +12,self_attn.o_proj,4,64,8,128,9.15625,34.03125,67108864 +12,self_attn.o_proj,4,128,8,128,10.40625,33.015625,67108864 +12,self_attn.o_proj,8,32,8,128,0.473876953125,68.0625,67108864 +12,self_attn.o_proj,8,64,8,128,0.55419921875,66.03125,67108864 +12,self_attn.o_proj,8,128,8,128,0.626953125,65.015625,67108864 +12,mlp.gate_proj,2,32,8,128,85.625,70.21875,234881024 +12,mlp.gate_proj,2,64,8,128,100.1875,63.109375,234881024 +12,mlp.gate_proj,2,128,8,128,113.5,59.5546875,234881024 +12,mlp.gate_proj,3,32,8,128,36.65625,98.21875,234881024 +12,mlp.gate_proj,3,64,8,128,42.90625,91.109375,234881024 +12,mlp.gate_proj,3,128,8,128,48.6875,87.5546875,234881024 +12,mlp.gate_proj,4,32,8,128,17.09375,126.21875,234881024 +12,mlp.gate_proj,4,64,8,128,20.03125,119.109375,234881024 +12,mlp.gate_proj,4,128,8,128,22.703125,115.5546875,234881024 +12,mlp.gate_proj,8,32,8,128,1.0400390625,238.21875,234881024 +12,mlp.gate_proj,8,64,8,128,1.21875,231.109375,234881024 +12,mlp.gate_proj,8,128,8,128,1.3779296875,227.5546875,234881024 +12,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024 +12,mlp.up_proj,2,64,8,128,94.5,63.109375,234881024 +12,mlp.up_proj,2,128,8,128,106.5625,59.5546875,234881024 +12,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024 +12,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024 +12,mlp.up_proj,3,128,8,128,45.625,87.5546875,234881024 +12,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024 +12,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024 +12,mlp.up_proj,4,128,8,128,21.296875,115.5546875,234881024 +12,mlp.up_proj,8,32,8,128,0.9833984375,238.21875,234881024 +12,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024 +12,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +12,mlp.down_proj,2,32,8,128,81.125,70.21875,234881024 +12,mlp.down_proj,2,64,8,128,94.375,63.109375,234881024 +12,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024 +12,mlp.down_proj,3,32,8,128,34.6875,98.21875,234881024 +12,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024 +12,mlp.down_proj,3,128,8,128,45.5625,87.5546875,234881024 +12,mlp.down_proj,4,32,8,128,16.171875,126.21875,234881024 +12,mlp.down_proj,4,64,8,128,18.875,119.109375,234881024 +12,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +12,mlp.down_proj,8,32,8,128,0.9833984375,238.21875,234881024 +12,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024 +12,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +13,self_attn.q_proj,2,32,8,128,48.9375,20.0625,67108864 +13,self_attn.q_proj,2,64,8,128,58.0,18.03125,67108864 +13,self_attn.q_proj,2,128,8,128,67.125,17.015625,67108864 +13,self_attn.q_proj,3,32,8,128,20.984375,28.0625,67108864 +13,self_attn.q_proj,3,64,8,128,24.953125,26.03125,67108864 +13,self_attn.q_proj,3,128,8,128,29.0625,25.015625,67108864 +13,self_attn.q_proj,4,32,8,128,9.7890625,36.0625,67108864 +13,self_attn.q_proj,4,64,8,128,11.640625,34.03125,67108864 +13,self_attn.q_proj,4,128,8,128,13.5625,33.015625,67108864 +13,self_attn.q_proj,8,32,8,128,0.5927734375,68.0625,67108864 +13,self_attn.q_proj,8,64,8,128,0.7041015625,66.03125,67108864 +13,self_attn.q_proj,8,128,8,128,0.8173828125,65.015625,67108864 +13,self_attn.v_proj,2,32,8,128,12.734375,2.5078125,8388608 +13,self_attn.v_proj,2,64,8,128,14.890625,2.25390625,8388608 +13,self_attn.v_proj,2,128,8,128,16.84375,2.126953125,8388608 +13,self_attn.v_proj,3,32,8,128,5.4453125,3.5078125,8388608 +13,self_attn.v_proj,3,64,8,128,6.3671875,3.25390625,8388608 +13,self_attn.v_proj,3,128,8,128,7.203125,3.126953125,8388608 +13,self_attn.v_proj,4,32,8,128,2.541015625,4.5078125,8388608 +13,self_attn.v_proj,4,64,8,128,2.970703125,4.25390625,8388608 +13,self_attn.v_proj,4,128,8,128,3.36328125,4.126953125,8388608 +13,self_attn.v_proj,8,32,8,128,0.15380859375,8.5078125,8388608 +13,self_attn.v_proj,8,64,8,128,0.1796875,8.25390625,8388608 +13,self_attn.v_proj,8,128,8,128,0.20263671875,8.126953125,8388608 +13,self_attn.k_proj,2,32,8,128,28.515625,2.5078125,8388608 +13,self_attn.k_proj,2,64,8,128,34.3125,2.25390625,8388608 +13,self_attn.k_proj,2,128,8,128,39.0625,2.126953125,8388608 +13,self_attn.k_proj,3,32,8,128,12.3671875,3.5078125,8388608 +13,self_attn.k_proj,3,64,8,128,15.1640625,3.25390625,8388608 +13,self_attn.k_proj,3,128,8,128,17.828125,3.126953125,8388608 +13,self_attn.k_proj,4,32,8,128,5.76953125,4.5078125,8388608 +13,self_attn.k_proj,4,64,8,128,7.078125,4.25390625,8388608 +13,self_attn.k_proj,4,128,8,128,8.3203125,4.126953125,8388608 +13,self_attn.k_proj,8,32,8,128,0.349365234375,8.5078125,8388608 +13,self_attn.k_proj,8,64,8,128,0.427734375,8.25390625,8388608 +13,self_attn.k_proj,8,128,8,128,0.50146484375,8.126953125,8388608 +13,self_attn.o_proj,2,32,8,128,39.71875,20.0625,67108864 +13,self_attn.o_proj,2,64,8,128,46.3125,18.03125,67108864 +13,self_attn.o_proj,2,128,8,128,52.3125,17.015625,67108864 +13,self_attn.o_proj,3,32,8,128,17.0,28.0625,67108864 +13,self_attn.o_proj,3,64,8,128,19.859375,26.03125,67108864 +13,self_attn.o_proj,3,128,8,128,22.484375,25.015625,67108864 +13,self_attn.o_proj,4,32,8,128,7.9375,36.0625,67108864 +13,self_attn.o_proj,4,64,8,128,9.2734375,34.03125,67108864 +13,self_attn.o_proj,4,128,8,128,10.5078125,33.015625,67108864 +13,self_attn.o_proj,8,32,8,128,0.48046875,68.0625,67108864 +13,self_attn.o_proj,8,64,8,128,0.56103515625,66.03125,67108864 +13,self_attn.o_proj,8,128,8,128,0.6337890625,65.015625,67108864 +13,mlp.gate_proj,2,32,8,128,85.0,70.21875,234881024 +13,mlp.gate_proj,2,64,8,128,99.4375,63.109375,234881024 +13,mlp.gate_proj,2,128,8,128,112.625,59.5546875,234881024 +13,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024 +13,mlp.gate_proj,3,64,8,128,42.5625,91.109375,234881024 +13,mlp.gate_proj,3,128,8,128,48.25,87.5546875,234881024 +13,mlp.gate_proj,4,32,8,128,16.953125,126.21875,234881024 +13,mlp.gate_proj,4,64,8,128,19.875,119.109375,234881024 +13,mlp.gate_proj,4,128,8,128,22.515625,115.5546875,234881024 +13,mlp.gate_proj,8,32,8,128,1.03125,238.21875,234881024 +13,mlp.gate_proj,8,64,8,128,1.208984375,231.109375,234881024 +13,mlp.gate_proj,8,128,8,128,1.3662109375,227.5546875,234881024 +13,mlp.up_proj,2,32,8,128,81.0,70.21875,234881024 +13,mlp.up_proj,2,64,8,128,94.3125,63.109375,234881024 +13,mlp.up_proj,2,128,8,128,106.375,59.5546875,234881024 +13,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024 +13,mlp.up_proj,3,64,8,128,40.375,91.109375,234881024 +13,mlp.up_proj,3,128,8,128,45.5,87.5546875,234881024 +13,mlp.up_proj,4,32,8,128,16.140625,126.21875,234881024 +13,mlp.up_proj,4,64,8,128,18.84375,119.109375,234881024 +13,mlp.up_proj,4,128,8,128,21.25,115.5546875,234881024 +13,mlp.up_proj,8,32,8,128,0.98193359375,238.21875,234881024 +13,mlp.up_proj,8,64,8,128,1.146484375,231.109375,234881024 +13,mlp.up_proj,8,128,8,128,1.2890625,227.5546875,234881024 +13,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024 +13,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024 +13,mlp.down_proj,2,128,8,128,106.75,59.5546875,234881024 +13,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +13,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024 +13,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024 +13,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +13,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024 +13,mlp.down_proj,4,128,8,128,21.3125,115.5546875,234881024 +13,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024 +13,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024 +13,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024 +14,self_attn.q_proj,2,32,8,128,49.21875,20.0625,67108864 +14,self_attn.q_proj,2,64,8,128,58.09375,18.03125,67108864 +14,self_attn.q_proj,2,128,8,128,66.8125,17.015625,67108864 +14,self_attn.q_proj,3,32,8,128,21.0625,28.0625,67108864 +14,self_attn.q_proj,3,64,8,128,24.90625,26.03125,67108864 +14,self_attn.q_proj,3,128,8,128,28.734375,25.015625,67108864 +14,self_attn.q_proj,4,32,8,128,9.828125,36.0625,67108864 +14,self_attn.q_proj,4,64,8,128,11.625,34.03125,67108864 +14,self_attn.q_proj,4,128,8,128,13.4140625,33.015625,67108864 +14,self_attn.q_proj,8,32,8,128,0.59521484375,68.0625,67108864 +14,self_attn.q_proj,8,64,8,128,0.703125,66.03125,67108864 +14,self_attn.q_proj,8,128,8,128,0.80859375,65.015625,67108864 +14,self_attn.v_proj,2,32,8,128,12.6875,2.5078125,8388608 +14,self_attn.v_proj,2,64,8,128,14.8125,2.25390625,8388608 +14,self_attn.v_proj,2,128,8,128,16.734375,2.126953125,8388608 +14,self_attn.v_proj,3,32,8,128,5.42578125,3.5078125,8388608 +14,self_attn.v_proj,3,64,8,128,6.3359375,3.25390625,8388608 +14,self_attn.v_proj,3,128,8,128,7.16015625,3.126953125,8388608 +14,self_attn.v_proj,4,32,8,128,2.529296875,4.5078125,8388608 +14,self_attn.v_proj,4,64,8,128,2.955078125,4.25390625,8388608 +14,self_attn.v_proj,4,128,8,128,3.341796875,4.126953125,8388608 +14,self_attn.v_proj,8,32,8,128,0.1533203125,8.5078125,8388608 +14,self_attn.v_proj,8,64,8,128,0.1788330078125,8.25390625,8388608 +14,self_attn.v_proj,8,128,8,128,0.201416015625,8.126953125,8388608 +14,self_attn.k_proj,2,32,8,128,26.953125,2.5078125,8388608 +14,self_attn.k_proj,2,64,8,128,32.59375,2.25390625,8388608 +14,self_attn.k_proj,2,128,8,128,37.4375,2.126953125,8388608 +14,self_attn.k_proj,3,32,8,128,11.5859375,3.5078125,8388608 +14,self_attn.k_proj,3,64,8,128,14.15625,3.25390625,8388608 +14,self_attn.k_proj,3,128,8,128,16.5625,3.126953125,8388608 +14,self_attn.k_proj,4,32,8,128,5.40625,4.5078125,8388608 +14,self_attn.k_proj,4,64,8,128,6.6015625,4.25390625,8388608 +14,self_attn.k_proj,4,128,8,128,7.73046875,4.126953125,8388608 +14,self_attn.k_proj,8,32,8,128,0.327392578125,8.5078125,8388608 +14,self_attn.k_proj,8,64,8,128,0.3994140625,8.25390625,8388608 +14,self_attn.k_proj,8,128,8,128,0.4658203125,8.126953125,8388608 +14,self_attn.o_proj,2,32,8,128,40.0625,20.0625,67108864 +14,self_attn.o_proj,2,64,8,128,46.65625,18.03125,67108864 +14,self_attn.o_proj,2,128,8,128,52.6875,17.015625,67108864 +14,self_attn.o_proj,3,32,8,128,17.125,28.0625,67108864 +14,self_attn.o_proj,3,64,8,128,20.0,26.03125,67108864 +14,self_attn.o_proj,3,128,8,128,22.609375,25.015625,67108864 +14,self_attn.o_proj,4,32,8,128,7.9921875,36.0625,67108864 +14,self_attn.o_proj,4,64,8,128,9.328125,34.03125,67108864 +14,self_attn.o_proj,4,128,8,128,10.5546875,33.015625,67108864 +14,self_attn.o_proj,8,32,8,128,0.48388671875,68.0625,67108864 +14,self_attn.o_proj,8,64,8,128,0.564453125,66.03125,67108864 +14,self_attn.o_proj,8,128,8,128,0.63623046875,65.015625,67108864 +14,mlp.gate_proj,2,32,8,128,85.25,70.21875,234881024 +14,mlp.gate_proj,2,64,8,128,99.6875,63.109375,234881024 +14,mlp.gate_proj,2,128,8,128,112.875,59.5546875,234881024 +14,mlp.gate_proj,3,32,8,128,36.46875,98.21875,234881024 +14,mlp.gate_proj,3,64,8,128,42.65625,91.109375,234881024 +14,mlp.gate_proj,3,128,8,128,48.375,87.5546875,234881024 +14,mlp.gate_proj,4,32,8,128,17.015625,126.21875,234881024 +14,mlp.gate_proj,4,64,8,128,19.921875,119.109375,234881024 +14,mlp.gate_proj,4,128,8,128,22.578125,115.5546875,234881024 +14,mlp.gate_proj,8,32,8,128,1.0341796875,238.21875,234881024 +14,mlp.gate_proj,8,64,8,128,1.212890625,231.109375,234881024 +14,mlp.gate_proj,8,128,8,128,1.3701171875,227.5546875,234881024 +14,mlp.up_proj,2,32,8,128,81.1875,70.21875,234881024 +14,mlp.up_proj,2,64,8,128,94.5,63.109375,234881024 +14,mlp.up_proj,2,128,8,128,106.625,59.5546875,234881024 +14,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024 +14,mlp.up_proj,3,64,8,128,40.46875,91.109375,234881024 +14,mlp.up_proj,3,128,8,128,45.625,87.5546875,234881024 +14,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024 +14,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024 +14,mlp.up_proj,4,128,8,128,21.296875,115.5546875,234881024 +14,mlp.up_proj,8,32,8,128,0.98388671875,238.21875,234881024 +14,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024 +14,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +14,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024 +14,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024 +14,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024 +14,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024 +14,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024 +14,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024 +14,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024 +14,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024 +14,mlp.down_proj,4,128,8,128,21.296875,115.5546875,234881024 +14,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024 +14,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +14,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +15,self_attn.q_proj,2,32,8,128,45.375,20.0625,67108864 +15,self_attn.q_proj,2,64,8,128,53.59375,18.03125,67108864 +15,self_attn.q_proj,2,128,8,128,62.0625,17.015625,67108864 +15,self_attn.q_proj,3,32,8,128,19.4375,28.0625,67108864 +15,self_attn.q_proj,3,64,8,128,23.03125,26.03125,67108864 +15,self_attn.q_proj,3,128,8,128,26.828125,25.015625,67108864 +15,self_attn.q_proj,4,32,8,128,9.0703125,36.0625,67108864 +15,self_attn.q_proj,4,64,8,128,10.7421875,34.03125,67108864 +15,self_attn.q_proj,4,128,8,128,12.5234375,33.015625,67108864 +15,self_attn.q_proj,8,32,8,128,0.54931640625,68.0625,67108864 +15,self_attn.q_proj,8,64,8,128,0.650390625,66.03125,67108864 +15,self_attn.q_proj,8,128,8,128,0.7548828125,65.015625,67108864 +15,self_attn.v_proj,2,32,8,128,13.75,2.5078125,8388608 +15,self_attn.v_proj,2,64,8,128,16.078125,2.25390625,8388608 +15,self_attn.v_proj,2,128,8,128,18.171875,2.126953125,8388608 +15,self_attn.v_proj,3,32,8,128,5.87890625,3.5078125,8388608 +15,self_attn.v_proj,3,64,8,128,6.875,3.25390625,8388608 +15,self_attn.v_proj,3,128,8,128,7.77734375,3.126953125,8388608 +15,self_attn.v_proj,4,32,8,128,2.740234375,4.5078125,8388608 +15,self_attn.v_proj,4,64,8,128,3.20703125,4.25390625,8388608 +15,self_attn.v_proj,4,128,8,128,3.630859375,4.126953125,8388608 +15,self_attn.v_proj,8,32,8,128,0.166015625,8.5078125,8388608 +15,self_attn.v_proj,8,64,8,128,0.194091796875,8.25390625,8388608 +15,self_attn.v_proj,8,128,8,128,0.21875,8.126953125,8388608 +15,self_attn.k_proj,2,32,8,128,25.953125,2.5078125,8388608 +15,self_attn.k_proj,2,64,8,128,31.40625,2.25390625,8388608 +15,self_attn.k_proj,2,128,8,128,35.96875,2.126953125,8388608 +15,self_attn.k_proj,3,32,8,128,11.25,3.5078125,8388608 +15,self_attn.k_proj,3,64,8,128,13.890625,3.25390625,8388608 +15,self_attn.k_proj,3,128,8,128,16.4375,3.126953125,8388608 +15,self_attn.k_proj,4,32,8,128,5.24609375,4.5078125,8388608 +15,self_attn.k_proj,4,64,8,128,6.484375,4.25390625,8388608 +15,self_attn.k_proj,4,128,8,128,7.671875,4.126953125,8388608 +15,self_attn.k_proj,8,32,8,128,0.317626953125,8.5078125,8388608 +15,self_attn.k_proj,8,64,8,128,0.391845703125,8.25390625,8388608 +15,self_attn.k_proj,8,128,8,128,0.46240234375,8.126953125,8388608 +15,self_attn.o_proj,2,32,8,128,40.40625,20.0625,67108864 +15,self_attn.o_proj,2,64,8,128,47.0625,18.03125,67108864 +15,self_attn.o_proj,2,128,8,128,53.09375,17.015625,67108864 +15,self_attn.o_proj,3,32,8,128,17.296875,28.0625,67108864 +15,self_attn.o_proj,3,64,8,128,20.171875,26.03125,67108864 +15,self_attn.o_proj,3,128,8,128,22.796875,25.015625,67108864 +15,self_attn.o_proj,4,32,8,128,8.078125,36.0625,67108864 +15,self_attn.o_proj,4,64,8,128,9.421875,34.03125,67108864 +15,self_attn.o_proj,4,128,8,128,10.6640625,33.015625,67108864 +15,self_attn.o_proj,8,32,8,128,0.48876953125,68.0625,67108864 +15,self_attn.o_proj,8,64,8,128,0.5703125,66.03125,67108864 +15,self_attn.o_proj,8,128,8,128,0.64306640625,65.015625,67108864 +15,mlp.gate_proj,2,32,8,128,85.0625,70.21875,234881024 +15,mlp.gate_proj,2,64,8,128,99.5,63.109375,234881024 +15,mlp.gate_proj,2,128,8,128,112.6875,59.5546875,234881024 +15,mlp.gate_proj,3,32,8,128,36.40625,98.21875,234881024 +15,mlp.gate_proj,3,64,8,128,42.59375,91.109375,234881024 +15,mlp.gate_proj,3,128,8,128,48.28125,87.5546875,234881024 +15,mlp.gate_proj,4,32,8,128,16.96875,126.21875,234881024 +15,mlp.gate_proj,4,64,8,128,19.890625,119.109375,234881024 +15,mlp.gate_proj,4,128,8,128,22.53125,115.5546875,234881024 +15,mlp.gate_proj,8,32,8,128,1.033203125,238.21875,234881024 +15,mlp.gate_proj,8,64,8,128,1.2099609375,231.109375,234881024 +15,mlp.gate_proj,8,128,8,128,1.3671875,227.5546875,234881024 +15,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024 +15,mlp.up_proj,2,64,8,128,94.4375,63.109375,234881024 +15,mlp.up_proj,2,128,8,128,106.5,59.5546875,234881024 +15,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024 +15,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024 +15,mlp.up_proj,3,128,8,128,45.59375,87.5546875,234881024 +15,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024 +15,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024 +15,mlp.up_proj,4,128,8,128,21.28125,115.5546875,234881024 +15,mlp.up_proj,8,32,8,128,0.9833984375,238.21875,234881024 +15,mlp.up_proj,8,64,8,128,1.1474609375,231.109375,234881024 +15,mlp.up_proj,8,128,8,128,1.291015625,227.5546875,234881024 +15,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024 +15,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024 +15,mlp.down_proj,2,128,8,128,106.75,59.5546875,234881024 +15,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +15,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024 +15,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024 +15,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +15,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024 +15,mlp.down_proj,4,128,8,128,21.3125,115.5546875,234881024 +15,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024 +15,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024 +15,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024 +16,self_attn.q_proj,2,32,8,128,46.375,20.0625,67108864 +16,self_attn.q_proj,2,64,8,128,55.0,18.03125,67108864 +16,self_attn.q_proj,2,128,8,128,63.6875,17.015625,67108864 +16,self_attn.q_proj,3,32,8,128,19.890625,28.0625,67108864 +16,self_attn.q_proj,3,64,8,128,23.671875,26.03125,67108864 +16,self_attn.q_proj,3,128,8,128,27.609375,25.015625,67108864 +16,self_attn.q_proj,4,32,8,128,9.28125,36.0625,67108864 +16,self_attn.q_proj,4,64,8,128,11.046875,34.03125,67108864 +16,self_attn.q_proj,4,128,8,128,12.8828125,33.015625,67108864 +16,self_attn.q_proj,8,32,8,128,0.56201171875,68.0625,67108864 +16,self_attn.q_proj,8,64,8,128,0.66845703125,66.03125,67108864 +16,self_attn.q_proj,8,128,8,128,0.7763671875,65.015625,67108864 +16,self_attn.v_proj,2,32,8,128,13.5859375,2.5078125,8388608 +16,self_attn.v_proj,2,64,8,128,15.828125,2.25390625,8388608 +16,self_attn.v_proj,2,128,8,128,17.859375,2.126953125,8388608 +16,self_attn.v_proj,3,32,8,128,5.8046875,3.5078125,8388608 +16,self_attn.v_proj,3,64,8,128,6.7734375,3.25390625,8388608 +16,self_attn.v_proj,3,128,8,128,7.64453125,3.126953125,8388608 +16,self_attn.v_proj,4,32,8,128,2.708984375,4.5078125,8388608 +16,self_attn.v_proj,4,64,8,128,3.16015625,4.25390625,8388608 +16,self_attn.v_proj,4,128,8,128,3.56640625,4.126953125,8388608 +16,self_attn.v_proj,8,32,8,128,0.1640625,8.5078125,8388608 +16,self_attn.v_proj,8,64,8,128,0.191162109375,8.25390625,8388608 +16,self_attn.v_proj,8,128,8,128,0.2149658203125,8.126953125,8388608 +16,self_attn.k_proj,2,32,8,128,26.9375,2.5078125,8388608 +16,self_attn.k_proj,2,64,8,128,32.5,2.25390625,8388608 +16,self_attn.k_proj,2,128,8,128,37.21875,2.126953125,8388608 +16,self_attn.k_proj,3,32,8,128,11.640625,3.5078125,8388608 +16,self_attn.k_proj,3,64,8,128,14.265625,3.25390625,8388608 +16,self_attn.k_proj,3,128,8,128,16.765625,3.126953125,8388608 +16,self_attn.k_proj,4,32,8,128,5.43359375,4.5078125,8388608 +16,self_attn.k_proj,4,64,8,128,6.65625,4.25390625,8388608 +16,self_attn.k_proj,4,128,8,128,7.83203125,4.126953125,8388608 +16,self_attn.k_proj,8,32,8,128,0.328857421875,8.5078125,8388608 +16,self_attn.k_proj,8,64,8,128,0.402587890625,8.25390625,8388608 +16,self_attn.k_proj,8,128,8,128,0.471923828125,8.126953125,8388608 +16,self_attn.o_proj,2,32,8,128,40.40625,20.0625,67108864 +16,self_attn.o_proj,2,64,8,128,47.125,18.03125,67108864 +16,self_attn.o_proj,2,128,8,128,53.25,17.015625,67108864 +16,self_attn.o_proj,3,32,8,128,17.296875,28.0625,67108864 +16,self_attn.o_proj,3,64,8,128,20.21875,26.03125,67108864 +16,self_attn.o_proj,3,128,8,128,22.921875,25.015625,67108864 +16,self_attn.o_proj,4,32,8,128,8.0703125,36.0625,67108864 +16,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864 +16,self_attn.o_proj,4,128,8,128,10.7109375,33.015625,67108864 +16,self_attn.o_proj,8,32,8,128,0.488525390625,68.0625,67108864 +16,self_attn.o_proj,8,64,8,128,0.57080078125,66.03125,67108864 +16,self_attn.o_proj,8,128,8,128,0.64501953125,65.015625,67108864 +16,mlp.gate_proj,2,32,8,128,84.9375,70.21875,234881024 +16,mlp.gate_proj,2,64,8,128,99.375,63.109375,234881024 +16,mlp.gate_proj,2,128,8,128,112.625,59.5546875,234881024 +16,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024 +16,mlp.gate_proj,3,64,8,128,42.5625,91.109375,234881024 +16,mlp.gate_proj,3,128,8,128,48.28125,87.5546875,234881024 +16,mlp.gate_proj,4,32,8,128,16.953125,126.21875,234881024 +16,mlp.gate_proj,4,64,8,128,19.875,119.109375,234881024 +16,mlp.gate_proj,4,128,8,128,22.53125,115.5546875,234881024 +16,mlp.gate_proj,8,32,8,128,1.03125,238.21875,234881024 +16,mlp.gate_proj,8,64,8,128,1.208984375,231.109375,234881024 +16,mlp.gate_proj,8,128,8,128,1.3671875,227.5546875,234881024 +16,mlp.up_proj,2,32,8,128,81.3125,70.21875,234881024 +16,mlp.up_proj,2,64,8,128,94.6875,63.109375,234881024 +16,mlp.up_proj,2,128,8,128,106.75,59.5546875,234881024 +16,mlp.up_proj,3,32,8,128,34.75,98.21875,234881024 +16,mlp.up_proj,3,64,8,128,40.53125,91.109375,234881024 +16,mlp.up_proj,3,128,8,128,45.71875,87.5546875,234881024 +16,mlp.up_proj,4,32,8,128,16.203125,126.21875,234881024 +16,mlp.up_proj,4,64,8,128,18.921875,119.109375,234881024 +16,mlp.up_proj,4,128,8,128,21.34375,115.5546875,234881024 +16,mlp.up_proj,8,32,8,128,0.98583984375,238.21875,234881024 +16,mlp.up_proj,8,64,8,128,1.150390625,231.109375,234881024 +16,mlp.up_proj,8,128,8,128,1.294921875,227.5546875,234881024 +16,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024 +16,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024 +16,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024 +16,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024 +16,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024 +16,mlp.down_proj,3,128,8,128,45.625,87.5546875,234881024 +16,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024 +16,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024 +16,mlp.down_proj,4,128,8,128,21.296875,115.5546875,234881024 +16,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024 +16,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +16,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +17,self_attn.q_proj,2,32,8,128,49.25,20.0625,67108864 +17,self_attn.q_proj,2,64,8,128,58.15625,18.03125,67108864 +17,self_attn.q_proj,2,128,8,128,67.3125,17.015625,67108864 +17,self_attn.q_proj,3,32,8,128,21.09375,28.0625,67108864 +17,self_attn.q_proj,3,64,8,128,24.953125,26.03125,67108864 +17,self_attn.q_proj,3,128,8,128,28.953125,25.015625,67108864 +17,self_attn.q_proj,4,32,8,128,9.84375,36.0625,67108864 +17,self_attn.q_proj,4,64,8,128,11.6484375,34.03125,67108864 +17,self_attn.q_proj,4,128,8,128,13.515625,33.015625,67108864 +17,self_attn.q_proj,8,32,8,128,0.59619140625,68.0625,67108864 +17,self_attn.q_proj,8,64,8,128,0.70458984375,66.03125,67108864 +17,self_attn.q_proj,8,128,8,128,0.814453125,65.015625,67108864 +17,self_attn.v_proj,2,32,8,128,13.4453125,2.5078125,8388608 +17,self_attn.v_proj,2,64,8,128,15.7734375,2.25390625,8388608 +17,self_attn.v_proj,2,128,8,128,17.90625,2.126953125,8388608 +17,self_attn.v_proj,3,32,8,128,5.74609375,3.5078125,8388608 +17,self_attn.v_proj,3,64,8,128,6.75,3.25390625,8388608 +17,self_attn.v_proj,3,128,8,128,7.6640625,3.126953125,8388608 +17,self_attn.v_proj,4,32,8,128,2.681640625,4.5078125,8388608 +17,self_attn.v_proj,4,64,8,128,3.1484375,4.25390625,8388608 +17,self_attn.v_proj,4,128,8,128,3.580078125,4.126953125,8388608 +17,self_attn.v_proj,8,32,8,128,0.1624755859375,8.5078125,8388608 +17,self_attn.v_proj,8,64,8,128,0.1904296875,8.25390625,8388608 +17,self_attn.v_proj,8,128,8,128,0.215576171875,8.126953125,8388608 +17,self_attn.k_proj,2,32,8,128,29.140625,2.5078125,8388608 +17,self_attn.k_proj,2,64,8,128,35.15625,2.25390625,8388608 +17,self_attn.k_proj,2,128,8,128,40.28125,2.126953125,8388608 +17,self_attn.k_proj,3,32,8,128,12.6015625,3.5078125,8388608 +17,self_attn.k_proj,3,64,8,128,15.4140625,3.25390625,8388608 +17,self_attn.k_proj,3,128,8,128,18.125,3.126953125,8388608 +17,self_attn.k_proj,4,32,8,128,5.87890625,4.5078125,8388608 +17,self_attn.k_proj,4,64,8,128,7.19140625,4.25390625,8388608 +17,self_attn.k_proj,4,128,8,128,8.4609375,4.126953125,8388608 +17,self_attn.k_proj,8,32,8,128,0.355712890625,8.5078125,8388608 +17,self_attn.k_proj,8,64,8,128,0.434814453125,8.25390625,8388608 +17,self_attn.k_proj,8,128,8,128,0.51025390625,8.126953125,8388608 +17,self_attn.o_proj,2,32,8,128,40.5625,20.0625,67108864 +17,self_attn.o_proj,2,64,8,128,47.25,18.03125,67108864 +17,self_attn.o_proj,2,128,8,128,53.34375,17.015625,67108864 +17,self_attn.o_proj,3,32,8,128,17.34375,28.0625,67108864 +17,self_attn.o_proj,3,64,8,128,20.234375,26.03125,67108864 +17,self_attn.o_proj,3,128,8,128,22.859375,25.015625,67108864 +17,self_attn.o_proj,4,32,8,128,8.09375,36.0625,67108864 +17,self_attn.o_proj,4,64,8,128,9.4453125,34.03125,67108864 +17,self_attn.o_proj,4,128,8,128,10.6796875,33.015625,67108864 +17,self_attn.o_proj,8,32,8,128,0.490234375,68.0625,67108864 +17,self_attn.o_proj,8,64,8,128,0.57177734375,66.03125,67108864 +17,self_attn.o_proj,8,128,8,128,0.6435546875,65.015625,67108864 +17,mlp.gate_proj,2,32,8,128,85.0,70.21875,234881024 +17,mlp.gate_proj,2,64,8,128,99.5,63.109375,234881024 +17,mlp.gate_proj,2,128,8,128,112.8125,59.5546875,234881024 +17,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024 +17,mlp.gate_proj,3,64,8,128,42.59375,91.109375,234881024 +17,mlp.gate_proj,3,128,8,128,48.34375,87.5546875,234881024 +17,mlp.gate_proj,4,32,8,128,16.96875,126.21875,234881024 +17,mlp.gate_proj,4,64,8,128,19.890625,119.109375,234881024 +17,mlp.gate_proj,4,128,8,128,22.5625,115.5546875,234881024 +17,mlp.gate_proj,8,32,8,128,1.0322265625,238.21875,234881024 +17,mlp.gate_proj,8,64,8,128,1.2099609375,231.109375,234881024 +17,mlp.gate_proj,8,128,8,128,1.369140625,227.5546875,234881024 +17,mlp.up_proj,2,32,8,128,81.0625,70.21875,234881024 +17,mlp.up_proj,2,64,8,128,94.375,63.109375,234881024 +17,mlp.up_proj,2,128,8,128,106.4375,59.5546875,234881024 +17,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024 +17,mlp.up_proj,3,64,8,128,40.40625,91.109375,234881024 +17,mlp.up_proj,3,128,8,128,45.5625,87.5546875,234881024 +17,mlp.up_proj,4,32,8,128,16.15625,126.21875,234881024 +17,mlp.up_proj,4,64,8,128,18.859375,119.109375,234881024 +17,mlp.up_proj,4,128,8,128,21.265625,115.5546875,234881024 +17,mlp.up_proj,8,32,8,128,0.982421875,238.21875,234881024 +17,mlp.up_proj,8,64,8,128,1.146484375,231.109375,234881024 +17,mlp.up_proj,8,128,8,128,1.2900390625,227.5546875,234881024 +17,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024 +17,mlp.down_proj,2,64,8,128,94.875,63.109375,234881024 +17,mlp.down_proj,2,128,8,128,106.9375,59.5546875,234881024 +17,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024 +17,mlp.down_proj,3,64,8,128,40.625,91.109375,234881024 +17,mlp.down_proj,3,128,8,128,45.75,87.5546875,234881024 +17,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024 +17,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024 +17,mlp.down_proj,4,128,8,128,21.359375,115.5546875,234881024 +17,mlp.down_proj,8,32,8,128,0.98876953125,238.21875,234881024 +17,mlp.down_proj,8,64,8,128,1.1533203125,231.109375,234881024 +17,mlp.down_proj,8,128,8,128,1.296875,227.5546875,234881024 +18,self_attn.q_proj,2,32,8,128,46.3125,20.0625,67108864 +18,self_attn.q_proj,2,64,8,128,54.71875,18.03125,67108864 +18,self_attn.q_proj,2,128,8,128,63.125,17.015625,67108864 +18,self_attn.q_proj,3,32,8,128,19.859375,28.0625,67108864 +18,self_attn.q_proj,3,64,8,128,23.53125,26.03125,67108864 +18,self_attn.q_proj,3,128,8,128,27.296875,25.015625,67108864 +18,self_attn.q_proj,4,32,8,128,9.265625,36.0625,67108864 +18,self_attn.q_proj,4,64,8,128,10.9765625,34.03125,67108864 +18,self_attn.q_proj,4,128,8,128,12.7421875,33.015625,67108864 +18,self_attn.q_proj,8,32,8,128,0.56103515625,68.0625,67108864 +18,self_attn.q_proj,8,64,8,128,0.6640625,66.03125,67108864 +18,self_attn.q_proj,8,128,8,128,0.76806640625,65.015625,67108864 +18,self_attn.v_proj,2,32,8,128,14.40625,2.5078125,8388608 +18,self_attn.v_proj,2,64,8,128,16.828125,2.25390625,8388608 +18,self_attn.v_proj,2,128,8,128,19.015625,2.126953125,8388608 +18,self_attn.v_proj,3,32,8,128,6.16015625,3.5078125,8388608 +18,self_attn.v_proj,3,64,8,128,7.1953125,3.25390625,8388608 +18,self_attn.v_proj,3,128,8,128,8.1328125,3.126953125,8388608 +18,self_attn.v_proj,4,32,8,128,2.873046875,4.5078125,8388608 +18,self_attn.v_proj,4,64,8,128,3.357421875,4.25390625,8388608 +18,self_attn.v_proj,4,128,8,128,3.798828125,4.126953125,8388608 +18,self_attn.v_proj,8,32,8,128,0.174072265625,8.5078125,8388608 +18,self_attn.v_proj,8,64,8,128,0.203125,8.25390625,8388608 +18,self_attn.v_proj,8,128,8,128,0.2288818359375,8.126953125,8388608 +18,self_attn.k_proj,2,32,8,128,26.359375,2.5078125,8388608 +18,self_attn.k_proj,2,64,8,128,31.640625,2.25390625,8388608 +18,self_attn.k_proj,2,128,8,128,36.03125,2.126953125,8388608 +18,self_attn.k_proj,3,32,8,128,11.3828125,3.5078125,8388608 +18,self_attn.k_proj,3,64,8,128,13.8515625,3.25390625,8388608 +18,self_attn.k_proj,3,128,8,128,16.203125,3.126953125,8388608 +18,self_attn.k_proj,4,32,8,128,5.3125,4.5078125,8388608 +18,self_attn.k_proj,4,64,8,128,6.46875,4.25390625,8388608 +18,self_attn.k_proj,4,128,8,128,7.56640625,4.126953125,8388608 +18,self_attn.k_proj,8,32,8,128,0.321533203125,8.5078125,8388608 +18,self_attn.k_proj,8,64,8,128,0.39111328125,8.25390625,8388608 +18,self_attn.k_proj,8,128,8,128,0.455810546875,8.126953125,8388608 +18,self_attn.o_proj,2,32,8,128,40.625,20.0625,67108864 +18,self_attn.o_proj,2,64,8,128,47.3125,18.03125,67108864 +18,self_attn.o_proj,2,128,8,128,53.375,17.015625,67108864 +18,self_attn.o_proj,3,32,8,128,17.375,28.0625,67108864 +18,self_attn.o_proj,3,64,8,128,20.265625,26.03125,67108864 +18,self_attn.o_proj,3,128,8,128,22.890625,25.015625,67108864 +18,self_attn.o_proj,4,32,8,128,8.109375,36.0625,67108864 +18,self_attn.o_proj,4,64,8,128,9.4609375,34.03125,67108864 +18,self_attn.o_proj,4,128,8,128,10.6953125,33.015625,67108864 +18,self_attn.o_proj,8,32,8,128,0.4912109375,68.0625,67108864 +18,self_attn.o_proj,8,64,8,128,0.57275390625,66.03125,67108864 +18,self_attn.o_proj,8,128,8,128,0.6455078125,65.015625,67108864 +18,mlp.gate_proj,2,32,8,128,85.25,70.21875,234881024 +18,mlp.gate_proj,2,64,8,128,99.75,63.109375,234881024 +18,mlp.gate_proj,2,128,8,128,113.125,59.5546875,234881024 +18,mlp.gate_proj,3,32,8,128,36.46875,98.21875,234881024 +18,mlp.gate_proj,3,64,8,128,42.71875,91.109375,234881024 +18,mlp.gate_proj,3,128,8,128,48.46875,87.5546875,234881024 +18,mlp.gate_proj,4,32,8,128,17.015625,126.21875,234881024 +18,mlp.gate_proj,4,64,8,128,19.9375,119.109375,234881024 +18,mlp.gate_proj,4,128,8,128,22.625,115.5546875,234881024 +18,mlp.gate_proj,8,32,8,128,1.03515625,238.21875,234881024 +18,mlp.gate_proj,8,64,8,128,1.212890625,231.109375,234881024 +18,mlp.gate_proj,8,128,8,128,1.3720703125,227.5546875,234881024 +18,mlp.up_proj,2,32,8,128,80.9375,70.21875,234881024 +18,mlp.up_proj,2,64,8,128,94.1875,63.109375,234881024 +18,mlp.up_proj,2,128,8,128,106.25,59.5546875,234881024 +18,mlp.up_proj,3,32,8,128,34.59375,98.21875,234881024 +18,mlp.up_proj,3,64,8,128,40.34375,91.109375,234881024 +18,mlp.up_proj,3,128,8,128,45.46875,87.5546875,234881024 +18,mlp.up_proj,4,32,8,128,16.125,126.21875,234881024 +18,mlp.up_proj,4,64,8,128,18.828125,119.109375,234881024 +18,mlp.up_proj,4,128,8,128,21.21875,115.5546875,234881024 +18,mlp.up_proj,8,32,8,128,0.98095703125,238.21875,234881024 +18,mlp.up_proj,8,64,8,128,1.14453125,231.109375,234881024 +18,mlp.up_proj,8,128,8,128,1.2880859375,227.5546875,234881024 +18,mlp.down_proj,2,32,8,128,81.625,70.21875,234881024 +18,mlp.down_proj,2,64,8,128,94.9375,63.109375,234881024 +18,mlp.down_proj,2,128,8,128,107.0625,59.5546875,234881024 +18,mlp.down_proj,3,32,8,128,34.875,98.21875,234881024 +18,mlp.down_proj,3,64,8,128,40.65625,91.109375,234881024 +18,mlp.down_proj,3,128,8,128,45.78125,87.5546875,234881024 +18,mlp.down_proj,4,32,8,128,16.265625,126.21875,234881024 +18,mlp.down_proj,4,64,8,128,18.96875,119.109375,234881024 +18,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024 +18,mlp.down_proj,8,32,8,128,0.98974609375,238.21875,234881024 +18,mlp.down_proj,8,64,8,128,1.154296875,231.109375,234881024 +18,mlp.down_proj,8,128,8,128,1.2978515625,227.5546875,234881024 +19,self_attn.q_proj,2,32,8,128,45.21875,20.0625,67108864 +19,self_attn.q_proj,2,64,8,128,53.4375,18.03125,67108864 +19,self_attn.q_proj,2,128,8,128,61.78125,17.015625,67108864 +19,self_attn.q_proj,3,32,8,128,19.375,28.0625,67108864 +19,self_attn.q_proj,3,64,8,128,22.953125,26.03125,67108864 +19,self_attn.q_proj,3,128,8,128,26.65625,25.015625,67108864 +19,self_attn.q_proj,4,32,8,128,9.0390625,36.0625,67108864 +19,self_attn.q_proj,4,64,8,128,10.7109375,34.03125,67108864 +19,self_attn.q_proj,4,128,8,128,12.4375,33.015625,67108864 +19,self_attn.q_proj,8,32,8,128,0.54736328125,68.0625,67108864 +19,self_attn.q_proj,8,64,8,128,0.64794921875,66.03125,67108864 +19,self_attn.q_proj,8,128,8,128,0.75,65.015625,67108864 +19,self_attn.v_proj,2,32,8,128,14.84375,2.5078125,8388608 +19,self_attn.v_proj,2,64,8,128,17.34375,2.25390625,8388608 +19,self_attn.v_proj,2,128,8,128,19.59375,2.126953125,8388608 +19,self_attn.v_proj,3,32,8,128,6.34375,3.5078125,8388608 +19,self_attn.v_proj,3,64,8,128,7.41796875,3.25390625,8388608 +19,self_attn.v_proj,3,128,8,128,8.3828125,3.126953125,8388608 +19,self_attn.v_proj,4,32,8,128,2.958984375,4.5078125,8388608 +19,self_attn.v_proj,4,64,8,128,3.4609375,4.25390625,8388608 +19,self_attn.v_proj,4,128,8,128,3.912109375,4.126953125,8388608 +19,self_attn.v_proj,8,32,8,128,0.1793212890625,8.5078125,8388608 +19,self_attn.v_proj,8,64,8,128,0.209228515625,8.25390625,8388608 +19,self_attn.v_proj,8,128,8,128,0.23583984375,8.126953125,8388608 +19,self_attn.k_proj,2,32,8,128,26.90625,2.5078125,8388608 +19,self_attn.k_proj,2,64,8,128,32.4375,2.25390625,8388608 +19,self_attn.k_proj,2,128,8,128,36.9375,2.126953125,8388608 +19,self_attn.k_proj,3,32,8,128,11.6796875,3.5078125,8388608 +19,self_attn.k_proj,3,64,8,128,14.34375,3.25390625,8388608 +19,self_attn.k_proj,3,128,8,128,16.875,3.126953125,8388608 +19,self_attn.k_proj,4,32,8,128,5.44921875,4.5078125,8388608 +19,self_attn.k_proj,4,64,8,128,6.69140625,4.25390625,8388608 +19,self_attn.k_proj,4,128,8,128,7.875,4.126953125,8388608 +19,self_attn.k_proj,8,32,8,128,0.329833984375,8.5078125,8388608 +19,self_attn.k_proj,8,64,8,128,0.40478515625,8.25390625,8388608 +19,self_attn.k_proj,8,128,8,128,0.474609375,8.126953125,8388608 +19,self_attn.o_proj,2,32,8,128,41.875,20.0625,67108864 +19,self_attn.o_proj,2,64,8,128,48.78125,18.03125,67108864 +19,self_attn.o_proj,2,128,8,128,55.03125,17.015625,67108864 +19,self_attn.o_proj,3,32,8,128,17.921875,28.0625,67108864 +19,self_attn.o_proj,3,64,8,128,20.90625,26.03125,67108864 +19,self_attn.o_proj,3,128,8,128,23.625,25.015625,67108864 +19,self_attn.o_proj,4,32,8,128,8.3671875,36.0625,67108864 +19,self_attn.o_proj,4,64,8,128,9.765625,34.03125,67108864 +19,self_attn.o_proj,4,128,8,128,11.0390625,33.015625,67108864 +19,self_attn.o_proj,8,32,8,128,0.50634765625,68.0625,67108864 +19,self_attn.o_proj,8,64,8,128,0.5908203125,66.03125,67108864 +19,self_attn.o_proj,8,128,8,128,0.66552734375,65.015625,67108864 +19,mlp.gate_proj,2,32,8,128,85.0625,70.21875,234881024 +19,mlp.gate_proj,2,64,8,128,99.375,63.109375,234881024 +19,mlp.gate_proj,2,128,8,128,112.5625,59.5546875,234881024 +19,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024 +19,mlp.gate_proj,3,64,8,128,42.5625,91.109375,234881024 +19,mlp.gate_proj,3,128,8,128,48.21875,87.5546875,234881024 +19,mlp.gate_proj,4,32,8,128,16.96875,126.21875,234881024 +19,mlp.gate_proj,4,64,8,128,19.859375,119.109375,234881024 +19,mlp.gate_proj,4,128,8,128,22.5,115.5546875,234881024 +19,mlp.gate_proj,8,32,8,128,1.0322265625,238.21875,234881024 +19,mlp.gate_proj,8,64,8,128,1.208984375,231.109375,234881024 +19,mlp.gate_proj,8,128,8,128,1.3642578125,227.5546875,234881024 +19,mlp.up_proj,2,32,8,128,81.25,70.21875,234881024 +19,mlp.up_proj,2,64,8,128,94.5625,63.109375,234881024 +19,mlp.up_proj,2,128,8,128,106.625,59.5546875,234881024 +19,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024 +19,mlp.up_proj,3,64,8,128,40.5,91.109375,234881024 +19,mlp.up_proj,3,128,8,128,45.625,87.5546875,234881024 +19,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024 +19,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024 +19,mlp.up_proj,4,128,8,128,21.296875,115.5546875,234881024 +19,mlp.up_proj,8,32,8,128,0.98486328125,238.21875,234881024 +19,mlp.up_proj,8,64,8,128,1.1494140625,231.109375,234881024 +19,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +19,mlp.down_proj,2,32,8,128,81.9375,70.21875,234881024 +19,mlp.down_proj,2,64,8,128,95.3125,63.109375,234881024 +19,mlp.down_proj,2,128,8,128,107.4375,59.5546875,234881024 +19,mlp.down_proj,3,32,8,128,35.03125,98.21875,234881024 +19,mlp.down_proj,3,64,8,128,40.8125,91.109375,234881024 +19,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024 +19,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024 +19,mlp.down_proj,4,64,8,128,19.046875,119.109375,234881024 +19,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024 +19,mlp.down_proj,8,32,8,128,0.9931640625,238.21875,234881024 +19,mlp.down_proj,8,64,8,128,1.158203125,231.109375,234881024 +19,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024 +20,self_attn.q_proj,2,32,8,128,45.46875,20.0625,67108864 +20,self_attn.q_proj,2,64,8,128,53.75,18.03125,67108864 +20,self_attn.q_proj,2,128,8,128,61.65625,17.015625,67108864 +20,self_attn.q_proj,3,32,8,128,19.5,28.0625,67108864 +20,self_attn.q_proj,3,64,8,128,23.140625,26.03125,67108864 +20,self_attn.q_proj,3,128,8,128,26.75,25.015625,67108864 +20,self_attn.q_proj,4,32,8,128,9.1015625,36.0625,67108864 +20,self_attn.q_proj,4,64,8,128,10.796875,34.03125,67108864 +20,self_attn.q_proj,4,128,8,128,12.484375,33.015625,67108864 +20,self_attn.q_proj,8,32,8,128,0.55126953125,68.0625,67108864 +20,self_attn.q_proj,8,64,8,128,0.6533203125,66.03125,67108864 +20,self_attn.q_proj,8,128,8,128,0.7529296875,65.015625,67108864 +20,self_attn.v_proj,2,32,8,128,13.9765625,2.5078125,8388608 +20,self_attn.v_proj,2,64,8,128,16.375,2.25390625,8388608 +20,self_attn.v_proj,2,128,8,128,18.53125,2.126953125,8388608 +20,self_attn.v_proj,3,32,8,128,5.9765625,3.5078125,8388608 +20,self_attn.v_proj,3,64,8,128,7.00390625,3.25390625,8388608 +20,self_attn.v_proj,3,128,8,128,7.9375,3.126953125,8388608 +20,self_attn.v_proj,4,32,8,128,2.7890625,4.5078125,8388608 +20,self_attn.v_proj,4,64,8,128,3.26953125,4.25390625,8388608 +20,self_attn.v_proj,4,128,8,128,3.703125,4.126953125,8388608 +20,self_attn.v_proj,8,32,8,128,0.1688232421875,8.5078125,8388608 +20,self_attn.v_proj,8,64,8,128,0.19775390625,8.25390625,8388608 +20,self_attn.v_proj,8,128,8,128,0.22314453125,8.126953125,8388608 +20,self_attn.k_proj,2,32,8,128,27.171875,2.5078125,8388608 +20,self_attn.k_proj,2,64,8,128,32.375,2.25390625,8388608 +20,self_attn.k_proj,2,128,8,128,36.53125,2.126953125,8388608 +20,self_attn.k_proj,3,32,8,128,11.84375,3.5078125,8388608 +20,self_attn.k_proj,3,64,8,128,14.4375,3.25390625,8388608 +20,self_attn.k_proj,3,128,8,128,16.875,3.126953125,8388608 +20,self_attn.k_proj,4,32,8,128,5.52734375,4.5078125,8388608 +20,self_attn.k_proj,4,64,8,128,6.7421875,4.25390625,8388608 +20,self_attn.k_proj,4,128,8,128,7.89453125,4.126953125,8388608 +20,self_attn.k_proj,8,32,8,128,0.33447265625,8.5078125,8388608 +20,self_attn.k_proj,8,64,8,128,0.40771484375,8.25390625,8388608 +20,self_attn.k_proj,8,128,8,128,0.475341796875,8.126953125,8388608 +20,self_attn.o_proj,2,32,8,128,39.84375,20.0625,67108864 +20,self_attn.o_proj,2,64,8,128,46.4375,18.03125,67108864 +20,self_attn.o_proj,2,128,8,128,52.40625,17.015625,67108864 +20,self_attn.o_proj,3,32,8,128,17.0625,28.0625,67108864 +20,self_attn.o_proj,3,64,8,128,19.921875,26.03125,67108864 +20,self_attn.o_proj,3,128,8,128,22.53125,25.015625,67108864 +20,self_attn.o_proj,4,32,8,128,7.96875,36.0625,67108864 +20,self_attn.o_proj,4,64,8,128,9.3125,34.03125,67108864 +20,self_attn.o_proj,4,128,8,128,10.5546875,33.015625,67108864 +20,self_attn.o_proj,8,32,8,128,0.4833984375,68.0625,67108864 +20,self_attn.o_proj,8,64,8,128,0.56494140625,66.03125,67108864 +20,self_attn.o_proj,8,128,8,128,0.6396484375,65.015625,67108864 +20,mlp.gate_proj,2,32,8,128,85.1875,70.21875,234881024 +20,mlp.gate_proj,2,64,8,128,99.625,63.109375,234881024 +20,mlp.gate_proj,2,128,8,128,112.8125,59.5546875,234881024 +20,mlp.gate_proj,3,32,8,128,36.46875,98.21875,234881024 +20,mlp.gate_proj,3,64,8,128,42.625,91.109375,234881024 +20,mlp.gate_proj,3,128,8,128,48.3125,87.5546875,234881024 +20,mlp.gate_proj,4,32,8,128,17.0,126.21875,234881024 +20,mlp.gate_proj,4,64,8,128,19.90625,119.109375,234881024 +20,mlp.gate_proj,4,128,8,128,22.546875,115.5546875,234881024 +20,mlp.gate_proj,8,32,8,128,1.0341796875,238.21875,234881024 +20,mlp.gate_proj,8,64,8,128,1.2109375,231.109375,234881024 +20,mlp.gate_proj,8,128,8,128,1.3681640625,227.5546875,234881024 +20,mlp.up_proj,2,32,8,128,81.4375,70.21875,234881024 +20,mlp.up_proj,2,64,8,128,94.8125,63.109375,234881024 +20,mlp.up_proj,2,128,8,128,106.9375,59.5546875,234881024 +20,mlp.up_proj,3,32,8,128,34.8125,98.21875,234881024 +20,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024 +20,mlp.up_proj,3,128,8,128,45.75,87.5546875,234881024 +20,mlp.up_proj,4,32,8,128,16.234375,126.21875,234881024 +20,mlp.up_proj,4,64,8,128,18.9375,119.109375,234881024 +20,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024 +20,mlp.up_proj,8,32,8,128,0.9873046875,238.21875,234881024 +20,mlp.up_proj,8,64,8,128,1.15234375,231.109375,234881024 +20,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +20,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024 +20,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024 +20,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024 +20,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024 +20,mlp.down_proj,3,64,8,128,40.78125,91.109375,234881024 +20,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024 +20,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024 +20,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024 +20,mlp.down_proj,4,128,8,128,21.4375,115.5546875,234881024 +20,mlp.down_proj,8,32,8,128,0.99267578125,238.21875,234881024 +20,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024 +20,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024 +21,self_attn.q_proj,2,32,8,128,44.0,20.0625,67108864 +21,self_attn.q_proj,2,64,8,128,52.125,18.03125,67108864 +21,self_attn.q_proj,2,128,8,128,59.9375,17.015625,67108864 +21,self_attn.q_proj,3,32,8,128,19.0,28.0625,67108864 +21,self_attn.q_proj,3,64,8,128,22.75,26.03125,67108864 +21,self_attn.q_proj,3,128,8,128,26.6875,25.015625,67108864 +21,self_attn.q_proj,4,32,8,128,8.859375,36.0625,67108864 +21,self_attn.q_proj,4,64,8,128,10.625,34.03125,67108864 +21,self_attn.q_proj,4,128,8,128,12.4609375,33.015625,67108864 +21,self_attn.q_proj,8,32,8,128,0.53662109375,68.0625,67108864 +21,self_attn.q_proj,8,64,8,128,0.64306640625,66.03125,67108864 +21,self_attn.q_proj,8,128,8,128,0.7509765625,65.015625,67108864 +21,self_attn.v_proj,2,32,8,128,13.7890625,2.5078125,8388608 +21,self_attn.v_proj,2,64,8,128,16.109375,2.25390625,8388608 +21,self_attn.v_proj,2,128,8,128,18.21875,2.126953125,8388608 +21,self_attn.v_proj,3,32,8,128,5.89453125,3.5078125,8388608 +21,self_attn.v_proj,3,64,8,128,6.8984375,3.25390625,8388608 +21,self_attn.v_proj,3,128,8,128,7.80078125,3.126953125,8388608 +21,self_attn.v_proj,4,32,8,128,2.75,4.5078125,8388608 +21,self_attn.v_proj,4,64,8,128,3.216796875,4.25390625,8388608 +21,self_attn.v_proj,4,128,8,128,3.642578125,4.126953125,8388608 +21,self_attn.v_proj,8,32,8,128,0.16650390625,8.5078125,8388608 +21,self_attn.v_proj,8,64,8,128,0.194580078125,8.25390625,8388608 +21,self_attn.v_proj,8,128,8,128,0.219482421875,8.126953125,8388608 +21,self_attn.k_proj,2,32,8,128,26.78125,2.5078125,8388608 +21,self_attn.k_proj,2,64,8,128,31.96875,2.25390625,8388608 +21,self_attn.k_proj,2,128,8,128,36.09375,2.126953125,8388608 +21,self_attn.k_proj,3,32,8,128,11.921875,3.5078125,8388608 +21,self_attn.k_proj,3,64,8,128,14.7734375,3.25390625,8388608 +21,self_attn.k_proj,3,128,8,128,17.484375,3.126953125,8388608 +21,self_attn.k_proj,4,32,8,128,5.56640625,4.5078125,8388608 +21,self_attn.k_proj,4,64,8,128,6.91015625,4.25390625,8388608 +21,self_attn.k_proj,4,128,8,128,8.2109375,4.126953125,8388608 +21,self_attn.k_proj,8,32,8,128,0.3369140625,8.5078125,8388608 +21,self_attn.k_proj,8,64,8,128,0.417724609375,8.25390625,8388608 +21,self_attn.k_proj,8,128,8,128,0.4951171875,8.126953125,8388608 +21,self_attn.o_proj,2,32,8,128,39.4375,20.0625,67108864 +21,self_attn.o_proj,2,64,8,128,45.96875,18.03125,67108864 +21,self_attn.o_proj,2,128,8,128,51.875,17.015625,67108864 +21,self_attn.o_proj,3,32,8,128,16.875,28.0625,67108864 +21,self_attn.o_proj,3,64,8,128,19.703125,26.03125,67108864 +21,self_attn.o_proj,3,128,8,128,22.28125,25.015625,67108864 +21,self_attn.o_proj,4,32,8,128,7.875,36.0625,67108864 +21,self_attn.o_proj,4,64,8,128,9.1953125,34.03125,67108864 +21,self_attn.o_proj,4,128,8,128,10.40625,33.015625,67108864 +21,self_attn.o_proj,8,32,8,128,0.476806640625,68.0625,67108864 +21,self_attn.o_proj,8,64,8,128,0.55615234375,66.03125,67108864 +21,self_attn.o_proj,8,128,8,128,0.62744140625,65.015625,67108864 +21,mlp.gate_proj,2,32,8,128,84.875,70.21875,234881024 +21,mlp.gate_proj,2,64,8,128,99.1875,63.109375,234881024 +21,mlp.gate_proj,2,128,8,128,112.25,59.5546875,234881024 +21,mlp.gate_proj,3,32,8,128,36.3125,98.21875,234881024 +21,mlp.gate_proj,3,64,8,128,42.4375,91.109375,234881024 +21,mlp.gate_proj,3,128,8,128,48.0625,87.5546875,234881024 +21,mlp.gate_proj,4,32,8,128,16.9375,126.21875,234881024 +21,mlp.gate_proj,4,64,8,128,19.828125,119.109375,234881024 +21,mlp.gate_proj,4,128,8,128,22.421875,115.5546875,234881024 +21,mlp.gate_proj,8,32,8,128,1.0302734375,238.21875,234881024 +21,mlp.gate_proj,8,64,8,128,1.2060546875,231.109375,234881024 +21,mlp.gate_proj,8,128,8,128,1.3603515625,227.5546875,234881024 +21,mlp.up_proj,2,32,8,128,81.6875,70.21875,234881024 +21,mlp.up_proj,2,64,8,128,95.0625,63.109375,234881024 +21,mlp.up_proj,2,128,8,128,107.25,59.5546875,234881024 +21,mlp.up_proj,3,32,8,128,34.9375,98.21875,234881024 +21,mlp.up_proj,3,64,8,128,40.71875,91.109375,234881024 +21,mlp.up_proj,3,128,8,128,45.90625,87.5546875,234881024 +21,mlp.up_proj,4,32,8,128,16.28125,126.21875,234881024 +21,mlp.up_proj,4,64,8,128,19.0,119.109375,234881024 +21,mlp.up_proj,4,128,8,128,21.421875,115.5546875,234881024 +21,mlp.up_proj,8,32,8,128,0.99072265625,238.21875,234881024 +21,mlp.up_proj,8,64,8,128,1.1552734375,231.109375,234881024 +21,mlp.up_proj,8,128,8,128,1.298828125,227.5546875,234881024 +21,mlp.down_proj,2,32,8,128,82.0,70.21875,234881024 +21,mlp.down_proj,2,64,8,128,95.375,63.109375,234881024 +21,mlp.down_proj,2,128,8,128,107.5,59.5546875,234881024 +21,mlp.down_proj,3,32,8,128,35.03125,98.21875,234881024 +21,mlp.down_proj,3,64,8,128,40.8125,91.109375,234881024 +21,mlp.down_proj,3,128,8,128,45.96875,87.5546875,234881024 +21,mlp.down_proj,4,32,8,128,16.34375,126.21875,234881024 +21,mlp.down_proj,4,64,8,128,19.0625,119.109375,234881024 +21,mlp.down_proj,4,128,8,128,21.46875,115.5546875,234881024 +21,mlp.down_proj,8,32,8,128,0.994140625,238.21875,234881024 +21,mlp.down_proj,8,64,8,128,1.1591796875,231.109375,234881024 +21,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024 +22,self_attn.q_proj,2,32,8,128,42.4375,20.0625,67108864 +22,self_attn.q_proj,2,64,8,128,50.625,18.03125,67108864 +22,self_attn.q_proj,2,128,8,128,58.65625,17.015625,67108864 +22,self_attn.q_proj,3,32,8,128,18.21875,28.0625,67108864 +22,self_attn.q_proj,3,64,8,128,21.859375,26.03125,67108864 +22,self_attn.q_proj,3,128,8,128,25.625,25.015625,67108864 +22,self_attn.q_proj,4,32,8,128,8.5,36.0625,67108864 +22,self_attn.q_proj,4,64,8,128,10.203125,34.03125,67108864 +22,self_attn.q_proj,4,128,8,128,11.9609375,33.015625,67108864 +22,self_attn.q_proj,8,32,8,128,0.5146484375,68.0625,67108864 +22,self_attn.q_proj,8,64,8,128,0.61767578125,66.03125,67108864 +22,self_attn.q_proj,8,128,8,128,0.72119140625,65.015625,67108864 +22,self_attn.v_proj,2,32,8,128,15.328125,2.5078125,8388608 +22,self_attn.v_proj,2,64,8,128,17.921875,2.25390625,8388608 +22,self_attn.v_proj,2,128,8,128,20.328125,2.126953125,8388608 +22,self_attn.v_proj,3,32,8,128,6.55078125,3.5078125,8388608 +22,self_attn.v_proj,3,64,8,128,7.671875,3.25390625,8388608 +22,self_attn.v_proj,3,128,8,128,8.703125,3.126953125,8388608 +22,self_attn.v_proj,4,32,8,128,3.056640625,4.5078125,8388608 +22,self_attn.v_proj,4,64,8,128,3.58203125,4.25390625,8388608 +22,self_attn.v_proj,4,128,8,128,4.0625,4.126953125,8388608 +22,self_attn.v_proj,8,32,8,128,0.1851806640625,8.5078125,8388608 +22,self_attn.v_proj,8,64,8,128,0.2166748046875,8.25390625,8388608 +22,self_attn.v_proj,8,128,8,128,0.2447509765625,8.126953125,8388608 +22,self_attn.k_proj,2,32,8,128,27.0,2.5078125,8388608 +22,self_attn.k_proj,2,64,8,128,32.5,2.25390625,8388608 +22,self_attn.k_proj,2,128,8,128,36.9375,2.126953125,8388608 +22,self_attn.k_proj,3,32,8,128,11.7734375,3.5078125,8388608 +22,self_attn.k_proj,3,64,8,128,14.4765625,3.25390625,8388608 +22,self_attn.k_proj,3,128,8,128,17.0625,3.126953125,8388608 +22,self_attn.k_proj,4,32,8,128,5.4921875,4.5078125,8388608 +22,self_attn.k_proj,4,64,8,128,6.75,4.25390625,8388608 +22,self_attn.k_proj,4,128,8,128,7.97265625,4.126953125,8388608 +22,self_attn.k_proj,8,32,8,128,0.33251953125,8.5078125,8388608 +22,self_attn.k_proj,8,64,8,128,0.408203125,8.25390625,8388608 +22,self_attn.k_proj,8,128,8,128,0.480224609375,8.126953125,8388608 +22,self_attn.o_proj,2,32,8,128,40.78125,20.0625,67108864 +22,self_attn.o_proj,2,64,8,128,47.5,18.03125,67108864 +22,self_attn.o_proj,2,128,8,128,53.53125,17.015625,67108864 +22,self_attn.o_proj,3,32,8,128,17.4375,28.0625,67108864 +22,self_attn.o_proj,3,64,8,128,20.3125,26.03125,67108864 +22,self_attn.o_proj,3,128,8,128,22.921875,25.015625,67108864 +22,self_attn.o_proj,4,32,8,128,8.1328125,36.0625,67108864 +22,self_attn.o_proj,4,64,8,128,9.484375,34.03125,67108864 +22,self_attn.o_proj,4,128,8,128,10.703125,33.015625,67108864 +22,self_attn.o_proj,8,32,8,128,0.49267578125,68.0625,67108864 +22,self_attn.o_proj,8,64,8,128,0.57373046875,66.03125,67108864 +22,self_attn.o_proj,8,128,8,128,0.64501953125,65.015625,67108864 +22,mlp.gate_proj,2,32,8,128,84.8125,70.21875,234881024 +22,mlp.gate_proj,2,64,8,128,99.0625,63.109375,234881024 +22,mlp.gate_proj,2,128,8,128,112.0625,59.5546875,234881024 +22,mlp.gate_proj,3,32,8,128,36.28125,98.21875,234881024 +22,mlp.gate_proj,3,64,8,128,42.40625,91.109375,234881024 +22,mlp.gate_proj,3,128,8,128,47.96875,87.5546875,234881024 +22,mlp.gate_proj,4,32,8,128,16.921875,126.21875,234881024 +22,mlp.gate_proj,4,64,8,128,19.796875,119.109375,234881024 +22,mlp.gate_proj,4,128,8,128,22.375,115.5546875,234881024 +22,mlp.gate_proj,8,32,8,128,1.029296875,238.21875,234881024 +22,mlp.gate_proj,8,64,8,128,1.2041015625,231.109375,234881024 +22,mlp.gate_proj,8,128,8,128,1.3583984375,227.5546875,234881024 +22,mlp.up_proj,2,32,8,128,81.8125,70.21875,234881024 +22,mlp.up_proj,2,64,8,128,95.25,63.109375,234881024 +22,mlp.up_proj,2,128,8,128,107.375,59.5546875,234881024 +22,mlp.up_proj,3,32,8,128,34.96875,98.21875,234881024 +22,mlp.up_proj,3,64,8,128,40.75,91.109375,234881024 +22,mlp.up_proj,3,128,8,128,45.9375,87.5546875,234881024 +22,mlp.up_proj,4,32,8,128,16.3125,126.21875,234881024 +22,mlp.up_proj,4,64,8,128,19.03125,119.109375,234881024 +22,mlp.up_proj,4,128,8,128,21.4375,115.5546875,234881024 +22,mlp.up_proj,8,32,8,128,0.9921875,238.21875,234881024 +22,mlp.up_proj,8,64,8,128,1.1572265625,231.109375,234881024 +22,mlp.up_proj,8,128,8,128,1.30078125,227.5546875,234881024 +22,mlp.down_proj,2,32,8,128,81.9375,70.21875,234881024 +22,mlp.down_proj,2,64,8,128,95.3125,63.109375,234881024 +22,mlp.down_proj,2,128,8,128,107.4375,59.5546875,234881024 +22,mlp.down_proj,3,32,8,128,35.03125,98.21875,234881024 +22,mlp.down_proj,3,64,8,128,40.8125,91.109375,234881024 +22,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024 +22,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024 +22,mlp.down_proj,4,64,8,128,19.046875,119.109375,234881024 +22,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024 +22,mlp.down_proj,8,32,8,128,0.99365234375,238.21875,234881024 +22,mlp.down_proj,8,64,8,128,1.158203125,231.109375,234881024 +22,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024 +23,self_attn.q_proj,2,32,8,128,44.6875,20.0625,67108864 +23,self_attn.q_proj,2,64,8,128,52.875,18.03125,67108864 +23,self_attn.q_proj,2,128,8,128,60.71875,17.015625,67108864 +23,self_attn.q_proj,3,32,8,128,19.140625,28.0625,67108864 +23,self_attn.q_proj,3,64,8,128,22.71875,26.03125,67108864 +23,self_attn.q_proj,3,128,8,128,26.234375,25.015625,67108864 +23,self_attn.q_proj,4,32,8,128,8.9296875,36.0625,67108864 +23,self_attn.q_proj,4,64,8,128,10.6015625,34.03125,67108864 +23,self_attn.q_proj,4,128,8,128,12.2421875,33.015625,67108864 +23,self_attn.q_proj,8,32,8,128,0.541015625,68.0625,67108864 +23,self_attn.q_proj,8,64,8,128,0.6416015625,66.03125,67108864 +23,self_attn.q_proj,8,128,8,128,0.73828125,65.015625,67108864 +23,self_attn.v_proj,2,32,8,128,14.8671875,2.5078125,8388608 +23,self_attn.v_proj,2,64,8,128,17.390625,2.25390625,8388608 +23,self_attn.v_proj,2,128,8,128,19.671875,2.126953125,8388608 +23,self_attn.v_proj,3,32,8,128,6.36328125,3.5078125,8388608 +23,self_attn.v_proj,3,64,8,128,7.44140625,3.25390625,8388608 +23,self_attn.v_proj,3,128,8,128,8.421875,3.126953125,8388608 +23,self_attn.v_proj,4,32,8,128,2.966796875,4.5078125,8388608 +23,self_attn.v_proj,4,64,8,128,3.470703125,4.25390625,8388608 +23,self_attn.v_proj,4,128,8,128,3.931640625,4.126953125,8388608 +23,self_attn.v_proj,8,32,8,128,0.1796875,8.5078125,8388608 +23,self_attn.v_proj,8,64,8,128,0.2100830078125,8.25390625,8388608 +23,self_attn.v_proj,8,128,8,128,0.2369384765625,8.126953125,8388608 +23,self_attn.k_proj,2,32,8,128,27.078125,2.5078125,8388608 +23,self_attn.k_proj,2,64,8,128,32.46875,2.25390625,8388608 +23,self_attn.k_proj,2,128,8,128,37.0625,2.126953125,8388608 +23,self_attn.k_proj,3,32,8,128,11.6953125,3.5078125,8388608 +23,self_attn.k_proj,3,64,8,128,14.203125,3.25390625,8388608 +23,self_attn.k_proj,3,128,8,128,16.59375,3.126953125,8388608 +23,self_attn.k_proj,4,32,8,128,5.45703125,4.5078125,8388608 +23,self_attn.k_proj,4,64,8,128,6.625,4.25390625,8388608 +23,self_attn.k_proj,4,128,8,128,7.75,4.126953125,8388608 +23,self_attn.k_proj,8,32,8,128,0.330322265625,8.5078125,8388608 +23,self_attn.k_proj,8,64,8,128,0.400634765625,8.25390625,8388608 +23,self_attn.k_proj,8,128,8,128,0.466552734375,8.126953125,8388608 +23,self_attn.o_proj,2,32,8,128,40.375,20.0625,67108864 +23,self_attn.o_proj,2,64,8,128,47.03125,18.03125,67108864 +23,self_attn.o_proj,2,128,8,128,53.03125,17.015625,67108864 +23,self_attn.o_proj,3,32,8,128,17.265625,28.0625,67108864 +23,self_attn.o_proj,3,64,8,128,20.15625,26.03125,67108864 +23,self_attn.o_proj,3,128,8,128,22.78125,25.015625,67108864 +23,self_attn.o_proj,4,32,8,128,8.0703125,36.0625,67108864 +23,self_attn.o_proj,4,64,8,128,9.4296875,34.03125,67108864 +23,self_attn.o_proj,4,128,8,128,10.6640625,33.015625,67108864 +23,self_attn.o_proj,8,32,8,128,0.4892578125,68.0625,67108864 +23,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864 +23,self_attn.o_proj,8,128,8,128,0.6455078125,65.015625,67108864 +23,mlp.gate_proj,2,32,8,128,84.8125,70.21875,234881024 +23,mlp.gate_proj,2,64,8,128,99.0625,63.109375,234881024 +23,mlp.gate_proj,2,128,8,128,112.125,59.5546875,234881024 +23,mlp.gate_proj,3,32,8,128,36.28125,98.21875,234881024 +23,mlp.gate_proj,3,64,8,128,42.40625,91.109375,234881024 +23,mlp.gate_proj,3,128,8,128,48.0,87.5546875,234881024 +23,mlp.gate_proj,4,32,8,128,16.921875,126.21875,234881024 +23,mlp.gate_proj,4,64,8,128,19.796875,119.109375,234881024 +23,mlp.gate_proj,4,128,8,128,22.40625,115.5546875,234881024 +23,mlp.gate_proj,8,32,8,128,1.029296875,238.21875,234881024 +23,mlp.gate_proj,8,64,8,128,1.205078125,231.109375,234881024 +23,mlp.gate_proj,8,128,8,128,1.359375,227.5546875,234881024 +23,mlp.up_proj,2,32,8,128,81.875,70.21875,234881024 +23,mlp.up_proj,2,64,8,128,95.3125,63.109375,234881024 +23,mlp.up_proj,2,128,8,128,107.4375,59.5546875,234881024 +23,mlp.up_proj,3,32,8,128,35.0,98.21875,234881024 +23,mlp.up_proj,3,64,8,128,40.78125,91.109375,234881024 +23,mlp.up_proj,3,128,8,128,45.96875,87.5546875,234881024 +23,mlp.up_proj,4,32,8,128,16.3125,126.21875,234881024 +23,mlp.up_proj,4,64,8,128,19.046875,119.109375,234881024 +23,mlp.up_proj,4,128,8,128,21.453125,115.5546875,234881024 +23,mlp.up_proj,8,32,8,128,0.99267578125,238.21875,234881024 +23,mlp.up_proj,8,64,8,128,1.158203125,231.109375,234881024 +23,mlp.up_proj,8,128,8,128,1.3017578125,227.5546875,234881024 +23,mlp.down_proj,2,32,8,128,81.8125,70.21875,234881024 +23,mlp.down_proj,2,64,8,128,95.1875,63.109375,234881024 +23,mlp.down_proj,2,128,8,128,107.3125,59.5546875,234881024 +23,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024 +23,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024 +23,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024 +23,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024 +23,mlp.down_proj,4,64,8,128,19.015625,119.109375,234881024 +23,mlp.down_proj,4,128,8,128,21.40625,115.5546875,234881024 +23,mlp.down_proj,8,32,8,128,0.99169921875,238.21875,234881024 +23,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024 +23,mlp.down_proj,8,128,8,128,1.2998046875,227.5546875,234881024 +24,self_attn.q_proj,2,32,8,128,45.46875,20.0625,67108864 +24,self_attn.q_proj,2,64,8,128,54.125,18.03125,67108864 +24,self_attn.q_proj,2,128,8,128,62.25,17.015625,67108864 +24,self_attn.q_proj,3,32,8,128,19.5,28.0625,67108864 +24,self_attn.q_proj,3,64,8,128,23.265625,26.03125,67108864 +24,self_attn.q_proj,3,128,8,128,26.984375,25.015625,67108864 +24,self_attn.q_proj,4,32,8,128,9.09375,36.0625,67108864 +24,self_attn.q_proj,4,64,8,128,10.859375,34.03125,67108864 +24,self_attn.q_proj,4,128,8,128,12.5859375,33.015625,67108864 +24,self_attn.q_proj,8,32,8,128,0.55078125,68.0625,67108864 +24,self_attn.q_proj,8,64,8,128,0.6572265625,66.03125,67108864 +24,self_attn.q_proj,8,128,8,128,0.75927734375,65.015625,67108864 +24,self_attn.v_proj,2,32,8,128,13.6640625,2.5078125,8388608 +24,self_attn.v_proj,2,64,8,128,15.953125,2.25390625,8388608 +24,self_attn.v_proj,2,128,8,128,18.015625,2.126953125,8388608 +24,self_attn.v_proj,3,32,8,128,5.84375,3.5078125,8388608 +24,self_attn.v_proj,3,64,8,128,6.8203125,3.25390625,8388608 +24,self_attn.v_proj,3,128,8,128,7.71484375,3.126953125,8388608 +24,self_attn.v_proj,4,32,8,128,2.7265625,4.5078125,8388608 +24,self_attn.v_proj,4,64,8,128,3.18359375,4.25390625,8388608 +24,self_attn.v_proj,4,128,8,128,3.599609375,4.126953125,8388608 +24,self_attn.v_proj,8,32,8,128,0.1651611328125,8.5078125,8388608 +24,self_attn.v_proj,8,64,8,128,0.192626953125,8.25390625,8388608 +24,self_attn.v_proj,8,128,8,128,0.217041015625,8.126953125,8388608 +24,self_attn.k_proj,2,32,8,128,26.734375,2.5078125,8388608 +24,self_attn.k_proj,2,64,8,128,32.0,2.25390625,8388608 +24,self_attn.k_proj,2,128,8,128,36.40625,2.126953125,8388608 +24,self_attn.k_proj,3,32,8,128,11.546875,3.5078125,8388608 +24,self_attn.k_proj,3,64,8,128,13.984375,3.25390625,8388608 +24,self_attn.k_proj,3,128,8,128,16.28125,3.126953125,8388608 +24,self_attn.k_proj,4,32,8,128,5.390625,4.5078125,8388608 +24,self_attn.k_proj,4,64,8,128,6.52734375,4.25390625,8388608 +24,self_attn.k_proj,4,128,8,128,7.60546875,4.126953125,8388608 +24,self_attn.k_proj,8,32,8,128,0.326171875,8.5078125,8388608 +24,self_attn.k_proj,8,64,8,128,0.394775390625,8.25390625,8388608 +24,self_attn.k_proj,8,128,8,128,0.45849609375,8.126953125,8388608 +24,self_attn.o_proj,2,32,8,128,39.0625,20.0625,67108864 +24,self_attn.o_proj,2,64,8,128,45.625,18.03125,67108864 +24,self_attn.o_proj,2,128,8,128,51.65625,17.015625,67108864 +24,self_attn.o_proj,3,32,8,128,16.71875,28.0625,67108864 +24,self_attn.o_proj,3,64,8,128,19.59375,26.03125,67108864 +24,self_attn.o_proj,3,128,8,128,22.296875,25.015625,67108864 +24,self_attn.o_proj,4,32,8,128,7.8046875,36.0625,67108864 +24,self_attn.o_proj,4,64,8,128,9.1484375,34.03125,67108864 +24,self_attn.o_proj,4,128,8,128,10.421875,33.015625,67108864 +24,self_attn.o_proj,8,32,8,128,0.472412109375,68.0625,67108864 +24,self_attn.o_proj,8,64,8,128,0.55322265625,66.03125,67108864 +24,self_attn.o_proj,8,128,8,128,0.6279296875,65.015625,67108864 +24,mlp.gate_proj,2,32,8,128,84.375,70.21875,234881024 +24,mlp.gate_proj,2,64,8,128,98.625,63.109375,234881024 +24,mlp.gate_proj,2,128,8,128,111.6875,59.5546875,234881024 +24,mlp.gate_proj,3,32,8,128,36.09375,98.21875,234881024 +24,mlp.gate_proj,3,64,8,128,42.21875,91.109375,234881024 +24,mlp.gate_proj,3,128,8,128,47.84375,87.5546875,234881024 +24,mlp.gate_proj,4,32,8,128,16.84375,126.21875,234881024 +24,mlp.gate_proj,4,64,8,128,19.71875,119.109375,234881024 +24,mlp.gate_proj,4,128,8,128,22.328125,115.5546875,234881024 +24,mlp.gate_proj,8,32,8,128,1.0244140625,238.21875,234881024 +24,mlp.gate_proj,8,64,8,128,1.19921875,231.109375,234881024 +24,mlp.gate_proj,8,128,8,128,1.3544921875,227.5546875,234881024 +24,mlp.up_proj,2,32,8,128,82.0,70.21875,234881024 +24,mlp.up_proj,2,64,8,128,95.5,63.109375,234881024 +24,mlp.up_proj,2,128,8,128,107.75,59.5546875,234881024 +24,mlp.up_proj,3,32,8,128,35.0625,98.21875,234881024 +24,mlp.up_proj,3,64,8,128,40.90625,91.109375,234881024 +24,mlp.up_proj,3,128,8,128,46.125,87.5546875,234881024 +24,mlp.up_proj,4,32,8,128,16.359375,126.21875,234881024 +24,mlp.up_proj,4,64,8,128,19.09375,119.109375,234881024 +24,mlp.up_proj,4,128,8,128,21.53125,115.5546875,234881024 +24,mlp.up_proj,8,32,8,128,0.99462890625,238.21875,234881024 +24,mlp.up_proj,8,64,8,128,1.1611328125,231.109375,234881024 +24,mlp.up_proj,8,128,8,128,1.3056640625,227.5546875,234881024 +24,mlp.down_proj,2,32,8,128,81.8125,70.21875,234881024 +24,mlp.down_proj,2,64,8,128,95.1875,63.109375,234881024 +24,mlp.down_proj,2,128,8,128,107.4375,59.5546875,234881024 +24,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024 +24,mlp.down_proj,3,64,8,128,40.78125,91.109375,234881024 +24,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024 +24,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024 +24,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024 +24,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024 +24,mlp.down_proj,8,32,8,128,0.9921875,238.21875,234881024 +24,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024 +24,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024 +25,self_attn.q_proj,2,32,8,128,48.09375,20.0625,67108864 +25,self_attn.q_proj,2,64,8,128,56.78125,18.03125,67108864 +25,self_attn.q_proj,2,128,8,128,65.9375,17.015625,67108864 +25,self_attn.q_proj,3,32,8,128,20.640625,28.0625,67108864 +25,self_attn.q_proj,3,64,8,128,24.515625,26.03125,67108864 +25,self_attn.q_proj,3,128,8,128,28.765625,25.015625,67108864 +25,self_attn.q_proj,4,32,8,128,9.6328125,36.0625,67108864 +25,self_attn.q_proj,4,64,8,128,11.4375,34.03125,67108864 +25,self_attn.q_proj,4,128,8,128,13.4296875,33.015625,67108864 +25,self_attn.q_proj,8,32,8,128,0.58349609375,68.0625,67108864 +25,self_attn.q_proj,8,64,8,128,0.69189453125,66.03125,67108864 +25,self_attn.q_proj,8,128,8,128,0.8095703125,65.015625,67108864 +25,self_attn.v_proj,2,32,8,128,13.2421875,2.5078125,8388608 +25,self_attn.v_proj,2,64,8,128,15.46875,2.25390625,8388608 +25,self_attn.v_proj,2,128,8,128,17.484375,2.126953125,8388608 +25,self_attn.v_proj,3,32,8,128,5.66015625,3.5078125,8388608 +25,self_attn.v_proj,3,64,8,128,6.6171875,3.25390625,8388608 +25,self_attn.v_proj,3,128,8,128,7.48046875,3.126953125,8388608 +25,self_attn.v_proj,4,32,8,128,2.640625,4.5078125,8388608 +25,self_attn.v_proj,4,64,8,128,3.0859375,4.25390625,8388608 +25,self_attn.v_proj,4,128,8,128,3.4921875,4.126953125,8388608 +25,self_attn.v_proj,8,32,8,128,0.159912109375,8.5078125,8388608 +25,self_attn.v_proj,8,64,8,128,0.186767578125,8.25390625,8388608 +25,self_attn.v_proj,8,128,8,128,0.2105712890625,8.126953125,8388608 +25,self_attn.k_proj,2,32,8,128,30.390625,2.5078125,8388608 +25,self_attn.k_proj,2,64,8,128,36.5625,2.25390625,8388608 +25,self_attn.k_proj,2,128,8,128,41.625,2.126953125,8388608 +25,self_attn.k_proj,3,32,8,128,13.296875,3.5078125,8388608 +25,self_attn.k_proj,3,64,8,128,16.4375,3.25390625,8388608 +25,self_attn.k_proj,3,128,8,128,19.5,3.126953125,8388608 +25,self_attn.k_proj,4,32,8,128,6.20703125,4.5078125,8388608 +25,self_attn.k_proj,4,64,8,128,7.6796875,4.25390625,8388608 +25,self_attn.k_proj,4,128,8,128,9.1171875,4.126953125,8388608 +25,self_attn.k_proj,8,32,8,128,0.37548828125,8.5078125,8388608 +25,self_attn.k_proj,8,64,8,128,0.464111328125,8.25390625,8388608 +25,self_attn.k_proj,8,128,8,128,0.54931640625,8.126953125,8388608 +25,self_attn.o_proj,2,32,8,128,39.8125,20.0625,67108864 +25,self_attn.o_proj,2,64,8,128,46.375,18.03125,67108864 +25,self_attn.o_proj,2,128,8,128,52.34375,17.015625,67108864 +25,self_attn.o_proj,3,32,8,128,17.03125,28.0625,67108864 +25,self_attn.o_proj,3,64,8,128,19.890625,26.03125,67108864 +25,self_attn.o_proj,3,128,8,128,22.484375,25.015625,67108864 +25,self_attn.o_proj,4,32,8,128,7.953125,36.0625,67108864 +25,self_attn.o_proj,4,64,8,128,9.2890625,34.03125,67108864 +25,self_attn.o_proj,4,128,8,128,10.5234375,33.015625,67108864 +25,self_attn.o_proj,8,32,8,128,0.481689453125,68.0625,67108864 +25,self_attn.o_proj,8,64,8,128,0.56201171875,66.03125,67108864 +25,self_attn.o_proj,8,128,8,128,0.634765625,65.015625,67108864 +25,mlp.gate_proj,2,32,8,128,83.875,70.21875,234881024 +25,mlp.gate_proj,2,64,8,128,98.0625,63.109375,234881024 +25,mlp.gate_proj,2,128,8,128,111.1875,59.5546875,234881024 +25,mlp.gate_proj,3,32,8,128,35.875,98.21875,234881024 +25,mlp.gate_proj,3,64,8,128,41.96875,91.109375,234881024 +25,mlp.gate_proj,3,128,8,128,47.625,87.5546875,234881024 +25,mlp.gate_proj,4,32,8,128,16.71875,126.21875,234881024 +25,mlp.gate_proj,4,64,8,128,19.609375,119.109375,234881024 +25,mlp.gate_proj,4,128,8,128,22.21875,115.5546875,234881024 +25,mlp.gate_proj,8,32,8,128,1.017578125,238.21875,234881024 +25,mlp.gate_proj,8,64,8,128,1.1923828125,231.109375,234881024 +25,mlp.gate_proj,8,128,8,128,1.34765625,227.5546875,234881024 +25,mlp.up_proj,2,32,8,128,81.6875,70.21875,234881024 +25,mlp.up_proj,2,64,8,128,95.125,63.109375,234881024 +25,mlp.up_proj,2,128,8,128,107.3125,59.5546875,234881024 +25,mlp.up_proj,3,32,8,128,34.9375,98.21875,234881024 +25,mlp.up_proj,3,64,8,128,40.71875,91.109375,234881024 +25,mlp.up_proj,3,128,8,128,45.9375,87.5546875,234881024 +25,mlp.up_proj,4,32,8,128,16.28125,126.21875,234881024 +25,mlp.up_proj,4,64,8,128,19.015625,119.109375,234881024 +25,mlp.up_proj,4,128,8,128,21.421875,115.5546875,234881024 +25,mlp.up_proj,8,32,8,128,0.99072265625,238.21875,234881024 +25,mlp.up_proj,8,64,8,128,1.15625,231.109375,234881024 +25,mlp.up_proj,8,128,8,128,1.30078125,227.5546875,234881024 +25,mlp.down_proj,2,32,8,128,82.0625,70.21875,234881024 +25,mlp.down_proj,2,64,8,128,95.5,63.109375,234881024 +25,mlp.down_proj,2,128,8,128,107.625,59.5546875,234881024 +25,mlp.down_proj,3,32,8,128,35.09375,98.21875,234881024 +25,mlp.down_proj,3,64,8,128,40.875,91.109375,234881024 +25,mlp.down_proj,3,128,8,128,46.0625,87.5546875,234881024 +25,mlp.down_proj,4,32,8,128,16.359375,126.21875,234881024 +25,mlp.down_proj,4,64,8,128,19.078125,119.109375,234881024 +25,mlp.down_proj,4,128,8,128,21.484375,115.5546875,234881024 +25,mlp.down_proj,8,32,8,128,0.9951171875,238.21875,234881024 +25,mlp.down_proj,8,64,8,128,1.16015625,231.109375,234881024 +25,mlp.down_proj,8,128,8,128,1.3037109375,227.5546875,234881024 +26,self_attn.q_proj,2,32,8,128,45.875,20.0625,67108864 +26,self_attn.q_proj,2,64,8,128,54.21875,18.03125,67108864 +26,self_attn.q_proj,2,128,8,128,62.71875,17.015625,67108864 +26,self_attn.q_proj,3,32,8,128,19.640625,28.0625,67108864 +26,self_attn.q_proj,3,64,8,128,23.265625,26.03125,67108864 +26,self_attn.q_proj,3,128,8,128,27.015625,25.015625,67108864 +26,self_attn.q_proj,4,32,8,128,9.1640625,36.0625,67108864 +26,self_attn.q_proj,4,64,8,128,10.859375,34.03125,67108864 +26,self_attn.q_proj,4,128,8,128,12.609375,33.015625,67108864 +26,self_attn.q_proj,8,32,8,128,0.55517578125,68.0625,67108864 +26,self_attn.q_proj,8,64,8,128,0.65673828125,66.03125,67108864 +26,self_attn.q_proj,8,128,8,128,0.76025390625,65.015625,67108864 +26,self_attn.v_proj,2,32,8,128,14.203125,2.5078125,8388608 +26,self_attn.v_proj,2,64,8,128,16.546875,2.25390625,8388608 +26,self_attn.v_proj,2,128,8,128,18.671875,2.126953125,8388608 +26,self_attn.v_proj,3,32,8,128,6.06640625,3.5078125,8388608 +26,self_attn.v_proj,3,64,8,128,7.08203125,3.25390625,8388608 +26,self_attn.v_proj,3,128,8,128,7.98828125,3.126953125,8388608 +26,self_attn.v_proj,4,32,8,128,2.830078125,4.5078125,8388608 +26,self_attn.v_proj,4,64,8,128,3.302734375,4.25390625,8388608 +26,self_attn.v_proj,4,128,8,128,3.728515625,4.126953125,8388608 +26,self_attn.v_proj,8,32,8,128,0.1715087890625,8.5078125,8388608 +26,self_attn.v_proj,8,64,8,128,0.1998291015625,8.25390625,8388608 +26,self_attn.v_proj,8,128,8,128,0.2247314453125,8.126953125,8388608 +26,self_attn.k_proj,2,32,8,128,27.265625,2.5078125,8388608 +26,self_attn.k_proj,2,64,8,128,32.8125,2.25390625,8388608 +26,self_attn.k_proj,2,128,8,128,37.5,2.126953125,8388608 +26,self_attn.k_proj,3,32,8,128,11.78125,3.5078125,8388608 +26,self_attn.k_proj,3,64,8,128,14.375,3.25390625,8388608 +26,self_attn.k_proj,3,128,8,128,16.84375,3.126953125,8388608 +26,self_attn.k_proj,4,32,8,128,5.5,4.5078125,8388608 +26,self_attn.k_proj,4,64,8,128,6.70703125,4.25390625,8388608 +26,self_attn.k_proj,4,128,8,128,7.86328125,4.126953125,8388608 +26,self_attn.k_proj,8,32,8,128,0.332763671875,8.5078125,8388608 +26,self_attn.k_proj,8,64,8,128,0.40576171875,8.25390625,8388608 +26,self_attn.k_proj,8,128,8,128,0.473876953125,8.126953125,8388608 +26,self_attn.o_proj,2,32,8,128,41.15625,20.0625,67108864 +26,self_attn.o_proj,2,64,8,128,47.9375,18.03125,67108864 +26,self_attn.o_proj,2,128,8,128,54.03125,17.015625,67108864 +26,self_attn.o_proj,3,32,8,128,17.609375,28.0625,67108864 +26,self_attn.o_proj,3,64,8,128,20.515625,26.03125,67108864 +26,self_attn.o_proj,3,128,8,128,23.15625,25.015625,67108864 +26,self_attn.o_proj,4,32,8,128,8.2109375,36.0625,67108864 +26,self_attn.o_proj,4,64,8,128,9.578125,34.03125,67108864 +26,self_attn.o_proj,4,128,8,128,10.8046875,33.015625,67108864 +26,self_attn.o_proj,8,32,8,128,0.497314453125,68.0625,67108864 +26,self_attn.o_proj,8,64,8,128,0.5791015625,66.03125,67108864 +26,self_attn.o_proj,8,128,8,128,0.6513671875,65.015625,67108864 +26,mlp.gate_proj,2,32,8,128,83.9375,70.21875,234881024 +26,mlp.gate_proj,2,64,8,128,98.125,63.109375,234881024 +26,mlp.gate_proj,2,128,8,128,111.1875,59.5546875,234881024 +26,mlp.gate_proj,3,32,8,128,35.90625,98.21875,234881024 +26,mlp.gate_proj,3,64,8,128,42.0,91.109375,234881024 +26,mlp.gate_proj,3,128,8,128,47.59375,87.5546875,234881024 +26,mlp.gate_proj,4,32,8,128,16.734375,126.21875,234881024 +26,mlp.gate_proj,4,64,8,128,19.609375,119.109375,234881024 +26,mlp.gate_proj,4,128,8,128,22.203125,115.5546875,234881024 +26,mlp.gate_proj,8,32,8,128,1.0185546875,238.21875,234881024 +26,mlp.gate_proj,8,64,8,128,1.193359375,231.109375,234881024 +26,mlp.gate_proj,8,128,8,128,1.34765625,227.5546875,234881024 +26,mlp.up_proj,2,32,8,128,81.625,70.21875,234881024 +26,mlp.up_proj,2,64,8,128,95.0,63.109375,234881024 +26,mlp.up_proj,2,128,8,128,107.1875,59.5546875,234881024 +26,mlp.up_proj,3,32,8,128,34.875,98.21875,234881024 +26,mlp.up_proj,3,64,8,128,40.65625,91.109375,234881024 +26,mlp.up_proj,3,128,8,128,45.84375,87.5546875,234881024 +26,mlp.up_proj,4,32,8,128,16.265625,126.21875,234881024 +26,mlp.up_proj,4,64,8,128,18.984375,119.109375,234881024 +26,mlp.up_proj,4,128,8,128,21.40625,115.5546875,234881024 +26,mlp.up_proj,8,32,8,128,0.9892578125,238.21875,234881024 +26,mlp.up_proj,8,64,8,128,1.154296875,231.109375,234881024 +26,mlp.up_proj,8,128,8,128,1.2978515625,227.5546875,234881024 +26,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024 +26,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024 +26,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024 +26,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024 +26,mlp.down_proj,3,64,8,128,40.78125,91.109375,234881024 +26,mlp.down_proj,3,128,8,128,45.90625,87.5546875,234881024 +26,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024 +26,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024 +26,mlp.down_proj,4,128,8,128,21.421875,115.5546875,234881024 +26,mlp.down_proj,8,32,8,128,0.99267578125,238.21875,234881024 +26,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024 +26,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024 +27,self_attn.q_proj,2,32,8,128,42.375,20.0625,67108864 +27,self_attn.q_proj,2,64,8,128,50.21875,18.03125,67108864 +27,self_attn.q_proj,2,128,8,128,57.84375,17.015625,67108864 +27,self_attn.q_proj,3,32,8,128,18.234375,28.0625,67108864 +27,self_attn.q_proj,3,64,8,128,21.765625,26.03125,67108864 +27,self_attn.q_proj,3,128,8,128,25.421875,25.015625,67108864 +27,self_attn.q_proj,4,32,8,128,8.5078125,36.0625,67108864 +27,self_attn.q_proj,4,64,8,128,10.1640625,34.03125,67108864 +27,self_attn.q_proj,4,128,8,128,11.875,33.015625,67108864 +27,self_attn.q_proj,8,32,8,128,0.51513671875,68.0625,67108864 +27,self_attn.q_proj,8,64,8,128,0.615234375,66.03125,67108864 +27,self_attn.q_proj,8,128,8,128,0.7158203125,65.015625,67108864 +27,self_attn.v_proj,2,32,8,128,15.1484375,2.5078125,8388608 +27,self_attn.v_proj,2,64,8,128,17.65625,2.25390625,8388608 +27,self_attn.v_proj,2,128,8,128,19.921875,2.126953125,8388608 +27,self_attn.v_proj,3,32,8,128,6.4765625,3.5078125,8388608 +27,self_attn.v_proj,3,64,8,128,7.5546875,3.25390625,8388608 +27,self_attn.v_proj,3,128,8,128,8.5234375,3.126953125,8388608 +27,self_attn.v_proj,4,32,8,128,3.021484375,4.5078125,8388608 +27,self_attn.v_proj,4,64,8,128,3.525390625,4.25390625,8388608 +27,self_attn.v_proj,4,128,8,128,3.98046875,4.126953125,8388608 +27,self_attn.v_proj,8,32,8,128,0.1829833984375,8.5078125,8388608 +27,self_attn.v_proj,8,64,8,128,0.2132568359375,8.25390625,8388608 +27,self_attn.v_proj,8,128,8,128,0.2398681640625,8.126953125,8388608 +27,self_attn.k_proj,2,32,8,128,26.453125,2.5078125,8388608 +27,self_attn.k_proj,2,64,8,128,31.65625,2.25390625,8388608 +27,self_attn.k_proj,2,128,8,128,35.875,2.126953125,8388608 +27,self_attn.k_proj,3,32,8,128,11.5078125,3.5078125,8388608 +27,self_attn.k_proj,3,64,8,128,14.0703125,3.25390625,8388608 +27,self_attn.k_proj,3,128,8,128,16.484375,3.126953125,8388608 +27,self_attn.k_proj,4,32,8,128,5.37109375,4.5078125,8388608 +27,self_attn.k_proj,4,64,8,128,6.56640625,4.25390625,8388608 +27,self_attn.k_proj,4,128,8,128,7.69921875,4.126953125,8388608 +27,self_attn.k_proj,8,32,8,128,0.324951171875,8.5078125,8388608 +27,self_attn.k_proj,8,64,8,128,0.397216796875,8.25390625,8388608 +27,self_attn.k_proj,8,128,8,128,0.464111328125,8.126953125,8388608 +27,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864 +27,self_attn.o_proj,2,64,8,128,46.78125,18.03125,67108864 +27,self_attn.o_proj,2,128,8,128,52.8125,17.015625,67108864 +27,self_attn.o_proj,3,32,8,128,17.1875,28.0625,67108864 +27,self_attn.o_proj,3,64,8,128,20.046875,26.03125,67108864 +27,self_attn.o_proj,3,128,8,128,22.640625,25.015625,67108864 +27,self_attn.o_proj,4,32,8,128,8.03125,36.0625,67108864 +27,self_attn.o_proj,4,64,8,128,9.375,34.03125,67108864 +27,self_attn.o_proj,4,128,8,128,10.609375,33.015625,67108864 +27,self_attn.o_proj,8,32,8,128,0.48828125,68.0625,67108864 +27,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864 +27,self_attn.o_proj,8,128,8,128,0.6474609375,65.015625,67108864 +27,mlp.gate_proj,2,32,8,128,84.0,70.21875,234881024 +27,mlp.gate_proj,2,64,8,128,98.1875,63.109375,234881024 +27,mlp.gate_proj,2,128,8,128,111.25,59.5546875,234881024 +27,mlp.gate_proj,3,32,8,128,35.9375,98.21875,234881024 +27,mlp.gate_proj,3,64,8,128,42.0,91.109375,234881024 +27,mlp.gate_proj,3,128,8,128,47.59375,87.5546875,234881024 +27,mlp.gate_proj,4,32,8,128,16.75,126.21875,234881024 +27,mlp.gate_proj,4,64,8,128,19.625,119.109375,234881024 +27,mlp.gate_proj,4,128,8,128,22.21875,115.5546875,234881024 +27,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024 +27,mlp.gate_proj,8,64,8,128,1.193359375,231.109375,234881024 +27,mlp.gate_proj,8,128,8,128,1.34765625,227.5546875,234881024 +27,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024 +27,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024 +27,mlp.up_proj,2,128,8,128,107.0625,59.5546875,234881024 +27,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024 +27,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024 +27,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024 +27,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024 +27,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024 +27,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024 +27,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024 +27,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +27,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024 +27,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024 +27,mlp.down_proj,2,64,8,128,95.125,63.109375,234881024 +27,mlp.down_proj,2,128,8,128,107.25,59.5546875,234881024 +27,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024 +27,mlp.down_proj,3,64,8,128,40.71875,91.109375,234881024 +27,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024 +27,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024 +27,mlp.down_proj,4,64,8,128,19.0,119.109375,234881024 +27,mlp.down_proj,4,128,8,128,21.40625,115.5546875,234881024 +27,mlp.down_proj,8,32,8,128,0.99169921875,238.21875,234881024 +27,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024 +27,mlp.down_proj,8,128,8,128,1.298828125,227.5546875,234881024 +28,self_attn.q_proj,2,32,8,128,43.3125,20.0625,67108864 +28,self_attn.q_proj,2,64,8,128,51.40625,18.03125,67108864 +28,self_attn.q_proj,2,128,8,128,59.53125,17.015625,67108864 +28,self_attn.q_proj,3,32,8,128,18.578125,28.0625,67108864 +28,self_attn.q_proj,3,64,8,128,22.140625,26.03125,67108864 +28,self_attn.q_proj,3,128,8,128,25.859375,25.015625,67108864 +28,self_attn.q_proj,4,32,8,128,8.671875,36.0625,67108864 +28,self_attn.q_proj,4,64,8,128,10.328125,34.03125,67108864 +28,self_attn.q_proj,4,128,8,128,12.0703125,33.015625,67108864 +28,self_attn.q_proj,8,32,8,128,0.52490234375,68.0625,67108864 +28,self_attn.q_proj,8,64,8,128,0.625,66.03125,67108864 +28,self_attn.q_proj,8,128,8,128,0.7275390625,65.015625,67108864 +28,self_attn.v_proj,2,32,8,128,15.625,2.5078125,8388608 +28,self_attn.v_proj,2,64,8,128,18.203125,2.25390625,8388608 +28,self_attn.v_proj,2,128,8,128,20.53125,2.126953125,8388608 +28,self_attn.v_proj,3,32,8,128,6.67578125,3.5078125,8388608 +28,self_attn.v_proj,3,64,8,128,7.78515625,3.25390625,8388608 +28,self_attn.v_proj,3,128,8,128,8.78125,3.126953125,8388608 +28,self_attn.v_proj,4,32,8,128,3.115234375,4.5078125,8388608 +28,self_attn.v_proj,4,64,8,128,3.6328125,4.25390625,8388608 +28,self_attn.v_proj,4,128,8,128,4.09765625,4.126953125,8388608 +28,self_attn.v_proj,8,32,8,128,0.188720703125,8.5078125,8388608 +28,self_attn.v_proj,8,64,8,128,0.2198486328125,8.25390625,8388608 +28,self_attn.v_proj,8,128,8,128,0.2471923828125,8.126953125,8388608 +28,self_attn.k_proj,2,32,8,128,25.515625,2.5078125,8388608 +28,self_attn.k_proj,2,64,8,128,30.84375,2.25390625,8388608 +28,self_attn.k_proj,2,128,8,128,35.40625,2.126953125,8388608 +28,self_attn.k_proj,3,32,8,128,11.0390625,3.5078125,8388608 +28,self_attn.k_proj,3,64,8,128,13.5,3.25390625,8388608 +28,self_attn.k_proj,3,128,8,128,15.890625,3.126953125,8388608 +28,self_attn.k_proj,4,32,8,128,5.1484375,4.5078125,8388608 +28,self_attn.k_proj,4,64,8,128,6.3046875,4.25390625,8388608 +28,self_attn.k_proj,4,128,8,128,7.421875,4.126953125,8388608 +28,self_attn.k_proj,8,32,8,128,0.3115234375,8.5078125,8388608 +28,self_attn.k_proj,8,64,8,128,0.381103515625,8.25390625,8388608 +28,self_attn.k_proj,8,128,8,128,0.447265625,8.126953125,8388608 +28,self_attn.o_proj,2,32,8,128,41.1875,20.0625,67108864 +28,self_attn.o_proj,2,64,8,128,47.96875,18.03125,67108864 +28,self_attn.o_proj,2,128,8,128,54.0625,17.015625,67108864 +28,self_attn.o_proj,3,32,8,128,17.609375,28.0625,67108864 +28,self_attn.o_proj,3,64,8,128,20.515625,26.03125,67108864 +28,self_attn.o_proj,3,128,8,128,23.140625,25.015625,67108864 +28,self_attn.o_proj,4,32,8,128,8.21875,36.0625,67108864 +28,self_attn.o_proj,4,64,8,128,9.578125,34.03125,67108864 +28,self_attn.o_proj,4,128,8,128,10.8046875,33.015625,67108864 +28,self_attn.o_proj,8,32,8,128,0.49755859375,68.0625,67108864 +28,self_attn.o_proj,8,64,8,128,0.5791015625,66.03125,67108864 +28,self_attn.o_proj,8,128,8,128,0.6513671875,65.015625,67108864 +28,mlp.gate_proj,2,32,8,128,84.0,70.21875,234881024 +28,mlp.gate_proj,2,64,8,128,98.1875,63.109375,234881024 +28,mlp.gate_proj,2,128,8,128,111.3125,59.5546875,234881024 +28,mlp.gate_proj,3,32,8,128,35.9375,98.21875,234881024 +28,mlp.gate_proj,3,64,8,128,42.03125,91.109375,234881024 +28,mlp.gate_proj,3,128,8,128,47.625,87.5546875,234881024 +28,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024 +28,mlp.gate_proj,4,64,8,128,19.625,119.109375,234881024 +28,mlp.gate_proj,4,128,8,128,22.234375,115.5546875,234881024 +28,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024 +28,mlp.gate_proj,8,64,8,128,1.1943359375,231.109375,234881024 +28,mlp.gate_proj,8,128,8,128,1.3486328125,227.5546875,234881024 +28,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024 +28,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024 +28,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024 +28,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024 +28,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024 +28,mlp.up_proj,3,128,8,128,45.8125,87.5546875,234881024 +28,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024 +28,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024 +28,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024 +28,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024 +28,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +28,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024 +28,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024 +28,mlp.down_proj,2,64,8,128,95.125,63.109375,234881024 +28,mlp.down_proj,2,128,8,128,107.1875,59.5546875,234881024 +28,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024 +28,mlp.down_proj,3,64,8,128,40.71875,91.109375,234881024 +28,mlp.down_proj,3,128,8,128,45.84375,87.5546875,234881024 +28,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024 +28,mlp.down_proj,4,64,8,128,19.0,119.109375,234881024 +28,mlp.down_proj,4,128,8,128,21.390625,115.5546875,234881024 +28,mlp.down_proj,8,32,8,128,0.99169921875,238.21875,234881024 +28,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024 +28,mlp.down_proj,8,128,8,128,1.298828125,227.5546875,234881024 +29,self_attn.q_proj,2,32,8,128,42.90625,20.0625,67108864 +29,self_attn.q_proj,2,64,8,128,51.1875,18.03125,67108864 +29,self_attn.q_proj,2,128,8,128,59.125,17.015625,67108864 +29,self_attn.q_proj,3,32,8,128,18.4375,28.0625,67108864 +29,self_attn.q_proj,3,64,8,128,22.15625,26.03125,67108864 +29,self_attn.q_proj,3,128,8,128,25.953125,25.015625,67108864 +29,self_attn.q_proj,4,32,8,128,8.6015625,36.0625,67108864 +29,self_attn.q_proj,4,64,8,128,10.3359375,34.03125,67108864 +29,self_attn.q_proj,4,128,8,128,12.109375,33.015625,67108864 +29,self_attn.q_proj,8,32,8,128,0.52099609375,68.0625,67108864 +29,self_attn.q_proj,8,64,8,128,0.62548828125,66.03125,67108864 +29,self_attn.q_proj,8,128,8,128,0.72998046875,65.015625,67108864 +29,self_attn.v_proj,2,32,8,128,15.2890625,2.5078125,8388608 +29,self_attn.v_proj,2,64,8,128,17.828125,2.25390625,8388608 +29,self_attn.v_proj,2,128,8,128,20.125,2.126953125,8388608 +29,self_attn.v_proj,3,32,8,128,6.5390625,3.5078125,8388608 +29,self_attn.v_proj,3,64,8,128,7.62890625,3.25390625,8388608 +29,self_attn.v_proj,3,128,8,128,8.609375,3.126953125,8388608 +29,self_attn.v_proj,4,32,8,128,3.05078125,4.5078125,8388608 +29,self_attn.v_proj,4,64,8,128,3.560546875,4.25390625,8388608 +29,self_attn.v_proj,4,128,8,128,4.01953125,4.126953125,8388608 +29,self_attn.v_proj,8,32,8,128,0.1846923828125,8.5078125,8388608 +29,self_attn.v_proj,8,64,8,128,0.21533203125,8.25390625,8388608 +29,self_attn.v_proj,8,128,8,128,0.2423095703125,8.126953125,8388608 +29,self_attn.k_proj,2,32,8,128,25.03125,2.5078125,8388608 +29,self_attn.k_proj,2,64,8,128,30.0,2.25390625,8388608 +29,self_attn.k_proj,2,128,8,128,34.0625,2.126953125,8388608 +29,self_attn.k_proj,3,32,8,128,10.8671875,3.5078125,8388608 +29,self_attn.k_proj,3,64,8,128,13.28125,3.25390625,8388608 +29,self_attn.k_proj,3,128,8,128,15.5625,3.126953125,8388608 +29,self_attn.k_proj,4,32,8,128,5.0703125,4.5078125,8388608 +29,self_attn.k_proj,4,64,8,128,6.1953125,4.25390625,8388608 +29,self_attn.k_proj,4,128,8,128,7.2734375,4.126953125,8388608 +29,self_attn.k_proj,8,32,8,128,0.306884765625,8.5078125,8388608 +29,self_attn.k_proj,8,64,8,128,0.374755859375,8.25390625,8388608 +29,self_attn.k_proj,8,128,8,128,0.438232421875,8.126953125,8388608 +29,self_attn.o_proj,2,32,8,128,40.9375,20.0625,67108864 +29,self_attn.o_proj,2,64,8,128,47.65625,18.03125,67108864 +29,self_attn.o_proj,2,128,8,128,53.71875,17.015625,67108864 +29,self_attn.o_proj,3,32,8,128,17.515625,28.0625,67108864 +29,self_attn.o_proj,3,64,8,128,20.421875,26.03125,67108864 +29,self_attn.o_proj,3,128,8,128,23.046875,25.015625,67108864 +29,self_attn.o_proj,4,32,8,128,8.1796875,36.0625,67108864 +29,self_attn.o_proj,4,64,8,128,9.5390625,34.03125,67108864 +29,self_attn.o_proj,4,128,8,128,10.78125,33.015625,67108864 +29,self_attn.o_proj,8,32,8,128,0.495361328125,68.0625,67108864 +29,self_attn.o_proj,8,64,8,128,0.57763671875,66.03125,67108864 +29,self_attn.o_proj,8,128,8,128,0.650390625,65.015625,67108864 +29,mlp.gate_proj,2,32,8,128,84.125,70.21875,234881024 +29,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024 +29,mlp.gate_proj,2,128,8,128,111.3125,59.5546875,234881024 +29,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024 +29,mlp.gate_proj,3,64,8,128,42.0625,91.109375,234881024 +29,mlp.gate_proj,3,128,8,128,47.65625,87.5546875,234881024 +29,mlp.gate_proj,4,32,8,128,16.78125,126.21875,234881024 +29,mlp.gate_proj,4,64,8,128,19.640625,119.109375,234881024 +29,mlp.gate_proj,4,128,8,128,22.234375,115.5546875,234881024 +29,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024 +29,mlp.gate_proj,8,64,8,128,1.1953125,231.109375,234881024 +29,mlp.gate_proj,8,128,8,128,1.3486328125,227.5546875,234881024 +29,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024 +29,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024 +29,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024 +29,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024 +29,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024 +29,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024 +29,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024 +29,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024 +29,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024 +29,mlp.up_proj,8,32,8,128,0.98779296875,238.21875,234881024 +29,mlp.up_proj,8,64,8,128,1.15234375,231.109375,234881024 +29,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +29,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024 +29,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024 +29,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024 +29,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024 +29,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024 +29,mlp.down_proj,3,128,8,128,45.8125,87.5546875,234881024 +29,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024 +29,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024 +29,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024 +29,mlp.down_proj,8,32,8,128,0.99072265625,238.21875,234881024 +29,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024 +29,mlp.down_proj,8,128,8,128,1.2978515625,227.5546875,234881024 +30,self_attn.q_proj,2,32,8,128,37.03125,20.0625,67108864 +30,self_attn.q_proj,2,64,8,128,44.40625,18.03125,67108864 +30,self_attn.q_proj,2,128,8,128,51.8125,17.015625,67108864 +30,self_attn.q_proj,3,32,8,128,15.9453125,28.0625,67108864 +30,self_attn.q_proj,3,64,8,128,19.3125,26.03125,67108864 +30,self_attn.q_proj,3,128,8,128,22.953125,25.015625,67108864 +30,self_attn.q_proj,4,32,8,128,7.4375,36.0625,67108864 +30,self_attn.q_proj,4,64,8,128,9.015625,34.03125,67108864 +30,self_attn.q_proj,4,128,8,128,10.71875,33.015625,67108864 +30,self_attn.q_proj,8,32,8,128,0.4501953125,68.0625,67108864 +30,self_attn.q_proj,8,64,8,128,0.54541015625,66.03125,67108864 +30,self_attn.q_proj,8,128,8,128,0.646484375,65.015625,67108864 +30,self_attn.v_proj,2,32,8,128,15.7109375,2.5078125,8388608 +30,self_attn.v_proj,2,64,8,128,18.3125,2.25390625,8388608 +30,self_attn.v_proj,2,128,8,128,20.65625,2.126953125,8388608 +30,self_attn.v_proj,3,32,8,128,6.71484375,3.5078125,8388608 +30,self_attn.v_proj,3,64,8,128,7.8359375,3.25390625,8388608 +30,self_attn.v_proj,3,128,8,128,8.84375,3.126953125,8388608 +30,self_attn.v_proj,4,32,8,128,3.134765625,4.5078125,8388608 +30,self_attn.v_proj,4,64,8,128,3.65625,4.25390625,8388608 +30,self_attn.v_proj,4,128,8,128,4.125,4.126953125,8388608 +30,self_attn.v_proj,8,32,8,128,0.1898193359375,8.5078125,8388608 +30,self_attn.v_proj,8,64,8,128,0.22119140625,8.25390625,8388608 +30,self_attn.v_proj,8,128,8,128,0.2486572265625,8.126953125,8388608 +30,self_attn.k_proj,2,32,8,128,23.078125,2.5078125,8388608 +30,self_attn.k_proj,2,64,8,128,27.921875,2.25390625,8388608 +30,self_attn.k_proj,2,128,8,128,32.03125,2.126953125,8388608 +30,self_attn.k_proj,3,32,8,128,10.2109375,3.5078125,8388608 +30,self_attn.k_proj,3,64,8,128,12.84375,3.25390625,8388608 +30,self_attn.k_proj,3,128,8,128,15.5390625,3.126953125,8388608 +30,self_attn.k_proj,4,32,8,128,4.765625,4.5078125,8388608 +30,self_attn.k_proj,4,64,8,128,5.99609375,4.25390625,8388608 +30,self_attn.k_proj,4,128,8,128,7.26953125,4.126953125,8388608 +30,self_attn.k_proj,8,32,8,128,0.288330078125,8.5078125,8388608 +30,self_attn.k_proj,8,64,8,128,0.362548828125,8.25390625,8388608 +30,self_attn.k_proj,8,128,8,128,0.437744140625,8.126953125,8388608 +30,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864 +30,self_attn.o_proj,2,64,8,128,46.78125,18.03125,67108864 +30,self_attn.o_proj,2,128,8,128,52.71875,17.015625,67108864 +30,self_attn.o_proj,3,32,8,128,17.171875,28.0625,67108864 +30,self_attn.o_proj,3,64,8,128,20.015625,26.03125,67108864 +30,self_attn.o_proj,3,128,8,128,22.578125,25.015625,67108864 +30,self_attn.o_proj,4,32,8,128,8.015625,36.0625,67108864 +30,self_attn.o_proj,4,64,8,128,9.34375,34.03125,67108864 +30,self_attn.o_proj,4,128,8,128,10.5390625,33.015625,67108864 +30,self_attn.o_proj,8,32,8,128,0.485107421875,68.0625,67108864 +30,self_attn.o_proj,8,64,8,128,0.56494140625,66.03125,67108864 +30,self_attn.o_proj,8,128,8,128,0.63525390625,65.015625,67108864 +30,mlp.gate_proj,2,32,8,128,84.3125,70.21875,234881024 +30,mlp.gate_proj,2,64,8,128,98.5625,63.109375,234881024 +30,mlp.gate_proj,2,128,8,128,111.625,59.5546875,234881024 +30,mlp.gate_proj,3,32,8,128,36.0625,98.21875,234881024 +30,mlp.gate_proj,3,64,8,128,42.1875,91.109375,234881024 +30,mlp.gate_proj,3,128,8,128,47.78125,87.5546875,234881024 +30,mlp.gate_proj,4,32,8,128,16.8125,126.21875,234881024 +30,mlp.gate_proj,4,64,8,128,19.6875,119.109375,234881024 +30,mlp.gate_proj,4,128,8,128,22.296875,115.5546875,234881024 +30,mlp.gate_proj,8,32,8,128,1.0234375,238.21875,234881024 +30,mlp.gate_proj,8,64,8,128,1.1982421875,231.109375,234881024 +30,mlp.gate_proj,8,128,8,128,1.3525390625,227.5546875,234881024 +30,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024 +30,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024 +30,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024 +30,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024 +30,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024 +30,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024 +30,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024 +30,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024 +30,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024 +30,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024 +30,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +30,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +30,mlp.down_proj,2,32,8,128,81.625,70.21875,234881024 +30,mlp.down_proj,2,64,8,128,94.9375,63.109375,234881024 +30,mlp.down_proj,2,128,8,128,107.0,59.5546875,234881024 +30,mlp.down_proj,3,32,8,128,34.875,98.21875,234881024 +30,mlp.down_proj,3,64,8,128,40.625,91.109375,234881024 +30,mlp.down_proj,3,128,8,128,45.75,87.5546875,234881024 +30,mlp.down_proj,4,32,8,128,16.265625,126.21875,234881024 +30,mlp.down_proj,4,64,8,128,18.96875,119.109375,234881024 +30,mlp.down_proj,4,128,8,128,21.359375,115.5546875,234881024 +30,mlp.down_proj,8,32,8,128,0.9892578125,238.21875,234881024 +30,mlp.down_proj,8,64,8,128,1.1533203125,231.109375,234881024 +30,mlp.down_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +31,self_attn.q_proj,2,32,8,128,42.375,20.0625,67108864 +31,self_attn.q_proj,2,64,8,128,50.46875,18.03125,67108864 +31,self_attn.q_proj,2,128,8,128,58.25,17.015625,67108864 +31,self_attn.q_proj,3,32,8,128,18.1875,28.0625,67108864 +31,self_attn.q_proj,3,64,8,128,21.75,26.03125,67108864 +31,self_attn.q_proj,3,128,8,128,25.328125,25.015625,67108864 +31,self_attn.q_proj,4,32,8,128,8.484375,36.0625,67108864 +31,self_attn.q_proj,4,64,8,128,10.15625,34.03125,67108864 +31,self_attn.q_proj,4,128,8,128,11.828125,33.015625,67108864 +31,self_attn.q_proj,8,32,8,128,0.513671875,68.0625,67108864 +31,self_attn.q_proj,8,64,8,128,0.6142578125,66.03125,67108864 +31,self_attn.q_proj,8,128,8,128,0.71337890625,65.015625,67108864 +31,self_attn.v_proj,2,32,8,128,15.1875,2.5078125,8388608 +31,self_attn.v_proj,2,64,8,128,17.75,2.25390625,8388608 +31,self_attn.v_proj,2,128,8,128,20.0625,2.126953125,8388608 +31,self_attn.v_proj,3,32,8,128,6.4921875,3.5078125,8388608 +31,self_attn.v_proj,3,64,8,128,7.58984375,3.25390625,8388608 +31,self_attn.v_proj,3,128,8,128,8.5859375,3.126953125,8388608 +31,self_attn.v_proj,4,32,8,128,3.03125,4.5078125,8388608 +31,self_attn.v_proj,4,64,8,128,3.54296875,4.25390625,8388608 +31,self_attn.v_proj,4,128,8,128,4.0078125,4.126953125,8388608 +31,self_attn.v_proj,8,32,8,128,0.18359375,8.5078125,8388608 +31,self_attn.v_proj,8,64,8,128,0.21435546875,8.25390625,8388608 +31,self_attn.v_proj,8,128,8,128,0.2415771484375,8.126953125,8388608 +31,self_attn.k_proj,2,32,8,128,24.921875,2.5078125,8388608 +31,self_attn.k_proj,2,64,8,128,29.90625,2.25390625,8388608 +31,self_attn.k_proj,2,128,8,128,33.96875,2.126953125,8388608 +31,self_attn.k_proj,3,32,8,128,10.7890625,3.5078125,8388608 +31,self_attn.k_proj,3,64,8,128,13.1640625,3.25390625,8388608 +31,self_attn.k_proj,3,128,8,128,15.40625,3.126953125,8388608 +31,self_attn.k_proj,4,32,8,128,5.03515625,4.5078125,8388608 +31,self_attn.k_proj,4,64,8,128,6.140625,4.25390625,8388608 +31,self_attn.k_proj,4,128,8,128,7.19921875,4.126953125,8388608 +31,self_attn.k_proj,8,32,8,128,0.3046875,8.5078125,8388608 +31,self_attn.k_proj,8,64,8,128,0.371337890625,8.25390625,8388608 +31,self_attn.k_proj,8,128,8,128,0.433349609375,8.126953125,8388608 +31,self_attn.o_proj,2,32,8,128,40.1875,20.0625,67108864 +31,self_attn.o_proj,2,64,8,128,46.78125,18.03125,67108864 +31,self_attn.o_proj,2,128,8,128,52.71875,17.015625,67108864 +31,self_attn.o_proj,3,32,8,128,17.1875,28.0625,67108864 +31,self_attn.o_proj,3,64,8,128,20.03125,26.03125,67108864 +31,self_attn.o_proj,3,128,8,128,22.59375,25.015625,67108864 +31,self_attn.o_proj,4,32,8,128,8.03125,36.0625,67108864 +31,self_attn.o_proj,4,64,8,128,9.3671875,34.03125,67108864 +31,self_attn.o_proj,4,128,8,128,10.59375,33.015625,67108864 +31,self_attn.o_proj,8,32,8,128,0.48779296875,68.0625,67108864 +31,self_attn.o_proj,8,64,8,128,0.56982421875,66.03125,67108864 +31,self_attn.o_proj,8,128,8,128,0.64404296875,65.015625,67108864 +31,mlp.gate_proj,2,32,8,128,84.375,70.21875,234881024 +31,mlp.gate_proj,2,64,8,128,98.5625,63.109375,234881024 +31,mlp.gate_proj,2,128,8,128,111.625,59.5546875,234881024 +31,mlp.gate_proj,3,32,8,128,36.09375,98.21875,234881024 +31,mlp.gate_proj,3,64,8,128,42.1875,91.109375,234881024 +31,mlp.gate_proj,3,128,8,128,47.78125,87.5546875,234881024 +31,mlp.gate_proj,4,32,8,128,16.828125,126.21875,234881024 +31,mlp.gate_proj,4,64,8,128,19.703125,119.109375,234881024 +31,mlp.gate_proj,4,128,8,128,22.296875,115.5546875,234881024 +31,mlp.gate_proj,8,32,8,128,1.0234375,238.21875,234881024 +31,mlp.gate_proj,8,64,8,128,1.1982421875,231.109375,234881024 +31,mlp.gate_proj,8,128,8,128,1.3525390625,227.5546875,234881024 +31,mlp.up_proj,2,32,8,128,81.5625,70.21875,234881024 +31,mlp.up_proj,2,64,8,128,94.9375,63.109375,234881024 +31,mlp.up_proj,2,128,8,128,107.0625,59.5546875,234881024 +31,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024 +31,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024 +31,mlp.up_proj,3,128,8,128,45.8125,87.5546875,234881024 +31,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024 +31,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024 +31,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024 +31,mlp.up_proj,8,32,8,128,0.98876953125,238.21875,234881024 +31,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +31,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024 +31,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024 +31,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024 +31,mlp.down_proj,2,128,8,128,106.8125,59.5546875,234881024 +31,mlp.down_proj,3,32,8,128,34.8125,98.21875,234881024 +31,mlp.down_proj,3,64,8,128,40.5625,91.109375,234881024 +31,mlp.down_proj,3,128,8,128,45.6875,87.5546875,234881024 +31,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024 +31,mlp.down_proj,4,64,8,128,18.9375,119.109375,234881024 +31,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024 +31,mlp.down_proj,8,32,8,128,0.98779296875,238.21875,234881024 +31,mlp.down_proj,8,64,8,128,1.1513671875,231.109375,234881024 +31,mlp.down_proj,8,128,8,128,1.2939453125,227.5546875,234881024 +32,self_attn.q_proj,2,32,8,128,43.125,20.0625,67108864 +32,self_attn.q_proj,2,64,8,128,51.0625,18.03125,67108864 +32,self_attn.q_proj,2,128,8,128,58.71875,17.015625,67108864 +32,self_attn.q_proj,3,32,8,128,18.46875,28.0625,67108864 +32,self_attn.q_proj,3,64,8,128,21.90625,26.03125,67108864 +32,self_attn.q_proj,3,128,8,128,25.296875,25.015625,67108864 +32,self_attn.q_proj,4,32,8,128,8.6171875,36.0625,67108864 +32,self_attn.q_proj,4,64,8,128,10.2265625,34.03125,67108864 +32,self_attn.q_proj,4,128,8,128,11.8046875,33.015625,67108864 +32,self_attn.q_proj,8,32,8,128,0.52197265625,68.0625,67108864 +32,self_attn.q_proj,8,64,8,128,0.61865234375,66.03125,67108864 +32,self_attn.q_proj,8,128,8,128,0.7119140625,65.015625,67108864 +32,self_attn.v_proj,2,32,8,128,15.078125,2.5078125,8388608 +32,self_attn.v_proj,2,64,8,128,17.578125,2.25390625,8388608 +32,self_attn.v_proj,2,128,8,128,19.84375,2.126953125,8388608 +32,self_attn.v_proj,3,32,8,128,6.4453125,3.5078125,8388608 +32,self_attn.v_proj,3,64,8,128,7.5234375,3.25390625,8388608 +32,self_attn.v_proj,3,128,8,128,8.4921875,3.126953125,8388608 +32,self_attn.v_proj,4,32,8,128,3.0078125,4.5078125,8388608 +32,self_attn.v_proj,4,64,8,128,3.5078125,4.25390625,8388608 +32,self_attn.v_proj,4,128,8,128,3.962890625,4.126953125,8388608 +32,self_attn.v_proj,8,32,8,128,0.18212890625,8.5078125,8388608 +32,self_attn.v_proj,8,64,8,128,0.2122802734375,8.25390625,8388608 +32,self_attn.v_proj,8,128,8,128,0.23876953125,8.126953125,8388608 +32,self_attn.k_proj,2,32,8,128,25.640625,2.5078125,8388608 +32,self_attn.k_proj,2,64,8,128,30.875,2.25390625,8388608 +32,self_attn.k_proj,2,128,8,128,35.3125,2.126953125,8388608 +32,self_attn.k_proj,3,32,8,128,11.0625,3.5078125,8388608 +32,self_attn.k_proj,3,64,8,128,13.484375,3.25390625,8388608 +32,self_attn.k_proj,3,128,8,128,15.8046875,3.126953125,8388608 +32,self_attn.k_proj,4,32,8,128,5.1640625,4.5078125,8388608 +32,self_attn.k_proj,4,64,8,128,6.296875,4.25390625,8388608 +32,self_attn.k_proj,4,128,8,128,7.3828125,4.126953125,8388608 +32,self_attn.k_proj,8,32,8,128,0.312744140625,8.5078125,8388608 +32,self_attn.k_proj,8,64,8,128,0.380615234375,8.25390625,8388608 +32,self_attn.k_proj,8,128,8,128,0.44482421875,8.126953125,8388608 +32,self_attn.o_proj,2,32,8,128,39.875,20.0625,67108864 +32,self_attn.o_proj,2,64,8,128,46.5,18.03125,67108864 +32,self_attn.o_proj,2,128,8,128,52.59375,17.015625,67108864 +32,self_attn.o_proj,3,32,8,128,17.0625,28.0625,67108864 +32,self_attn.o_proj,3,64,8,128,19.953125,26.03125,67108864 +32,self_attn.o_proj,3,128,8,128,22.625,25.015625,67108864 +32,self_attn.o_proj,4,32,8,128,7.9609375,36.0625,67108864 +32,self_attn.o_proj,4,64,8,128,9.3125,34.03125,67108864 +32,self_attn.o_proj,4,128,8,128,10.5703125,33.015625,67108864 +32,self_attn.o_proj,8,32,8,128,0.482177734375,68.0625,67108864 +32,self_attn.o_proj,8,64,8,128,0.5634765625,66.03125,67108864 +32,self_attn.o_proj,8,128,8,128,0.63720703125,65.015625,67108864 +32,mlp.gate_proj,2,32,8,128,84.3125,70.21875,234881024 +32,mlp.gate_proj,2,64,8,128,98.4375,63.109375,234881024 +32,mlp.gate_proj,2,128,8,128,111.4375,59.5546875,234881024 +32,mlp.gate_proj,3,32,8,128,36.0625,98.21875,234881024 +32,mlp.gate_proj,3,64,8,128,42.125,91.109375,234881024 +32,mlp.gate_proj,3,128,8,128,47.71875,87.5546875,234881024 +32,mlp.gate_proj,4,32,8,128,16.8125,126.21875,234881024 +32,mlp.gate_proj,4,64,8,128,19.6875,119.109375,234881024 +32,mlp.gate_proj,4,128,8,128,22.265625,115.5546875,234881024 +32,mlp.gate_proj,8,32,8,128,1.0224609375,238.21875,234881024 +32,mlp.gate_proj,8,64,8,128,1.197265625,231.109375,234881024 +32,mlp.gate_proj,8,128,8,128,1.3505859375,227.5546875,234881024 +32,mlp.up_proj,2,32,8,128,81.625,70.21875,234881024 +32,mlp.up_proj,2,64,8,128,95.0,63.109375,234881024 +32,mlp.up_proj,2,128,8,128,107.1875,59.5546875,234881024 +32,mlp.up_proj,3,32,8,128,34.875,98.21875,234881024 +32,mlp.up_proj,3,64,8,128,40.6875,91.109375,234881024 +32,mlp.up_proj,3,128,8,128,45.84375,87.5546875,234881024 +32,mlp.up_proj,4,32,8,128,16.265625,126.21875,234881024 +32,mlp.up_proj,4,64,8,128,18.984375,119.109375,234881024 +32,mlp.up_proj,4,128,8,128,21.40625,115.5546875,234881024 +32,mlp.up_proj,8,32,8,128,0.98974609375,238.21875,234881024 +32,mlp.up_proj,8,64,8,128,1.154296875,231.109375,234881024 +32,mlp.up_proj,8,128,8,128,1.2978515625,227.5546875,234881024 +32,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024 +32,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024 +32,mlp.down_proj,2,128,8,128,106.875,59.5546875,234881024 +32,mlp.down_proj,3,32,8,128,34.8125,98.21875,234881024 +32,mlp.down_proj,3,64,8,128,40.5625,91.109375,234881024 +32,mlp.down_proj,3,128,8,128,45.6875,87.5546875,234881024 +32,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024 +32,mlp.down_proj,4,64,8,128,18.9375,119.109375,234881024 +32,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024 +32,mlp.down_proj,8,32,8,128,0.98779296875,238.21875,234881024 +32,mlp.down_proj,8,64,8,128,1.1513671875,231.109375,234881024 +32,mlp.down_proj,8,128,8,128,1.2939453125,227.5546875,234881024 +33,self_attn.q_proj,2,32,8,128,45.8125,20.0625,67108864 +33,self_attn.q_proj,2,64,8,128,53.9375,18.03125,67108864 +33,self_attn.q_proj,2,128,8,128,62.1875,17.015625,67108864 +33,self_attn.q_proj,3,32,8,128,19.609375,28.0625,67108864 +33,self_attn.q_proj,3,64,8,128,23.125,26.03125,67108864 +33,self_attn.q_proj,3,128,8,128,26.75,25.015625,67108864 +33,self_attn.q_proj,4,32,8,128,9.1484375,36.0625,67108864 +33,self_attn.q_proj,4,64,8,128,10.7890625,34.03125,67108864 +33,self_attn.q_proj,4,128,8,128,12.484375,33.015625,67108864 +33,self_attn.q_proj,8,32,8,128,0.55419921875,68.0625,67108864 +33,self_attn.q_proj,8,64,8,128,0.65283203125,66.03125,67108864 +33,self_attn.q_proj,8,128,8,128,0.7529296875,65.015625,67108864 +33,self_attn.v_proj,2,32,8,128,14.828125,2.5078125,8388608 +33,self_attn.v_proj,2,64,8,128,17.34375,2.25390625,8388608 +33,self_attn.v_proj,2,128,8,128,19.65625,2.126953125,8388608 +33,self_attn.v_proj,3,32,8,128,6.33984375,3.5078125,8388608 +33,self_attn.v_proj,3,64,8,128,7.421875,3.25390625,8388608 +33,self_attn.v_proj,3,128,8,128,8.40625,3.126953125,8388608 +33,self_attn.v_proj,4,32,8,128,2.95703125,4.5078125,8388608 +33,self_attn.v_proj,4,64,8,128,3.46484375,4.25390625,8388608 +33,self_attn.v_proj,4,128,8,128,3.92578125,4.126953125,8388608 +33,self_attn.v_proj,8,32,8,128,0.17919921875,8.5078125,8388608 +33,self_attn.v_proj,8,64,8,128,0.20947265625,8.25390625,8388608 +33,self_attn.v_proj,8,128,8,128,0.236572265625,8.126953125,8388608 +33,self_attn.k_proj,2,32,8,128,26.15625,2.5078125,8388608 +33,self_attn.k_proj,2,64,8,128,31.578125,2.25390625,8388608 +33,self_attn.k_proj,2,128,8,128,36.21875,2.126953125,8388608 +33,self_attn.k_proj,3,32,8,128,11.328125,3.5078125,8388608 +33,self_attn.k_proj,3,64,8,128,13.8671875,3.25390625,8388608 +33,self_attn.k_proj,3,128,8,128,16.34375,3.126953125,8388608 +33,self_attn.k_proj,4,32,8,128,5.28515625,4.5078125,8388608 +33,self_attn.k_proj,4,64,8,128,6.47265625,4.25390625,8388608 +33,self_attn.k_proj,4,128,8,128,7.63671875,4.126953125,8388608 +33,self_attn.k_proj,8,32,8,128,0.31982421875,8.5078125,8388608 +33,self_attn.k_proj,8,64,8,128,0.391357421875,8.25390625,8388608 +33,self_attn.k_proj,8,128,8,128,0.4599609375,8.126953125,8388608 +33,self_attn.o_proj,2,32,8,128,40.46875,20.0625,67108864 +33,self_attn.o_proj,2,64,8,128,47.15625,18.03125,67108864 +33,self_attn.o_proj,2,128,8,128,53.21875,17.015625,67108864 +33,self_attn.o_proj,3,32,8,128,17.3125,28.0625,67108864 +33,self_attn.o_proj,3,64,8,128,20.21875,26.03125,67108864 +33,self_attn.o_proj,3,128,8,128,22.84375,25.015625,67108864 +33,self_attn.o_proj,4,32,8,128,8.0859375,36.0625,67108864 +33,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864 +33,self_attn.o_proj,4,128,8,128,10.6796875,33.015625,67108864 +33,self_attn.o_proj,8,32,8,128,0.489501953125,68.0625,67108864 +33,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864 +33,self_attn.o_proj,8,128,8,128,0.64453125,65.015625,67108864 +33,mlp.gate_proj,2,32,8,128,84.125,70.21875,234881024 +33,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024 +33,mlp.gate_proj,2,128,8,128,111.3125,59.5546875,234881024 +33,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024 +33,mlp.gate_proj,3,64,8,128,42.0625,91.109375,234881024 +33,mlp.gate_proj,3,128,8,128,47.65625,87.5546875,234881024 +33,mlp.gate_proj,4,32,8,128,16.78125,126.21875,234881024 +33,mlp.gate_proj,4,64,8,128,19.640625,119.109375,234881024 +33,mlp.gate_proj,4,128,8,128,22.25,115.5546875,234881024 +33,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024 +33,mlp.gate_proj,8,64,8,128,1.1953125,231.109375,234881024 +33,mlp.gate_proj,8,128,8,128,1.349609375,227.5546875,234881024 +33,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024 +33,mlp.up_proj,2,64,8,128,94.9375,63.109375,234881024 +33,mlp.up_proj,2,128,8,128,107.0625,59.5546875,234881024 +33,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024 +33,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024 +33,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024 +33,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024 +33,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024 +33,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024 +33,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024 +33,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +33,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024 +33,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024 +33,mlp.down_proj,2,64,8,128,94.875,63.109375,234881024 +33,mlp.down_proj,2,128,8,128,106.875,59.5546875,234881024 +33,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024 +33,mlp.down_proj,3,64,8,128,40.59375,91.109375,234881024 +33,mlp.down_proj,3,128,8,128,45.71875,87.5546875,234881024 +33,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024 +33,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024 +33,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024 +33,mlp.down_proj,8,32,8,128,0.98828125,238.21875,234881024 +33,mlp.down_proj,8,64,8,128,1.15234375,231.109375,234881024 +33,mlp.down_proj,8,128,8,128,1.294921875,227.5546875,234881024 +34,self_attn.q_proj,2,32,8,128,44.0625,20.0625,67108864 +34,self_attn.q_proj,2,64,8,128,52.4375,18.03125,67108864 +34,self_attn.q_proj,2,128,8,128,60.53125,17.015625,67108864 +34,self_attn.q_proj,3,32,8,128,18.96875,28.0625,67108864 +34,self_attn.q_proj,3,64,8,128,22.734375,26.03125,67108864 +34,self_attn.q_proj,3,128,8,128,26.625,25.015625,67108864 +34,self_attn.q_proj,4,32,8,128,8.8515625,36.0625,67108864 +34,self_attn.q_proj,4,64,8,128,10.609375,34.03125,67108864 +34,self_attn.q_proj,4,128,8,128,12.4296875,33.015625,67108864 +34,self_attn.q_proj,8,32,8,128,0.53564453125,68.0625,67108864 +34,self_attn.q_proj,8,64,8,128,0.6416015625,66.03125,67108864 +34,self_attn.q_proj,8,128,8,128,0.74951171875,65.015625,67108864 +34,self_attn.v_proj,2,32,8,128,15.0859375,2.5078125,8388608 +34,self_attn.v_proj,2,64,8,128,17.625,2.25390625,8388608 +34,self_attn.v_proj,2,128,8,128,19.921875,2.126953125,8388608 +34,self_attn.v_proj,3,32,8,128,6.44921875,3.5078125,8388608 +34,self_attn.v_proj,3,64,8,128,7.54296875,3.25390625,8388608 +34,self_attn.v_proj,3,128,8,128,8.53125,3.126953125,8388608 +34,self_attn.v_proj,4,32,8,128,3.009765625,4.5078125,8388608 +34,self_attn.v_proj,4,64,8,128,3.51953125,4.25390625,8388608 +34,self_attn.v_proj,4,128,8,128,3.98046875,4.126953125,8388608 +34,self_attn.v_proj,8,32,8,128,0.1822509765625,8.5078125,8388608 +34,self_attn.v_proj,8,64,8,128,0.2127685546875,8.25390625,8388608 +34,self_attn.v_proj,8,128,8,128,0.2398681640625,8.126953125,8388608 +34,self_attn.k_proj,2,32,8,128,27.109375,2.5078125,8388608 +34,self_attn.k_proj,2,64,8,128,32.625,2.25390625,8388608 +34,self_attn.k_proj,2,128,8,128,37.15625,2.126953125,8388608 +34,self_attn.k_proj,3,32,8,128,11.7890625,3.5078125,8388608 +34,self_attn.k_proj,3,64,8,128,14.4453125,3.25390625,8388608 +34,self_attn.k_proj,3,128,8,128,16.984375,3.126953125,8388608 +34,self_attn.k_proj,4,32,8,128,5.50390625,4.5078125,8388608 +34,self_attn.k_proj,4,64,8,128,6.73828125,4.25390625,8388608 +34,self_attn.k_proj,4,128,8,128,7.9296875,4.126953125,8388608 +34,self_attn.k_proj,8,32,8,128,0.3330078125,8.5078125,8388608 +34,self_attn.k_proj,8,64,8,128,0.407470703125,8.25390625,8388608 +34,self_attn.k_proj,8,128,8,128,0.477783203125,8.126953125,8388608 +34,self_attn.o_proj,2,32,8,128,39.875,20.0625,67108864 +34,self_attn.o_proj,2,64,8,128,46.46875,18.03125,67108864 +34,self_attn.o_proj,2,128,8,128,52.40625,17.015625,67108864 +34,self_attn.o_proj,3,32,8,128,17.046875,28.0625,67108864 +34,self_attn.o_proj,3,64,8,128,19.890625,26.03125,67108864 +34,self_attn.o_proj,3,128,8,128,22.46875,25.015625,67108864 +34,self_attn.o_proj,4,32,8,128,7.95703125,36.0625,67108864 +34,self_attn.o_proj,4,64,8,128,9.28125,34.03125,67108864 +34,self_attn.o_proj,4,128,8,128,10.484375,33.015625,67108864 +34,self_attn.o_proj,8,32,8,128,0.481689453125,68.0625,67108864 +34,self_attn.o_proj,8,64,8,128,0.5615234375,66.03125,67108864 +34,self_attn.o_proj,8,128,8,128,0.63232421875,65.015625,67108864 +34,mlp.gate_proj,2,32,8,128,84.0625,70.21875,234881024 +34,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024 +34,mlp.gate_proj,2,128,8,128,111.5,59.5546875,234881024 +34,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024 +34,mlp.gate_proj,3,64,8,128,42.09375,91.109375,234881024 +34,mlp.gate_proj,3,128,8,128,47.75,87.5546875,234881024 +34,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024 +34,mlp.gate_proj,4,64,8,128,19.65625,119.109375,234881024 +34,mlp.gate_proj,4,128,8,128,22.28125,115.5546875,234881024 +34,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024 +34,mlp.gate_proj,8,64,8,128,1.1953125,231.109375,234881024 +34,mlp.gate_proj,8,128,8,128,1.3515625,227.5546875,234881024 +34,mlp.up_proj,2,32,8,128,81.3125,70.21875,234881024 +34,mlp.up_proj,2,64,8,128,94.625,63.109375,234881024 +34,mlp.up_proj,2,128,8,128,106.75,59.5546875,234881024 +34,mlp.up_proj,3,32,8,128,34.75,98.21875,234881024 +34,mlp.up_proj,3,64,8,128,40.53125,91.109375,234881024 +34,mlp.up_proj,3,128,8,128,45.65625,87.5546875,234881024 +34,mlp.up_proj,4,32,8,128,16.203125,126.21875,234881024 +34,mlp.up_proj,4,64,8,128,18.90625,119.109375,234881024 +34,mlp.up_proj,4,128,8,128,21.3125,115.5546875,234881024 +34,mlp.up_proj,8,32,8,128,0.98583984375,238.21875,234881024 +34,mlp.up_proj,8,64,8,128,1.150390625,231.109375,234881024 +34,mlp.up_proj,8,128,8,128,1.29296875,227.5546875,234881024 +34,mlp.down_proj,2,32,8,128,81.4375,70.21875,234881024 +34,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024 +34,mlp.down_proj,2,128,8,128,106.75,59.5546875,234881024 +34,mlp.down_proj,3,32,8,128,34.8125,98.21875,234881024 +34,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024 +34,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024 +34,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024 +34,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024 +34,mlp.down_proj,4,128,8,128,21.3125,115.5546875,234881024 +34,mlp.down_proj,8,32,8,128,0.98681640625,238.21875,234881024 +34,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024 +34,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024 +35,self_attn.q_proj,2,32,8,128,43.4375,20.0625,67108864 +35,self_attn.q_proj,2,64,8,128,51.75,18.03125,67108864 +35,self_attn.q_proj,2,128,8,128,59.71875,17.015625,67108864 +35,self_attn.q_proj,3,32,8,128,18.65625,28.0625,67108864 +35,self_attn.q_proj,3,64,8,128,22.359375,26.03125,67108864 +35,self_attn.q_proj,3,128,8,128,26.125,25.015625,67108864 +35,self_attn.q_proj,4,32,8,128,8.703125,36.0625,67108864 +35,self_attn.q_proj,4,64,8,128,10.4296875,34.03125,67108864 +35,self_attn.q_proj,4,128,8,128,12.1875,33.015625,67108864 +35,self_attn.q_proj,8,32,8,128,0.52734375,68.0625,67108864 +35,self_attn.q_proj,8,64,8,128,0.63134765625,66.03125,67108864 +35,self_attn.q_proj,8,128,8,128,0.73486328125,65.015625,67108864 +35,self_attn.v_proj,2,32,8,128,15.046875,2.5078125,8388608 +35,self_attn.v_proj,2,64,8,128,17.625,2.25390625,8388608 +35,self_attn.v_proj,2,128,8,128,19.96875,2.126953125,8388608 +35,self_attn.v_proj,3,32,8,128,6.4375,3.5078125,8388608 +35,self_attn.v_proj,3,64,8,128,7.5390625,3.25390625,8388608 +35,self_attn.v_proj,3,128,8,128,8.546875,3.126953125,8388608 +35,self_attn.v_proj,4,32,8,128,3.001953125,4.5078125,8388608 +35,self_attn.v_proj,4,64,8,128,3.515625,4.25390625,8388608 +35,self_attn.v_proj,4,128,8,128,3.986328125,4.126953125,8388608 +35,self_attn.v_proj,8,32,8,128,0.181884765625,8.5078125,8388608 +35,self_attn.v_proj,8,64,8,128,0.2127685546875,8.25390625,8388608 +35,self_attn.v_proj,8,128,8,128,0.2403564453125,8.126953125,8388608 +35,self_attn.k_proj,2,32,8,128,24.84375,2.5078125,8388608 +35,self_attn.k_proj,2,64,8,128,29.875,2.25390625,8388608 +35,self_attn.k_proj,2,128,8,128,34.0,2.126953125,8388608 +35,self_attn.k_proj,3,32,8,128,10.8046875,3.5078125,8388608 +35,self_attn.k_proj,3,64,8,128,13.2578125,3.25390625,8388608 +35,self_attn.k_proj,3,128,8,128,15.6015625,3.126953125,8388608 +35,self_attn.k_proj,4,32,8,128,5.04296875,4.5078125,8388608 +35,self_attn.k_proj,4,64,8,128,6.19140625,4.25390625,8388608 +35,self_attn.k_proj,4,128,8,128,7.29296875,4.126953125,8388608 +35,self_attn.k_proj,8,32,8,128,0.30517578125,8.5078125,8388608 +35,self_attn.k_proj,8,64,8,128,0.37451171875,8.25390625,8388608 +35,self_attn.k_proj,8,128,8,128,0.439453125,8.126953125,8388608 +35,self_attn.o_proj,2,32,8,128,39.4375,20.0625,67108864 +35,self_attn.o_proj,2,64,8,128,46.0,18.03125,67108864 +35,self_attn.o_proj,2,128,8,128,51.96875,17.015625,67108864 +35,self_attn.o_proj,3,32,8,128,16.890625,28.0625,67108864 +35,self_attn.o_proj,3,64,8,128,19.734375,26.03125,67108864 +35,self_attn.o_proj,3,128,8,128,22.359375,25.015625,67108864 +35,self_attn.o_proj,4,32,8,128,7.8828125,36.0625,67108864 +35,self_attn.o_proj,4,64,8,128,9.2109375,34.03125,67108864 +35,self_attn.o_proj,4,128,8,128,10.4453125,33.015625,67108864 +35,self_attn.o_proj,8,32,8,128,0.477294921875,68.0625,67108864 +35,self_attn.o_proj,8,64,8,128,0.5576171875,66.03125,67108864 +35,self_attn.o_proj,8,128,8,128,0.63037109375,65.015625,67108864 +35,mlp.gate_proj,2,32,8,128,84.0625,70.21875,234881024 +35,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024 +35,mlp.gate_proj,2,128,8,128,111.375,59.5546875,234881024 +35,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024 +35,mlp.gate_proj,3,64,8,128,42.0625,91.109375,234881024 +35,mlp.gate_proj,3,128,8,128,47.6875,87.5546875,234881024 +35,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024 +35,mlp.gate_proj,4,64,8,128,19.640625,119.109375,234881024 +35,mlp.gate_proj,4,128,8,128,22.25,115.5546875,234881024 +35,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024 +35,mlp.gate_proj,8,64,8,128,1.1943359375,231.109375,234881024 +35,mlp.gate_proj,8,128,8,128,1.349609375,227.5546875,234881024 +35,mlp.up_proj,2,32,8,128,81.1875,70.21875,234881024 +35,mlp.up_proj,2,64,8,128,94.5625,63.109375,234881024 +35,mlp.up_proj,2,128,8,128,106.625,59.5546875,234881024 +35,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024 +35,mlp.up_proj,3,64,8,128,40.46875,91.109375,234881024 +35,mlp.up_proj,3,128,8,128,45.59375,87.5546875,234881024 +35,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024 +35,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024 +35,mlp.up_proj,4,128,8,128,21.28125,115.5546875,234881024 +35,mlp.up_proj,8,32,8,128,0.984375,238.21875,234881024 +35,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024 +35,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +35,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024 +35,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024 +35,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024 +35,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +35,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024 +35,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024 +35,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +35,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024 +35,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +35,mlp.down_proj,8,32,8,128,0.98583984375,238.21875,234881024 +35,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +35,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +36,self_attn.q_proj,2,32,8,128,44.28125,20.0625,67108864 +36,self_attn.q_proj,2,64,8,128,52.46875,18.03125,67108864 +36,self_attn.q_proj,2,128,8,128,60.28125,17.015625,67108864 +36,self_attn.q_proj,3,32,8,128,19.015625,28.0625,67108864 +36,self_attn.q_proj,3,64,8,128,22.640625,26.03125,67108864 +36,self_attn.q_proj,3,128,8,128,26.28125,25.015625,67108864 +36,self_attn.q_proj,4,32,8,128,8.875,36.0625,67108864 +36,self_attn.q_proj,4,64,8,128,10.5703125,34.03125,67108864 +36,self_attn.q_proj,4,128,8,128,12.2734375,33.015625,67108864 +36,self_attn.q_proj,8,32,8,128,0.537109375,68.0625,67108864 +36,self_attn.q_proj,8,64,8,128,0.6396484375,66.03125,67108864 +36,self_attn.q_proj,8,128,8,128,0.73974609375,65.015625,67108864 +36,self_attn.v_proj,2,32,8,128,15.265625,2.5078125,8388608 +36,self_attn.v_proj,2,64,8,128,17.84375,2.25390625,8388608 +36,self_attn.v_proj,2,128,8,128,20.1875,2.126953125,8388608 +36,self_attn.v_proj,3,32,8,128,6.5234375,3.5078125,8388608 +36,self_attn.v_proj,3,64,8,128,7.63671875,3.25390625,8388608 +36,self_attn.v_proj,3,128,8,128,8.6484375,3.126953125,8388608 +36,self_attn.v_proj,4,32,8,128,3.044921875,4.5078125,8388608 +36,self_attn.v_proj,4,64,8,128,3.5625,4.25390625,8388608 +36,self_attn.v_proj,4,128,8,128,4.03125,4.126953125,8388608 +36,self_attn.v_proj,8,32,8,128,0.184326171875,8.5078125,8388608 +36,self_attn.v_proj,8,64,8,128,0.215576171875,8.25390625,8388608 +36,self_attn.v_proj,8,128,8,128,0.242919921875,8.126953125,8388608 +36,self_attn.k_proj,2,32,8,128,25.546875,2.5078125,8388608 +36,self_attn.k_proj,2,64,8,128,30.53125,2.25390625,8388608 +36,self_attn.k_proj,2,128,8,128,34.46875,2.126953125,8388608 +36,self_attn.k_proj,3,32,8,128,11.1328125,3.5078125,8388608 +36,self_attn.k_proj,3,64,8,128,13.6015625,3.25390625,8388608 +36,self_attn.k_proj,3,128,8,128,15.890625,3.126953125,8388608 +36,self_attn.k_proj,4,32,8,128,5.1953125,4.5078125,8388608 +36,self_attn.k_proj,4,64,8,128,6.34765625,4.25390625,8388608 +36,self_attn.k_proj,4,128,8,128,7.4296875,4.126953125,8388608 +36,self_attn.k_proj,8,32,8,128,0.314453125,8.5078125,8388608 +36,self_attn.k_proj,8,64,8,128,0.3837890625,8.25390625,8388608 +36,self_attn.k_proj,8,128,8,128,0.44775390625,8.126953125,8388608 +36,self_attn.o_proj,2,32,8,128,39.375,20.0625,67108864 +36,self_attn.o_proj,2,64,8,128,45.9375,18.03125,67108864 +36,self_attn.o_proj,2,128,8,128,51.9375,17.015625,67108864 +36,self_attn.o_proj,3,32,8,128,16.84375,28.0625,67108864 +36,self_attn.o_proj,3,64,8,128,19.6875,26.03125,67108864 +36,self_attn.o_proj,3,128,8,128,22.3125,25.015625,67108864 +36,self_attn.o_proj,4,32,8,128,7.86328125,36.0625,67108864 +36,self_attn.o_proj,4,64,8,128,9.1953125,34.03125,67108864 +36,self_attn.o_proj,4,128,8,128,10.421875,33.015625,67108864 +36,self_attn.o_proj,8,32,8,128,0.47607421875,68.0625,67108864 +36,self_attn.o_proj,8,64,8,128,0.55615234375,66.03125,67108864 +36,self_attn.o_proj,8,128,8,128,0.62841796875,65.015625,67108864 +36,mlp.gate_proj,2,32,8,128,84.125,70.21875,234881024 +36,mlp.gate_proj,2,64,8,128,98.4375,63.109375,234881024 +36,mlp.gate_proj,2,128,8,128,111.6875,59.5546875,234881024 +36,mlp.gate_proj,3,32,8,128,36.0,98.21875,234881024 +36,mlp.gate_proj,3,64,8,128,42.15625,91.109375,234881024 +36,mlp.gate_proj,3,128,8,128,47.8125,87.5546875,234881024 +36,mlp.gate_proj,4,32,8,128,16.78125,126.21875,234881024 +36,mlp.gate_proj,4,64,8,128,19.6875,119.109375,234881024 +36,mlp.gate_proj,4,128,8,128,22.3125,115.5546875,234881024 +36,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024 +36,mlp.gate_proj,8,64,8,128,1.197265625,231.109375,234881024 +36,mlp.gate_proj,8,128,8,128,1.3544921875,227.5546875,234881024 +36,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024 +36,mlp.up_proj,2,64,8,128,94.375,63.109375,234881024 +36,mlp.up_proj,2,128,8,128,106.5,59.5546875,234881024 +36,mlp.up_proj,3,32,8,128,34.65625,98.21875,234881024 +36,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024 +36,mlp.up_proj,3,128,8,128,45.5625,87.5546875,234881024 +36,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024 +36,mlp.up_proj,4,64,8,128,18.859375,119.109375,234881024 +36,mlp.up_proj,4,128,8,128,21.265625,115.5546875,234881024 +36,mlp.up_proj,8,32,8,128,0.98291015625,238.21875,234881024 +36,mlp.up_proj,8,64,8,128,1.1474609375,231.109375,234881024 +36,mlp.up_proj,8,128,8,128,1.2900390625,227.5546875,234881024 +36,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024 +36,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024 +36,mlp.down_proj,2,128,8,128,106.6875,59.5546875,234881024 +36,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +36,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024 +36,mlp.down_proj,3,128,8,128,45.625,87.5546875,234881024 +36,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +36,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024 +36,mlp.down_proj,4,128,8,128,21.296875,115.5546875,234881024 +36,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024 +36,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024 +36,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +37,self_attn.q_proj,2,32,8,128,44.90625,20.0625,67108864 +37,self_attn.q_proj,2,64,8,128,52.96875,18.03125,67108864 +37,self_attn.q_proj,2,128,8,128,60.71875,17.015625,67108864 +37,self_attn.q_proj,3,32,8,128,19.234375,28.0625,67108864 +37,self_attn.q_proj,3,64,8,128,22.75,26.03125,67108864 +37,self_attn.q_proj,3,128,8,128,26.203125,25.015625,67108864 +37,self_attn.q_proj,4,32,8,128,8.9765625,36.0625,67108864 +37,self_attn.q_proj,4,64,8,128,10.6171875,34.03125,67108864 +37,self_attn.q_proj,4,128,8,128,12.234375,33.015625,67108864 +37,self_attn.q_proj,8,32,8,128,0.54345703125,68.0625,67108864 +37,self_attn.q_proj,8,64,8,128,0.642578125,66.03125,67108864 +37,self_attn.q_proj,8,128,8,128,0.7373046875,65.015625,67108864 +37,self_attn.v_proj,2,32,8,128,15.421875,2.5078125,8388608 +37,self_attn.v_proj,2,64,8,128,18.03125,2.25390625,8388608 +37,self_attn.v_proj,2,128,8,128,20.390625,2.126953125,8388608 +37,self_attn.v_proj,3,32,8,128,6.59375,3.5078125,8388608 +37,self_attn.v_proj,3,64,8,128,7.7109375,3.25390625,8388608 +37,self_attn.v_proj,3,128,8,128,8.71875,3.126953125,8388608 +37,self_attn.v_proj,4,32,8,128,3.076171875,4.5078125,8388608 +37,self_attn.v_proj,4,64,8,128,3.59765625,4.25390625,8388608 +37,self_attn.v_proj,4,128,8,128,4.06640625,4.126953125,8388608 +37,self_attn.v_proj,8,32,8,128,0.186279296875,8.5078125,8388608 +37,self_attn.v_proj,8,64,8,128,0.2176513671875,8.25390625,8388608 +37,self_attn.v_proj,8,128,8,128,0.2452392578125,8.126953125,8388608 +37,self_attn.k_proj,2,32,8,128,25.671875,2.5078125,8388608 +37,self_attn.k_proj,2,64,8,128,30.65625,2.25390625,8388608 +37,self_attn.k_proj,2,128,8,128,34.75,2.126953125,8388608 +37,self_attn.k_proj,3,32,8,128,11.1171875,3.5078125,8388608 +37,self_attn.k_proj,3,64,8,128,13.4765625,3.25390625,8388608 +37,self_attn.k_proj,3,128,8,128,15.703125,3.126953125,8388608 +37,self_attn.k_proj,4,32,8,128,5.1875,4.5078125,8388608 +37,self_attn.k_proj,4,64,8,128,6.296875,4.25390625,8388608 +37,self_attn.k_proj,4,128,8,128,7.34375,4.126953125,8388608 +37,self_attn.k_proj,8,32,8,128,0.314208984375,8.5078125,8388608 +37,self_attn.k_proj,8,64,8,128,0.380615234375,8.25390625,8388608 +37,self_attn.k_proj,8,128,8,128,0.4423828125,8.126953125,8388608 +37,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864 +37,self_attn.o_proj,2,64,8,128,46.84375,18.03125,67108864 +37,self_attn.o_proj,2,128,8,128,52.9375,17.015625,67108864 +37,self_attn.o_proj,3,32,8,128,17.171875,28.0625,67108864 +37,self_attn.o_proj,3,64,8,128,20.0625,26.03125,67108864 +37,self_attn.o_proj,3,128,8,128,22.703125,25.015625,67108864 +37,self_attn.o_proj,4,32,8,128,8.015625,36.0625,67108864 +37,self_attn.o_proj,4,64,8,128,9.359375,34.03125,67108864 +37,self_attn.o_proj,4,128,8,128,10.6015625,33.015625,67108864 +37,self_attn.o_proj,8,32,8,128,0.485107421875,68.0625,67108864 +37,self_attn.o_proj,8,64,8,128,0.56640625,66.03125,67108864 +37,self_attn.o_proj,8,128,8,128,0.63916015625,65.015625,67108864 +37,mlp.gate_proj,2,32,8,128,84.25,70.21875,234881024 +37,mlp.gate_proj,2,64,8,128,98.5625,63.109375,234881024 +37,mlp.gate_proj,2,128,8,128,111.875,59.5546875,234881024 +37,mlp.gate_proj,3,32,8,128,36.03125,98.21875,234881024 +37,mlp.gate_proj,3,64,8,128,42.1875,91.109375,234881024 +37,mlp.gate_proj,3,128,8,128,47.90625,87.5546875,234881024 +37,mlp.gate_proj,4,32,8,128,16.796875,126.21875,234881024 +37,mlp.gate_proj,4,64,8,128,19.703125,119.109375,234881024 +37,mlp.gate_proj,4,128,8,128,22.359375,115.5546875,234881024 +37,mlp.gate_proj,8,32,8,128,1.0224609375,238.21875,234881024 +37,mlp.gate_proj,8,64,8,128,1.19921875,231.109375,234881024 +37,mlp.gate_proj,8,128,8,128,1.3564453125,227.5546875,234881024 +37,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024 +37,mlp.up_proj,2,64,8,128,94.4375,63.109375,234881024 +37,mlp.up_proj,2,128,8,128,106.5625,59.5546875,234881024 +37,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024 +37,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024 +37,mlp.up_proj,3,128,8,128,45.59375,87.5546875,234881024 +37,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024 +37,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024 +37,mlp.up_proj,4,128,8,128,21.28125,115.5546875,234881024 +37,mlp.up_proj,8,32,8,128,0.98388671875,238.21875,234881024 +37,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024 +37,mlp.up_proj,8,128,8,128,1.291015625,227.5546875,234881024 +37,mlp.down_proj,2,32,8,128,81.4375,70.21875,234881024 +37,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024 +37,mlp.down_proj,2,128,8,128,106.8125,59.5546875,234881024 +37,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +37,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024 +37,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024 +37,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024 +37,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024 +37,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024 +37,mlp.down_proj,8,32,8,128,0.98681640625,238.21875,234881024 +37,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024 +37,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024 +38,self_attn.q_proj,2,32,8,128,46.5625,20.0625,67108864 +38,self_attn.q_proj,2,64,8,128,55.25,18.03125,67108864 +38,self_attn.q_proj,2,128,8,128,63.65625,17.015625,67108864 +38,self_attn.q_proj,3,32,8,128,19.96875,28.0625,67108864 +38,self_attn.q_proj,3,64,8,128,23.765625,26.03125,67108864 +38,self_attn.q_proj,3,128,8,128,27.59375,25.015625,67108864 +38,self_attn.q_proj,4,32,8,128,9.3125,36.0625,67108864 +38,self_attn.q_proj,4,64,8,128,11.0859375,34.03125,67108864 +38,self_attn.q_proj,4,128,8,128,12.8828125,33.015625,67108864 +38,self_attn.q_proj,8,32,8,128,0.56396484375,68.0625,67108864 +38,self_attn.q_proj,8,64,8,128,0.6708984375,66.03125,67108864 +38,self_attn.q_proj,8,128,8,128,0.7763671875,65.015625,67108864 +38,self_attn.v_proj,2,32,8,128,14.828125,2.5078125,8388608 +38,self_attn.v_proj,2,64,8,128,17.34375,2.25390625,8388608 +38,self_attn.v_proj,2,128,8,128,19.625,2.126953125,8388608 +38,self_attn.v_proj,3,32,8,128,6.33984375,3.5078125,8388608 +38,self_attn.v_proj,3,64,8,128,7.4140625,3.25390625,8388608 +38,self_attn.v_proj,3,128,8,128,8.390625,3.126953125,8388608 +38,self_attn.v_proj,4,32,8,128,2.95703125,4.5078125,8388608 +38,self_attn.v_proj,4,64,8,128,3.4609375,4.25390625,8388608 +38,self_attn.v_proj,4,128,8,128,3.916015625,4.126953125,8388608 +38,self_attn.v_proj,8,32,8,128,0.1790771484375,8.5078125,8388608 +38,self_attn.v_proj,8,64,8,128,0.20947265625,8.25390625,8388608 +38,self_attn.v_proj,8,128,8,128,0.236083984375,8.126953125,8388608 +38,self_attn.k_proj,2,32,8,128,25.828125,2.5078125,8388608 +38,self_attn.k_proj,2,64,8,128,30.90625,2.25390625,8388608 +38,self_attn.k_proj,2,128,8,128,35.0,2.126953125,8388608 +38,self_attn.k_proj,3,32,8,128,11.1796875,3.5078125,8388608 +38,self_attn.k_proj,3,64,8,128,13.625,3.25390625,8388608 +38,self_attn.k_proj,3,128,8,128,15.890625,3.126953125,8388608 +38,self_attn.k_proj,4,32,8,128,5.21484375,4.5078125,8388608 +38,self_attn.k_proj,4,64,8,128,6.359375,4.25390625,8388608 +38,self_attn.k_proj,4,128,8,128,7.42578125,4.126953125,8388608 +38,self_attn.k_proj,8,32,8,128,0.315673828125,8.5078125,8388608 +38,self_attn.k_proj,8,64,8,128,0.384521484375,8.25390625,8388608 +38,self_attn.k_proj,8,128,8,128,0.447509765625,8.126953125,8388608 +38,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864 +38,self_attn.o_proj,2,64,8,128,46.84375,18.03125,67108864 +38,self_attn.o_proj,2,128,8,128,52.96875,17.015625,67108864 +38,self_attn.o_proj,3,32,8,128,17.171875,28.0625,67108864 +38,self_attn.o_proj,3,64,8,128,20.078125,26.03125,67108864 +38,self_attn.o_proj,3,128,8,128,22.78125,25.015625,67108864 +38,self_attn.o_proj,4,32,8,128,8.015625,36.0625,67108864 +38,self_attn.o_proj,4,64,8,128,9.375,34.03125,67108864 +38,self_attn.o_proj,4,128,8,128,10.6328125,33.015625,67108864 +38,self_attn.o_proj,8,32,8,128,0.4853515625,68.0625,67108864 +38,self_attn.o_proj,8,64,8,128,0.56689453125,66.03125,67108864 +38,self_attn.o_proj,8,128,8,128,0.64111328125,65.015625,67108864 +38,mlp.gate_proj,2,32,8,128,84.375,70.21875,234881024 +38,mlp.gate_proj,2,64,8,128,98.8125,63.109375,234881024 +38,mlp.gate_proj,2,128,8,128,112.1875,59.5546875,234881024 +38,mlp.gate_proj,3,32,8,128,36.09375,98.21875,234881024 +38,mlp.gate_proj,3,64,8,128,42.28125,91.109375,234881024 +38,mlp.gate_proj,3,128,8,128,48.03125,87.5546875,234881024 +38,mlp.gate_proj,4,32,8,128,16.828125,126.21875,234881024 +38,mlp.gate_proj,4,64,8,128,19.75,119.109375,234881024 +38,mlp.gate_proj,4,128,8,128,22.421875,115.5546875,234881024 +38,mlp.gate_proj,8,32,8,128,1.0234375,238.21875,234881024 +38,mlp.gate_proj,8,64,8,128,1.201171875,231.109375,234881024 +38,mlp.gate_proj,8,128,8,128,1.3603515625,227.5546875,234881024 +38,mlp.up_proj,2,32,8,128,81.25,70.21875,234881024 +38,mlp.up_proj,2,64,8,128,94.5625,63.109375,234881024 +38,mlp.up_proj,2,128,8,128,106.6875,59.5546875,234881024 +38,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024 +38,mlp.up_proj,3,64,8,128,40.46875,91.109375,234881024 +38,mlp.up_proj,3,128,8,128,45.65625,87.5546875,234881024 +38,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024 +38,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024 +38,mlp.up_proj,4,128,8,128,21.3125,115.5546875,234881024 +38,mlp.up_proj,8,32,8,128,0.98486328125,238.21875,234881024 +38,mlp.up_proj,8,64,8,128,1.1494140625,231.109375,234881024 +38,mlp.up_proj,8,128,8,128,1.29296875,227.5546875,234881024 +38,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024 +38,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024 +38,mlp.down_proj,2,128,8,128,106.875,59.5546875,234881024 +38,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024 +38,mlp.down_proj,3,64,8,128,40.59375,91.109375,234881024 +38,mlp.down_proj,3,128,8,128,45.71875,87.5546875,234881024 +38,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024 +38,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024 +38,mlp.down_proj,4,128,8,128,21.34375,115.5546875,234881024 +38,mlp.down_proj,8,32,8,128,0.98828125,238.21875,234881024 +38,mlp.down_proj,8,64,8,128,1.15234375,231.109375,234881024 +38,mlp.down_proj,8,128,8,128,1.294921875,227.5546875,234881024 +39,self_attn.q_proj,2,32,8,128,48.0,20.0625,67108864 +39,self_attn.q_proj,2,64,8,128,56.46875,18.03125,67108864 +39,self_attn.q_proj,2,128,8,128,65.0,17.015625,67108864 +39,self_attn.q_proj,3,32,8,128,20.546875,28.0625,67108864 +39,self_attn.q_proj,3,64,8,128,24.203125,26.03125,67108864 +39,self_attn.q_proj,3,128,8,128,27.953125,25.015625,67108864 +39,self_attn.q_proj,4,32,8,128,9.5859375,36.0625,67108864 +39,self_attn.q_proj,4,64,8,128,11.2890625,34.03125,67108864 +39,self_attn.q_proj,4,128,8,128,13.0390625,33.015625,67108864 +39,self_attn.q_proj,8,32,8,128,0.58056640625,68.0625,67108864 +39,self_attn.q_proj,8,64,8,128,0.68310546875,66.03125,67108864 +39,self_attn.q_proj,8,128,8,128,0.7861328125,65.015625,67108864 +39,self_attn.v_proj,2,32,8,128,15.125,2.5078125,8388608 +39,self_attn.v_proj,2,64,8,128,17.6875,2.25390625,8388608 +39,self_attn.v_proj,2,128,8,128,20.0,2.126953125,8388608 +39,self_attn.v_proj,3,32,8,128,6.46875,3.5078125,8388608 +39,self_attn.v_proj,3,64,8,128,7.56640625,3.25390625,8388608 +39,self_attn.v_proj,3,128,8,128,8.5625,3.126953125,8388608 +39,self_attn.v_proj,4,32,8,128,3.017578125,4.5078125,8388608 +39,self_attn.v_proj,4,64,8,128,3.53125,4.25390625,8388608 +39,self_attn.v_proj,4,128,8,128,3.99609375,4.126953125,8388608 +39,self_attn.v_proj,8,32,8,128,0.1827392578125,8.5078125,8388608 +39,self_attn.v_proj,8,64,8,128,0.213623046875,8.25390625,8388608 +39,self_attn.v_proj,8,128,8,128,0.240966796875,8.126953125,8388608 +39,self_attn.k_proj,2,32,8,128,25.8125,2.5078125,8388608 +39,self_attn.k_proj,2,64,8,128,30.96875,2.25390625,8388608 +39,self_attn.k_proj,2,128,8,128,35.375,2.126953125,8388608 +39,self_attn.k_proj,3,32,8,128,11.09375,3.5078125,8388608 +39,self_attn.k_proj,3,64,8,128,13.390625,3.25390625,8388608 +39,self_attn.k_proj,3,128,8,128,15.5234375,3.126953125,8388608 +39,self_attn.k_proj,4,32,8,128,5.17578125,4.5078125,8388608 +39,self_attn.k_proj,4,64,8,128,6.25,4.25390625,8388608 +39,self_attn.k_proj,4,128,8,128,7.25,4.126953125,8388608 +39,self_attn.k_proj,8,32,8,128,0.313232421875,8.5078125,8388608 +39,self_attn.k_proj,8,64,8,128,0.3779296875,8.25390625,8388608 +39,self_attn.k_proj,8,128,8,128,0.436767578125,8.126953125,8388608 +39,self_attn.o_proj,2,32,8,128,41.1875,20.0625,67108864 +39,self_attn.o_proj,2,64,8,128,48.0625,18.03125,67108864 +39,self_attn.o_proj,2,128,8,128,54.34375,17.015625,67108864 +39,self_attn.o_proj,3,32,8,128,17.625,28.0625,67108864 +39,self_attn.o_proj,3,64,8,128,20.59375,26.03125,67108864 +39,self_attn.o_proj,3,128,8,128,23.359375,25.015625,67108864 +39,self_attn.o_proj,4,32,8,128,8.21875,36.0625,67108864 +39,self_attn.o_proj,4,64,8,128,9.6171875,34.03125,67108864 +39,self_attn.o_proj,4,128,8,128,10.90625,33.015625,67108864 +39,self_attn.o_proj,8,32,8,128,0.497802734375,68.0625,67108864 +39,self_attn.o_proj,8,64,8,128,0.58203125,66.03125,67108864 +39,self_attn.o_proj,8,128,8,128,0.6572265625,65.015625,67108864 +39,mlp.gate_proj,2,32,8,128,84.625,70.21875,234881024 +39,mlp.gate_proj,2,64,8,128,99.125,63.109375,234881024 +39,mlp.gate_proj,2,128,8,128,112.6875,59.5546875,234881024 +39,mlp.gate_proj,3,32,8,128,36.1875,98.21875,234881024 +39,mlp.gate_proj,3,64,8,128,42.4375,91.109375,234881024 +39,mlp.gate_proj,3,128,8,128,48.28125,87.5546875,234881024 +39,mlp.gate_proj,4,32,8,128,16.875,126.21875,234881024 +39,mlp.gate_proj,4,64,8,128,19.8125,119.109375,234881024 +39,mlp.gate_proj,4,128,8,128,22.53125,115.5546875,234881024 +39,mlp.gate_proj,8,32,8,128,1.0263671875,238.21875,234881024 +39,mlp.gate_proj,8,64,8,128,1.2060546875,231.109375,234881024 +39,mlp.gate_proj,8,128,8,128,1.3671875,227.5546875,234881024 +39,mlp.up_proj,2,32,8,128,81.4375,70.21875,234881024 +39,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024 +39,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024 +39,mlp.up_proj,3,32,8,128,34.8125,98.21875,234881024 +39,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024 +39,mlp.up_proj,3,128,8,128,45.8125,87.5546875,234881024 +39,mlp.up_proj,4,32,8,128,16.234375,126.21875,234881024 +39,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024 +39,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024 +39,mlp.up_proj,8,32,8,128,0.98779296875,238.21875,234881024 +39,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +39,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024 +39,mlp.down_proj,2,32,8,128,81.8125,70.21875,234881024 +39,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024 +39,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024 +39,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024 +39,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024 +39,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024 +39,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024 +39,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024 +39,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024 +39,mlp.down_proj,8,32,8,128,0.9921875,238.21875,234881024 +39,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024 +39,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024 +40,self_attn.q_proj,2,32,8,128,47.28125,20.0625,67108864 +40,self_attn.q_proj,2,64,8,128,55.96875,18.03125,67108864 +40,self_attn.q_proj,2,128,8,128,64.6875,17.015625,67108864 +40,self_attn.q_proj,3,32,8,128,20.25,28.0625,67108864 +40,self_attn.q_proj,3,64,8,128,24.015625,26.03125,67108864 +40,self_attn.q_proj,3,128,8,128,27.875,25.015625,67108864 +40,self_attn.q_proj,4,32,8,128,9.4453125,36.0625,67108864 +40,self_attn.q_proj,4,64,8,128,11.203125,34.03125,67108864 +40,self_attn.q_proj,4,128,8,128,13.0078125,33.015625,67108864 +40,self_attn.q_proj,8,32,8,128,0.572265625,68.0625,67108864 +40,self_attn.q_proj,8,64,8,128,0.677734375,66.03125,67108864 +40,self_attn.q_proj,8,128,8,128,0.7841796875,65.015625,67108864 +40,self_attn.v_proj,2,32,8,128,16.09375,2.5078125,8388608 +40,self_attn.v_proj,2,64,8,128,18.828125,2.25390625,8388608 +40,self_attn.v_proj,2,128,8,128,21.3125,2.126953125,8388608 +40,self_attn.v_proj,3,32,8,128,6.8828125,3.5078125,8388608 +40,self_attn.v_proj,3,64,8,128,8.0546875,3.25390625,8388608 +40,self_attn.v_proj,3,128,8,128,9.1171875,3.126953125,8388608 +40,self_attn.v_proj,4,32,8,128,3.2109375,4.5078125,8388608 +40,self_attn.v_proj,4,64,8,128,3.7578125,4.25390625,8388608 +40,self_attn.v_proj,4,128,8,128,4.25390625,4.126953125,8388608 +40,self_attn.v_proj,8,32,8,128,0.1944580078125,8.5078125,8388608 +40,self_attn.v_proj,8,64,8,128,0.2274169921875,8.25390625,8388608 +40,self_attn.v_proj,8,128,8,128,0.25634765625,8.126953125,8388608 +40,self_attn.k_proj,2,32,8,128,25.421875,2.5078125,8388608 +40,self_attn.k_proj,2,64,8,128,30.59375,2.25390625,8388608 +40,self_attn.k_proj,2,128,8,128,34.875,2.126953125,8388608 +40,self_attn.k_proj,3,32,8,128,10.9453125,3.5078125,8388608 +40,self_attn.k_proj,3,64,8,128,13.3046875,3.25390625,8388608 +40,self_attn.k_proj,3,128,8,128,15.484375,3.126953125,8388608 +40,self_attn.k_proj,4,32,8,128,5.109375,4.5078125,8388608 +40,self_attn.k_proj,4,64,8,128,6.2109375,4.25390625,8388608 +40,self_attn.k_proj,4,128,8,128,7.23046875,4.126953125,8388608 +40,self_attn.k_proj,8,32,8,128,0.309326171875,8.5078125,8388608 +40,self_attn.k_proj,8,64,8,128,0.375732421875,8.25390625,8388608 +40,self_attn.k_proj,8,128,8,128,0.435546875,8.126953125,8388608 +40,self_attn.o_proj,2,32,8,128,43.09375,20.0625,67108864 +40,self_attn.o_proj,2,64,8,128,50.1875,18.03125,67108864 +40,self_attn.o_proj,2,128,8,128,56.59375,17.015625,67108864 +40,self_attn.o_proj,3,32,8,128,18.421875,28.0625,67108864 +40,self_attn.o_proj,3,64,8,128,21.484375,26.03125,67108864 +40,self_attn.o_proj,3,128,8,128,24.25,25.015625,67108864 +40,self_attn.o_proj,4,32,8,128,8.59375,36.0625,67108864 +40,self_attn.o_proj,4,64,8,128,10.0234375,34.03125,67108864 +40,self_attn.o_proj,4,128,8,128,11.3203125,33.015625,67108864 +40,self_attn.o_proj,8,32,8,128,0.5205078125,68.0625,67108864 +40,self_attn.o_proj,8,64,8,128,0.6064453125,66.03125,67108864 +40,self_attn.o_proj,8,128,8,128,0.68212890625,65.015625,67108864 +40,mlp.gate_proj,2,32,8,128,84.5,70.21875,234881024 +40,mlp.gate_proj,2,64,8,128,98.8125,63.109375,234881024 +40,mlp.gate_proj,2,128,8,128,112.125,59.5546875,234881024 +40,mlp.gate_proj,3,32,8,128,36.125,98.21875,234881024 +40,mlp.gate_proj,3,64,8,128,42.3125,91.109375,234881024 +40,mlp.gate_proj,3,128,8,128,48.03125,87.5546875,234881024 +40,mlp.gate_proj,4,32,8,128,16.84375,126.21875,234881024 +40,mlp.gate_proj,4,64,8,128,19.75,119.109375,234881024 +40,mlp.gate_proj,4,128,8,128,22.421875,115.5546875,234881024 +40,mlp.gate_proj,8,32,8,128,1.025390625,238.21875,234881024 +40,mlp.gate_proj,8,64,8,128,1.2021484375,231.109375,234881024 +40,mlp.gate_proj,8,128,8,128,1.359375,227.5546875,234881024 +40,mlp.up_proj,2,32,8,128,81.4375,70.21875,234881024 +40,mlp.up_proj,2,64,8,128,94.8125,63.109375,234881024 +40,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024 +40,mlp.up_proj,3,32,8,128,34.8125,98.21875,234881024 +40,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024 +40,mlp.up_proj,3,128,8,128,45.75,87.5546875,234881024 +40,mlp.up_proj,4,32,8,128,16.234375,126.21875,234881024 +40,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024 +40,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024 +40,mlp.up_proj,8,32,8,128,0.9873046875,238.21875,234881024 +40,mlp.up_proj,8,64,8,128,1.15234375,231.109375,234881024 +40,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +40,mlp.down_proj,2,32,8,128,82.0,70.21875,234881024 +40,mlp.down_proj,2,64,8,128,95.4375,63.109375,234881024 +40,mlp.down_proj,2,128,8,128,107.625,59.5546875,234881024 +40,mlp.down_proj,3,32,8,128,35.0625,98.21875,234881024 +40,mlp.down_proj,3,64,8,128,40.84375,91.109375,234881024 +40,mlp.down_proj,3,128,8,128,46.03125,87.5546875,234881024 +40,mlp.down_proj,4,32,8,128,16.34375,126.21875,234881024 +40,mlp.down_proj,4,64,8,128,19.078125,119.109375,234881024 +40,mlp.down_proj,4,128,8,128,21.5,115.5546875,234881024 +40,mlp.down_proj,8,32,8,128,0.99462890625,238.21875,234881024 +40,mlp.down_proj,8,64,8,128,1.16015625,231.109375,234881024 +40,mlp.down_proj,8,128,8,128,1.3037109375,227.5546875,234881024 +41,self_attn.q_proj,2,32,8,128,45.75,20.0625,67108864 +41,self_attn.q_proj,2,64,8,128,54.0,18.03125,67108864 +41,self_attn.q_proj,2,128,8,128,62.34375,17.015625,67108864 +41,self_attn.q_proj,3,32,8,128,19.59375,28.0625,67108864 +41,self_attn.q_proj,3,64,8,128,23.1875,26.03125,67108864 +41,self_attn.q_proj,3,128,8,128,26.9375,25.015625,67108864 +41,self_attn.q_proj,4,32,8,128,9.1484375,36.0625,67108864 +41,self_attn.q_proj,4,64,8,128,10.828125,34.03125,67108864 +41,self_attn.q_proj,4,128,8,128,12.5703125,33.015625,67108864 +41,self_attn.q_proj,8,32,8,128,0.5537109375,68.0625,67108864 +41,self_attn.q_proj,8,64,8,128,0.6552734375,66.03125,67108864 +41,self_attn.q_proj,8,128,8,128,0.7578125,65.015625,67108864 +41,self_attn.v_proj,2,32,8,128,15.5546875,2.5078125,8388608 +41,self_attn.v_proj,2,64,8,128,18.15625,2.25390625,8388608 +41,self_attn.v_proj,2,128,8,128,20.515625,2.126953125,8388608 +41,self_attn.v_proj,3,32,8,128,6.65234375,3.5078125,8388608 +41,self_attn.v_proj,3,64,8,128,7.765625,3.25390625,8388608 +41,self_attn.v_proj,3,128,8,128,8.78125,3.126953125,8388608 +41,self_attn.v_proj,4,32,8,128,3.1015625,4.5078125,8388608 +41,self_attn.v_proj,4,64,8,128,3.625,4.25390625,8388608 +41,self_attn.v_proj,4,128,8,128,4.09765625,4.126953125,8388608 +41,self_attn.v_proj,8,32,8,128,0.1878662109375,8.5078125,8388608 +41,self_attn.v_proj,8,64,8,128,0.2193603515625,8.25390625,8388608 +41,self_attn.v_proj,8,128,8,128,0.2469482421875,8.126953125,8388608 +41,self_attn.k_proj,2,32,8,128,26.21875,2.5078125,8388608 +41,self_attn.k_proj,2,64,8,128,31.71875,2.25390625,8388608 +41,self_attn.k_proj,2,128,8,128,36.34375,2.126953125,8388608 +41,self_attn.k_proj,3,32,8,128,11.34375,3.5078125,8388608 +41,self_attn.k_proj,3,64,8,128,13.953125,3.25390625,8388608 +41,self_attn.k_proj,3,128,8,128,16.453125,3.126953125,8388608 +41,self_attn.k_proj,4,32,8,128,5.2890625,4.5078125,8388608 +41,self_attn.k_proj,4,64,8,128,6.51171875,4.25390625,8388608 +41,self_attn.k_proj,4,128,8,128,7.6875,4.126953125,8388608 +41,self_attn.k_proj,8,32,8,128,0.3203125,8.5078125,8388608 +41,self_attn.k_proj,8,64,8,128,0.393798828125,8.25390625,8388608 +41,self_attn.k_proj,8,128,8,128,0.463134765625,8.126953125,8388608 +41,self_attn.o_proj,2,32,8,128,41.625,20.0625,67108864 +41,self_attn.o_proj,2,64,8,128,48.53125,18.03125,67108864 +41,self_attn.o_proj,2,128,8,128,54.8125,17.015625,67108864 +41,self_attn.o_proj,3,32,8,128,17.8125,28.0625,67108864 +41,self_attn.o_proj,3,64,8,128,20.796875,26.03125,67108864 +41,self_attn.o_proj,3,128,8,128,23.53125,25.015625,67108864 +41,self_attn.o_proj,4,32,8,128,8.3203125,36.0625,67108864 +41,self_attn.o_proj,4,64,8,128,9.734375,34.03125,67108864 +41,self_attn.o_proj,4,128,8,128,11.03125,33.015625,67108864 +41,self_attn.o_proj,8,32,8,128,0.5048828125,68.0625,67108864 +41,self_attn.o_proj,8,64,8,128,0.5908203125,66.03125,67108864 +41,self_attn.o_proj,8,128,8,128,0.66845703125,65.015625,67108864 +41,mlp.gate_proj,2,32,8,128,85.125,70.21875,234881024 +41,mlp.gate_proj,2,64,8,128,99.625,63.109375,234881024 +41,mlp.gate_proj,2,128,8,128,112.9375,59.5546875,234881024 +41,mlp.gate_proj,3,32,8,128,36.4375,98.21875,234881024 +41,mlp.gate_proj,3,64,8,128,42.625,91.109375,234881024 +41,mlp.gate_proj,3,128,8,128,48.375,87.5546875,234881024 +41,mlp.gate_proj,4,32,8,128,16.984375,126.21875,234881024 +41,mlp.gate_proj,4,64,8,128,19.90625,119.109375,234881024 +41,mlp.gate_proj,4,128,8,128,22.578125,115.5546875,234881024 +41,mlp.gate_proj,8,32,8,128,1.033203125,238.21875,234881024 +41,mlp.gate_proj,8,64,8,128,1.2109375,231.109375,234881024 +41,mlp.gate_proj,8,128,8,128,1.369140625,227.5546875,234881024 +41,mlp.up_proj,2,32,8,128,80.875,70.21875,234881024 +41,mlp.up_proj,2,64,8,128,94.1875,63.109375,234881024 +41,mlp.up_proj,2,128,8,128,106.25,59.5546875,234881024 +41,mlp.up_proj,3,32,8,128,34.59375,98.21875,234881024 +41,mlp.up_proj,3,64,8,128,40.3125,91.109375,234881024 +41,mlp.up_proj,3,128,8,128,45.4375,87.5546875,234881024 +41,mlp.up_proj,4,32,8,128,16.125,126.21875,234881024 +41,mlp.up_proj,4,64,8,128,18.8125,119.109375,234881024 +41,mlp.up_proj,4,128,8,128,21.21875,115.5546875,234881024 +41,mlp.up_proj,8,32,8,128,0.98095703125,238.21875,234881024 +41,mlp.up_proj,8,64,8,128,1.14453125,231.109375,234881024 +41,mlp.up_proj,8,128,8,128,1.287109375,227.5546875,234881024 +41,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024 +41,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024 +41,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024 +41,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024 +41,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024 +41,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024 +41,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024 +41,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024 +41,mlp.down_proj,4,128,8,128,21.4375,115.5546875,234881024 +41,mlp.down_proj,8,32,8,128,0.9921875,238.21875,234881024 +41,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024 +41,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024 +42,self_attn.q_proj,2,32,8,128,44.5,20.0625,67108864 +42,self_attn.q_proj,2,64,8,128,52.4375,18.03125,67108864 +42,self_attn.q_proj,2,128,8,128,60.5625,17.015625,67108864 +42,self_attn.q_proj,3,32,8,128,19.0625,28.0625,67108864 +42,self_attn.q_proj,3,64,8,128,22.484375,26.03125,67108864 +42,self_attn.q_proj,3,128,8,128,26.0625,25.015625,67108864 +42,self_attn.q_proj,4,32,8,128,8.890625,36.0625,67108864 +42,self_attn.q_proj,4,64,8,128,10.5,34.03125,67108864 +42,self_attn.q_proj,4,128,8,128,12.1640625,33.015625,67108864 +42,self_attn.q_proj,8,32,8,128,0.53857421875,68.0625,67108864 +42,self_attn.q_proj,8,64,8,128,0.63525390625,66.03125,67108864 +42,self_attn.q_proj,8,128,8,128,0.7333984375,65.015625,67108864 +42,self_attn.v_proj,2,32,8,128,16.25,2.5078125,8388608 +42,self_attn.v_proj,2,64,8,128,19.0625,2.25390625,8388608 +42,self_attn.v_proj,2,128,8,128,21.625,2.126953125,8388608 +42,self_attn.v_proj,3,32,8,128,6.94921875,3.5078125,8388608 +42,self_attn.v_proj,3,64,8,128,8.15625,3.25390625,8388608 +42,self_attn.v_proj,3,128,8,128,9.2578125,3.126953125,8388608 +42,self_attn.v_proj,4,32,8,128,3.2421875,4.5078125,8388608 +42,self_attn.v_proj,4,64,8,128,3.8046875,4.25390625,8388608 +42,self_attn.v_proj,4,128,8,128,4.3203125,4.126953125,8388608 +42,self_attn.v_proj,8,32,8,128,0.1964111328125,8.5078125,8388608 +42,self_attn.v_proj,8,64,8,128,0.230224609375,8.25390625,8388608 +42,self_attn.v_proj,8,128,8,128,0.26025390625,8.126953125,8388608 +42,self_attn.k_proj,2,32,8,128,25.078125,2.5078125,8388608 +42,self_attn.k_proj,2,64,8,128,30.296875,2.25390625,8388608 +42,self_attn.k_proj,2,128,8,128,34.59375,2.126953125,8388608 +42,self_attn.k_proj,3,32,8,128,10.875,3.5078125,8388608 +42,self_attn.k_proj,3,64,8,128,13.40625,3.25390625,8388608 +42,self_attn.k_proj,3,128,8,128,15.796875,3.126953125,8388608 +42,self_attn.k_proj,4,32,8,128,5.07421875,4.5078125,8388608 +42,self_attn.k_proj,4,64,8,128,6.2578125,4.25390625,8388608 +42,self_attn.k_proj,4,128,8,128,7.37890625,4.126953125,8388608 +42,self_attn.k_proj,8,32,8,128,0.30712890625,8.5078125,8388608 +42,self_attn.k_proj,8,64,8,128,0.37841796875,8.25390625,8388608 +42,self_attn.k_proj,8,128,8,128,0.44482421875,8.126953125,8388608 +42,self_attn.o_proj,2,32,8,128,41.90625,20.0625,67108864 +42,self_attn.o_proj,2,64,8,128,48.875,18.03125,67108864 +42,self_attn.o_proj,2,128,8,128,55.25,17.015625,67108864 +42,self_attn.o_proj,3,32,8,128,17.96875,28.0625,67108864 +42,self_attn.o_proj,3,64,8,128,21.03125,26.03125,67108864 +42,self_attn.o_proj,3,128,8,128,23.859375,25.015625,67108864 +42,self_attn.o_proj,4,32,8,128,8.3984375,36.0625,67108864 +42,self_attn.o_proj,4,64,8,128,9.8515625,34.03125,67108864 +42,self_attn.o_proj,4,128,8,128,11.2109375,33.015625,67108864 +42,self_attn.o_proj,8,32,8,128,0.50927734375,68.0625,67108864 +42,self_attn.o_proj,8,64,8,128,0.59716796875,66.03125,67108864 +42,self_attn.o_proj,8,128,8,128,0.67822265625,65.015625,67108864 +42,mlp.gate_proj,2,32,8,128,86.0,70.21875,234881024 +42,mlp.gate_proj,2,64,8,128,100.5625,63.109375,234881024 +42,mlp.gate_proj,2,128,8,128,113.875,59.5546875,234881024 +42,mlp.gate_proj,3,32,8,128,36.8125,98.21875,234881024 +42,mlp.gate_proj,3,64,8,128,43.03125,91.109375,234881024 +42,mlp.gate_proj,3,128,8,128,48.78125,87.5546875,234881024 +42,mlp.gate_proj,4,32,8,128,17.15625,126.21875,234881024 +42,mlp.gate_proj,4,64,8,128,20.09375,119.109375,234881024 +42,mlp.gate_proj,4,128,8,128,22.75,115.5546875,234881024 +42,mlp.gate_proj,8,32,8,128,1.0439453125,238.21875,234881024 +42,mlp.gate_proj,8,64,8,128,1.22265625,231.109375,234881024 +42,mlp.gate_proj,8,128,8,128,1.380859375,227.5546875,234881024 +42,mlp.up_proj,2,32,8,128,80.5625,70.21875,234881024 +42,mlp.up_proj,2,64,8,128,93.8125,63.109375,234881024 +42,mlp.up_proj,2,128,8,128,105.75,59.5546875,234881024 +42,mlp.up_proj,3,32,8,128,34.4375,98.21875,234881024 +42,mlp.up_proj,3,64,8,128,40.15625,91.109375,234881024 +42,mlp.up_proj,3,128,8,128,45.25,87.5546875,234881024 +42,mlp.up_proj,4,32,8,128,16.0625,126.21875,234881024 +42,mlp.up_proj,4,64,8,128,18.734375,119.109375,234881024 +42,mlp.up_proj,4,128,8,128,21.125,115.5546875,234881024 +42,mlp.up_proj,8,32,8,128,0.9765625,238.21875,234881024 +42,mlp.up_proj,8,64,8,128,1.1396484375,231.109375,234881024 +42,mlp.up_proj,8,128,8,128,1.28125,227.5546875,234881024 +42,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024 +42,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024 +42,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024 +42,mlp.down_proj,3,32,8,128,34.90625,98.21875,234881024 +42,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024 +42,mlp.down_proj,3,128,8,128,45.84375,87.5546875,234881024 +42,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024 +42,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024 +42,mlp.down_proj,4,128,8,128,21.390625,115.5546875,234881024 +42,mlp.down_proj,8,32,8,128,0.990234375,238.21875,234881024 +42,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024 +42,mlp.down_proj,8,128,8,128,1.2978515625,227.5546875,234881024 +43,self_attn.q_proj,2,32,8,128,44.625,20.0625,67108864 +43,self_attn.q_proj,2,64,8,128,53.125,18.03125,67108864 +43,self_attn.q_proj,2,128,8,128,61.53125,17.015625,67108864 +43,self_attn.q_proj,3,32,8,128,19.171875,28.0625,67108864 +43,self_attn.q_proj,3,64,8,128,22.953125,26.03125,67108864 +43,self_attn.q_proj,3,128,8,128,26.90625,25.015625,67108864 +43,self_attn.q_proj,4,32,8,128,8.9453125,36.0625,67108864 +43,self_attn.q_proj,4,64,8,128,10.7109375,34.03125,67108864 +43,self_attn.q_proj,4,128,8,128,12.5625,33.015625,67108864 +43,self_attn.q_proj,8,32,8,128,0.54150390625,68.0625,67108864 +43,self_attn.q_proj,8,64,8,128,0.6484375,66.03125,67108864 +43,self_attn.q_proj,8,128,8,128,0.75732421875,65.015625,67108864 +43,self_attn.v_proj,2,32,8,128,16.046875,2.5078125,8388608 +43,self_attn.v_proj,2,64,8,128,18.765625,2.25390625,8388608 +43,self_attn.v_proj,2,128,8,128,21.25,2.126953125,8388608 +43,self_attn.v_proj,3,32,8,128,6.86328125,3.5078125,8388608 +43,self_attn.v_proj,3,64,8,128,8.03125,3.25390625,8388608 +43,self_attn.v_proj,3,128,8,128,9.09375,3.126953125,8388608 +43,self_attn.v_proj,4,32,8,128,3.201171875,4.5078125,8388608 +43,self_attn.v_proj,4,64,8,128,3.748046875,4.25390625,8388608 +43,self_attn.v_proj,4,128,8,128,4.24609375,4.126953125,8388608 +43,self_attn.v_proj,8,32,8,128,0.1939697265625,8.5078125,8388608 +43,self_attn.v_proj,8,64,8,128,0.2266845703125,8.25390625,8388608 +43,self_attn.v_proj,8,128,8,128,0.256103515625,8.126953125,8388608 +43,self_attn.k_proj,2,32,8,128,24.953125,2.5078125,8388608 +43,self_attn.k_proj,2,64,8,128,30.046875,2.25390625,8388608 +43,self_attn.k_proj,2,128,8,128,34.3125,2.126953125,8388608 +43,self_attn.k_proj,3,32,8,128,10.765625,3.5078125,8388608 +43,self_attn.k_proj,3,64,8,128,13.1328125,3.25390625,8388608 +43,self_attn.k_proj,3,128,8,128,15.3515625,3.126953125,8388608 +43,self_attn.k_proj,4,32,8,128,5.01953125,4.5078125,8388608 +43,self_attn.k_proj,4,64,8,128,6.12890625,4.25390625,8388608 +43,self_attn.k_proj,4,128,8,128,7.17578125,4.126953125,8388608 +43,self_attn.k_proj,8,32,8,128,0.303955078125,8.5078125,8388608 +43,self_attn.k_proj,8,64,8,128,0.370849609375,8.25390625,8388608 +43,self_attn.k_proj,8,128,8,128,0.43212890625,8.126953125,8388608 +43,self_attn.o_proj,2,32,8,128,41.0,20.0625,67108864 +43,self_attn.o_proj,2,64,8,128,47.8125,18.03125,67108864 +43,self_attn.o_proj,2,128,8,128,54.0,17.015625,67108864 +43,self_attn.o_proj,3,32,8,128,17.53125,28.0625,67108864 +43,self_attn.o_proj,3,64,8,128,20.46875,26.03125,67108864 +43,self_attn.o_proj,3,128,8,128,23.171875,25.015625,67108864 +43,self_attn.o_proj,4,32,8,128,8.1796875,36.0625,67108864 +43,self_attn.o_proj,4,64,8,128,9.5546875,34.03125,67108864 +43,self_attn.o_proj,4,128,8,128,10.8203125,33.015625,67108864 +43,self_attn.o_proj,8,32,8,128,0.495361328125,68.0625,67108864 +43,self_attn.o_proj,8,64,8,128,0.578125,66.03125,67108864 +43,self_attn.o_proj,8,128,8,128,0.65234375,65.015625,67108864 +43,mlp.gate_proj,2,32,8,128,86.6875,70.21875,234881024 +43,mlp.gate_proj,2,64,8,128,101.4375,63.109375,234881024 +43,mlp.gate_proj,2,128,8,128,114.875,59.5546875,234881024 +43,mlp.gate_proj,3,32,8,128,37.09375,98.21875,234881024 +43,mlp.gate_proj,3,64,8,128,43.375,91.109375,234881024 +43,mlp.gate_proj,3,128,8,128,49.21875,87.5546875,234881024 +43,mlp.gate_proj,4,32,8,128,17.296875,126.21875,234881024 +43,mlp.gate_proj,4,64,8,128,20.265625,119.109375,234881024 +43,mlp.gate_proj,4,128,8,128,22.953125,115.5546875,234881024 +43,mlp.gate_proj,8,32,8,128,1.052734375,238.21875,234881024 +43,mlp.gate_proj,8,64,8,128,1.2333984375,231.109375,234881024 +43,mlp.gate_proj,8,128,8,128,1.392578125,227.5546875,234881024 +43,mlp.up_proj,2,32,8,128,80.625,70.21875,234881024 +43,mlp.up_proj,2,64,8,128,93.875,63.109375,234881024 +43,mlp.up_proj,2,128,8,128,105.875,59.5546875,234881024 +43,mlp.up_proj,3,32,8,128,34.46875,98.21875,234881024 +43,mlp.up_proj,3,64,8,128,40.1875,91.109375,234881024 +43,mlp.up_proj,3,128,8,128,45.34375,87.5546875,234881024 +43,mlp.up_proj,4,32,8,128,16.0625,126.21875,234881024 +43,mlp.up_proj,4,64,8,128,18.765625,119.109375,234881024 +43,mlp.up_proj,4,128,8,128,21.15625,115.5546875,234881024 +43,mlp.up_proj,8,32,8,128,0.97705078125,238.21875,234881024 +43,mlp.up_proj,8,64,8,128,1.140625,231.109375,234881024 +43,mlp.up_proj,8,128,8,128,1.283203125,227.5546875,234881024 +43,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024 +43,mlp.down_proj,2,64,8,128,95.125,63.109375,234881024 +43,mlp.down_proj,2,128,8,128,107.25,59.5546875,234881024 +43,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024 +43,mlp.down_proj,3,64,8,128,40.71875,91.109375,234881024 +43,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024 +43,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024 +43,mlp.down_proj,4,64,8,128,19.015625,119.109375,234881024 +43,mlp.down_proj,4,128,8,128,21.421875,115.5546875,234881024 +43,mlp.down_proj,8,32,8,128,0.9912109375,238.21875,234881024 +43,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024 +43,mlp.down_proj,8,128,8,128,1.2998046875,227.5546875,234881024 +44,self_attn.q_proj,2,32,8,128,45.71875,20.0625,67108864 +44,self_attn.q_proj,2,64,8,128,53.8125,18.03125,67108864 +44,self_attn.q_proj,2,128,8,128,62.28125,17.015625,67108864 +44,self_attn.q_proj,3,32,8,128,19.578125,28.0625,67108864 +44,self_attn.q_proj,3,64,8,128,23.0625,26.03125,67108864 +44,self_attn.q_proj,3,128,8,128,26.796875,25.015625,67108864 +44,self_attn.q_proj,4,32,8,128,9.1328125,36.0625,67108864 +44,self_attn.q_proj,4,64,8,128,10.765625,34.03125,67108864 +44,self_attn.q_proj,4,128,8,128,12.5,33.015625,67108864 +44,self_attn.q_proj,8,32,8,128,0.55322265625,68.0625,67108864 +44,self_attn.q_proj,8,64,8,128,0.65185546875,66.03125,67108864 +44,self_attn.q_proj,8,128,8,128,0.75390625,65.015625,67108864 +44,self_attn.v_proj,2,32,8,128,16.65625,2.5078125,8388608 +44,self_attn.v_proj,2,64,8,128,19.484375,2.25390625,8388608 +44,self_attn.v_proj,2,128,8,128,22.046875,2.126953125,8388608 +44,self_attn.v_proj,3,32,8,128,7.125,3.5078125,8388608 +44,self_attn.v_proj,3,64,8,128,8.3359375,3.25390625,8388608 +44,self_attn.v_proj,3,128,8,128,9.4375,3.126953125,8388608 +44,self_attn.v_proj,4,32,8,128,3.32421875,4.5078125,8388608 +44,self_attn.v_proj,4,64,8,128,3.888671875,4.25390625,8388608 +44,self_attn.v_proj,4,128,8,128,4.40234375,4.126953125,8388608 +44,self_attn.v_proj,8,32,8,128,0.201416015625,8.5078125,8388608 +44,self_attn.v_proj,8,64,8,128,0.2353515625,8.25390625,8388608 +44,self_attn.v_proj,8,128,8,128,0.265380859375,8.126953125,8388608 +44,self_attn.k_proj,2,32,8,128,24.78125,2.5078125,8388608 +44,self_attn.k_proj,2,64,8,128,29.78125,2.25390625,8388608 +44,self_attn.k_proj,2,128,8,128,33.59375,2.126953125,8388608 +44,self_attn.k_proj,3,32,8,128,10.765625,3.5078125,8388608 +44,self_attn.k_proj,3,64,8,128,13.28125,3.25390625,8388608 +44,self_attn.k_proj,3,128,8,128,15.53125,3.126953125,8388608 +44,self_attn.k_proj,4,32,8,128,5.0234375,4.5078125,8388608 +44,self_attn.k_proj,4,64,8,128,6.203125,4.25390625,8388608 +44,self_attn.k_proj,4,128,8,128,7.2578125,4.126953125,8388608 +44,self_attn.k_proj,8,32,8,128,0.30419921875,8.5078125,8388608 +44,self_attn.k_proj,8,64,8,128,0.375244140625,8.25390625,8388608 +44,self_attn.k_proj,8,128,8,128,0.4375,8.126953125,8388608 +44,self_attn.o_proj,2,32,8,128,44.03125,20.0625,67108864 +44,self_attn.o_proj,2,64,8,128,51.3125,18.03125,67108864 +44,self_attn.o_proj,2,128,8,128,57.875,17.015625,67108864 +44,self_attn.o_proj,3,32,8,128,18.875,28.0625,67108864 +44,self_attn.o_proj,3,64,8,128,22.046875,26.03125,67108864 +44,self_attn.o_proj,3,128,8,128,24.953125,25.015625,67108864 +44,self_attn.o_proj,4,32,8,128,8.8359375,36.0625,67108864 +44,self_attn.o_proj,4,64,8,128,10.3359375,34.03125,67108864 +44,self_attn.o_proj,4,128,8,128,11.7265625,33.015625,67108864 +44,self_attn.o_proj,8,32,8,128,0.53515625,68.0625,67108864 +44,self_attn.o_proj,8,64,8,128,0.6259765625,66.03125,67108864 +44,self_attn.o_proj,8,128,8,128,0.70849609375,65.015625,67108864 +44,mlp.gate_proj,2,32,8,128,87.8125,70.21875,234881024 +44,mlp.gate_proj,2,64,8,128,102.625,63.109375,234881024 +44,mlp.gate_proj,2,128,8,128,116.0,59.5546875,234881024 +44,mlp.gate_proj,3,32,8,128,37.59375,98.21875,234881024 +44,mlp.gate_proj,3,64,8,128,43.875,91.109375,234881024 +44,mlp.gate_proj,3,128,8,128,49.6875,87.5546875,234881024 +44,mlp.gate_proj,4,32,8,128,17.515625,126.21875,234881024 +44,mlp.gate_proj,4,64,8,128,20.484375,119.109375,234881024 +44,mlp.gate_proj,4,128,8,128,23.171875,115.5546875,234881024 +44,mlp.gate_proj,8,32,8,128,1.06640625,238.21875,234881024 +44,mlp.gate_proj,8,64,8,128,1.2470703125,231.109375,234881024 +44,mlp.gate_proj,8,128,8,128,1.40625,227.5546875,234881024 +44,mlp.up_proj,2,32,8,128,79.8125,70.21875,234881024 +44,mlp.up_proj,2,64,8,128,92.875,63.109375,234881024 +44,mlp.up_proj,2,128,8,128,104.75,59.5546875,234881024 +44,mlp.up_proj,3,32,8,128,34.09375,98.21875,234881024 +44,mlp.up_proj,3,64,8,128,39.78125,91.109375,234881024 +44,mlp.up_proj,3,128,8,128,44.8125,87.5546875,234881024 +44,mlp.up_proj,4,32,8,128,15.90625,126.21875,234881024 +44,mlp.up_proj,4,64,8,128,18.5625,119.109375,234881024 +44,mlp.up_proj,4,128,8,128,20.921875,115.5546875,234881024 +44,mlp.up_proj,8,32,8,128,0.96728515625,238.21875,234881024 +44,mlp.up_proj,8,64,8,128,1.12890625,231.109375,234881024 +44,mlp.up_proj,8,128,8,128,1.26953125,227.5546875,234881024 +44,mlp.down_proj,2,32,8,128,81.3125,70.21875,234881024 +44,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024 +44,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024 +44,mlp.down_proj,3,32,8,128,34.75,98.21875,234881024 +44,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024 +44,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024 +44,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024 +44,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024 +44,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +44,mlp.down_proj,8,32,8,128,0.98583984375,238.21875,234881024 +44,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +44,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +45,self_attn.q_proj,2,32,8,128,40.46875,20.0625,67108864 +45,self_attn.q_proj,2,64,8,128,47.71875,18.03125,67108864 +45,self_attn.q_proj,2,128,8,128,55.40625,17.015625,67108864 +45,self_attn.q_proj,3,32,8,128,17.328125,28.0625,67108864 +45,self_attn.q_proj,3,64,8,128,20.484375,26.03125,67108864 +45,self_attn.q_proj,3,128,8,128,23.890625,25.015625,67108864 +45,self_attn.q_proj,4,32,8,128,8.078125,36.0625,67108864 +45,self_attn.q_proj,4,64,8,128,9.5546875,34.03125,67108864 +45,self_attn.q_proj,4,128,8,128,11.140625,33.015625,67108864 +45,self_attn.q_proj,8,32,8,128,0.4892578125,68.0625,67108864 +45,self_attn.q_proj,8,64,8,128,0.57861328125,66.03125,67108864 +45,self_attn.q_proj,8,128,8,128,0.671875,65.015625,67108864 +45,self_attn.v_proj,2,32,8,128,17.234375,2.5078125,8388608 +45,self_attn.v_proj,2,64,8,128,20.140625,2.25390625,8388608 +45,self_attn.v_proj,2,128,8,128,22.78125,2.126953125,8388608 +45,self_attn.v_proj,3,32,8,128,7.37109375,3.5078125,8388608 +45,self_attn.v_proj,3,64,8,128,8.6171875,3.25390625,8388608 +45,self_attn.v_proj,3,128,8,128,9.75,3.126953125,8388608 +45,self_attn.v_proj,4,32,8,128,3.4375,4.5078125,8388608 +45,self_attn.v_proj,4,64,8,128,4.01953125,4.25390625,8388608 +45,self_attn.v_proj,4,128,8,128,4.55078125,4.126953125,8388608 +45,self_attn.v_proj,8,32,8,128,0.208251953125,8.5078125,8388608 +45,self_attn.v_proj,8,64,8,128,0.243408203125,8.25390625,8388608 +45,self_attn.v_proj,8,128,8,128,0.2744140625,8.126953125,8388608 +45,self_attn.k_proj,2,32,8,128,23.234375,2.5078125,8388608 +45,self_attn.k_proj,2,64,8,128,28.171875,2.25390625,8388608 +45,self_attn.k_proj,2,128,8,128,32.5,2.126953125,8388608 +45,self_attn.k_proj,3,32,8,128,10.015625,3.5078125,8388608 +45,self_attn.k_proj,3,64,8,128,12.3203125,3.25390625,8388608 +45,self_attn.k_proj,3,128,8,128,14.546875,3.126953125,8388608 +45,self_attn.k_proj,4,32,8,128,4.671875,4.5078125,8388608 +45,self_attn.k_proj,4,64,8,128,5.75390625,4.25390625,8388608 +45,self_attn.k_proj,4,128,8,128,6.796875,4.126953125,8388608 +45,self_attn.k_proj,8,32,8,128,0.282958984375,8.5078125,8388608 +45,self_attn.k_proj,8,64,8,128,0.34814453125,8.25390625,8388608 +45,self_attn.k_proj,8,128,8,128,0.40966796875,8.126953125,8388608 +45,self_attn.o_proj,2,32,8,128,42.21875,20.0625,67108864 +45,self_attn.o_proj,2,64,8,128,49.25,18.03125,67108864 +45,self_attn.o_proj,2,128,8,128,55.65625,17.015625,67108864 +45,self_attn.o_proj,3,32,8,128,18.109375,28.0625,67108864 +45,self_attn.o_proj,3,64,8,128,21.171875,26.03125,67108864 +45,self_attn.o_proj,3,128,8,128,24.03125,25.015625,67108864 +45,self_attn.o_proj,4,32,8,128,8.4609375,36.0625,67108864 +45,self_attn.o_proj,4,64,8,128,9.90625,34.03125,67108864 +45,self_attn.o_proj,4,128,8,128,11.25,33.015625,67108864 +45,self_attn.o_proj,8,32,8,128,0.51220703125,68.0625,67108864 +45,self_attn.o_proj,8,64,8,128,0.59912109375,66.03125,67108864 +45,self_attn.o_proj,8,128,8,128,0.67822265625,65.015625,67108864 +45,mlp.gate_proj,2,32,8,128,88.3125,70.21875,234881024 +45,mlp.gate_proj,2,64,8,128,103.125,63.109375,234881024 +45,mlp.gate_proj,2,128,8,128,116.5,59.5546875,234881024 +45,mlp.gate_proj,3,32,8,128,37.8125,98.21875,234881024 +45,mlp.gate_proj,3,64,8,128,44.09375,91.109375,234881024 +45,mlp.gate_proj,3,128,8,128,49.90625,87.5546875,234881024 +45,mlp.gate_proj,4,32,8,128,17.625,126.21875,234881024 +45,mlp.gate_proj,4,64,8,128,20.59375,119.109375,234881024 +45,mlp.gate_proj,4,128,8,128,23.28125,115.5546875,234881024 +45,mlp.gate_proj,8,32,8,128,1.072265625,238.21875,234881024 +45,mlp.gate_proj,8,64,8,128,1.2529296875,231.109375,234881024 +45,mlp.gate_proj,8,128,8,128,1.412109375,227.5546875,234881024 +45,mlp.up_proj,2,32,8,128,79.75,70.21875,234881024 +45,mlp.up_proj,2,64,8,128,92.8125,63.109375,234881024 +45,mlp.up_proj,2,128,8,128,104.6875,59.5546875,234881024 +45,mlp.up_proj,3,32,8,128,34.09375,98.21875,234881024 +45,mlp.up_proj,3,64,8,128,39.75,91.109375,234881024 +45,mlp.up_proj,3,128,8,128,44.78125,87.5546875,234881024 +45,mlp.up_proj,4,32,8,128,15.8984375,126.21875,234881024 +45,mlp.up_proj,4,64,8,128,18.546875,119.109375,234881024 +45,mlp.up_proj,4,128,8,128,20.890625,115.5546875,234881024 +45,mlp.up_proj,8,32,8,128,0.966796875,238.21875,234881024 +45,mlp.up_proj,8,64,8,128,1.1279296875,231.109375,234881024 +45,mlp.up_proj,8,128,8,128,1.2685546875,227.5546875,234881024 +45,mlp.down_proj,2,32,8,128,81.1875,70.21875,234881024 +45,mlp.down_proj,2,64,8,128,94.5,63.109375,234881024 +45,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024 +45,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024 +45,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024 +45,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024 +45,mlp.down_proj,4,32,8,128,16.1875,126.21875,234881024 +45,mlp.down_proj,4,64,8,128,18.875,119.109375,234881024 +45,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024 +45,mlp.down_proj,8,32,8,128,0.984375,238.21875,234881024 +45,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024 +45,mlp.down_proj,8,128,8,128,1.2890625,227.5546875,234881024 +46,self_attn.q_proj,2,32,8,128,39.03125,20.0625,67108864 +46,self_attn.q_proj,2,64,8,128,45.9375,18.03125,67108864 +46,self_attn.q_proj,2,128,8,128,53.15625,17.015625,67108864 +46,self_attn.q_proj,3,32,8,128,16.703125,28.0625,67108864 +46,self_attn.q_proj,3,64,8,128,19.703125,26.03125,67108864 +46,self_attn.q_proj,3,128,8,128,22.859375,25.015625,67108864 +46,self_attn.q_proj,4,32,8,128,7.79296875,36.0625,67108864 +46,self_attn.q_proj,4,64,8,128,9.1953125,34.03125,67108864 +46,self_attn.q_proj,4,128,8,128,10.6640625,33.015625,67108864 +46,self_attn.q_proj,8,32,8,128,0.471923828125,68.0625,67108864 +46,self_attn.q_proj,8,64,8,128,0.55615234375,66.03125,67108864 +46,self_attn.q_proj,8,128,8,128,0.64306640625,65.015625,67108864 +46,self_attn.v_proj,2,32,8,128,17.0,2.5078125,8388608 +46,self_attn.v_proj,2,64,8,128,19.796875,2.25390625,8388608 +46,self_attn.v_proj,2,128,8,128,22.3125,2.126953125,8388608 +46,self_attn.v_proj,3,32,8,128,7.2734375,3.5078125,8388608 +46,self_attn.v_proj,3,64,8,128,8.46875,3.25390625,8388608 +46,self_attn.v_proj,3,128,8,128,9.546875,3.126953125,8388608 +46,self_attn.v_proj,4,32,8,128,3.392578125,4.5078125,8388608 +46,self_attn.v_proj,4,64,8,128,3.951171875,4.25390625,8388608 +46,self_attn.v_proj,4,128,8,128,4.45703125,4.126953125,8388608 +46,self_attn.v_proj,8,32,8,128,0.2054443359375,8.5078125,8388608 +46,self_attn.v_proj,8,64,8,128,0.2391357421875,8.25390625,8388608 +46,self_attn.v_proj,8,128,8,128,0.2685546875,8.126953125,8388608 +46,self_attn.k_proj,2,32,8,128,22.765625,2.5078125,8388608 +46,self_attn.k_proj,2,64,8,128,27.484375,2.25390625,8388608 +46,self_attn.k_proj,2,128,8,128,31.296875,2.126953125,8388608 +46,self_attn.k_proj,3,32,8,128,9.828125,3.5078125,8388608 +46,self_attn.k_proj,3,64,8,128,12.0390625,3.25390625,8388608 +46,self_attn.k_proj,3,128,8,128,14.078125,3.126953125,8388608 +46,self_attn.k_proj,4,32,8,128,4.58203125,4.5078125,8388608 +46,self_attn.k_proj,4,64,8,128,5.6171875,4.25390625,8388608 +46,self_attn.k_proj,4,128,8,128,6.57421875,4.126953125,8388608 +46,self_attn.k_proj,8,32,8,128,0.277587890625,8.5078125,8388608 +46,self_attn.k_proj,8,64,8,128,0.33984375,8.25390625,8388608 +46,self_attn.k_proj,8,128,8,128,0.396240234375,8.126953125,8388608 +46,self_attn.o_proj,2,32,8,128,42.3125,20.0625,67108864 +46,self_attn.o_proj,2,64,8,128,49.28125,18.03125,67108864 +46,self_attn.o_proj,2,128,8,128,55.59375,17.015625,67108864 +46,self_attn.o_proj,3,32,8,128,18.125,28.0625,67108864 +46,self_attn.o_proj,3,64,8,128,21.140625,26.03125,67108864 +46,self_attn.o_proj,3,128,8,128,23.90625,25.015625,67108864 +46,self_attn.o_proj,4,32,8,128,8.4609375,36.0625,67108864 +46,self_attn.o_proj,4,64,8,128,9.8828125,34.03125,67108864 +46,self_attn.o_proj,4,128,8,128,11.1875,33.015625,67108864 +46,self_attn.o_proj,8,32,8,128,0.51220703125,68.0625,67108864 +46,self_attn.o_proj,8,64,8,128,0.59765625,66.03125,67108864 +46,self_attn.o_proj,8,128,8,128,0.6748046875,65.015625,67108864 +46,mlp.gate_proj,2,32,8,128,88.9375,70.21875,234881024 +46,mlp.gate_proj,2,64,8,128,103.875,63.109375,234881024 +46,mlp.gate_proj,2,128,8,128,117.3125,59.5546875,234881024 +46,mlp.gate_proj,3,32,8,128,38.0625,98.21875,234881024 +46,mlp.gate_proj,3,64,8,128,44.40625,91.109375,234881024 +46,mlp.gate_proj,3,128,8,128,50.25,87.5546875,234881024 +46,mlp.gate_proj,4,32,8,128,17.75,126.21875,234881024 +46,mlp.gate_proj,4,64,8,128,20.71875,119.109375,234881024 +46,mlp.gate_proj,4,128,8,128,23.453125,115.5546875,234881024 +46,mlp.gate_proj,8,32,8,128,1.080078125,238.21875,234881024 +46,mlp.gate_proj,8,64,8,128,1.26171875,231.109375,234881024 +46,mlp.gate_proj,8,128,8,128,1.421875,227.5546875,234881024 +46,mlp.up_proj,2,32,8,128,79.6875,70.21875,234881024 +46,mlp.up_proj,2,64,8,128,92.6875,63.109375,234881024 +46,mlp.up_proj,2,128,8,128,104.5,59.5546875,234881024 +46,mlp.up_proj,3,32,8,128,34.03125,98.21875,234881024 +46,mlp.up_proj,3,64,8,128,39.6875,91.109375,234881024 +46,mlp.up_proj,3,128,8,128,44.6875,87.5546875,234881024 +46,mlp.up_proj,4,32,8,128,15.875,126.21875,234881024 +46,mlp.up_proj,4,64,8,128,18.515625,119.109375,234881024 +46,mlp.up_proj,4,128,8,128,20.859375,115.5546875,234881024 +46,mlp.up_proj,8,32,8,128,0.96533203125,238.21875,234881024 +46,mlp.up_proj,8,64,8,128,1.1259765625,231.109375,234881024 +46,mlp.up_proj,8,128,8,128,1.265625,227.5546875,234881024 +46,mlp.down_proj,2,32,8,128,81.0,70.21875,234881024 +46,mlp.down_proj,2,64,8,128,94.25,63.109375,234881024 +46,mlp.down_proj,2,128,8,128,106.1875,59.5546875,234881024 +46,mlp.down_proj,3,32,8,128,34.625,98.21875,234881024 +46,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024 +46,mlp.down_proj,3,128,8,128,45.4375,87.5546875,234881024 +46,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024 +46,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024 +46,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024 +46,mlp.down_proj,8,32,8,128,0.98193359375,238.21875,234881024 +46,mlp.down_proj,8,64,8,128,1.14453125,231.109375,234881024 +46,mlp.down_proj,8,128,8,128,1.2861328125,227.5546875,234881024 +47,self_attn.q_proj,2,32,8,128,44.0625,20.0625,67108864 +47,self_attn.q_proj,2,64,8,128,52.0,18.03125,67108864 +47,self_attn.q_proj,2,128,8,128,59.84375,17.015625,67108864 +47,self_attn.q_proj,3,32,8,128,18.875,28.0625,67108864 +47,self_attn.q_proj,3,64,8,128,22.328125,26.03125,67108864 +47,self_attn.q_proj,3,128,8,128,25.828125,25.015625,67108864 +47,self_attn.q_proj,4,32,8,128,8.8046875,36.0625,67108864 +47,self_attn.q_proj,4,64,8,128,10.421875,34.03125,67108864 +47,self_attn.q_proj,4,128,8,128,12.0546875,33.015625,67108864 +47,self_attn.q_proj,8,32,8,128,0.533203125,68.0625,67108864 +47,self_attn.q_proj,8,64,8,128,0.63037109375,66.03125,67108864 +47,self_attn.q_proj,8,128,8,128,0.72705078125,65.015625,67108864 +47,self_attn.v_proj,2,32,8,128,16.75,2.5078125,8388608 +47,self_attn.v_proj,2,64,8,128,19.625,2.25390625,8388608 +47,self_attn.v_proj,2,128,8,128,22.234375,2.126953125,8388608 +47,self_attn.v_proj,3,32,8,128,7.1640625,3.5078125,8388608 +47,self_attn.v_proj,3,64,8,128,8.390625,3.25390625,8388608 +47,self_attn.v_proj,3,128,8,128,9.5078125,3.126953125,8388608 +47,self_attn.v_proj,4,32,8,128,3.341796875,4.5078125,8388608 +47,self_attn.v_proj,4,64,8,128,3.9140625,4.25390625,8388608 +47,self_attn.v_proj,4,128,8,128,4.44140625,4.126953125,8388608 +47,self_attn.v_proj,8,32,8,128,0.202392578125,8.5078125,8388608 +47,self_attn.v_proj,8,64,8,128,0.2369384765625,8.25390625,8388608 +47,self_attn.v_proj,8,128,8,128,0.267578125,8.126953125,8388608 +47,self_attn.k_proj,2,32,8,128,25.765625,2.5078125,8388608 +47,self_attn.k_proj,2,64,8,128,31.03125,2.25390625,8388608 +47,self_attn.k_proj,2,128,8,128,35.4375,2.126953125,8388608 +47,self_attn.k_proj,3,32,8,128,11.109375,3.5078125,8388608 +47,self_attn.k_proj,3,64,8,128,13.546875,3.25390625,8388608 +47,self_attn.k_proj,3,128,8,128,15.8203125,3.126953125,8388608 +47,self_attn.k_proj,4,32,8,128,5.18359375,4.5078125,8388608 +47,self_attn.k_proj,4,64,8,128,6.3203125,4.25390625,8388608 +47,self_attn.k_proj,4,128,8,128,7.390625,4.126953125,8388608 +47,self_attn.k_proj,8,32,8,128,0.313720703125,8.5078125,8388608 +47,self_attn.k_proj,8,64,8,128,0.38232421875,8.25390625,8388608 +47,self_attn.k_proj,8,128,8,128,0.4453125,8.126953125,8388608 +47,self_attn.o_proj,2,32,8,128,42.0,20.0625,67108864 +47,self_attn.o_proj,2,64,8,128,48.96875,18.03125,67108864 +47,self_attn.o_proj,2,128,8,128,55.34375,17.015625,67108864 +47,self_attn.o_proj,3,32,8,128,18.015625,28.0625,67108864 +47,self_attn.o_proj,3,64,8,128,21.078125,26.03125,67108864 +47,self_attn.o_proj,3,128,8,128,23.9375,25.015625,67108864 +47,self_attn.o_proj,4,32,8,128,8.4453125,36.0625,67108864 +47,self_attn.o_proj,4,64,8,128,9.90625,34.03125,67108864 +47,self_attn.o_proj,4,128,8,128,11.296875,33.015625,67108864 +47,self_attn.o_proj,8,32,8,128,0.51416015625,68.0625,67108864 +47,self_attn.o_proj,8,64,8,128,0.60546875,66.03125,67108864 +47,self_attn.o_proj,8,128,8,128,0.69287109375,65.015625,67108864 +47,mlp.gate_proj,2,32,8,128,89.375,70.21875,234881024 +47,mlp.gate_proj,2,64,8,128,104.375,63.109375,234881024 +47,mlp.gate_proj,2,128,8,128,117.875,59.5546875,234881024 +47,mlp.gate_proj,3,32,8,128,38.25,98.21875,234881024 +47,mlp.gate_proj,3,64,8,128,44.65625,91.109375,234881024 +47,mlp.gate_proj,3,128,8,128,50.46875,87.5546875,234881024 +47,mlp.gate_proj,4,32,8,128,17.84375,126.21875,234881024 +47,mlp.gate_proj,4,64,8,128,20.828125,119.109375,234881024 +47,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024 +47,mlp.gate_proj,8,32,8,128,1.0859375,238.21875,234881024 +47,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024 +47,mlp.gate_proj,8,128,8,128,1.4287109375,227.5546875,234881024 +47,mlp.up_proj,2,32,8,128,79.5,70.21875,234881024 +47,mlp.up_proj,2,64,8,128,92.5,63.109375,234881024 +47,mlp.up_proj,2,128,8,128,104.25,59.5546875,234881024 +47,mlp.up_proj,3,32,8,128,33.96875,98.21875,234881024 +47,mlp.up_proj,3,64,8,128,39.59375,91.109375,234881024 +47,mlp.up_proj,3,128,8,128,44.59375,87.5546875,234881024 +47,mlp.up_proj,4,32,8,128,15.8359375,126.21875,234881024 +47,mlp.up_proj,4,64,8,128,18.46875,119.109375,234881024 +47,mlp.up_proj,4,128,8,128,20.828125,115.5546875,234881024 +47,mlp.up_proj,8,32,8,128,0.962890625,238.21875,234881024 +47,mlp.up_proj,8,64,8,128,1.123046875,231.109375,234881024 +47,mlp.up_proj,8,128,8,128,1.263671875,227.5546875,234881024 +47,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024 +47,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024 +47,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024 +47,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024 +47,mlp.down_proj,3,64,8,128,40.28125,91.109375,234881024 +47,mlp.down_proj,3,128,8,128,45.34375,87.5546875,234881024 +47,mlp.down_proj,4,32,8,128,16.125,126.21875,234881024 +47,mlp.down_proj,4,64,8,128,18.796875,119.109375,234881024 +47,mlp.down_proj,4,128,8,128,21.171875,115.5546875,234881024 +47,mlp.down_proj,8,32,8,128,0.98046875,238.21875,234881024 +47,mlp.down_proj,8,64,8,128,1.1435546875,231.109375,234881024 +47,mlp.down_proj,8,128,8,128,1.2841796875,227.5546875,234881024 +48,self_attn.q_proj,2,32,8,128,37.4375,20.0625,67108864 +48,self_attn.q_proj,2,64,8,128,44.21875,18.03125,67108864 +48,self_attn.q_proj,2,128,8,128,51.0,17.015625,67108864 +48,self_attn.q_proj,3,32,8,128,16.03125,28.0625,67108864 +48,self_attn.q_proj,3,64,8,128,19.0,26.03125,67108864 +48,self_attn.q_proj,3,128,8,128,22.0625,25.015625,67108864 +48,self_attn.q_proj,4,32,8,128,7.484375,36.0625,67108864 +48,self_attn.q_proj,4,64,8,128,8.8671875,34.03125,67108864 +48,self_attn.q_proj,4,128,8,128,10.3046875,33.015625,67108864 +48,self_attn.q_proj,8,32,8,128,0.453125,68.0625,67108864 +48,self_attn.q_proj,8,64,8,128,0.53662109375,66.03125,67108864 +48,self_attn.q_proj,8,128,8,128,0.62109375,65.015625,67108864 +48,self_attn.v_proj,2,32,8,128,17.5,2.5078125,8388608 +48,self_attn.v_proj,2,64,8,128,20.453125,2.25390625,8388608 +48,self_attn.v_proj,2,128,8,128,23.125,2.126953125,8388608 +48,self_attn.v_proj,3,32,8,128,7.484375,3.5078125,8388608 +48,self_attn.v_proj,3,64,8,128,8.75,3.25390625,8388608 +48,self_attn.v_proj,3,128,8,128,9.90625,3.126953125,8388608 +48,self_attn.v_proj,4,32,8,128,3.4921875,4.5078125,8388608 +48,self_attn.v_proj,4,64,8,128,4.08203125,4.25390625,8388608 +48,self_attn.v_proj,4,128,8,128,4.62109375,4.126953125,8388608 +48,self_attn.v_proj,8,32,8,128,0.21142578125,8.5078125,8388608 +48,self_attn.v_proj,8,64,8,128,0.2470703125,8.25390625,8388608 +48,self_attn.v_proj,8,128,8,128,0.278564453125,8.126953125,8388608 +48,self_attn.k_proj,2,32,8,128,22.171875,2.5078125,8388608 +48,self_attn.k_proj,2,64,8,128,26.515625,2.25390625,8388608 +48,self_attn.k_proj,2,128,8,128,29.859375,2.126953125,8388608 +48,self_attn.k_proj,3,32,8,128,9.71875,3.5078125,8388608 +48,self_attn.k_proj,3,64,8,128,12.0,3.25390625,8388608 +48,self_attn.k_proj,3,128,8,128,14.1328125,3.126953125,8388608 +48,self_attn.k_proj,4,32,8,128,4.53515625,4.5078125,8388608 +48,self_attn.k_proj,4,64,8,128,5.60546875,4.25390625,8388608 +48,self_attn.k_proj,4,128,8,128,6.609375,4.126953125,8388608 +48,self_attn.k_proj,8,32,8,128,0.2744140625,8.5078125,8388608 +48,self_attn.k_proj,8,64,8,128,0.3388671875,8.25390625,8388608 +48,self_attn.k_proj,8,128,8,128,0.3984375,8.126953125,8388608 +48,self_attn.o_proj,2,32,8,128,42.09375,20.0625,67108864 +48,self_attn.o_proj,2,64,8,128,49.03125,18.03125,67108864 +48,self_attn.o_proj,2,128,8,128,55.25,17.015625,67108864 +48,self_attn.o_proj,3,32,8,128,18.015625,28.0625,67108864 +48,self_attn.o_proj,3,64,8,128,21.03125,26.03125,67108864 +48,self_attn.o_proj,3,128,8,128,23.75,25.015625,67108864 +48,self_attn.o_proj,4,32,8,128,8.4140625,36.0625,67108864 +48,self_attn.o_proj,4,64,8,128,9.8203125,34.03125,67108864 +48,self_attn.o_proj,4,128,8,128,11.1171875,33.015625,67108864 +48,self_attn.o_proj,8,32,8,128,0.509765625,68.0625,67108864 +48,self_attn.o_proj,8,64,8,128,0.5947265625,66.03125,67108864 +48,self_attn.o_proj,8,128,8,128,0.6708984375,65.015625,67108864 +48,mlp.gate_proj,2,32,8,128,89.5625,70.21875,234881024 +48,mlp.gate_proj,2,64,8,128,104.5625,63.109375,234881024 +48,mlp.gate_proj,2,128,8,128,118.0625,59.5546875,234881024 +48,mlp.gate_proj,3,32,8,128,38.34375,98.21875,234881024 +48,mlp.gate_proj,3,64,8,128,44.71875,91.109375,234881024 +48,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024 +48,mlp.gate_proj,4,32,8,128,17.875,126.21875,234881024 +48,mlp.gate_proj,4,64,8,128,20.875,119.109375,234881024 +48,mlp.gate_proj,4,128,8,128,23.59375,115.5546875,234881024 +48,mlp.gate_proj,8,32,8,128,1.087890625,238.21875,234881024 +48,mlp.gate_proj,8,64,8,128,1.2705078125,231.109375,234881024 +48,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +48,mlp.up_proj,2,32,8,128,79.5625,70.21875,234881024 +48,mlp.up_proj,2,64,8,128,92.5625,63.109375,234881024 +48,mlp.up_proj,2,128,8,128,104.3125,59.5546875,234881024 +48,mlp.up_proj,3,32,8,128,34.0,98.21875,234881024 +48,mlp.up_proj,3,64,8,128,39.625,91.109375,234881024 +48,mlp.up_proj,3,128,8,128,44.65625,87.5546875,234881024 +48,mlp.up_proj,4,32,8,128,15.8515625,126.21875,234881024 +48,mlp.up_proj,4,64,8,128,18.484375,119.109375,234881024 +48,mlp.up_proj,4,128,8,128,20.84375,115.5546875,234881024 +48,mlp.up_proj,8,32,8,128,0.9638671875,238.21875,234881024 +48,mlp.up_proj,8,64,8,128,1.1240234375,231.109375,234881024 +48,mlp.up_proj,8,128,8,128,1.2646484375,227.5546875,234881024 +48,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024 +48,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024 +48,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024 +48,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024 +48,mlp.down_proj,3,64,8,128,40.25,91.109375,234881024 +48,mlp.down_proj,3,128,8,128,45.3125,87.5546875,234881024 +48,mlp.down_proj,4,32,8,128,16.109375,126.21875,234881024 +48,mlp.down_proj,4,64,8,128,18.78125,119.109375,234881024 +48,mlp.down_proj,4,128,8,128,21.15625,115.5546875,234881024 +48,mlp.down_proj,8,32,8,128,0.97998046875,238.21875,234881024 +48,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024 +48,mlp.down_proj,8,128,8,128,1.283203125,227.5546875,234881024 +49,self_attn.q_proj,2,32,8,128,34.875,20.0625,67108864 +49,self_attn.q_proj,2,64,8,128,41.625,18.03125,67108864 +49,self_attn.q_proj,2,128,8,128,48.09375,17.015625,67108864 +49,self_attn.q_proj,3,32,8,128,15.0078125,28.0625,67108864 +49,self_attn.q_proj,3,64,8,128,18.0625,26.03125,67108864 +49,self_attn.q_proj,3,128,8,128,21.140625,25.015625,67108864 +49,self_attn.q_proj,4,32,8,128,7.00390625,36.0625,67108864 +49,self_attn.q_proj,4,64,8,128,8.4296875,34.03125,67108864 +49,self_attn.q_proj,4,128,8,128,9.875,33.015625,67108864 +49,self_attn.q_proj,8,32,8,128,0.423828125,68.0625,67108864 +49,self_attn.q_proj,8,64,8,128,0.51025390625,66.03125,67108864 +49,self_attn.q_proj,8,128,8,128,0.595703125,65.015625,67108864 +49,self_attn.v_proj,2,32,8,128,17.171875,2.5078125,8388608 +49,self_attn.v_proj,2,64,8,128,20.0625,2.25390625,8388608 +49,self_attn.v_proj,2,128,8,128,22.6875,2.126953125,8388608 +49,self_attn.v_proj,3,32,8,128,7.33984375,3.5078125,8388608 +49,self_attn.v_proj,3,64,8,128,8.578125,3.25390625,8388608 +49,self_attn.v_proj,3,128,8,128,9.703125,3.126953125,8388608 +49,self_attn.v_proj,4,32,8,128,3.42578125,4.5078125,8388608 +49,self_attn.v_proj,4,64,8,128,4.00390625,4.25390625,8388608 +49,self_attn.v_proj,4,128,8,128,4.52734375,4.126953125,8388608 +49,self_attn.v_proj,8,32,8,128,0.20751953125,8.5078125,8388608 +49,self_attn.v_proj,8,64,8,128,0.2421875,8.25390625,8388608 +49,self_attn.v_proj,8,128,8,128,0.27294921875,8.126953125,8388608 +49,self_attn.k_proj,2,32,8,128,21.328125,2.5078125,8388608 +49,self_attn.k_proj,2,64,8,128,25.625,2.25390625,8388608 +49,self_attn.k_proj,2,128,8,128,29.171875,2.126953125,8388608 +49,self_attn.k_proj,3,32,8,128,9.28125,3.5078125,8388608 +49,self_attn.k_proj,3,64,8,128,11.40625,3.25390625,8388608 +49,self_attn.k_proj,3,128,8,128,13.4296875,3.126953125,8388608 +49,self_attn.k_proj,4,32,8,128,4.33203125,4.5078125,8388608 +49,self_attn.k_proj,4,64,8,128,5.32421875,4.25390625,8388608 +49,self_attn.k_proj,4,128,8,128,6.28125,4.126953125,8388608 +49,self_attn.k_proj,8,32,8,128,0.262451171875,8.5078125,8388608 +49,self_attn.k_proj,8,64,8,128,0.322021484375,8.25390625,8388608 +49,self_attn.k_proj,8,128,8,128,0.37841796875,8.126953125,8388608 +49,self_attn.o_proj,2,32,8,128,40.0,20.0625,67108864 +49,self_attn.o_proj,2,64,8,128,46.65625,18.03125,67108864 +49,self_attn.o_proj,2,128,8,128,52.75,17.015625,67108864 +49,self_attn.o_proj,3,32,8,128,17.140625,28.0625,67108864 +49,self_attn.o_proj,3,64,8,128,20.046875,26.03125,67108864 +49,self_attn.o_proj,3,128,8,128,22.734375,25.015625,67108864 +49,self_attn.o_proj,4,32,8,128,8.0,36.0625,67108864 +49,self_attn.o_proj,4,64,8,128,9.359375,34.03125,67108864 +49,self_attn.o_proj,4,128,8,128,10.6328125,33.015625,67108864 +49,self_attn.o_proj,8,32,8,128,0.484375,68.0625,67108864 +49,self_attn.o_proj,8,64,8,128,0.56640625,66.03125,67108864 +49,self_attn.o_proj,8,128,8,128,0.6416015625,65.015625,67108864 +49,mlp.gate_proj,2,32,8,128,89.6875,70.21875,234881024 +49,mlp.gate_proj,2,64,8,128,104.6875,63.109375,234881024 +49,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024 +49,mlp.gate_proj,3,32,8,128,38.375,98.21875,234881024 +49,mlp.gate_proj,3,64,8,128,44.78125,91.109375,234881024 +49,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024 +49,mlp.gate_proj,4,32,8,128,17.890625,126.21875,234881024 +49,mlp.gate_proj,4,64,8,128,20.890625,119.109375,234881024 +49,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +49,mlp.gate_proj,8,32,8,128,1.0888671875,238.21875,234881024 +49,mlp.gate_proj,8,64,8,128,1.271484375,231.109375,234881024 +49,mlp.gate_proj,8,128,8,128,1.4326171875,227.5546875,234881024 +49,mlp.up_proj,2,32,8,128,79.6875,70.21875,234881024 +49,mlp.up_proj,2,64,8,128,92.75,63.109375,234881024 +49,mlp.up_proj,2,128,8,128,104.5625,59.5546875,234881024 +49,mlp.up_proj,3,32,8,128,34.0625,98.21875,234881024 +49,mlp.up_proj,3,64,8,128,39.71875,91.109375,234881024 +49,mlp.up_proj,3,128,8,128,44.75,87.5546875,234881024 +49,mlp.up_proj,4,32,8,128,15.8828125,126.21875,234881024 +49,mlp.up_proj,4,64,8,128,18.515625,119.109375,234881024 +49,mlp.up_proj,4,128,8,128,20.875,115.5546875,234881024 +49,mlp.up_proj,8,32,8,128,0.9658203125,238.21875,234881024 +49,mlp.up_proj,8,64,8,128,1.126953125,231.109375,234881024 +49,mlp.up_proj,8,128,8,128,1.2666015625,227.5546875,234881024 +49,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024 +49,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024 +49,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024 +49,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024 +49,mlp.down_proj,3,64,8,128,40.28125,91.109375,234881024 +49,mlp.down_proj,3,128,8,128,45.34375,87.5546875,234881024 +49,mlp.down_proj,4,32,8,128,16.125,126.21875,234881024 +49,mlp.down_proj,4,64,8,128,18.796875,119.109375,234881024 +49,mlp.down_proj,4,128,8,128,21.15625,115.5546875,234881024 +49,mlp.down_proj,8,32,8,128,0.98046875,238.21875,234881024 +49,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024 +49,mlp.down_proj,8,128,8,128,1.2841796875,227.5546875,234881024 +50,self_attn.q_proj,2,32,8,128,34.96875,20.0625,67108864 +50,self_attn.q_proj,2,64,8,128,40.96875,18.03125,67108864 +50,self_attn.q_proj,2,128,8,128,47.53125,17.015625,67108864 +50,self_attn.q_proj,3,32,8,128,14.9921875,28.0625,67108864 +50,self_attn.q_proj,3,64,8,128,17.640625,26.03125,67108864 +50,self_attn.q_proj,3,128,8,128,20.609375,25.015625,67108864 +50,self_attn.q_proj,4,32,8,128,6.99609375,36.0625,67108864 +50,self_attn.q_proj,4,64,8,128,8.234375,34.03125,67108864 +50,self_attn.q_proj,4,128,8,128,9.6171875,33.015625,67108864 +50,self_attn.q_proj,8,32,8,128,0.42333984375,68.0625,67108864 +50,self_attn.q_proj,8,64,8,128,0.498046875,66.03125,67108864 +50,self_attn.q_proj,8,128,8,128,0.580078125,65.015625,67108864 +50,self_attn.v_proj,2,32,8,128,17.59375,2.5078125,8388608 +50,self_attn.v_proj,2,64,8,128,20.5625,2.25390625,8388608 +50,self_attn.v_proj,2,128,8,128,23.234375,2.126953125,8388608 +50,self_attn.v_proj,3,32,8,128,7.51953125,3.5078125,8388608 +50,self_attn.v_proj,3,64,8,128,8.7890625,3.25390625,8388608 +50,self_attn.v_proj,3,128,8,128,9.9375,3.126953125,8388608 +50,self_attn.v_proj,4,32,8,128,3.5078125,4.5078125,8388608 +50,self_attn.v_proj,4,64,8,128,4.1015625,4.25390625,8388608 +50,self_attn.v_proj,4,128,8,128,4.63671875,4.126953125,8388608 +50,self_attn.v_proj,8,32,8,128,0.2125244140625,8.5078125,8388608 +50,self_attn.v_proj,8,64,8,128,0.2481689453125,8.25390625,8388608 +50,self_attn.v_proj,8,128,8,128,0.279541015625,8.126953125,8388608 +50,self_attn.k_proj,2,32,8,128,20.46875,2.5078125,8388608 +50,self_attn.k_proj,2,64,8,128,24.515625,2.25390625,8388608 +50,self_attn.k_proj,2,128,8,128,27.734375,2.126953125,8388608 +50,self_attn.k_proj,3,32,8,128,8.9609375,3.5078125,8388608 +50,self_attn.k_proj,3,64,8,128,11.0625,3.25390625,8388608 +50,self_attn.k_proj,3,128,8,128,13.0078125,3.126953125,8388608 +50,self_attn.k_proj,4,32,8,128,4.18359375,4.5078125,8388608 +50,self_attn.k_proj,4,64,8,128,5.1796875,4.25390625,8388608 +50,self_attn.k_proj,4,128,8,128,6.109375,4.126953125,8388608 +50,self_attn.k_proj,8,32,8,128,0.253173828125,8.5078125,8388608 +50,self_attn.k_proj,8,64,8,128,0.31298828125,8.25390625,8388608 +50,self_attn.k_proj,8,128,8,128,0.368408203125,8.126953125,8388608 +50,self_attn.o_proj,2,32,8,128,41.15625,20.0625,67108864 +50,self_attn.o_proj,2,64,8,128,47.90625,18.03125,67108864 +50,self_attn.o_proj,2,128,8,128,53.96875,17.015625,67108864 +50,self_attn.o_proj,3,32,8,128,17.609375,28.0625,67108864 +50,self_attn.o_proj,3,64,8,128,20.515625,26.03125,67108864 +50,self_attn.o_proj,3,128,8,128,23.125,25.015625,67108864 +50,self_attn.o_proj,4,32,8,128,8.2109375,36.0625,67108864 +50,self_attn.o_proj,4,64,8,128,9.578125,34.03125,67108864 +50,self_attn.o_proj,4,128,8,128,10.796875,33.015625,67108864 +50,self_attn.o_proj,8,32,8,128,0.497314453125,68.0625,67108864 +50,self_attn.o_proj,8,64,8,128,0.5791015625,66.03125,67108864 +50,self_attn.o_proj,8,128,8,128,0.65087890625,65.015625,67108864 +50,mlp.gate_proj,2,32,8,128,89.8125,70.21875,234881024 +50,mlp.gate_proj,2,64,8,128,104.8125,63.109375,234881024 +50,mlp.gate_proj,2,128,8,128,118.3125,59.5546875,234881024 +50,mlp.gate_proj,3,32,8,128,38.4375,98.21875,234881024 +50,mlp.gate_proj,3,64,8,128,44.8125,91.109375,234881024 +50,mlp.gate_proj,3,128,8,128,50.6875,87.5546875,234881024 +50,mlp.gate_proj,4,32,8,128,17.921875,126.21875,234881024 +50,mlp.gate_proj,4,64,8,128,20.921875,119.109375,234881024 +50,mlp.gate_proj,4,128,8,128,23.640625,115.5546875,234881024 +50,mlp.gate_proj,8,32,8,128,1.0908203125,238.21875,234881024 +50,mlp.gate_proj,8,64,8,128,1.2734375,231.109375,234881024 +50,mlp.gate_proj,8,128,8,128,1.4345703125,227.5546875,234881024 +50,mlp.up_proj,2,32,8,128,79.8125,70.21875,234881024 +50,mlp.up_proj,2,64,8,128,92.8125,63.109375,234881024 +50,mlp.up_proj,2,128,8,128,104.625,59.5546875,234881024 +50,mlp.up_proj,3,32,8,128,34.09375,98.21875,234881024 +50,mlp.up_proj,3,64,8,128,39.75,91.109375,234881024 +50,mlp.up_proj,3,128,8,128,44.75,87.5546875,234881024 +50,mlp.up_proj,4,32,8,128,15.8984375,126.21875,234881024 +50,mlp.up_proj,4,64,8,128,18.53125,119.109375,234881024 +50,mlp.up_proj,4,128,8,128,20.890625,115.5546875,234881024 +50,mlp.up_proj,8,32,8,128,0.966796875,238.21875,234881024 +50,mlp.up_proj,8,64,8,128,1.126953125,231.109375,234881024 +50,mlp.up_proj,8,128,8,128,1.267578125,227.5546875,234881024 +50,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024 +50,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024 +50,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024 +50,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024 +50,mlp.down_proj,3,64,8,128,40.25,91.109375,234881024 +50,mlp.down_proj,3,128,8,128,45.3125,87.5546875,234881024 +50,mlp.down_proj,4,32,8,128,16.125,126.21875,234881024 +50,mlp.down_proj,4,64,8,128,18.78125,119.109375,234881024 +50,mlp.down_proj,4,128,8,128,21.15625,115.5546875,234881024 +50,mlp.down_proj,8,32,8,128,0.97998046875,238.21875,234881024 +50,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024 +50,mlp.down_proj,8,128,8,128,1.283203125,227.5546875,234881024 +51,self_attn.q_proj,2,32,8,128,37.65625,20.0625,67108864 +51,self_attn.q_proj,2,64,8,128,44.4375,18.03125,67108864 +51,self_attn.q_proj,2,128,8,128,51.3125,17.015625,67108864 +51,self_attn.q_proj,3,32,8,128,16.140625,28.0625,67108864 +51,self_attn.q_proj,3,64,8,128,19.109375,26.03125,67108864 +51,self_attn.q_proj,3,128,8,128,22.21875,25.015625,67108864 +51,self_attn.q_proj,4,32,8,128,7.53125,36.0625,67108864 +51,self_attn.q_proj,4,64,8,128,8.9140625,34.03125,67108864 +51,self_attn.q_proj,4,128,8,128,10.375,33.015625,67108864 +51,self_attn.q_proj,8,32,8,128,0.4560546875,68.0625,67108864 +51,self_attn.q_proj,8,64,8,128,0.53955078125,66.03125,67108864 +51,self_attn.q_proj,8,128,8,128,0.62548828125,65.015625,67108864 +51,self_attn.v_proj,2,32,8,128,18.046875,2.5078125,8388608 +51,self_attn.v_proj,2,64,8,128,21.109375,2.25390625,8388608 +51,self_attn.v_proj,2,128,8,128,23.890625,2.126953125,8388608 +51,self_attn.v_proj,3,32,8,128,7.71484375,3.5078125,8388608 +51,self_attn.v_proj,3,64,8,128,9.0234375,3.25390625,8388608 +51,self_attn.v_proj,3,128,8,128,10.21875,3.126953125,8388608 +51,self_attn.v_proj,4,32,8,128,3.59765625,4.5078125,8388608 +51,self_attn.v_proj,4,64,8,128,4.2109375,4.25390625,8388608 +51,self_attn.v_proj,4,128,8,128,4.76953125,4.126953125,8388608 +51,self_attn.v_proj,8,32,8,128,0.218017578125,8.5078125,8388608 +51,self_attn.v_proj,8,64,8,128,0.2548828125,8.25390625,8388608 +51,self_attn.v_proj,8,128,8,128,0.28759765625,8.126953125,8388608 +51,self_attn.k_proj,2,32,8,128,21.640625,2.5078125,8388608 +51,self_attn.k_proj,2,64,8,128,26.203125,2.25390625,8388608 +51,self_attn.k_proj,2,128,8,128,30.03125,2.126953125,8388608 +51,self_attn.k_proj,3,32,8,128,9.3515625,3.5078125,8388608 +51,self_attn.k_proj,3,64,8,128,11.515625,3.25390625,8388608 +51,self_attn.k_proj,3,128,8,128,13.5625,3.126953125,8388608 +51,self_attn.k_proj,4,32,8,128,4.36328125,4.5078125,8388608 +51,self_attn.k_proj,4,64,8,128,5.375,4.25390625,8388608 +51,self_attn.k_proj,4,128,8,128,6.33203125,4.126953125,8388608 +51,self_attn.k_proj,8,32,8,128,0.26416015625,8.5078125,8388608 +51,self_attn.k_proj,8,64,8,128,0.3251953125,8.25390625,8388608 +51,self_attn.k_proj,8,128,8,128,0.381591796875,8.126953125,8388608 +51,self_attn.o_proj,2,32,8,128,40.8125,20.0625,67108864 +51,self_attn.o_proj,2,64,8,128,47.53125,18.03125,67108864 +51,self_attn.o_proj,2,128,8,128,53.59375,17.015625,67108864 +51,self_attn.o_proj,3,32,8,128,17.46875,28.0625,67108864 +51,self_attn.o_proj,3,64,8,128,20.359375,26.03125,67108864 +51,self_attn.o_proj,3,128,8,128,22.96875,25.015625,67108864 +51,self_attn.o_proj,4,32,8,128,8.1484375,36.0625,67108864 +51,self_attn.o_proj,4,64,8,128,9.5,34.03125,67108864 +51,self_attn.o_proj,4,128,8,128,10.7265625,33.015625,67108864 +51,self_attn.o_proj,8,32,8,128,0.493408203125,68.0625,67108864 +51,self_attn.o_proj,8,64,8,128,0.57470703125,66.03125,67108864 +51,self_attn.o_proj,8,128,8,128,0.64697265625,65.015625,67108864 +51,mlp.gate_proj,2,32,8,128,89.6875,70.21875,234881024 +51,mlp.gate_proj,2,64,8,128,104.6875,63.109375,234881024 +51,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024 +51,mlp.gate_proj,3,32,8,128,38.375,98.21875,234881024 +51,mlp.gate_proj,3,64,8,128,44.75,91.109375,234881024 +51,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024 +51,mlp.gate_proj,4,32,8,128,17.90625,126.21875,234881024 +51,mlp.gate_proj,4,64,8,128,20.890625,119.109375,234881024 +51,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +51,mlp.gate_proj,8,32,8,128,1.0888671875,238.21875,234881024 +51,mlp.gate_proj,8,64,8,128,1.271484375,231.109375,234881024 +51,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +51,mlp.up_proj,2,32,8,128,80.0,70.21875,234881024 +51,mlp.up_proj,2,64,8,128,93.0,63.109375,234881024 +51,mlp.up_proj,2,128,8,128,104.8125,59.5546875,234881024 +51,mlp.up_proj,3,32,8,128,34.15625,98.21875,234881024 +51,mlp.up_proj,3,64,8,128,39.8125,91.109375,234881024 +51,mlp.up_proj,3,128,8,128,44.84375,87.5546875,234881024 +51,mlp.up_proj,4,32,8,128,15.9375,126.21875,234881024 +51,mlp.up_proj,4,64,8,128,18.578125,119.109375,234881024 +51,mlp.up_proj,4,128,8,128,20.9375,115.5546875,234881024 +51,mlp.up_proj,8,32,8,128,0.96923828125,238.21875,234881024 +51,mlp.up_proj,8,64,8,128,1.1298828125,231.109375,234881024 +51,mlp.up_proj,8,128,8,128,1.2705078125,227.5546875,234881024 +51,mlp.down_proj,2,32,8,128,80.9375,70.21875,234881024 +51,mlp.down_proj,2,64,8,128,94.1875,63.109375,234881024 +51,mlp.down_proj,2,128,8,128,106.125,59.5546875,234881024 +51,mlp.down_proj,3,32,8,128,34.59375,98.21875,234881024 +51,mlp.down_proj,3,64,8,128,40.3125,91.109375,234881024 +51,mlp.down_proj,3,128,8,128,45.375,87.5546875,234881024 +51,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024 +51,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024 +51,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024 +51,mlp.down_proj,8,32,8,128,0.9814453125,238.21875,234881024 +51,mlp.down_proj,8,64,8,128,1.1435546875,231.109375,234881024 +51,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024 +52,self_attn.q_proj,2,32,8,128,40.40625,20.0625,67108864 +52,self_attn.q_proj,2,64,8,128,47.34375,18.03125,67108864 +52,self_attn.q_proj,2,128,8,128,54.6875,17.015625,67108864 +52,self_attn.q_proj,3,32,8,128,17.296875,28.0625,67108864 +52,self_attn.q_proj,3,64,8,128,20.3125,26.03125,67108864 +52,self_attn.q_proj,3,128,8,128,23.546875,25.015625,67108864 +52,self_attn.q_proj,4,32,8,128,8.0703125,36.0625,67108864 +52,self_attn.q_proj,4,64,8,128,9.484375,34.03125,67108864 +52,self_attn.q_proj,4,128,8,128,11.0,33.015625,67108864 +52,self_attn.q_proj,8,32,8,128,0.48876953125,68.0625,67108864 +52,self_attn.q_proj,8,64,8,128,0.57373046875,66.03125,67108864 +52,self_attn.q_proj,8,128,8,128,0.66357421875,65.015625,67108864 +52,self_attn.v_proj,2,32,8,128,17.765625,2.5078125,8388608 +52,self_attn.v_proj,2,64,8,128,20.78125,2.25390625,8388608 +52,self_attn.v_proj,2,128,8,128,23.515625,2.126953125,8388608 +52,self_attn.v_proj,3,32,8,128,7.59765625,3.5078125,8388608 +52,self_attn.v_proj,3,64,8,128,8.890625,3.25390625,8388608 +52,self_attn.v_proj,3,128,8,128,10.0546875,3.126953125,8388608 +52,self_attn.v_proj,4,32,8,128,3.54296875,4.5078125,8388608 +52,self_attn.v_proj,4,64,8,128,4.1484375,4.25390625,8388608 +52,self_attn.v_proj,4,128,8,128,4.69140625,4.126953125,8388608 +52,self_attn.v_proj,8,32,8,128,0.214599609375,8.5078125,8388608 +52,self_attn.v_proj,8,64,8,128,0.2509765625,8.25390625,8388608 +52,self_attn.v_proj,8,128,8,128,0.282958984375,8.126953125,8388608 +52,self_attn.k_proj,2,32,8,128,22.265625,2.5078125,8388608 +52,self_attn.k_proj,2,64,8,128,26.6875,2.25390625,8388608 +52,self_attn.k_proj,2,128,8,128,30.09375,2.126953125,8388608 +52,self_attn.k_proj,3,32,8,128,9.6796875,3.5078125,8388608 +52,self_attn.k_proj,3,64,8,128,11.8984375,3.25390625,8388608 +52,self_attn.k_proj,3,128,8,128,13.90625,3.126953125,8388608 +52,self_attn.k_proj,4,32,8,128,4.515625,4.5078125,8388608 +52,self_attn.k_proj,4,64,8,128,5.55859375,4.25390625,8388608 +52,self_attn.k_proj,4,128,8,128,6.5,4.126953125,8388608 +52,self_attn.k_proj,8,32,8,128,0.2734375,8.5078125,8388608 +52,self_attn.k_proj,8,64,8,128,0.3359375,8.25390625,8388608 +52,self_attn.k_proj,8,128,8,128,0.391845703125,8.126953125,8388608 +52,self_attn.o_proj,2,32,8,128,41.8125,20.0625,67108864 +52,self_attn.o_proj,2,64,8,128,48.71875,18.03125,67108864 +52,self_attn.o_proj,2,128,8,128,55.0,17.015625,67108864 +52,self_attn.o_proj,3,32,8,128,17.9375,28.0625,67108864 +52,self_attn.o_proj,3,64,8,128,20.96875,26.03125,67108864 +52,self_attn.o_proj,3,128,8,128,23.75,25.015625,67108864 +52,self_attn.o_proj,4,32,8,128,8.421875,36.0625,67108864 +52,self_attn.o_proj,4,64,8,128,9.875,34.03125,67108864 +52,self_attn.o_proj,4,128,8,128,11.234375,33.015625,67108864 +52,self_attn.o_proj,8,32,8,128,0.51123046875,68.0625,67108864 +52,self_attn.o_proj,8,64,8,128,0.60009765625,66.03125,67108864 +52,self_attn.o_proj,8,128,8,128,0.68359375,65.015625,67108864 +52,mlp.gate_proj,2,32,8,128,89.75,70.21875,234881024 +52,mlp.gate_proj,2,64,8,128,104.8125,63.109375,234881024 +52,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024 +52,mlp.gate_proj,3,32,8,128,38.40625,98.21875,234881024 +52,mlp.gate_proj,3,64,8,128,44.78125,91.109375,234881024 +52,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024 +52,mlp.gate_proj,4,32,8,128,17.921875,126.21875,234881024 +52,mlp.gate_proj,4,64,8,128,20.90625,119.109375,234881024 +52,mlp.gate_proj,4,128,8,128,23.625,115.5546875,234881024 +52,mlp.gate_proj,8,32,8,128,1.08984375,238.21875,234881024 +52,mlp.gate_proj,8,64,8,128,1.2724609375,231.109375,234881024 +52,mlp.gate_proj,8,128,8,128,1.4326171875,227.5546875,234881024 +52,mlp.up_proj,2,32,8,128,80.0625,70.21875,234881024 +52,mlp.up_proj,2,64,8,128,93.125,63.109375,234881024 +52,mlp.up_proj,2,128,8,128,104.9375,59.5546875,234881024 +52,mlp.up_proj,3,32,8,128,34.1875,98.21875,234881024 +52,mlp.up_proj,3,64,8,128,39.875,91.109375,234881024 +52,mlp.up_proj,3,128,8,128,44.90625,87.5546875,234881024 +52,mlp.up_proj,4,32,8,128,15.953125,126.21875,234881024 +52,mlp.up_proj,4,64,8,128,18.59375,119.109375,234881024 +52,mlp.up_proj,4,128,8,128,20.953125,115.5546875,234881024 +52,mlp.up_proj,8,32,8,128,0.9697265625,238.21875,234881024 +52,mlp.up_proj,8,64,8,128,1.130859375,231.109375,234881024 +52,mlp.up_proj,8,128,8,128,1.271484375,227.5546875,234881024 +52,mlp.down_proj,2,32,8,128,80.9375,70.21875,234881024 +52,mlp.down_proj,2,64,8,128,94.1875,63.109375,234881024 +52,mlp.down_proj,2,128,8,128,106.125,59.5546875,234881024 +52,mlp.down_proj,3,32,8,128,34.59375,98.21875,234881024 +52,mlp.down_proj,3,64,8,128,40.3125,91.109375,234881024 +52,mlp.down_proj,3,128,8,128,45.375,87.5546875,234881024 +52,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024 +52,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024 +52,mlp.down_proj,4,128,8,128,21.171875,115.5546875,234881024 +52,mlp.down_proj,8,32,8,128,0.9814453125,238.21875,234881024 +52,mlp.down_proj,8,64,8,128,1.1435546875,231.109375,234881024 +52,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024 +53,self_attn.q_proj,2,32,8,128,33.6875,20.0625,67108864 +53,self_attn.q_proj,2,64,8,128,39.71875,18.03125,67108864 +53,self_attn.q_proj,2,128,8,128,46.0,17.015625,67108864 +53,self_attn.q_proj,3,32,8,128,14.4609375,28.0625,67108864 +53,self_attn.q_proj,3,64,8,128,17.125,26.03125,67108864 +53,self_attn.q_proj,3,128,8,128,19.96875,25.015625,67108864 +53,self_attn.q_proj,4,32,8,128,6.75,36.0625,67108864 +53,self_attn.q_proj,4,64,8,128,7.98828125,34.03125,67108864 +53,self_attn.q_proj,4,128,8,128,9.3203125,33.015625,67108864 +53,self_attn.q_proj,8,32,8,128,0.408447265625,68.0625,67108864 +53,self_attn.q_proj,8,64,8,128,0.4833984375,66.03125,67108864 +53,self_attn.q_proj,8,128,8,128,0.56201171875,65.015625,67108864 +53,self_attn.v_proj,2,32,8,128,18.265625,2.5078125,8388608 +53,self_attn.v_proj,2,64,8,128,21.40625,2.25390625,8388608 +53,self_attn.v_proj,2,128,8,128,24.28125,2.126953125,8388608 +53,self_attn.v_proj,3,32,8,128,7.8125,3.5078125,8388608 +53,self_attn.v_proj,3,64,8,128,9.15625,3.25390625,8388608 +53,self_attn.v_proj,3,128,8,128,10.390625,3.126953125,8388608 +53,self_attn.v_proj,4,32,8,128,3.64453125,4.5078125,8388608 +53,self_attn.v_proj,4,64,8,128,4.2734375,4.25390625,8388608 +53,self_attn.v_proj,4,128,8,128,4.84765625,4.126953125,8388608 +53,self_attn.v_proj,8,32,8,128,0.2208251953125,8.5078125,8388608 +53,self_attn.v_proj,8,64,8,128,0.258544921875,8.25390625,8388608 +53,self_attn.v_proj,8,128,8,128,0.2919921875,8.126953125,8388608 +53,self_attn.k_proj,2,32,8,128,20.03125,2.5078125,8388608 +53,self_attn.k_proj,2,64,8,128,24.0625,2.25390625,8388608 +53,self_attn.k_proj,2,128,8,128,27.328125,2.126953125,8388608 +53,self_attn.k_proj,3,32,8,128,8.8125,3.5078125,8388608 +53,self_attn.k_proj,3,64,8,128,10.953125,3.25390625,8388608 +53,self_attn.k_proj,3,128,8,128,13.0078125,3.126953125,8388608 +53,self_attn.k_proj,4,32,8,128,4.1171875,4.5078125,8388608 +53,self_attn.k_proj,4,64,8,128,5.12109375,4.25390625,8388608 +53,self_attn.k_proj,4,128,8,128,6.10546875,4.126953125,8388608 +53,self_attn.k_proj,8,32,8,128,0.2491455078125,8.5078125,8388608 +53,self_attn.k_proj,8,64,8,128,0.309814453125,8.25390625,8388608 +53,self_attn.k_proj,8,128,8,128,0.3681640625,8.126953125,8388608 +53,self_attn.o_proj,2,32,8,128,41.4375,20.0625,67108864 +53,self_attn.o_proj,2,64,8,128,48.25,18.03125,67108864 +53,self_attn.o_proj,2,128,8,128,54.375,17.015625,67108864 +53,self_attn.o_proj,3,32,8,128,17.71875,28.0625,67108864 +53,self_attn.o_proj,3,64,8,128,20.65625,26.03125,67108864 +53,self_attn.o_proj,3,128,8,128,23.3125,25.015625,67108864 +53,self_attn.o_proj,4,32,8,128,8.265625,36.0625,67108864 +53,self_attn.o_proj,4,64,8,128,9.640625,34.03125,67108864 +53,self_attn.o_proj,4,128,8,128,10.8828125,33.015625,67108864 +53,self_attn.o_proj,8,32,8,128,0.50048828125,68.0625,67108864 +53,self_attn.o_proj,8,64,8,128,0.5830078125,66.03125,67108864 +53,self_attn.o_proj,8,128,8,128,0.65625,65.015625,67108864 +53,mlp.gate_proj,2,32,8,128,89.75,70.21875,234881024 +53,mlp.gate_proj,2,64,8,128,104.8125,63.109375,234881024 +53,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024 +53,mlp.gate_proj,3,32,8,128,38.40625,98.21875,234881024 +53,mlp.gate_proj,3,64,8,128,44.8125,91.109375,234881024 +53,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024 +53,mlp.gate_proj,4,32,8,128,17.921875,126.21875,234881024 +53,mlp.gate_proj,4,64,8,128,20.90625,119.109375,234881024 +53,mlp.gate_proj,4,128,8,128,23.625,115.5546875,234881024 +53,mlp.gate_proj,8,32,8,128,1.08984375,238.21875,234881024 +53,mlp.gate_proj,8,64,8,128,1.2724609375,231.109375,234881024 +53,mlp.gate_proj,8,128,8,128,1.4326171875,227.5546875,234881024 +53,mlp.up_proj,2,32,8,128,80.1875,70.21875,234881024 +53,mlp.up_proj,2,64,8,128,93.3125,63.109375,234881024 +53,mlp.up_proj,2,128,8,128,105.125,59.5546875,234881024 +53,mlp.up_proj,3,32,8,128,34.25,98.21875,234881024 +53,mlp.up_proj,3,64,8,128,39.9375,91.109375,234881024 +53,mlp.up_proj,3,128,8,128,44.96875,87.5546875,234881024 +53,mlp.up_proj,4,32,8,128,15.9765625,126.21875,234881024 +53,mlp.up_proj,4,64,8,128,18.625,119.109375,234881024 +53,mlp.up_proj,4,128,8,128,20.984375,115.5546875,234881024 +53,mlp.up_proj,8,32,8,128,0.9716796875,238.21875,234881024 +53,mlp.up_proj,8,64,8,128,1.1328125,231.109375,234881024 +53,mlp.up_proj,8,128,8,128,1.2734375,227.5546875,234881024 +53,mlp.down_proj,2,32,8,128,81.0,70.21875,234881024 +53,mlp.down_proj,2,64,8,128,94.25,63.109375,234881024 +53,mlp.down_proj,2,128,8,128,106.1875,59.5546875,234881024 +53,mlp.down_proj,3,32,8,128,34.625,98.21875,234881024 +53,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024 +53,mlp.down_proj,3,128,8,128,45.40625,87.5546875,234881024 +53,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024 +53,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024 +53,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024 +53,mlp.down_proj,8,32,8,128,0.98193359375,238.21875,234881024 +53,mlp.down_proj,8,64,8,128,1.14453125,231.109375,234881024 +53,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024 +54,self_attn.q_proj,2,32,8,128,34.46875,20.0625,67108864 +54,self_attn.q_proj,2,64,8,128,40.625,18.03125,67108864 +54,self_attn.q_proj,2,128,8,128,47.125,17.015625,67108864 +54,self_attn.q_proj,3,32,8,128,14.78125,28.0625,67108864 +54,self_attn.q_proj,3,64,8,128,17.5,26.03125,67108864 +54,self_attn.q_proj,3,128,8,128,20.484375,25.015625,67108864 +54,self_attn.q_proj,4,32,8,128,6.8984375,36.0625,67108864 +54,self_attn.q_proj,4,64,8,128,8.171875,34.03125,67108864 +54,self_attn.q_proj,4,128,8,128,9.5625,33.015625,67108864 +54,self_attn.q_proj,8,32,8,128,0.41748046875,68.0625,67108864 +54,self_attn.q_proj,8,64,8,128,0.494384765625,66.03125,67108864 +54,self_attn.q_proj,8,128,8,128,0.5771484375,65.015625,67108864 +54,self_attn.v_proj,2,32,8,128,18.25,2.5078125,8388608 +54,self_attn.v_proj,2,64,8,128,21.421875,2.25390625,8388608 +54,self_attn.v_proj,2,128,8,128,24.296875,2.126953125,8388608 +54,self_attn.v_proj,3,32,8,128,7.8046875,3.5078125,8388608 +54,self_attn.v_proj,3,64,8,128,9.1640625,3.25390625,8388608 +54,self_attn.v_proj,3,128,8,128,10.3984375,3.126953125,8388608 +54,self_attn.v_proj,4,32,8,128,3.640625,4.5078125,8388608 +54,self_attn.v_proj,4,64,8,128,4.2734375,4.25390625,8388608 +54,self_attn.v_proj,4,128,8,128,4.8515625,4.126953125,8388608 +54,self_attn.v_proj,8,32,8,128,0.220458984375,8.5078125,8388608 +54,self_attn.v_proj,8,64,8,128,0.258544921875,8.25390625,8388608 +54,self_attn.v_proj,8,128,8,128,0.29248046875,8.126953125,8388608 +54,self_attn.k_proj,2,32,8,128,19.71875,2.5078125,8388608 +54,self_attn.k_proj,2,64,8,128,23.78125,2.25390625,8388608 +54,self_attn.k_proj,2,128,8,128,27.046875,2.126953125,8388608 +54,self_attn.k_proj,3,32,8,128,8.6484375,3.5078125,8388608 +54,self_attn.k_proj,3,64,8,128,10.765625,3.25390625,8388608 +54,self_attn.k_proj,3,128,8,128,12.7890625,3.126953125,8388608 +54,self_attn.k_proj,4,32,8,128,4.03515625,4.5078125,8388608 +54,self_attn.k_proj,4,64,8,128,5.03125,4.25390625,8388608 +54,self_attn.k_proj,4,128,8,128,5.98828125,4.126953125,8388608 +54,self_attn.k_proj,8,32,8,128,0.2442626953125,8.5078125,8388608 +54,self_attn.k_proj,8,64,8,128,0.30419921875,8.25390625,8388608 +54,self_attn.k_proj,8,128,8,128,0.360595703125,8.126953125,8388608 +54,self_attn.o_proj,2,32,8,128,40.59375,20.0625,67108864 +54,self_attn.o_proj,2,64,8,128,47.25,18.03125,67108864 +54,self_attn.o_proj,2,128,8,128,53.28125,17.015625,67108864 +54,self_attn.o_proj,3,32,8,128,17.359375,28.0625,67108864 +54,self_attn.o_proj,3,64,8,128,20.21875,26.03125,67108864 +54,self_attn.o_proj,3,128,8,128,22.796875,25.015625,67108864 +54,self_attn.o_proj,4,32,8,128,8.1015625,36.0625,67108864 +54,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864 +54,self_attn.o_proj,4,128,8,128,10.6484375,33.015625,67108864 +54,self_attn.o_proj,8,32,8,128,0.490478515625,68.0625,67108864 +54,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864 +54,self_attn.o_proj,8,128,8,128,0.64208984375,65.015625,67108864 +54,mlp.gate_proj,2,32,8,128,89.6875,70.21875,234881024 +54,mlp.gate_proj,2,64,8,128,104.6875,63.109375,234881024 +54,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024 +54,mlp.gate_proj,3,32,8,128,38.40625,98.21875,234881024 +54,mlp.gate_proj,3,64,8,128,44.75,91.109375,234881024 +54,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024 +54,mlp.gate_proj,4,32,8,128,17.90625,126.21875,234881024 +54,mlp.gate_proj,4,64,8,128,20.890625,119.109375,234881024 +54,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +54,mlp.gate_proj,8,32,8,128,1.0888671875,238.21875,234881024 +54,mlp.gate_proj,8,64,8,128,1.271484375,231.109375,234881024 +54,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +54,mlp.up_proj,2,32,8,128,80.375,70.21875,234881024 +54,mlp.up_proj,2,64,8,128,93.5,63.109375,234881024 +54,mlp.up_proj,2,128,8,128,105.375,59.5546875,234881024 +54,mlp.up_proj,3,32,8,128,34.34375,98.21875,234881024 +54,mlp.up_proj,3,64,8,128,40.03125,91.109375,234881024 +54,mlp.up_proj,3,128,8,128,45.0625,87.5546875,234881024 +54,mlp.up_proj,4,32,8,128,16.015625,126.21875,234881024 +54,mlp.up_proj,4,64,8,128,18.671875,119.109375,234881024 +54,mlp.up_proj,4,128,8,128,21.03125,115.5546875,234881024 +54,mlp.up_proj,8,32,8,128,0.9736328125,238.21875,234881024 +54,mlp.up_proj,8,64,8,128,1.1357421875,231.109375,234881024 +54,mlp.up_proj,8,128,8,128,1.2763671875,227.5546875,234881024 +54,mlp.down_proj,2,32,8,128,81.0625,70.21875,234881024 +54,mlp.down_proj,2,64,8,128,94.3125,63.109375,234881024 +54,mlp.down_proj,2,128,8,128,106.3125,59.5546875,234881024 +54,mlp.down_proj,3,32,8,128,34.65625,98.21875,234881024 +54,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024 +54,mlp.down_proj,3,128,8,128,45.4375,87.5546875,234881024 +54,mlp.down_proj,4,32,8,128,16.15625,126.21875,234881024 +54,mlp.down_proj,4,64,8,128,18.828125,119.109375,234881024 +54,mlp.down_proj,4,128,8,128,21.203125,115.5546875,234881024 +54,mlp.down_proj,8,32,8,128,0.98291015625,238.21875,234881024 +54,mlp.down_proj,8,64,8,128,1.1455078125,231.109375,234881024 +54,mlp.down_proj,8,128,8,128,1.2861328125,227.5546875,234881024 +55,self_attn.q_proj,2,32,8,128,38.9375,20.0625,67108864 +55,self_attn.q_proj,2,64,8,128,45.9375,18.03125,67108864 +55,self_attn.q_proj,2,128,8,128,53.3125,17.015625,67108864 +55,self_attn.q_proj,3,32,8,128,16.734375,28.0625,67108864 +55,self_attn.q_proj,3,64,8,128,19.84375,26.03125,67108864 +55,self_attn.q_proj,3,128,8,128,23.25,25.015625,67108864 +55,self_attn.q_proj,4,32,8,128,7.8046875,36.0625,67108864 +55,self_attn.q_proj,4,64,8,128,9.2578125,34.03125,67108864 +55,self_attn.q_proj,4,128,8,128,10.8515625,33.015625,67108864 +55,self_attn.q_proj,8,32,8,128,0.47265625,68.0625,67108864 +55,self_attn.q_proj,8,64,8,128,0.560546875,66.03125,67108864 +55,self_attn.q_proj,8,128,8,128,0.65478515625,65.015625,67108864 +55,self_attn.v_proj,2,32,8,128,18.609375,2.5078125,8388608 +55,self_attn.v_proj,2,64,8,128,21.71875,2.25390625,8388608 +55,self_attn.v_proj,2,128,8,128,24.5625,2.126953125,8388608 +55,self_attn.v_proj,3,32,8,128,7.95703125,3.5078125,8388608 +55,self_attn.v_proj,3,64,8,128,9.296875,3.25390625,8388608 +55,self_attn.v_proj,3,128,8,128,10.5078125,3.126953125,8388608 +55,self_attn.v_proj,4,32,8,128,3.712890625,4.5078125,8388608 +55,self_attn.v_proj,4,64,8,128,4.3359375,4.25390625,8388608 +55,self_attn.v_proj,4,128,8,128,4.90625,4.126953125,8388608 +55,self_attn.v_proj,8,32,8,128,0.224853515625,8.5078125,8388608 +55,self_attn.v_proj,8,64,8,128,0.262451171875,8.25390625,8388608 +55,self_attn.v_proj,8,128,8,128,0.295654296875,8.126953125,8388608 +55,self_attn.k_proj,2,32,8,128,22.234375,2.5078125,8388608 +55,self_attn.k_proj,2,64,8,128,26.609375,2.25390625,8388608 +55,self_attn.k_proj,2,128,8,128,30.140625,2.126953125,8388608 +55,self_attn.k_proj,3,32,8,128,9.703125,3.5078125,8388608 +55,self_attn.k_proj,3,64,8,128,11.890625,3.25390625,8388608 +55,self_attn.k_proj,3,128,8,128,13.9765625,3.126953125,8388608 +55,self_attn.k_proj,4,32,8,128,4.52734375,4.5078125,8388608 +55,self_attn.k_proj,4,64,8,128,5.55078125,4.25390625,8388608 +55,self_attn.k_proj,4,128,8,128,6.52734375,4.126953125,8388608 +55,self_attn.k_proj,8,32,8,128,0.274169921875,8.5078125,8388608 +55,self_attn.k_proj,8,64,8,128,0.335693359375,8.25390625,8388608 +55,self_attn.k_proj,8,128,8,128,0.3935546875,8.126953125,8388608 +55,self_attn.o_proj,2,32,8,128,43.15625,20.0625,67108864 +55,self_attn.o_proj,2,64,8,128,50.25,18.03125,67108864 +55,self_attn.o_proj,2,128,8,128,56.59375,17.015625,67108864 +55,self_attn.o_proj,3,32,8,128,18.453125,28.0625,67108864 +55,self_attn.o_proj,3,64,8,128,21.5,26.03125,67108864 +55,self_attn.o_proj,3,128,8,128,24.234375,25.015625,67108864 +55,self_attn.o_proj,4,32,8,128,8.609375,36.0625,67108864 +55,self_attn.o_proj,4,64,8,128,10.03125,34.03125,67108864 +55,self_attn.o_proj,4,128,8,128,11.3125,33.015625,67108864 +55,self_attn.o_proj,8,32,8,128,0.521484375,68.0625,67108864 +55,self_attn.o_proj,8,64,8,128,0.60693359375,66.03125,67108864 +55,self_attn.o_proj,8,128,8,128,0.681640625,65.015625,67108864 +55,mlp.gate_proj,2,32,8,128,89.5625,70.21875,234881024 +55,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024 +55,mlp.gate_proj,2,128,8,128,117.9375,59.5546875,234881024 +55,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024 +55,mlp.gate_proj,3,64,8,128,44.6875,91.109375,234881024 +55,mlp.gate_proj,3,128,8,128,50.5,87.5546875,234881024 +55,mlp.gate_proj,4,32,8,128,17.875,126.21875,234881024 +55,mlp.gate_proj,4,64,8,128,20.859375,119.109375,234881024 +55,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024 +55,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024 +55,mlp.gate_proj,8,64,8,128,1.26953125,231.109375,234881024 +55,mlp.gate_proj,8,128,8,128,1.4287109375,227.5546875,234881024 +55,mlp.up_proj,2,32,8,128,80.625,70.21875,234881024 +55,mlp.up_proj,2,64,8,128,93.8125,63.109375,234881024 +55,mlp.up_proj,2,128,8,128,105.6875,59.5546875,234881024 +55,mlp.up_proj,3,32,8,128,34.46875,98.21875,234881024 +55,mlp.up_proj,3,64,8,128,40.15625,91.109375,234881024 +55,mlp.up_proj,3,128,8,128,45.21875,87.5546875,234881024 +55,mlp.up_proj,4,32,8,128,16.0625,126.21875,234881024 +55,mlp.up_proj,4,64,8,128,18.734375,119.109375,234881024 +55,mlp.up_proj,4,128,8,128,21.109375,115.5546875,234881024 +55,mlp.up_proj,8,32,8,128,0.97705078125,238.21875,234881024 +55,mlp.up_proj,8,64,8,128,1.1396484375,231.109375,234881024 +55,mlp.up_proj,8,128,8,128,1.2802734375,227.5546875,234881024 +55,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024 +55,mlp.down_proj,2,64,8,128,94.5,63.109375,234881024 +55,mlp.down_proj,2,128,8,128,106.4375,59.5546875,234881024 +55,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024 +55,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024 +55,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024 +55,mlp.down_proj,4,32,8,128,16.1875,126.21875,234881024 +55,mlp.down_proj,4,64,8,128,18.875,119.109375,234881024 +55,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024 +55,mlp.down_proj,8,32,8,128,0.98486328125,238.21875,234881024 +55,mlp.down_proj,8,64,8,128,1.1484375,231.109375,234881024 +55,mlp.down_proj,8,128,8,128,1.2890625,227.5546875,234881024 +56,self_attn.q_proj,2,32,8,128,36.5,20.0625,67108864 +56,self_attn.q_proj,2,64,8,128,43.28125,18.03125,67108864 +56,self_attn.q_proj,2,128,8,128,50.125,17.015625,67108864 +56,self_attn.q_proj,3,32,8,128,15.65625,28.0625,67108864 +56,self_attn.q_proj,3,64,8,128,18.65625,26.03125,67108864 +56,self_attn.q_proj,3,128,8,128,21.8125,25.015625,67108864 +56,self_attn.q_proj,4,32,8,128,7.30859375,36.0625,67108864 +56,self_attn.q_proj,4,64,8,128,8.7109375,34.03125,67108864 +56,self_attn.q_proj,4,128,8,128,10.1875,33.015625,67108864 +56,self_attn.q_proj,8,32,8,128,0.4423828125,68.0625,67108864 +56,self_attn.q_proj,8,64,8,128,0.52734375,66.03125,67108864 +56,self_attn.q_proj,8,128,8,128,0.6142578125,65.015625,67108864 +56,self_attn.v_proj,2,32,8,128,18.21875,2.5078125,8388608 +56,self_attn.v_proj,2,64,8,128,21.359375,2.25390625,8388608 +56,self_attn.v_proj,2,128,8,128,24.1875,2.126953125,8388608 +56,self_attn.v_proj,3,32,8,128,7.7890625,3.5078125,8388608 +56,self_attn.v_proj,3,64,8,128,9.1328125,3.25390625,8388608 +56,self_attn.v_proj,3,128,8,128,10.34375,3.126953125,8388608 +56,self_attn.v_proj,4,32,8,128,3.6328125,4.5078125,8388608 +56,self_attn.v_proj,4,64,8,128,4.2578125,4.25390625,8388608 +56,self_attn.v_proj,4,128,8,128,4.828125,4.126953125,8388608 +56,self_attn.v_proj,8,32,8,128,0.219970703125,8.5078125,8388608 +56,self_attn.v_proj,8,64,8,128,0.257568359375,8.25390625,8388608 +56,self_attn.v_proj,8,128,8,128,0.291015625,8.126953125,8388608 +56,self_attn.k_proj,2,32,8,128,22.421875,2.5078125,8388608 +56,self_attn.k_proj,2,64,8,128,26.84375,2.25390625,8388608 +56,self_attn.k_proj,2,128,8,128,30.40625,2.126953125,8388608 +56,self_attn.k_proj,3,32,8,128,9.8359375,3.5078125,8388608 +56,self_attn.k_proj,3,64,8,128,12.140625,3.25390625,8388608 +56,self_attn.k_proj,3,128,8,128,14.34375,3.126953125,8388608 +56,self_attn.k_proj,4,32,8,128,4.59375,4.5078125,8388608 +56,self_attn.k_proj,4,64,8,128,5.67578125,4.25390625,8388608 +56,self_attn.k_proj,4,128,8,128,6.7109375,4.126953125,8388608 +56,self_attn.k_proj,8,32,8,128,0.278076171875,8.5078125,8388608 +56,self_attn.k_proj,8,64,8,128,0.343017578125,8.25390625,8388608 +56,self_attn.k_proj,8,128,8,128,0.404541015625,8.126953125,8388608 +56,self_attn.o_proj,2,32,8,128,40.84375,20.0625,67108864 +56,self_attn.o_proj,2,64,8,128,47.65625,18.03125,67108864 +56,self_attn.o_proj,2,128,8,128,53.8125,17.015625,67108864 +56,self_attn.o_proj,3,32,8,128,17.53125,28.0625,67108864 +56,self_attn.o_proj,3,64,8,128,20.5,26.03125,67108864 +56,self_attn.o_proj,3,128,8,128,23.265625,25.015625,67108864 +56,self_attn.o_proj,4,32,8,128,8.203125,36.0625,67108864 +56,self_attn.o_proj,4,64,8,128,9.6171875,34.03125,67108864 +56,self_attn.o_proj,4,128,8,128,10.9609375,33.015625,67108864 +56,self_attn.o_proj,8,32,8,128,0.499267578125,68.0625,67108864 +56,self_attn.o_proj,8,64,8,128,0.58642578125,66.03125,67108864 +56,self_attn.o_proj,8,128,8,128,0.66943359375,65.015625,67108864 +56,mlp.gate_proj,2,32,8,128,89.5,70.21875,234881024 +56,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024 +56,mlp.gate_proj,2,128,8,128,118.0,59.5546875,234881024 +56,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024 +56,mlp.gate_proj,3,64,8,128,44.65625,91.109375,234881024 +56,mlp.gate_proj,3,128,8,128,50.5,87.5546875,234881024 +56,mlp.gate_proj,4,32,8,128,17.859375,126.21875,234881024 +56,mlp.gate_proj,4,64,8,128,20.84375,119.109375,234881024 +56,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024 +56,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024 +56,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024 +56,mlp.gate_proj,8,128,8,128,1.4287109375,227.5546875,234881024 +56,mlp.up_proj,2,32,8,128,80.75,70.21875,234881024 +56,mlp.up_proj,2,64,8,128,93.9375,63.109375,234881024 +56,mlp.up_proj,2,128,8,128,105.875,59.5546875,234881024 +56,mlp.up_proj,3,32,8,128,34.5,98.21875,234881024 +56,mlp.up_proj,3,64,8,128,40.21875,91.109375,234881024 +56,mlp.up_proj,3,128,8,128,45.28125,87.5546875,234881024 +56,mlp.up_proj,4,32,8,128,16.09375,126.21875,234881024 +56,mlp.up_proj,4,64,8,128,18.765625,119.109375,234881024 +56,mlp.up_proj,4,128,8,128,21.140625,115.5546875,234881024 +56,mlp.up_proj,8,32,8,128,0.97900390625,238.21875,234881024 +56,mlp.up_proj,8,64,8,128,1.1416015625,231.109375,234881024 +56,mlp.up_proj,8,128,8,128,1.2822265625,227.5546875,234881024 +56,mlp.down_proj,2,32,8,128,81.3125,70.21875,234881024 +56,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024 +56,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024 +56,mlp.down_proj,3,32,8,128,34.75,98.21875,234881024 +56,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024 +56,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024 +56,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024 +56,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024 +56,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024 +56,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024 +56,mlp.down_proj,8,64,8,128,1.1484375,231.109375,234881024 +56,mlp.down_proj,8,128,8,128,1.2900390625,227.5546875,234881024 +57,self_attn.q_proj,2,32,8,128,34.75,20.0625,67108864 +57,self_attn.q_proj,2,64,8,128,40.90625,18.03125,67108864 +57,self_attn.q_proj,2,128,8,128,47.28125,17.015625,67108864 +57,self_attn.q_proj,3,32,8,128,14.8828125,28.0625,67108864 +57,self_attn.q_proj,3,64,8,128,17.546875,26.03125,67108864 +57,self_attn.q_proj,3,128,8,128,20.390625,25.015625,67108864 +57,self_attn.q_proj,4,32,8,128,6.9453125,36.0625,67108864 +57,self_attn.q_proj,4,64,8,128,8.1953125,34.03125,67108864 +57,self_attn.q_proj,4,128,8,128,9.515625,33.015625,67108864 +57,self_attn.q_proj,8,32,8,128,0.42041015625,68.0625,67108864 +57,self_attn.q_proj,8,64,8,128,0.49560546875,66.03125,67108864 +57,self_attn.q_proj,8,128,8,128,0.57421875,65.015625,67108864 +57,self_attn.v_proj,2,32,8,128,18.65625,2.5078125,8388608 +57,self_attn.v_proj,2,64,8,128,21.78125,2.25390625,8388608 +57,self_attn.v_proj,2,128,8,128,24.59375,2.126953125,8388608 +57,self_attn.v_proj,3,32,8,128,7.96484375,3.5078125,8388608 +57,self_attn.v_proj,3,64,8,128,9.3125,3.25390625,8388608 +57,self_attn.v_proj,3,128,8,128,10.53125,3.126953125,8388608 +57,self_attn.v_proj,4,32,8,128,3.71875,4.5078125,8388608 +57,self_attn.v_proj,4,64,8,128,4.34765625,4.25390625,8388608 +57,self_attn.v_proj,4,128,8,128,4.9140625,4.126953125,8388608 +57,self_attn.v_proj,8,32,8,128,0.2252197265625,8.5078125,8388608 +57,self_attn.v_proj,8,64,8,128,0.262939453125,8.25390625,8388608 +57,self_attn.v_proj,8,128,8,128,0.296142578125,8.126953125,8388608 +57,self_attn.k_proj,2,32,8,128,20.46875,2.5078125,8388608 +57,self_attn.k_proj,2,64,8,128,24.796875,2.25390625,8388608 +57,self_attn.k_proj,2,128,8,128,28.28125,2.126953125,8388608 +57,self_attn.k_proj,3,32,8,128,8.921875,3.5078125,8388608 +57,self_attn.k_proj,3,64,8,128,11.1015625,3.25390625,8388608 +57,self_attn.k_proj,3,128,8,128,13.1796875,3.126953125,8388608 +57,self_attn.k_proj,4,32,8,128,4.16015625,4.5078125,8388608 +57,self_attn.k_proj,4,64,8,128,5.1796875,4.25390625,8388608 +57,self_attn.k_proj,4,128,8,128,6.15625,4.126953125,8388608 +57,self_attn.k_proj,8,32,8,128,0.251953125,8.5078125,8388608 +57,self_attn.k_proj,8,64,8,128,0.313232421875,8.25390625,8388608 +57,self_attn.k_proj,8,128,8,128,0.37109375,8.126953125,8388608 +57,self_attn.o_proj,2,32,8,128,42.21875,20.0625,67108864 +57,self_attn.o_proj,2,64,8,128,49.125,18.03125,67108864 +57,self_attn.o_proj,2,128,8,128,55.34375,17.015625,67108864 +57,self_attn.o_proj,3,32,8,128,18.046875,28.0625,67108864 +57,self_attn.o_proj,3,64,8,128,21.015625,26.03125,67108864 +57,self_attn.o_proj,3,128,8,128,23.6875,25.015625,67108864 +57,self_attn.o_proj,4,32,8,128,8.421875,36.0625,67108864 +57,self_attn.o_proj,4,64,8,128,9.8125,34.03125,67108864 +57,self_attn.o_proj,4,128,8,128,11.0546875,33.015625,67108864 +57,self_attn.o_proj,8,32,8,128,0.509765625,68.0625,67108864 +57,self_attn.o_proj,8,64,8,128,0.59326171875,66.03125,67108864 +57,self_attn.o_proj,8,128,8,128,0.66650390625,65.015625,67108864 +57,mlp.gate_proj,2,32,8,128,89.5,70.21875,234881024 +57,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024 +57,mlp.gate_proj,2,128,8,128,118.0,59.5546875,234881024 +57,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024 +57,mlp.gate_proj,3,64,8,128,44.6875,91.109375,234881024 +57,mlp.gate_proj,3,128,8,128,50.5625,87.5546875,234881024 +57,mlp.gate_proj,4,32,8,128,17.859375,126.21875,234881024 +57,mlp.gate_proj,4,64,8,128,20.859375,119.109375,234881024 +57,mlp.gate_proj,4,128,8,128,23.578125,115.5546875,234881024 +57,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024 +57,mlp.gate_proj,8,64,8,128,1.26953125,231.109375,234881024 +57,mlp.gate_proj,8,128,8,128,1.4306640625,227.5546875,234881024 +57,mlp.up_proj,2,32,8,128,80.9375,70.21875,234881024 +57,mlp.up_proj,2,64,8,128,94.125,63.109375,234881024 +57,mlp.up_proj,2,128,8,128,106.125,59.5546875,234881024 +57,mlp.up_proj,3,32,8,128,34.59375,98.21875,234881024 +57,mlp.up_proj,3,64,8,128,40.3125,91.109375,234881024 +57,mlp.up_proj,3,128,8,128,45.40625,87.5546875,234881024 +57,mlp.up_proj,4,32,8,128,16.125,126.21875,234881024 +57,mlp.up_proj,4,64,8,128,18.8125,119.109375,234881024 +57,mlp.up_proj,4,128,8,128,21.1875,115.5546875,234881024 +57,mlp.up_proj,8,32,8,128,0.98095703125,238.21875,234881024 +57,mlp.up_proj,8,64,8,128,1.1435546875,231.109375,234881024 +57,mlp.up_proj,8,128,8,128,1.28515625,227.5546875,234881024 +57,mlp.down_proj,2,32,8,128,81.3125,70.21875,234881024 +57,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024 +57,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024 +57,mlp.down_proj,3,32,8,128,34.75,98.21875,234881024 +57,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024 +57,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024 +57,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +57,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024 +57,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +57,mlp.down_proj,8,32,8,128,0.98583984375,238.21875,234881024 +57,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +57,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +58,self_attn.q_proj,2,32,8,128,34.375,20.0625,67108864 +58,self_attn.q_proj,2,64,8,128,40.875,18.03125,67108864 +58,self_attn.q_proj,2,128,8,128,47.6875,17.015625,67108864 +58,self_attn.q_proj,3,32,8,128,14.8203125,28.0625,67108864 +58,self_attn.q_proj,3,64,8,128,17.765625,26.03125,67108864 +58,self_attn.q_proj,3,128,8,128,21.0625,25.015625,67108864 +58,self_attn.q_proj,4,32,8,128,6.9140625,36.0625,67108864 +58,self_attn.q_proj,4,64,8,128,8.296875,34.03125,67108864 +58,self_attn.q_proj,4,128,8,128,9.8359375,33.015625,67108864 +58,self_attn.q_proj,8,32,8,128,0.41845703125,68.0625,67108864 +58,self_attn.q_proj,8,64,8,128,0.501953125,66.03125,67108864 +58,self_attn.q_proj,8,128,8,128,0.59326171875,65.015625,67108864 +58,self_attn.v_proj,2,32,8,128,17.875,2.5078125,8388608 +58,self_attn.v_proj,2,64,8,128,20.890625,2.25390625,8388608 +58,self_attn.v_proj,2,128,8,128,23.625,2.126953125,8388608 +58,self_attn.v_proj,3,32,8,128,7.64453125,3.5078125,8388608 +58,self_attn.v_proj,3,64,8,128,8.9375,3.25390625,8388608 +58,self_attn.v_proj,3,128,8,128,10.1171875,3.126953125,8388608 +58,self_attn.v_proj,4,32,8,128,3.56640625,4.5078125,8388608 +58,self_attn.v_proj,4,64,8,128,4.171875,4.25390625,8388608 +58,self_attn.v_proj,4,128,8,128,4.72265625,4.126953125,8388608 +58,self_attn.v_proj,8,32,8,128,0.2161865234375,8.5078125,8388608 +58,self_attn.v_proj,8,64,8,128,0.25244140625,8.25390625,8388608 +58,self_attn.v_proj,8,128,8,128,0.28466796875,8.126953125,8388608 +58,self_attn.k_proj,2,32,8,128,20.4375,2.5078125,8388608 +58,self_attn.k_proj,2,64,8,128,24.609375,2.25390625,8388608 +58,self_attn.k_proj,2,128,8,128,28.109375,2.126953125,8388608 +58,self_attn.k_proj,3,32,8,128,9.0546875,3.5078125,8388608 +58,self_attn.k_proj,3,64,8,128,11.328125,3.25390625,8388608 +58,self_attn.k_proj,3,128,8,128,13.5703125,3.126953125,8388608 +58,self_attn.k_proj,4,32,8,128,4.2265625,4.5078125,8388608 +58,self_attn.k_proj,4,64,8,128,5.30078125,4.25390625,8388608 +58,self_attn.k_proj,4,128,8,128,6.36328125,4.126953125,8388608 +58,self_attn.k_proj,8,32,8,128,0.255615234375,8.5078125,8388608 +58,self_attn.k_proj,8,64,8,128,0.3203125,8.25390625,8388608 +58,self_attn.k_proj,8,128,8,128,0.38330078125,8.126953125,8388608 +58,self_attn.o_proj,2,32,8,128,40.78125,20.0625,67108864 +58,self_attn.o_proj,2,64,8,128,47.5,18.03125,67108864 +58,self_attn.o_proj,2,128,8,128,53.5625,17.015625,67108864 +58,self_attn.o_proj,3,32,8,128,17.453125,28.0625,67108864 +58,self_attn.o_proj,3,64,8,128,20.359375,26.03125,67108864 +58,self_attn.o_proj,3,128,8,128,23.0,25.015625,67108864 +58,self_attn.o_proj,4,32,8,128,8.140625,36.0625,67108864 +58,self_attn.o_proj,4,64,8,128,9.5078125,34.03125,67108864 +58,self_attn.o_proj,4,128,8,128,10.75,33.015625,67108864 +58,self_attn.o_proj,8,32,8,128,0.4931640625,68.0625,67108864 +58,self_attn.o_proj,8,64,8,128,0.5751953125,66.03125,67108864 +58,self_attn.o_proj,8,128,8,128,0.64794921875,65.015625,67108864 +58,mlp.gate_proj,2,32,8,128,89.5625,70.21875,234881024 +58,mlp.gate_proj,2,64,8,128,104.5625,63.109375,234881024 +58,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024 +58,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024 +58,mlp.gate_proj,3,64,8,128,44.71875,91.109375,234881024 +58,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024 +58,mlp.gate_proj,4,32,8,128,17.875,126.21875,234881024 +58,mlp.gate_proj,4,64,8,128,20.875,119.109375,234881024 +58,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +58,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024 +58,mlp.gate_proj,8,64,8,128,1.2705078125,231.109375,234881024 +58,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +58,mlp.up_proj,2,32,8,128,81.0625,70.21875,234881024 +58,mlp.up_proj,2,64,8,128,94.3125,63.109375,234881024 +58,mlp.up_proj,2,128,8,128,106.25,59.5546875,234881024 +58,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024 +58,mlp.up_proj,3,64,8,128,40.375,91.109375,234881024 +58,mlp.up_proj,3,128,8,128,45.46875,87.5546875,234881024 +58,mlp.up_proj,4,32,8,128,16.15625,126.21875,234881024 +58,mlp.up_proj,4,64,8,128,18.828125,119.109375,234881024 +58,mlp.up_proj,4,128,8,128,21.21875,115.5546875,234881024 +58,mlp.up_proj,8,32,8,128,0.982421875,238.21875,234881024 +58,mlp.up_proj,8,64,8,128,1.1455078125,231.109375,234881024 +58,mlp.up_proj,8,128,8,128,1.287109375,227.5546875,234881024 +58,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024 +58,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024 +58,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024 +58,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +58,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024 +58,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024 +58,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +58,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024 +58,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +58,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024 +58,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024 +58,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024 +59,self_attn.q_proj,2,32,8,128,33.15625,20.0625,67108864 +59,self_attn.q_proj,2,64,8,128,39.125,18.03125,67108864 +59,self_attn.q_proj,2,128,8,128,45.59375,17.015625,67108864 +59,self_attn.q_proj,3,32,8,128,14.21875,28.0625,67108864 +59,self_attn.q_proj,3,64,8,128,16.84375,26.03125,67108864 +59,self_attn.q_proj,3,128,8,128,19.796875,25.015625,67108864 +59,self_attn.q_proj,4,32,8,128,6.63671875,36.0625,67108864 +59,self_attn.q_proj,4,64,8,128,7.8671875,34.03125,67108864 +59,self_attn.q_proj,4,128,8,128,9.25,33.015625,67108864 +59,self_attn.q_proj,8,32,8,128,0.401611328125,68.0625,67108864 +59,self_attn.q_proj,8,64,8,128,0.475830078125,66.03125,67108864 +59,self_attn.q_proj,8,128,8,128,0.55810546875,65.015625,67108864 +59,self_attn.v_proj,2,32,8,128,18.171875,2.5078125,8388608 +59,self_attn.v_proj,2,64,8,128,21.234375,2.25390625,8388608 +59,self_attn.v_proj,2,128,8,128,24.0,2.126953125,8388608 +59,self_attn.v_proj,3,32,8,128,7.765625,3.5078125,8388608 +59,self_attn.v_proj,3,64,8,128,9.078125,3.25390625,8388608 +59,self_attn.v_proj,3,128,8,128,10.2734375,3.126953125,8388608 +59,self_attn.v_proj,4,32,8,128,3.623046875,4.5078125,8388608 +59,self_attn.v_proj,4,64,8,128,4.23828125,4.25390625,8388608 +59,self_attn.v_proj,4,128,8,128,4.79296875,4.126953125,8388608 +59,self_attn.v_proj,8,32,8,128,0.219482421875,8.5078125,8388608 +59,self_attn.v_proj,8,64,8,128,0.25634765625,8.25390625,8388608 +59,self_attn.v_proj,8,128,8,128,0.288818359375,8.126953125,8388608 +59,self_attn.k_proj,2,32,8,128,20.1875,2.5078125,8388608 +59,self_attn.k_proj,2,64,8,128,24.40625,2.25390625,8388608 +59,self_attn.k_proj,2,128,8,128,28.0,2.126953125,8388608 +59,self_attn.k_proj,3,32,8,128,8.8515625,3.5078125,8388608 +59,self_attn.k_proj,3,64,8,128,11.03125,3.25390625,8388608 +59,self_attn.k_proj,3,128,8,128,13.1953125,3.126953125,8388608 +59,self_attn.k_proj,4,32,8,128,4.1328125,4.5078125,8388608 +59,self_attn.k_proj,4,64,8,128,5.15234375,4.25390625,8388608 +59,self_attn.k_proj,4,128,8,128,6.171875,4.126953125,8388608 +59,self_attn.k_proj,8,32,8,128,0.25,8.5078125,8388608 +59,self_attn.k_proj,8,64,8,128,0.3115234375,8.25390625,8388608 +59,self_attn.k_proj,8,128,8,128,0.371826171875,8.126953125,8388608 +59,self_attn.o_proj,2,32,8,128,40.5625,20.0625,67108864 +59,self_attn.o_proj,2,64,8,128,47.21875,18.03125,67108864 +59,self_attn.o_proj,2,128,8,128,53.25,17.015625,67108864 +59,self_attn.o_proj,3,32,8,128,17.34375,28.0625,67108864 +59,self_attn.o_proj,3,64,8,128,20.234375,26.03125,67108864 +59,self_attn.o_proj,3,128,8,128,22.84375,25.015625,67108864 +59,self_attn.o_proj,4,32,8,128,8.09375,36.0625,67108864 +59,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864 +59,self_attn.o_proj,4,128,8,128,10.671875,33.015625,67108864 +59,self_attn.o_proj,8,32,8,128,0.489990234375,68.0625,67108864 +59,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864 +59,self_attn.o_proj,8,128,8,128,0.642578125,65.015625,67108864 +59,mlp.gate_proj,2,32,8,128,89.5,70.21875,234881024 +59,mlp.gate_proj,2,64,8,128,104.625,63.109375,234881024 +59,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024 +59,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024 +59,mlp.gate_proj,3,64,8,128,44.71875,91.109375,234881024 +59,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024 +59,mlp.gate_proj,4,32,8,128,17.859375,126.21875,234881024 +59,mlp.gate_proj,4,64,8,128,20.875,119.109375,234881024 +59,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +59,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024 +59,mlp.gate_proj,8,64,8,128,1.2705078125,231.109375,234881024 +59,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +59,mlp.up_proj,2,32,8,128,81.1875,70.21875,234881024 +59,mlp.up_proj,2,64,8,128,94.5,63.109375,234881024 +59,mlp.up_proj,2,128,8,128,106.5,59.5546875,234881024 +59,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024 +59,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024 +59,mlp.up_proj,3,128,8,128,45.53125,87.5546875,234881024 +59,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024 +59,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024 +59,mlp.up_proj,4,128,8,128,21.265625,115.5546875,234881024 +59,mlp.up_proj,8,32,8,128,0.984375,238.21875,234881024 +59,mlp.up_proj,8,64,8,128,1.1474609375,231.109375,234881024 +59,mlp.up_proj,8,128,8,128,1.2900390625,227.5546875,234881024 +59,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024 +59,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024 +59,mlp.down_proj,2,128,8,128,106.6875,59.5546875,234881024 +59,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024 +59,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024 +59,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024 +59,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024 +59,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024 +59,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024 +59,mlp.down_proj,8,32,8,128,0.98681640625,238.21875,234881024 +59,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024 +59,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024 +60,self_attn.q_proj,2,32,8,128,26.671875,20.0625,67108864 +60,self_attn.q_proj,2,64,8,128,31.390625,18.03125,67108864 +60,self_attn.q_proj,2,128,8,128,35.96875,17.015625,67108864 +60,self_attn.q_proj,3,32,8,128,11.4609375,28.0625,67108864 +60,self_attn.q_proj,3,64,8,128,13.5859375,26.03125,67108864 +60,self_attn.q_proj,3,128,8,128,15.7265625,25.015625,67108864 +60,self_attn.q_proj,4,32,8,128,5.34765625,36.0625,67108864 +60,self_attn.q_proj,4,64,8,128,6.34375,34.03125,67108864 +60,self_attn.q_proj,4,128,8,128,7.34375,33.015625,67108864 +60,self_attn.q_proj,8,32,8,128,0.323974609375,68.0625,67108864 +60,self_attn.q_proj,8,64,8,128,0.383544921875,66.03125,67108864 +60,self_attn.q_proj,8,128,8,128,0.44287109375,65.015625,67108864 +60,self_attn.v_proj,2,32,8,128,18.1875,2.5078125,8388608 +60,self_attn.v_proj,2,64,8,128,21.25,2.25390625,8388608 +60,self_attn.v_proj,2,128,8,128,24.046875,2.126953125,8388608 +60,self_attn.v_proj,3,32,8,128,7.7734375,3.5078125,8388608 +60,self_attn.v_proj,3,64,8,128,9.09375,3.25390625,8388608 +60,self_attn.v_proj,3,128,8,128,10.28125,3.126953125,8388608 +60,self_attn.v_proj,4,32,8,128,3.626953125,4.5078125,8388608 +60,self_attn.v_proj,4,64,8,128,4.2421875,4.25390625,8388608 +60,self_attn.v_proj,4,128,8,128,4.80078125,4.126953125,8388608 +60,self_attn.v_proj,8,32,8,128,0.2197265625,8.5078125,8388608 +60,self_attn.v_proj,8,64,8,128,0.256591796875,8.25390625,8388608 +60,self_attn.v_proj,8,128,8,128,0.289306640625,8.126953125,8388608 +60,self_attn.k_proj,2,32,8,128,16.640625,2.5078125,8388608 +60,self_attn.k_proj,2,64,8,128,19.96875,2.25390625,8388608 +60,self_attn.k_proj,2,128,8,128,22.671875,2.126953125,8388608 +60,self_attn.k_proj,3,32,8,128,7.30078125,3.5078125,8388608 +60,self_attn.k_proj,3,64,8,128,8.9921875,3.25390625,8388608 +60,self_attn.k_proj,3,128,8,128,10.5234375,3.126953125,8388608 +60,self_attn.k_proj,4,32,8,128,3.416015625,4.5078125,8388608 +60,self_attn.k_proj,4,64,8,128,4.22265625,4.25390625,8388608 +60,self_attn.k_proj,4,128,8,128,4.96875,4.126953125,8388608 +60,self_attn.k_proj,8,32,8,128,0.206787109375,8.5078125,8388608 +60,self_attn.k_proj,8,64,8,128,0.25537109375,8.25390625,8388608 +60,self_attn.k_proj,8,128,8,128,0.2998046875,8.126953125,8388608 +60,self_attn.o_proj,2,32,8,128,39.34375,20.0625,67108864 +60,self_attn.o_proj,2,64,8,128,45.78125,18.03125,67108864 +60,self_attn.o_proj,2,128,8,128,51.53125,17.015625,67108864 +60,self_attn.o_proj,3,32,8,128,16.8125,28.0625,67108864 +60,self_attn.o_proj,3,64,8,128,19.59375,26.03125,67108864 +60,self_attn.o_proj,3,128,8,128,22.078125,25.015625,67108864 +60,self_attn.o_proj,4,32,8,128,7.84765625,36.0625,67108864 +60,self_attn.o_proj,4,64,8,128,9.140625,34.03125,67108864 +60,self_attn.o_proj,4,128,8,128,10.3046875,33.015625,67108864 +60,self_attn.o_proj,8,32,8,128,0.47509765625,68.0625,67108864 +60,self_attn.o_proj,8,64,8,128,0.55322265625,66.03125,67108864 +60,self_attn.o_proj,8,128,8,128,0.62109375,65.015625,67108864 +60,mlp.gate_proj,2,32,8,128,89.4375,70.21875,234881024 +60,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024 +60,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024 +60,mlp.gate_proj,3,32,8,128,38.28125,98.21875,234881024 +60,mlp.gate_proj,3,64,8,128,44.6875,91.109375,234881024 +60,mlp.gate_proj,3,128,8,128,50.5625,87.5546875,234881024 +60,mlp.gate_proj,4,32,8,128,17.84375,126.21875,234881024 +60,mlp.gate_proj,4,64,8,128,20.859375,119.109375,234881024 +60,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +60,mlp.gate_proj,8,32,8,128,1.0859375,238.21875,234881024 +60,mlp.gate_proj,8,64,8,128,1.26953125,231.109375,234881024 +60,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +60,mlp.up_proj,2,32,8,128,81.375,70.21875,234881024 +60,mlp.up_proj,2,64,8,128,94.6875,63.109375,234881024 +60,mlp.up_proj,2,128,8,128,106.75,59.5546875,234881024 +60,mlp.up_proj,3,32,8,128,34.78125,98.21875,234881024 +60,mlp.up_proj,3,64,8,128,40.53125,91.109375,234881024 +60,mlp.up_proj,3,128,8,128,45.65625,87.5546875,234881024 +60,mlp.up_proj,4,32,8,128,16.21875,126.21875,234881024 +60,mlp.up_proj,4,64,8,128,18.921875,119.109375,234881024 +60,mlp.up_proj,4,128,8,128,21.3125,115.5546875,234881024 +60,mlp.up_proj,8,32,8,128,0.98681640625,238.21875,234881024 +60,mlp.up_proj,8,64,8,128,1.150390625,231.109375,234881024 +60,mlp.up_proj,8,128,8,128,1.29296875,227.5546875,234881024 +60,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024 +60,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024 +60,mlp.down_proj,2,128,8,128,106.8125,59.5546875,234881024 +60,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024 +60,mlp.down_proj,3,64,8,128,40.5625,91.109375,234881024 +60,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024 +60,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024 +60,mlp.down_proj,4,64,8,128,18.9375,119.109375,234881024 +60,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024 +60,mlp.down_proj,8,32,8,128,0.98779296875,238.21875,234881024 +60,mlp.down_proj,8,64,8,128,1.1513671875,231.109375,234881024 +60,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024 +61,self_attn.q_proj,2,32,8,128,31.328125,20.0625,67108864 +61,self_attn.q_proj,2,64,8,128,37.21875,18.03125,67108864 +61,self_attn.q_proj,2,128,8,128,43.0625,17.015625,67108864 +61,self_attn.q_proj,3,32,8,128,13.421875,28.0625,67108864 +61,self_attn.q_proj,3,64,8,128,16.015625,26.03125,67108864 +61,self_attn.q_proj,3,128,8,128,18.640625,25.015625,67108864 +61,self_attn.q_proj,4,32,8,128,6.265625,36.0625,67108864 +61,self_attn.q_proj,4,64,8,128,7.47265625,34.03125,67108864 +61,self_attn.q_proj,4,128,8,128,8.6953125,33.015625,67108864 +61,self_attn.q_proj,8,32,8,128,0.379150390625,68.0625,67108864 +61,self_attn.q_proj,8,64,8,128,0.451904296875,66.03125,67108864 +61,self_attn.q_proj,8,128,8,128,0.52490234375,65.015625,67108864 +61,self_attn.v_proj,2,32,8,128,18.53125,2.5078125,8388608 +61,self_attn.v_proj,2,64,8,128,21.640625,2.25390625,8388608 +61,self_attn.v_proj,2,128,8,128,24.4375,2.126953125,8388608 +61,self_attn.v_proj,3,32,8,128,7.921875,3.5078125,8388608 +61,self_attn.v_proj,3,64,8,128,9.2578125,3.25390625,8388608 +61,self_attn.v_proj,3,128,8,128,10.4609375,3.126953125,8388608 +61,self_attn.v_proj,4,32,8,128,3.697265625,4.5078125,8388608 +61,self_attn.v_proj,4,64,8,128,4.31640625,4.25390625,8388608 +61,self_attn.v_proj,4,128,8,128,4.87890625,4.126953125,8388608 +61,self_attn.v_proj,8,32,8,128,0.223876953125,8.5078125,8388608 +61,self_attn.v_proj,8,64,8,128,0.26123046875,8.25390625,8388608 +61,self_attn.v_proj,8,128,8,128,0.294189453125,8.126953125,8388608 +61,self_attn.k_proj,2,32,8,128,19.453125,2.5078125,8388608 +61,self_attn.k_proj,2,64,8,128,23.359375,2.25390625,8388608 +61,self_attn.k_proj,2,128,8,128,26.578125,2.126953125,8388608 +61,self_attn.k_proj,3,32,8,128,8.546875,3.5078125,8388608 +61,self_attn.k_proj,3,64,8,128,10.6015625,3.25390625,8388608 +61,self_attn.k_proj,3,128,8,128,12.5234375,3.126953125,8388608 +61,self_attn.k_proj,4,32,8,128,3.990234375,4.5078125,8388608 +61,self_attn.k_proj,4,64,8,128,4.953125,4.25390625,8388608 +61,self_attn.k_proj,4,128,8,128,5.8671875,4.126953125,8388608 +61,self_attn.k_proj,8,32,8,128,0.241455078125,8.5078125,8388608 +61,self_attn.k_proj,8,64,8,128,0.299560546875,8.25390625,8388608 +61,self_attn.k_proj,8,128,8,128,0.353759765625,8.126953125,8388608 +61,self_attn.o_proj,2,32,8,128,40.96875,20.0625,67108864 +61,self_attn.o_proj,2,64,8,128,47.6875,18.03125,67108864 +61,self_attn.o_proj,2,128,8,128,53.6875,17.015625,67108864 +61,self_attn.o_proj,3,32,8,128,17.515625,28.0625,67108864 +61,self_attn.o_proj,3,64,8,128,20.390625,26.03125,67108864 +61,self_attn.o_proj,3,128,8,128,22.984375,25.015625,67108864 +61,self_attn.o_proj,4,32,8,128,8.171875,36.0625,67108864 +61,self_attn.o_proj,4,64,8,128,9.5234375,34.03125,67108864 +61,self_attn.o_proj,4,128,8,128,10.7265625,33.015625,67108864 +61,self_attn.o_proj,8,32,8,128,0.494873046875,68.0625,67108864 +61,self_attn.o_proj,8,64,8,128,0.57568359375,66.03125,67108864 +61,self_attn.o_proj,8,128,8,128,0.646484375,65.015625,67108864 +61,mlp.gate_proj,2,32,8,128,89.375,70.21875,234881024 +61,mlp.gate_proj,2,64,8,128,104.4375,63.109375,234881024 +61,mlp.gate_proj,2,128,8,128,118.0625,59.5546875,234881024 +61,mlp.gate_proj,3,32,8,128,38.25,98.21875,234881024 +61,mlp.gate_proj,3,64,8,128,44.65625,91.109375,234881024 +61,mlp.gate_proj,3,128,8,128,50.5625,87.5546875,234881024 +61,mlp.gate_proj,4,32,8,128,17.828125,126.21875,234881024 +61,mlp.gate_proj,4,64,8,128,20.84375,119.109375,234881024 +61,mlp.gate_proj,4,128,8,128,23.578125,115.5546875,234881024 +61,mlp.gate_proj,8,32,8,128,1.0849609375,238.21875,234881024 +61,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024 +61,mlp.gate_proj,8,128,8,128,1.4306640625,227.5546875,234881024 +61,mlp.up_proj,2,32,8,128,81.5625,70.21875,234881024 +61,mlp.up_proj,2,64,8,128,94.9375,63.109375,234881024 +61,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024 +61,mlp.up_proj,3,32,8,128,34.875,98.21875,234881024 +61,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024 +61,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024 +61,mlp.up_proj,4,32,8,128,16.265625,126.21875,234881024 +61,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024 +61,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024 +61,mlp.up_proj,8,32,8,128,0.9892578125,238.21875,234881024 +61,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024 +61,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +61,mlp.down_proj,2,32,8,128,81.625,70.21875,234881024 +61,mlp.down_proj,2,64,8,128,94.9375,63.109375,234881024 +61,mlp.down_proj,2,128,8,128,107.0,59.5546875,234881024 +61,mlp.down_proj,3,32,8,128,34.875,98.21875,234881024 +61,mlp.down_proj,3,64,8,128,40.625,91.109375,234881024 +61,mlp.down_proj,3,128,8,128,45.71875,87.5546875,234881024 +61,mlp.down_proj,4,32,8,128,16.265625,126.21875,234881024 +61,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024 +61,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024 +61,mlp.down_proj,8,32,8,128,0.9892578125,238.21875,234881024 +61,mlp.down_proj,8,64,8,128,1.1533203125,231.109375,234881024 +61,mlp.down_proj,8,128,8,128,1.294921875,227.5546875,234881024 +62,self_attn.q_proj,2,32,8,128,32.375,20.0625,67108864 +62,self_attn.q_proj,2,64,8,128,38.5,18.03125,67108864 +62,self_attn.q_proj,2,128,8,128,44.875,17.015625,67108864 +62,self_attn.q_proj,3,32,8,128,13.9140625,28.0625,67108864 +62,self_attn.q_proj,3,64,8,128,16.640625,26.03125,67108864 +62,self_attn.q_proj,3,128,8,128,19.640625,25.015625,67108864 +62,self_attn.q_proj,4,32,8,128,6.48828125,36.0625,67108864 +62,self_attn.q_proj,4,64,8,128,7.765625,34.03125,67108864 +62,self_attn.q_proj,4,128,8,128,9.1640625,33.015625,67108864 +62,self_attn.q_proj,8,32,8,128,0.392822265625,68.0625,67108864 +62,self_attn.q_proj,8,64,8,128,0.4697265625,66.03125,67108864 +62,self_attn.q_proj,8,128,8,128,0.552734375,65.015625,67108864 +62,self_attn.v_proj,2,32,8,128,18.25,2.5078125,8388608 +62,self_attn.v_proj,2,64,8,128,21.28125,2.25390625,8388608 +62,self_attn.v_proj,2,128,8,128,24.015625,2.126953125,8388608 +62,self_attn.v_proj,3,32,8,128,7.8046875,3.5078125,8388608 +62,self_attn.v_proj,3,64,8,128,9.1015625,3.25390625,8388608 +62,self_attn.v_proj,3,128,8,128,10.28125,3.126953125,8388608 +62,self_attn.v_proj,4,32,8,128,3.642578125,4.5078125,8388608 +62,self_attn.v_proj,4,64,8,128,4.25,4.25390625,8388608 +62,self_attn.v_proj,4,128,8,128,4.80078125,4.126953125,8388608 +62,self_attn.v_proj,8,32,8,128,0.2205810546875,8.5078125,8388608 +62,self_attn.v_proj,8,64,8,128,0.257080078125,8.25390625,8388608 +62,self_attn.v_proj,8,128,8,128,0.289306640625,8.126953125,8388608 +62,self_attn.k_proj,2,32,8,128,20.046875,2.5078125,8388608 +62,self_attn.k_proj,2,64,8,128,24.140625,2.25390625,8388608 +62,self_attn.k_proj,2,128,8,128,27.5625,2.126953125,8388608 +62,self_attn.k_proj,3,32,8,128,8.8828125,3.5078125,8388608 +62,self_attn.k_proj,3,64,8,128,11.1484375,3.25390625,8388608 +62,self_attn.k_proj,3,128,8,128,13.3515625,3.126953125,8388608 +62,self_attn.k_proj,4,32,8,128,4.14453125,4.5078125,8388608 +62,self_attn.k_proj,4,64,8,128,5.2109375,4.25390625,8388608 +62,self_attn.k_proj,4,128,8,128,6.265625,4.126953125,8388608 +62,self_attn.k_proj,8,32,8,128,0.250732421875,8.5078125,8388608 +62,self_attn.k_proj,8,64,8,128,0.315185546875,8.25390625,8388608 +62,self_attn.k_proj,8,128,8,128,0.377685546875,8.126953125,8388608 +62,self_attn.o_proj,2,32,8,128,40.40625,20.0625,67108864 +62,self_attn.o_proj,2,64,8,128,47.03125,18.03125,67108864 +62,self_attn.o_proj,2,128,8,128,52.96875,17.015625,67108864 +62,self_attn.o_proj,3,32,8,128,17.265625,28.0625,67108864 +62,self_attn.o_proj,3,64,8,128,20.125,26.03125,67108864 +62,self_attn.o_proj,3,128,8,128,22.671875,25.015625,67108864 +62,self_attn.o_proj,4,32,8,128,8.0546875,36.0625,67108864 +62,self_attn.o_proj,4,64,8,128,9.3828125,34.03125,67108864 +62,self_attn.o_proj,4,128,8,128,10.578125,33.015625,67108864 +62,self_attn.o_proj,8,32,8,128,0.48779296875,68.0625,67108864 +62,self_attn.o_proj,8,64,8,128,0.56787109375,66.03125,67108864 +62,self_attn.o_proj,8,128,8,128,0.6376953125,65.015625,67108864 +62,mlp.gate_proj,2,32,8,128,89.3125,70.21875,234881024 +62,mlp.gate_proj,2,64,8,128,104.375,63.109375,234881024 +62,mlp.gate_proj,2,128,8,128,118.0,59.5546875,234881024 +62,mlp.gate_proj,3,32,8,128,38.21875,98.21875,234881024 +62,mlp.gate_proj,3,64,8,128,44.625,91.109375,234881024 +62,mlp.gate_proj,3,128,8,128,50.53125,87.5546875,234881024 +62,mlp.gate_proj,4,32,8,128,17.8125,126.21875,234881024 +62,mlp.gate_proj,4,64,8,128,20.828125,119.109375,234881024 +62,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024 +62,mlp.gate_proj,8,32,8,128,1.083984375,238.21875,234881024 +62,mlp.gate_proj,8,64,8,128,1.267578125,231.109375,234881024 +62,mlp.gate_proj,8,128,8,128,1.4296875,227.5546875,234881024 +62,mlp.up_proj,2,32,8,128,81.75,70.21875,234881024 +62,mlp.up_proj,2,64,8,128,95.125,63.109375,234881024 +62,mlp.up_proj,2,128,8,128,107.25,59.5546875,234881024 +62,mlp.up_proj,3,32,8,128,34.9375,98.21875,234881024 +62,mlp.up_proj,3,64,8,128,40.71875,91.109375,234881024 +62,mlp.up_proj,3,128,8,128,45.875,87.5546875,234881024 +62,mlp.up_proj,4,32,8,128,16.296875,126.21875,234881024 +62,mlp.up_proj,4,64,8,128,19.0,119.109375,234881024 +62,mlp.up_proj,4,128,8,128,21.40625,115.5546875,234881024 +62,mlp.up_proj,8,32,8,128,0.9912109375,238.21875,234881024 +62,mlp.up_proj,8,64,8,128,1.15625,231.109375,234881024 +62,mlp.up_proj,8,128,8,128,1.298828125,227.5546875,234881024 +62,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024 +62,mlp.down_proj,2,64,8,128,95.0,63.109375,234881024 +62,mlp.down_proj,2,128,8,128,107.0625,59.5546875,234881024 +62,mlp.down_proj,3,32,8,128,34.90625,98.21875,234881024 +62,mlp.down_proj,3,64,8,128,40.65625,91.109375,234881024 +62,mlp.down_proj,3,128,8,128,45.78125,87.5546875,234881024 +62,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024 +62,mlp.down_proj,4,64,8,128,18.96875,119.109375,234881024 +62,mlp.down_proj,4,128,8,128,21.359375,115.5546875,234881024 +62,mlp.down_proj,8,32,8,128,0.990234375,238.21875,234881024 +62,mlp.down_proj,8,64,8,128,1.154296875,231.109375,234881024 +62,mlp.down_proj,8,128,8,128,1.2958984375,227.5546875,234881024 +63,self_attn.q_proj,2,32,8,128,31.921875,20.0625,67108864 +63,self_attn.q_proj,2,64,8,128,37.90625,18.03125,67108864 +63,self_attn.q_proj,2,128,8,128,43.84375,17.015625,67108864 +63,self_attn.q_proj,3,32,8,128,13.7421875,28.0625,67108864 +63,self_attn.q_proj,3,64,8,128,16.421875,26.03125,67108864 +63,self_attn.q_proj,3,128,8,128,19.25,25.015625,67108864 +63,self_attn.q_proj,4,32,8,128,6.4140625,36.0625,67108864 +63,self_attn.q_proj,4,64,8,128,7.671875,34.03125,67108864 +63,self_attn.q_proj,4,128,8,128,9.0,33.015625,67108864 +63,self_attn.q_proj,8,32,8,128,0.38818359375,68.0625,67108864 +63,self_attn.q_proj,8,64,8,128,0.464111328125,66.03125,67108864 +63,self_attn.q_proj,8,128,8,128,0.54296875,65.015625,67108864 +63,self_attn.v_proj,2,32,8,128,18.125,2.5078125,8388608 +63,self_attn.v_proj,2,64,8,128,21.28125,2.25390625,8388608 +63,self_attn.v_proj,2,128,8,128,24.203125,2.126953125,8388608 +63,self_attn.v_proj,3,32,8,128,7.75,3.5078125,8388608 +63,self_attn.v_proj,3,64,8,128,9.109375,3.25390625,8388608 +63,self_attn.v_proj,3,128,8,128,10.359375,3.126953125,8388608 +63,self_attn.v_proj,4,32,8,128,3.615234375,4.5078125,8388608 +63,self_attn.v_proj,4,64,8,128,4.25,4.25390625,8388608 +63,self_attn.v_proj,4,128,8,128,4.83203125,4.126953125,8388608 +63,self_attn.v_proj,8,32,8,128,0.218994140625,8.5078125,8388608 +63,self_attn.v_proj,8,64,8,128,0.257080078125,8.25390625,8388608 +63,self_attn.v_proj,8,128,8,128,0.291259765625,8.126953125,8388608 +63,self_attn.k_proj,2,32,8,128,19.3125,2.5078125,8388608 +63,self_attn.k_proj,2,64,8,128,23.109375,2.25390625,8388608 +63,self_attn.k_proj,2,128,8,128,26.109375,2.126953125,8388608 +63,self_attn.k_proj,3,32,8,128,8.5859375,3.5078125,8388608 +63,self_attn.k_proj,3,64,8,128,10.640625,3.25390625,8388608 +63,self_attn.k_proj,3,128,8,128,12.5390625,3.126953125,8388608 +63,self_attn.k_proj,4,32,8,128,4.015625,4.5078125,8388608 +63,self_attn.k_proj,4,64,8,128,4.9921875,4.25390625,8388608 +63,self_attn.k_proj,4,128,8,128,5.9140625,4.126953125,8388608 +63,self_attn.k_proj,8,32,8,128,0.2431640625,8.5078125,8388608 +63,self_attn.k_proj,8,64,8,128,0.3017578125,8.25390625,8388608 +63,self_attn.k_proj,8,128,8,128,0.356689453125,8.126953125,8388608 +63,self_attn.o_proj,2,32,8,128,38.96875,20.0625,67108864 +63,self_attn.o_proj,2,64,8,128,45.34375,18.03125,67108864 +63,self_attn.o_proj,2,128,8,128,51.125,17.015625,67108864 +63,self_attn.o_proj,3,32,8,128,16.671875,28.0625,67108864 +63,self_attn.o_proj,3,64,8,128,19.4375,26.03125,67108864 +63,self_attn.o_proj,3,128,8,128,21.9375,25.015625,67108864 +63,self_attn.o_proj,4,32,8,128,7.77734375,36.0625,67108864 +63,self_attn.o_proj,4,64,8,128,9.0703125,34.03125,67108864 +63,self_attn.o_proj,4,128,8,128,10.25,33.015625,67108864 +63,self_attn.o_proj,8,32,8,128,0.47119140625,68.0625,67108864 +63,self_attn.o_proj,8,64,8,128,0.54931640625,66.03125,67108864 +63,self_attn.o_proj,8,128,8,128,0.6181640625,65.015625,67108864 +63,mlp.gate_proj,2,32,8,128,89.3125,70.21875,234881024 +63,mlp.gate_proj,2,64,8,128,104.375,63.109375,234881024 +63,mlp.gate_proj,2,128,8,128,118.0625,59.5546875,234881024 +63,mlp.gate_proj,3,32,8,128,38.21875,98.21875,234881024 +63,mlp.gate_proj,3,64,8,128,44.625,91.109375,234881024 +63,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024 +63,mlp.gate_proj,4,32,8,128,17.8125,126.21875,234881024 +63,mlp.gate_proj,4,64,8,128,20.84375,119.109375,234881024 +63,mlp.gate_proj,4,128,8,128,23.59375,115.5546875,234881024 +63,mlp.gate_proj,8,32,8,128,1.083984375,238.21875,234881024 +63,mlp.gate_proj,8,64,8,128,1.267578125,231.109375,234881024 +63,mlp.gate_proj,8,128,8,128,1.4306640625,227.5546875,234881024 +63,mlp.up_proj,2,32,8,128,81.875,70.21875,234881024 +63,mlp.up_proj,2,64,8,128,95.3125,63.109375,234881024 +63,mlp.up_proj,2,128,8,128,107.4375,59.5546875,234881024 +63,mlp.up_proj,3,32,8,128,35.0,98.21875,234881024 +63,mlp.up_proj,3,64,8,128,40.78125,91.109375,234881024 +63,mlp.up_proj,3,128,8,128,45.9375,87.5546875,234881024 +63,mlp.up_proj,4,32,8,128,16.328125,126.21875,234881024 +63,mlp.up_proj,4,64,8,128,19.03125,119.109375,234881024 +63,mlp.up_proj,4,128,8,128,21.4375,115.5546875,234881024 +63,mlp.up_proj,8,32,8,128,0.99267578125,238.21875,234881024 +63,mlp.up_proj,8,64,8,128,1.158203125,231.109375,234881024 +63,mlp.up_proj,8,128,8,128,1.30078125,227.5546875,234881024 +63,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024 +63,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024 +63,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024 +63,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024 +63,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024 +63,mlp.down_proj,3,128,8,128,45.78125,87.5546875,234881024 +63,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024 +63,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024 +63,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024 +63,mlp.down_proj,8,32,8,128,0.99072265625,238.21875,234881024 +63,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024 +63,mlp.down_proj,8,128,8,128,1.296875,227.5546875,234881024 +64,self_attn.q_proj,2,32,8,128,34.875,20.0625,67108864 +64,self_attn.q_proj,2,64,8,128,41.46875,18.03125,67108864 +64,self_attn.q_proj,2,128,8,128,48.0625,17.015625,67108864 +64,self_attn.q_proj,3,32,8,128,14.9765625,28.0625,67108864 +64,self_attn.q_proj,3,64,8,128,17.90625,26.03125,67108864 +64,self_attn.q_proj,3,128,8,128,20.984375,25.015625,67108864 +64,self_attn.q_proj,4,32,8,128,6.98828125,36.0625,67108864 +64,self_attn.q_proj,4,64,8,128,8.359375,34.03125,67108864 +64,self_attn.q_proj,4,128,8,128,9.796875,33.015625,67108864 +64,self_attn.q_proj,8,32,8,128,0.4228515625,68.0625,67108864 +64,self_attn.q_proj,8,64,8,128,0.505859375,66.03125,67108864 +64,self_attn.q_proj,8,128,8,128,0.5908203125,65.015625,67108864 +64,self_attn.v_proj,2,32,8,128,18.640625,2.5078125,8388608 +64,self_attn.v_proj,2,64,8,128,21.71875,2.25390625,8388608 +64,self_attn.v_proj,2,128,8,128,24.484375,2.126953125,8388608 +64,self_attn.v_proj,3,32,8,128,7.96875,3.5078125,8388608 +64,self_attn.v_proj,3,64,8,128,9.2890625,3.25390625,8388608 +64,self_attn.v_proj,3,128,8,128,10.46875,3.126953125,8388608 +64,self_attn.v_proj,4,32,8,128,3.71875,4.5078125,8388608 +64,self_attn.v_proj,4,64,8,128,4.33203125,4.25390625,8388608 +64,self_attn.v_proj,4,128,8,128,4.88671875,4.126953125,8388608 +64,self_attn.v_proj,8,32,8,128,0.225341796875,8.5078125,8388608 +64,self_attn.v_proj,8,64,8,128,0.26220703125,8.25390625,8388608 +64,self_attn.v_proj,8,128,8,128,0.294677734375,8.126953125,8388608 +64,self_attn.k_proj,2,32,8,128,21.859375,2.5078125,8388608 +64,self_attn.k_proj,2,64,8,128,26.1875,2.25390625,8388608 +64,self_attn.k_proj,2,128,8,128,29.6875,2.126953125,8388608 +64,self_attn.k_proj,3,32,8,128,9.6171875,3.5078125,8388608 +64,self_attn.k_proj,3,64,8,128,11.859375,3.25390625,8388608 +64,self_attn.k_proj,3,128,8,128,14.015625,3.126953125,8388608 +64,self_attn.k_proj,4,32,8,128,4.48828125,4.5078125,8388608 +64,self_attn.k_proj,4,64,8,128,5.54296875,4.25390625,8388608 +64,self_attn.k_proj,4,128,8,128,6.55078125,4.126953125,8388608 +64,self_attn.k_proj,8,32,8,128,0.271728515625,8.5078125,8388608 +64,self_attn.k_proj,8,64,8,128,0.3349609375,8.25390625,8388608 +64,self_attn.k_proj,8,128,8,128,0.39501953125,8.126953125,8388608 +64,self_attn.o_proj,2,32,8,128,41.84375,20.0625,67108864 +64,self_attn.o_proj,2,64,8,128,48.6875,18.03125,67108864 +64,self_attn.o_proj,2,128,8,128,54.84375,17.015625,67108864 +64,self_attn.o_proj,3,32,8,128,17.890625,28.0625,67108864 +64,self_attn.o_proj,3,64,8,128,20.828125,26.03125,67108864 +64,self_attn.o_proj,3,128,8,128,23.484375,25.015625,67108864 +64,self_attn.o_proj,4,32,8,128,8.34375,36.0625,67108864 +64,self_attn.o_proj,4,64,8,128,9.71875,34.03125,67108864 +64,self_attn.o_proj,4,128,8,128,10.9609375,33.015625,67108864 +64,self_attn.o_proj,8,32,8,128,0.50537109375,68.0625,67108864 +64,self_attn.o_proj,8,64,8,128,0.58837890625,66.03125,67108864 +64,self_attn.o_proj,8,128,8,128,0.66064453125,65.015625,67108864 +64,mlp.gate_proj,2,32,8,128,89.25,70.21875,234881024 +64,mlp.gate_proj,2,64,8,128,104.4375,63.109375,234881024 +64,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024 +64,mlp.gate_proj,3,32,8,128,38.21875,98.21875,234881024 +64,mlp.gate_proj,3,64,8,128,44.625,91.109375,234881024 +64,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024 +64,mlp.gate_proj,4,32,8,128,17.8125,126.21875,234881024 +64,mlp.gate_proj,4,64,8,128,20.828125,119.109375,234881024 +64,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024 +64,mlp.gate_proj,8,32,8,128,1.083984375,238.21875,234881024 +64,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024 +64,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024 +64,mlp.up_proj,2,32,8,128,82.0,70.21875,234881024 +64,mlp.up_proj,2,64,8,128,95.4375,63.109375,234881024 +64,mlp.up_proj,2,128,8,128,107.625,59.5546875,234881024 +64,mlp.up_proj,3,32,8,128,35.0625,98.21875,234881024 +64,mlp.up_proj,3,64,8,128,40.875,91.109375,234881024 +64,mlp.up_proj,3,128,8,128,46.03125,87.5546875,234881024 +64,mlp.up_proj,4,32,8,128,16.359375,126.21875,234881024 +64,mlp.up_proj,4,64,8,128,19.078125,119.109375,234881024 +64,mlp.up_proj,4,128,8,128,21.484375,115.5546875,234881024 +64,mlp.up_proj,8,32,8,128,0.99462890625,238.21875,234881024 +64,mlp.up_proj,8,64,8,128,1.16015625,231.109375,234881024 +64,mlp.up_proj,8,128,8,128,1.3037109375,227.5546875,234881024 +64,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024 +64,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024 +64,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024 +64,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024 +64,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024 +64,mlp.down_proj,3,128,8,128,45.84375,87.5546875,234881024 +64,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024 +64,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024 +64,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024 +64,mlp.down_proj,8,32,8,128,0.9912109375,238.21875,234881024 +64,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024 +64,mlp.down_proj,8,128,8,128,1.296875,227.5546875,234881024 +65,self_attn.q_proj,2,32,8,128,28.375,20.0625,67108864 +65,self_attn.q_proj,2,64,8,128,33.5625,18.03125,67108864 +65,self_attn.q_proj,2,128,8,128,38.375,17.015625,67108864 +65,self_attn.q_proj,3,32,8,128,12.21875,28.0625,67108864 +65,self_attn.q_proj,3,64,8,128,14.5625,26.03125,67108864 +65,self_attn.q_proj,3,128,8,128,16.875,25.015625,67108864 +65,self_attn.q_proj,4,32,8,128,5.69921875,36.0625,67108864 +65,self_attn.q_proj,4,64,8,128,6.796875,34.03125,67108864 +65,self_attn.q_proj,4,128,8,128,7.8828125,33.015625,67108864 +65,self_attn.q_proj,8,32,8,128,0.34521484375,68.0625,67108864 +65,self_attn.q_proj,8,64,8,128,0.4111328125,66.03125,67108864 +65,self_attn.q_proj,8,128,8,128,0.4755859375,65.015625,67108864 +65,self_attn.v_proj,2,32,8,128,17.84375,2.5078125,8388608 +65,self_attn.v_proj,2,64,8,128,20.953125,2.25390625,8388608 +65,self_attn.v_proj,2,128,8,128,23.8125,2.126953125,8388608 +65,self_attn.v_proj,3,32,8,128,7.6328125,3.5078125,8388608 +65,self_attn.v_proj,3,64,8,128,8.9609375,3.25390625,8388608 +65,self_attn.v_proj,3,128,8,128,10.1875,3.126953125,8388608 +65,self_attn.v_proj,4,32,8,128,3.55859375,4.5078125,8388608 +65,self_attn.v_proj,4,64,8,128,4.18359375,4.25390625,8388608 +65,self_attn.v_proj,4,128,8,128,4.75390625,4.126953125,8388608 +65,self_attn.v_proj,8,32,8,128,0.2154541015625,8.5078125,8388608 +65,self_attn.v_proj,8,64,8,128,0.2529296875,8.25390625,8388608 +65,self_attn.v_proj,8,128,8,128,0.28662109375,8.126953125,8388608 +65,self_attn.k_proj,2,32,8,128,18.078125,2.5078125,8388608 +65,self_attn.k_proj,2,64,8,128,21.609375,2.25390625,8388608 +65,self_attn.k_proj,2,128,8,128,24.46875,2.126953125,8388608 +65,self_attn.k_proj,3,32,8,128,8.0859375,3.5078125,8388608 +65,self_attn.k_proj,3,64,8,128,9.9765625,3.25390625,8388608 +65,self_attn.k_proj,3,128,8,128,11.7109375,3.126953125,8388608 +65,self_attn.k_proj,4,32,8,128,3.791015625,4.5078125,8388608 +65,self_attn.k_proj,4,64,8,128,4.70703125,4.25390625,8388608 +65,self_attn.k_proj,4,128,8,128,5.5703125,4.126953125,8388608 +65,self_attn.k_proj,8,32,8,128,0.2293701171875,8.5078125,8388608 +65,self_attn.k_proj,8,64,8,128,0.284912109375,8.25390625,8388608 +65,self_attn.k_proj,8,128,8,128,0.33642578125,8.126953125,8388608 +65,self_attn.o_proj,2,32,8,128,37.8125,20.0625,67108864 +65,self_attn.o_proj,2,64,8,128,44.03125,18.03125,67108864 +65,self_attn.o_proj,2,128,8,128,49.625,17.015625,67108864 +65,self_attn.o_proj,3,32,8,128,16.171875,28.0625,67108864 +65,self_attn.o_proj,3,64,8,128,18.828125,26.03125,67108864 +65,self_attn.o_proj,3,128,8,128,21.25,25.015625,67108864 +65,self_attn.o_proj,4,32,8,128,7.546875,36.0625,67108864 +65,self_attn.o_proj,4,64,8,128,8.7890625,34.03125,67108864 +65,self_attn.o_proj,4,128,8,128,9.9140625,33.015625,67108864 +65,self_attn.o_proj,8,32,8,128,0.456787109375,68.0625,67108864 +65,self_attn.o_proj,8,64,8,128,0.53173828125,66.03125,67108864 +65,self_attn.o_proj,8,128,8,128,0.59765625,65.015625,67108864 +65,mlp.gate_proj,2,32,8,128,89.125,70.21875,234881024 +65,mlp.gate_proj,2,64,8,128,104.25,63.109375,234881024 +65,mlp.gate_proj,2,128,8,128,117.9375,59.5546875,234881024 +65,mlp.gate_proj,3,32,8,128,38.15625,98.21875,234881024 +65,mlp.gate_proj,3,64,8,128,44.5625,91.109375,234881024 +65,mlp.gate_proj,3,128,8,128,50.5,87.5546875,234881024 +65,mlp.gate_proj,4,32,8,128,17.796875,126.21875,234881024 +65,mlp.gate_proj,4,64,8,128,20.796875,119.109375,234881024 +65,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024 +65,mlp.gate_proj,8,32,8,128,1.08203125,238.21875,234881024 +65,mlp.gate_proj,8,64,8,128,1.2666015625,231.109375,234881024 +65,mlp.gate_proj,8,128,8,128,1.4296875,227.5546875,234881024 +65,mlp.up_proj,2,32,8,128,82.25,70.21875,234881024 +65,mlp.up_proj,2,64,8,128,95.6875,63.109375,234881024 +65,mlp.up_proj,2,128,8,128,107.9375,59.5546875,234881024 +65,mlp.up_proj,3,32,8,128,35.15625,98.21875,234881024 +65,mlp.up_proj,3,64,8,128,40.96875,91.109375,234881024 +65,mlp.up_proj,3,128,8,128,46.15625,87.5546875,234881024 +65,mlp.up_proj,4,32,8,128,16.390625,126.21875,234881024 +65,mlp.up_proj,4,64,8,128,19.125,119.109375,234881024 +65,mlp.up_proj,4,128,8,128,21.546875,115.5546875,234881024 +65,mlp.up_proj,8,32,8,128,0.9970703125,238.21875,234881024 +65,mlp.up_proj,8,64,8,128,1.1630859375,231.109375,234881024 +65,mlp.up_proj,8,128,8,128,1.306640625,227.5546875,234881024 +65,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024 +65,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024 +65,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024 +65,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024 +65,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024 +65,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024 +65,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024 +65,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024 +65,mlp.down_proj,4,128,8,128,21.421875,115.5546875,234881024 +65,mlp.down_proj,8,32,8,128,0.99267578125,238.21875,234881024 +65,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024 +65,mlp.down_proj,8,128,8,128,1.2998046875,227.5546875,234881024 +66,self_attn.q_proj,2,32,8,128,34.4375,20.0625,67108864 +66,self_attn.q_proj,2,64,8,128,41.375,18.03125,67108864 +66,self_attn.q_proj,2,128,8,128,48.28125,17.015625,67108864 +66,self_attn.q_proj,3,32,8,128,14.828125,28.0625,67108864 +66,self_attn.q_proj,3,64,8,128,17.96875,26.03125,67108864 +66,self_attn.q_proj,3,128,8,128,21.3125,25.015625,67108864 +66,self_attn.q_proj,4,32,8,128,6.91796875,36.0625,67108864 +66,self_attn.q_proj,4,64,8,128,8.3828125,34.03125,67108864 +66,self_attn.q_proj,4,128,8,128,9.953125,33.015625,67108864 +66,self_attn.q_proj,8,32,8,128,0.418701171875,68.0625,67108864 +66,self_attn.q_proj,8,64,8,128,0.50732421875,66.03125,67108864 +66,self_attn.q_proj,8,128,8,128,0.60009765625,65.015625,67108864 +66,self_attn.v_proj,2,32,8,128,17.921875,2.5078125,8388608 +66,self_attn.v_proj,2,64,8,128,20.890625,2.25390625,8388608 +66,self_attn.v_proj,2,128,8,128,23.578125,2.126953125,8388608 +66,self_attn.v_proj,3,32,8,128,7.66015625,3.5078125,8388608 +66,self_attn.v_proj,3,64,8,128,8.9453125,3.25390625,8388608 +66,self_attn.v_proj,3,128,8,128,10.0859375,3.126953125,8388608 +66,self_attn.v_proj,4,32,8,128,3.57421875,4.5078125,8388608 +66,self_attn.v_proj,4,64,8,128,4.16796875,4.25390625,8388608 +66,self_attn.v_proj,4,128,8,128,4.70703125,4.126953125,8388608 +66,self_attn.v_proj,8,32,8,128,0.2164306640625,8.5078125,8388608 +66,self_attn.v_proj,8,64,8,128,0.25244140625,8.25390625,8388608 +66,self_attn.v_proj,8,128,8,128,0.28369140625,8.126953125,8388608 +66,self_attn.k_proj,2,32,8,128,20.59375,2.5078125,8388608 +66,self_attn.k_proj,2,64,8,128,24.984375,2.25390625,8388608 +66,self_attn.k_proj,2,128,8,128,28.6875,2.126953125,8388608 +66,self_attn.k_proj,3,32,8,128,9.09375,3.5078125,8388608 +66,self_attn.k_proj,3,64,8,128,11.4140625,3.25390625,8388608 +66,self_attn.k_proj,3,128,8,128,13.734375,3.126953125,8388608 +66,self_attn.k_proj,4,32,8,128,4.2421875,4.5078125,8388608 +66,self_attn.k_proj,4,64,8,128,5.33203125,4.25390625,8388608 +66,self_attn.k_proj,4,128,8,128,6.421875,4.126953125,8388608 +66,self_attn.k_proj,8,32,8,128,0.25634765625,8.5078125,8388608 +66,self_attn.k_proj,8,64,8,128,0.322265625,8.25390625,8388608 +66,self_attn.k_proj,8,128,8,128,0.386962890625,8.126953125,8388608 +66,self_attn.o_proj,2,32,8,128,40.6875,20.0625,67108864 +66,self_attn.o_proj,2,64,8,128,47.375,18.03125,67108864 +66,self_attn.o_proj,2,128,8,128,53.375,17.015625,67108864 +66,self_attn.o_proj,3,32,8,128,17.40625,28.0625,67108864 +66,self_attn.o_proj,3,64,8,128,20.265625,26.03125,67108864 +66,self_attn.o_proj,3,128,8,128,22.859375,25.015625,67108864 +66,self_attn.o_proj,4,32,8,128,8.1171875,36.0625,67108864 +66,self_attn.o_proj,4,64,8,128,9.4609375,34.03125,67108864 +66,self_attn.o_proj,4,128,8,128,10.671875,33.015625,67108864 +66,self_attn.o_proj,8,32,8,128,0.49169921875,68.0625,67108864 +66,self_attn.o_proj,8,64,8,128,0.572265625,66.03125,67108864 +66,self_attn.o_proj,8,128,8,128,0.64306640625,65.015625,67108864 +66,mlp.gate_proj,2,32,8,128,89.0,70.21875,234881024 +66,mlp.gate_proj,2,64,8,128,104.125,63.109375,234881024 +66,mlp.gate_proj,2,128,8,128,117.8125,59.5546875,234881024 +66,mlp.gate_proj,3,32,8,128,38.09375,98.21875,234881024 +66,mlp.gate_proj,3,64,8,128,44.5,91.109375,234881024 +66,mlp.gate_proj,3,128,8,128,50.4375,87.5546875,234881024 +66,mlp.gate_proj,4,32,8,128,17.75,126.21875,234881024 +66,mlp.gate_proj,4,64,8,128,20.765625,119.109375,234881024 +66,mlp.gate_proj,4,128,8,128,23.53125,115.5546875,234881024 +66,mlp.gate_proj,8,32,8,128,1.080078125,238.21875,234881024 +66,mlp.gate_proj,8,64,8,128,1.2646484375,231.109375,234881024 +66,mlp.gate_proj,8,128,8,128,1.427734375,227.5546875,234881024 +66,mlp.up_proj,2,32,8,128,82.4375,70.21875,234881024 +66,mlp.up_proj,2,64,8,128,95.9375,63.109375,234881024 +66,mlp.up_proj,2,128,8,128,108.1875,59.5546875,234881024 +66,mlp.up_proj,3,32,8,128,35.25,98.21875,234881024 +66,mlp.up_proj,3,64,8,128,41.0625,91.109375,234881024 +66,mlp.up_proj,3,128,8,128,46.25,87.5546875,234881024 +66,mlp.up_proj,4,32,8,128,16.4375,126.21875,234881024 +66,mlp.up_proj,4,64,8,128,19.171875,119.109375,234881024 +66,mlp.up_proj,4,128,8,128,21.578125,115.5546875,234881024 +66,mlp.up_proj,8,32,8,128,0.99951171875,238.21875,234881024 +66,mlp.up_proj,8,64,8,128,1.166015625,231.109375,234881024 +66,mlp.up_proj,8,128,8,128,1.3095703125,227.5546875,234881024 +66,mlp.down_proj,2,32,8,128,82.0625,70.21875,234881024 +66,mlp.down_proj,2,64,8,128,95.4375,63.109375,234881024 +66,mlp.down_proj,2,128,8,128,107.5,59.5546875,234881024 +66,mlp.down_proj,3,32,8,128,35.0625,98.21875,234881024 +66,mlp.down_proj,3,64,8,128,40.84375,91.109375,234881024 +66,mlp.down_proj,3,128,8,128,45.96875,87.5546875,234881024 +66,mlp.down_proj,4,32,8,128,16.34375,126.21875,234881024 +66,mlp.down_proj,4,64,8,128,19.0625,119.109375,234881024 +66,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024 +66,mlp.down_proj,8,32,8,128,0.99462890625,238.21875,234881024 +66,mlp.down_proj,8,64,8,128,1.1591796875,231.109375,234881024 +66,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024 +67,self_attn.q_proj,2,32,8,128,29.65625,20.0625,67108864 +67,self_attn.q_proj,2,64,8,128,35.28125,18.03125,67108864 +67,self_attn.q_proj,2,128,8,128,40.8125,17.015625,67108864 +67,self_attn.q_proj,3,32,8,128,12.859375,28.0625,67108864 +67,self_attn.q_proj,3,64,8,128,15.5234375,26.03125,67108864 +67,self_attn.q_proj,3,128,8,128,18.375,25.015625,67108864 +67,self_attn.q_proj,4,32,8,128,6.0,36.0625,67108864 +67,self_attn.q_proj,4,64,8,128,7.24609375,34.03125,67108864 +67,self_attn.q_proj,4,128,8,128,8.5859375,33.015625,67108864 +67,self_attn.q_proj,8,32,8,128,0.36328125,68.0625,67108864 +67,self_attn.q_proj,8,64,8,128,0.4384765625,66.03125,67108864 +67,self_attn.q_proj,8,128,8,128,0.51806640625,65.015625,67108864 +67,self_attn.v_proj,2,32,8,128,18.140625,2.5078125,8388608 +67,self_attn.v_proj,2,64,8,128,21.5,2.25390625,8388608 +67,self_attn.v_proj,2,128,8,128,24.515625,2.126953125,8388608 +67,self_attn.v_proj,3,32,8,128,7.765625,3.5078125,8388608 +67,self_attn.v_proj,3,64,8,128,9.21875,3.25390625,8388608 +67,self_attn.v_proj,3,128,8,128,10.546875,3.126953125,8388608 +67,self_attn.v_proj,4,32,8,128,3.623046875,4.5078125,8388608 +67,self_attn.v_proj,4,64,8,128,4.30078125,4.25390625,8388608 +67,self_attn.v_proj,4,128,8,128,4.92578125,4.126953125,8388608 +67,self_attn.v_proj,8,32,8,128,0.2193603515625,8.5078125,8388608 +67,self_attn.v_proj,8,64,8,128,0.260009765625,8.25390625,8388608 +67,self_attn.v_proj,8,128,8,128,0.296875,8.126953125,8388608 +67,self_attn.k_proj,2,32,8,128,19.78125,2.5078125,8388608 +67,self_attn.k_proj,2,64,8,128,23.4375,2.25390625,8388608 +67,self_attn.k_proj,2,128,8,128,26.453125,2.126953125,8388608 +67,self_attn.k_proj,3,32,8,128,8.9453125,3.5078125,8388608 +67,self_attn.k_proj,3,64,8,128,11.03125,3.25390625,8388608 +67,self_attn.k_proj,3,128,8,128,12.8203125,3.126953125,8388608 +67,self_attn.k_proj,4,32,8,128,4.2265625,4.5078125,8388608 +67,self_attn.k_proj,4,64,8,128,5.2734375,4.25390625,8388608 +67,self_attn.k_proj,4,128,8,128,6.20703125,4.126953125,8388608 +67,self_attn.k_proj,8,32,8,128,0.256591796875,8.5078125,8388608 +67,self_attn.k_proj,8,64,8,128,0.32177734375,8.25390625,8388608 +67,self_attn.k_proj,8,128,8,128,0.3798828125,8.126953125,8388608 +67,self_attn.o_proj,2,32,8,128,36.25,20.0625,67108864 +67,self_attn.o_proj,2,64,8,128,42.3125,18.03125,67108864 +67,self_attn.o_proj,2,128,8,128,47.90625,17.015625,67108864 +67,self_attn.o_proj,3,32,8,128,15.546875,28.0625,67108864 +67,self_attn.o_proj,3,64,8,128,18.21875,26.03125,67108864 +67,self_attn.o_proj,3,128,8,128,20.765625,25.015625,67108864 +67,self_attn.o_proj,4,32,8,128,7.26953125,36.0625,67108864 +67,self_attn.o_proj,4,64,8,128,8.53125,34.03125,67108864 +67,self_attn.o_proj,4,128,8,128,9.734375,33.015625,67108864 +67,self_attn.o_proj,8,32,8,128,0.4404296875,68.0625,67108864 +67,self_attn.o_proj,8,64,8,128,0.51708984375,66.03125,67108864 +67,self_attn.o_proj,8,128,8,128,0.58984375,65.015625,67108864 +67,mlp.gate_proj,2,32,8,128,88.8125,70.21875,234881024 +67,mlp.gate_proj,2,64,8,128,103.8125,63.109375,234881024 +67,mlp.gate_proj,2,128,8,128,117.4375,59.5546875,234881024 +67,mlp.gate_proj,3,32,8,128,38.0,98.21875,234881024 +67,mlp.gate_proj,3,64,8,128,44.40625,91.109375,234881024 +67,mlp.gate_proj,3,128,8,128,50.3125,87.5546875,234881024 +67,mlp.gate_proj,4,32,8,128,17.71875,126.21875,234881024 +67,mlp.gate_proj,4,64,8,128,20.71875,119.109375,234881024 +67,mlp.gate_proj,4,128,8,128,23.46875,115.5546875,234881024 +67,mlp.gate_proj,8,32,8,128,1.078125,238.21875,234881024 +67,mlp.gate_proj,8,64,8,128,1.26171875,231.109375,234881024 +67,mlp.gate_proj,8,128,8,128,1.423828125,227.5546875,234881024 +67,mlp.up_proj,2,32,8,128,82.6875,70.21875,234881024 +67,mlp.up_proj,2,64,8,128,96.25,63.109375,234881024 +67,mlp.up_proj,2,128,8,128,108.5,59.5546875,234881024 +67,mlp.up_proj,3,32,8,128,35.34375,98.21875,234881024 +67,mlp.up_proj,3,64,8,128,41.1875,91.109375,234881024 +67,mlp.up_proj,3,128,8,128,46.40625,87.5546875,234881024 +67,mlp.up_proj,4,32,8,128,16.484375,126.21875,234881024 +67,mlp.up_proj,4,64,8,128,19.21875,119.109375,234881024 +67,mlp.up_proj,4,128,8,128,21.65625,115.5546875,234881024 +67,mlp.up_proj,8,32,8,128,1.0029296875,238.21875,234881024 +67,mlp.up_proj,8,64,8,128,1.1689453125,231.109375,234881024 +67,mlp.up_proj,8,128,8,128,1.3134765625,227.5546875,234881024 +67,mlp.down_proj,2,32,8,128,82.1875,70.21875,234881024 +67,mlp.down_proj,2,64,8,128,95.625,63.109375,234881024 +67,mlp.down_proj,2,128,8,128,107.75,59.5546875,234881024 +67,mlp.down_proj,3,32,8,128,35.125,98.21875,234881024 +67,mlp.down_proj,3,64,8,128,40.90625,91.109375,234881024 +67,mlp.down_proj,3,128,8,128,46.0625,87.5546875,234881024 +67,mlp.down_proj,4,32,8,128,16.390625,126.21875,234881024 +67,mlp.down_proj,4,64,8,128,19.09375,119.109375,234881024 +67,mlp.down_proj,4,128,8,128,21.5,115.5546875,234881024 +67,mlp.down_proj,8,32,8,128,0.99658203125,238.21875,234881024 +67,mlp.down_proj,8,64,8,128,1.162109375,231.109375,234881024 +67,mlp.down_proj,8,128,8,128,1.3046875,227.5546875,234881024 +68,self_attn.q_proj,2,32,8,128,35.21875,20.0625,67108864 +68,self_attn.q_proj,2,64,8,128,41.5,18.03125,67108864 +68,self_attn.q_proj,2,128,8,128,48.0625,17.015625,67108864 +68,self_attn.q_proj,3,32,8,128,15.078125,28.0625,67108864 +68,self_attn.q_proj,3,64,8,128,17.796875,26.03125,67108864 +68,self_attn.q_proj,3,128,8,128,20.6875,25.015625,67108864 +68,self_attn.q_proj,4,32,8,128,7.03515625,36.0625,67108864 +68,self_attn.q_proj,4,64,8,128,8.3046875,34.03125,67108864 +68,self_attn.q_proj,4,128,8,128,9.65625,33.015625,67108864 +68,self_attn.q_proj,8,32,8,128,0.42578125,68.0625,67108864 +68,self_attn.q_proj,8,64,8,128,0.50244140625,66.03125,67108864 +68,self_attn.q_proj,8,128,8,128,0.58251953125,65.015625,67108864 +68,self_attn.v_proj,2,32,8,128,20.0,2.5078125,8388608 +68,self_attn.v_proj,2,64,8,128,23.515625,2.25390625,8388608 +68,self_attn.v_proj,2,128,8,128,26.78125,2.126953125,8388608 +68,self_attn.v_proj,3,32,8,128,8.5546875,3.5078125,8388608 +68,self_attn.v_proj,3,64,8,128,10.0625,3.25390625,8388608 +68,self_attn.v_proj,3,128,8,128,11.4609375,3.126953125,8388608 +68,self_attn.v_proj,4,32,8,128,3.9921875,4.5078125,8388608 +68,self_attn.v_proj,4,64,8,128,4.6953125,4.25390625,8388608 +68,self_attn.v_proj,4,128,8,128,5.34765625,4.126953125,8388608 +68,self_attn.v_proj,8,32,8,128,0.24169921875,8.5078125,8388608 +68,self_attn.v_proj,8,64,8,128,0.2841796875,8.25390625,8388608 +68,self_attn.v_proj,8,128,8,128,0.322265625,8.126953125,8388608 +68,self_attn.k_proj,2,32,8,128,21.234375,2.5078125,8388608 +68,self_attn.k_proj,2,64,8,128,25.640625,2.25390625,8388608 +68,self_attn.k_proj,2,128,8,128,29.390625,2.126953125,8388608 +68,self_attn.k_proj,3,32,8,128,9.1796875,3.5078125,8388608 +68,self_attn.k_proj,3,64,8,128,11.25,3.25390625,8388608 +68,self_attn.k_proj,3,128,8,128,13.2421875,3.126953125,8388608 +68,self_attn.k_proj,4,32,8,128,4.28515625,4.5078125,8388608 +68,self_attn.k_proj,4,64,8,128,5.25390625,4.25390625,8388608 +68,self_attn.k_proj,4,128,8,128,6.1796875,4.126953125,8388608 +68,self_attn.k_proj,8,32,8,128,0.25927734375,8.5078125,8388608 +68,self_attn.k_proj,8,64,8,128,0.317626953125,8.25390625,8388608 +68,self_attn.k_proj,8,128,8,128,0.37255859375,8.126953125,8388608 +68,self_attn.o_proj,2,32,8,128,42.1875,20.0625,67108864 +68,self_attn.o_proj,2,64,8,128,49.09375,18.03125,67108864 +68,self_attn.o_proj,2,128,8,128,55.34375,17.015625,67108864 +68,self_attn.o_proj,3,32,8,128,18.046875,28.0625,67108864 +68,self_attn.o_proj,3,64,8,128,21.015625,26.03125,67108864 +68,self_attn.o_proj,3,128,8,128,23.6875,25.015625,67108864 +68,self_attn.o_proj,4,32,8,128,8.4140625,36.0625,67108864 +68,self_attn.o_proj,4,64,8,128,9.8046875,34.03125,67108864 +68,self_attn.o_proj,4,128,8,128,11.0546875,33.015625,67108864 +68,self_attn.o_proj,8,32,8,128,0.509765625,68.0625,67108864 +68,self_attn.o_proj,8,64,8,128,0.59326171875,66.03125,67108864 +68,self_attn.o_proj,8,128,8,128,0.66650390625,65.015625,67108864 +68,mlp.gate_proj,2,32,8,128,88.6875,70.21875,234881024 +68,mlp.gate_proj,2,64,8,128,103.6875,63.109375,234881024 +68,mlp.gate_proj,2,128,8,128,117.375,59.5546875,234881024 +68,mlp.gate_proj,3,32,8,128,37.9375,98.21875,234881024 +68,mlp.gate_proj,3,64,8,128,44.375,91.109375,234881024 +68,mlp.gate_proj,3,128,8,128,50.28125,87.5546875,234881024 +68,mlp.gate_proj,4,32,8,128,17.6875,126.21875,234881024 +68,mlp.gate_proj,4,64,8,128,20.703125,119.109375,234881024 +68,mlp.gate_proj,4,128,8,128,23.453125,115.5546875,234881024 +68,mlp.gate_proj,8,32,8,128,1.076171875,238.21875,234881024 +68,mlp.gate_proj,8,64,8,128,1.259765625,231.109375,234881024 +68,mlp.gate_proj,8,128,8,128,1.4228515625,227.5546875,234881024 +68,mlp.up_proj,2,32,8,128,82.875,70.21875,234881024 +68,mlp.up_proj,2,64,8,128,96.4375,63.109375,234881024 +68,mlp.up_proj,2,128,8,128,108.75,59.5546875,234881024 +68,mlp.up_proj,3,32,8,128,35.4375,98.21875,234881024 +68,mlp.up_proj,3,64,8,128,41.28125,91.109375,234881024 +68,mlp.up_proj,3,128,8,128,46.53125,87.5546875,234881024 +68,mlp.up_proj,4,32,8,128,16.515625,126.21875,234881024 +68,mlp.up_proj,4,64,8,128,19.265625,119.109375,234881024 +68,mlp.up_proj,4,128,8,128,21.703125,115.5546875,234881024 +68,mlp.up_proj,8,32,8,128,1.0048828125,238.21875,234881024 +68,mlp.up_proj,8,64,8,128,1.171875,231.109375,234881024 +68,mlp.up_proj,8,128,8,128,1.3173828125,227.5546875,234881024 +68,mlp.down_proj,2,32,8,128,82.3125,70.21875,234881024 +68,mlp.down_proj,2,64,8,128,95.75,63.109375,234881024 +68,mlp.down_proj,2,128,8,128,107.9375,59.5546875,234881024 +68,mlp.down_proj,3,32,8,128,35.1875,98.21875,234881024 +68,mlp.down_proj,3,64,8,128,40.96875,91.109375,234881024 +68,mlp.down_proj,3,128,8,128,46.125,87.5546875,234881024 +68,mlp.down_proj,4,32,8,128,16.40625,126.21875,234881024 +68,mlp.down_proj,4,64,8,128,19.125,119.109375,234881024 +68,mlp.down_proj,4,128,8,128,21.53125,115.5546875,234881024 +68,mlp.down_proj,8,32,8,128,0.998046875,238.21875,234881024 +68,mlp.down_proj,8,64,8,128,1.1640625,231.109375,234881024 +68,mlp.down_proj,8,128,8,128,1.306640625,227.5546875,234881024 +69,self_attn.q_proj,2,32,8,128,38.34375,20.0625,67108864 +69,self_attn.q_proj,2,64,8,128,45.1875,18.03125,67108864 +69,self_attn.q_proj,2,128,8,128,52.25,17.015625,67108864 +69,self_attn.q_proj,3,32,8,128,16.421875,28.0625,67108864 +69,self_attn.q_proj,3,64,8,128,19.390625,26.03125,67108864 +69,self_attn.q_proj,3,128,8,128,22.5,25.015625,67108864 +69,self_attn.q_proj,4,32,8,128,7.6640625,36.0625,67108864 +69,self_attn.q_proj,4,64,8,128,9.0546875,34.03125,67108864 +69,self_attn.q_proj,4,128,8,128,10.5078125,33.015625,67108864 +69,self_attn.q_proj,8,32,8,128,0.4638671875,68.0625,67108864 +69,self_attn.q_proj,8,64,8,128,0.5478515625,66.03125,67108864 +69,self_attn.q_proj,8,128,8,128,0.6337890625,65.015625,67108864 +69,self_attn.v_proj,2,32,8,128,20.828125,2.5078125,8388608 +69,self_attn.v_proj,2,64,8,128,24.359375,2.25390625,8388608 +69,self_attn.v_proj,2,128,8,128,27.546875,2.126953125,8388608 +69,self_attn.v_proj,3,32,8,128,8.90625,3.5078125,8388608 +69,self_attn.v_proj,3,64,8,128,10.421875,3.25390625,8388608 +69,self_attn.v_proj,3,128,8,128,11.7890625,3.126953125,8388608 +69,self_attn.v_proj,4,32,8,128,4.15625,4.5078125,8388608 +69,self_attn.v_proj,4,64,8,128,4.859375,4.25390625,8388608 +69,self_attn.v_proj,4,128,8,128,5.5,4.126953125,8388608 +69,self_attn.v_proj,8,32,8,128,0.251708984375,8.5078125,8388608 +69,self_attn.v_proj,8,64,8,128,0.294189453125,8.25390625,8388608 +69,self_attn.v_proj,8,128,8,128,0.33154296875,8.126953125,8388608 +69,self_attn.k_proj,2,32,8,128,22.328125,2.5078125,8388608 +69,self_attn.k_proj,2,64,8,128,26.796875,2.25390625,8388608 +69,self_attn.k_proj,2,128,8,128,30.84375,2.126953125,8388608 +69,self_attn.k_proj,3,32,8,128,9.578125,3.5078125,8388608 +69,self_attn.k_proj,3,64,8,128,11.5625,3.25390625,8388608 +69,self_attn.k_proj,3,128,8,128,13.46875,3.126953125,8388608 +69,self_attn.k_proj,4,32,8,128,4.46875,4.5078125,8388608 +69,self_attn.k_proj,4,64,8,128,5.39453125,4.25390625,8388608 +69,self_attn.k_proj,4,128,8,128,6.28515625,4.126953125,8388608 +69,self_attn.k_proj,8,32,8,128,0.270751953125,8.5078125,8388608 +69,self_attn.k_proj,8,64,8,128,0.326416015625,8.25390625,8388608 +69,self_attn.k_proj,8,128,8,128,0.37890625,8.126953125,8388608 +69,self_attn.o_proj,2,32,8,128,44.28125,20.0625,67108864 +69,self_attn.o_proj,2,64,8,128,51.5625,18.03125,67108864 +69,self_attn.o_proj,2,128,8,128,58.125,17.015625,67108864 +69,self_attn.o_proj,3,32,8,128,18.9375,28.0625,67108864 +69,self_attn.o_proj,3,64,8,128,22.0625,26.03125,67108864 +69,self_attn.o_proj,3,128,8,128,24.890625,25.015625,67108864 +69,self_attn.o_proj,4,32,8,128,8.8359375,36.0625,67108864 +69,self_attn.o_proj,4,64,8,128,10.296875,34.03125,67108864 +69,self_attn.o_proj,4,128,8,128,11.6171875,33.015625,67108864 +69,self_attn.o_proj,8,32,8,128,0.53515625,68.0625,67108864 +69,self_attn.o_proj,8,64,8,128,0.623046875,66.03125,67108864 +69,self_attn.o_proj,8,128,8,128,0.7001953125,65.015625,67108864 +69,mlp.gate_proj,2,32,8,128,88.375,70.21875,234881024 +69,mlp.gate_proj,2,64,8,128,103.3125,63.109375,234881024 +69,mlp.gate_proj,2,128,8,128,116.875,59.5546875,234881024 +69,mlp.gate_proj,3,32,8,128,37.8125,98.21875,234881024 +69,mlp.gate_proj,3,64,8,128,44.1875,91.109375,234881024 +69,mlp.gate_proj,3,128,8,128,50.0625,87.5546875,234881024 +69,mlp.gate_proj,4,32,8,128,17.625,126.21875,234881024 +69,mlp.gate_proj,4,64,8,128,20.625,119.109375,234881024 +69,mlp.gate_proj,4,128,8,128,23.359375,115.5546875,234881024 +69,mlp.gate_proj,8,32,8,128,1.072265625,238.21875,234881024 +69,mlp.gate_proj,8,64,8,128,1.2548828125,231.109375,234881024 +69,mlp.gate_proj,8,128,8,128,1.4169921875,227.5546875,234881024 +69,mlp.up_proj,2,32,8,128,83.1875,70.21875,234881024 +69,mlp.up_proj,2,64,8,128,96.875,63.109375,234881024 +69,mlp.up_proj,2,128,8,128,109.1875,59.5546875,234881024 +69,mlp.up_proj,3,32,8,128,35.5625,98.21875,234881024 +69,mlp.up_proj,3,64,8,128,41.4375,91.109375,234881024 +69,mlp.up_proj,3,128,8,128,46.71875,87.5546875,234881024 +69,mlp.up_proj,4,32,8,128,16.59375,126.21875,234881024 +69,mlp.up_proj,4,64,8,128,19.34375,119.109375,234881024 +69,mlp.up_proj,4,128,8,128,21.796875,115.5546875,234881024 +69,mlp.up_proj,8,32,8,128,1.0087890625,238.21875,234881024 +69,mlp.up_proj,8,64,8,128,1.1767578125,231.109375,234881024 +69,mlp.up_proj,8,128,8,128,1.322265625,227.5546875,234881024 +69,mlp.down_proj,2,32,8,128,82.625,70.21875,234881024 +69,mlp.down_proj,2,64,8,128,96.1875,63.109375,234881024 +69,mlp.down_proj,2,128,8,128,108.375,59.5546875,234881024 +69,mlp.down_proj,3,32,8,128,35.34375,98.21875,234881024 +69,mlp.down_proj,3,64,8,128,41.125,91.109375,234881024 +69,mlp.down_proj,3,128,8,128,46.3125,87.5546875,234881024 +69,mlp.down_proj,4,32,8,128,16.484375,126.21875,234881024 +69,mlp.down_proj,4,64,8,128,19.203125,119.109375,234881024 +69,mlp.down_proj,4,128,8,128,21.625,115.5546875,234881024 +69,mlp.down_proj,8,32,8,128,1.001953125,238.21875,234881024 +69,mlp.down_proj,8,64,8,128,1.16796875,231.109375,234881024 +69,mlp.down_proj,8,128,8,128,1.3115234375,227.5546875,234881024 +70,self_attn.q_proj,2,32,8,128,38.1875,20.0625,67108864 +70,self_attn.q_proj,2,64,8,128,45.15625,18.03125,67108864 +70,self_attn.q_proj,2,128,8,128,52.03125,17.015625,67108864 +70,self_attn.q_proj,3,32,8,128,16.359375,28.0625,67108864 +70,self_attn.q_proj,3,64,8,128,19.421875,26.03125,67108864 +70,self_attn.q_proj,3,128,8,128,22.46875,25.015625,67108864 +70,self_attn.q_proj,4,32,8,128,7.63671875,36.0625,67108864 +70,self_attn.q_proj,4,64,8,128,9.0625,34.03125,67108864 +70,self_attn.q_proj,4,128,8,128,10.4921875,33.015625,67108864 +70,self_attn.q_proj,8,32,8,128,0.46240234375,68.0625,67108864 +70,self_attn.q_proj,8,64,8,128,0.54833984375,66.03125,67108864 +70,self_attn.q_proj,8,128,8,128,0.6328125,65.015625,67108864 +70,self_attn.v_proj,2,32,8,128,19.4375,2.5078125,8388608 +70,self_attn.v_proj,2,64,8,128,22.71875,2.25390625,8388608 +70,self_attn.v_proj,2,128,8,128,25.71875,2.126953125,8388608 +70,self_attn.v_proj,3,32,8,128,8.3125,3.5078125,8388608 +70,self_attn.v_proj,3,64,8,128,9.71875,3.25390625,8388608 +70,self_attn.v_proj,3,128,8,128,11.0078125,3.126953125,8388608 +70,self_attn.v_proj,4,32,8,128,3.876953125,4.5078125,8388608 +70,self_attn.v_proj,4,64,8,128,4.53515625,4.25390625,8388608 +70,self_attn.v_proj,4,128,8,128,5.13671875,4.126953125,8388608 +70,self_attn.v_proj,8,32,8,128,0.23486328125,8.5078125,8388608 +70,self_attn.v_proj,8,64,8,128,0.2744140625,8.25390625,8388608 +70,self_attn.v_proj,8,128,8,128,0.3095703125,8.126953125,8388608 +70,self_attn.k_proj,2,32,8,128,21.90625,2.5078125,8388608 +70,self_attn.k_proj,2,64,8,128,26.3125,2.25390625,8388608 +70,self_attn.k_proj,2,128,8,128,30.1875,2.126953125,8388608 +70,self_attn.k_proj,3,32,8,128,9.4375,3.5078125,8388608 +70,self_attn.k_proj,3,64,8,128,11.4609375,3.25390625,8388608 +70,self_attn.k_proj,3,128,8,128,13.3828125,3.126953125,8388608 +70,self_attn.k_proj,4,32,8,128,4.40234375,4.5078125,8388608 +70,self_attn.k_proj,4,64,8,128,5.34765625,4.25390625,8388608 +70,self_attn.k_proj,4,128,8,128,6.24609375,4.126953125,8388608 +70,self_attn.k_proj,8,32,8,128,0.2666015625,8.5078125,8388608 +70,self_attn.k_proj,8,64,8,128,0.323486328125,8.25390625,8388608 +70,self_attn.k_proj,8,128,8,128,0.376708984375,8.126953125,8388608 +70,self_attn.o_proj,2,32,8,128,41.8125,20.0625,67108864 +70,self_attn.o_proj,2,64,8,128,48.71875,18.03125,67108864 +70,self_attn.o_proj,2,128,8,128,54.9375,17.015625,67108864 +70,self_attn.o_proj,3,32,8,128,17.875,28.0625,67108864 +70,self_attn.o_proj,3,64,8,128,20.859375,26.03125,67108864 +70,self_attn.o_proj,3,128,8,128,23.546875,25.015625,67108864 +70,self_attn.o_proj,4,32,8,128,8.34375,36.0625,67108864 +70,self_attn.o_proj,4,64,8,128,9.734375,34.03125,67108864 +70,self_attn.o_proj,4,128,8,128,10.9921875,33.015625,67108864 +70,self_attn.o_proj,8,32,8,128,0.50537109375,68.0625,67108864 +70,self_attn.o_proj,8,64,8,128,0.5888671875,66.03125,67108864 +70,self_attn.o_proj,8,128,8,128,0.66259765625,65.015625,67108864 +70,mlp.gate_proj,2,32,8,128,88.25,70.21875,234881024 +70,mlp.gate_proj,2,64,8,128,103.125,63.109375,234881024 +70,mlp.gate_proj,2,128,8,128,116.75,59.5546875,234881024 +70,mlp.gate_proj,3,32,8,128,37.75,98.21875,234881024 +70,mlp.gate_proj,3,64,8,128,44.125,91.109375,234881024 +70,mlp.gate_proj,3,128,8,128,50.0,87.5546875,234881024 +70,mlp.gate_proj,4,32,8,128,17.609375,126.21875,234881024 +70,mlp.gate_proj,4,64,8,128,20.59375,119.109375,234881024 +70,mlp.gate_proj,4,128,8,128,23.328125,115.5546875,234881024 +70,mlp.gate_proj,8,32,8,128,1.0712890625,238.21875,234881024 +70,mlp.gate_proj,8,64,8,128,1.2529296875,231.109375,234881024 +70,mlp.gate_proj,8,128,8,128,1.4150390625,227.5546875,234881024 +70,mlp.up_proj,2,32,8,128,83.4375,70.21875,234881024 +70,mlp.up_proj,2,64,8,128,97.125,63.109375,234881024 +70,mlp.up_proj,2,128,8,128,109.5625,59.5546875,234881024 +70,mlp.up_proj,3,32,8,128,35.6875,98.21875,234881024 +70,mlp.up_proj,3,64,8,128,41.5625,91.109375,234881024 +70,mlp.up_proj,3,128,8,128,46.875,87.5546875,234881024 +70,mlp.up_proj,4,32,8,128,16.640625,126.21875,234881024 +70,mlp.up_proj,4,64,8,128,19.40625,119.109375,234881024 +70,mlp.up_proj,4,128,8,128,21.859375,115.5546875,234881024 +70,mlp.up_proj,8,32,8,128,1.01171875,238.21875,234881024 +70,mlp.up_proj,8,64,8,128,1.1806640625,231.109375,234881024 +70,mlp.up_proj,8,128,8,128,1.326171875,227.5546875,234881024 +70,mlp.down_proj,2,32,8,128,82.875,70.21875,234881024 +70,mlp.down_proj,2,64,8,128,96.4375,63.109375,234881024 +70,mlp.down_proj,2,128,8,128,108.6875,59.5546875,234881024 +70,mlp.down_proj,3,32,8,128,35.4375,98.21875,234881024 +70,mlp.down_proj,3,64,8,128,41.25,91.109375,234881024 +70,mlp.down_proj,3,128,8,128,46.5,87.5546875,234881024 +70,mlp.down_proj,4,32,8,128,16.515625,126.21875,234881024 +70,mlp.down_proj,4,64,8,128,19.265625,119.109375,234881024 +70,mlp.down_proj,4,128,8,128,21.6875,115.5546875,234881024 +70,mlp.down_proj,8,32,8,128,1.0048828125,238.21875,234881024 +70,mlp.down_proj,8,64,8,128,1.171875,231.109375,234881024 +70,mlp.down_proj,8,128,8,128,1.3154296875,227.5546875,234881024 +71,self_attn.q_proj,2,32,8,128,39.53125,20.0625,67108864 +71,self_attn.q_proj,2,64,8,128,46.46875,18.03125,67108864 +71,self_attn.q_proj,2,128,8,128,53.5625,17.015625,67108864 +71,self_attn.q_proj,3,32,8,128,16.90625,28.0625,67108864 +71,self_attn.q_proj,3,64,8,128,19.9375,26.03125,67108864 +71,self_attn.q_proj,3,128,8,128,23.046875,25.015625,67108864 +71,self_attn.q_proj,4,32,8,128,7.890625,36.0625,67108864 +71,self_attn.q_proj,4,64,8,128,9.296875,34.03125,67108864 +71,self_attn.q_proj,4,128,8,128,10.75,33.015625,67108864 +71,self_attn.q_proj,8,32,8,128,0.477783203125,68.0625,67108864 +71,self_attn.q_proj,8,64,8,128,0.5625,66.03125,67108864 +71,self_attn.q_proj,8,128,8,128,0.6484375,65.015625,67108864 +71,self_attn.v_proj,2,32,8,128,19.265625,2.5078125,8388608 +71,self_attn.v_proj,2,64,8,128,22.546875,2.25390625,8388608 +71,self_attn.v_proj,2,128,8,128,25.546875,2.126953125,8388608 +71,self_attn.v_proj,3,32,8,128,8.2421875,3.5078125,8388608 +71,self_attn.v_proj,3,64,8,128,9.6484375,3.25390625,8388608 +71,self_attn.v_proj,3,128,8,128,10.9296875,3.126953125,8388608 +71,self_attn.v_proj,4,32,8,128,3.841796875,4.5078125,8388608 +71,self_attn.v_proj,4,64,8,128,4.5,4.25390625,8388608 +71,self_attn.v_proj,4,128,8,128,5.1015625,4.126953125,8388608 +71,self_attn.v_proj,8,32,8,128,0.2327880859375,8.5078125,8388608 +71,self_attn.v_proj,8,64,8,128,0.272216796875,8.25390625,8388608 +71,self_attn.v_proj,8,128,8,128,0.307373046875,8.126953125,8388608 +71,self_attn.k_proj,2,32,8,128,22.703125,2.5078125,8388608 +71,self_attn.k_proj,2,64,8,128,27.296875,2.25390625,8388608 +71,self_attn.k_proj,2,128,8,128,31.359375,2.126953125,8388608 +71,self_attn.k_proj,3,32,8,128,9.75,3.5078125,8388608 +71,self_attn.k_proj,3,64,8,128,11.8125,3.25390625,8388608 +71,self_attn.k_proj,3,128,8,128,13.7734375,3.126953125,8388608 +71,self_attn.k_proj,4,32,8,128,4.546875,4.5078125,8388608 +71,self_attn.k_proj,4,64,8,128,5.51171875,4.25390625,8388608 +71,self_attn.k_proj,4,128,8,128,6.42578125,4.126953125,8388608 +71,self_attn.k_proj,8,32,8,128,0.275390625,8.5078125,8388608 +71,self_attn.k_proj,8,64,8,128,0.33349609375,8.25390625,8388608 +71,self_attn.k_proj,8,128,8,128,0.38720703125,8.126953125,8388608 +71,self_attn.o_proj,2,32,8,128,42.59375,20.0625,67108864 +71,self_attn.o_proj,2,64,8,128,49.625,18.03125,67108864 +71,self_attn.o_proj,2,128,8,128,55.96875,17.015625,67108864 +71,self_attn.o_proj,3,32,8,128,18.234375,28.0625,67108864 +71,self_attn.o_proj,3,64,8,128,21.265625,26.03125,67108864 +71,self_attn.o_proj,3,128,8,128,24.015625,25.015625,67108864 +71,self_attn.o_proj,4,32,8,128,8.5078125,36.0625,67108864 +71,self_attn.o_proj,4,64,8,128,9.9296875,34.03125,67108864 +71,self_attn.o_proj,4,128,8,128,11.21875,33.015625,67108864 +71,self_attn.o_proj,8,32,8,128,0.51513671875,68.0625,67108864 +71,self_attn.o_proj,8,64,8,128,0.6005859375,66.03125,67108864 +71,self_attn.o_proj,8,128,8,128,0.67626953125,65.015625,67108864 +71,mlp.gate_proj,2,32,8,128,88.1875,70.21875,234881024 +71,mlp.gate_proj,2,64,8,128,103.125,63.109375,234881024 +71,mlp.gate_proj,2,128,8,128,116.6875,59.5546875,234881024 +71,mlp.gate_proj,3,32,8,128,37.75,98.21875,234881024 +71,mlp.gate_proj,3,64,8,128,44.09375,91.109375,234881024 +71,mlp.gate_proj,3,128,8,128,50.0,87.5546875,234881024 +71,mlp.gate_proj,4,32,8,128,17.59375,126.21875,234881024 +71,mlp.gate_proj,4,64,8,128,20.578125,119.109375,234881024 +71,mlp.gate_proj,4,128,8,128,23.3125,115.5546875,234881024 +71,mlp.gate_proj,8,32,8,128,1.0703125,238.21875,234881024 +71,mlp.gate_proj,8,64,8,128,1.2529296875,231.109375,234881024 +71,mlp.gate_proj,8,128,8,128,1.4150390625,227.5546875,234881024 +71,mlp.up_proj,2,32,8,128,83.5,70.21875,234881024 +71,mlp.up_proj,2,64,8,128,97.25,63.109375,234881024 +71,mlp.up_proj,2,128,8,128,109.6875,59.5546875,234881024 +71,mlp.up_proj,3,32,8,128,35.71875,98.21875,234881024 +71,mlp.up_proj,3,64,8,128,41.625,91.109375,234881024 +71,mlp.up_proj,3,128,8,128,46.9375,87.5546875,234881024 +71,mlp.up_proj,4,32,8,128,16.65625,126.21875,234881024 +71,mlp.up_proj,4,64,8,128,19.4375,119.109375,234881024 +71,mlp.up_proj,4,128,8,128,21.890625,115.5546875,234881024 +71,mlp.up_proj,8,32,8,128,1.0126953125,238.21875,234881024 +71,mlp.up_proj,8,64,8,128,1.1826171875,231.109375,234881024 +71,mlp.up_proj,8,128,8,128,1.3291015625,227.5546875,234881024 +71,mlp.down_proj,2,32,8,128,82.875,70.21875,234881024 +71,mlp.down_proj,2,64,8,128,96.4375,63.109375,234881024 +71,mlp.down_proj,2,128,8,128,108.6875,59.5546875,234881024 +71,mlp.down_proj,3,32,8,128,35.4375,98.21875,234881024 +71,mlp.down_proj,3,64,8,128,41.28125,91.109375,234881024 +71,mlp.down_proj,3,128,8,128,46.46875,87.5546875,234881024 +71,mlp.down_proj,4,32,8,128,16.53125,126.21875,234881024 +71,mlp.down_proj,4,64,8,128,19.265625,119.109375,234881024 +71,mlp.down_proj,4,128,8,128,21.6875,115.5546875,234881024 +71,mlp.down_proj,8,32,8,128,1.0048828125,238.21875,234881024 +71,mlp.down_proj,8,64,8,128,1.171875,231.109375,234881024 +71,mlp.down_proj,8,128,8,128,1.31640625,227.5546875,234881024 +72,self_attn.q_proj,2,32,8,128,40.96875,20.0625,67108864 +72,self_attn.q_proj,2,64,8,128,48.15625,18.03125,67108864 +72,self_attn.q_proj,2,128,8,128,55.125,17.015625,67108864 +72,self_attn.q_proj,3,32,8,128,17.515625,28.0625,67108864 +72,self_attn.q_proj,3,64,8,128,20.625,26.03125,67108864 +72,self_attn.q_proj,3,128,8,128,23.65625,25.015625,67108864 +72,self_attn.q_proj,4,32,8,128,8.1796875,36.0625,67108864 +72,self_attn.q_proj,4,64,8,128,9.625,34.03125,67108864 +72,self_attn.q_proj,4,128,8,128,11.0390625,33.015625,67108864 +72,self_attn.q_proj,8,32,8,128,0.4951171875,68.0625,67108864 +72,self_attn.q_proj,8,64,8,128,0.58251953125,66.03125,67108864 +72,self_attn.q_proj,8,128,8,128,0.66552734375,65.015625,67108864 +72,self_attn.v_proj,2,32,8,128,19.03125,2.5078125,8388608 +72,self_attn.v_proj,2,64,8,128,22.203125,2.25390625,8388608 +72,self_attn.v_proj,2,128,8,128,25.09375,2.126953125,8388608 +72,self_attn.v_proj,3,32,8,128,8.140625,3.5078125,8388608 +72,self_attn.v_proj,3,64,8,128,9.5078125,3.25390625,8388608 +72,self_attn.v_proj,3,128,8,128,10.7421875,3.126953125,8388608 +72,self_attn.v_proj,4,32,8,128,3.796875,4.5078125,8388608 +72,self_attn.v_proj,4,64,8,128,4.43359375,4.25390625,8388608 +72,self_attn.v_proj,4,128,8,128,5.01171875,4.126953125,8388608 +72,self_attn.v_proj,8,32,8,128,0.22998046875,8.5078125,8388608 +72,self_attn.v_proj,8,64,8,128,0.268310546875,8.25390625,8388608 +72,self_attn.v_proj,8,128,8,128,0.30224609375,8.126953125,8388608 +72,self_attn.k_proj,2,32,8,128,23.71875,2.5078125,8388608 +72,self_attn.k_proj,2,64,8,128,28.5,2.25390625,8388608 +72,self_attn.k_proj,2,128,8,128,32.625,2.126953125,8388608 +72,self_attn.k_proj,3,32,8,128,10.1875,3.5078125,8388608 +72,self_attn.k_proj,3,64,8,128,12.328125,3.25390625,8388608 +72,self_attn.k_proj,3,128,8,128,14.3203125,3.126953125,8388608 +72,self_attn.k_proj,4,32,8,128,4.75390625,4.5078125,8388608 +72,self_attn.k_proj,4,64,8,128,5.75390625,4.25390625,8388608 +72,self_attn.k_proj,4,128,8,128,6.6875,4.126953125,8388608 +72,self_attn.k_proj,8,32,8,128,0.287841796875,8.5078125,8388608 +72,self_attn.k_proj,8,64,8,128,0.34814453125,8.25390625,8388608 +72,self_attn.k_proj,8,128,8,128,0.403076171875,8.126953125,8388608 +72,self_attn.o_proj,2,32,8,128,42.71875,20.0625,67108864 +72,self_attn.o_proj,2,64,8,128,49.75,18.03125,67108864 +72,self_attn.o_proj,2,128,8,128,56.09375,17.015625,67108864 +72,self_attn.o_proj,3,32,8,128,18.265625,28.0625,67108864 +72,self_attn.o_proj,3,64,8,128,21.296875,26.03125,67108864 +72,self_attn.o_proj,3,128,8,128,24.015625,25.015625,67108864 +72,self_attn.o_proj,4,32,8,128,8.5234375,36.0625,67108864 +72,self_attn.o_proj,4,64,8,128,9.9375,34.03125,67108864 +72,self_attn.o_proj,4,128,8,128,11.2109375,33.015625,67108864 +72,self_attn.o_proj,8,32,8,128,0.51611328125,68.0625,67108864 +72,self_attn.o_proj,8,64,8,128,0.60107421875,66.03125,67108864 +72,self_attn.o_proj,8,128,8,128,0.67578125,65.015625,67108864 +72,mlp.gate_proj,2,32,8,128,88.0,70.21875,234881024 +72,mlp.gate_proj,2,64,8,128,102.875,63.109375,234881024 +72,mlp.gate_proj,2,128,8,128,116.4375,59.5546875,234881024 +72,mlp.gate_proj,3,32,8,128,37.65625,98.21875,234881024 +72,mlp.gate_proj,3,64,8,128,44.0,91.109375,234881024 +72,mlp.gate_proj,3,128,8,128,49.875,87.5546875,234881024 +72,mlp.gate_proj,4,32,8,128,17.5625,126.21875,234881024 +72,mlp.gate_proj,4,64,8,128,20.546875,119.109375,234881024 +72,mlp.gate_proj,4,128,8,128,23.28125,115.5546875,234881024 +72,mlp.gate_proj,8,32,8,128,1.068359375,238.21875,234881024 +72,mlp.gate_proj,8,64,8,128,1.25,231.109375,234881024 +72,mlp.gate_proj,8,128,8,128,1.4111328125,227.5546875,234881024 +72,mlp.up_proj,2,32,8,128,83.75,70.21875,234881024 +72,mlp.up_proj,2,64,8,128,97.5,63.109375,234881024 +72,mlp.up_proj,2,128,8,128,110.0,59.5546875,234881024 +72,mlp.up_proj,3,32,8,128,35.8125,98.21875,234881024 +72,mlp.up_proj,3,64,8,128,41.71875,91.109375,234881024 +72,mlp.up_proj,3,128,8,128,47.03125,87.5546875,234881024 +72,mlp.up_proj,4,32,8,128,16.6875,126.21875,234881024 +72,mlp.up_proj,4,64,8,128,19.484375,119.109375,234881024 +72,mlp.up_proj,4,128,8,128,21.953125,115.5546875,234881024 +72,mlp.up_proj,8,32,8,128,1.015625,238.21875,234881024 +72,mlp.up_proj,8,64,8,128,1.1845703125,231.109375,234881024 +72,mlp.up_proj,8,128,8,128,1.33203125,227.5546875,234881024 +72,mlp.down_proj,2,32,8,128,83.0625,70.21875,234881024 +72,mlp.down_proj,2,64,8,128,96.6875,63.109375,234881024 +72,mlp.down_proj,2,128,8,128,109.0,59.5546875,234881024 +72,mlp.down_proj,3,32,8,128,35.53125,98.21875,234881024 +72,mlp.down_proj,3,64,8,128,41.375,91.109375,234881024 +72,mlp.down_proj,3,128,8,128,46.59375,87.5546875,234881024 +72,mlp.down_proj,4,32,8,128,16.5625,126.21875,234881024 +72,mlp.down_proj,4,64,8,128,19.3125,119.109375,234881024 +72,mlp.down_proj,4,128,8,128,21.75,115.5546875,234881024 +72,mlp.down_proj,8,32,8,128,1.0078125,238.21875,234881024 +72,mlp.down_proj,8,64,8,128,1.1748046875,231.109375,234881024 +72,mlp.down_proj,8,128,8,128,1.3193359375,227.5546875,234881024 +73,self_attn.q_proj,2,32,8,128,38.75,20.0625,67108864 +73,self_attn.q_proj,2,64,8,128,45.4375,18.03125,67108864 +73,self_attn.q_proj,2,128,8,128,52.0625,17.015625,67108864 +73,self_attn.q_proj,3,32,8,128,16.578125,28.0625,67108864 +73,self_attn.q_proj,3,64,8,128,19.46875,26.03125,67108864 +73,self_attn.q_proj,3,128,8,128,22.328125,25.015625,67108864 +73,self_attn.q_proj,4,32,8,128,7.734375,36.0625,67108864 +73,self_attn.q_proj,4,64,8,128,9.0859375,34.03125,67108864 +73,self_attn.q_proj,4,128,8,128,10.4296875,33.015625,67108864 +73,self_attn.q_proj,8,32,8,128,0.46826171875,68.0625,67108864 +73,self_attn.q_proj,8,64,8,128,0.5498046875,66.03125,67108864 +73,self_attn.q_proj,8,128,8,128,0.62890625,65.015625,67108864 +73,self_attn.v_proj,2,32,8,128,20.296875,2.5078125,8388608 +73,self_attn.v_proj,2,64,8,128,23.703125,2.25390625,8388608 +73,self_attn.v_proj,2,128,8,128,26.796875,2.126953125,8388608 +73,self_attn.v_proj,3,32,8,128,8.6796875,3.5078125,8388608 +73,self_attn.v_proj,3,64,8,128,10.140625,3.25390625,8388608 +73,self_attn.v_proj,3,128,8,128,11.4609375,3.126953125,8388608 +73,self_attn.v_proj,4,32,8,128,4.046875,4.5078125,8388608 +73,self_attn.v_proj,4,64,8,128,4.73046875,4.25390625,8388608 +73,self_attn.v_proj,4,128,8,128,5.3515625,4.126953125,8388608 +73,self_attn.v_proj,8,32,8,128,0.2452392578125,8.5078125,8388608 +73,self_attn.v_proj,8,64,8,128,0.2861328125,8.25390625,8388608 +73,self_attn.v_proj,8,128,8,128,0.322509765625,8.126953125,8388608 +73,self_attn.k_proj,2,32,8,128,22.453125,2.5078125,8388608 +73,self_attn.k_proj,2,64,8,128,26.921875,2.25390625,8388608 +73,self_attn.k_proj,2,128,8,128,30.921875,2.126953125,8388608 +73,self_attn.k_proj,3,32,8,128,9.640625,3.5078125,8388608 +73,self_attn.k_proj,3,64,8,128,11.625,3.25390625,8388608 +73,self_attn.k_proj,3,128,8,128,13.5078125,3.126953125,8388608 +73,self_attn.k_proj,4,32,8,128,4.49609375,4.5078125,8388608 +73,self_attn.k_proj,4,64,8,128,5.4296875,4.25390625,8388608 +73,self_attn.k_proj,4,128,8,128,6.3046875,4.126953125,8388608 +73,self_attn.k_proj,8,32,8,128,0.272216796875,8.5078125,8388608 +73,self_attn.k_proj,8,64,8,128,0.328369140625,8.25390625,8388608 +73,self_attn.k_proj,8,128,8,128,0.3798828125,8.126953125,8388608 +73,self_attn.o_proj,2,32,8,128,42.46875,20.0625,67108864 +73,self_attn.o_proj,2,64,8,128,49.4375,18.03125,67108864 +73,self_attn.o_proj,2,128,8,128,55.75,17.015625,67108864 +73,self_attn.o_proj,3,32,8,128,18.171875,28.0625,67108864 +73,self_attn.o_proj,3,64,8,128,21.1875,26.03125,67108864 +73,self_attn.o_proj,3,128,8,128,23.9375,25.015625,67108864 +73,self_attn.o_proj,4,32,8,128,8.4765625,36.0625,67108864 +73,self_attn.o_proj,4,64,8,128,9.890625,34.03125,67108864 +73,self_attn.o_proj,4,128,8,128,11.1796875,33.015625,67108864 +73,self_attn.o_proj,8,32,8,128,0.51318359375,68.0625,67108864 +73,self_attn.o_proj,8,64,8,128,0.59814453125,66.03125,67108864 +73,self_attn.o_proj,8,128,8,128,0.67333984375,65.015625,67108864 +73,mlp.gate_proj,2,32,8,128,87.9375,70.21875,234881024 +73,mlp.gate_proj,2,64,8,128,102.8125,63.109375,234881024 +73,mlp.gate_proj,2,128,8,128,116.375,59.5546875,234881024 +73,mlp.gate_proj,3,32,8,128,37.625,98.21875,234881024 +73,mlp.gate_proj,3,64,8,128,43.96875,91.109375,234881024 +73,mlp.gate_proj,3,128,8,128,49.875,87.5546875,234881024 +73,mlp.gate_proj,4,32,8,128,17.546875,126.21875,234881024 +73,mlp.gate_proj,4,64,8,128,20.53125,119.109375,234881024 +73,mlp.gate_proj,4,128,8,128,23.265625,115.5546875,234881024 +73,mlp.gate_proj,8,32,8,128,1.0673828125,238.21875,234881024 +73,mlp.gate_proj,8,64,8,128,1.25,231.109375,234881024 +73,mlp.gate_proj,8,128,8,128,1.4111328125,227.5546875,234881024 +73,mlp.up_proj,2,32,8,128,83.9375,70.21875,234881024 +73,mlp.up_proj,2,64,8,128,97.75,63.109375,234881024 +73,mlp.up_proj,2,128,8,128,110.25,59.5546875,234881024 +73,mlp.up_proj,3,32,8,128,35.90625,98.21875,234881024 +73,mlp.up_proj,3,64,8,128,41.8125,91.109375,234881024 +73,mlp.up_proj,3,128,8,128,47.1875,87.5546875,234881024 +73,mlp.up_proj,4,32,8,128,16.734375,126.21875,234881024 +73,mlp.up_proj,4,64,8,128,19.53125,119.109375,234881024 +73,mlp.up_proj,4,128,8,128,22.015625,115.5546875,234881024 +73,mlp.up_proj,8,32,8,128,1.0185546875,238.21875,234881024 +73,mlp.up_proj,8,64,8,128,1.1884765625,231.109375,234881024 +73,mlp.up_proj,8,128,8,128,1.3359375,227.5546875,234881024 +73,mlp.down_proj,2,32,8,128,83.25,70.21875,234881024 +73,mlp.down_proj,2,64,8,128,96.875,63.109375,234881024 +73,mlp.down_proj,2,128,8,128,109.25,59.5546875,234881024 +73,mlp.down_proj,3,32,8,128,35.59375,98.21875,234881024 +73,mlp.down_proj,3,64,8,128,41.46875,91.109375,234881024 +73,mlp.down_proj,3,128,8,128,46.71875,87.5546875,234881024 +73,mlp.down_proj,4,32,8,128,16.59375,126.21875,234881024 +73,mlp.down_proj,4,64,8,128,19.359375,119.109375,234881024 +73,mlp.down_proj,4,128,8,128,21.796875,115.5546875,234881024 +73,mlp.down_proj,8,32,8,128,1.009765625,238.21875,234881024 +73,mlp.down_proj,8,64,8,128,1.177734375,231.109375,234881024 +73,mlp.down_proj,8,128,8,128,1.322265625,227.5546875,234881024 +74,self_attn.q_proj,2,32,8,128,42.78125,20.0625,67108864 +74,self_attn.q_proj,2,64,8,128,50.59375,18.03125,67108864 +74,self_attn.q_proj,2,128,8,128,58.28125,17.015625,67108864 +74,self_attn.q_proj,3,32,8,128,18.3125,28.0625,67108864 +74,self_attn.q_proj,3,64,8,128,21.71875,26.03125,67108864 +74,self_attn.q_proj,3,128,8,128,25.125,25.015625,67108864 +74,self_attn.q_proj,4,32,8,128,8.546875,36.0625,67108864 +74,self_attn.q_proj,4,64,8,128,10.1328125,34.03125,67108864 +74,self_attn.q_proj,4,128,8,128,11.7265625,33.015625,67108864 +74,self_attn.q_proj,8,32,8,128,0.517578125,68.0625,67108864 +74,self_attn.q_proj,8,64,8,128,0.61328125,66.03125,67108864 +74,self_attn.q_proj,8,128,8,128,0.70703125,65.015625,67108864 +74,self_attn.v_proj,2,32,8,128,19.25,2.5078125,8388608 +74,self_attn.v_proj,2,64,8,128,22.5625,2.25390625,8388608 +74,self_attn.v_proj,2,128,8,128,25.5625,2.126953125,8388608 +74,self_attn.v_proj,3,32,8,128,8.2265625,3.5078125,8388608 +74,self_attn.v_proj,3,64,8,128,9.6484375,3.25390625,8388608 +74,self_attn.v_proj,3,128,8,128,10.9453125,3.126953125,8388608 +74,self_attn.v_proj,4,32,8,128,3.837890625,4.5078125,8388608 +74,self_attn.v_proj,4,64,8,128,4.5,4.25390625,8388608 +74,self_attn.v_proj,4,128,8,128,5.10546875,4.126953125,8388608 +74,self_attn.v_proj,8,32,8,128,0.2325439453125,8.5078125,8388608 +74,self_attn.v_proj,8,64,8,128,0.2724609375,8.25390625,8388608 +74,self_attn.v_proj,8,128,8,128,0.3076171875,8.126953125,8388608 +74,self_attn.k_proj,2,32,8,128,23.296875,2.5078125,8388608 +74,self_attn.k_proj,2,64,8,128,27.96875,2.25390625,8388608 +74,self_attn.k_proj,2,128,8,128,32.0625,2.126953125,8388608 +74,self_attn.k_proj,3,32,8,128,10.0078125,3.5078125,8388608 +74,self_attn.k_proj,3,64,8,128,12.1171875,3.25390625,8388608 +74,self_attn.k_proj,3,128,8,128,14.0859375,3.126953125,8388608 +74,self_attn.k_proj,4,32,8,128,4.671875,4.5078125,8388608 +74,self_attn.k_proj,4,64,8,128,5.65625,4.25390625,8388608 +74,self_attn.k_proj,4,128,8,128,6.578125,4.126953125,8388608 +74,self_attn.k_proj,8,32,8,128,0.28271484375,8.5078125,8388608 +74,self_attn.k_proj,8,64,8,128,0.342041015625,8.25390625,8388608 +74,self_attn.k_proj,8,128,8,128,0.396484375,8.126953125,8388608 +74,self_attn.o_proj,2,32,8,128,42.6875,20.0625,67108864 +74,self_attn.o_proj,2,64,8,128,49.8125,18.03125,67108864 +74,self_attn.o_proj,2,128,8,128,56.3125,17.015625,67108864 +74,self_attn.o_proj,3,32,8,128,18.28125,28.0625,67108864 +74,self_attn.o_proj,3,64,8,128,21.359375,26.03125,67108864 +74,self_attn.o_proj,3,128,8,128,24.234375,25.015625,67108864 +74,self_attn.o_proj,4,32,8,128,8.5234375,36.0625,67108864 +74,self_attn.o_proj,4,64,8,128,9.9765625,34.03125,67108864 +74,self_attn.o_proj,4,128,8,128,11.3203125,33.015625,67108864 +74,self_attn.o_proj,8,32,8,128,0.5166015625,68.0625,67108864 +74,self_attn.o_proj,8,64,8,128,0.603515625,66.03125,67108864 +74,self_attn.o_proj,8,128,8,128,0.68212890625,65.015625,67108864 +74,mlp.gate_proj,2,32,8,128,87.8125,70.21875,234881024 +74,mlp.gate_proj,2,64,8,128,102.6875,63.109375,234881024 +74,mlp.gate_proj,2,128,8,128,116.1875,59.5546875,234881024 +74,mlp.gate_proj,3,32,8,128,37.5625,98.21875,234881024 +74,mlp.gate_proj,3,64,8,128,43.90625,91.109375,234881024 +74,mlp.gate_proj,3,128,8,128,49.78125,87.5546875,234881024 +74,mlp.gate_proj,4,32,8,128,17.515625,126.21875,234881024 +74,mlp.gate_proj,4,64,8,128,20.5,119.109375,234881024 +74,mlp.gate_proj,4,128,8,128,23.234375,115.5546875,234881024 +74,mlp.gate_proj,8,32,8,128,1.06640625,238.21875,234881024 +74,mlp.gate_proj,8,64,8,128,1.248046875,231.109375,234881024 +74,mlp.gate_proj,8,128,8,128,1.4091796875,227.5546875,234881024 +74,mlp.up_proj,2,32,8,128,84.0,70.21875,234881024 +74,mlp.up_proj,2,64,8,128,97.875,63.109375,234881024 +74,mlp.up_proj,2,128,8,128,110.4375,59.5546875,234881024 +74,mlp.up_proj,3,32,8,128,35.9375,98.21875,234881024 +74,mlp.up_proj,3,64,8,128,41.875,91.109375,234881024 +74,mlp.up_proj,3,128,8,128,47.25,87.5546875,234881024 +74,mlp.up_proj,4,32,8,128,16.75,126.21875,234881024 +74,mlp.up_proj,4,64,8,128,19.5625,119.109375,234881024 +74,mlp.up_proj,4,128,8,128,22.046875,115.5546875,234881024 +74,mlp.up_proj,8,32,8,128,1.01953125,238.21875,234881024 +74,mlp.up_proj,8,64,8,128,1.189453125,231.109375,234881024 +74,mlp.up_proj,8,128,8,128,1.337890625,227.5546875,234881024 +74,mlp.down_proj,2,32,8,128,83.3125,70.21875,234881024 +74,mlp.down_proj,2,64,8,128,97.0,63.109375,234881024 +74,mlp.down_proj,2,128,8,128,109.3125,59.5546875,234881024 +74,mlp.down_proj,3,32,8,128,35.625,98.21875,234881024 +74,mlp.down_proj,3,64,8,128,41.5,91.109375,234881024 +74,mlp.down_proj,3,128,8,128,46.75,87.5546875,234881024 +74,mlp.down_proj,4,32,8,128,16.609375,126.21875,234881024 +74,mlp.down_proj,4,64,8,128,19.375,119.109375,234881024 +74,mlp.down_proj,4,128,8,128,21.828125,115.5546875,234881024 +74,mlp.down_proj,8,32,8,128,1.0107421875,238.21875,234881024 +74,mlp.down_proj,8,64,8,128,1.1787109375,231.109375,234881024 +74,mlp.down_proj,8,128,8,128,1.32421875,227.5546875,234881024 +75,self_attn.q_proj,2,32,8,128,40.1875,20.0625,67108864 +75,self_attn.q_proj,2,64,8,128,47.375,18.03125,67108864 +75,self_attn.q_proj,2,128,8,128,54.375,17.015625,67108864 +75,self_attn.q_proj,3,32,8,128,17.203125,28.0625,67108864 +75,self_attn.q_proj,3,64,8,128,20.296875,26.03125,67108864 +75,self_attn.q_proj,3,128,8,128,23.34375,25.015625,67108864 +75,self_attn.q_proj,4,32,8,128,8.03125,36.0625,67108864 +75,self_attn.q_proj,4,64,8,128,9.4765625,34.03125,67108864 +75,self_attn.q_proj,4,128,8,128,10.90625,33.015625,67108864 +75,self_attn.q_proj,8,32,8,128,0.486083984375,68.0625,67108864 +75,self_attn.q_proj,8,64,8,128,0.57373046875,66.03125,67108864 +75,self_attn.q_proj,8,128,8,128,0.65771484375,65.015625,67108864 +75,self_attn.v_proj,2,32,8,128,21.140625,2.5078125,8388608 +75,self_attn.v_proj,2,64,8,128,24.875,2.25390625,8388608 +75,self_attn.v_proj,2,128,8,128,28.375,2.126953125,8388608 +75,self_attn.v_proj,3,32,8,128,9.0390625,3.5078125,8388608 +75,self_attn.v_proj,3,64,8,128,10.640625,3.25390625,8388608 +75,self_attn.v_proj,3,128,8,128,12.140625,3.126953125,8388608 +75,self_attn.v_proj,4,32,8,128,4.21484375,4.5078125,8388608 +75,self_attn.v_proj,4,64,8,128,4.96875,4.25390625,8388608 +75,self_attn.v_proj,4,128,8,128,5.6640625,4.126953125,8388608 +75,self_attn.v_proj,8,32,8,128,0.25537109375,8.5078125,8388608 +75,self_attn.v_proj,8,64,8,128,0.300537109375,8.25390625,8388608 +75,self_attn.v_proj,8,128,8,128,0.341552734375,8.126953125,8388608 +75,self_attn.k_proj,2,32,8,128,21.78125,2.5078125,8388608 +75,self_attn.k_proj,2,64,8,128,26.15625,2.25390625,8388608 +75,self_attn.k_proj,2,128,8,128,30.046875,2.126953125,8388608 +75,self_attn.k_proj,3,32,8,128,9.3515625,3.5078125,8388608 +75,self_attn.k_proj,3,64,8,128,11.296875,3.25390625,8388608 +75,self_attn.k_proj,3,128,8,128,13.1328125,3.126953125,8388608 +75,self_attn.k_proj,4,32,8,128,4.359375,4.5078125,8388608 +75,self_attn.k_proj,4,64,8,128,5.26953125,4.25390625,8388608 +75,self_attn.k_proj,4,128,8,128,6.12890625,4.126953125,8388608 +75,self_attn.k_proj,8,32,8,128,0.26416015625,8.5078125,8388608 +75,self_attn.k_proj,8,64,8,128,0.31884765625,8.25390625,8388608 +75,self_attn.k_proj,8,128,8,128,0.36962890625,8.126953125,8388608 +75,self_attn.o_proj,2,32,8,128,42.78125,20.0625,67108864 +75,self_attn.o_proj,2,64,8,128,49.90625,18.03125,67108864 +75,self_attn.o_proj,2,128,8,128,56.40625,17.015625,67108864 +75,self_attn.o_proj,3,32,8,128,18.328125,28.0625,67108864 +75,self_attn.o_proj,3,64,8,128,21.421875,26.03125,67108864 +75,self_attn.o_proj,3,128,8,128,24.265625,25.015625,67108864 +75,self_attn.o_proj,4,32,8,128,8.5546875,36.0625,67108864 +75,self_attn.o_proj,4,64,8,128,10.0078125,34.03125,67108864 +75,self_attn.o_proj,4,128,8,128,11.3515625,33.015625,67108864 +75,self_attn.o_proj,8,32,8,128,0.51806640625,68.0625,67108864 +75,self_attn.o_proj,8,64,8,128,0.60498046875,66.03125,67108864 +75,self_attn.o_proj,8,128,8,128,0.68408203125,65.015625,67108864 +75,mlp.gate_proj,2,32,8,128,87.875,70.21875,234881024 +75,mlp.gate_proj,2,64,8,128,102.8125,63.109375,234881024 +75,mlp.gate_proj,2,128,8,128,116.375,59.5546875,234881024 +75,mlp.gate_proj,3,32,8,128,37.625,98.21875,234881024 +75,mlp.gate_proj,3,64,8,128,44.0,91.109375,234881024 +75,mlp.gate_proj,3,128,8,128,49.875,87.5546875,234881024 +75,mlp.gate_proj,4,32,8,128,17.53125,126.21875,234881024 +75,mlp.gate_proj,4,64,8,128,20.53125,119.109375,234881024 +75,mlp.gate_proj,4,128,8,128,23.28125,115.5546875,234881024 +75,mlp.gate_proj,8,32,8,128,1.0673828125,238.21875,234881024 +75,mlp.gate_proj,8,64,8,128,1.25,231.109375,234881024 +75,mlp.gate_proj,8,128,8,128,1.412109375,227.5546875,234881024 +75,mlp.up_proj,2,32,8,128,84.0625,70.21875,234881024 +75,mlp.up_proj,2,64,8,128,98.0,63.109375,234881024 +75,mlp.up_proj,2,128,8,128,110.625,59.5546875,234881024 +75,mlp.up_proj,3,32,8,128,35.96875,98.21875,234881024 +75,mlp.up_proj,3,64,8,128,41.9375,91.109375,234881024 +75,mlp.up_proj,3,128,8,128,47.34375,87.5546875,234881024 +75,mlp.up_proj,4,32,8,128,16.765625,126.21875,234881024 +75,mlp.up_proj,4,64,8,128,19.59375,119.109375,234881024 +75,mlp.up_proj,4,128,8,128,22.09375,115.5546875,234881024 +75,mlp.up_proj,8,32,8,128,1.0205078125,238.21875,234881024 +75,mlp.up_proj,8,64,8,128,1.19140625,231.109375,234881024 +75,mlp.up_proj,8,128,8,128,1.33984375,227.5546875,234881024 +75,mlp.down_proj,2,32,8,128,83.25,70.21875,234881024 +75,mlp.down_proj,2,64,8,128,96.9375,63.109375,234881024 +75,mlp.down_proj,2,128,8,128,109.3125,59.5546875,234881024 +75,mlp.down_proj,3,32,8,128,35.59375,98.21875,234881024 +75,mlp.down_proj,3,64,8,128,41.5,91.109375,234881024 +75,mlp.down_proj,3,128,8,128,46.78125,87.5546875,234881024 +75,mlp.down_proj,4,32,8,128,16.609375,126.21875,234881024 +75,mlp.down_proj,4,64,8,128,19.375,119.109375,234881024 +75,mlp.down_proj,4,128,8,128,21.84375,115.5546875,234881024 +75,mlp.down_proj,8,32,8,128,1.009765625,238.21875,234881024 +75,mlp.down_proj,8,64,8,128,1.1787109375,231.109375,234881024 +75,mlp.down_proj,8,128,8,128,1.326171875,227.5546875,234881024 +76,self_attn.q_proj,2,32,8,128,43.75,20.0625,67108864 +76,self_attn.q_proj,2,64,8,128,51.34375,18.03125,67108864 +76,self_attn.q_proj,2,128,8,128,58.9375,17.015625,67108864 +76,self_attn.q_proj,3,32,8,128,18.71875,28.0625,67108864 +76,self_attn.q_proj,3,64,8,128,21.984375,26.03125,67108864 +76,self_attn.q_proj,3,128,8,128,25.296875,25.015625,67108864 +76,self_attn.q_proj,4,32,8,128,8.734375,36.0625,67108864 +76,self_attn.q_proj,4,64,8,128,10.265625,34.03125,67108864 +76,self_attn.q_proj,4,128,8,128,11.8046875,33.015625,67108864 +76,self_attn.q_proj,8,32,8,128,0.52880859375,68.0625,67108864 +76,self_attn.q_proj,8,64,8,128,0.62109375,66.03125,67108864 +76,self_attn.q_proj,8,128,8,128,0.7119140625,65.015625,67108864 +76,self_attn.v_proj,2,32,8,128,22.40625,2.5078125,8388608 +76,self_attn.v_proj,2,64,8,128,26.40625,2.25390625,8388608 +76,self_attn.v_proj,2,128,8,128,30.15625,2.126953125,8388608 +76,self_attn.v_proj,3,32,8,128,9.578125,3.5078125,8388608 +76,self_attn.v_proj,3,64,8,128,11.3046875,3.25390625,8388608 +76,self_attn.v_proj,3,128,8,128,12.9140625,3.126953125,8388608 +76,self_attn.v_proj,4,32,8,128,4.46875,4.5078125,8388608 +76,self_attn.v_proj,4,64,8,128,5.2734375,4.25390625,8388608 +76,self_attn.v_proj,4,128,8,128,6.0234375,4.126953125,8388608 +76,self_attn.v_proj,8,32,8,128,0.270751953125,8.5078125,8388608 +76,self_attn.v_proj,8,64,8,128,0.319091796875,8.25390625,8388608 +76,self_attn.v_proj,8,128,8,128,0.363037109375,8.126953125,8388608 +76,self_attn.k_proj,2,32,8,128,22.375,2.5078125,8388608 +76,self_attn.k_proj,2,64,8,128,26.765625,2.25390625,8388608 +76,self_attn.k_proj,2,128,8,128,30.671875,2.126953125,8388608 +76,self_attn.k_proj,3,32,8,128,9.5859375,3.5078125,8388608 +76,self_attn.k_proj,3,64,8,128,11.5078125,3.25390625,8388608 +76,self_attn.k_proj,3,128,8,128,13.265625,3.126953125,8388608 +76,self_attn.k_proj,4,32,8,128,4.47265625,4.5078125,8388608 +76,self_attn.k_proj,4,64,8,128,5.37109375,4.25390625,8388608 +76,self_attn.k_proj,4,128,8,128,6.19140625,4.126953125,8388608 +76,self_attn.k_proj,8,32,8,128,0.270751953125,8.5078125,8388608 +76,self_attn.k_proj,8,64,8,128,0.324951171875,8.25390625,8388608 +76,self_attn.k_proj,8,128,8,128,0.373046875,8.126953125,8388608 +76,self_attn.o_proj,2,32,8,128,46.375,20.0625,67108864 +76,self_attn.o_proj,2,64,8,128,54.03125,18.03125,67108864 +76,self_attn.o_proj,2,128,8,128,60.96875,17.015625,67108864 +76,self_attn.o_proj,3,32,8,128,19.828125,28.0625,67108864 +76,self_attn.o_proj,3,64,8,128,23.140625,26.03125,67108864 +76,self_attn.o_proj,3,128,8,128,26.140625,25.015625,67108864 +76,self_attn.o_proj,4,32,8,128,9.25,36.0625,67108864 +76,self_attn.o_proj,4,64,8,128,10.796875,34.03125,67108864 +76,self_attn.o_proj,4,128,8,128,12.2109375,33.015625,67108864 +76,self_attn.o_proj,8,32,8,128,0.560546875,68.0625,67108864 +76,self_attn.o_proj,8,64,8,128,0.65283203125,66.03125,67108864 +76,self_attn.o_proj,8,128,8,128,0.7353515625,65.015625,67108864 +76,mlp.gate_proj,2,32,8,128,88.5,70.21875,234881024 +76,mlp.gate_proj,2,64,8,128,103.5625,63.109375,234881024 +76,mlp.gate_proj,2,128,8,128,117.375,59.5546875,234881024 +76,mlp.gate_proj,3,32,8,128,37.875,98.21875,234881024 +76,mlp.gate_proj,3,64,8,128,44.3125,91.109375,234881024 +76,mlp.gate_proj,3,128,8,128,50.3125,87.5546875,234881024 +76,mlp.gate_proj,4,32,8,128,17.65625,126.21875,234881024 +76,mlp.gate_proj,4,64,8,128,20.6875,119.109375,234881024 +76,mlp.gate_proj,4,128,8,128,23.484375,115.5546875,234881024 +76,mlp.gate_proj,8,32,8,128,1.07421875,238.21875,234881024 +76,mlp.gate_proj,8,64,8,128,1.2587890625,231.109375,234881024 +76,mlp.gate_proj,8,128,8,128,1.423828125,227.5546875,234881024 +76,mlp.up_proj,2,32,8,128,84.25,70.21875,234881024 +76,mlp.up_proj,2,64,8,128,98.25,63.109375,234881024 +76,mlp.up_proj,2,128,8,128,111.0,59.5546875,234881024 +76,mlp.up_proj,3,32,8,128,36.03125,98.21875,234881024 +76,mlp.up_proj,3,64,8,128,42.0625,91.109375,234881024 +76,mlp.up_proj,3,128,8,128,47.53125,87.5546875,234881024 +76,mlp.up_proj,4,32,8,128,16.796875,126.21875,234881024 +76,mlp.up_proj,4,64,8,128,19.640625,119.109375,234881024 +76,mlp.up_proj,4,128,8,128,22.203125,115.5546875,234881024 +76,mlp.up_proj,8,32,8,128,1.0224609375,238.21875,234881024 +76,mlp.up_proj,8,64,8,128,1.1943359375,231.109375,234881024 +76,mlp.up_proj,8,128,8,128,1.345703125,227.5546875,234881024 +76,mlp.down_proj,2,32,8,128,83.125,70.21875,234881024 +76,mlp.down_proj,2,64,8,128,96.8125,63.109375,234881024 +76,mlp.down_proj,2,128,8,128,109.3125,59.5546875,234881024 +76,mlp.down_proj,3,32,8,128,35.5625,98.21875,234881024 +76,mlp.down_proj,3,64,8,128,41.46875,91.109375,234881024 +76,mlp.down_proj,3,128,8,128,46.84375,87.5546875,234881024 +76,mlp.down_proj,4,32,8,128,16.578125,126.21875,234881024 +76,mlp.down_proj,4,64,8,128,19.375,119.109375,234881024 +76,mlp.down_proj,4,128,8,128,21.859375,115.5546875,234881024 +76,mlp.down_proj,8,32,8,128,1.0087890625,238.21875,234881024 +76,mlp.down_proj,8,64,8,128,1.1787109375,231.109375,234881024 +76,mlp.down_proj,8,128,8,128,1.3271484375,227.5546875,234881024 +77,self_attn.q_proj,2,32,8,128,41.84375,20.0625,67108864 +77,self_attn.q_proj,2,64,8,128,49.3125,18.03125,67108864 +77,self_attn.q_proj,2,128,8,128,56.8125,17.015625,67108864 +77,self_attn.q_proj,3,32,8,128,17.921875,28.0625,67108864 +77,self_attn.q_proj,3,64,8,128,21.140625,26.03125,67108864 +77,self_attn.q_proj,3,128,8,128,24.4375,25.015625,67108864 +77,self_attn.q_proj,4,32,8,128,8.359375,36.0625,67108864 +77,self_attn.q_proj,4,64,8,128,9.875,34.03125,67108864 +77,self_attn.q_proj,4,128,8,128,11.40625,33.015625,67108864 +77,self_attn.q_proj,8,32,8,128,0.50634765625,68.0625,67108864 +77,self_attn.q_proj,8,64,8,128,0.59716796875,66.03125,67108864 +77,self_attn.q_proj,8,128,8,128,0.6875,65.015625,67108864 +77,self_attn.v_proj,2,32,8,128,21.484375,2.5078125,8388608 +77,self_attn.v_proj,2,64,8,128,25.28125,2.25390625,8388608 +77,self_attn.v_proj,2,128,8,128,28.765625,2.126953125,8388608 +77,self_attn.v_proj,3,32,8,128,9.1875,3.5078125,8388608 +77,self_attn.v_proj,3,64,8,128,10.8203125,3.25390625,8388608 +77,self_attn.v_proj,3,128,8,128,12.3125,3.126953125,8388608 +77,self_attn.v_proj,4,32,8,128,4.28515625,4.5078125,8388608 +77,self_attn.v_proj,4,64,8,128,5.046875,4.25390625,8388608 +77,self_attn.v_proj,4,128,8,128,5.74609375,4.126953125,8388608 +77,self_attn.v_proj,8,32,8,128,0.259521484375,8.5078125,8388608 +77,self_attn.v_proj,8,64,8,128,0.305419921875,8.25390625,8388608 +77,self_attn.v_proj,8,128,8,128,0.346435546875,8.126953125,8388608 +77,self_attn.k_proj,2,32,8,128,22.234375,2.5078125,8388608 +77,self_attn.k_proj,2,64,8,128,26.75,2.25390625,8388608 +77,self_attn.k_proj,2,128,8,128,30.828125,2.126953125,8388608 +77,self_attn.k_proj,3,32,8,128,9.546875,3.5078125,8388608 +77,self_attn.k_proj,3,64,8,128,11.5546875,3.25390625,8388608 +77,self_attn.k_proj,3,128,8,128,13.46875,3.126953125,8388608 +77,self_attn.k_proj,4,32,8,128,4.45703125,4.5078125,8388608 +77,self_attn.k_proj,4,64,8,128,5.390625,4.25390625,8388608 +77,self_attn.k_proj,4,128,8,128,6.28125,4.126953125,8388608 +77,self_attn.k_proj,8,32,8,128,0.269775390625,8.5078125,8388608 +77,self_attn.k_proj,8,64,8,128,0.326171875,8.25390625,8388608 +77,self_attn.k_proj,8,128,8,128,0.378662109375,8.126953125,8388608 +77,self_attn.o_proj,2,32,8,128,43.8125,20.0625,67108864 +77,self_attn.o_proj,2,64,8,128,51.09375,18.03125,67108864 +77,self_attn.o_proj,2,128,8,128,57.75,17.015625,67108864 +77,self_attn.o_proj,3,32,8,128,18.75,28.0625,67108864 +77,self_attn.o_proj,3,64,8,128,21.90625,26.03125,67108864 +77,self_attn.o_proj,3,128,8,128,24.828125,25.015625,67108864 +77,self_attn.o_proj,4,32,8,128,8.75,36.0625,67108864 +77,self_attn.o_proj,4,64,8,128,10.2265625,34.03125,67108864 +77,self_attn.o_proj,4,128,8,128,11.5859375,33.015625,67108864 +77,self_attn.o_proj,8,32,8,128,0.52978515625,68.0625,67108864 +77,self_attn.o_proj,8,64,8,128,0.61865234375,66.03125,67108864 +77,self_attn.o_proj,8,128,8,128,0.6982421875,65.015625,67108864 +77,mlp.gate_proj,2,32,8,128,88.8125,70.21875,234881024 +77,mlp.gate_proj,2,64,8,128,103.9375,63.109375,234881024 +77,mlp.gate_proj,2,128,8,128,117.75,59.5546875,234881024 +77,mlp.gate_proj,3,32,8,128,38.03125,98.21875,234881024 +77,mlp.gate_proj,3,64,8,128,44.5,91.109375,234881024 +77,mlp.gate_proj,3,128,8,128,50.46875,87.5546875,234881024 +77,mlp.gate_proj,4,32,8,128,17.734375,126.21875,234881024 +77,mlp.gate_proj,4,64,8,128,20.765625,119.109375,234881024 +77,mlp.gate_proj,4,128,8,128,23.578125,115.5546875,234881024 +77,mlp.gate_proj,8,32,8,128,1.0791015625,238.21875,234881024 +77,mlp.gate_proj,8,64,8,128,1.263671875,231.109375,234881024 +77,mlp.gate_proj,8,128,8,128,1.4296875,227.5546875,234881024 +77,mlp.up_proj,2,32,8,128,84.75,70.21875,234881024 +77,mlp.up_proj,2,64,8,128,98.8125,63.109375,234881024 +77,mlp.up_proj,2,128,8,128,111.625,59.5546875,234881024 +77,mlp.up_proj,3,32,8,128,36.25,98.21875,234881024 +77,mlp.up_proj,3,64,8,128,42.28125,91.109375,234881024 +77,mlp.up_proj,3,128,8,128,47.8125,87.5546875,234881024 +77,mlp.up_proj,4,32,8,128,16.90625,126.21875,234881024 +77,mlp.up_proj,4,64,8,128,19.75,119.109375,234881024 +77,mlp.up_proj,4,128,8,128,22.328125,115.5546875,234881024 +77,mlp.up_proj,8,32,8,128,1.0283203125,238.21875,234881024 +77,mlp.up_proj,8,64,8,128,1.2021484375,231.109375,234881024 +77,mlp.up_proj,8,128,8,128,1.353515625,227.5546875,234881024 +77,mlp.down_proj,2,32,8,128,83.625,70.21875,234881024 +77,mlp.down_proj,2,64,8,128,97.5,63.109375,234881024 +77,mlp.down_proj,2,128,8,128,110.3125,59.5546875,234881024 +77,mlp.down_proj,3,32,8,128,35.78125,98.21875,234881024 +77,mlp.down_proj,3,64,8,128,41.8125,91.109375,234881024 +77,mlp.down_proj,3,128,8,128,47.375,87.5546875,234881024 +77,mlp.down_proj,4,32,8,128,16.703125,126.21875,234881024 +77,mlp.down_proj,4,64,8,128,19.546875,119.109375,234881024 +77,mlp.down_proj,4,128,8,128,22.15625,115.5546875,234881024 +77,mlp.down_proj,8,32,8,128,1.0166015625,238.21875,234881024 +77,mlp.down_proj,8,64,8,128,1.189453125,231.109375,234881024 +77,mlp.down_proj,8,128,8,128,1.3447265625,227.5546875,234881024 +78,self_attn.q_proj,2,32,8,128,40.78125,20.0625,67108864 +78,self_attn.q_proj,2,64,8,128,48.09375,18.03125,67108864 +78,self_attn.q_proj,2,128,8,128,55.0625,17.015625,67108864 +78,self_attn.q_proj,3,32,8,128,17.46875,28.0625,67108864 +78,self_attn.q_proj,3,64,8,128,20.640625,26.03125,67108864 +78,self_attn.q_proj,3,128,8,128,23.734375,25.015625,67108864 +78,self_attn.q_proj,4,32,8,128,8.1484375,36.0625,67108864 +78,self_attn.q_proj,4,64,8,128,9.6328125,34.03125,67108864 +78,self_attn.q_proj,4,128,8,128,11.0859375,33.015625,67108864 +78,self_attn.q_proj,8,32,8,128,0.493408203125,68.0625,67108864 +78,self_attn.q_proj,8,64,8,128,0.5830078125,66.03125,67108864 +78,self_attn.q_proj,8,128,8,128,0.66796875,65.015625,67108864 +78,self_attn.v_proj,2,32,8,128,19.75,2.5078125,8388608 +78,self_attn.v_proj,2,64,8,128,23.25,2.25390625,8388608 +78,self_attn.v_proj,2,128,8,128,26.515625,2.126953125,8388608 +78,self_attn.v_proj,3,32,8,128,8.4453125,3.5078125,8388608 +78,self_attn.v_proj,3,64,8,128,9.953125,3.25390625,8388608 +78,self_attn.v_proj,3,128,8,128,11.359375,3.126953125,8388608 +78,self_attn.v_proj,4,32,8,128,3.939453125,4.5078125,8388608 +78,self_attn.v_proj,4,64,8,128,4.64453125,4.25390625,8388608 +78,self_attn.v_proj,4,128,8,128,5.30078125,4.126953125,8388608 +78,self_attn.v_proj,8,32,8,128,0.238525390625,8.5078125,8388608 +78,self_attn.v_proj,8,64,8,128,0.281005859375,8.25390625,8388608 +78,self_attn.v_proj,8,128,8,128,0.319580078125,8.126953125,8388608 +78,self_attn.k_proj,2,32,8,128,22.203125,2.5078125,8388608 +78,self_attn.k_proj,2,64,8,128,26.671875,2.25390625,8388608 +78,self_attn.k_proj,2,128,8,128,30.546875,2.126953125,8388608 +78,self_attn.k_proj,3,32,8,128,9.53125,3.5078125,8388608 +78,self_attn.k_proj,3,64,8,128,11.515625,3.25390625,8388608 +78,self_attn.k_proj,3,128,8,128,13.3359375,3.126953125,8388608 +78,self_attn.k_proj,4,32,8,128,4.4453125,4.5078125,8388608 +78,self_attn.k_proj,4,64,8,128,5.37109375,4.25390625,8388608 +78,self_attn.k_proj,4,128,8,128,6.23046875,4.126953125,8388608 +78,self_attn.k_proj,8,32,8,128,0.269287109375,8.5078125,8388608 +78,self_attn.k_proj,8,64,8,128,0.324951171875,8.25390625,8388608 +78,self_attn.k_proj,8,128,8,128,0.375244140625,8.126953125,8388608 +78,self_attn.o_proj,2,32,8,128,41.0625,20.0625,67108864 +78,self_attn.o_proj,2,64,8,128,47.875,18.03125,67108864 +78,self_attn.o_proj,2,128,8,128,54.125,17.015625,67108864 +78,self_attn.o_proj,3,32,8,128,17.59375,28.0625,67108864 +78,self_attn.o_proj,3,64,8,128,20.5625,26.03125,67108864 +78,self_attn.o_proj,3,128,8,128,23.3125,25.015625,67108864 +78,self_attn.o_proj,4,32,8,128,8.21875,36.0625,67108864 +78,self_attn.o_proj,4,64,8,128,9.625,34.03125,67108864 +78,self_attn.o_proj,4,128,8,128,10.9375,33.015625,67108864 +78,self_attn.o_proj,8,32,8,128,0.4990234375,68.0625,67108864 +78,self_attn.o_proj,8,64,8,128,0.5849609375,66.03125,67108864 +78,self_attn.o_proj,8,128,8,128,0.6640625,65.015625,67108864 +78,mlp.gate_proj,2,32,8,128,90.125,70.21875,234881024 +78,mlp.gate_proj,2,64,8,128,105.6875,63.109375,234881024 +78,mlp.gate_proj,2,128,8,128,119.875,59.5546875,234881024 +78,mlp.gate_proj,3,32,8,128,38.59375,98.21875,234881024 +78,mlp.gate_proj,3,64,8,128,45.21875,91.109375,234881024 +78,mlp.gate_proj,3,128,8,128,51.5,87.5546875,234881024 +78,mlp.gate_proj,4,32,8,128,18.0,126.21875,234881024 +78,mlp.gate_proj,4,64,8,128,21.109375,119.109375,234881024 +78,mlp.gate_proj,4,128,8,128,24.03125,115.5546875,234881024 +78,mlp.gate_proj,8,32,8,128,1.0947265625,238.21875,234881024 +78,mlp.gate_proj,8,64,8,128,1.28515625,231.109375,234881024 +78,mlp.gate_proj,8,128,8,128,1.45703125,227.5546875,234881024 +78,mlp.up_proj,2,32,8,128,85.5,70.21875,234881024 +78,mlp.up_proj,2,64,8,128,99.75,63.109375,234881024 +78,mlp.up_proj,2,128,8,128,112.625,59.5546875,234881024 +78,mlp.up_proj,3,32,8,128,36.59375,98.21875,234881024 +78,mlp.up_proj,3,64,8,128,42.71875,91.109375,234881024 +78,mlp.up_proj,3,128,8,128,48.28125,87.5546875,234881024 +78,mlp.up_proj,4,32,8,128,17.0625,126.21875,234881024 +78,mlp.up_proj,4,64,8,128,19.9375,119.109375,234881024 +78,mlp.up_proj,4,128,8,128,22.53125,115.5546875,234881024 +78,mlp.up_proj,8,32,8,128,1.0380859375,238.21875,234881024 +78,mlp.up_proj,8,64,8,128,1.2138671875,231.109375,234881024 +78,mlp.up_proj,8,128,8,128,1.3671875,227.5546875,234881024 +78,mlp.down_proj,2,32,8,128,84.125,70.21875,234881024 +78,mlp.down_proj,2,64,8,128,98.375,63.109375,234881024 +78,mlp.down_proj,2,128,8,128,111.5,59.5546875,234881024 +78,mlp.down_proj,3,32,8,128,36.0625,98.21875,234881024 +78,mlp.down_proj,3,64,8,128,42.28125,91.109375,234881024 +78,mlp.down_proj,3,128,8,128,48.15625,87.5546875,234881024 +78,mlp.down_proj,4,32,8,128,16.828125,126.21875,234881024 +78,mlp.down_proj,4,64,8,128,19.765625,119.109375,234881024 +78,mlp.down_proj,4,128,8,128,22.515625,115.5546875,234881024 +78,mlp.down_proj,8,32,8,128,1.0234375,238.21875,234881024 +78,mlp.down_proj,8,64,8,128,1.2021484375,231.109375,234881024 +78,mlp.down_proj,8,128,8,128,1.3671875,227.5546875,234881024 +79,self_attn.q_proj,2,32,8,128,39.71875,20.0625,67108864 +79,self_attn.q_proj,2,64,8,128,46.875,18.03125,67108864 +79,self_attn.q_proj,2,128,8,128,53.8125,17.015625,67108864 +79,self_attn.q_proj,3,32,8,128,17.015625,28.0625,67108864 +79,self_attn.q_proj,3,64,8,128,20.15625,26.03125,67108864 +79,self_attn.q_proj,3,128,8,128,23.296875,25.015625,67108864 +79,self_attn.q_proj,4,32,8,128,7.94140625,36.0625,67108864 +79,self_attn.q_proj,4,64,8,128,9.40625,34.03125,67108864 +79,self_attn.q_proj,4,128,8,128,10.875,33.015625,67108864 +79,self_attn.q_proj,8,32,8,128,0.48095703125,68.0625,67108864 +79,self_attn.q_proj,8,64,8,128,0.5693359375,66.03125,67108864 +79,self_attn.q_proj,8,128,8,128,0.65576171875,65.015625,67108864 +79,self_attn.v_proj,2,32,8,128,16.671875,2.5078125,8388608 +79,self_attn.v_proj,2,64,8,128,19.859375,2.25390625,8388608 +79,self_attn.v_proj,2,128,8,128,22.859375,2.126953125,8388608 +79,self_attn.v_proj,3,32,8,128,7.13671875,3.5078125,8388608 +79,self_attn.v_proj,3,64,8,128,8.515625,3.25390625,8388608 +79,self_attn.v_proj,3,128,8,128,9.828125,3.126953125,8388608 +79,self_attn.v_proj,4,32,8,128,3.330078125,4.5078125,8388608 +79,self_attn.v_proj,4,64,8,128,3.974609375,4.25390625,8388608 +79,self_attn.v_proj,4,128,8,128,4.5859375,4.126953125,8388608 +79,self_attn.v_proj,8,32,8,128,0.20166015625,8.5078125,8388608 +79,self_attn.v_proj,8,64,8,128,0.240478515625,8.25390625,8388608 +79,self_attn.v_proj,8,128,8,128,0.276611328125,8.126953125,8388608 +79,self_attn.k_proj,2,32,8,128,22.890625,2.5078125,8388608 +79,self_attn.k_proj,2,64,8,128,27.640625,2.25390625,8388608 +79,self_attn.k_proj,2,128,8,128,31.828125,2.126953125,8388608 +79,self_attn.k_proj,3,32,8,128,9.828125,3.5078125,8388608 +79,self_attn.k_proj,3,64,8,128,11.9609375,3.25390625,8388608 +79,self_attn.k_proj,3,128,8,128,13.984375,3.126953125,8388608 +79,self_attn.k_proj,4,32,8,128,4.5859375,4.5078125,8388608 +79,self_attn.k_proj,4,64,8,128,5.58203125,4.25390625,8388608 +79,self_attn.k_proj,4,128,8,128,6.52734375,4.126953125,8388608 +79,self_attn.k_proj,8,32,8,128,0.277587890625,8.5078125,8388608 +79,self_attn.k_proj,8,64,8,128,0.337646484375,8.25390625,8388608 +79,self_attn.k_proj,8,128,8,128,0.393310546875,8.126953125,8388608 +79,self_attn.o_proj,2,32,8,128,35.375,20.0625,67108864 +79,self_attn.o_proj,2,64,8,128,41.625,18.03125,67108864 +79,self_attn.o_proj,2,128,8,128,47.6875,17.015625,67108864 +79,self_attn.o_proj,3,32,8,128,15.2109375,28.0625,67108864 +79,self_attn.o_proj,3,64,8,128,18.03125,26.03125,67108864 +79,self_attn.o_proj,3,128,8,128,20.890625,25.015625,67108864 +79,self_attn.o_proj,4,32,8,128,7.1015625,36.0625,67108864 +79,self_attn.o_proj,4,64,8,128,8.421875,34.03125,67108864 +79,self_attn.o_proj,4,128,8,128,9.78125,33.015625,67108864 +79,self_attn.o_proj,8,32,8,128,0.4296875,68.0625,67108864 +79,self_attn.o_proj,8,64,8,128,0.509765625,66.03125,67108864 +79,self_attn.o_proj,8,128,8,128,0.59033203125,65.015625,67108864 +79,mlp.gate_proj,2,32,8,128,92.375,70.21875,234881024 +79,mlp.gate_proj,2,64,8,128,108.5,63.109375,234881024 +79,mlp.gate_proj,2,128,8,128,123.375,59.5546875,234881024 +79,mlp.gate_proj,3,32,8,128,39.5625,98.21875,234881024 +79,mlp.gate_proj,3,64,8,128,46.46875,91.109375,234881024 +79,mlp.gate_proj,3,128,8,128,53.0,87.5546875,234881024 +79,mlp.gate_proj,4,32,8,128,18.453125,126.21875,234881024 +79,mlp.gate_proj,4,64,8,128,21.671875,119.109375,234881024 +79,mlp.gate_proj,4,128,8,128,24.734375,115.5546875,234881024 +79,mlp.gate_proj,8,32,8,128,1.1220703125,238.21875,234881024 +79,mlp.gate_proj,8,64,8,128,1.3193359375,231.109375,234881024 +79,mlp.gate_proj,8,128,8,128,1.5,227.5546875,234881024 +79,mlp.up_proj,2,32,8,128,88.1875,70.21875,234881024 +79,mlp.up_proj,2,64,8,128,103.25,63.109375,234881024 +79,mlp.up_proj,2,128,8,128,116.875,59.5546875,234881024 +79,mlp.up_proj,3,32,8,128,37.78125,98.21875,234881024 +79,mlp.up_proj,3,64,8,128,44.25,91.109375,234881024 +79,mlp.up_proj,3,128,8,128,50.21875,87.5546875,234881024 +79,mlp.up_proj,4,32,8,128,17.609375,126.21875,234881024 +79,mlp.up_proj,4,64,8,128,20.65625,119.109375,234881024 +79,mlp.up_proj,4,128,8,128,23.453125,115.5546875,234881024 +79,mlp.up_proj,8,32,8,128,1.0712890625,238.21875,234881024 +79,mlp.up_proj,8,64,8,128,1.2568359375,231.109375,234881024 +79,mlp.up_proj,8,128,8,128,1.421875,227.5546875,234881024 +79,mlp.down_proj,2,32,8,128,84.375,70.21875,234881024 +79,mlp.down_proj,2,64,8,128,98.875,63.109375,234881024 +79,mlp.down_proj,2,128,8,128,112.4375,59.5546875,234881024 +79,mlp.down_proj,3,32,8,128,36.28125,98.21875,234881024 +79,mlp.down_proj,3,64,8,128,42.71875,91.109375,234881024 +79,mlp.down_proj,3,128,8,128,49.0,87.5546875,234881024 +79,mlp.down_proj,4,32,8,128,16.96875,126.21875,234881024 +79,mlp.down_proj,4,64,8,128,20.046875,119.109375,234881024 +79,mlp.down_proj,4,128,8,128,23.0625,115.5546875,234881024 +79,mlp.down_proj,8,32,8,128,1.033203125,238.21875,234881024 +79,mlp.down_proj,8,64,8,128,1.22265625,231.109375,234881024 +79,mlp.down_proj,8,128,8,128,1.404296875,227.5546875,234881024 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-2-7b-hf.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-2-7b-hf.csv new file mode 100644 index 0000000000000000000000000000000000000000..ce4e494f6b667c1f232578ceba14f77d24bed98f --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-2-7b-hf.csv @@ -0,0 +1,2689 @@ +layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params,kurtosis,cost_sensi,sensitivity +0,self_attn.q_proj,2,32,8,128,23.5625,5.015625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,2,64,8,128,28.234375,4.5078125,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,2,128,8,128,32.5625,4.25390625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,3,32,8,128,10.5,7.015625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,3,64,8,128,13.1171875,6.5078125,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,3,128,8,128,16.15625,6.25390625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,4,32,8,128,4.91015625,9.015625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,4,64,8,128,6.16015625,8.5078125,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,4,128,8,128,7.6484375,8.25390625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,8,32,8,128,0.297119140625,17.015625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,8,64,8,128,0.372314453125,16.5078125,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.q_proj,8,128,8,128,0.46142578125,16.25390625,16777216,70.89063407817052,1,3.511252089083428e-7 +0,self_attn.v_proj,2,32,8,128,19.15625,5.015625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,2,64,8,128,23.359375,4.5078125,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,2,128,8,128,27.140625,4.25390625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,3,32,8,128,8.3046875,7.015625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,3,64,8,128,10.296875,6.5078125,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,3,128,8,128,12.328125,6.25390625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,4,32,8,128,3.875,9.015625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,4,64,8,128,4.8046875,8.5078125,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,4,128,8,128,5.76171875,8.25390625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,8,32,8,128,0.234375,17.015625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,8,64,8,128,0.29052734375,16.5078125,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.v_proj,8,128,8,128,0.34716796875,16.25390625,16777216,6.17988231863005,1,3.511252089083428e-7 +0,self_attn.k_proj,2,32,8,128,25.65625,5.015625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,2,64,8,128,30.6875,4.5078125,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,2,128,8,128,35.65625,4.25390625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,3,32,8,128,11.1640625,7.015625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,3,64,8,128,13.6328125,6.5078125,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,3,128,8,128,16.453125,6.25390625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,4,32,8,128,5.21484375,9.015625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,4,64,8,128,6.375,8.5078125,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,4,128,8,128,7.6953125,8.25390625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,8,32,8,128,0.3154296875,17.015625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,8,64,8,128,0.38525390625,16.5078125,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.k_proj,8,128,8,128,0.463623046875,16.25390625,16777216,38.24344781783628,1,3.511252089083428e-7 +0,self_attn.o_proj,2,32,8,128,11.1171875,5.015625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,2,64,8,128,13.140625,4.5078125,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,2,128,8,128,15.078125,4.25390625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,3,32,8,128,4.82421875,7.015625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,3,64,8,128,5.76171875,6.5078125,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,3,128,8,128,6.73046875,6.25390625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,4,32,8,128,2.26171875,9.015625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,4,64,8,128,2.712890625,8.5078125,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,4,128,8,128,3.1953125,8.25390625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,8,32,8,128,0.13720703125,17.015625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,8,64,8,128,0.16455078125,16.5078125,16777216,39.606632106657415,1,5.996923846396385e-7 +0,self_attn.o_proj,8,128,8,128,0.193603515625,16.25390625,16777216,39.606632106657415,1,5.996923846396385e-7 +0,mlp.gate_proj,2,32,8,128,41.34375,13.4794921875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,2,64,8,128,48.5625,12.11474609375,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,2,128,8,128,55.34375,11.432373046875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,3,32,8,128,17.703125,18.8544921875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,3,64,8,128,20.859375,17.48974609375,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,3,128,8,128,23.890625,16.807373046875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,4,32,8,128,8.265625,24.2294921875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,4,64,8,128,9.7421875,22.86474609375,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,4,128,8,128,11.1640625,22.182373046875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,8,32,8,128,0.5,45.7294921875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,8,64,8,128,0.5888671875,44.36474609375,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.gate_proj,8,128,8,128,0.6728515625,43.682373046875,45088768,4.269190067451742,1,1.3132982701336005e-7 +0,mlp.up_proj,2,32,8,128,40.375,13.4794921875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,2,64,8,128,47.09375,12.11474609375,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,2,128,8,128,53.21875,11.432373046875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,3,32,8,128,17.265625,18.8544921875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,3,64,8,128,20.15625,17.48974609375,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,3,128,8,128,22.8125,16.807373046875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,4,32,8,128,8.0546875,24.2294921875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,4,64,8,128,9.4140625,22.86474609375,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,4,128,8,128,10.6484375,22.182373046875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,8,32,8,128,0.487548828125,45.7294921875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,8,64,8,128,0.5693359375,44.36474609375,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.up_proj,8,128,8,128,0.6416015625,43.682373046875,45088768,3.1257809370322978,1,1.3132982701336005e-7 +0,mlp.down_proj,2,32,8,128,42.21875,13.4794921875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,2,64,8,128,49.40625,12.11474609375,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,2,128,8,128,56.0625,11.432373046875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,3,32,8,128,18.09375,18.8544921875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,3,64,8,128,21.21875,17.48974609375,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,3,128,8,128,24.171875,16.807373046875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,4,32,8,128,8.4453125,24.2294921875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,4,64,8,128,9.9140625,22.86474609375,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,4,128,8,128,11.296875,22.182373046875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,8,32,8,128,0.51123046875,45.7294921875,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,8,64,8,128,0.59912109375,44.36474609375,45088768,3.941024313230316,1,1.116492285291315e-6 +0,mlp.down_proj,8,128,8,128,0.68115234375,43.682373046875,45088768,3.941024313230316,1,1.116492285291315e-6 +1,self_attn.q_proj,2,32,8,128,45.1875,5.015625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,2,64,8,128,53.125,4.5078125,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,2,128,8,128,61.40625,4.25390625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,3,32,8,128,20.640625,7.015625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,3,64,8,128,25.1875,6.5078125,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,3,128,8,128,31.21875,6.25390625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,4,32,8,128,9.6796875,9.015625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,4,64,8,128,11.8984375,8.5078125,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,4,128,8,128,14.8671875,8.25390625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,8,32,8,128,0.5859375,17.015625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,8,64,8,128,0.72021484375,16.5078125,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.q_proj,8,128,8,128,0.896484375,16.25390625,16777216,9.506592860422105,1,1.1230303016418477e-6 +1,self_attn.v_proj,2,32,8,128,16.875,5.015625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,2,64,8,128,20.203125,4.5078125,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,2,128,8,128,23.109375,4.25390625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,3,32,8,128,7.25390625,7.015625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,3,64,8,128,8.7421875,6.5078125,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,3,128,8,128,10.15625,6.25390625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,4,32,8,128,3.384765625,9.015625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,4,64,8,128,4.08203125,8.5078125,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,4,128,8,128,4.7421875,8.25390625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,8,32,8,128,0.204833984375,17.015625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,8,64,8,128,0.246826171875,16.5078125,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.v_proj,8,128,8,128,0.285888671875,16.25390625,16777216,4.8384166345295005,1,1.1230303016418477e-6 +1,self_attn.k_proj,2,32,8,128,45.3125,5.015625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,2,64,8,128,53.53125,4.5078125,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,2,128,8,128,62.75,4.25390625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,3,32,8,128,20.390625,7.015625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,3,64,8,128,24.9375,6.5078125,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,3,128,8,128,31.53125,6.25390625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,4,32,8,128,9.53125,9.015625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,4,64,8,128,11.7109375,8.5078125,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,4,128,8,128,14.875,8.25390625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,8,32,8,128,0.5771484375,17.015625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,8,64,8,128,0.7080078125,16.5078125,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.k_proj,8,128,8,128,0.896484375,16.25390625,16777216,9.93537137402608,1,1.1230303016418477e-6 +1,self_attn.o_proj,2,32,8,128,12.671875,5.015625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,2,64,8,128,14.9140625,4.5078125,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,2,128,8,128,17.015625,4.25390625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,3,32,8,128,5.4765625,7.015625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,3,64,8,128,6.51171875,6.5078125,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,3,128,8,128,7.55859375,6.25390625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,4,32,8,128,2.564453125,9.015625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,4,64,8,128,3.056640625,8.5078125,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,4,128,8,128,3.572265625,8.25390625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,8,32,8,128,0.155517578125,17.015625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,8,64,8,128,0.1849365234375,16.5078125,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,self_attn.o_proj,8,128,8,128,0.2156982421875,16.25390625,16777216,26.69787713965271,1,1.9699409676832147e-6 +1,mlp.gate_proj,2,32,8,128,45.71875,13.4794921875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,2,64,8,128,53.65625,12.11474609375,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,2,128,8,128,61.125,11.432373046875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,3,32,8,128,19.609375,18.8544921875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,3,64,8,128,23.125,17.48974609375,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,3,128,8,128,26.546875,16.807373046875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,4,32,8,128,9.15625,24.2294921875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,4,64,8,128,10.8046875,22.86474609375,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,4,128,8,128,12.4140625,22.182373046875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,8,32,8,128,0.5546875,45.7294921875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,8,64,8,128,0.65380859375,44.36474609375,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.gate_proj,8,128,8,128,0.74951171875,43.682373046875,45088768,6.280002848563622,1,3.190103598171845e-5 +1,mlp.up_proj,2,32,8,128,42.84375,13.4794921875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,2,64,8,128,49.96875,12.11474609375,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,2,128,8,128,56.375,11.432373046875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,3,32,8,128,18.34375,18.8544921875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,3,64,8,128,21.421875,17.48974609375,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,3,128,8,128,24.265625,16.807373046875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,4,32,8,128,8.5546875,24.2294921875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,4,64,8,128,10,22.86474609375,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,4,128,8,128,11.328125,22.182373046875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,8,32,8,128,0.51806640625,45.7294921875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,8,64,8,128,0.60498046875,44.36474609375,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.up_proj,8,128,8,128,0.6826171875,43.682373046875,45088768,3.4447561314119977,1,3.190103598171845e-5 +1,mlp.down_proj,2,32,8,128,43.4375,13.4794921875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,2,64,8,128,50.625,12.11474609375,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,2,128,8,128,57.125,11.432373046875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,3,32,8,128,18.59375,18.8544921875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,3,64,8,128,21.703125,17.48974609375,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,3,128,8,128,24.546875,16.807373046875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,4,32,8,128,8.6796875,24.2294921875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,4,64,8,128,10.1328125,22.86474609375,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,4,128,8,128,11.46875,22.182373046875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,8,32,8,128,0.525390625,45.7294921875,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,8,64,8,128,0.61328125,44.36474609375,45088768,7.580293538212413,2,0.01109052263200283 +1,mlp.down_proj,8,128,8,128,0.69189453125,43.682373046875,45088768,7.580293538212413,2,0.01109052263200283 +2,self_attn.q_proj,2,32,8,128,43.4375,5.015625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,2,64,8,128,51.46875,4.5078125,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,2,128,8,128,60.75,4.25390625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,3,32,8,128,18.703125,7.015625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,3,64,8,128,22.28125,6.5078125,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,3,128,8,128,27.3125,6.25390625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,4,32,8,128,8.7265625,9.015625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,4,64,8,128,10.40625,8.5078125,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,4,128,8,128,12.7734375,8.25390625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,8,32,8,128,0.5283203125,17.015625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,8,64,8,128,0.62939453125,16.5078125,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.q_proj,8,128,8,128,0.76953125,16.25390625,16777216,5.845498006156931,1,4.232647825119784e-6 +2,self_attn.v_proj,2,32,8,128,22.046875,5.015625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,2,64,8,128,25.75,4.5078125,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,2,128,8,128,29.109375,4.25390625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,3,32,8,128,9.421875,7.015625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,3,64,8,128,11.015625,6.5078125,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,3,128,8,128,12.46875,6.25390625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,4,32,8,128,4.39453125,9.015625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,4,64,8,128,5.140625,8.5078125,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,4,128,8,128,5.81640625,8.25390625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,8,32,8,128,0.26611328125,17.015625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,8,64,8,128,0.31103515625,16.5078125,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.v_proj,8,128,8,128,0.3505859375,16.25390625,16777216,3.1704948733015765,1,4.232647825119784e-6 +2,self_attn.k_proj,2,32,8,128,45.875,5.015625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,2,64,8,128,54.3125,4.5078125,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,2,128,8,128,63.65625,4.25390625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,3,32,8,128,19.84375,7.015625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,3,64,8,128,23.703125,6.5078125,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,3,128,8,128,29.390625,6.25390625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,4,32,8,128,9.265625,9.015625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,4,64,8,128,11.078125,8.5078125,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,4,128,8,128,13.828125,8.25390625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,8,32,8,128,0.56103515625,17.015625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,8,64,8,128,0.669921875,16.5078125,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.k_proj,8,128,8,128,0.83349609375,16.25390625,16777216,5.422568329046552,1,4.232647825119784e-6 +2,self_attn.o_proj,2,32,8,128,21.28125,5.015625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,2,64,8,128,24.921875,4.5078125,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,2,128,8,128,28.3125,4.25390625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,3,32,8,128,9.140625,7.015625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,3,64,8,128,10.75,6.5078125,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,3,128,8,128,12.3046875,6.25390625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,4,32,8,128,4.26953125,9.015625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,4,64,8,128,5.02734375,8.5078125,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,4,128,8,128,5.76953125,8.25390625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,8,32,8,128,0.25830078125,17.015625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,8,64,8,128,0.303955078125,16.5078125,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,self_attn.o_proj,8,128,8,128,0.34765625,16.25390625,16777216,7.172113174858918,1,1.6720242683732067e-6 +2,mlp.gate_proj,2,32,8,128,46.625,13.4794921875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,2,64,8,128,54.34375,12.11474609375,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,2,128,8,128,61.34375,11.432373046875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,3,32,8,128,19.9375,18.8544921875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,3,64,8,128,23.28125,17.48974609375,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,3,128,8,128,26.3125,16.807373046875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,4,32,8,128,9.3046875,24.2294921875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,4,64,8,128,10.8671875,22.86474609375,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,4,128,8,128,12.2890625,22.182373046875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,8,32,8,128,0.5634765625,45.7294921875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,8,64,8,128,0.6572265625,44.36474609375,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.gate_proj,8,128,8,128,0.74072265625,43.682373046875,45088768,3.252795019767423,1,3.831796675513033e-6 +2,mlp.up_proj,2,32,8,128,43.5,13.4794921875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,2,64,8,128,50.65625,12.11474609375,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,2,128,8,128,57.03125,11.432373046875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,3,32,8,128,18.609375,18.8544921875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,3,64,8,128,21.6875,17.48974609375,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,3,128,8,128,24.4375,16.807373046875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,4,32,8,128,8.6796875,24.2294921875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,4,64,8,128,10.1171875,22.86474609375,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,4,128,8,128,11.4140625,22.182373046875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,8,32,8,128,0.525390625,45.7294921875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,8,64,8,128,0.61181640625,44.36474609375,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.up_proj,8,128,8,128,0.6875,43.682373046875,45088768,3.0643926831566968,1,3.831796675513033e-6 +2,mlp.down_proj,2,32,8,128,43.78125,13.4794921875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,2,64,8,128,51,12.11474609375,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,2,128,8,128,57.46875,11.432373046875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,3,32,8,128,18.734375,18.8544921875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,3,64,8,128,21.828125,17.48974609375,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,3,128,8,128,24.625,16.807373046875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,4,32,8,128,8.7421875,24.2294921875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,4,64,8,128,10.1875,22.86474609375,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,4,128,8,128,11.5,22.182373046875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,8,32,8,128,0.529296875,45.7294921875,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,8,64,8,128,0.6162109375,44.36474609375,45088768,3.500094635203027,1,5.077923560747877e-6 +2,mlp.down_proj,8,128,8,128,0.69287109375,43.682373046875,45088768,3.500094635203027,1,5.077923560747877e-6 +3,self_attn.q_proj,2,32,8,128,40.3125,5.015625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,2,64,8,128,47.65625,4.5078125,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,2,128,8,128,56.46875,4.25390625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,3,32,8,128,17.28125,7.015625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,3,64,8,128,20.484375,6.5078125,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,3,128,8,128,24.8125,6.25390625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,4,32,8,128,8.0625,9.015625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,4,64,8,128,9.5546875,8.5078125,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,4,128,8,128,11.5859375,8.25390625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,8,32,8,128,0.488037109375,17.015625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,8,64,8,128,0.57763671875,16.5078125,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.q_proj,8,128,8,128,0.6982421875,16.25390625,16777216,5.747449112728132,1,8.20862169348402e-6 +3,self_attn.v_proj,2,32,8,128,21,5.015625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,2,64,8,128,24.5625,4.5078125,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,2,128,8,128,27.78125,4.25390625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,3,32,8,128,8.9765625,7.015625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,3,64,8,128,10.5,6.5078125,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,3,128,8,128,11.8828125,6.25390625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,4,32,8,128,4.1875,9.015625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,4,64,8,128,4.90234375,8.5078125,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,4,128,8,128,5.55078125,8.25390625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,8,32,8,128,0.253662109375,17.015625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,8,64,8,128,0.29638671875,16.5078125,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.v_proj,8,128,8,128,0.334228515625,16.25390625,16777216,3.210137666145488,1,8.20862169348402e-6 +3,self_attn.k_proj,2,32,8,128,42.03125,5.015625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,2,64,8,128,49.65625,4.5078125,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,2,128,8,128,58.71875,4.25390625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,3,32,8,128,18.03125,7.015625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,3,64,8,128,21.375,6.5078125,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,3,128,8,128,26.078125,6.25390625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,4,32,8,128,8.421875,9.015625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,4,64,8,128,9.9765625,8.5078125,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,4,128,8,128,12.1953125,8.25390625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,8,32,8,128,0.509765625,17.015625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,8,64,8,128,0.603515625,16.5078125,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.k_proj,8,128,8,128,0.73486328125,16.25390625,16777216,4.552550541532545,1,8.20862169348402e-6 +3,self_attn.o_proj,2,32,8,128,20.3125,5.015625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,2,64,8,128,23.75,4.5078125,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,2,128,8,128,26.90625,4.25390625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,3,32,8,128,8.71875,7.015625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,3,64,8,128,10.234375,6.5078125,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,3,128,8,128,11.671875,6.25390625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,4,32,8,128,4.0703125,9.015625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,4,64,8,128,4.78125,8.5078125,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,4,128,8,128,5.46484375,8.25390625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,8,32,8,128,0.2464599609375,17.015625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,8,64,8,128,0.289306640625,16.5078125,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,self_attn.o_proj,8,128,8,128,0.329345703125,16.25390625,16777216,6.738655132097817,1,3.2434377317258622e-6 +3,mlp.gate_proj,2,32,8,128,47.5,13.4794921875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,2,64,8,128,55.46875,12.11474609375,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,2,128,8,128,62.71875,11.432373046875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,3,32,8,128,20.3125,18.8544921875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,3,64,8,128,23.75,17.48974609375,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,3,128,8,128,26.921875,16.807373046875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,4,32,8,128,9.484375,24.2294921875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,4,64,8,128,11.0859375,22.86474609375,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,4,128,8,128,12.5703125,22.182373046875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,8,32,8,128,0.57373046875,45.7294921875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,8,64,8,128,0.67041015625,44.36474609375,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.gate_proj,8,128,8,128,0.75732421875,43.682373046875,45088768,3.269010367998764,1,9.4121887741494e-6 +3,mlp.up_proj,2,32,8,128,43.84375,13.4794921875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,2,64,8,128,51.0625,12.11474609375,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,2,128,8,128,57.53125,11.432373046875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,3,32,8,128,18.75,18.8544921875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,3,64,8,128,21.859375,17.48974609375,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,3,128,8,128,24.65625,16.807373046875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,4,32,8,128,8.75,24.2294921875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,4,64,8,128,10.203125,22.86474609375,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,4,128,8,128,11.5078125,22.182373046875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,8,32,8,128,0.52978515625,45.7294921875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,8,64,8,128,0.6171875,44.36474609375,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.up_proj,8,128,8,128,0.693359375,43.682373046875,45088768,3.11530955415524,1,9.4121887741494e-6 +3,mlp.down_proj,2,32,8,128,43.9375,13.4794921875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,2,64,8,128,51.15625,12.11474609375,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,2,128,8,128,57.65625,11.432373046875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,3,32,8,128,18.796875,18.8544921875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,3,64,8,128,21.90625,17.48974609375,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,3,128,8,128,24.71875,16.807373046875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,4,32,8,128,8.7734375,24.2294921875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,4,64,8,128,10.2265625,22.86474609375,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,4,128,8,128,11.5390625,22.182373046875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,8,32,8,128,0.53076171875,45.7294921875,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,8,64,8,128,0.6181640625,44.36474609375,45088768,3.522186223965613,1,9.163330105366184e-6 +3,mlp.down_proj,8,128,8,128,0.69580078125,43.682373046875,45088768,3.522186223965613,1,9.163330105366184e-6 +4,self_attn.q_proj,2,32,8,128,41.625,5.015625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,2,64,8,128,49.0625,4.5078125,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,2,128,8,128,58.0625,4.25390625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,3,32,8,128,17.8125,7.015625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,3,64,8,128,21.046875,6.5078125,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,3,128,8,128,25.25,6.25390625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,4,32,8,128,8.3125,9.015625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,4,64,8,128,9.8203125,8.5078125,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,4,128,8,128,11.7890625,8.25390625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,8,32,8,128,0.50341796875,17.015625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,8,64,8,128,0.59423828125,16.5078125,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.q_proj,8,128,8,128,0.71044921875,16.25390625,16777216,5.076595574419752,1,1.840256118157413e-5 +4,self_attn.v_proj,2,32,8,128,21.984375,5.015625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,2,64,8,128,25.671875,4.5078125,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,2,128,8,128,29,4.25390625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,3,32,8,128,9.3984375,7.015625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,3,64,8,128,10.9765625,6.5078125,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,3,128,8,128,12.421875,6.25390625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,4,32,8,128,4.38671875,9.015625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,4,64,8,128,5.125,8.5078125,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,4,128,8,128,5.79296875,8.25390625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,8,32,8,128,0.265625,17.015625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,8,64,8,128,0.309814453125,16.5078125,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.v_proj,8,128,8,128,0.34912109375,16.25390625,16777216,3.12290612284217,1,1.840256118157413e-5 +4,self_attn.k_proj,2,32,8,128,42.59375,5.015625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,2,64,8,128,50.21875,4.5078125,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,2,128,8,128,59.53125,4.25390625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,3,32,8,128,18.265625,7.015625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,3,64,8,128,21.5625,6.5078125,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,3,128,8,128,26.171875,6.25390625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,4,32,8,128,8.5234375,9.015625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,4,64,8,128,10.0703125,8.5078125,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,4,128,8,128,12.2265625,8.25390625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,8,32,8,128,0.51611328125,17.015625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,8,64,8,128,0.60888671875,16.5078125,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.k_proj,8,128,8,128,0.7373046875,16.25390625,16777216,4.330934358734191,1,1.840256118157413e-5 +4,self_attn.o_proj,2,32,8,128,21.421875,5.015625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,2,64,8,128,25.03125,4.5078125,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,2,128,8,128,28.34375,4.25390625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,3,32,8,128,9.1875,7.015625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,3,64,8,128,10.7734375,6.5078125,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,3,128,8,128,12.265625,6.25390625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,4,32,8,128,4.2890625,9.015625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,4,64,8,128,5.03515625,8.5078125,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,4,128,8,128,5.7421875,8.25390625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,8,32,8,128,0.259765625,17.015625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,8,64,8,128,0.304443359375,16.5078125,16777216,5.993817704977201,1,6.374161330313655e-6 +4,self_attn.o_proj,8,128,8,128,0.345703125,16.25390625,16777216,5.993817704977201,1,6.374161330313655e-6 +4,mlp.gate_proj,2,32,8,128,48.75,13.4794921875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,2,64,8,128,57.0625,12.11474609375,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,2,128,8,128,64.75,11.432373046875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,3,32,8,128,20.84375,18.8544921875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,3,64,8,128,24.453125,17.48974609375,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,3,128,8,128,27.796875,16.807373046875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,4,32,8,128,9.734375,24.2294921875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,4,64,8,128,11.4140625,22.86474609375,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,4,128,8,128,12.9765625,22.182373046875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,8,32,8,128,0.58935546875,45.7294921875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,8,64,8,128,0.68994140625,44.36474609375,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.gate_proj,8,128,8,128,0.78173828125,43.682373046875,45088768,3.3662009159295945,1,1.8175354853156023e-5 +4,mlp.up_proj,2,32,8,128,43.5625,13.4794921875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,2,64,8,128,50.75,12.11474609375,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,2,128,8,128,57.1875,11.432373046875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,3,32,8,128,18.640625,18.8544921875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,3,64,8,128,21.71875,17.48974609375,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,3,128,8,128,24.515625,16.807373046875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,4,32,8,128,8.6953125,24.2294921875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,4,64,8,128,10.140625,22.86474609375,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,4,128,8,128,11.4453125,22.182373046875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,8,32,8,128,0.5263671875,45.7294921875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,8,64,8,128,0.61328125,44.36474609375,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.up_proj,8,128,8,128,0.689453125,43.682373046875,45088768,3.103154995797977,1,1.8175354853156023e-5 +4,mlp.down_proj,2,32,8,128,43.4375,13.4794921875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,2,64,8,128,50.59375,12.11474609375,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,2,128,8,128,57.0625,11.432373046875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,3,32,8,128,18.578125,18.8544921875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,3,64,8,128,21.671875,17.48974609375,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,3,128,8,128,24.453125,16.807373046875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,4,32,8,128,8.671875,24.2294921875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,4,64,8,128,10.1171875,22.86474609375,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,4,128,8,128,11.4296875,22.182373046875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,8,32,8,128,0.52490234375,45.7294921875,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,8,64,8,128,0.61181640625,44.36474609375,45088768,3.950713876071597,1,2.3200484065455385e-5 +4,mlp.down_proj,8,128,8,128,0.6884765625,43.682373046875,45088768,3.950713876071597,1,2.3200484065455385e-5 +5,self_attn.q_proj,2,32,8,128,41.59375,5.015625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,2,64,8,128,48.96875,4.5078125,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,2,128,8,128,57.875,4.25390625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,3,32,8,128,17.8125,7.015625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,3,64,8,128,20.96875,6.5078125,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,3,128,8,128,25.03125,6.25390625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,4,32,8,128,8.3046875,9.015625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,4,64,8,128,9.7890625,8.5078125,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,4,128,8,128,11.6875,8.25390625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,8,32,8,128,0.5029296875,17.015625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,8,64,8,128,0.59228515625,16.5078125,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.q_proj,8,128,8,128,0.7041015625,16.25390625,16777216,4.239130739572246,1,3.1707564630778506e-5 +5,self_attn.v_proj,2,32,8,128,22.65625,5.015625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,2,64,8,128,26.453125,4.5078125,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,2,128,8,128,29.875,4.25390625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,3,32,8,128,9.6796875,7.015625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,3,64,8,128,11.3125,6.5078125,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,3,128,8,128,12.7734375,6.25390625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,4,32,8,128,4.51953125,9.015625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,4,64,8,128,5.27734375,8.5078125,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,4,128,8,128,5.96484375,8.25390625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,8,32,8,128,0.273681640625,17.015625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,8,64,8,128,0.319091796875,16.5078125,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.v_proj,8,128,8,128,0.359375,16.25390625,16777216,3.166712534307685,1,3.1707564630778506e-5 +5,self_attn.k_proj,2,32,8,128,43.5,5.015625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,2,64,8,128,51.1875,4.5078125,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,2,128,8,128,60.90625,4.25390625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,3,32,8,128,18.609375,7.015625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,3,64,8,128,21.9375,6.5078125,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,3,128,8,128,26.5625,6.25390625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,4,32,8,128,8.6875,9.015625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,4,64,8,128,10.2421875,8.5078125,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,4,128,8,128,12.40625,8.25390625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,8,32,8,128,0.52587890625,17.015625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,8,64,8,128,0.619140625,16.5078125,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.k_proj,8,128,8,128,0.74755859375,16.25390625,16777216,4.188035260197244,1,3.1707564630778506e-5 +5,self_attn.o_proj,2,32,8,128,21.96875,5.015625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,2,64,8,128,25.640625,4.5078125,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,2,128,8,128,29,4.25390625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,3,32,8,128,9.4140625,7.015625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,3,64,8,128,11.03125,6.5078125,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,3,128,8,128,12.53125,6.25390625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,4,32,8,128,4.39453125,9.015625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,4,64,8,128,5.1484375,8.5078125,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,4,128,8,128,5.859375,8.25390625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,8,32,8,128,0.26611328125,17.015625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,8,64,8,128,0.311279296875,16.5078125,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,self_attn.o_proj,8,128,8,128,0.35302734375,16.25390625,16777216,5.158751141507659,1,1.2589415746333543e-5 +5,mlp.gate_proj,2,32,8,128,49.25,13.4794921875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,2,64,8,128,57.78125,12.11474609375,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,2,128,8,128,65.6875,11.432373046875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,3,32,8,128,21.0625,18.8544921875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,3,64,8,128,24.75,17.48974609375,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,3,128,8,128,28.1875,16.807373046875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,4,32,8,128,9.8359375,24.2294921875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,4,64,8,128,11.5546875,22.86474609375,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,4,128,8,128,13.15625,22.182373046875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,8,32,8,128,0.59521484375,45.7294921875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,8,64,8,128,0.6982421875,44.36474609375,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.gate_proj,8,128,8,128,0.79248046875,43.682373046875,45088768,3.3132728389187305,1,2.797519300656859e-5 +5,mlp.up_proj,2,32,8,128,43.59375,13.4794921875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,2,64,8,128,50.78125,12.11474609375,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,2,128,8,128,57.25,11.432373046875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,3,32,8,128,18.65625,18.8544921875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,3,64,8,128,21.75,17.48974609375,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,3,128,8,128,24.53125,16.807373046875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,4,32,8,128,8.703125,24.2294921875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,4,64,8,128,10.1484375,22.86474609375,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,4,128,8,128,11.453125,22.182373046875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,8,32,8,128,0.52685546875,45.7294921875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,8,64,8,128,0.61376953125,44.36474609375,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.up_proj,8,128,8,128,0.6904296875,43.682373046875,45088768,3.1074866507472088,1,2.797519300656859e-5 +5,mlp.down_proj,2,32,8,128,43.5,13.4794921875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,2,64,8,128,50.65625,12.11474609375,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,2,128,8,128,57.09375,11.432373046875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,3,32,8,128,18.59375,18.8544921875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,3,64,8,128,21.6875,17.48974609375,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,3,128,8,128,24.484375,16.807373046875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,4,32,8,128,8.6796875,24.2294921875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,4,64,8,128,10.1171875,22.86474609375,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,4,128,8,128,11.4296875,22.182373046875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,8,32,8,128,0.525390625,45.7294921875,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,8,64,8,128,0.6123046875,44.36474609375,45088768,3.503131607870768,1,2.4447845135000534e-5 +5,mlp.down_proj,8,128,8,128,0.6884765625,43.682373046875,45088768,3.503131607870768,1,2.4447845135000534e-5 +6,self_attn.q_proj,2,32,8,128,39.5625,5.015625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,2,64,8,128,46.71875,4.5078125,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,2,128,8,128,56.28125,4.25390625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,3,32,8,128,16.953125,7.015625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,3,64,8,128,20.0625,6.5078125,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,3,128,8,128,24.59375,6.25390625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,4,32,8,128,7.90625,9.015625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,4,64,8,128,9.359375,8.5078125,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,4,128,8,128,11.484375,8.25390625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,8,32,8,128,0.478515625,17.015625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,8,64,8,128,0.56591796875,16.5078125,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.q_proj,8,128,8,128,0.69189453125,16.25390625,16777216,4.859155894110683,1,4.1241120925405994e-5 +6,self_attn.v_proj,2,32,8,128,21.203125,5.015625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,2,64,8,128,24.859375,4.5078125,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,2,128,8,128,28.203125,4.25390625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,3,32,8,128,9.0625,7.015625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,3,64,8,128,10.640625,6.5078125,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,3,128,8,128,12.078125,6.25390625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,4,32,8,128,4.23046875,9.015625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,4,64,8,128,4.96484375,8.5078125,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,4,128,8,128,5.63671875,8.25390625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,8,32,8,128,0.256103515625,17.015625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,8,64,8,128,0.30029296875,16.5078125,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.v_proj,8,128,8,128,0.339599609375,16.25390625,16777216,3.301832949774581,1,4.1241120925405994e-5 +6,self_attn.k_proj,2,32,8,128,40.375,5.015625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,2,64,8,128,47.65625,4.5078125,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,2,128,8,128,57.5,4.25390625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,3,32,8,128,17.28125,7.015625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,3,64,8,128,20.421875,6.5078125,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,3,128,8,128,25.234375,6.25390625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,4,32,8,128,8.0625,9.015625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,4,64,8,128,9.53125,8.5078125,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,4,128,8,128,11.78125,8.25390625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,8,32,8,128,0.48779296875,17.015625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,8,64,8,128,0.57666015625,16.5078125,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.k_proj,8,128,8,128,0.7099609375,16.25390625,16777216,4.478034883058803,1,4.1241120925405994e-5 +6,self_attn.o_proj,2,32,8,128,20.546875,5.015625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,2,64,8,128,24,4.5078125,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,2,128,8,128,27.140625,4.25390625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,3,32,8,128,8.8125,7.015625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,3,64,8,128,10.3203125,6.5078125,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,3,128,8,128,11.734375,6.25390625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,4,32,8,128,4.1171875,9.015625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,4,64,8,128,4.82421875,8.5078125,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,4,128,8,128,5.49609375,8.25390625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,8,32,8,128,0.2491455078125,17.015625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,8,64,8,128,0.291748046875,16.5078125,16777216,7.088338485259551,1,1.534799230284989e-5 +6,self_attn.o_proj,8,128,8,128,0.331298828125,16.25390625,16777216,7.088338485259551,1,1.534799230284989e-5 +6,mlp.gate_proj,2,32,8,128,50.1875,13.4794921875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,2,64,8,128,58.9375,12.11474609375,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,2,128,8,128,67.0625,11.432373046875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,3,32,8,128,21.46875,18.8544921875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,3,64,8,128,25.25,17.48974609375,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,3,128,8,128,28.765625,16.807373046875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,4,32,8,128,10.0234375,24.2294921875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,4,64,8,128,11.78125,22.86474609375,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,4,128,8,128,13.4296875,22.182373046875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,8,32,8,128,0.6064453125,45.7294921875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,8,64,8,128,0.71240234375,44.36474609375,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.gate_proj,8,128,8,128,0.80908203125,43.682373046875,45088768,3.3562287806004885,1,4.1680410504341125e-5 +6,mlp.up_proj,2,32,8,128,43.46875,13.4794921875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,2,64,8,128,50.625,12.11474609375,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,2,128,8,128,57.0625,11.432373046875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,3,32,8,128,18.59375,18.8544921875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,3,64,8,128,21.671875,17.48974609375,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,3,128,8,128,24.453125,16.807373046875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,4,32,8,128,8.671875,24.2294921875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,4,64,8,128,10.1171875,22.86474609375,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,4,128,8,128,11.4140625,22.182373046875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,8,32,8,128,0.52490234375,45.7294921875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,8,64,8,128,0.61181640625,44.36474609375,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.up_proj,8,128,8,128,0.68798828125,43.682373046875,45088768,3.1103515391488963,1,4.1680410504341125e-5 +6,mlp.down_proj,2,32,8,128,43.25,13.4794921875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,2,64,8,128,50.375,12.11474609375,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,2,128,8,128,56.78125,11.432373046875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,3,32,8,128,18.5,18.8544921875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,3,64,8,128,21.546875,17.48974609375,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,3,128,8,128,24.328125,16.807373046875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,4,32,8,128,8.6328125,24.2294921875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,4,64,8,128,10.0625,22.86474609375,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,4,128,8,128,11.3515625,22.182373046875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,8,32,8,128,0.5224609375,45.7294921875,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,8,64,8,128,0.60888671875,44.36474609375,45088768,3.676690922512981,1,3.688649303512648e-5 +6,mlp.down_proj,8,128,8,128,0.68408203125,43.682373046875,45088768,3.676690922512981,1,3.688649303512648e-5 +7,self_attn.q_proj,2,32,8,128,39.65625,5.015625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,2,64,8,128,46.90625,4.5078125,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,2,128,8,128,56.34375,4.25390625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,3,32,8,128,16.984375,7.015625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,3,64,8,128,20.125,6.5078125,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,3,128,8,128,24.625,6.25390625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,4,32,8,128,7.92578125,9.015625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,4,64,8,128,9.390625,8.5078125,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,4,128,8,128,11.4921875,8.25390625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,8,32,8,128,0.479736328125,17.015625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,8,64,8,128,0.56787109375,16.5078125,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.q_proj,8,128,8,128,0.6923828125,16.25390625,16777216,4.7117156949789765,1,5.036446964368224e-5 +7,self_attn.v_proj,2,32,8,128,21.546875,5.015625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,2,64,8,128,25.296875,4.5078125,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,2,128,8,128,28.71875,4.25390625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,3,32,8,128,9.21875,7.015625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,3,64,8,128,10.828125,6.5078125,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,3,128,8,128,12.3046875,6.25390625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,4,32,8,128,4.30078125,9.015625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,4,64,8,128,5.05078125,8.5078125,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,4,128,8,128,5.7421875,8.25390625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,8,32,8,128,0.26025390625,17.015625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,8,64,8,128,0.305419921875,16.5078125,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.v_proj,8,128,8,128,0.345947265625,16.25390625,16777216,3.35458205981544,1,5.036446964368224e-5 +7,self_attn.k_proj,2,32,8,128,39.75,5.015625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,2,64,8,128,46.9375,4.5078125,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,2,128,8,128,56.3125,4.25390625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,3,32,8,128,17.015625,7.015625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,3,64,8,128,20.109375,6.5078125,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,3,128,8,128,24.609375,6.25390625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,4,32,8,128,7.9375,9.015625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,4,64,8,128,9.3828125,8.5078125,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,4,128,8,128,11.4921875,8.25390625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,8,32,8,128,0.480712890625,17.015625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,8,64,8,128,0.56787109375,16.5078125,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.k_proj,8,128,8,128,0.6923828125,16.25390625,16777216,4.368184768466018,1,5.036446964368224e-5 +7,self_attn.o_proj,2,32,8,128,20.703125,5.015625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,2,64,8,128,24.15625,4.5078125,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,2,128,8,128,27.3125,4.25390625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,3,32,8,128,8.8828125,7.015625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,3,64,8,128,10.390625,6.5078125,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,3,128,8,128,11.8125,6.25390625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,4,32,8,128,4.14453125,9.015625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,4,64,8,128,4.8515625,8.5078125,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,4,128,8,128,5.51953125,8.25390625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,8,32,8,128,0.250732421875,17.015625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,8,64,8,128,0.29345703125,16.5078125,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,self_attn.o_proj,8,128,8,128,0.33251953125,16.25390625,16777216,5.347610561393708,1,2.1903808374190703e-5 +7,mlp.gate_proj,2,32,8,128,50.28125,13.4794921875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,2,64,8,128,59.125,12.11474609375,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,2,128,8,128,67.3125,11.432373046875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,3,32,8,128,21.515625,18.8544921875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,3,64,8,128,25.3125,17.48974609375,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,3,128,8,128,28.890625,16.807373046875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,4,32,8,128,10.0390625,24.2294921875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,4,64,8,128,11.8203125,22.86474609375,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,4,128,8,128,13.484375,22.182373046875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,8,32,8,128,0.60791015625,45.7294921875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,8,64,8,128,0.71435546875,44.36474609375,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.gate_proj,8,128,8,128,0.8125,43.682373046875,45088768,3.3854765386900025,1,5.7778179325396195e-5 +7,mlp.up_proj,2,32,8,128,43.625,13.4794921875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,2,64,8,128,50.8125,12.11474609375,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,2,128,8,128,57.28125,11.432373046875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,3,32,8,128,18.65625,18.8544921875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,3,64,8,128,21.75,17.48974609375,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,3,128,8,128,24.546875,16.807373046875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,4,32,8,128,8.703125,24.2294921875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,4,64,8,128,10.15625,22.86474609375,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,4,128,8,128,11.4609375,22.182373046875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,8,32,8,128,0.52685546875,45.7294921875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,8,64,8,128,0.61376953125,44.36474609375,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.up_proj,8,128,8,128,0.6904296875,43.682373046875,45088768,3.0859186463660886,1,5.7778179325396195e-5 +7,mlp.down_proj,2,32,8,128,43.3125,13.4794921875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,2,64,8,128,50.46875,12.11474609375,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,2,128,8,128,56.90625,11.432373046875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,3,32,8,128,18.53125,18.8544921875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,3,64,8,128,21.609375,17.48974609375,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,3,128,8,128,24.390625,16.807373046875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,4,32,8,128,8.6484375,24.2294921875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,4,64,8,128,10.0859375,22.86474609375,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,4,128,8,128,11.390625,22.182373046875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,8,32,8,128,0.5234375,45.7294921875,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,8,64,8,128,0.60986328125,44.36474609375,45088768,3.6398775576866766,1,4.949917274643667e-5 +7,mlp.down_proj,8,128,8,128,0.68603515625,43.682373046875,45088768,3.6398775576866766,1,4.949917274643667e-5 +8,self_attn.q_proj,2,32,8,128,39.9375,5.015625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,2,64,8,128,47.28125,4.5078125,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,2,128,8,128,57.09375,4.25390625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,3,32,8,128,17.109375,7.015625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,3,64,8,128,20.3125,6.5078125,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,3,128,8,128,24.96875,6.25390625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,4,32,8,128,7.98828125,9.015625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,4,64,8,128,9.484375,8.5078125,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,4,128,8,128,11.65625,8.25390625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,8,32,8,128,0.4833984375,17.015625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,8,64,8,128,0.5732421875,16.5078125,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.q_proj,8,128,8,128,0.7021484375,16.25390625,16777216,4.825607138911687,1,8.275920117739588e-5 +8,self_attn.v_proj,2,32,8,128,21.9375,5.015625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,2,64,8,128,25.6875,4.5078125,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,2,128,8,128,29.109375,4.25390625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,3,32,8,128,9.3828125,7.015625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,3,64,8,128,10.9921875,6.5078125,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,3,128,8,128,12.4609375,6.25390625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,4,32,8,128,4.37890625,9.015625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,4,64,8,128,5.12890625,8.5078125,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,4,128,8,128,5.81640625,8.25390625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,8,32,8,128,0.26513671875,17.015625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,8,64,8,128,0.310302734375,16.5078125,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.v_proj,8,128,8,128,0.350341796875,16.25390625,16777216,3.224550114501631,1,8.275920117739588e-5 +8,self_attn.k_proj,2,32,8,128,39.875,5.015625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,2,64,8,128,47.03125,4.5078125,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,2,128,8,128,56.625,4.25390625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,3,32,8,128,17.078125,7.015625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,3,64,8,128,20.140625,6.5078125,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,3,128,8,128,24.6875,6.25390625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,4,32,8,128,7.96484375,9.015625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,4,64,8,128,9.3984375,8.5078125,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,4,128,8,128,11.5234375,8.25390625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,8,32,8,128,0.482421875,17.015625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,8,64,8,128,0.56884765625,16.5078125,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.k_proj,8,128,8,128,0.6943359375,16.25390625,16777216,4.302162251441214,1,8.275920117739588e-5 +8,self_attn.o_proj,2,32,8,128,21.296875,5.015625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,2,64,8,128,24.828125,4.5078125,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,2,128,8,128,28.03125,4.25390625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,3,32,8,128,9.1171875,7.015625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,3,64,8,128,10.6484375,6.5078125,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,3,128,8,128,12.0703125,6.25390625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,4,32,8,128,4.2578125,9.015625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,4,64,8,128,4.98046875,8.5078125,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,4,128,8,128,5.6484375,8.25390625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,8,32,8,128,0.2578125,17.015625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,8,64,8,128,0.301025390625,16.5078125,16777216,6.007940412405506,1,3.149401891278103e-5 +8,self_attn.o_proj,8,128,8,128,0.34033203125,16.25390625,16777216,6.007940412405506,1,3.149401891278103e-5 +8,mlp.gate_proj,2,32,8,128,49.5625,13.4794921875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,2,64,8,128,58.3125,12.11474609375,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,2,128,8,128,66.4375,11.432373046875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,3,32,8,128,21.203125,18.8544921875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,3,64,8,128,24.984375,17.48974609375,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,3,128,8,128,28.515625,16.807373046875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,4,32,8,128,9.8984375,24.2294921875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,4,64,8,128,11.65625,22.86474609375,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,4,128,8,128,13.3125,22.182373046875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,8,32,8,128,0.59912109375,45.7294921875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,8,64,8,128,0.70458984375,44.36474609375,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.gate_proj,8,128,8,128,0.8017578125,43.682373046875,45088768,3.3917166195373265,1,7.177540828706697e-5 +8,mlp.up_proj,2,32,8,128,44.1875,13.4794921875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,2,64,8,128,51.5,12.11474609375,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,2,128,8,128,58.0625,11.432373046875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,3,32,8,128,18.890625,18.8544921875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,3,64,8,128,22.03125,17.48974609375,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,3,128,8,128,24.875,16.807373046875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,4,32,8,128,8.8125,24.2294921875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,4,64,8,128,10.28125,22.86474609375,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,4,128,8,128,11.609375,22.182373046875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,8,32,8,128,0.53369140625,45.7294921875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,8,64,8,128,0.6220703125,44.36474609375,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.up_proj,8,128,8,128,0.69970703125,43.682373046875,45088768,3.143100821324133,1,7.177540828706697e-5 +8,mlp.down_proj,2,32,8,128,43.78125,13.4794921875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,2,64,8,128,51,12.11474609375,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,2,128,8,128,57.53125,11.432373046875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,3,32,8,128,18.71875,18.8544921875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,3,64,8,128,21.84375,17.48974609375,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,3,128,8,128,24.65625,16.807373046875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,4,32,8,128,8.734375,24.2294921875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,4,64,8,128,10.1953125,22.86474609375,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,4,128,8,128,11.515625,22.182373046875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,8,32,8,128,0.52880859375,45.7294921875,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,8,64,8,128,0.61669921875,44.36474609375,45088768,3.648714903862577,1,6.067876165616326e-5 +8,mlp.down_proj,8,128,8,128,0.69384765625,43.682373046875,45088768,3.648714903862577,1,6.067876165616326e-5 +9,self_attn.q_proj,2,32,8,128,39.4375,5.015625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,2,64,8,128,46.375,4.5078125,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,2,128,8,128,55.4375,4.25390625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,3,32,8,128,16.875,7.015625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,3,64,8,128,19.875,6.5078125,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,3,128,8,128,23.96875,6.25390625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,4,32,8,128,7.87109375,9.015625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,4,64,8,128,9.2734375,8.5078125,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,4,128,8,128,11.1875,8.25390625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,8,32,8,128,0.4765625,17.015625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,8,64,8,128,0.56103515625,16.5078125,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.q_proj,8,128,8,128,0.67431640625,16.25390625,16777216,4.141120940492358,1,1.2172867718618e-4 +9,self_attn.v_proj,2,32,8,128,22.421875,5.015625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,2,64,8,128,26.265625,4.5078125,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,2,128,8,128,29.78125,4.25390625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,3,32,8,128,9.5859375,7.015625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,3,64,8,128,11.234375,6.5078125,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,3,128,8,128,12.75,6.25390625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,4,32,8,128,4.47265625,9.015625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,4,64,8,128,5.2421875,8.5078125,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,4,128,8,128,5.94921875,8.25390625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,8,32,8,128,0.270751953125,17.015625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,8,64,8,128,0.317138671875,16.5078125,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.v_proj,8,128,8,128,0.358642578125,16.25390625,16777216,3.254566641438937,1,1.2172867718618e-4 +9,self_attn.k_proj,2,32,8,128,40.5625,5.015625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,2,64,8,128,47.75,4.5078125,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,2,128,8,128,57.3125,4.25390625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,3,32,8,128,17.34375,7.015625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,3,64,8,128,20.453125,6.5078125,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,3,128,8,128,24.859375,6.25390625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,4,32,8,128,8.09375,9.015625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,4,64,8,128,9.5390625,8.5078125,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,4,128,8,128,11.6015625,8.25390625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,8,32,8,128,0.490234375,17.015625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,8,64,8,128,0.5771484375,16.5078125,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.k_proj,8,128,8,128,0.69921875,16.25390625,16777216,4.097917798059499,1,1.2172867718618e-4 +9,self_attn.o_proj,2,32,8,128,21.796875,5.015625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,2,64,8,128,25.421875,4.5078125,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,2,128,8,128,28.703125,4.25390625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,3,32,8,128,9.3359375,7.015625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,3,64,8,128,10.921875,6.5078125,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,3,128,8,128,12.375,6.25390625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,4,32,8,128,4.359375,9.015625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,4,64,8,128,5.09765625,8.5078125,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,4,128,8,128,5.78515625,8.25390625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,8,32,8,128,0.263916015625,17.015625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,8,64,8,128,0.308349609375,16.5078125,16777216,4.903297282365117,1,4.24713252868969e-5 +9,self_attn.o_proj,8,128,8,128,0.3486328125,16.25390625,16777216,4.903297282365117,1,4.24713252868969e-5 +9,mlp.gate_proj,2,32,8,128,49.25,13.4794921875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,2,64,8,128,58.03125,12.11474609375,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,2,128,8,128,66.3125,11.432373046875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,3,32,8,128,21.0625,18.8544921875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,3,64,8,128,24.859375,17.48974609375,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,3,128,8,128,28.46875,16.807373046875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,4,32,8,128,9.828125,24.2294921875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,4,64,8,128,11.6015625,22.86474609375,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,4,128,8,128,13.2890625,22.182373046875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,8,32,8,128,0.59521484375,45.7294921875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,8,64,8,128,0.70166015625,44.36474609375,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.gate_proj,8,128,8,128,0.80029296875,43.682373046875,45088768,3.4635676641917645,1,8.42771987663582e-5 +9,mlp.up_proj,2,32,8,128,44.5,13.4794921875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,2,64,8,128,51.875,12.11474609375,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,2,128,8,128,58.5,11.432373046875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,3,32,8,128,19.03125,18.8544921875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,3,64,8,128,22.203125,17.48974609375,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,3,128,8,128,25.0625,16.807373046875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,4,32,8,128,8.8828125,24.2294921875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,4,64,8,128,10.359375,22.86474609375,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,4,128,8,128,11.703125,22.182373046875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,8,32,8,128,0.53759765625,45.7294921875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,8,64,8,128,0.626953125,44.36474609375,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.up_proj,8,128,8,128,0.705078125,43.682373046875,45088768,3.1384390669911246,1,8.42771987663582e-5 +9,mlp.down_proj,2,32,8,128,44.09375,13.4794921875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,2,64,8,128,51.375,12.11474609375,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,2,128,8,128,57.9375,11.432373046875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,3,32,8,128,18.859375,18.8544921875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,3,64,8,128,22,17.48974609375,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,3,128,8,128,24.84375,16.807373046875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,4,32,8,128,8.796875,24.2294921875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,4,64,8,128,10.265625,22.86474609375,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,4,128,8,128,11.59375,22.182373046875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,8,32,8,128,0.53271484375,45.7294921875,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,8,64,8,128,0.62109375,44.36474609375,45088768,3.607272014668365,1,6.981325714150444e-5 +9,mlp.down_proj,8,128,8,128,0.69873046875,43.682373046875,45088768,3.607272014668365,1,6.981325714150444e-5 +10,self_attn.q_proj,2,32,8,128,39.25,5.015625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,2,64,8,128,46.125,4.5078125,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,2,128,8,128,55.6875,4.25390625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,3,32,8,128,16.796875,7.015625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,3,64,8,128,19.75,6.5078125,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,3,128,8,128,24.109375,6.25390625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,4,32,8,128,7.8359375,9.015625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,4,64,8,128,9.2109375,8.5078125,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,4,128,8,128,11.25,8.25390625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,8,32,8,128,0.474365234375,17.015625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,8,64,8,128,0.55712890625,16.5078125,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.q_proj,8,128,8,128,0.67822265625,16.25390625,16777216,4.133870396866107,1,1.2341282854322e-4 +10,self_attn.v_proj,2,32,8,128,22.078125,5.015625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,2,64,8,128,25.828125,4.5078125,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,2,128,8,128,29.21875,4.25390625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,3,32,8,128,9.4453125,7.015625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,3,64,8,128,11.0546875,6.5078125,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,3,128,8,128,12.5078125,6.25390625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,4,32,8,128,4.40625,9.015625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,4,64,8,128,5.15625,8.5078125,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,4,128,8,128,5.84375,8.25390625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,8,32,8,128,0.266845703125,17.015625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,8,64,8,128,0.31201171875,16.5078125,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.v_proj,8,128,8,128,0.351806640625,16.25390625,16777216,3.221364068191206,1,1.2341282854322e-4 +10,self_attn.k_proj,2,32,8,128,40.71875,5.015625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,2,64,8,128,47.9375,4.5078125,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,2,128,8,128,58.0625,4.25390625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,3,32,8,128,17.421875,7.015625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,3,64,8,128,20.515625,6.5078125,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,3,128,8,128,25.328125,6.25390625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,4,32,8,128,8.125,9.015625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,4,64,8,128,9.5703125,8.5078125,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,4,128,8,128,11.8203125,8.25390625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,8,32,8,128,0.491943359375,17.015625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,8,64,8,128,0.5791015625,16.5078125,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.k_proj,8,128,8,128,0.71240234375,16.25390625,16777216,4.262353252802395,1,1.2341282854322e-4 +10,self_attn.o_proj,2,32,8,128,21.6875,5.015625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,2,64,8,128,25.28125,4.5078125,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,2,128,8,128,28.515625,4.25390625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,3,32,8,128,9.28125,7.015625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,3,64,8,128,10.8359375,6.5078125,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,3,128,8,128,12.2578125,6.25390625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,4,32,8,128,4.33203125,9.015625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,4,64,8,128,5.05859375,8.5078125,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,4,128,8,128,5.7265625,8.25390625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,8,32,8,128,0.26220703125,17.015625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,8,64,8,128,0.305908203125,16.5078125,16777216,4.063452398003821,1,5.917765520280227e-5 +10,self_attn.o_proj,8,128,8,128,0.344970703125,16.25390625,16777216,4.063452398003821,1,5.917765520280227e-5 +10,mlp.gate_proj,2,32,8,128,49.4375,13.4794921875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,2,64,8,128,58.4375,12.11474609375,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,2,128,8,128,67.0625,11.432373046875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,3,32,8,128,21.15625,18.8544921875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,3,64,8,128,25.0625,17.48974609375,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,3,128,8,128,28.84375,16.807373046875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,4,32,8,128,9.8671875,24.2294921875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,4,64,8,128,11.6953125,22.86474609375,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,4,128,8,128,13.46875,22.182373046875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,8,32,8,128,0.59765625,45.7294921875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,8,64,8,128,0.70703125,44.36474609375,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.gate_proj,8,128,8,128,0.8115234375,43.682373046875,45088768,3.659082896762817,1,9.913711255649103e-5 +10,mlp.up_proj,2,32,8,128,45.03125,13.4794921875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,2,64,8,128,52.5,12.11474609375,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,2,128,8,128,59.25,11.432373046875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,3,32,8,128,19.25,18.8544921875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,3,64,8,128,22.46875,17.48974609375,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,3,128,8,128,25.375,16.807373046875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,4,32,8,128,8.984375,24.2294921875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,4,64,8,128,10.484375,22.86474609375,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,4,128,8,128,11.84375,22.182373046875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,8,32,8,128,0.5439453125,45.7294921875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,8,64,8,128,0.63427734375,44.36474609375,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.up_proj,8,128,8,128,0.7138671875,43.682373046875,45088768,3.0941643300420067,1,9.913711255649103e-5 +10,mlp.down_proj,2,32,8,128,44.46875,13.4794921875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,2,64,8,128,51.8125,12.11474609375,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,2,128,8,128,58.40625,11.432373046875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,3,32,8,128,19.015625,18.8544921875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,3,64,8,128,22.171875,17.48974609375,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,3,128,8,128,25.03125,16.807373046875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,4,32,8,128,8.875,24.2294921875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,4,64,8,128,10.3515625,22.86474609375,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,4,128,8,128,11.6875,22.182373046875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,8,32,8,128,0.537109375,45.7294921875,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,8,64,8,128,0.6259765625,44.36474609375,45088768,3.633526000703724,1,8.062620327109471e-5 +10,mlp.down_proj,8,128,8,128,0.70458984375,43.682373046875,45088768,3.633526000703724,1,8.062620327109471e-5 +11,self_attn.q_proj,2,32,8,128,38.03125,5.015625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,2,64,8,128,45.03125,4.5078125,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,2,128,8,128,55.03125,4.25390625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,3,32,8,128,16.296875,7.015625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,3,64,8,128,19.34375,6.5078125,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,3,128,8,128,24.15625,6.25390625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,4,32,8,128,7.6015625,9.015625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,4,64,8,128,9.0234375,8.5078125,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,4,128,8,128,11.28125,8.25390625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,8,32,8,128,0.460205078125,17.015625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,8,64,8,128,0.5458984375,16.5078125,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.q_proj,8,128,8,128,0.6796875,16.25390625,16777216,5.116813149153919,1,1.4837335038464e-4 +11,self_attn.v_proj,2,32,8,128,23.15625,5.015625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,2,64,8,128,27.140625,4.5078125,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,2,128,8,128,30.78125,4.25390625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,3,32,8,128,9.90625,7.015625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,3,64,8,128,11.6171875,6.5078125,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,3,128,8,128,13.1796875,6.25390625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,4,32,8,128,4.62109375,9.015625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,4,64,8,128,5.421875,8.5078125,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,4,128,8,128,6.15234375,8.25390625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,8,32,8,128,0.27978515625,17.015625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,8,64,8,128,0.327880859375,16.5078125,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.v_proj,8,128,8,128,0.370849609375,16.25390625,16777216,3.3207023327508245,1,1.4837335038464e-4 +11,self_attn.k_proj,2,32,8,128,37.875,5.015625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,2,64,8,128,44.75,4.5078125,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,2,128,8,128,54.375,4.25390625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,3,32,8,128,16.203125,7.015625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,3,64,8,128,19.15625,6.5078125,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,3,128,8,128,23.84375,6.25390625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,4,32,8,128,7.55859375,9.015625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,4,64,8,128,8.9453125,8.5078125,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,4,128,8,128,11.1328125,8.25390625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,8,32,8,128,0.457763671875,17.015625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,8,64,8,128,0.541015625,16.5078125,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.k_proj,8,128,8,128,0.67041015625,16.25390625,16777216,4.630848587525195,1,1.4837335038464e-4 +11,self_attn.o_proj,2,32,8,128,22.578125,5.015625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,2,64,8,128,26.328125,4.5078125,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,2,128,8,128,29.75,4.25390625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,3,32,8,128,9.6796875,7.015625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,3,64,8,128,11.3203125,6.5078125,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,3,128,8,128,12.8515625,6.25390625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,4,32,8,128,4.51953125,9.015625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,4,64,8,128,5.2890625,8.5078125,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,4,128,8,128,6.01171875,8.25390625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,8,32,8,128,0.273681640625,17.015625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,8,64,8,128,0.31982421875,16.5078125,16777216,6.151251257937108,1,6.026827759342268e-5 +11,self_attn.o_proj,8,128,8,128,0.362060546875,16.25390625,16777216,6.151251257937108,1,6.026827759342268e-5 +11,mlp.gate_proj,2,32,8,128,49.0625,13.4794921875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,2,64,8,128,57.9375,12.11474609375,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,2,128,8,128,66.375,11.432373046875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,3,32,8,128,21,18.8544921875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,3,64,8,128,24.84375,17.48974609375,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,3,128,8,128,28.53125,16.807373046875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,4,32,8,128,9.796875,24.2294921875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,4,64,8,128,11.59375,22.86474609375,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,4,128,8,128,13.3203125,22.182373046875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,8,32,8,128,0.5927734375,45.7294921875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,8,64,8,128,0.70068359375,44.36474609375,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.gate_proj,8,128,8,128,0.80224609375,43.682373046875,45088768,3.578670038605152,1,1.1187846394022e-4 +11,mlp.up_proj,2,32,8,128,45.375,13.4794921875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,2,64,8,128,52.90625,12.11474609375,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,2,128,8,128,59.6875,11.432373046875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,3,32,8,128,19.40625,18.8544921875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,3,64,8,128,22.640625,17.48974609375,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,3,128,8,128,25.578125,16.807373046875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,4,32,8,128,9.0546875,24.2294921875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,4,64,8,128,10.5703125,22.86474609375,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,4,128,8,128,11.9375,22.182373046875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,8,32,8,128,0.54833984375,45.7294921875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,8,64,8,128,0.63916015625,44.36474609375,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.up_proj,8,128,8,128,0.71923828125,43.682373046875,45088768,3.1151256150814888,1,1.1187846394022e-4 +11,mlp.down_proj,2,32,8,128,44.75,13.4794921875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,2,64,8,128,52.15625,12.11474609375,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,2,128,8,128,58.8125,11.432373046875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,3,32,8,128,19.140625,18.8544921875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,3,64,8,128,22.328125,17.48974609375,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,3,128,8,128,25.21875,16.807373046875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,4,32,8,128,8.9296875,24.2294921875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,4,64,8,128,10.421875,22.86474609375,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,4,128,8,128,11.7734375,22.182373046875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,8,32,8,128,0.541015625,45.7294921875,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,8,64,8,128,0.63037109375,44.36474609375,45088768,3.553004124944969,1,9.217795013682915e-5 +11,mlp.down_proj,8,128,8,128,0.70947265625,43.682373046875,45088768,3.553004124944969,1,9.217795013682915e-5 +12,self_attn.q_proj,2,32,8,128,38.125,5.015625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,2,64,8,128,44.84375,4.5078125,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,2,128,8,128,54.0625,4.25390625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,3,32,8,128,16.328125,7.015625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,3,64,8,128,19.21875,6.5078125,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,3,128,8,128,23.390625,6.25390625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,4,32,8,128,7.61328125,9.015625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,4,64,8,128,8.96875,8.5078125,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,4,128,8,128,10.9296875,8.25390625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,8,32,8,128,0.461181640625,17.015625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,8,64,8,128,0.54296875,16.5078125,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.q_proj,8,128,8,128,0.658203125,16.25390625,16777216,4.227513682256377,1,1.6132026212289e-4 +12,self_attn.v_proj,2,32,8,128,22.875,5.015625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,2,64,8,128,26.796875,4.5078125,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,2,128,8,128,30.421875,4.25390625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,3,32,8,128,9.78125,7.015625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,3,64,8,128,11.46875,6.5078125,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,3,128,8,128,13.0234375,6.25390625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,4,32,8,128,4.5625,9.015625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,4,64,8,128,5.3515625,8.5078125,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,4,128,8,128,6.08203125,8.25390625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,8,32,8,128,0.2763671875,17.015625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,8,64,8,128,0.32373046875,16.5078125,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.v_proj,8,128,8,128,0.3662109375,16.25390625,16777216,3.291719617992136,1,1.6132026212289e-4 +12,self_attn.k_proj,2,32,8,128,39.4375,5.015625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,2,64,8,128,46.40625,4.5078125,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,2,128,8,128,56.125,4.25390625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,3,32,8,128,16.875,7.015625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,3,64,8,128,19.875,6.5078125,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,3,128,8,128,24.375,6.25390625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,4,32,8,128,7.87109375,9.015625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,4,64,8,128,9.265625,8.5078125,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,4,128,8,128,11.375,8.25390625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,8,32,8,128,0.476806640625,17.015625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,8,64,8,128,0.56103515625,16.5078125,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.k_proj,8,128,8,128,0.685546875,16.25390625,16777216,4.210345039001077,1,1.6132026212289e-4 +12,self_attn.o_proj,2,32,8,128,22.328125,5.015625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,2,64,8,128,26.03125,4.5078125,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,2,128,8,128,29.390625,4.25390625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,3,32,8,128,9.5625,7.015625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,3,64,8,128,11.171875,6.5078125,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,3,128,8,128,12.6484375,6.25390625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,4,32,8,128,4.4609375,9.015625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,4,64,8,128,5.21484375,8.5078125,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,4,128,8,128,5.91015625,8.25390625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,8,32,8,128,0.270263671875,17.015625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,8,64,8,128,0.3154296875,16.5078125,16777216,4.434055632045808,1,7.075531175360084e-5 +12,self_attn.o_proj,8,128,8,128,0.35595703125,16.25390625,16777216,4.434055632045808,1,7.075531175360084e-5 +12,mlp.gate_proj,2,32,8,128,48.625,13.4794921875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,2,64,8,128,57.375,12.11474609375,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,2,128,8,128,65.625,11.432373046875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,3,32,8,128,20.8125,18.8544921875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,3,64,8,128,24.59375,17.48974609375,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,3,128,8,128,28.203125,16.807373046875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,4,32,8,128,9.7109375,24.2294921875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,4,64,8,128,11.4765625,22.86474609375,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,4,128,8,128,13.1640625,22.182373046875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,8,32,8,128,0.587890625,45.7294921875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,8,64,8,128,0.69384765625,44.36474609375,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.gate_proj,8,128,8,128,0.79296875,43.682373046875,45088768,3.523107162105424,1,1.2630679702851e-4 +12,mlp.up_proj,2,32,8,128,45.78125,13.4794921875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,2,64,8,128,53.40625,12.11474609375,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,2,128,8,128,60.25,11.432373046875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,3,32,8,128,19.578125,18.8544921875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,3,64,8,128,22.84375,17.48974609375,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,3,128,8,128,25.796875,16.807373046875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,4,32,8,128,9.1328125,24.2294921875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,4,64,8,128,10.6640625,22.86474609375,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,4,128,8,128,12.0390625,22.182373046875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,8,32,8,128,0.55322265625,45.7294921875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,8,64,8,128,0.64453125,44.36474609375,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.up_proj,8,128,8,128,0.7255859375,43.682373046875,45088768,3.0757519740562933,1,1.2630679702851e-4 +12,mlp.down_proj,2,32,8,128,45.125,13.4794921875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,2,64,8,128,52.5625,12.11474609375,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,2,128,8,128,59.25,11.432373046875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,3,32,8,128,19.296875,18.8544921875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,3,64,8,128,22.484375,17.48974609375,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,3,128,8,128,25.390625,16.807373046875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,4,32,8,128,9.0078125,24.2294921875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,4,64,8,128,10.5,22.86474609375,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,4,128,8,128,11.8515625,22.182373046875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,8,32,8,128,0.544921875,45.7294921875,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,8,64,8,128,0.634765625,44.36474609375,45088768,3.291699935375262,1,1.0519409261178e-4 +12,mlp.down_proj,8,128,8,128,0.7138671875,43.682373046875,45088768,3.291699935375262,1,1.0519409261178e-4 +13,self_attn.q_proj,2,32,8,128,37.78125,5.015625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,2,64,8,128,44.46875,4.5078125,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,2,128,8,128,53.40625,4.25390625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,3,32,8,128,16.171875,7.015625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,3,64,8,128,19.0625,6.5078125,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,3,128,8,128,23.078125,6.25390625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,4,32,8,128,7.546875,9.015625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,4,64,8,128,8.890625,8.5078125,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,4,128,8,128,10.78125,8.25390625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,8,32,8,128,0.456787109375,17.015625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,8,64,8,128,0.53759765625,16.5078125,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.q_proj,8,128,8,128,0.6494140625,16.25390625,16777216,4.246715043246877,1,1.5230204735416e-4 +13,self_attn.v_proj,2,32,8,128,23.65625,5.015625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,2,64,8,128,27.703125,4.5078125,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,2,128,8,128,31.390625,4.25390625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,3,32,8,128,10.1171875,7.015625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,3,64,8,128,11.859375,6.5078125,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,3,128,8,128,13.4375,6.25390625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,4,32,8,128,4.71875,9.015625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,4,64,8,128,5.53515625,8.5078125,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,4,128,8,128,6.2734375,8.25390625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,8,32,8,128,0.285888671875,17.015625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,8,64,8,128,0.334716796875,16.5078125,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.v_proj,8,128,8,128,0.378173828125,16.25390625,16777216,3.240945054885195,1,1.5230204735416e-4 +13,self_attn.k_proj,2,32,8,128,38.625,5.015625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,2,64,8,128,45.5,4.5078125,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,2,128,8,128,54.8125,4.25390625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,3,32,8,128,16.515625,7.015625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,3,64,8,128,19.46875,6.5078125,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,3,128,8,128,23.75,6.25390625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,4,32,8,128,7.70703125,9.015625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,4,64,8,128,9.0859375,8.5078125,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,4,128,8,128,11.09375,8.25390625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,8,32,8,128,0.466552734375,17.015625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,8,64,8,128,0.5498046875,16.5078125,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.k_proj,8,128,8,128,0.66796875,16.25390625,16777216,4.160297130333501,1,1.5230204735416e-4 +13,self_attn.o_proj,2,32,8,128,23.109375,5.015625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,2,64,8,128,26.9375,4.5078125,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,2,128,8,128,30.453125,4.25390625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,3,32,8,128,9.8984375,7.015625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,3,64,8,128,11.5703125,6.5078125,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,3,128,8,128,13.1171875,6.25390625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,4,32,8,128,4.62109375,9.015625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,4,64,8,128,5.40234375,8.5078125,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,4,128,8,128,6.125,8.25390625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,8,32,8,128,0.27978515625,17.015625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,8,64,8,128,0.32666015625,16.5078125,16777216,4.47524457880676,1,7.67381425248459e-5 +13,self_attn.o_proj,8,128,8,128,0.369140625,16.25390625,16777216,4.47524457880676,1,7.67381425248459e-5 +13,mlp.gate_proj,2,32,8,128,48.46875,13.4794921875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,2,64,8,128,57.15625,12.11474609375,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,2,128,8,128,65.375,11.432373046875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,3,32,8,128,20.734375,18.8544921875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,3,64,8,128,24.484375,17.48974609375,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,3,128,8,128,28.078125,16.807373046875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,4,32,8,128,9.671875,24.2294921875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,4,64,8,128,11.4296875,22.86474609375,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,4,128,8,128,13.1015625,22.182373046875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,8,32,8,128,0.58544921875,45.7294921875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,8,64,8,128,0.69091796875,44.36474609375,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.gate_proj,8,128,8,128,0.78955078125,43.682373046875,45088768,3.492987813719813,1,1.5027161862235e-4 +13,mlp.up_proj,2,32,8,128,46.28125,13.4794921875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,2,64,8,128,53.96875,12.11474609375,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,2,128,8,128,60.90625,11.432373046875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,3,32,8,128,19.796875,18.8544921875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,3,64,8,128,23.09375,17.48974609375,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,3,128,8,128,26.09375,16.807373046875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,4,32,8,128,9.234375,24.2294921875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,4,64,8,128,10.7734375,22.86474609375,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,4,128,8,128,12.1796875,22.182373046875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,8,32,8,128,0.55908203125,45.7294921875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,8,64,8,128,0.65185546875,44.36474609375,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.up_proj,8,128,8,128,0.7333984375,43.682373046875,45088768,3.099709982186448,1,1.5027161862235e-4 +13,mlp.down_proj,2,32,8,128,45.46875,13.4794921875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,2,64,8,128,52.9375,12.11474609375,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,2,128,8,128,59.6875,11.432373046875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,3,32,8,128,19.4375,18.8544921875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,3,64,8,128,22.65625,17.48974609375,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,3,128,8,128,25.5625,16.807373046875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,4,32,8,128,9.0703125,24.2294921875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,4,64,8,128,10.578125,22.86474609375,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,4,128,8,128,11.9296875,22.182373046875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,8,32,8,128,0.54931640625,45.7294921875,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,8,64,8,128,0.6396484375,44.36474609375,45088768,3.647359138499682,1,1.2898707063868e-4 +13,mlp.down_proj,8,128,8,128,0.71923828125,43.682373046875,45088768,3.647359138499682,1,1.2898707063868e-4 +14,self_attn.q_proj,2,32,8,128,37.5625,5.015625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,2,64,8,128,44.15625,4.5078125,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,2,128,8,128,53.25,4.25390625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,3,32,8,128,16.078125,7.015625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,3,64,8,128,18.90625,6.5078125,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,3,128,8,128,23.015625,6.25390625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,4,32,8,128,7.5,9.015625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,4,64,8,128,8.828125,8.5078125,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,4,128,8,128,10.7421875,8.25390625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,8,32,8,128,0.4541015625,17.015625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,8,64,8,128,0.53369140625,16.5078125,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.q_proj,8,128,8,128,0.6474609375,16.25390625,16777216,4.172159714823189,1,1.7738113820087e-4 +14,self_attn.v_proj,2,32,8,128,23.15625,5.015625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,2,64,8,128,27.09375,4.5078125,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,2,128,8,128,30.703125,4.25390625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,3,32,8,128,9.8984375,7.015625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,3,64,8,128,11.59375,6.5078125,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,3,128,8,128,13.140625,6.25390625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,4,32,8,128,4.62109375,9.015625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,4,64,8,128,5.41015625,8.5078125,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,4,128,8,128,6.1328125,8.25390625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,8,32,8,128,0.27978515625,17.015625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,8,64,8,128,0.3271484375,16.5078125,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.v_proj,8,128,8,128,0.36962890625,16.25390625,16777216,3.220789266884044,1,1.7738113820087e-4 +14,self_attn.k_proj,2,32,8,128,38.40625,5.015625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,2,64,8,128,45.15625,4.5078125,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,2,128,8,128,54.625,4.25390625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,3,32,8,128,16.4375,7.015625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,3,64,8,128,19.328125,6.5078125,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,3,128,8,128,23.6875,6.25390625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,4,32,8,128,7.6640625,9.015625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,4,64,8,128,9.0234375,8.5078125,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,4,128,8,128,11.0546875,8.25390625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,8,32,8,128,0.464111328125,17.015625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,8,64,8,128,0.54541015625,16.5078125,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.k_proj,8,128,8,128,0.666015625,16.25390625,16777216,4.20591811396014,1,1.7738113820087e-4 +14,self_attn.o_proj,2,32,8,128,22.640625,5.015625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,2,64,8,128,26.375,4.5078125,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,2,128,8,128,29.765625,4.25390625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,3,32,8,128,9.6875,7.015625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,3,64,8,128,11.3125,6.5078125,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,3,128,8,128,12.7890625,6.25390625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,4,32,8,128,4.5234375,9.015625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,4,64,8,128,5.28515625,8.5078125,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,4,128,8,128,5.98046875,8.25390625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,8,32,8,128,0.27392578125,17.015625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,8,64,8,128,0.319580078125,16.5078125,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,self_attn.o_proj,8,128,8,128,0.3603515625,16.25390625,16777216,4.3913518343431885,1,8.662589243613183e-5 +14,mlp.gate_proj,2,32,8,128,48.3125,13.4794921875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,2,64,8,128,56.96875,12.11474609375,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,2,128,8,128,65.25,11.432373046875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,3,32,8,128,20.671875,18.8544921875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,3,64,8,128,24.421875,17.48974609375,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,3,128,8,128,28.03125,16.807373046875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,4,32,8,128,9.6484375,24.2294921875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,4,64,8,128,11.3984375,22.86474609375,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,4,128,8,128,13.0859375,22.182373046875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,8,32,8,128,0.583984375,45.7294921875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,8,64,8,128,0.68896484375,44.36474609375,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.gate_proj,8,128,8,128,0.7880859375,43.682373046875,45088768,3.5157555990507983,1,1.6938253247644e-4 +14,mlp.up_proj,2,32,8,128,46.3125,13.4794921875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,2,64,8,128,54,12.11474609375,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,2,128,8,128,60.9375,11.432373046875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,3,32,8,128,19.8125,18.8544921875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,3,64,8,128,23.109375,17.48974609375,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,3,128,8,128,26.09375,16.807373046875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,4,32,8,128,9.2421875,24.2294921875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,4,64,8,128,10.78125,22.86474609375,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,4,128,8,128,12.1796875,22.182373046875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,8,32,8,128,0.5595703125,45.7294921875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,8,64,8,128,0.65234375,44.36474609375,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.up_proj,8,128,8,128,0.73388671875,43.682373046875,45088768,3.088386720975056,1,1.6938253247644e-4 +14,mlp.down_proj,2,32,8,128,45.5,13.4794921875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,2,64,8,128,53,12.11474609375,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,2,128,8,128,59.75,11.432373046875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,3,32,8,128,19.453125,18.8544921875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,3,64,8,128,22.671875,17.48974609375,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,3,128,8,128,25.578125,16.807373046875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,4,32,8,128,9.078125,24.2294921875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,4,64,8,128,10.5859375,22.86474609375,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,4,128,8,128,11.9375,22.182373046875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,8,32,8,128,0.5498046875,45.7294921875,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,8,64,8,128,0.6396484375,44.36474609375,45088768,3.240919165543396,1,1.5278153296094e-4 +14,mlp.down_proj,8,128,8,128,0.71923828125,43.682373046875,45088768,3.240919165543396,1,1.5278153296094e-4 +15,self_attn.q_proj,2,32,8,128,37.125,5.015625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,2,64,8,128,43.625,4.5078125,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,2,128,8,128,51.96875,4.25390625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,3,32,8,128,15.8671875,7.015625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,3,64,8,128,18.671875,6.5078125,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,3,128,8,128,22.4375,6.25390625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,4,32,8,128,7.40625,9.015625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,4,64,8,128,8.7109375,8.5078125,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,4,128,8,128,10.46875,8.25390625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,8,32,8,128,0.4482421875,17.015625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,8,64,8,128,0.52734375,16.5078125,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.q_proj,8,128,8,128,0.630859375,16.25390625,16777216,4.180196395053585,1,1.9123419770039e-4 +15,self_attn.v_proj,2,32,8,128,24.015625,5.015625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,2,64,8,128,28.0625,4.5078125,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,2,128,8,128,31.734375,4.25390625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,3,32,8,128,10.2734375,7.015625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,3,64,8,128,12.0078125,6.5078125,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,3,128,8,128,13.5859375,6.25390625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,4,32,8,128,4.79296875,9.015625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,4,64,8,128,5.60546875,8.5078125,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,4,128,8,128,6.3359375,8.25390625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,8,32,8,128,0.290283203125,17.015625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,8,64,8,128,0.339111328125,16.5078125,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.v_proj,8,128,8,128,0.382080078125,16.25390625,16777216,3.1698077407619936,1,1.9123419770039e-4 +15,self_attn.k_proj,2,32,8,128,38.53125,5.015625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,2,64,8,128,45.3125,4.5078125,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,2,128,8,128,54.375,4.25390625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,3,32,8,128,16.484375,7.015625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,3,64,8,128,19.40625,6.5078125,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,3,128,8,128,23.515625,6.25390625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,4,32,8,128,7.69140625,9.015625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,4,64,8,128,9.046875,8.5078125,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,4,128,8,128,10.9765625,8.25390625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,8,32,8,128,0.465576171875,17.015625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,8,64,8,128,0.54736328125,16.5078125,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.k_proj,8,128,8,128,0.6611328125,16.25390625,16777216,4.031791099881734,1,1.9123419770039e-4 +15,self_attn.o_proj,2,32,8,128,23.515625,5.015625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,2,64,8,128,27.4375,4.5078125,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,2,128,8,128,30.984375,4.25390625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,3,32,8,128,10.0703125,7.015625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,3,64,8,128,11.765625,6.5078125,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,3,128,8,128,13.328125,6.25390625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,4,32,8,128,4.69921875,9.015625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,4,64,8,128,5.4921875,8.5078125,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,4,128,8,128,6.2265625,8.25390625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,8,32,8,128,0.284423828125,17.015625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,8,64,8,128,0.33203125,16.5078125,16777216,3.962050750665648,1,9.319519449491054e-5 +15,self_attn.o_proj,8,128,8,128,0.375244140625,16.25390625,16777216,3.962050750665648,1,9.319519449491054e-5 +15,mlp.gate_proj,2,32,8,128,48.625,13.4794921875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,2,64,8,128,57.375,12.11474609375,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,2,128,8,128,65.6875,11.432373046875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,3,32,8,128,20.796875,18.8544921875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,3,64,8,128,24.59375,17.48974609375,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,3,128,8,128,28.21875,16.807373046875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,4,32,8,128,9.7109375,24.2294921875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,4,64,8,128,11.4765625,22.86474609375,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,4,128,8,128,13.171875,22.182373046875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,8,32,8,128,0.587890625,45.7294921875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,8,64,8,128,0.69384765625,44.36474609375,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.gate_proj,8,128,8,128,0.79345703125,43.682373046875,45088768,3.495786797250684,1,2.0993905491195e-4 +15,mlp.up_proj,2,32,8,128,46.59375,13.4794921875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,2,64,8,128,54.375,12.11474609375,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,2,128,8,128,61.40625,11.432373046875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,3,32,8,128,19.921875,18.8544921875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,3,64,8,128,23.265625,17.48974609375,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,3,128,8,128,26.28125,16.807373046875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,4,32,8,128,9.296875,24.2294921875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,4,64,8,128,10.859375,22.86474609375,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,4,128,8,128,12.265625,22.182373046875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,8,32,8,128,0.56298828125,45.7294921875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,8,64,8,128,0.65625,44.36474609375,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.up_proj,8,128,8,128,0.73876953125,43.682373046875,45088768,3.103608058016212,1,2.0993905491195e-4 +15,mlp.down_proj,2,32,8,128,45.65625,13.4794921875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,2,64,8,128,53.15625,12.11474609375,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,2,128,8,128,59.9375,11.432373046875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,3,32,8,128,19.53125,18.8544921875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,3,64,8,128,22.765625,17.48974609375,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,3,128,8,128,25.671875,16.807373046875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,4,32,8,128,9.109375,24.2294921875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,4,64,8,128,10.625,22.86474609375,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,4,128,8,128,11.984375,22.182373046875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,8,32,8,128,0.5517578125,45.7294921875,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,8,64,8,128,0.642578125,44.36474609375,45088768,3.5596160792985456,1,1.9223659182898e-4 +15,mlp.down_proj,8,128,8,128,0.72216796875,43.682373046875,45088768,3.5596160792985456,1,1.9223659182898e-4 +16,self_attn.q_proj,2,32,8,128,36.96875,5.015625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,2,64,8,128,43.5,4.5078125,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,2,128,8,128,52.25,4.25390625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,3,32,8,128,15.8203125,7.015625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,3,64,8,128,18.65625,6.5078125,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,3,128,8,128,22.59375,6.25390625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,4,32,8,128,7.3828125,9.015625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,4,64,8,128,8.703125,8.5078125,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,4,128,8,128,10.5546875,8.25390625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,8,32,8,128,0.447021484375,17.015625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,8,64,8,128,0.5263671875,16.5078125,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.q_proj,8,128,8,128,0.63623046875,16.25390625,16777216,4.938775650774272,1,2.2740692656952e-4 +16,self_attn.v_proj,2,32,8,128,25.03125,5.015625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,2,64,8,128,29.265625,4.5078125,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,2,128,8,128,33.125,4.25390625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,3,32,8,128,10.7109375,7.015625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,3,64,8,128,12.5234375,6.5078125,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,3,128,8,128,14.1796875,6.25390625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,4,32,8,128,4.99609375,9.015625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,4,64,8,128,5.84375,8.5078125,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,4,128,8,128,6.62109375,8.25390625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,8,32,8,128,0.302490234375,17.015625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,8,64,8,128,0.353515625,16.5078125,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.v_proj,8,128,8,128,0.39892578125,16.25390625,16777216,3.1732431739777383,1,2.2740692656952e-4 +16,self_attn.k_proj,2,32,8,128,38,5.015625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,2,64,8,128,44.71875,4.5078125,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,2,128,8,128,53.875,4.25390625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,3,32,8,128,16.25,7.015625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,3,64,8,128,19.125,6.5078125,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,3,128,8,128,23.34375,6.25390625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,4,32,8,128,7.5859375,9.015625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,4,64,8,128,8.9296875,8.5078125,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,4,128,8,128,10.8984375,8.25390625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,8,32,8,128,0.459228515625,17.015625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,8,64,8,128,0.5400390625,16.5078125,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.k_proj,8,128,8,128,0.65673828125,16.25390625,16777216,4.181085379327221,1,2.2740692656952e-4 +16,self_attn.o_proj,2,32,8,128,24.5625,5.015625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,2,64,8,128,28.625,4.5078125,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,2,128,8,128,32.3125,4.25390625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,3,32,8,128,10.515625,7.015625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,3,64,8,128,12.28125,6.5078125,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,3,128,8,128,13.90625,6.25390625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,4,32,8,128,4.90625,9.015625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,4,64,8,128,5.73046875,8.5078125,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,4,128,8,128,6.49609375,8.25390625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,8,32,8,128,0.297119140625,17.015625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,8,64,8,128,0.3466796875,16.5078125,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,self_attn.o_proj,8,128,8,128,0.391357421875,16.25390625,16777216,3.9618297565651854,1,1.3398118608165e-4 +16,mlp.gate_proj,2,32,8,128,48.84375,13.4794921875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,2,64,8,128,57.59375,12.11474609375,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,2,128,8,128,65.9375,11.432373046875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,3,32,8,128,20.890625,18.8544921875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,3,64,8,128,24.6875,17.48974609375,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,3,128,8,128,28.3125,16.807373046875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,4,32,8,128,9.75,24.2294921875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,4,64,8,128,11.515625,22.86474609375,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,4,128,8,128,13.2109375,22.182373046875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,8,32,8,128,0.59033203125,45.7294921875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,8,64,8,128,0.6962890625,44.36474609375,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.gate_proj,8,128,8,128,0.7958984375,43.682373046875,45088768,3.4721134762605304,1,2.7203984791412e-4 +16,mlp.up_proj,2,32,8,128,46.5,13.4794921875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,2,64,8,128,54.21875,12.11474609375,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,2,128,8,128,61.15625,11.432373046875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,3,32,8,128,19.890625,18.8544921875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,3,64,8,128,23.1875,17.48974609375,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,3,128,8,128,26.1875,16.807373046875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,4,32,8,128,9.28125,24.2294921875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,4,64,8,128,10.828125,22.86474609375,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,4,128,8,128,12.21875,22.182373046875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,8,32,8,128,0.5615234375,45.7294921875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,8,64,8,128,0.65478515625,44.36474609375,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.up_proj,8,128,8,128,0.736328125,43.682373046875,45088768,3.0796057455877506,1,2.7203984791412e-4 +16,mlp.down_proj,2,32,8,128,45.59375,13.4794921875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,2,64,8,128,53.125,12.11474609375,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,2,128,8,128,59.875,11.432373046875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,3,32,8,128,19.515625,18.8544921875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,3,64,8,128,22.734375,17.48974609375,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,3,128,8,128,25.640625,16.807373046875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,4,32,8,128,9.1015625,24.2294921875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,4,64,8,128,10.609375,22.86474609375,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,4,128,8,128,11.96875,22.182373046875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,8,32,8,128,0.55126953125,45.7294921875,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,8,64,8,128,0.6416015625,44.36474609375,45088768,3.307626413448843,1,2.8085691155865e-4 +16,mlp.down_proj,8,128,8,128,0.72119140625,43.682373046875,45088768,3.307626413448843,1,2.8085691155865e-4 +17,self_attn.q_proj,2,32,8,128,36.625,5.015625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,2,64,8,128,43.09375,4.5078125,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,2,128,8,128,50.84375,4.25390625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,3,32,8,128,15.6640625,7.015625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,3,64,8,128,18.453125,6.5078125,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,3,128,8,128,21.890625,6.25390625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,4,32,8,128,7.3125,9.015625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,4,64,8,128,8.6171875,8.5078125,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,4,128,8,128,10.2265625,8.25390625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,8,32,8,128,0.44287109375,17.015625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,8,64,8,128,0.521484375,16.5078125,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.q_proj,8,128,8,128,0.6162109375,16.25390625,16777216,4.41058356494222,1,1.819304889068e-4 +17,self_attn.v_proj,2,32,8,128,24.875,5.015625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,2,64,8,128,29.03125,4.5078125,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,2,128,8,128,32.8125,4.25390625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,3,32,8,128,10.6328125,7.015625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,3,64,8,128,12.421875,6.5078125,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,3,128,8,128,14.0390625,6.25390625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,4,32,8,128,4.96484375,9.015625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,4,64,8,128,5.796875,8.5078125,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,4,128,8,128,6.5546875,8.25390625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,8,32,8,128,0.300537109375,17.015625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,8,64,8,128,0.3505859375,16.5078125,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.v_proj,8,128,8,128,0.39501953125,16.25390625,16777216,3.131738031845088,1,1.819304889068e-4 +17,self_attn.k_proj,2,32,8,128,37.625,5.015625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,2,64,8,128,44.28125,4.5078125,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,2,128,8,128,52.5625,4.25390625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,3,32,8,128,16.078125,7.015625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,3,64,8,128,18.953125,6.5078125,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,3,128,8,128,22.6875,6.25390625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,4,32,8,128,7.5078125,9.015625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,4,64,8,128,8.8515625,8.5078125,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,4,128,8,128,10.59375,8.25390625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,8,32,8,128,0.45458984375,17.015625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,8,64,8,128,0.53515625,16.5078125,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.k_proj,8,128,8,128,0.63818359375,16.25390625,16777216,4.011172975365525,1,1.819304889068e-4 +17,self_attn.o_proj,2,32,8,128,24.59375,5.015625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,2,64,8,128,28.671875,4.5078125,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,2,128,8,128,32.375,4.25390625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,3,32,8,128,10.5390625,7.015625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,3,64,8,128,12.3125,6.5078125,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,3,128,8,128,13.953125,6.25390625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,4,32,8,128,4.9140625,9.015625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,4,64,8,128,5.74609375,8.5078125,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,4,128,8,128,6.51953125,8.25390625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,8,32,8,128,0.297607421875,17.015625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,8,64,8,128,0.34765625,16.5078125,16777216,4.40230065747996,1,9.5199175120797e-5 +17,self_attn.o_proj,8,128,8,128,0.392822265625,16.25390625,16777216,4.40230065747996,1,9.5199175120797e-5 +17,mlp.gate_proj,2,32,8,128,49.15625,13.4794921875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,2,64,8,128,57.875,12.11474609375,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,2,128,8,128,66.125,11.432373046875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,3,32,8,128,21.015625,18.8544921875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,3,64,8,128,24.78125,17.48974609375,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,3,128,8,128,28.359375,16.807373046875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,4,32,8,128,9.8125,24.2294921875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,4,64,8,128,11.5625,22.86474609375,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,4,128,8,128,13.234375,22.182373046875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,8,32,8,128,0.59375,45.7294921875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,8,64,8,128,0.69921875,44.36474609375,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.gate_proj,8,128,8,128,0.79736328125,43.682373046875,45088768,3.379672209074253,1,3.0208012321963e-4 +17,mlp.up_proj,2,32,8,128,46.28125,13.4794921875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,2,64,8,128,53.90625,12.11474609375,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,2,128,8,128,60.78125,11.432373046875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,3,32,8,128,19.78125,18.8544921875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,3,64,8,128,23.0625,17.48974609375,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,3,128,8,128,26.03125,16.807373046875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,4,32,8,128,9.234375,24.2294921875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,4,64,8,128,10.765625,22.86474609375,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,4,128,8,128,12.1484375,22.182373046875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,8,32,8,128,0.55908203125,45.7294921875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,8,64,8,128,0.65087890625,44.36474609375,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.up_proj,8,128,8,128,0.73193359375,43.682373046875,45088768,3.05398357984898,1,3.0208012321963e-4 +17,mlp.down_proj,2,32,8,128,45.5625,13.4794921875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,2,64,8,128,53.03125,12.11474609375,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,2,128,8,128,59.75,11.432373046875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,3,32,8,128,19.46875,18.8544921875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,3,64,8,128,22.6875,17.48974609375,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,3,128,8,128,25.59375,16.807373046875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,4,32,8,128,9.0859375,24.2294921875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,4,64,8,128,10.5859375,22.86474609375,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,4,128,8,128,11.9375,22.182373046875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,8,32,8,128,0.55029296875,45.7294921875,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,8,64,8,128,0.640625,44.36474609375,45088768,3.467642156087784,1,2.9216750408522e-4 +17,mlp.down_proj,8,128,8,128,0.7197265625,43.682373046875,45088768,3.467642156087784,1,2.9216750408522e-4 +18,self_attn.q_proj,2,32,8,128,35.65625,5.015625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,2,64,8,128,41.875,4.5078125,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,2,128,8,128,49.09375,4.25390625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,3,32,8,128,15.25,7.015625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,3,64,8,128,17.9375,6.5078125,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,3,128,8,128,21.125,6.25390625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,4,32,8,128,7.12109375,9.015625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,4,64,8,128,8.375,8.5078125,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,4,128,8,128,9.8671875,8.25390625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,8,32,8,128,0.430908203125,17.015625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,8,64,8,128,0.50634765625,16.5078125,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.q_proj,8,128,8,128,0.59423828125,16.25390625,16777216,4.331741585407725,1,1.9963789964094e-4 +18,self_attn.v_proj,2,32,8,128,26.078125,5.015625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,2,64,8,128,30.46875,4.5078125,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,2,128,8,128,34.4375,4.25390625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,3,32,8,128,11.15625,7.015625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,3,64,8,128,13.0390625,6.5078125,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,3,128,8,128,14.7421875,6.25390625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,4,32,8,128,5.203125,9.015625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,4,64,8,128,6.08203125,8.5078125,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,4,128,8,128,6.8828125,8.25390625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,8,32,8,128,0.315185546875,17.015625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,8,64,8,128,0.367919921875,16.5078125,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.v_proj,8,128,8,128,0.414794921875,16.25390625,16777216,3.1396861815281687,1,1.9963789964094e-4 +18,self_attn.k_proj,2,32,8,128,36.59375,5.015625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,2,64,8,128,43.03125,4.5078125,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,2,128,8,128,50.8125,4.25390625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,3,32,8,128,15.65625,7.015625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,3,64,8,128,18.421875,6.5078125,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,3,128,8,128,21.890625,6.25390625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,4,32,8,128,7.3046875,9.015625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,4,64,8,128,8.59375,8.5078125,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,4,128,8,128,10.21875,8.25390625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,8,32,8,128,0.4423828125,17.015625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,8,64,8,128,0.52001953125,16.5078125,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.k_proj,8,128,8,128,0.61572265625,16.25390625,16777216,3.852686131364871,1,1.9963789964094e-4 +18,self_attn.o_proj,2,32,8,128,25.546875,5.015625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,2,64,8,128,29.78125,4.5078125,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,2,128,8,128,33.625,4.25390625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,3,32,8,128,10.9296875,7.015625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,3,64,8,128,12.7734375,6.5078125,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,3,128,8,128,14.4609375,6.25390625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,4,32,8,128,5.1015625,9.015625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,4,64,8,128,5.9609375,8.5078125,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,4,128,8,128,6.7578125,8.25390625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,8,32,8,128,0.308837890625,17.015625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,8,64,8,128,0.360595703125,16.5078125,16777216,4.205083278063674,1,1.1155115498695e-4 +18,self_attn.o_proj,8,128,8,128,0.406982421875,16.25390625,16777216,4.205083278063674,1,1.1155115498695e-4 +18,mlp.gate_proj,2,32,8,128,49.53125,13.4794921875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,2,64,8,128,58.28125,12.11474609375,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,2,128,8,128,66.5,11.432373046875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,3,32,8,128,21.1875,18.8544921875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,3,64,8,128,24.953125,17.48974609375,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,3,128,8,128,28.53125,16.807373046875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,4,32,8,128,9.8828125,24.2294921875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,4,64,8,128,11.6484375,22.86474609375,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,4,128,8,128,13.3203125,22.182373046875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,8,32,8,128,0.59814453125,45.7294921875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,8,64,8,128,0.7041015625,44.36474609375,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.gate_proj,8,128,8,128,0.80224609375,43.682373046875,45088768,3.363501779614211,1,3.5903716343455e-4 +18,mlp.up_proj,2,32,8,128,46.09375,13.4794921875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,2,64,8,128,53.71875,12.11474609375,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,2,128,8,128,60.53125,11.432373046875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,3,32,8,128,19.71875,18.8544921875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,3,64,8,128,22.984375,17.48974609375,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,3,128,8,128,25.921875,16.807373046875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,4,32,8,128,9.1953125,24.2294921875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,4,64,8,128,10.71875,22.86474609375,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,4,128,8,128,12.09375,22.182373046875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,8,32,8,128,0.556640625,45.7294921875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,8,64,8,128,0.6484375,44.36474609375,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.up_proj,8,128,8,128,0.728515625,43.682373046875,45088768,3.0475381520216764,1,3.5903716343455e-4 +18,mlp.down_proj,2,32,8,128,45.5,13.4794921875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,2,64,8,128,52.96875,12.11474609375,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,2,128,8,128,59.6875,11.432373046875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,3,32,8,128,19.453125,18.8544921875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,3,64,8,128,22.65625,17.48974609375,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,3,128,8,128,25.546875,16.807373046875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,4,32,8,128,9.078125,24.2294921875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,4,64,8,128,10.578125,22.86474609375,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,4,128,8,128,11.9296875,22.182373046875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,8,32,8,128,0.5498046875,45.7294921875,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,8,64,8,128,0.6396484375,44.36474609375,45088768,3.47087722889434,1,3.5338316229172e-4 +18,mlp.down_proj,8,128,8,128,0.71875,43.682373046875,45088768,3.47087722889434,1,3.5338316229172e-4 +19,self_attn.q_proj,2,32,8,128,35.1875,5.015625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,2,64,8,128,41.3125,4.5078125,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,2,128,8,128,48.75,4.25390625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,3,32,8,128,15.0625,7.015625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,3,64,8,128,17.6875,6.5078125,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,3,128,8,128,20.984375,6.25390625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,4,32,8,128,7.02734375,9.015625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,4,64,8,128,8.2578125,8.5078125,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,4,128,8,128,9.796875,8.25390625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,8,32,8,128,0.425537109375,17.015625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,8,64,8,128,0.499267578125,16.5078125,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.q_proj,8,128,8,128,0.59033203125,16.25390625,16777216,4.501974087132231,1,1.6247964231297e-4 +19,self_attn.v_proj,2,32,8,128,26.34375,5.015625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,2,64,8,128,30.796875,4.5078125,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,2,128,8,128,34.84375,4.25390625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,3,32,8,128,11.265625,7.015625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,3,64,8,128,13.171875,6.5078125,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,3,128,8,128,14.90625,6.25390625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,4,32,8,128,5.25390625,9.015625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,4,64,8,128,6.1484375,8.5078125,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,4,128,8,128,6.9609375,8.25390625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,8,32,8,128,0.318359375,17.015625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,8,64,8,128,0.371826171875,16.5078125,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.v_proj,8,128,8,128,0.41943359375,16.25390625,16777216,3.1479206835009523,1,1.6247964231297e-4 +19,self_attn.k_proj,2,32,8,128,36.03125,5.015625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,2,64,8,128,42.375,4.5078125,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,2,128,8,128,50.21875,4.25390625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,3,32,8,128,15.4140625,7.015625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,3,64,8,128,18.125,6.5078125,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,3,128,8,128,21.65625,6.25390625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,4,32,8,128,7.19140625,9.015625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,4,64,8,128,8.4609375,8.5078125,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,4,128,8,128,10.109375,8.25390625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,8,32,8,128,0.435546875,17.015625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,8,64,8,128,0.51171875,16.5078125,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.k_proj,8,128,8,128,0.609375,16.25390625,16777216,3.95344327211766,1,1.6247964231297e-4 +19,self_attn.o_proj,2,32,8,128,25.859375,5.015625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,2,64,8,128,30.140625,4.5078125,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,2,128,8,128,34.03125,4.25390625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,3,32,8,128,11.0703125,7.015625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,3,64,8,128,12.9296875,6.5078125,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,3,128,8,128,14.640625,6.25390625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,4,32,8,128,5.16796875,9.015625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,4,64,8,128,6.03515625,8.5078125,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,4,128,8,128,6.83984375,8.25390625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,8,32,8,128,0.31298828125,17.015625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,8,64,8,128,0.364990234375,16.5078125,16777216,4.21699893959325,1,9.622184006730096e-5 +19,self_attn.o_proj,8,128,8,128,0.412109375,16.25390625,16777216,4.21699893959325,1,9.622184006730096e-5 +19,mlp.gate_proj,2,32,8,128,49.5625,13.4794921875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,2,64,8,128,58.25,12.11474609375,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,2,128,8,128,66.375,11.432373046875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,3,32,8,128,21.203125,18.8544921875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,3,64,8,128,24.953125,17.48974609375,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,3,128,8,128,28.453125,16.807373046875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,4,32,8,128,9.8984375,24.2294921875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,4,64,8,128,11.640625,22.86474609375,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,4,128,8,128,13.2890625,22.182373046875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,8,32,8,128,0.59912109375,45.7294921875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,8,64,8,128,0.70361328125,44.36474609375,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.gate_proj,8,128,8,128,0.80029296875,43.682373046875,45088768,3.305941434985796,1,4.1256987606175e-4 +19,mlp.up_proj,2,32,8,128,46.09375,13.4794921875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,2,64,8,128,53.6875,12.11474609375,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,2,128,8,128,60.5,11.432373046875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,3,32,8,128,19.703125,18.8544921875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,3,64,8,128,22.96875,17.48974609375,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,3,128,8,128,25.90625,16.807373046875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,4,32,8,128,9.1953125,24.2294921875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,4,64,8,128,10.71875,22.86474609375,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,4,128,8,128,12.09375,22.182373046875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,8,32,8,128,0.556640625,45.7294921875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,8,64,8,128,0.6484375,44.36474609375,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.up_proj,8,128,8,128,0.728515625,43.682373046875,45088768,3.07692631709512,1,4.1256987606175e-4 +19,mlp.down_proj,2,32,8,128,45.59375,13.4794921875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,2,64,8,128,53.09375,12.11474609375,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,2,128,8,128,59.78125,11.432373046875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,3,32,8,128,19.5,18.8544921875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,3,64,8,128,22.703125,17.48974609375,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,3,128,8,128,25.59375,16.807373046875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,4,32,8,128,9.09375,24.2294921875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,4,64,8,128,10.6015625,22.86474609375,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,4,128,8,128,11.953125,22.182373046875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,8,32,8,128,0.55078125,45.7294921875,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,8,64,8,128,0.64111328125,44.36474609375,45088768,3.350168069059772,1,3.9766280679032e-4 +19,mlp.down_proj,8,128,8,128,0.72021484375,43.682373046875,45088768,3.350168069059772,1,3.9766280679032e-4 +20,self_attn.q_proj,2,32,8,128,35.625,5.015625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,2,64,8,128,41.90625,4.5078125,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,2,128,8,128,49.28125,4.25390625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,3,32,8,128,15.25,7.015625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,3,64,8,128,17.953125,6.5078125,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,3,128,8,128,21.25,6.25390625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,4,32,8,128,7.12109375,9.015625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,4,64,8,128,8.3828125,8.5078125,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,4,128,8,128,9.921875,8.25390625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,8,32,8,128,0.430908203125,17.015625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,8,64,8,128,0.50732421875,16.5078125,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.q_proj,8,128,8,128,0.59814453125,16.25390625,16777216,5.089448547787887,1,1.9958704069722e-4 +20,self_attn.v_proj,2,32,8,128,26.578125,5.015625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,2,64,8,128,31,4.5078125,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,2,128,8,128,34.96875,4.25390625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,3,32,8,128,11.3671875,7.015625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,3,64,8,128,13.265625,6.5078125,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,3,128,8,128,14.9609375,6.25390625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,4,32,8,128,5.30078125,9.015625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,4,64,8,128,6.18359375,8.5078125,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,4,128,8,128,6.984375,8.25390625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,8,32,8,128,0.3212890625,17.015625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,8,64,8,128,0.374267578125,16.5078125,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.v_proj,8,128,8,128,0.4208984375,16.25390625,16777216,3.077329271332318,1,1.9958704069722e-4 +20,self_attn.k_proj,2,32,8,128,36.40625,5.015625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,2,64,8,128,42.78125,4.5078125,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,2,128,8,128,50.46875,4.25390625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,3,32,8,128,15.5625,7.015625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,3,64,8,128,18.3125,6.5078125,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,3,128,8,128,21.8125,6.25390625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,4,32,8,128,7.26171875,9.015625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,4,64,8,128,8.546875,8.5078125,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,4,128,8,128,10.1796875,8.25390625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,8,32,8,128,0.439697265625,17.015625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,8,64,8,128,0.51708984375,16.5078125,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.k_proj,8,128,8,128,0.61328125,16.25390625,16777216,4.017856732818786,1,1.9958704069722e-4 +20,self_attn.o_proj,2,32,8,128,26.296875,5.015625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,2,64,8,128,30.65625,4.5078125,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,2,128,8,128,34.59375,4.25390625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,3,32,8,128,11.296875,7.015625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,3,64,8,128,13.203125,6.5078125,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,3,128,8,128,14.9609375,6.25390625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,4,32,8,128,5.2734375,9.015625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,4,64,8,128,6.171875,8.5078125,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,4,128,8,128,7.0078125,8.25390625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,8,32,8,128,0.3193359375,17.015625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,8,64,8,128,0.373046875,16.5078125,16777216,6.20978041938933,1,1.1364324745954e-4 +20,self_attn.o_proj,8,128,8,128,0.422119140625,16.25390625,16777216,6.20978041938933,1,1.1364324745954e-4 +20,mlp.gate_proj,2,32,8,128,49.65625,13.4794921875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,2,64,8,128,58.25,12.11474609375,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,2,128,8,128,66.25,11.432373046875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,3,32,8,128,21.234375,18.8544921875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,3,64,8,128,24.9375,17.48974609375,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,3,128,8,128,28.375,16.807373046875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,4,32,8,128,9.90625,24.2294921875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,4,64,8,128,11.6328125,22.86474609375,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,4,128,8,128,13.2421875,22.182373046875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,8,32,8,128,0.599609375,45.7294921875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,8,64,8,128,0.70361328125,44.36474609375,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.gate_proj,8,128,8,128,0.7978515625,43.682373046875,45088768,3.24058842202935,1,4.8459292156621e-4 +20,mlp.up_proj,2,32,8,128,46.03125,13.4794921875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,2,64,8,128,53.59375,12.11474609375,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,2,128,8,128,60.375,11.432373046875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,3,32,8,128,19.6875,18.8544921875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,3,64,8,128,22.9375,17.48974609375,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,3,128,8,128,25.84375,16.807373046875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,4,32,8,128,9.1796875,24.2294921875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,4,64,8,128,10.703125,22.86474609375,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,4,128,8,128,12.0625,22.182373046875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,8,32,8,128,0.5556640625,45.7294921875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,8,64,8,128,0.64697265625,44.36474609375,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.up_proj,8,128,8,128,0.72705078125,43.682373046875,45088768,3.037644738477705,1,4.8459292156621e-4 +20,mlp.down_proj,2,32,8,128,45.59375,13.4794921875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,2,64,8,128,53.09375,12.11474609375,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,2,128,8,128,59.8125,11.432373046875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,3,32,8,128,19.5,18.8544921875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,3,64,8,128,22.71875,17.48974609375,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,3,128,8,128,25.59375,16.807373046875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,4,32,8,128,9.1015625,24.2294921875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,4,64,8,128,10.6015625,22.86474609375,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,4,128,8,128,11.953125,22.182373046875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,8,32,8,128,0.55078125,45.7294921875,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,8,64,8,128,0.64111328125,44.36474609375,45088768,3.5542677087714925,1,4.8140791477635e-4 +20,mlp.down_proj,8,128,8,128,0.72021484375,43.682373046875,45088768,3.5542677087714925,1,4.8140791477635e-4 +21,self_attn.q_proj,2,32,8,128,34.84375,5.015625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,2,64,8,128,41,4.5078125,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,2,128,8,128,48.09375,4.25390625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,3,32,8,128,14.921875,7.015625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,3,64,8,128,17.5625,6.5078125,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,3,128,8,128,20.765625,6.25390625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,4,32,8,128,6.96484375,9.015625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,4,64,8,128,8.203125,8.5078125,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,4,128,8,128,9.703125,8.25390625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,8,32,8,128,0.421630859375,17.015625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,8,64,8,128,0.49609375,16.5078125,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.q_proj,8,128,8,128,0.58447265625,16.25390625,16777216,5.023500600377066,1,1.6971878358162e-4 +21,self_attn.v_proj,2,32,8,128,27.4375,5.015625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,2,64,8,128,32,4.5078125,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,2,128,8,128,36.125,4.25390625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,3,32,8,128,11.734375,7.015625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,3,64,8,128,13.6953125,6.5078125,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,3,128,8,128,15.46875,6.25390625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,4,32,8,128,5.47265625,9.015625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,4,64,8,128,6.390625,8.5078125,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,4,128,8,128,7.21484375,8.25390625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,8,32,8,128,0.33154296875,17.015625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,8,64,8,128,0.386474609375,16.5078125,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.v_proj,8,128,8,128,0.43505859375,16.25390625,16777216,3.0914145141503155,1,1.6971878358162e-4 +21,self_attn.k_proj,2,32,8,128,35.40625,5.015625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,2,64,8,128,41.65625,4.5078125,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,2,128,8,128,49,4.25390625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,3,32,8,128,15.1484375,7.015625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,3,64,8,128,17.84375,6.5078125,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,3,128,8,128,21.1875,6.25390625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,4,32,8,128,7.06640625,9.015625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,4,64,8,128,8.328125,8.5078125,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,4,128,8,128,9.890625,8.25390625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,8,32,8,128,0.42822265625,17.015625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,8,64,8,128,0.50390625,16.5078125,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.k_proj,8,128,8,128,0.59619140625,16.25390625,16777216,4.043552781047404,1,1.6971878358162e-4 +21,self_attn.o_proj,2,32,8,128,26.921875,5.015625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,2,64,8,128,31.375,4.5078125,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,2,128,8,128,35.40625,4.25390625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,3,32,8,128,11.53125,7.015625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,3,64,8,128,13.46875,6.5078125,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,3,128,8,128,15.25,6.25390625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,4,32,8,128,5.3828125,9.015625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,4,64,8,128,6.2890625,8.5078125,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,4,128,8,128,7.12890625,8.25390625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,8,32,8,128,0.325927734375,17.015625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,8,64,8,128,0.38037109375,16.5078125,16777216,4.176771420961363,1,1.0500274220248e-4 +21,self_attn.o_proj,8,128,8,128,0.429443359375,16.25390625,16777216,4.176771420961363,1,1.0500274220248e-4 +21,mlp.gate_proj,2,32,8,128,49.9375,13.4794921875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,2,64,8,128,58.5,12.11474609375,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,2,128,8,128,66.5,11.432373046875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,3,32,8,128,21.359375,18.8544921875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,3,64,8,128,25.046875,17.48974609375,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,3,128,8,128,28.484375,16.807373046875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,4,32,8,128,9.9609375,24.2294921875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,4,64,8,128,11.6875,22.86474609375,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,4,128,8,128,13.296875,22.182373046875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,8,32,8,128,0.603515625,45.7294921875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,8,64,8,128,0.70654296875,44.36474609375,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.gate_proj,8,128,8,128,0.80126953125,43.682373046875,45088768,3.2252613589641217,1,5.3334806580096e-4 +21,mlp.up_proj,2,32,8,128,45.9375,13.4794921875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,2,64,8,128,53.5,12.11474609375,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,2,128,8,128,60.25,11.432373046875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,3,32,8,128,19.640625,18.8544921875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,3,64,8,128,22.890625,17.48974609375,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,3,128,8,128,25.796875,16.807373046875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,4,32,8,128,9.1640625,24.2294921875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,4,64,8,128,10.6796875,22.86474609375,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,4,128,8,128,12.0390625,22.182373046875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,8,32,8,128,0.5546875,45.7294921875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,8,64,8,128,0.6455078125,44.36474609375,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.up_proj,8,128,8,128,0.72509765625,43.682373046875,45088768,3.0317359434824303,1,5.3334806580096e-4 +21,mlp.down_proj,2,32,8,128,45.59375,13.4794921875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,2,64,8,128,53.0625,12.11474609375,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,2,128,8,128,59.75,11.432373046875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,3,32,8,128,19.5,18.8544921875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,3,64,8,128,22.703125,17.48974609375,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,3,128,8,128,25.59375,16.807373046875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,4,32,8,128,9.09375,24.2294921875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,4,64,8,128,10.59375,22.86474609375,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,4,128,8,128,11.9453125,22.182373046875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,8,32,8,128,0.55078125,45.7294921875,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,8,64,8,128,0.640625,44.36474609375,45088768,3.2126358273768942,1,4.8393046017736e-4 +21,mlp.down_proj,8,128,8,128,0.71923828125,43.682373046875,45088768,3.2126358273768942,1,4.8393046017736e-4 +22,self_attn.q_proj,2,32,8,128,35.65625,5.015625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,2,64,8,128,41.9375,4.5078125,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,2,128,8,128,48.625,4.25390625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,3,32,8,128,15.265625,7.015625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,3,64,8,128,17.96875,6.5078125,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,3,128,8,128,20.921875,6.25390625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,4,32,8,128,7.12109375,9.015625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,4,64,8,128,8.390625,8.5078125,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,4,128,8,128,9.7734375,8.25390625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,8,32,8,128,0.43115234375,17.015625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,8,64,8,128,0.50732421875,16.5078125,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.q_proj,8,128,8,128,0.5888671875,16.25390625,16777216,4.650421015682584,1,2.7104874607175e-4 +22,self_attn.v_proj,2,32,8,128,27.578125,5.015625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,2,64,8,128,32.1875,4.5078125,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,2,128,8,128,36.34375,4.25390625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,3,32,8,128,11.7890625,7.015625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,3,64,8,128,13.7734375,6.5078125,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,3,128,8,128,15.5546875,6.25390625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,4,32,8,128,5.50390625,9.015625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,4,64,8,128,6.42578125,8.5078125,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,4,128,8,128,7.26171875,8.25390625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,8,32,8,128,0.333251953125,17.015625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,8,64,8,128,0.388671875,16.5078125,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.v_proj,8,128,8,128,0.4375,16.25390625,16777216,3.1028997163538112,1,2.7104874607175e-4 +22,self_attn.k_proj,2,32,8,128,36.25,5.015625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,2,64,8,128,42.59375,4.5078125,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,2,128,8,128,49.46875,4.25390625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,3,32,8,128,15.5078125,7.015625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,3,64,8,128,18.25,6.5078125,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,3,128,8,128,21.34375,6.25390625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,4,32,8,128,7.234375,9.015625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,4,64,8,128,8.515625,8.5078125,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,4,128,8,128,9.9609375,8.25390625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,8,32,8,128,0.437744140625,17.015625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,8,64,8,128,0.51513671875,16.5078125,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.k_proj,8,128,8,128,0.60009765625,16.25390625,16777216,3.856286168587325,1,2.7104874607175e-4 +22,self_attn.o_proj,2,32,8,128,26.890625,5.015625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,2,64,8,128,31.34375,4.5078125,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,2,128,8,128,35.375,4.25390625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,3,32,8,128,11.53125,7.015625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,3,64,8,128,13.46875,6.5078125,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,3,128,8,128,15.25,6.25390625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,4,32,8,128,5.40625,9.015625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,4,64,8,128,6.33203125,8.5078125,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,4,128,8,128,7.19921875,8.25390625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,8,32,8,128,0.327392578125,17.015625,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,8,64,8,128,0.383056640625,16.5078125,16777216,12.72973753381282,1,1.315164117841e-4 +22,self_attn.o_proj,8,128,8,128,0.434814453125,16.25390625,16777216,12.72973753381282,1,1.315164117841e-4 +22,mlp.gate_proj,2,32,8,128,50.125,13.4794921875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,2,64,8,128,58.71875,12.11474609375,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,2,128,8,128,66.625,11.432373046875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,3,32,8,128,21.4375,18.8544921875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,3,64,8,128,25.125,17.48974609375,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,3,128,8,128,28.546875,16.807373046875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,4,32,8,128,10.0078125,24.2294921875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,4,64,8,128,11.7265625,22.86474609375,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,4,128,8,128,13.3203125,22.182373046875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,8,32,8,128,0.60595703125,45.7294921875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,8,64,8,128,0.708984375,44.36474609375,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.gate_proj,8,128,8,128,0.80224609375,43.682373046875,45088768,3.1997543042303,1,6.0864095576107e-4 +22,mlp.up_proj,2,32,8,128,45.875,13.4794921875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,2,64,8,128,53.40625,12.11474609375,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,2,128,8,128,60.15625,11.432373046875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,3,32,8,128,19.625,18.8544921875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,3,64,8,128,22.859375,17.48974609375,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,3,128,8,128,25.765625,16.807373046875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,4,32,8,128,9.15625,24.2294921875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,4,64,8,128,10.6640625,22.86474609375,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,4,128,8,128,12.0234375,22.182373046875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,8,32,8,128,0.55419921875,45.7294921875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,8,64,8,128,0.64501953125,44.36474609375,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.up_proj,8,128,8,128,0.72412109375,43.682373046875,45088768,3.026070663498454,1,6.0864095576107e-4 +22,mlp.down_proj,2,32,8,128,45.59375,13.4794921875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,2,64,8,128,53.0625,12.11474609375,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,2,128,8,128,59.78125,11.432373046875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,3,32,8,128,19.5,18.8544921875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,3,64,8,128,22.703125,17.48974609375,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,3,128,8,128,25.578125,16.807373046875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,4,32,8,128,9.09375,24.2294921875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,4,64,8,128,10.59375,22.86474609375,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,4,128,8,128,11.9375,22.182373046875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,8,32,8,128,0.55078125,45.7294921875,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,8,64,8,128,0.640625,44.36474609375,45088768,3.045221148790187,1,5.3965946426615e-4 +22,mlp.down_proj,8,128,8,128,0.71923828125,43.682373046875,45088768,3.045221148790187,1,5.3965946426615e-4 +23,self_attn.q_proj,2,32,8,128,35.25,5.015625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,2,64,8,128,41.375,4.5078125,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,2,128,8,128,47.8125,4.25390625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,3,32,8,128,15.0859375,7.015625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,3,64,8,128,17.71875,6.5078125,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,3,128,8,128,20.546875,6.25390625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,4,32,8,128,7.0390625,9.015625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,4,64,8,128,8.2734375,8.5078125,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,4,128,8,128,9.6015625,8.25390625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,8,32,8,128,0.42626953125,17.015625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,8,64,8,128,0.5,16.5078125,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.q_proj,8,128,8,128,0.578125,16.25390625,16777216,4.622973048321524,1,2.2279600671026e-4 +23,self_attn.v_proj,2,32,8,128,28.875,5.015625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,2,64,8,128,33.6875,4.5078125,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,2,128,8,128,38.0625,4.25390625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,3,32,8,128,12.3515625,7.015625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,3,64,8,128,14.4140625,6.5078125,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,3,128,8,128,16.28125,6.25390625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,4,32,8,128,5.76171875,9.015625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,4,64,8,128,6.7265625,8.5078125,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,4,128,8,128,7.6015625,8.25390625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,8,32,8,128,0.348876953125,17.015625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,8,64,8,128,0.40673828125,16.5078125,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.v_proj,8,128,8,128,0.4580078125,16.25390625,16777216,3.1018698150322948,1,2.2279600671026e-4 +23,self_attn.k_proj,2,32,8,128,35.625,5.015625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,2,64,8,128,41.84375,4.5078125,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,2,128,8,128,48.4375,4.25390625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,3,32,8,128,15.25,7.015625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,3,64,8,128,17.921875,6.5078125,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,3,128,8,128,20.84375,6.25390625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,4,32,8,128,7.109375,9.015625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,4,64,8,128,8.359375,8.5078125,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,4,128,8,128,9.734375,8.25390625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,8,32,8,128,0.4306640625,17.015625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,8,64,8,128,0.505859375,16.5078125,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.k_proj,8,128,8,128,0.58642578125,16.25390625,16777216,3.751007505803854,1,2.2279600671026e-4 +23,self_attn.o_proj,2,32,8,128,28.34375,5.015625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,2,64,8,128,33.03125,4.5078125,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,2,128,8,128,37.25,4.25390625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,3,32,8,128,12.140625,7.015625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,3,64,8,128,14.171875,6.5078125,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,3,128,8,128,16.015625,6.25390625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,4,32,8,128,5.6640625,9.015625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,4,64,8,128,6.61328125,8.5078125,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,4,128,8,128,7.484375,8.25390625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,8,32,8,128,0.343017578125,17.015625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,8,64,8,128,0.39990234375,16.5078125,16777216,3.976658721156604,1,1.1295548756606e-4 +23,self_attn.o_proj,8,128,8,128,0.450927734375,16.25390625,16777216,3.976658721156604,1,1.1295548756606e-4 +23,mlp.gate_proj,2,32,8,128,50.03125,13.4794921875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,2,64,8,128,58.46875,12.11474609375,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,2,128,8,128,66.25,11.432373046875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,3,32,8,128,21.390625,18.8544921875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,3,64,8,128,25.015625,17.48974609375,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,3,128,8,128,28.359375,16.807373046875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,4,32,8,128,9.9765625,24.2294921875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,4,64,8,128,11.671875,22.86474609375,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,4,128,8,128,13.2421875,22.182373046875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,8,32,8,128,0.60400390625,45.7294921875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,8,64,8,128,0.7060546875,44.36474609375,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.gate_proj,8,128,8,128,0.79736328125,43.682373046875,45088768,3.152225075876393,1,6.790806655772e-4 +23,mlp.up_proj,2,32,8,128,46.09375,13.4794921875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,2,64,8,128,53.65625,12.11474609375,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,2,128,8,128,60.4375,11.432373046875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,3,32,8,128,19.71875,18.8544921875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,3,64,8,128,22.953125,17.48974609375,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,3,128,8,128,25.875,16.807373046875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,4,32,8,128,9.203125,24.2294921875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,4,64,8,128,10.7109375,22.86474609375,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,4,128,8,128,12.078125,22.182373046875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,8,32,8,128,0.556640625,45.7294921875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,8,64,8,128,0.64794921875,44.36474609375,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.up_proj,8,128,8,128,0.7275390625,43.682373046875,45088768,3.029466900689184,1,6.790806655772e-4 +23,mlp.down_proj,2,32,8,128,45.875,13.4794921875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,2,64,8,128,53.375,12.11474609375,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,2,128,8,128,60.09375,11.432373046875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,3,32,8,128,19.609375,18.8544921875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,3,64,8,128,22.84375,17.48974609375,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,3,128,8,128,25.71875,16.807373046875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,4,32,8,128,9.1484375,24.2294921875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,4,64,8,128,10.65625,22.86474609375,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,4,128,8,128,12.0078125,22.182373046875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,8,32,8,128,0.55419921875,45.7294921875,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,8,64,8,128,0.64453125,44.36474609375,45088768,3.661347930223575,1,5.731675773859e-4 +23,mlp.down_proj,8,128,8,128,0.7236328125,43.682373046875,45088768,3.661347930223575,1,5.731675773859e-4 +24,self_attn.q_proj,2,32,8,128,34.625,5.015625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,2,64,8,128,40.78125,4.5078125,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,2,128,8,128,47.65625,4.25390625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,3,32,8,128,14.828125,7.015625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,3,64,8,128,17.5,6.5078125,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,3,128,8,128,20.609375,6.25390625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,4,32,8,128,6.91796875,9.015625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,4,64,8,128,8.171875,8.5078125,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,4,128,8,128,9.625,8.25390625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,8,32,8,128,0.418701171875,17.015625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,8,64,8,128,0.494140625,16.5078125,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.q_proj,8,128,8,128,0.580078125,16.25390625,16777216,4.810796636988315,1,2.7459222474135e-4 +24,self_attn.v_proj,2,32,8,128,28.578125,5.015625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,2,64,8,128,33.34375,4.5078125,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,2,128,8,128,37.65625,4.25390625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,3,32,8,128,12.21875,7.015625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,3,64,8,128,14.265625,6.5078125,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,3,128,8,128,16.09375,6.25390625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,4,32,8,128,5.69921875,9.015625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,4,64,8,128,6.65625,8.5078125,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,4,128,8,128,7.515625,8.25390625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,8,32,8,128,0.34521484375,17.015625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,8,64,8,128,0.40234375,16.5078125,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.v_proj,8,128,8,128,0.452880859375,16.25390625,16777216,3.08752802550685,1,2.7459222474135e-4 +24,self_attn.k_proj,2,32,8,128,34.96875,5.015625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,2,64,8,128,41.1875,4.5078125,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,2,128,8,128,48.15625,4.25390625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,3,32,8,128,14.9609375,7.015625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,3,64,8,128,17.671875,6.5078125,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,3,128,8,128,20.890625,6.25390625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,4,32,8,128,6.984375,9.015625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,4,64,8,128,8.25,8.5078125,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,4,128,8,128,9.75,8.25390625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,8,32,8,128,0.422607421875,17.015625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,8,64,8,128,0.498779296875,16.5078125,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.k_proj,8,128,8,128,0.58740234375,16.25390625,16777216,4.161086490091737,1,2.7459222474135e-4 +24,self_attn.o_proj,2,32,8,128,27.90625,5.015625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,2,64,8,128,32.53125,4.5078125,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,2,128,8,128,36.6875,4.25390625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,3,32,8,128,11.984375,7.015625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,3,64,8,128,14,6.5078125,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,3,128,8,128,15.8671875,6.25390625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,4,32,8,128,5.59375,9.015625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,4,64,8,128,6.54296875,8.5078125,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,4,128,8,128,7.4296875,8.25390625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,8,32,8,128,0.3388671875,17.015625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,8,64,8,128,0.395751953125,16.5078125,16777216,6.397808422117449,1,1.4576167450286e-4 +24,self_attn.o_proj,8,128,8,128,0.44775390625,16.25390625,16777216,6.397808422117449,1,1.4576167450286e-4 +24,mlp.gate_proj,2,32,8,128,50.15625,13.4794921875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,2,64,8,128,58.6875,12.11474609375,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,2,128,8,128,66.5,11.432373046875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,3,32,8,128,21.453125,18.8544921875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,3,64,8,128,25.109375,17.48974609375,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,3,128,8,128,28.484375,16.807373046875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,4,32,8,128,10.0078125,24.2294921875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,4,64,8,128,11.71875,22.86474609375,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,4,128,8,128,13.296875,22.182373046875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,8,32,8,128,0.60595703125,45.7294921875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,8,64,8,128,0.70849609375,44.36474609375,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.gate_proj,8,128,8,128,0.80078125,43.682373046875,45088768,3.159957907684576,1,7.426660740748e-4 +24,mlp.up_proj,2,32,8,128,46.25,13.4794921875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,2,64,8,128,53.8125,12.11474609375,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,2,128,8,128,60.59375,11.432373046875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,3,32,8,128,19.765625,18.8544921875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,3,64,8,128,23.03125,17.48974609375,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,3,128,8,128,25.9375,16.807373046875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,4,32,8,128,9.2265625,24.2294921875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,4,64,8,128,10.7421875,22.86474609375,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,4,128,8,128,12.109375,22.182373046875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,8,32,8,128,0.55859375,45.7294921875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,8,64,8,128,0.64990234375,44.36474609375,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.up_proj,8,128,8,128,0.7294921875,43.682373046875,45088768,3.020966721362208,1,7.426660740748e-4 +24,mlp.down_proj,2,32,8,128,46.03125,13.4794921875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,2,64,8,128,53.5625,12.11474609375,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,2,128,8,128,60.3125,11.432373046875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,3,32,8,128,19.6875,18.8544921875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,3,64,8,128,22.921875,17.48974609375,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,3,128,8,128,25.8125,16.807373046875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,4,32,8,128,9.1875,24.2294921875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,4,64,8,128,10.6953125,22.86474609375,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,4,128,8,128,12.046875,22.182373046875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,8,32,8,128,0.5556640625,45.7294921875,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,8,64,8,128,0.646484375,44.36474609375,45088768,3.1393332037439903,1,5.9406989021226e-4 +24,mlp.down_proj,8,128,8,128,0.7255859375,43.682373046875,45088768,3.1393332037439903,1,5.9406989021226e-4 +25,self_attn.q_proj,2,32,8,128,34.53125,5.015625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,2,64,8,128,40.53125,4.5078125,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,2,128,8,128,47.28125,4.25390625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,3,32,8,128,14.7734375,7.015625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,3,64,8,128,17.359375,6.5078125,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,3,128,8,128,20.328125,6.25390625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,4,32,8,128,6.89453125,9.015625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,4,64,8,128,8.1015625,8.5078125,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,4,128,8,128,9.484375,8.25390625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,8,32,8,128,0.417236328125,17.015625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,8,64,8,128,0.489990234375,16.5078125,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.q_proj,8,128,8,128,0.57177734375,16.25390625,16777216,4.259720660607395,1,2.3969505855347e-4 +25,self_attn.v_proj,2,32,8,128,29.640625,5.015625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,2,64,8,128,34.59375,4.5078125,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,2,128,8,128,39.03125,4.25390625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,3,32,8,128,12.671875,7.015625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,3,64,8,128,14.7890625,6.5078125,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,3,128,8,128,16.703125,6.25390625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,4,32,8,128,5.9140625,9.015625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,4,64,8,128,6.90234375,8.5078125,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,4,128,8,128,7.79296875,8.25390625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,8,32,8,128,0.358154296875,17.015625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,8,64,8,128,0.41748046875,16.5078125,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.v_proj,8,128,8,128,0.4697265625,16.25390625,16777216,3.0825314450000945,1,2.3969505855347e-4 +25,self_attn.k_proj,2,32,8,128,34.6875,5.015625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,2,64,8,128,40.75,4.5078125,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,2,128,8,128,47.5625,4.25390625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,3,32,8,128,14.8515625,7.015625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,3,64,8,128,17.453125,6.5078125,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,3,128,8,128,20.46875,6.25390625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,4,32,8,128,6.9296875,9.015625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,4,64,8,128,8.140625,8.5078125,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,4,128,8,128,9.5546875,8.25390625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,8,32,8,128,0.41943359375,17.015625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,8,64,8,128,0.492431640625,16.5078125,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.k_proj,8,128,8,128,0.57568359375,16.25390625,16777216,3.828115065648799,1,2.3969505855347e-4 +25,self_attn.o_proj,2,32,8,128,29.109375,5.015625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,2,64,8,128,33.875,4.5078125,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,2,128,8,128,38.21875,4.25390625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,3,32,8,128,12.4453125,7.015625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,3,64,8,128,14.5078125,6.5078125,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,3,128,8,128,16.375,6.25390625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,4,32,8,128,5.8046875,9.015625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,4,64,8,128,6.76953125,8.5078125,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,4,128,8,128,7.64453125,8.25390625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,8,32,8,128,0.3515625,17.015625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,8,64,8,128,0.409423828125,16.5078125,16777216,3.41399610775646,1,1.1872268805745e-4 +25,self_attn.o_proj,8,128,8,128,0.460693359375,16.25390625,16777216,3.41399610775646,1,1.1872268805745e-4 +25,mlp.gate_proj,2,32,8,128,50.34375,13.4794921875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,2,64,8,128,58.9375,12.11474609375,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,2,128,8,128,66.875,11.432373046875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,3,32,8,128,21.53125,18.8544921875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,3,64,8,128,25.234375,17.48974609375,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,3,128,8,128,28.640625,16.807373046875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,4,32,8,128,10.0546875,24.2294921875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,4,64,8,128,11.78125,22.86474609375,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,4,128,8,128,13.3671875,22.182373046875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,8,32,8,128,0.6083984375,45.7294921875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,8,64,8,128,0.7119140625,44.36474609375,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.gate_proj,8,128,8,128,0.80517578125,43.682373046875,45088768,3.2140774706633204,1,8.3609303692355e-4 +25,mlp.up_proj,2,32,8,128,46.4375,13.4794921875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,2,64,8,128,54.0625,12.11474609375,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,2,128,8,128,60.875,11.432373046875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,3,32,8,128,19.859375,18.8544921875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,3,64,8,128,23.125,17.48974609375,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,3,128,8,128,26.0625,16.807373046875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,4,32,8,128,9.265625,24.2294921875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,4,64,8,128,10.7890625,22.86474609375,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,4,128,8,128,12.1640625,22.182373046875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,8,32,8,128,0.56103515625,45.7294921875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,8,64,8,128,0.65234375,44.36474609375,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.up_proj,8,128,8,128,0.73291015625,43.682373046875,45088768,3.02615172988415,1,8.3609303692355e-4 +25,mlp.down_proj,2,32,8,128,46.25,13.4794921875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,2,64,8,128,53.84375,12.11474609375,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,2,128,8,128,60.59375,11.432373046875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,3,32,8,128,19.78125,18.8544921875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,3,64,8,128,23.03125,17.48974609375,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,3,128,8,128,25.9375,16.807373046875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,4,32,8,128,9.2265625,24.2294921875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,4,64,8,128,10.75,22.86474609375,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,4,128,8,128,12.1015625,22.182373046875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,8,32,8,128,0.55859375,45.7294921875,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,8,64,8,128,0.64990234375,44.36474609375,45088768,3.257651311132728,1,0.00064206763636320005 +25,mlp.down_proj,8,128,8,128,0.7294921875,43.682373046875,45088768,3.257651311132728,1,0.00064206763636320005 +26,self_attn.q_proj,2,32,8,128,34.0625,5.015625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,2,64,8,128,40.03125,4.5078125,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,2,128,8,128,46.78125,4.25390625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,3,32,8,128,14.5703125,7.015625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,3,64,8,128,17.140625,6.5078125,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,3,128,8,128,20.15625,6.25390625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,4,32,8,128,6.80078125,9.015625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,4,64,8,128,8,8.5078125,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,4,128,8,128,9.40625,8.25390625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,8,32,8,128,0.41162109375,17.015625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,8,64,8,128,0.484130859375,16.5078125,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.q_proj,8,128,8,128,0.56689453125,16.25390625,16777216,4.746416094050959,1,3.9320776704698e-4 +26,self_attn.v_proj,2,32,8,128,30.359375,5.015625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,2,64,8,128,35.53125,4.5078125,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,2,128,8,128,40.25,4.25390625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,3,32,8,128,12.9765625,7.015625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,3,64,8,128,15.1953125,6.5078125,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,3,128,8,128,17.21875,6.25390625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,4,32,8,128,6.0546875,9.015625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,4,64,8,128,7.09375,8.5078125,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,4,128,8,128,8.0390625,8.25390625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,8,32,8,128,0.36669921875,17.015625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,8,64,8,128,0.4287109375,16.5078125,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.v_proj,8,128,8,128,0.484375,16.25390625,16777216,3.174601908030728,1,3.9320776704698e-4 +26,self_attn.k_proj,2,32,8,128,34.4375,5.015625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,2,64,8,128,40.46875,4.5078125,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,2,128,8,128,47.34375,4.25390625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,3,32,8,128,14.7265625,7.015625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,3,64,8,128,17.328125,6.5078125,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,3,128,8,128,20.421875,6.25390625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,4,32,8,128,6.875,9.015625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,4,64,8,128,8.0859375,8.5078125,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,4,128,8,128,9.53125,8.25390625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,8,32,8,128,0.416015625,17.015625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,8,64,8,128,0.489013671875,16.5078125,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.k_proj,8,128,8,128,0.57470703125,16.25390625,16777216,3.950877539540366,1,3.9320776704698e-4 +26,self_attn.o_proj,2,32,8,128,29.75,5.015625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,2,64,8,128,34.65625,4.5078125,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,2,128,8,128,39.125,4.25390625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,3,32,8,128,12.765625,7.015625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,3,64,8,128,14.90625,6.5078125,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,3,128,8,128,16.875,6.25390625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,4,32,8,128,5.9609375,9.015625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,4,64,8,128,6.96484375,8.5078125,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,4,128,8,128,7.90234375,8.25390625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,8,32,8,128,0.36083984375,17.015625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,8,64,8,128,0.42138671875,16.5078125,16777216,5.404761630435047,1,2.0280107855796e-4 +26,self_attn.o_proj,8,128,8,128,0.476318359375,16.25390625,16777216,5.404761630435047,1,2.0280107855796e-4 +26,mlp.gate_proj,2,32,8,128,50.78125,13.4794921875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,2,64,8,128,59.59375,12.11474609375,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,2,128,8,128,67.875,11.432373046875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,3,32,8,128,21.71875,18.8544921875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,3,64,8,128,25.515625,17.48974609375,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,3,128,8,128,29.0625,16.807373046875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,4,32,8,128,10.1328125,24.2294921875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,4,64,8,128,11.90625,22.86474609375,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,4,128,8,128,13.5625,22.182373046875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,8,32,8,128,0.61328125,45.7294921875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,8,64,8,128,0.7197265625,44.36474609375,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.gate_proj,8,128,8,128,0.81689453125,43.682373046875,45088768,3.258833850594769,1,9.3857629690319e-4 +26,mlp.up_proj,2,32,8,128,46.625,13.4794921875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,2,64,8,128,54.28125,12.11474609375,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,2,128,8,128,61.125,11.432373046875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,3,32,8,128,19.9375,18.8544921875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,3,64,8,128,23.21875,17.48974609375,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,3,128,8,128,26.171875,16.807373046875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,4,32,8,128,9.3046875,24.2294921875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,4,64,8,128,10.8359375,22.86474609375,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,4,128,8,128,12.2109375,22.182373046875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,8,32,8,128,0.56298828125,45.7294921875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,8,64,8,128,0.6552734375,44.36474609375,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.up_proj,8,128,8,128,0.73583984375,43.682373046875,45088768,3.0306968402468804,1,9.3857629690319e-4 +26,mlp.down_proj,2,32,8,128,46.40625,13.4794921875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,2,64,8,128,54,12.11474609375,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,2,128,8,128,60.8125,11.432373046875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,3,32,8,128,19.84375,18.8544921875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,3,64,8,128,23.09375,17.48974609375,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,3,128,8,128,26.03125,16.807373046875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,4,32,8,128,9.2578125,24.2294921875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,4,64,8,128,10.78125,22.86474609375,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,4,128,8,128,12.1484375,22.182373046875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,8,32,8,128,0.560546875,45.7294921875,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,8,64,8,128,0.65185546875,44.36474609375,45088768,3.0934423947000096,1,6.9413584424182e-4 +26,mlp.down_proj,8,128,8,128,0.73193359375,43.682373046875,45088768,3.0934423947000096,1,6.9413584424182e-4 +27,self_attn.q_proj,2,32,8,128,34.9375,5.015625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,2,64,8,128,40.9375,4.5078125,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,2,128,8,128,47.3125,4.25390625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,3,32,8,128,14.953125,7.015625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,3,64,8,128,17.515625,6.5078125,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,3,128,8,128,20.296875,6.25390625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,4,32,8,128,6.98046875,9.015625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,4,64,8,128,8.1796875,8.5078125,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,4,128,8,128,9.484375,8.25390625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,8,32,8,128,0.422607421875,17.015625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,8,64,8,128,0.494873046875,16.5078125,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.q_proj,8,128,8,128,0.5712890625,16.25390625,16777216,4.456513964473084,1,3.1607458367943e-4 +27,self_attn.v_proj,2,32,8,128,30.203125,5.015625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,2,64,8,128,35.25,4.5078125,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,2,128,8,128,39.8125,4.25390625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,3,32,8,128,12.921875,7.015625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,3,64,8,128,15.078125,6.5078125,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,3,128,8,128,17.03125,6.25390625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,4,32,8,128,6.0234375,9.015625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,4,64,8,128,7.03515625,8.5078125,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,4,128,8,128,7.94921875,8.25390625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,8,32,8,128,0.364990234375,17.015625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,8,64,8,128,0.425537109375,16.5078125,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.v_proj,8,128,8,128,0.47900390625,16.25390625,16777216,3.0942072739706443,1,3.1607458367943e-4 +27,self_attn.k_proj,2,32,8,128,35.34375,5.015625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,2,64,8,128,41.40625,4.5078125,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,2,128,8,128,47.9375,4.25390625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,3,32,8,128,15.109375,7.015625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,3,64,8,128,17.71875,6.5078125,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,3,128,8,128,20.546875,6.25390625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,4,32,8,128,7.05078125,9.015625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,4,64,8,128,8.265625,8.5078125,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,4,128,8,128,9.59375,8.25390625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,8,32,8,128,0.4267578125,17.015625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,8,64,8,128,0.5,16.5078125,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.k_proj,8,128,8,128,0.578125,16.25390625,16777216,3.5343563483836373,1,3.1607458367943e-4 +27,self_attn.o_proj,2,32,8,128,29.96875,5.015625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,2,64,8,128,34.90625,4.5078125,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,2,128,8,128,39.375,4.25390625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,3,32,8,128,12.828125,7.015625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,3,64,8,128,14.96875,6.5078125,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,3,128,8,128,16.921875,6.25390625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,4,32,8,128,5.98828125,9.015625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,4,64,8,128,6.98828125,8.5078125,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,4,128,8,128,7.90234375,8.25390625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,8,32,8,128,0.362548828125,17.015625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,8,64,8,128,0.422607421875,16.5078125,16777216,3.855701436089548,1,1.8604610522743e-4 +27,self_attn.o_proj,8,128,8,128,0.476318359375,16.25390625,16777216,3.855701436089548,1,1.8604610522743e-4 +27,mlp.gate_proj,2,32,8,128,51.0625,13.4794921875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,2,64,8,128,60.09375,12.11474609375,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,2,128,8,128,68.5625,11.432373046875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,3,32,8,128,21.84375,18.8544921875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,3,64,8,128,25.734375,17.48974609375,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,3,128,8,128,29.40625,16.807373046875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,4,32,8,128,10.1875,24.2294921875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,4,64,8,128,12.0078125,22.86474609375,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,4,128,8,128,13.7265625,22.182373046875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,8,32,8,128,0.61669921875,45.7294921875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,8,64,8,128,0.72607421875,44.36474609375,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.gate_proj,8,128,8,128,0.8271484375,43.682373046875,45088768,3.357166204682404,1,0.00107777502853423 +27,mlp.up_proj,2,32,8,128,46.90625,13.4794921875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,2,64,8,128,54.59375,12.11474609375,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,2,128,8,128,61.5,11.432373046875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,3,32,8,128,20.046875,18.8544921875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,3,64,8,128,23.359375,17.48974609375,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,3,128,8,128,26.328125,16.807373046875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,4,32,8,128,9.3515625,24.2294921875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,4,64,8,128,10.8984375,22.86474609375,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,4,128,8,128,12.2890625,22.182373046875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,8,32,8,128,0.56640625,45.7294921875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,8,64,8,128,0.65869140625,44.36474609375,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.up_proj,8,128,8,128,0.740234375,43.682373046875,45088768,3.032937516667663,1,0.00107777502853423 +27,mlp.down_proj,2,32,8,128,46.65625,13.4794921875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,2,64,8,128,54.3125,12.11474609375,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,2,128,8,128,61.125,11.432373046875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,3,32,8,128,19.953125,18.8544921875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,3,64,8,128,23.21875,17.48974609375,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,3,128,8,128,26.171875,16.807373046875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,4,32,8,128,9.3046875,24.2294921875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,4,64,8,128,10.84375,22.86474609375,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,4,128,8,128,12.21875,22.182373046875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,8,32,8,128,0.5634765625,45.7294921875,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,8,64,8,128,0.6552734375,44.36474609375,45088768,3.149947949147703,1,8.0323434667661e-4 +27,mlp.down_proj,8,128,8,128,0.73583984375,43.682373046875,45088768,3.149947949147703,1,8.0323434667661e-4 +28,self_attn.q_proj,2,32,8,128,34.1875,5.015625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,2,64,8,128,40.09375,4.5078125,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,2,128,8,128,46.46875,4.25390625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,3,32,8,128,14.6171875,7.015625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,3,64,8,128,17.15625,6.5078125,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,3,128,8,128,19.953125,6.25390625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,4,32,8,128,6.82421875,9.015625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,4,64,8,128,8.015625,8.5078125,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,4,128,8,128,9.3125,8.25390625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,8,32,8,128,0.4130859375,17.015625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,8,64,8,128,0.484619140625,16.5078125,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.q_proj,8,128,8,128,0.5615234375,16.25390625,16777216,4.9206986159336,1,3.718503867276e-4 +28,self_attn.v_proj,2,32,8,128,31.34375,5.015625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,2,64,8,128,36.65625,4.5078125,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,2,128,8,128,41.5,4.25390625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,3,32,8,128,13.40625,7.015625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,3,64,8,128,15.671875,6.5078125,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,3,128,8,128,17.75,6.25390625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,4,32,8,128,6.25390625,9.015625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,4,64,8,128,7.3125,8.5078125,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,4,128,8,128,8.28125,8.25390625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,8,32,8,128,0.37841796875,17.015625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,8,64,8,128,0.4423828125,16.5078125,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.v_proj,8,128,8,128,0.499267578125,16.25390625,16777216,3.15623756263924,1,3.718503867276e-4 +28,self_attn.k_proj,2,32,8,128,34.625,5.015625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,2,64,8,128,40.625,4.5078125,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,2,128,8,128,47.21875,4.25390625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,3,32,8,128,14.8046875,7.015625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,3,64,8,128,17.390625,6.5078125,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,3,128,8,128,20.28125,6.25390625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,4,32,8,128,6.90625,9.015625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,4,64,8,128,8.109375,8.5078125,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,4,128,8,128,9.46875,8.25390625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,8,32,8,128,0.418212890625,17.015625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,8,64,8,128,0.49072265625,16.5078125,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.k_proj,8,128,8,128,0.5703125,16.25390625,16777216,3.7013362450844207,1,3.718503867276e-4 +28,self_attn.o_proj,2,32,8,128,30.875,5.015625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,2,64,8,128,36,4.5078125,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,2,128,8,128,40.65625,4.25390625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,3,32,8,128,13.2109375,7.015625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,3,64,8,128,15.4296875,6.5078125,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,3,128,8,128,17.453125,6.25390625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,4,32,8,128,6.16796875,9.015625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,4,64,8,128,7.19921875,8.5078125,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,4,128,8,128,8.1484375,8.25390625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,8,32,8,128,0.373291015625,17.015625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,8,64,8,128,0.435546875,16.5078125,16777216,3.815716480743954,1,2.2103457013145e-4 +28,self_attn.o_proj,8,128,8,128,0.4912109375,16.25390625,16777216,3.815716480743954,1,2.2103457013145e-4 +28,mlp.gate_proj,2,32,8,128,51.03125,13.4794921875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,2,64,8,128,60.1875,12.11474609375,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,2,128,8,128,68.9375,11.432373046875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,3,32,8,128,21.828125,18.8544921875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,3,64,8,128,25.796875,17.48974609375,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,3,128,8,128,29.5625,16.807373046875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,4,32,8,128,10.1875,24.2294921875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,4,64,8,128,12.03125,22.86474609375,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,4,128,8,128,13.796875,22.182373046875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,8,32,8,128,0.61669921875,45.7294921875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,8,64,8,128,0.7275390625,44.36474609375,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.gate_proj,8,128,8,128,0.83154296875,43.682373046875,45088768,3.412267812021293,1,0.0012836423702538 +28,mlp.up_proj,2,32,8,128,47.375,13.4794921875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,2,64,8,128,55.1875,12.11474609375,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,2,128,8,128,62.1875,11.432373046875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,3,32,8,128,20.25,18.8544921875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,3,64,8,128,23.609375,17.48974609375,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,3,128,8,128,26.625,16.807373046875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,4,32,8,128,9.453125,24.2294921875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,4,64,8,128,11.015625,22.86474609375,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,4,128,8,128,12.421875,22.182373046875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,8,32,8,128,0.572265625,45.7294921875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,8,64,8,128,0.666015625,44.36474609375,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.up_proj,8,128,8,128,0.74853515625,43.682373046875,45088768,3.045774527398766,1,0.0012836423702538 +28,mlp.down_proj,2,32,8,128,46.96875,13.4794921875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,2,64,8,128,54.65625,12.11474609375,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,2,128,8,128,61.5625,11.432373046875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,3,32,8,128,20.078125,18.8544921875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,3,64,8,128,23.390625,17.48974609375,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,3,128,8,128,26.359375,16.807373046875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,4,32,8,128,9.3671875,24.2294921875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,4,64,8,128,10.9140625,22.86474609375,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,4,128,8,128,12.3046875,22.182373046875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,8,32,8,128,0.5673828125,45.7294921875,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,8,64,8,128,0.66015625,44.36474609375,45088768,3.3256110936488867,1,0.0010198998497799 +28,mlp.down_proj,8,128,8,128,0.74169921875,43.682373046875,45088768,3.3256110936488867,1,0.0010198998497799 +29,self_attn.q_proj,2,32,8,128,33.34375,5.015625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,2,64,8,128,39.125,4.5078125,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,2,128,8,128,45.96875,4.25390625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,3,32,8,128,14.265625,7.015625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,3,64,8,128,16.75,6.5078125,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,3,128,8,128,19.765625,6.25390625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,4,32,8,128,6.65625,9.015625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,4,64,8,128,7.8203125,8.5078125,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,4,128,8,128,9.234375,8.25390625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,8,32,8,128,0.4033203125,17.015625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,8,64,8,128,0.47314453125,16.5078125,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.q_proj,8,128,8,128,0.556640625,16.25390625,16777216,5.75206972772167,1,3.9460937841795e-4 +29,self_attn.v_proj,2,32,8,128,31.328125,5.015625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,2,64,8,128,36.59375,4.5078125,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,2,128,8,128,41.375,4.25390625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,3,32,8,128,13.3984375,7.015625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,3,64,8,128,15.6484375,6.5078125,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,3,128,8,128,17.6875,6.25390625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,4,32,8,128,6.25,9.015625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,4,64,8,128,7.3046875,8.5078125,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,4,128,8,128,8.2578125,8.25390625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,8,32,8,128,0.37841796875,17.015625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,8,64,8,128,0.441650390625,16.5078125,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.v_proj,8,128,8,128,0.497802734375,16.25390625,16777216,3.113633082966048,1,3.9460937841795e-4 +29,self_attn.k_proj,2,32,8,128,33.75,5.015625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,2,64,8,128,39.625,4.5078125,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,2,128,8,128,46.59375,4.25390625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,3,32,8,128,14.4296875,7.015625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,3,64,8,128,16.953125,6.5078125,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,3,128,8,128,20.0625,6.25390625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,4,32,8,128,6.73046875,9.015625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,4,64,8,128,7.9140625,8.5078125,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,4,128,8,128,9.3671875,8.25390625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,8,32,8,128,0.40771484375,17.015625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,8,64,8,128,0.478515625,16.5078125,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.k_proj,8,128,8,128,0.564453125,16.25390625,16777216,3.9551189681736383,1,3.9460937841795e-4 +29,self_attn.o_proj,2,32,8,128,31.234375,5.015625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,2,64,8,128,36.40625,4.5078125,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,2,128,8,128,41.125,4.25390625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,3,32,8,128,13.375,7.015625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,3,64,8,128,15.625,6.5078125,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,3,128,8,128,17.703125,6.25390625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,4,32,8,128,6.2421875,9.015625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,4,64,8,128,7.296875,8.5078125,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,4,128,8,128,8.2734375,8.25390625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,8,32,8,128,0.3779296875,17.015625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,8,64,8,128,0.441162109375,16.5078125,16777216,4.135413539616198,1,2.2196362260729e-4 +29,self_attn.o_proj,8,128,8,128,0.498291015625,16.25390625,16777216,4.135413539616198,1,2.2196362260729e-4 +29,mlp.gate_proj,2,32,8,128,51.4375,13.4794921875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,2,64,8,128,60.875,12.11474609375,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,2,128,8,128,69.875,11.432373046875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,3,32,8,128,22.015625,18.8544921875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,3,64,8,128,26.09375,17.48974609375,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,3,128,8,128,30.03125,16.807373046875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,4,32,8,128,10.2734375,24.2294921875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,4,64,8,128,12.1796875,22.86474609375,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,4,128,8,128,14.0234375,22.182373046875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,8,32,8,128,0.62158203125,45.7294921875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,8,64,8,128,0.73583984375,44.36474609375,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.gate_proj,8,128,8,128,0.8447265625,43.682373046875,45088768,3.605206129991087,1,0.00156129512470215 +29,mlp.up_proj,2,32,8,128,47.90625,13.4794921875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,2,64,8,128,55.875,12.11474609375,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,2,128,8,128,63.09375,11.432373046875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,3,32,8,128,20.5,18.8544921875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,3,64,8,128,23.9375,17.48974609375,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,3,128,8,128,27.09375,16.807373046875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,4,32,8,128,9.5703125,24.2294921875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,4,64,8,128,11.1796875,22.86474609375,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,4,128,8,128,12.640625,22.182373046875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,8,32,8,128,0.5791015625,45.7294921875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,8,64,8,128,0.67578125,44.36474609375,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.up_proj,8,128,8,128,0.76171875,43.682373046875,45088768,3.3965450154438708,1,0.00156129512470215 +29,mlp.down_proj,2,32,8,128,47.125,13.4794921875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,2,64,8,128,54.90625,12.11474609375,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,2,128,8,128,61.90625,11.432373046875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,3,32,8,128,20.15625,18.8544921875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,3,64,8,128,23.5,17.48974609375,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,3,128,8,128,26.515625,16.807373046875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,4,32,8,128,9.40625,24.2294921875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,4,64,8,128,10.96875,22.86474609375,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,4,128,8,128,12.3828125,22.182373046875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,8,32,8,128,0.5693359375,45.7294921875,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,8,64,8,128,0.6630859375,44.36474609375,45088768,3.547261426279374,1,0.00144272239413112 +29,mlp.down_proj,8,128,8,128,0.74609375,43.682373046875,45088768,3.547261426279374,1,0.00144272239413112 +30,self_attn.q_proj,2,32,8,128,33.3125,5.015625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,2,64,8,128,39.03125,4.5078125,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,2,128,8,128,45.375,4.25390625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,3,32,8,128,14.25,7.015625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,3,64,8,128,16.71875,6.5078125,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,3,128,8,128,19.453125,6.25390625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,4,32,8,128,6.65234375,9.015625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,4,64,8,128,7.8046875,8.5078125,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,4,128,8,128,9.0859375,8.25390625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,8,32,8,128,0.40283203125,17.015625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,8,64,8,128,0.471923828125,16.5078125,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.q_proj,8,128,8,128,0.54736328125,16.25390625,16777216,5.794882486073459,1,4.4782337499782e-4 +30,self_attn.v_proj,2,32,8,128,32,5.015625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,2,64,8,128,37.34375,4.5078125,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,2,128,8,128,42.125,4.25390625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,3,32,8,128,13.6796875,7.015625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,3,64,8,128,15.96875,6.5078125,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,3,128,8,128,18.03125,6.25390625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,4,32,8,128,6.38671875,9.015625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,4,64,8,128,7.453125,8.5078125,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,4,128,8,128,8.4140625,8.25390625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,8,32,8,128,0.38671875,17.015625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,8,64,8,128,0.450927734375,16.5078125,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.v_proj,8,128,8,128,0.50732421875,16.25390625,16777216,3.1510945883178296,1,4.4782337499782e-4 +30,self_attn.k_proj,2,32,8,128,33.78125,5.015625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,2,64,8,128,39.59375,4.5078125,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,2,128,8,128,46.125,4.25390625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,3,32,8,128,14.4453125,7.015625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,3,64,8,128,16.9375,6.5078125,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,3,128,8,128,19.78125,6.25390625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,4,32,8,128,6.7421875,9.015625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,4,64,8,128,7.90625,8.5078125,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,4,128,8,128,9.234375,8.25390625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,8,32,8,128,0.408203125,17.015625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,8,64,8,128,0.47802734375,16.5078125,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.k_proj,8,128,8,128,0.55615234375,16.25390625,16777216,3.65192595962886,1,4.4782337499782e-4 +30,self_attn.o_proj,2,32,8,128,32.0625,5.015625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,2,64,8,128,37.4375,4.5078125,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,2,128,8,128,42.375,4.25390625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,3,32,8,128,13.7421875,7.015625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,3,64,8,128,16.078125,6.5078125,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,3,128,8,128,18.265625,6.25390625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,4,32,8,128,6.4140625,9.015625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,4,64,8,128,7.50390625,8.5078125,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,4,128,8,128,8.5234375,8.25390625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,8,32,8,128,0.38818359375,17.015625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,8,64,8,128,0.45361328125,16.5078125,16777216,3.989114407498837,1,2.9881519731134e-4 +30,self_attn.o_proj,8,128,8,128,0.513671875,16.25390625,16777216,3.989114407498837,1,2.9881519731134e-4 +30,mlp.gate_proj,2,32,8,128,52.875,13.4794921875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,2,64,8,128,62.8125,12.11474609375,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,2,128,8,128,72.5,11.432373046875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,3,32,8,128,22.8125,18.8544921875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,3,64,8,128,27.296875,17.48974609375,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,3,128,8,128,31.875,16.807373046875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,4,32,8,128,10.6484375,24.2294921875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,4,64,8,128,12.7578125,22.86474609375,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,4,128,8,128,14.9375,22.182373046875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,8,32,8,128,0.64453125,45.7294921875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,8,64,8,128,0.77099609375,44.36474609375,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.gate_proj,8,128,8,128,0.89990234375,43.682373046875,45088768,8.489964918198787,1,0.00259080668911337 +30,mlp.up_proj,2,32,8,128,48.78125,13.4794921875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,2,64,8,128,57.15625,12.11474609375,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,2,128,8,128,64.9375,11.432373046875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,3,32,8,128,21.09375,18.8544921875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,3,64,8,128,24.90625,17.48974609375,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,3,128,8,128,28.671875,16.807373046875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,4,32,8,128,9.8515625,24.2294921875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,4,64,8,128,11.6484375,22.86474609375,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,4,128,8,128,13.453125,22.182373046875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,8,32,8,128,0.59619140625,45.7294921875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,8,64,8,128,0.7041015625,44.36474609375,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.up_proj,8,128,8,128,0.810546875,43.682373046875,45088768,9.524281481179829,1,0.00259080668911337 +30,mlp.down_proj,2,32,8,128,46.90625,13.4794921875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,2,64,8,128,54.6875,12.11474609375,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,2,128,8,128,61.75,11.432373046875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,3,32,8,128,20.078125,18.8544921875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,3,64,8,128,23.4375,17.48974609375,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,3,128,8,128,26.53125,16.807373046875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,4,32,8,128,9.375,24.2294921875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,4,64,8,128,10.953125,22.86474609375,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,4,128,8,128,12.3984375,22.182373046875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,8,32,8,128,0.56787109375,45.7294921875,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,8,64,8,128,0.66259765625,44.36474609375,45088768,5.107840285620544,2,0.0062037156894803 +30,mlp.down_proj,8,128,8,128,0.748046875,43.682373046875,45088768,5.107840285620544,2,0.0062037156894803 +31,self_attn.q_proj,2,32,8,128,33.78125,5.015625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,2,64,8,128,39.65625,4.5078125,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,2,128,8,128,46.21875,4.25390625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,3,32,8,128,14.4453125,7.015625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,3,64,8,128,16.953125,6.5078125,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,3,128,8,128,19.84375,6.25390625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,4,32,8,128,6.7421875,9.015625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,4,64,8,128,7.91796875,8.5078125,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,4,128,8,128,9.25,8.25390625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,8,32,8,128,0.408447265625,17.015625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,8,64,8,128,0.478759765625,16.5078125,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.q_proj,8,128,8,128,0.55810546875,16.25390625,16777216,4.2604406701808495,1,0.00205261004157364 +31,self_attn.v_proj,2,32,8,128,29.34375,5.015625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,2,64,8,128,34.3125,4.5078125,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,2,128,8,128,38.875,4.25390625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,3,32,8,128,12.5546875,7.015625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,3,64,8,128,14.6875,6.5078125,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,3,128,8,128,16.640625,6.25390625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,4,32,8,128,5.859375,9.015625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,4,64,8,128,6.859375,8.5078125,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,4,128,8,128,7.7734375,8.25390625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,8,32,8,128,0.35498046875,17.015625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,8,64,8,128,0.415283203125,16.5078125,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.v_proj,8,128,8,128,0.469482421875,16.25390625,16777216,3.423163795614051,1,0.00205261004157364 +31,self_attn.k_proj,2,32,8,128,35.3125,5.015625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,2,64,8,128,41.53125,4.5078125,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,2,128,8,128,48.75,4.25390625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,3,32,8,128,15.109375,7.015625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,3,64,8,128,17.78125,6.5078125,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,3,128,8,128,20.984375,6.25390625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,4,32,8,128,7.05078125,9.015625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,4,64,8,128,8.296875,8.5078125,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,4,128,8,128,9.796875,8.25390625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,8,32,8,128,0.4267578125,17.015625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,8,64,8,128,0.50146484375,16.5078125,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.k_proj,8,128,8,128,0.59033203125,16.25390625,16777216,4.044775676830005,1,0.00205261004157364 +31,self_attn.o_proj,2,32,8,128,29.359375,5.015625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,2,64,8,128,34.40625,4.5078125,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,2,128,8,128,39.21875,4.25390625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,3,32,8,128,12.640625,7.015625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,3,64,8,128,14.9140625,6.5078125,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,3,128,8,128,17.171875,6.25390625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,4,32,8,128,5.9296875,9.015625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,4,64,8,128,7.01171875,8.5078125,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,4,128,8,128,8.1171875,8.25390625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,8,32,8,128,0.36083984375,17.015625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,8,64,8,128,0.42822265625,16.5078125,16777216,38.917877019289406,1,4.0688703302294e-4 +31,self_attn.o_proj,8,128,8,128,0.4970703125,16.25390625,16777216,38.917877019289406,1,4.0688703302294e-4 +31,mlp.gate_proj,2,32,8,128,55.125,13.4794921875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,2,64,8,128,65.25,12.11474609375,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,2,128,8,128,75,11.432373046875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,3,32,8,128,23.578125,18.8544921875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,3,64,8,128,27.984375,17.48974609375,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,3,128,8,128,32.25,16.807373046875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,4,32,8,128,11.0078125,24.2294921875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,4,64,8,128,13.0546875,22.86474609375,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,4,128,8,128,15.0390625,22.182373046875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,8,32,8,128,0.666015625,45.7294921875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,8,64,8,128,0.7890625,44.36474609375,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.gate_proj,8,128,8,128,0.90625,43.682373046875,45088768,3.59690663694663,2,0.0053017633035779 +31,mlp.up_proj,2,32,8,128,51.21875,13.4794921875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,2,64,8,128,60.0625,12.11474609375,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,2,128,8,128,68.3125,11.432373046875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,3,32,8,128,21.90625,18.8544921875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,3,64,8,128,25.765625,17.48974609375,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,3,128,8,128,29.40625,16.807373046875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,4,32,8,128,10.2265625,24.2294921875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,4,64,8,128,12.0234375,22.86474609375,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,4,128,8,128,13.734375,22.182373046875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,8,32,8,128,0.619140625,45.7294921875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,8,64,8,128,0.72705078125,44.36474609375,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.up_proj,8,128,8,128,0.8271484375,43.682373046875,45088768,3.714970630664952,2,0.0053017633035779 +31,mlp.down_proj,2,32,8,128,46.96875,13.4794921875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,2,64,8,128,54.875,12.11474609375,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,2,128,8,128,62.125,11.432373046875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,3,32,8,128,20.125,18.8544921875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,3,64,8,128,23.578125,17.48974609375,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,3,128,8,128,26.8125,16.807373046875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,4,32,8,128,9.3984375,24.2294921875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,4,64,8,128,11.015625,22.86474609375,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,4,128,8,128,12.5390625,22.182373046875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,8,32,8,128,0.5693359375,45.7294921875,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,8,64,8,128,0.6669921875,44.36474609375,45088768,8.550558646527769,2,0.01145612820982933 +31,mlp.down_proj,8,128,8,128,0.75732421875,43.682373046875,45088768,8.550558646527769,2,0.01145612820982933 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-3.1-8B.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-3.1-8B.csv new file mode 100644 index 0000000000000000000000000000000000000000..9b29986aee98ac32777edad03b363cd26a2f17a4 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Llama-3.1-8B.csv @@ -0,0 +1,2689 @@ +layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params +0,self_attn.q_proj,2,32,8,128,31.353914260864258,5.015625,16777216 +0,self_attn.q_proj,2,64,8,128,37.95658493041992,4.5078125,16777216 +0,self_attn.q_proj,2,128,8,128,44.016265869140625,4.25390625,16777216 +0,self_attn.q_proj,3,32,8,128,13.658820152282715,7.015625,16777216 +0,self_attn.q_proj,3,64,8,128,16.869216918945312,6.5078125,16777216 +0,self_attn.q_proj,3,128,8,128,20.302486419677734,6.25390625,16777216 +0,self_attn.q_proj,4,32,8,128,6.373652458190918,9.015625,16777216 +0,self_attn.q_proj,4,64,8,128,7.8845295906066895,8.5078125,16777216 +0,self_attn.q_proj,4,128,8,128,9.499629020690918,8.25390625,16777216 +0,self_attn.q_proj,8,32,8,128,0.38628098368644714,17.015625,16777216 +0,self_attn.q_proj,8,64,8,128,0.4774124324321747,16.5078125,16777216 +0,self_attn.q_proj,8,128,8,128,0.5732570290565491,16.25390625,16777216 +0,self_attn.v_proj,2,32,8,128,5.73164701461792,1.25390625,4194304 +0,self_attn.v_proj,2,64,8,128,6.755260944366455,1.126953125,4194304 +0,self_attn.v_proj,2,128,8,128,7.683543682098389,1.0634765625,4194304 +0,self_attn.v_proj,3,32,8,128,2.453540802001953,1.75390625,4194304 +0,self_attn.v_proj,3,64,8,128,2.8974449634552,1.626953125,4194304 +0,self_attn.v_proj,3,128,8,128,3.3091328144073486,1.5634765625,4194304 +0,self_attn.v_proj,4,32,8,128,1.1455694437026978,2.25390625,4194304 +0,self_attn.v_proj,4,64,8,128,1.3514484167099,2.126953125,4194304 +0,self_attn.v_proj,4,128,8,128,1.5437763929367065,2.0634765625,4194304 +0,self_attn.v_proj,8,32,8,128,0.070158451795578,4.25390625,4194304 +0,self_attn.v_proj,8,64,8,128,0.0823262557387352,4.126953125,4194304 +0,self_attn.v_proj,8,128,8,128,0.0934486985206604,4.0634765625,4194304 +0,self_attn.k_proj,2,32,8,128,22.8813533782959,1.25390625,4194304 +0,self_attn.k_proj,2,64,8,128,27.585908889770508,1.126953125,4194304 +0,self_attn.k_proj,2,128,8,128,31.700037002563477,1.0634765625,4194304 +0,self_attn.k_proj,3,32,8,128,10.011412620544434,1.75390625,4194304 +0,self_attn.k_proj,3,64,8,128,12.338346481323242,1.626953125,4194304 +0,self_attn.k_proj,3,128,8,128,14.586329460144043,1.5634765625,4194304 +0,self_attn.k_proj,4,32,8,128,4.676162242889404,2.25390625,4194304 +0,self_attn.k_proj,4,64,8,128,5.7611188888549805,2.126953125,4194304 +0,self_attn.k_proj,4,128,8,128,6.841525554656982,2.0634765625,4194304 +0,self_attn.k_proj,8,32,8,128,0.2830941081047058,4.25390625,4194304 +0,self_attn.k_proj,8,64,8,128,0.3489919900894165,4.126953125,4194304 +0,self_attn.k_proj,8,128,8,128,0.4125772714614868,4.0634765625,4194304 +0,self_attn.o_proj,2,32,8,128,13.097525596618652,5.015625,16777216 +0,self_attn.o_proj,2,64,8,128,15.490410804748535,4.5078125,16777216 +0,self_attn.o_proj,2,128,8,128,17.747562408447266,4.25390625,16777216 +0,self_attn.o_proj,3,32,8,128,5.651535987854004,7.015625,16777216 +0,self_attn.o_proj,3,64,8,128,6.748816967010498,6.5078125,16777216 +0,self_attn.o_proj,3,128,8,128,7.85412073135376,6.25390625,16777216 +0,self_attn.o_proj,4,32,8,128,2.6453964710235596,9.015625,16777216 +0,self_attn.o_proj,4,64,8,128,3.166229248046875,8.5078125,16777216 +0,self_attn.o_proj,4,128,8,128,3.7004642486572266,8.25390625,16777216 +0,self_attn.o_proj,8,32,8,128,0.16013142466545105,17.015625,16777216 +0,self_attn.o_proj,8,64,8,128,0.1917829066514969,16.5078125,16777216 +0,self_attn.o_proj,8,128,8,128,0.22378630936145782,16.25390625,16777216 +0,mlp.gate_proj,2,32,8,128,37.102561950683594,17.5546875,58720256 +0,mlp.gate_proj,2,64,8,128,43.699092864990234,15.77734375,58720256 +0,mlp.gate_proj,2,128,8,128,49.80133819580078,14.888671875,58720256 +0,mlp.gate_proj,3,32,8,128,15.898781776428223,24.5546875,58720256 +0,mlp.gate_proj,3,64,8,128,18.750398635864258,22.77734375,58720256 +0,mlp.gate_proj,3,128,8,128,21.506258010864258,21.888671875,58720256 +0,mlp.gate_proj,4,32,8,128,7.4153618812561035,31.5546875,58720256 +0,mlp.gate_proj,4,64,8,128,8.750203132629395,29.77734375,58720256 +0,mlp.gate_proj,4,128,8,128,10.037043571472168,28.888671875,58720256 +0,mlp.gate_proj,8,32,8,128,0.4499224126338959,59.5546875,58720256 +0,mlp.gate_proj,8,64,8,128,0.5307692885398865,57.77734375,58720256 +0,mlp.gate_proj,8,128,8,128,0.60727459192276,56.888671875,58720256 +0,mlp.up_proj,2,32,8,128,33.78168487548828,17.5546875,58720256 +0,mlp.up_proj,2,64,8,128,39.53192138671875,15.77734375,58720256 +0,mlp.up_proj,2,128,8,128,44.79423141479492,14.888671875,58720256 +0,mlp.up_proj,3,32,8,128,14.448440551757812,24.5546875,58720256 +0,mlp.up_proj,3,64,8,128,16.92987823486328,22.77734375,58720256 +0,mlp.up_proj,3,128,8,128,19.200571060180664,21.888671875,58720256 +0,mlp.up_proj,4,32,8,128,6.739186763763428,31.5546875,58720256 +0,mlp.up_proj,4,64,8,128,7.901976108551025,29.77734375,58720256 +0,mlp.up_proj,4,128,8,128,8.960665702819824,28.888671875,58720256 +0,mlp.up_proj,8,32,8,128,0.4092637896537781,59.5546875,58720256 +0,mlp.up_proj,8,64,8,128,0.47888556122779846,57.77734375,58720256 +0,mlp.up_proj,8,128,8,128,0.541621208190918,56.888671875,58720256 +0,mlp.down_proj,2,32,8,128,33.93179702758789,17.5546875,58720256 +0,mlp.down_proj,2,64,8,128,39.87704849243164,15.77734375,58720256 +0,mlp.down_proj,2,128,8,128,45.44186019897461,14.888671875,58720256 +0,mlp.down_proj,3,32,8,128,14.55285930633545,24.5546875,58720256 +0,mlp.down_proj,3,64,8,128,17.161346435546875,22.77734375,58720256 +0,mlp.down_proj,3,128,8,128,19.66796112060547,21.888671875,58720256 +0,mlp.down_proj,4,32,8,128,6.793258190155029,31.5546875,58720256 +0,mlp.down_proj,4,64,8,128,8.025272369384766,29.77734375,58720256 +0,mlp.down_proj,4,128,8,128,9.213508605957031,28.888671875,58720256 +0,mlp.down_proj,8,32,8,128,0.41281500458717346,59.5546875,58720256 +0,mlp.down_proj,8,64,8,128,0.4869184195995331,57.77734375,58720256 +0,mlp.down_proj,8,128,8,128,0.5582616925239563,56.888671875,58720256 +1,self_attn.q_proj,2,32,8,128,32.29652404785156,5.015625,16777216 +1,self_attn.q_proj,2,64,8,128,38.70115661621094,4.5078125,16777216 +1,self_attn.q_proj,2,128,8,128,44.72345733642578,4.25390625,16777216 +1,self_attn.q_proj,3,32,8,128,14.052138328552246,7.015625,16777216 +1,self_attn.q_proj,3,64,8,128,17.109827041625977,6.5078125,16777216 +1,self_attn.q_proj,3,128,8,128,20.420045852661133,6.25390625,16777216 +1,self_attn.q_proj,4,32,8,128,6.563356876373291,9.015625,16777216 +1,self_attn.q_proj,4,64,8,128,8.004384994506836,8.5078125,16777216 +1,self_attn.q_proj,4,128,8,128,9.557613372802734,8.25390625,16777216 +1,self_attn.q_proj,8,32,8,128,0.3971712291240692,17.015625,16777216 +1,self_attn.q_proj,8,64,8,128,0.4838508665561676,16.5078125,16777216 +1,self_attn.q_proj,8,128,8,128,0.5765863656997681,16.25390625,16777216 +1,self_attn.v_proj,2,32,8,128,6.992136001586914,1.25390625,4194304 +1,self_attn.v_proj,2,64,8,128,8.214609146118164,1.126953125,4194304 +1,self_attn.v_proj,2,128,8,128,9.343364715576172,1.0634765625,4194304 +1,self_attn.v_proj,3,32,8,128,2.990298271179199,1.75390625,4194304 +1,self_attn.v_proj,3,64,8,128,3.515022039413452,1.626953125,4194304 +1,self_attn.v_proj,3,128,8,128,3.999699592590332,1.5634765625,4194304 +1,self_attn.v_proj,4,32,8,128,1.3951102495193481,2.25390625,4194304 +1,self_attn.v_proj,4,64,8,128,1.6407121419906616,2.126953125,4194304 +1,self_attn.v_proj,4,128,8,128,1.8681185245513916,2.0634765625,4194304 +1,self_attn.v_proj,8,32,8,128,0.08454109728336334,4.25390625,4194304 +1,self_attn.v_proj,8,64,8,128,0.09935352951288223,4.126953125,4194304 +1,self_attn.v_proj,8,128,8,128,0.11260196566581726,4.0634765625,4194304 +1,self_attn.k_proj,2,32,8,128,23.162338256835938,1.25390625,4194304 +1,self_attn.k_proj,2,64,8,128,27.537050247192383,1.126953125,4194304 +1,self_attn.k_proj,2,128,8,128,31.056779861450195,1.0634765625,4194304 +1,self_attn.k_proj,3,32,8,128,10.139037132263184,1.75390625,4194304 +1,self_attn.k_proj,3,64,8,128,12.381231307983398,1.626953125,4194304 +1,self_attn.k_proj,3,128,8,128,14.396084785461426,1.5634765625,4194304 +1,self_attn.k_proj,4,32,8,128,4.743703365325928,2.25390625,4194304 +1,self_attn.k_proj,4,64,8,128,5.807139873504639,2.126953125,4194304 +1,self_attn.k_proj,4,128,8,128,6.786295413970947,2.0634765625,4194304 +1,self_attn.k_proj,8,32,8,128,0.2868131995201111,4.25390625,4194304 +1,self_attn.k_proj,8,64,8,128,0.35116565227508545,4.126953125,4194304 +1,self_attn.k_proj,8,128,8,128,0.4091583788394928,4.0634765625,4194304 +1,self_attn.o_proj,2,32,8,128,15.152460098266602,5.015625,16777216 +1,self_attn.o_proj,2,64,8,128,18.016952514648438,4.5078125,16777216 +1,self_attn.o_proj,2,128,8,128,20.729021072387695,4.25390625,16777216 +1,self_attn.o_proj,3,32,8,128,6.609400749206543,7.015625,16777216 +1,self_attn.o_proj,3,64,8,128,8.003070831298828,6.5078125,16777216 +1,self_attn.o_proj,3,128,8,128,9.439093589782715,6.25390625,16777216 +1,self_attn.o_proj,4,32,8,128,3.09663724899292,9.015625,16777216 +1,self_attn.o_proj,4,64,8,128,3.7703700065612793,8.5078125,16777216 +1,self_attn.o_proj,4,128,8,128,4.493071556091309,8.25390625,16777216 +1,self_attn.o_proj,8,32,8,128,0.18734721839427948,17.015625,16777216 +1,self_attn.o_proj,8,64,8,128,0.22794853150844574,16.5078125,16777216 +1,self_attn.o_proj,8,128,8,128,0.27150189876556396,16.25390625,16777216 +1,mlp.gate_proj,2,32,8,128,37.304813385009766,17.5546875,58720256 +1,mlp.gate_proj,2,64,8,128,43.75423812866211,15.77734375,58720256 +1,mlp.gate_proj,2,128,8,128,49.68382263183594,14.888671875,58720256 +1,mlp.gate_proj,3,32,8,128,15.982694625854492,24.5546875,58720256 +1,mlp.gate_proj,3,64,8,128,18.7584171295166,22.77734375,58720256 +1,mlp.gate_proj,3,128,8,128,21.395015716552734,21.888671875,58720256 +1,mlp.gate_proj,4,32,8,128,7.454606533050537,31.5546875,58720256 +1,mlp.gate_proj,4,64,8,128,8.757988929748535,29.77734375,58720256 +1,mlp.gate_proj,4,128,8,128,9.991157531738281,28.888671875,58720256 +1,mlp.gate_proj,8,32,8,128,0.4522514045238495,59.5546875,58720256 +1,mlp.gate_proj,8,64,8,128,0.5311040878295898,57.77734375,58720256 +1,mlp.gate_proj,8,128,8,128,0.604612410068512,56.888671875,58720256 +1,mlp.up_proj,2,32,8,128,34.21645736694336,17.5546875,58720256 +1,mlp.up_proj,2,64,8,128,39.97964859008789,15.77734375,58720256 +1,mlp.up_proj,2,128,8,128,45.251304626464844,14.888671875,58720256 +1,mlp.up_proj,3,32,8,128,14.641050338745117,24.5546875,58720256 +1,mlp.up_proj,3,64,8,128,17.131967544555664,22.77734375,58720256 +1,mlp.up_proj,3,128,8,128,19.439006805419922,21.888671875,58720256 +1,mlp.up_proj,4,32,8,128,6.83050012588501,31.5546875,58720256 +1,mlp.up_proj,4,64,8,128,8.003508567810059,29.77734375,58720256 +1,mlp.up_proj,4,128,8,128,9.07853889465332,28.888671875,58720256 +1,mlp.up_proj,8,32,8,128,0.414794921875,59.5546875,58720256 +1,mlp.up_proj,8,64,8,128,0.4848926365375519,57.77734375,58720256 +1,mlp.up_proj,8,128,8,128,0.5488119125366211,56.888671875,58720256 +1,mlp.down_proj,2,32,8,128,34.15717697143555,17.5546875,58720256 +1,mlp.down_proj,2,64,8,128,39.90151596069336,15.77734375,58720256 +1,mlp.down_proj,2,128,8,128,45.13018035888672,14.888671875,58720256 +1,mlp.down_proj,3,32,8,128,14.628009796142578,24.5546875,58720256 +1,mlp.down_proj,3,64,8,128,17.132530212402344,22.77734375,58720256 +1,mlp.down_proj,3,128,8,128,19.45292091369629,21.888671875,58720256 +1,mlp.down_proj,4,32,8,128,6.82521390914917,31.5546875,58720256 +1,mlp.down_proj,4,64,8,128,8.005558967590332,29.77734375,58720256 +1,mlp.down_proj,4,128,8,128,9.090758323669434,28.888671875,58720256 +1,mlp.down_proj,8,32,8,128,0.41456136107444763,59.5546875,58720256 +1,mlp.down_proj,8,64,8,128,0.4853489398956299,57.77734375,58720256 +1,mlp.down_proj,8,128,8,128,0.5503469705581665,56.888671875,58720256 +2,self_attn.q_proj,2,32,8,128,30.365509033203125,5.015625,16777216 +2,self_attn.q_proj,2,64,8,128,36.27523422241211,4.5078125,16777216 +2,self_attn.q_proj,2,128,8,128,41.893245697021484,4.25390625,16777216 +2,self_attn.q_proj,3,32,8,128,13.078288078308105,7.015625,16777216 +2,self_attn.q_proj,3,64,8,128,15.747833251953125,6.5078125,16777216 +2,self_attn.q_proj,3,128,8,128,18.530900955200195,6.25390625,16777216 +2,self_attn.q_proj,4,32,8,128,6.105013370513916,9.015625,16777216 +2,self_attn.q_proj,4,64,8,128,7.3519439697265625,8.5078125,16777216 +2,self_attn.q_proj,4,128,8,128,8.660855293273926,8.25390625,16777216 +2,self_attn.q_proj,8,32,8,128,0.36959201097488403,17.015625,16777216 +2,self_attn.q_proj,8,64,8,128,0.44454026222229004,16.5078125,16777216 +2,self_attn.q_proj,8,128,8,128,0.5220497846603394,16.25390625,16777216 +2,self_attn.v_proj,2,32,8,128,5.865646839141846,1.25390625,4194304 +2,self_attn.v_proj,2,64,8,128,6.912929534912109,1.126953125,4194304 +2,self_attn.v_proj,2,128,8,128,7.873420715332031,1.0634765625,4194304 +2,self_attn.v_proj,3,32,8,128,2.508667469024658,1.75390625,4194304 +2,self_attn.v_proj,3,64,8,128,2.9588942527770996,1.626953125,4194304 +2,self_attn.v_proj,3,128,8,128,3.3745577335357666,1.5634765625,4194304 +2,self_attn.v_proj,4,32,8,128,1.1705979108810425,2.25390625,4194304 +2,self_attn.v_proj,4,64,8,128,1.3809363842010498,2.126953125,4194304 +2,self_attn.v_proj,4,128,8,128,1.5750237703323364,2.0634765625,4194304 +2,self_attn.v_proj,8,32,8,128,0.07090398669242859,4.25390625,4194304 +2,self_attn.v_proj,8,64,8,128,0.08354182541370392,4.126953125,4194304 +2,self_attn.v_proj,8,128,8,128,0.09496123343706131,4.0634765625,4194304 +2,self_attn.k_proj,2,32,8,128,23.66606903076172,1.25390625,4194304 +2,self_attn.k_proj,2,64,8,128,28.22458839416504,1.126953125,4194304 +2,self_attn.k_proj,2,128,8,128,31.91000747680664,1.0634765625,4194304 +2,self_attn.k_proj,3,32,8,128,10.266057014465332,1.75390625,4194304 +2,self_attn.k_proj,3,64,8,128,12.421545028686523,1.626953125,4194304 +2,self_attn.k_proj,3,128,8,128,14.415032386779785,1.5634765625,4194304 +2,self_attn.k_proj,4,32,8,128,4.791152000427246,2.25390625,4194304 +2,self_attn.k_proj,4,64,8,128,5.805240154266357,2.126953125,4194304 +2,self_attn.k_proj,4,128,8,128,6.744692802429199,2.0634765625,4194304 +2,self_attn.k_proj,8,32,8,128,0.29007431864738464,4.25390625,4194304 +2,self_attn.k_proj,8,64,8,128,0.35100963711738586,4.126953125,4194304 +2,self_attn.k_proj,8,128,8,128,0.4065532982349396,4.0634765625,4194304 +2,self_attn.o_proj,2,32,8,128,13.74080753326416,5.015625,16777216 +2,self_attn.o_proj,2,64,8,128,16.259204864501953,4.5078125,16777216 +2,self_attn.o_proj,2,128,8,128,18.660329818725586,4.25390625,16777216 +2,self_attn.o_proj,3,32,8,128,5.905372142791748,7.015625,16777216 +2,self_attn.o_proj,3,64,8,128,7.029297351837158,6.5078125,16777216 +2,self_attn.o_proj,3,128,8,128,8.160127639770508,6.25390625,16777216 +2,self_attn.o_proj,4,32,8,128,2.7578015327453613,9.015625,16777216 +2,self_attn.o_proj,4,64,8,128,3.2829208374023438,8.5078125,16777216 +2,self_attn.o_proj,4,128,8,128,3.8164286613464355,8.25390625,16777216 +2,self_attn.o_proj,8,32,8,128,0.1668560802936554,17.015625,16777216 +2,self_attn.o_proj,8,64,8,128,0.1985287219285965,16.5078125,16777216 +2,self_attn.o_proj,8,128,8,128,0.2299499213695526,16.25390625,16777216 +2,mlp.gate_proj,2,32,8,128,38.17426681518555,17.5546875,58720256 +2,mlp.gate_proj,2,64,8,128,44.794532775878906,15.77734375,58720256 +2,mlp.gate_proj,2,128,8,128,50.85295486450195,14.888671875,58720256 +2,mlp.gate_proj,3,32,8,128,16.34511947631836,24.5546875,58720256 +2,mlp.gate_proj,3,64,8,128,19.193567276000977,22.77734375,58720256 +2,mlp.gate_proj,3,128,8,128,21.89304542541504,21.888671875,58720256 +2,mlp.gate_proj,4,32,8,128,7.6252827644348145,31.5546875,58720256 +2,mlp.gate_proj,4,64,8,128,8.959787368774414,29.77734375,58720256 +2,mlp.gate_proj,4,128,8,128,10.218497276306152,28.888671875,58720256 +2,mlp.gate_proj,8,32,8,128,0.4624970555305481,59.5546875,58720256 +2,mlp.gate_proj,8,64,8,128,0.5432219505310059,57.77734375,58720256 +2,mlp.gate_proj,8,128,8,128,0.6183617115020752,56.888671875,58720256 +2,mlp.up_proj,2,32,8,128,33.983341217041016,17.5546875,58720256 +2,mlp.up_proj,2,64,8,128,39.65127944946289,15.77734375,58720256 +2,mlp.up_proj,2,128,8,128,44.813541412353516,14.888671875,58720256 +2,mlp.up_proj,3,32,8,128,14.534832000732422,24.5546875,58720256 +2,mlp.up_proj,3,64,8,128,16.98261833190918,22.77734375,58720256 +2,mlp.up_proj,3,128,8,128,19.21963882446289,21.888671875,58720256 +2,mlp.up_proj,4,32,8,128,6.779207229614258,31.5546875,58720256 +2,mlp.up_proj,4,64,8,128,7.926984786987305,29.77734375,58720256 +2,mlp.up_proj,4,128,8,128,8.969118118286133,28.888671875,58720256 +2,mlp.up_proj,8,32,8,128,0.41166192293167114,59.5546875,58720256 +2,mlp.up_proj,8,64,8,128,0.48040977120399475,57.77734375,58720256 +2,mlp.up_proj,8,128,8,128,0.5421519875526428,56.888671875,58720256 +2,mlp.down_proj,2,32,8,128,34.12712097167969,17.5546875,58720256 +2,mlp.down_proj,2,64,8,128,39.839595794677734,15.77734375,58720256 +2,mlp.down_proj,2,128,8,128,45.02960205078125,14.888671875,58720256 +2,mlp.down_proj,3,32,8,128,14.610933303833008,24.5546875,58720256 +2,mlp.down_proj,3,64,8,128,17.088600158691406,22.77734375,58720256 +2,mlp.down_proj,3,128,8,128,19.363445281982422,21.888671875,58720256 +2,mlp.down_proj,4,32,8,128,6.817477226257324,31.5546875,58720256 +2,mlp.down_proj,4,64,8,128,7.985670566558838,29.77734375,58720256 +2,mlp.down_proj,4,128,8,128,9.051961898803711,28.888671875,58720256 +2,mlp.down_proj,8,32,8,128,0.4140346050262451,59.5546875,58720256 +2,mlp.down_proj,8,64,8,128,0.4839620888233185,57.77734375,58720256 +2,mlp.down_proj,8,128,8,128,0.5475106835365295,56.888671875,58720256 +3,self_attn.q_proj,2,32,8,128,30.0765438079834,5.015625,16777216 +3,self_attn.q_proj,2,64,8,128,35.90376663208008,4.5078125,16777216 +3,self_attn.q_proj,2,128,8,128,41.742164611816406,4.25390625,16777216 +3,self_attn.q_proj,3,32,8,128,12.903072357177734,7.015625,16777216 +3,self_attn.q_proj,3,64,8,128,15.450603485107422,6.5078125,16777216 +3,self_attn.q_proj,3,128,8,128,18.135257720947266,6.25390625,16777216 +3,self_attn.q_proj,4,32,8,128,6.0236029624938965,9.015625,16777216 +3,self_attn.q_proj,4,64,8,128,7.211884498596191,8.5078125,16777216 +3,self_attn.q_proj,4,128,8,128,8.466845512390137,8.25390625,16777216 +3,self_attn.q_proj,8,32,8,128,0.36464881896972656,17.015625,16777216 +3,self_attn.q_proj,8,64,8,128,0.4361165463924408,16.5078125,16777216 +3,self_attn.q_proj,8,128,8,128,0.5101926326751709,16.25390625,16777216 +3,self_attn.v_proj,2,32,8,128,6.825829982757568,1.25390625,4194304 +3,self_attn.v_proj,2,64,8,128,8.05012321472168,1.126953125,4194304 +3,self_attn.v_proj,2,128,8,128,9.182071685791016,1.0634765625,4194304 +3,self_attn.v_proj,3,32,8,128,2.9221222400665283,1.75390625,4194304 +3,self_attn.v_proj,3,64,8,128,3.447613477706909,1.626953125,4194304 +3,self_attn.v_proj,3,128,8,128,3.9407715797424316,1.5634765625,4194304 +3,self_attn.v_proj,4,32,8,128,1.3632123470306396,2.25390625,4194304 +3,self_attn.v_proj,4,64,8,128,1.6084686517715454,2.126953125,4194304 +3,self_attn.v_proj,4,128,8,128,1.8388642072677612,2.0634765625,4194304 +3,self_attn.v_proj,8,32,8,128,0.08255447447299957,4.25390625,4194304 +3,self_attn.v_proj,8,64,8,128,0.09733613580465317,4.126953125,4194304 +3,self_attn.v_proj,8,128,8,128,0.11077219992876053,4.0634765625,4194304 +3,self_attn.k_proj,2,32,8,128,23.270971298217773,1.25390625,4194304 +3,self_attn.k_proj,2,64,8,128,27.877880096435547,1.126953125,4194304 +3,self_attn.k_proj,2,128,8,128,31.943754196166992,1.0634765625,4194304 +3,self_attn.k_proj,3,32,8,128,10.016742706298828,1.75390625,4194304 +3,self_attn.k_proj,3,64,8,128,12.083196640014648,1.626953125,4194304 +3,self_attn.k_proj,3,128,8,128,14.049163818359375,1.5634765625,4194304 +3,self_attn.k_proj,4,32,8,128,4.673532485961914,2.25390625,4194304 +3,self_attn.k_proj,4,64,8,128,5.63783073425293,2.126953125,4194304 +3,self_attn.k_proj,4,128,8,128,6.559664249420166,2.0634765625,4194304 +3,self_attn.k_proj,8,32,8,128,0.28303390741348267,4.25390625,4194304 +3,self_attn.k_proj,8,64,8,128,0.3410465121269226,4.126953125,4194304 +3,self_attn.k_proj,8,128,8,128,0.3952634334564209,4.0634765625,4194304 +3,self_attn.o_proj,2,32,8,128,15.478205680847168,5.015625,16777216 +3,self_attn.o_proj,2,64,8,128,18.152347564697266,4.5078125,16777216 +3,self_attn.o_proj,2,128,8,128,20.656272888183594,4.25390625,16777216 +3,self_attn.o_proj,3,32,8,128,6.63338565826416,7.015625,16777216 +3,self_attn.o_proj,3,64,8,128,7.80942964553833,6.5078125,16777216 +3,self_attn.o_proj,3,128,8,128,8.927901268005371,6.25390625,16777216 +3,self_attn.o_proj,4,32,8,128,3.0960311889648438,9.015625,16777216 +3,self_attn.o_proj,4,64,8,128,3.64548921585083,8.5078125,16777216 +3,self_attn.o_proj,4,128,8,128,4.170829772949219,8.25390625,16777216 +3,self_attn.o_proj,8,32,8,128,0.18740607798099518,17.015625,16777216 +3,self_attn.o_proj,8,64,8,128,0.22039665281772614,16.5078125,16777216 +3,self_attn.o_proj,8,128,8,128,0.25134262442588806,16.25390625,16777216 +3,mlp.gate_proj,2,32,8,128,40.5650520324707,17.5546875,58720256 +3,mlp.gate_proj,2,64,8,128,47.76633071899414,15.77734375,58720256 +3,mlp.gate_proj,2,128,8,128,54.47903823852539,14.888671875,58720256 +3,mlp.gate_proj,3,32,8,128,17.36041259765625,24.5546875,58720256 +3,mlp.gate_proj,3,64,8,128,20.488161087036133,22.77734375,58720256 +3,mlp.gate_proj,3,128,8,128,23.468496322631836,21.888671875,58720256 +3,mlp.gate_proj,4,32,8,128,8.104728698730469,31.5546875,58720256 +3,mlp.gate_proj,4,64,8,128,9.559508323669434,29.77734375,58720256 +3,mlp.gate_proj,4,128,8,128,10.962554931640625,28.888671875,58720256 +3,mlp.gate_proj,8,32,8,128,0.49138230085372925,59.5546875,58720256 +3,mlp.gate_proj,8,64,8,128,0.5801472067832947,57.77734375,58720256 +3,mlp.gate_proj,8,128,8,128,0.6634387969970703,56.888671875,58720256 +3,mlp.up_proj,2,32,8,128,33.47959899902344,17.5546875,58720256 +3,mlp.up_proj,2,64,8,128,39.0748176574707,15.77734375,58720256 +3,mlp.up_proj,2,128,8,128,44.18793487548828,14.888671875,58720256 +3,mlp.up_proj,3,32,8,128,14.3134126663208,24.5546875,58720256 +3,mlp.up_proj,3,64,8,128,16.737089157104492,22.77734375,58720256 +3,mlp.up_proj,3,128,8,128,18.952110290527344,21.888671875,58720256 +3,mlp.up_proj,4,32,8,128,6.677464962005615,31.5546875,58720256 +3,mlp.up_proj,4,64,8,128,7.812717437744141,29.77734375,58720256 +3,mlp.up_proj,4,128,8,128,8.847103118896484,28.888671875,58720256 +3,mlp.up_proj,8,32,8,128,0.40553200244903564,59.5546875,58720256 +3,mlp.up_proj,8,64,8,128,0.4735206961631775,57.77734375,58720256 +3,mlp.up_proj,8,128,8,128,0.5345169901847839,56.888671875,58720256 +3,mlp.down_proj,2,32,8,128,33.44816207885742,17.5546875,58720256 +3,mlp.down_proj,2,64,8,128,39.041778564453125,15.77734375,58720256 +3,mlp.down_proj,2,128,8,128,44.118682861328125,14.888671875,58720256 +3,mlp.down_proj,3,32,8,128,14.30600357055664,24.5546875,58720256 +3,mlp.down_proj,3,64,8,128,16.7259464263916,22.77734375,58720256 +3,mlp.down_proj,3,128,8,128,18.936351776123047,21.888671875,58720256 +3,mlp.down_proj,4,32,8,128,6.675703048706055,31.5546875,58720256 +3,mlp.down_proj,4,64,8,128,7.809521675109863,29.77734375,58720256 +3,mlp.down_proj,4,128,8,128,8.839631080627441,28.888671875,58720256 +3,mlp.down_proj,8,32,8,128,0.4053632318973541,59.5546875,58720256 +3,mlp.down_proj,8,64,8,128,0.47341373562812805,57.77734375,58720256 +3,mlp.down_proj,8,128,8,128,0.5344170331954956,56.888671875,58720256 +4,self_attn.q_proj,2,32,8,128,30.083614349365234,5.015625,16777216 +4,self_attn.q_proj,2,64,8,128,35.95115661621094,4.5078125,16777216 +4,self_attn.q_proj,2,128,8,128,41.91539001464844,4.25390625,16777216 +4,self_attn.q_proj,3,32,8,128,12.93169116973877,7.015625,16777216 +4,self_attn.q_proj,3,64,8,128,15.548795700073242,6.5078125,16777216 +4,self_attn.q_proj,3,128,8,128,18.41219139099121,6.25390625,16777216 +4,self_attn.q_proj,4,32,8,128,6.038930416107178,9.015625,16777216 +4,self_attn.q_proj,4,64,8,128,7.2613348960876465,8.5078125,16777216 +4,self_attn.q_proj,4,128,8,128,8.604525566101074,8.25390625,16777216 +4,self_attn.q_proj,8,32,8,128,0.3654842972755432,17.015625,16777216 +4,self_attn.q_proj,8,64,8,128,0.438870906829834,16.5078125,16777216 +4,self_attn.q_proj,8,128,8,128,0.5186217427253723,16.25390625,16777216 +4,self_attn.v_proj,2,32,8,128,7.416920185089111,1.25390625,4194304 +4,self_attn.v_proj,2,64,8,128,8.764520645141602,1.126953125,4194304 +4,self_attn.v_proj,2,128,8,128,10.00966739654541,1.0634765625,4194304 +4,self_attn.v_proj,3,32,8,128,3.172041177749634,1.75390625,4194304 +4,self_attn.v_proj,3,64,8,128,3.7507846355438232,1.626953125,4194304 +4,self_attn.v_proj,3,128,8,128,4.286296367645264,1.5634765625,4194304 +4,self_attn.v_proj,4,32,8,128,1.4802603721618652,2.25390625,4194304 +4,self_attn.v_proj,4,64,8,128,1.7502737045288086,2.126953125,4194304 +4,self_attn.v_proj,4,128,8,128,2.001307487487793,2.0634765625,4194304 +4,self_attn.v_proj,8,32,8,128,0.0895981565117836,4.25390625,4194304 +4,self_attn.v_proj,8,64,8,128,0.10592576861381531,4.126953125,4194304 +4,self_attn.v_proj,8,128,8,128,0.12058984488248825,4.0634765625,4194304 +4,self_attn.k_proj,2,32,8,128,22.736724853515625,1.25390625,4194304 +4,self_attn.k_proj,2,64,8,128,27.162885665893555,1.126953125,4194304 +4,self_attn.k_proj,2,128,8,128,31.053537368774414,1.0634765625,4194304 +4,self_attn.k_proj,3,32,8,128,9.771358489990234,1.75390625,4194304 +4,self_attn.k_proj,3,64,8,128,11.750617980957031,1.626953125,4194304 +4,self_attn.k_proj,3,128,8,128,13.632905006408691,1.5634765625,4194304 +4,self_attn.k_proj,4,32,8,128,4.561568260192871,2.25390625,4194304 +4,self_attn.k_proj,4,64,8,128,5.486782073974609,2.126953125,4194304 +4,self_attn.k_proj,4,128,8,128,6.36852502822876,2.0634765625,4194304 +4,self_attn.k_proj,8,32,8,128,0.2761555314064026,4.25390625,4194304 +4,self_attn.k_proj,8,64,8,128,0.33179372549057007,4.126953125,4194304 +4,self_attn.k_proj,8,128,8,128,0.38395005464553833,4.0634765625,4194304 +4,self_attn.o_proj,2,32,8,128,15.92138385772705,5.015625,16777216 +4,self_attn.o_proj,2,64,8,128,18.749645233154297,4.5078125,16777216 +4,self_attn.o_proj,2,128,8,128,21.425750732421875,4.25390625,16777216 +4,self_attn.o_proj,3,32,8,128,6.8580427169799805,7.015625,16777216 +4,self_attn.o_proj,3,64,8,128,8.154813766479492,6.5078125,16777216 +4,self_attn.o_proj,3,128,8,128,9.444148063659668,6.25390625,16777216 +4,self_attn.o_proj,4,32,8,128,3.20137357711792,9.015625,16777216 +4,self_attn.o_proj,4,64,8,128,3.811955690383911,8.5078125,16777216 +4,self_attn.o_proj,4,128,8,128,4.429438591003418,8.25390625,16777216 +4,self_attn.o_proj,8,32,8,128,0.1937759816646576,17.015625,16777216 +4,self_attn.o_proj,8,64,8,128,0.23040039837360382,16.5078125,16777216 +4,self_attn.o_proj,8,128,8,128,0.26690956950187683,16.25390625,16777216 +4,mlp.gate_proj,2,32,8,128,42.83198165893555,17.5546875,58720256 +4,mlp.gate_proj,2,64,8,128,50.48828887939453,15.77734375,58720256 +4,mlp.gate_proj,2,128,8,128,57.70125198364258,14.888671875,58720256 +4,mlp.gate_proj,3,32,8,128,18.325246810913086,24.5546875,58720256 +4,mlp.gate_proj,3,64,8,128,21.675304412841797,22.77734375,58720256 +4,mlp.gate_proj,3,128,8,128,24.83637046813965,21.888671875,58720256 +4,mlp.gate_proj,4,32,8,128,8.552212715148926,31.5546875,58720256 +4,mlp.gate_proj,4,64,8,128,10.11113452911377,29.77734375,58720256 +4,mlp.gate_proj,4,128,8,128,11.601034164428711,28.888671875,58720256 +4,mlp.gate_proj,8,32,8,128,0.5187771320343018,59.5546875,58720256 +4,mlp.gate_proj,8,64,8,128,0.6137517094612122,57.77734375,58720256 +4,mlp.gate_proj,8,128,8,128,0.7021692395210266,56.888671875,58720256 +4,mlp.up_proj,2,32,8,128,32.911781311035156,17.5546875,58720256 +4,mlp.up_proj,2,64,8,128,38.47255325317383,15.77734375,58720256 +4,mlp.up_proj,2,128,8,128,43.55121994018555,14.888671875,58720256 +4,mlp.up_proj,3,32,8,128,14.081363677978516,24.5546875,58720256 +4,mlp.up_proj,3,64,8,128,16.50337791442871,22.77734375,58720256 +4,mlp.up_proj,3,128,8,128,18.734643936157227,21.888671875,58720256 +4,mlp.up_proj,4,32,8,128,6.567831039428711,31.5546875,58720256 +4,mlp.up_proj,4,64,8,128,7.701935768127441,29.77734375,58720256 +4,mlp.up_proj,4,128,8,128,8.745016098022461,28.888671875,58720256 +4,mlp.up_proj,8,32,8,128,0.3990180194377899,59.5546875,58720256 +4,mlp.up_proj,8,64,8,128,0.4669491648674011,57.77734375,58720256 +4,mlp.up_proj,8,128,8,128,0.5286247730255127,56.888671875,58720256 +4,mlp.down_proj,2,32,8,128,32.81414031982422,17.5546875,58720256 +4,mlp.down_proj,2,64,8,128,38.33388137817383,15.77734375,58720256 +4,mlp.down_proj,2,128,8,128,43.37004470825195,14.888671875,58720256 +4,mlp.down_proj,3,32,8,128,14.036075592041016,24.5546875,58720256 +4,mlp.down_proj,3,64,8,128,16.439712524414062,22.77734375,58720256 +4,mlp.down_proj,3,128,8,128,18.641700744628906,21.888671875,58720256 +4,mlp.down_proj,4,32,8,128,6.548447608947754,31.5546875,58720256 +4,mlp.down_proj,4,64,8,128,7.675734043121338,29.77734375,58720256 +4,mlp.down_proj,4,128,8,128,8.7052640914917,28.888671875,58720256 +4,mlp.down_proj,8,32,8,128,0.39785444736480713,59.5546875,58720256 +4,mlp.down_proj,8,64,8,128,0.4653557240962982,57.77734375,58720256 +4,mlp.down_proj,8,128,8,128,0.5262950658798218,56.888671875,58720256 +5,self_attn.q_proj,2,32,8,128,29.222444534301758,5.015625,16777216 +5,self_attn.q_proj,2,64,8,128,35.029354095458984,4.5078125,16777216 +5,self_attn.q_proj,2,128,8,128,41.1779670715332,4.25390625,16777216 +5,self_attn.q_proj,3,32,8,128,12.571595191955566,7.015625,16777216 +5,self_attn.q_proj,3,64,8,128,15.12048625946045,6.5078125,16777216 +5,self_attn.q_proj,3,128,8,128,18.071041107177734,6.25390625,16777216 +5,self_attn.q_proj,4,32,8,128,5.8744001388549805,9.015625,16777216 +5,self_attn.q_proj,4,64,8,128,7.0671467781066895,8.5078125,16777216 +5,self_attn.q_proj,4,128,8,128,8.446761131286621,8.25390625,16777216 +5,self_attn.q_proj,8,32,8,128,0.3555201292037964,17.015625,16777216 +5,self_attn.q_proj,8,64,8,128,0.4272916615009308,16.5078125,16777216 +5,self_attn.q_proj,8,128,8,128,0.5090081691741943,16.25390625,16777216 +5,self_attn.v_proj,2,32,8,128,6.084989070892334,1.25390625,4194304 +5,self_attn.v_proj,2,64,8,128,7.141707897186279,1.126953125,4194304 +5,self_attn.v_proj,2,128,8,128,8.107608795166016,1.0634765625,4194304 +5,self_attn.v_proj,3,32,8,128,2.6033425331115723,1.75390625,4194304 +5,self_attn.v_proj,3,64,8,128,3.0585944652557373,1.626953125,4194304 +5,self_attn.v_proj,3,128,8,128,3.477306365966797,1.5634765625,4194304 +5,self_attn.v_proj,4,32,8,128,1.214025855064392,2.25390625,4194304 +5,self_attn.v_proj,4,64,8,128,1.427322506904602,2.126953125,4194304 +5,self_attn.v_proj,4,128,8,128,1.6234699487686157,2.0634765625,4194304 +5,self_attn.v_proj,8,32,8,128,0.07356319576501846,4.25390625,4194304 +5,self_attn.v_proj,8,64,8,128,0.08635125309228897,4.126953125,4194304 +5,self_attn.v_proj,8,128,8,128,0.09787572175264359,4.0634765625,4194304 +5,self_attn.k_proj,2,32,8,128,22.873891830444336,1.25390625,4194304 +5,self_attn.k_proj,2,64,8,128,27.271949768066406,1.126953125,4194304 +5,self_attn.k_proj,2,128,8,128,31.07009506225586,1.0634765625,4194304 +5,self_attn.k_proj,3,32,8,128,9.906036376953125,1.75390625,4194304 +5,self_attn.k_proj,3,64,8,128,11.95331859588623,1.626953125,4194304 +5,self_attn.k_proj,3,128,8,128,13.895054817199707,1.5634765625,4194304 +5,self_attn.k_proj,4,32,8,128,4.628349781036377,2.25390625,4194304 +5,self_attn.k_proj,4,64,8,128,5.594888687133789,2.126953125,4194304 +5,self_attn.k_proj,4,128,8,128,6.535345077514648,2.0634765625,4194304 +5,self_attn.k_proj,8,32,8,128,0.2800804078578949,4.25390625,4194304 +5,self_attn.k_proj,8,64,8,128,0.3381898105144501,4.126953125,4194304 +5,self_attn.k_proj,8,128,8,128,0.39380884170532227,4.0634765625,4194304 +5,self_attn.o_proj,2,32,8,128,14.622654914855957,5.015625,16777216 +5,self_attn.o_proj,2,64,8,128,17.11866569519043,4.5078125,16777216 +5,self_attn.o_proj,2,128,8,128,19.441051483154297,4.25390625,16777216 +5,self_attn.o_proj,3,32,8,128,6.255242347717285,7.015625,16777216 +5,self_attn.o_proj,3,64,8,128,7.34019660949707,6.5078125,16777216 +5,self_attn.o_proj,3,128,8,128,8.356011390686035,6.25390625,16777216 +5,self_attn.o_proj,4,32,8,128,2.918644666671753,9.015625,16777216 +5,self_attn.o_proj,4,64,8,128,3.4252405166625977,8.5078125,16777216 +5,self_attn.o_proj,4,128,8,128,3.9014854431152344,8.25390625,16777216 +5,self_attn.o_proj,8,32,8,128,0.17674587666988373,17.015625,16777216 +5,self_attn.o_proj,8,64,8,128,0.2071458399295807,16.5078125,16777216 +5,self_attn.o_proj,8,128,8,128,0.23505708575248718,16.25390625,16777216 +5,mlp.gate_proj,2,32,8,128,42.56002426147461,17.5546875,58720256 +5,mlp.gate_proj,2,64,8,128,50.084197998046875,15.77734375,58720256 +5,mlp.gate_proj,2,128,8,128,57.10890579223633,14.888671875,58720256 +5,mlp.gate_proj,3,32,8,128,18.202054977416992,24.5546875,58720256 +5,mlp.gate_proj,3,64,8,128,21.47345733642578,22.77734375,58720256 +5,mlp.gate_proj,3,128,8,128,24.525468826293945,21.888671875,58720256 +5,mlp.gate_proj,4,32,8,128,8.49460506439209,31.5546875,58720256 +5,mlp.gate_proj,4,64,8,128,10.01638412475586,29.77734375,58720256 +5,mlp.gate_proj,4,128,8,128,11.453516006469727,28.888671875,58720256 +5,mlp.gate_proj,8,32,8,128,0.5152372717857361,59.5546875,58720256 +5,mlp.gate_proj,8,64,8,128,0.6080232262611389,57.77734375,58720256 +5,mlp.gate_proj,8,128,8,128,0.6934967041015625,56.888671875,58720256 +5,mlp.up_proj,2,32,8,128,33.044776916503906,17.5546875,58720256 +5,mlp.up_proj,2,64,8,128,38.59564971923828,15.77734375,58720256 +5,mlp.up_proj,2,128,8,128,43.66254806518555,14.888671875,58720256 +5,mlp.up_proj,3,32,8,128,14.136244773864746,24.5546875,58720256 +5,mlp.up_proj,3,64,8,128,16.543148040771484,22.77734375,58720256 +5,mlp.up_proj,3,128,8,128,18.75366973876953,21.888671875,58720256 +5,mlp.up_proj,4,32,8,128,6.592376232147217,31.5546875,58720256 +5,mlp.up_proj,4,64,8,128,7.72108268737793,29.77734375,58720256 +5,mlp.up_proj,4,128,8,128,8.751051902770996,28.888671875,58720256 +5,mlp.up_proj,8,32,8,128,0.40038764476776123,59.5546875,58720256 +5,mlp.up_proj,8,64,8,128,0.468051016330719,57.77734375,58720256 +5,mlp.up_proj,8,128,8,128,0.5288262963294983,56.888671875,58720256 +5,mlp.down_proj,2,32,8,128,32.94956970214844,17.5546875,58720256 +5,mlp.down_proj,2,64,8,128,38.47535705566406,15.77734375,58720256 +5,mlp.down_proj,2,128,8,128,43.51252365112305,14.888671875,58720256 +5,mlp.down_proj,3,32,8,128,14.094295501708984,24.5546875,58720256 +5,mlp.down_proj,3,64,8,128,16.492313385009766,22.77734375,58720256 +5,mlp.down_proj,3,128,8,128,18.676166534423828,21.888671875,58720256 +5,mlp.down_proj,4,32,8,128,6.577403545379639,31.5546875,58720256 +5,mlp.down_proj,4,64,8,128,7.703035831451416,29.77734375,58720256 +5,mlp.down_proj,4,128,8,128,8.73043155670166,28.888671875,58720256 +5,mlp.down_proj,8,32,8,128,0.3995763063430786,59.5546875,58720256 +5,mlp.down_proj,8,64,8,128,0.4670538306236267,57.77734375,58720256 +5,mlp.down_proj,8,128,8,128,0.5279058814048767,56.888671875,58720256 +6,self_attn.q_proj,2,32,8,128,30.357757568359375,5.015625,16777216 +6,self_attn.q_proj,2,64,8,128,36.35004806518555,4.5078125,16777216 +6,self_attn.q_proj,2,128,8,128,42.761268615722656,4.25390625,16777216 +6,self_attn.q_proj,3,32,8,128,13.027424812316895,7.015625,16777216 +6,self_attn.q_proj,3,64,8,128,15.665924072265625,6.5078125,16777216 +6,self_attn.q_proj,3,128,8,128,18.68075180053711,6.25390625,16777216 +6,self_attn.q_proj,4,32,8,128,6.079878330230713,9.015625,16777216 +6,self_attn.q_proj,4,64,8,128,7.312449932098389,8.5078125,16777216 +6,self_attn.q_proj,4,128,8,128,8.7217435836792,8.25390625,16777216 +6,self_attn.q_proj,8,32,8,128,0.3680567443370819,17.015625,16777216 +6,self_attn.q_proj,8,64,8,128,0.44200587272644043,16.5078125,16777216 +6,self_attn.q_proj,8,128,8,128,0.5257716774940491,16.25390625,16777216 +6,self_attn.v_proj,2,32,8,128,6.581231117248535,1.25390625,4194304 +6,self_attn.v_proj,2,64,8,128,7.726805686950684,1.126953125,4194304 +6,self_attn.v_proj,2,128,8,128,8.79161548614502,1.0634765625,4194304 +6,self_attn.v_proj,3,32,8,128,2.8154382705688477,1.75390625,4194304 +6,self_attn.v_proj,3,64,8,128,3.3060097694396973,1.626953125,4194304 +6,self_attn.v_proj,3,128,8,128,3.7630813121795654,1.5634765625,4194304 +6,self_attn.v_proj,4,32,8,128,1.3137822151184082,2.25390625,4194304 +6,self_attn.v_proj,4,64,8,128,1.543904185295105,2.126953125,4194304 +6,self_attn.v_proj,4,128,8,128,1.756843090057373,2.0634765625,4194304 +6,self_attn.v_proj,8,32,8,128,0.07959981262683868,4.25390625,4194304 +6,self_attn.v_proj,8,64,8,128,0.09338797628879547,4.126953125,4194304 +6,self_attn.v_proj,8,128,8,128,0.10586458444595337,4.0634765625,4194304 +6,self_attn.k_proj,2,32,8,128,22.507909774780273,1.25390625,4194304 +6,self_attn.k_proj,2,64,8,128,26.86285972595215,1.126953125,4194304 +6,self_attn.k_proj,2,128,8,128,30.850976943969727,1.0634765625,4194304 +6,self_attn.k_proj,3,32,8,128,9.651057243347168,1.75390625,4194304 +6,self_attn.k_proj,3,64,8,128,11.548900604248047,1.626953125,4194304 +6,self_attn.k_proj,3,128,8,128,13.359161376953125,1.5634765625,4194304 +6,self_attn.k_proj,4,32,8,128,4.501798152923584,2.25390625,4194304 +6,self_attn.k_proj,4,64,8,128,5.387714862823486,2.126953125,4194304 +6,self_attn.k_proj,4,128,8,128,6.230097770690918,2.0634765625,4194304 +6,self_attn.k_proj,8,32,8,128,0.27238741517066956,4.25390625,4194304 +6,self_attn.k_proj,8,64,8,128,0.3257833421230316,4.126953125,4194304 +6,self_attn.k_proj,8,128,8,128,0.37550345063209534,4.0634765625,4194304 +6,self_attn.o_proj,2,32,8,128,15.235309600830078,5.015625,16777216 +6,self_attn.o_proj,2,64,8,128,17.822856903076172,4.5078125,16777216 +6,self_attn.o_proj,2,128,8,128,20.22065544128418,4.25390625,16777216 +6,self_attn.o_proj,3,32,8,128,6.5180745124816895,7.015625,16777216 +6,self_attn.o_proj,3,64,8,128,7.63830041885376,6.5078125,16777216 +6,self_attn.o_proj,3,128,8,128,8.683662414550781,6.25390625,16777216 +6,self_attn.o_proj,4,32,8,128,3.042389154434204,9.015625,16777216 +6,self_attn.o_proj,4,64,8,128,3.5667383670806885,8.5078125,16777216 +6,self_attn.o_proj,4,128,8,128,4.057885646820068,8.25390625,16777216 +6,self_attn.o_proj,8,32,8,128,0.18422940373420715,17.015625,16777216 +6,self_attn.o_proj,8,64,8,128,0.21565783023834229,16.5078125,16777216 +6,self_attn.o_proj,8,128,8,128,0.24448902904987335,16.25390625,16777216 +6,mlp.gate_proj,2,32,8,128,42.934268951416016,17.5546875,58720256 +6,mlp.gate_proj,2,64,8,128,50.62862777709961,15.77734375,58720256 +6,mlp.gate_proj,2,128,8,128,57.87238311767578,14.888671875,58720256 +6,mlp.gate_proj,3,32,8,128,18.36046600341797,24.5546875,58720256 +6,mlp.gate_proj,3,64,8,128,21.716197967529297,22.77734375,58720256 +6,mlp.gate_proj,3,128,8,128,24.871429443359375,21.888671875,58720256 +6,mlp.gate_proj,4,32,8,128,8.567111015319824,31.5546875,58720256 +6,mlp.gate_proj,4,64,8,128,10.129088401794434,29.77734375,58720256 +6,mlp.gate_proj,4,128,8,128,11.613856315612793,28.888671875,58720256 +6,mlp.gate_proj,8,32,8,128,0.5197127461433411,59.5546875,58720256 +6,mlp.gate_proj,8,64,8,128,0.6149697303771973,57.77734375,58720256 +6,mlp.gate_proj,8,128,8,128,0.7032941579818726,56.888671875,58720256 +6,mlp.up_proj,2,32,8,128,33.0184211730957,17.5546875,58720256 +6,mlp.up_proj,2,64,8,128,38.55435562133789,15.77734375,58720256 +6,mlp.up_proj,2,128,8,128,43.6105842590332,14.888671875,58720256 +6,mlp.up_proj,3,32,8,128,14.120134353637695,24.5546875,58720256 +6,mlp.up_proj,3,64,8,128,16.521259307861328,22.77734375,58720256 +6,mlp.up_proj,3,128,8,128,18.705928802490234,21.888671875,58720256 +6,mlp.up_proj,4,32,8,128,6.586504936218262,31.5546875,58720256 +6,mlp.up_proj,4,64,8,128,7.709926605224609,29.77734375,58720256 +6,mlp.up_proj,4,128,8,128,8.73668098449707,28.888671875,58720256 +6,mlp.up_proj,8,32,8,128,0.39997151494026184,59.5546875,58720256 +6,mlp.up_proj,8,64,8,128,0.4673685133457184,57.77734375,58720256 +6,mlp.up_proj,8,128,8,128,0.527724027633667,56.888671875,58720256 +6,mlp.down_proj,2,32,8,128,32.97812271118164,17.5546875,58720256 +6,mlp.down_proj,2,64,8,128,38.53191375732422,15.77734375,58720256 +6,mlp.down_proj,2,128,8,128,43.60712814331055,14.888671875,58720256 +6,mlp.down_proj,3,32,8,128,14.102401733398438,24.5546875,58720256 +6,mlp.down_proj,3,64,8,128,16.512012481689453,22.77734375,58720256 +6,mlp.down_proj,3,128,8,128,18.72007179260254,21.888671875,58720256 +6,mlp.down_proj,4,32,8,128,6.580263137817383,31.5546875,58720256 +6,mlp.down_proj,4,64,8,128,7.708827495574951,29.77734375,58720256 +6,mlp.down_proj,4,128,8,128,8.741865158081055,28.888671875,58720256 +6,mlp.down_proj,8,32,8,128,0.39963650703430176,59.5546875,58720256 +6,mlp.down_proj,8,64,8,128,0.46740472316741943,57.77734375,58720256 +6,mlp.down_proj,8,128,8,128,0.528446614742279,56.888671875,58720256 +7,self_attn.q_proj,2,32,8,128,29.098302841186523,5.015625,16777216 +7,self_attn.q_proj,2,64,8,128,34.797054290771484,4.5078125,16777216 +7,self_attn.q_proj,2,128,8,128,40.82132339477539,4.25390625,16777216 +7,self_attn.q_proj,3,32,8,128,12.4752779006958,7.015625,16777216 +7,self_attn.q_proj,3,64,8,128,14.974203109741211,6.5078125,16777216 +7,self_attn.q_proj,3,128,8,128,17.80384063720703,6.25390625,16777216 +7,self_attn.q_proj,4,32,8,128,5.822228908538818,9.015625,16777216 +7,self_attn.q_proj,4,64,8,128,6.986598014831543,8.5078125,16777216 +7,self_attn.q_proj,4,128,8,128,8.310455322265625,8.25390625,16777216 +7,self_attn.q_proj,8,32,8,128,0.3524138629436493,17.015625,16777216 +7,self_attn.q_proj,8,64,8,128,0.42246007919311523,16.5078125,16777216 +7,self_attn.q_proj,8,128,8,128,0.5007156729698181,16.25390625,16777216 +7,self_attn.v_proj,2,32,8,128,6.50254487991333,1.25390625,4194304 +7,self_attn.v_proj,2,64,8,128,7.637937545776367,1.126953125,4194304 +7,self_attn.v_proj,2,128,8,128,8.673030853271484,1.0634765625,4194304 +7,self_attn.v_proj,3,32,8,128,2.7816123962402344,1.75390625,4194304 +7,self_attn.v_proj,3,64,8,128,3.2684326171875,1.626953125,4194304 +7,self_attn.v_proj,3,128,8,128,3.7165303230285645,1.5634765625,4194304 +7,self_attn.v_proj,4,32,8,128,1.2977458238601685,2.25390625,4194304 +7,self_attn.v_proj,4,64,8,128,1.5260562896728516,2.126953125,4194304 +7,self_attn.v_proj,4,128,8,128,1.7341516017913818,2.0634765625,4194304 +7,self_attn.v_proj,8,32,8,128,0.07861447334289551,4.25390625,4194304 +7,self_attn.v_proj,8,64,8,128,0.09226477146148682,4.126953125,4194304 +7,self_attn.v_proj,8,128,8,128,0.10453595966100693,4.0634765625,4194304 +7,self_attn.k_proj,2,32,8,128,22.58738136291504,1.25390625,4194304 +7,self_attn.k_proj,2,64,8,128,26.90680503845215,1.126953125,4194304 +7,self_attn.k_proj,2,128,8,128,30.803823471069336,1.0634765625,4194304 +7,self_attn.k_proj,3,32,8,128,9.674205780029297,1.75390625,4194304 +7,self_attn.k_proj,3,64,8,128,11.544671058654785,1.626953125,4194304 +7,self_attn.k_proj,3,128,8,128,13.281217575073242,1.5634765625,4194304 +7,self_attn.k_proj,4,32,8,128,4.517162799835205,2.25390625,4194304 +7,self_attn.k_proj,4,64,8,128,5.38930082321167,2.126953125,4194304 +7,self_attn.k_proj,4,128,8,128,6.19756555557251,2.0634765625,4194304 +7,self_attn.k_proj,8,32,8,128,0.2733948826789856,4.25390625,4194304 +7,self_attn.k_proj,8,64,8,128,0.32574233412742615,4.126953125,4194304 +7,self_attn.k_proj,8,128,8,128,0.3735117018222809,4.0634765625,4194304 +7,self_attn.o_proj,2,32,8,128,15.576741218566895,5.015625,16777216 +7,self_attn.o_proj,2,64,8,128,18.285734176635742,4.5078125,16777216 +7,self_attn.o_proj,2,128,8,128,20.816131591796875,4.25390625,16777216 +7,self_attn.o_proj,3,32,8,128,6.668774604797363,7.015625,16777216 +7,self_attn.o_proj,3,64,8,128,7.849410057067871,6.5078125,16777216 +7,self_attn.o_proj,3,128,8,128,8.964725494384766,6.25390625,16777216 +7,self_attn.o_proj,4,32,8,128,3.1115434169769287,9.015625,16777216 +7,self_attn.o_proj,4,64,8,128,3.6639904975891113,8.5078125,16777216 +7,self_attn.o_proj,4,128,8,128,4.188023567199707,8.25390625,16777216 +7,self_attn.o_proj,8,32,8,128,0.18835203349590302,17.015625,16777216 +7,self_attn.o_proj,8,64,8,128,0.2213946431875229,16.5078125,16777216 +7,self_attn.o_proj,8,128,8,128,0.2523128390312195,16.25390625,16777216 +7,mlp.gate_proj,2,32,8,128,42.35660934448242,17.5546875,58720256 +7,mlp.gate_proj,2,64,8,128,50.031158447265625,15.77734375,58720256 +7,mlp.gate_proj,2,128,8,128,57.23368453979492,14.888671875,58720256 +7,mlp.gate_proj,3,32,8,128,18.117908477783203,24.5546875,58720256 +7,mlp.gate_proj,3,64,8,128,21.453506469726562,22.77734375,58720256 +7,mlp.gate_proj,3,128,8,128,24.602251052856445,21.888671875,58720256 +7,mlp.gate_proj,4,32,8,128,8.454544067382812,31.5546875,58720256 +7,mlp.gate_proj,4,64,8,128,10.010523796081543,29.77734375,58720256 +7,mlp.gate_proj,4,128,8,128,11.485529899597168,28.888671875,58720256 +7,mlp.gate_proj,8,32,8,128,0.5129130482673645,59.5546875,58720256 +7,mlp.gate_proj,8,64,8,128,0.6076484322547913,57.77734375,58720256 +7,mlp.gate_proj,8,128,8,128,0.6954354643821716,56.888671875,58720256 +7,mlp.up_proj,2,32,8,128,33.50232696533203,17.5546875,58720256 +7,mlp.up_proj,2,64,8,128,39.179927825927734,15.77734375,58720256 +7,mlp.up_proj,2,128,8,128,44.362144470214844,14.888671875,58720256 +7,mlp.up_proj,3,32,8,128,14.333677291870117,24.5546875,58720256 +7,mlp.up_proj,3,64,8,128,16.797828674316406,22.77734375,58720256 +7,mlp.up_proj,3,128,8,128,19.068626403808594,21.888671875,58720256 +7,mlp.up_proj,4,32,8,128,6.685086250305176,31.5546875,58720256 +7,mlp.up_proj,4,64,8,128,7.839568138122559,29.77734375,58720256 +7,mlp.up_proj,4,128,8,128,8.898309707641602,28.888671875,58720256 +7,mlp.up_proj,8,32,8,128,0.40605342388153076,59.5546875,58720256 +7,mlp.up_proj,8,64,8,128,0.4752146899700165,57.77734375,58720256 +7,mlp.up_proj,8,128,8,128,0.5378113389015198,56.888671875,58720256 +7,mlp.down_proj,2,32,8,128,33.53489303588867,17.5546875,58720256 +7,mlp.down_proj,2,64,8,128,39.26102066040039,15.77734375,58720256 +7,mlp.down_proj,2,128,8,128,44.49696731567383,14.888671875,58720256 +7,mlp.down_proj,3,32,8,128,14.347747802734375,24.5546875,58720256 +7,mlp.down_proj,3,64,8,128,16.82952308654785,22.77734375,58720256 +7,mlp.down_proj,3,128,8,128,19.135120391845703,21.888671875,58720256 +7,mlp.down_proj,4,32,8,128,6.6952009201049805,31.5546875,58720256 +7,mlp.down_proj,4,64,8,128,7.862168312072754,29.77734375,58720256 +7,mlp.down_proj,4,128,8,128,8.940099716186523,28.888671875,58720256 +7,mlp.down_proj,8,32,8,128,0.40658998489379883,59.5546875,58720256 +7,mlp.down_proj,8,64,8,128,0.47651708126068115,57.77734375,58720256 +7,mlp.down_proj,8,128,8,128,0.5403572916984558,56.888671875,58720256 +8,self_attn.q_proj,2,32,8,128,29.102739334106445,5.015625,16777216 +8,self_attn.q_proj,2,64,8,128,34.96165084838867,4.5078125,16777216 +8,self_attn.q_proj,2,128,8,128,41.559139251708984,4.25390625,16777216 +8,self_attn.q_proj,3,32,8,128,12.530718803405762,7.015625,16777216 +8,self_attn.q_proj,3,64,8,128,15.17061710357666,6.5078125,16777216 +8,self_attn.q_proj,3,128,8,128,18.44765853881836,6.25390625,16777216 +8,self_attn.q_proj,4,32,8,128,5.8522162437438965,9.015625,16777216 +8,self_attn.q_proj,4,64,8,128,7.095270156860352,8.5078125,16777216 +8,self_attn.q_proj,4,128,8,128,8.648637771606445,8.25390625,16777216 +8,self_attn.q_proj,8,32,8,128,0.3541480004787445,17.015625,16777216 +8,self_attn.q_proj,8,64,8,128,0.42901280522346497,16.5078125,16777216 +8,self_attn.q_proj,8,128,8,128,0.5214547514915466,16.25390625,16777216 +8,self_attn.v_proj,2,32,8,128,7.0695881843566895,1.25390625,4194304 +8,self_attn.v_proj,2,64,8,128,8.404984474182129,1.126953125,4194304 +8,self_attn.v_proj,2,128,8,128,9.669179916381836,1.0634765625,4194304 +8,self_attn.v_proj,3,32,8,128,3.027573347091675,1.75390625,4194304 +8,self_attn.v_proj,3,64,8,128,3.604905605316162,1.626953125,4194304 +8,self_attn.v_proj,3,128,8,128,4.154642581939697,1.5634765625,4194304 +8,self_attn.v_proj,4,32,8,128,1.4120959043502808,2.25390625,4194304 +8,self_attn.v_proj,4,64,8,128,1.6825671195983887,2.126953125,4194304 +8,self_attn.v_proj,4,128,8,128,1.9394460916519165,2.0634765625,4194304 +8,self_attn.v_proj,8,32,8,128,0.08554039895534515,4.25390625,4194304 +8,self_attn.v_proj,8,64,8,128,0.10174846649169922,4.126953125,4194304 +8,self_attn.v_proj,8,128,8,128,0.11690159887075424,4.0634765625,4194304 +8,self_attn.k_proj,2,32,8,128,22.94489288330078,1.25390625,4194304 +8,self_attn.k_proj,2,64,8,128,27.573701858520508,1.126953125,4194304 +8,self_attn.k_proj,2,128,8,128,31.673152923583984,1.0634765625,4194304 +8,self_attn.k_proj,3,32,8,128,9.911046028137207,1.75390625,4194304 +8,self_attn.k_proj,3,64,8,128,12.030993461608887,1.626953125,4194304 +8,self_attn.k_proj,3,128,8,128,14.083611488342285,1.5634765625,4194304 +8,self_attn.k_proj,4,32,8,128,4.6252217292785645,2.25390625,4194304 +8,self_attn.k_proj,4,64,8,128,5.614022254943848,2.126953125,4194304 +8,self_attn.k_proj,4,128,8,128,6.581591606140137,2.0634765625,4194304 +8,self_attn.k_proj,8,32,8,128,0.2798959016799927,4.25390625,4194304 +8,self_attn.k_proj,8,64,8,128,0.33947625756263733,4.126953125,4194304 +8,self_attn.k_proj,8,128,8,128,0.3966178297996521,4.0634765625,4194304 +8,self_attn.o_proj,2,32,8,128,15.93042278289795,5.015625,16777216 +8,self_attn.o_proj,2,64,8,128,18.701812744140625,4.5078125,16777216 +8,self_attn.o_proj,2,128,8,128,21.272045135498047,4.25390625,16777216 +8,self_attn.o_proj,3,32,8,128,6.82534122467041,7.015625,16777216 +8,self_attn.o_proj,3,64,8,128,8.035728454589844,6.5078125,16777216 +8,self_attn.o_proj,3,128,8,128,9.178585052490234,6.25390625,16777216 +8,self_attn.o_proj,4,32,8,128,3.1863675117492676,9.015625,16777216 +8,self_attn.o_proj,4,64,8,128,3.753875732421875,8.5078125,16777216 +8,self_attn.o_proj,4,128,8,128,4.293275356292725,8.25390625,16777216 +8,self_attn.o_proj,8,32,8,128,0.1928136646747589,17.015625,16777216 +8,self_attn.o_proj,8,64,8,128,0.22684866189956665,16.5078125,16777216 +8,self_attn.o_proj,8,128,8,128,0.25873854756355286,16.25390625,16777216 +8,mlp.gate_proj,2,32,8,128,42.63568878173828,17.5546875,58720256 +8,mlp.gate_proj,2,64,8,128,50.424198150634766,15.77734375,58720256 +8,mlp.gate_proj,2,128,8,128,57.76854705810547,14.888671875,58720256 +8,mlp.gate_proj,3,32,8,128,18.235918045043945,24.5546875,58720256 +8,mlp.gate_proj,3,64,8,128,21.62849998474121,22.77734375,58720256 +8,mlp.gate_proj,3,128,8,128,24.843730926513672,21.888671875,58720256 +8,mlp.gate_proj,4,32,8,128,8.50918197631836,31.5546875,58720256 +8,mlp.gate_proj,4,64,8,128,10.089552879333496,29.77734375,58720256 +8,mlp.gate_proj,4,128,8,128,11.599563598632812,28.888671875,58720256 +8,mlp.gate_proj,8,32,8,128,0.5162365436553955,59.5546875,58720256 +8,mlp.gate_proj,8,64,8,128,0.6125434041023254,57.77734375,58720256 +8,mlp.gate_proj,8,128,8,128,0.7024494409561157,56.888671875,58720256 +8,mlp.up_proj,2,32,8,128,33.355831146240234,17.5546875,58720256 +8,mlp.up_proj,2,64,8,128,39.005863189697266,15.77734375,58720256 +8,mlp.up_proj,2,128,8,128,44.16631317138672,14.888671875,58720256 +8,mlp.up_proj,3,32,8,128,14.269635200500488,24.5546875,58720256 +8,mlp.up_proj,3,64,8,128,16.717241287231445,22.77734375,58720256 +8,mlp.up_proj,3,128,8,128,18.965198516845703,21.888671875,58720256 +8,mlp.up_proj,4,32,8,128,6.65586519241333,31.5546875,58720256 +8,mlp.up_proj,4,64,8,128,7.80333948135376,29.77734375,58720256 +8,mlp.up_proj,4,128,8,128,8.854433059692383,28.888671875,58720256 +8,mlp.up_proj,8,32,8,128,0.4042971432209015,59.5546875,58720256 +8,mlp.up_proj,8,64,8,128,0.47299468517303467,57.77734375,58720256 +8,mlp.up_proj,8,128,8,128,0.5351264476776123,56.888671875,58720256 +8,mlp.down_proj,2,32,8,128,33.46714782714844,17.5546875,58720256 +8,mlp.down_proj,2,64,8,128,39.17340850830078,15.77734375,58720256 +8,mlp.down_proj,2,128,8,128,44.3709716796875,14.888671875,58720256 +8,mlp.down_proj,3,32,8,128,14.323638916015625,24.5546875,58720256 +8,mlp.down_proj,3,64,8,128,16.796981811523438,22.77734375,58720256 +8,mlp.down_proj,3,128,8,128,19.07370948791504,21.888671875,58720256 +8,mlp.down_proj,4,32,8,128,6.683643817901611,31.5546875,58720256 +8,mlp.down_proj,4,64,8,128,7.845089435577393,29.77734375,58720256 +8,mlp.down_proj,4,128,8,128,8.913344383239746,28.888671875,58720256 +8,mlp.down_proj,8,32,8,128,0.4059576988220215,59.5546875,58720256 +8,mlp.down_proj,8,64,8,128,0.47558698058128357,57.77734375,58720256 +8,mlp.down_proj,8,128,8,128,0.5389354825019836,56.888671875,58720256 +9,self_attn.q_proj,2,32,8,128,28.970827102661133,5.015625,16777216 +9,self_attn.q_proj,2,64,8,128,34.773170471191406,4.5078125,16777216 +9,self_attn.q_proj,2,128,8,128,41.48375701904297,4.25390625,16777216 +9,self_attn.q_proj,3,32,8,128,12.423651695251465,7.015625,16777216 +9,self_attn.q_proj,3,64,8,128,14.960939407348633,6.5078125,16777216 +9,self_attn.q_proj,3,128,8,128,18.188318252563477,6.25390625,16777216 +9,self_attn.q_proj,4,32,8,128,5.796566486358643,9.015625,16777216 +9,self_attn.q_proj,4,64,8,128,6.983687400817871,8.5078125,16777216 +9,self_attn.q_proj,4,128,8,128,8.491787910461426,8.25390625,16777216 +9,self_attn.q_proj,8,32,8,128,0.35094621777534485,17.015625,16777216 +9,self_attn.q_proj,8,64,8,128,0.4223078787326813,16.5078125,16777216 +9,self_attn.q_proj,8,128,8,128,0.5118732452392578,16.25390625,16777216 +9,self_attn.v_proj,2,32,8,128,8.430035591125488,1.25390625,4194304 +9,self_attn.v_proj,2,64,8,128,10.15719985961914,1.126953125,4194304 +9,self_attn.v_proj,2,128,8,128,11.790925025939941,1.0634765625,4194304 +9,self_attn.v_proj,3,32,8,128,3.6167452335357666,1.75390625,4194304 +9,self_attn.v_proj,3,64,8,128,4.3756561279296875,1.626953125,4194304 +9,self_attn.v_proj,3,128,8,128,5.129218578338623,1.5634765625,4194304 +9,self_attn.v_proj,4,32,8,128,1.688912034034729,2.25390625,4194304 +9,self_attn.v_proj,4,64,8,128,2.0433149337768555,2.126953125,4194304 +9,self_attn.v_proj,4,128,8,128,2.3952956199645996,2.0634765625,4194304 +9,self_attn.v_proj,8,32,8,128,0.10217446833848953,4.25390625,4194304 +9,self_attn.v_proj,8,64,8,128,0.12355350703001022,4.126953125,4194304 +9,self_attn.v_proj,8,128,8,128,0.1442720890045166,4.0634765625,4194304 +9,self_attn.k_proj,2,32,8,128,22.62237548828125,1.25390625,4194304 +9,self_attn.k_proj,2,64,8,128,27.09912872314453,1.126953125,4194304 +9,self_attn.k_proj,2,128,8,128,31.047313690185547,1.0634765625,4194304 +9,self_attn.k_proj,3,32,8,128,9.723644256591797,1.75390625,4194304 +9,self_attn.k_proj,3,64,8,128,11.704264640808105,1.626953125,4194304 +9,self_attn.k_proj,3,128,8,128,13.561904907226562,1.5634765625,4194304 +9,self_attn.k_proj,4,32,8,128,4.5324883460998535,2.25390625,4194304 +9,self_attn.k_proj,4,64,8,128,5.463091850280762,2.126953125,4194304 +9,self_attn.k_proj,4,128,8,128,6.332092761993408,2.0634765625,4194304 +9,self_attn.k_proj,8,32,8,128,0.27470558881759644,4.25390625,4194304 +9,self_attn.k_proj,8,64,8,128,0.33033686876296997,4.126953125,4194304 +9,self_attn.k_proj,8,128,8,128,0.38159799575805664,4.0634765625,4194304 +9,self_attn.o_proj,2,32,8,128,17.05085563659668,5.015625,16777216 +9,self_attn.o_proj,2,64,8,128,19.99981689453125,4.5078125,16777216 +9,self_attn.o_proj,2,128,8,128,22.716672897338867,4.25390625,16777216 +9,self_attn.o_proj,3,32,8,128,7.2985076904296875,7.015625,16777216 +9,self_attn.o_proj,3,64,8,128,8.575852394104004,6.5078125,16777216 +9,self_attn.o_proj,3,128,8,128,9.767566680908203,6.25390625,16777216 +9,self_attn.o_proj,4,32,8,128,3.4048287868499756,9.015625,16777216 +9,self_attn.o_proj,4,64,8,128,4.003286361694336,8.5078125,16777216 +9,self_attn.o_proj,4,128,8,128,4.560546875,8.25390625,16777216 +9,self_attn.o_proj,8,32,8,128,0.20610328018665314,17.015625,16777216 +9,self_attn.o_proj,8,64,8,128,0.24192199110984802,16.5078125,16777216 +9,self_attn.o_proj,8,128,8,128,0.2747935950756073,16.25390625,16777216 +9,mlp.gate_proj,2,32,8,128,43.36977767944336,17.5546875,58720256 +9,mlp.gate_proj,2,64,8,128,51.428401947021484,15.77734375,58720256 +9,mlp.gate_proj,2,128,8,128,59.06992721557617,14.888671875,58720256 +9,mlp.gate_proj,3,32,8,128,18.55809783935547,24.5546875,58720256 +9,mlp.gate_proj,3,64,8,128,22.089975357055664,22.77734375,58720256 +9,mlp.gate_proj,3,128,8,128,25.46958351135254,21.888671875,58720256 +9,mlp.gate_proj,4,32,8,128,8.660787582397461,31.5546875,58720256 +9,mlp.gate_proj,4,64,8,128,10.305378913879395,29.77734375,58720256 +9,mlp.gate_proj,4,128,8,128,11.895636558532715,28.888671875,58720256 +9,mlp.gate_proj,8,32,8,128,0.5254871249198914,59.5546875,58720256 +9,mlp.gate_proj,8,64,8,128,0.6257582306861877,57.77734375,58720256 +9,mlp.gate_proj,8,128,8,128,0.720163106918335,56.888671875,58720256 +9,mlp.up_proj,2,32,8,128,33.76575469970703,17.5546875,58720256 +9,mlp.up_proj,2,64,8,128,39.51215744018555,15.77734375,58720256 +9,mlp.up_proj,2,128,8,128,44.77749252319336,14.888671875,58720256 +9,mlp.up_proj,3,32,8,128,14.45068073272705,24.5546875,58720256 +9,mlp.up_proj,3,64,8,128,16.942270278930664,22.77734375,58720256 +9,mlp.up_proj,3,128,8,128,19.249910354614258,21.888671875,58720256 +9,mlp.up_proj,4,32,8,128,6.739861488342285,31.5546875,58720256 +9,mlp.up_proj,4,64,8,128,7.9095282554626465,29.77734375,58720256 +9,mlp.up_proj,4,128,8,128,8.984209060668945,28.888671875,58720256 +9,mlp.up_proj,8,32,8,128,0.40932872891426086,59.5546875,58720256 +9,mlp.up_proj,8,64,8,128,0.479402095079422,57.77734375,58720256 +9,mlp.up_proj,8,128,8,128,0.5432174801826477,56.888671875,58720256 +9,mlp.down_proj,2,32,8,128,33.639041900634766,17.5546875,58720256 +9,mlp.down_proj,2,64,8,128,39.37888717651367,15.77734375,58720256 +9,mlp.down_proj,2,128,8,128,44.63368225097656,14.888671875,58720256 +9,mlp.down_proj,3,32,8,128,14.392541885375977,24.5546875,58720256 +9,mlp.down_proj,3,64,8,128,16.87546157836914,22.77734375,58720256 +9,mlp.down_proj,3,128,8,128,19.167049407958984,21.888671875,58720256 +9,mlp.down_proj,4,32,8,128,6.715319633483887,31.5546875,58720256 +9,mlp.down_proj,4,64,8,128,7.88001823425293,29.77734375,58720256 +9,mlp.down_proj,4,128,8,128,8.955047607421875,28.888671875,58720256 +9,mlp.down_proj,8,32,8,128,0.4078029692173004,59.5546875,58720256 +9,mlp.down_proj,8,64,8,128,0.4777718484401703,57.77734375,58720256 +9,mlp.down_proj,8,128,8,128,0.5413944721221924,56.888671875,58720256 +10,self_attn.q_proj,2,32,8,128,29.74867820739746,5.015625,16777216 +10,self_attn.q_proj,2,64,8,128,35.734153747558594,4.5078125,16777216 +10,self_attn.q_proj,2,128,8,128,42.222740173339844,4.25390625,16777216 +10,self_attn.q_proj,3,32,8,128,12.815299987792969,7.015625,16777216 +10,self_attn.q_proj,3,64,8,128,15.518683433532715,6.5078125,16777216 +10,self_attn.q_proj,3,128,8,128,18.86678695678711,6.25390625,16777216 +10,self_attn.q_proj,4,32,8,128,5.987305164337158,9.015625,16777216 +10,self_attn.q_proj,4,64,8,128,7.257505416870117,8.5078125,16777216 +10,self_attn.q_proj,4,128,8,128,8.844426155090332,8.25390625,16777216 +10,self_attn.q_proj,8,32,8,128,0.36258378624916077,17.015625,16777216 +10,self_attn.q_proj,8,64,8,128,0.43899136781692505,16.5078125,16777216 +10,self_attn.q_proj,8,128,8,128,0.5332654118537903,16.25390625,16777216 +10,self_attn.v_proj,2,32,8,128,7.087030410766602,1.25390625,4194304 +10,self_attn.v_proj,2,64,8,128,8.432042121887207,1.126953125,4194304 +10,self_attn.v_proj,2,128,8,128,9.697478294372559,1.0634765625,4194304 +10,self_attn.v_proj,3,32,8,128,3.0343551635742188,1.75390625,4194304 +10,self_attn.v_proj,3,64,8,128,3.614154577255249,1.626953125,4194304 +10,self_attn.v_proj,3,128,8,128,4.1670355796813965,1.5634765625,4194304 +10,self_attn.v_proj,4,32,8,128,1.4159622192382812,2.25390625,4194304 +10,self_attn.v_proj,4,64,8,128,1.6870222091674805,2.126953125,4194304 +10,self_attn.v_proj,4,128,8,128,1.9449712038040161,2.0634765625,4194304 +10,self_attn.v_proj,8,32,8,128,0.08575884997844696,4.25390625,4194304 +10,self_attn.v_proj,8,64,8,128,0.1020272895693779,4.126953125,4194304 +10,self_attn.v_proj,8,128,8,128,0.11721678823232651,4.0634765625,4194304 +10,self_attn.k_proj,2,32,8,128,23.049606323242188,1.25390625,4194304 +10,self_attn.k_proj,2,64,8,128,27.577190399169922,1.126953125,4194304 +10,self_attn.k_proj,2,128,8,128,31.56557273864746,1.0634765625,4194304 +10,self_attn.k_proj,3,32,8,128,9.962592124938965,1.75390625,4194304 +10,self_attn.k_proj,3,64,8,128,12.061654090881348,1.626953125,4194304 +10,self_attn.k_proj,3,128,8,128,14.062702178955078,1.5634765625,4194304 +10,self_attn.k_proj,4,32,8,128,4.6485443115234375,2.25390625,4194304 +10,self_attn.k_proj,4,64,8,128,5.634191989898682,2.126953125,4194304 +10,self_attn.k_proj,4,128,8,128,6.587673187255859,2.0634765625,4194304 +10,self_attn.k_proj,8,32,8,128,0.2816416919231415,4.25390625,4194304 +10,self_attn.k_proj,8,64,8,128,0.3408685326576233,4.126953125,4194304 +10,self_attn.k_proj,8,128,8,128,0.39702314138412476,4.0634765625,4194304 +10,self_attn.o_proj,2,32,8,128,15.789702415466309,5.015625,16777216 +10,self_attn.o_proj,2,64,8,128,18.495756149291992,4.5078125,16777216 +10,self_attn.o_proj,2,128,8,128,20.98558235168457,4.25390625,16777216 +10,self_attn.o_proj,3,32,8,128,6.760126113891602,7.015625,16777216 +10,self_attn.o_proj,3,64,8,128,7.935427665710449,6.5078125,16777216 +10,self_attn.o_proj,3,128,8,128,9.019789695739746,6.25390625,16777216 +10,self_attn.o_proj,4,32,8,128,3.155888080596924,9.015625,16777216 +10,self_attn.o_proj,4,64,8,128,3.706847906112671,8.5078125,16777216 +10,self_attn.o_proj,4,128,8,128,4.218595504760742,8.25390625,16777216 +10,self_attn.o_proj,8,32,8,128,0.19101321697235107,17.015625,16777216 +10,self_attn.o_proj,8,64,8,128,0.22407148778438568,16.5078125,16777216 +10,self_attn.o_proj,8,128,8,128,0.25413307547569275,16.25390625,16777216 +10,mlp.gate_proj,2,32,8,128,42.18458557128906,17.5546875,58720256 +10,mlp.gate_proj,2,64,8,128,49.968502044677734,15.77734375,58720256 +10,mlp.gate_proj,2,128,8,128,57.33974075317383,14.888671875,58720256 +10,mlp.gate_proj,3,32,8,128,18.048921585083008,24.5546875,58720256 +10,mlp.gate_proj,3,64,8,128,21.444515228271484,22.77734375,58720256 +10,mlp.gate_proj,3,128,8,128,24.679664611816406,21.888671875,58720256 +10,mlp.gate_proj,4,32,8,128,8.422057151794434,31.5546875,58720256 +10,mlp.gate_proj,4,64,8,128,10.00362491607666,29.77734375,58720256 +10,mlp.gate_proj,4,128,8,128,11.52573299407959,28.888671875,58720256 +10,mlp.gate_proj,8,32,8,128,0.5109154582023621,59.5546875,58720256 +10,mlp.gate_proj,8,64,8,128,0.6073028445243835,57.77734375,58720256 +10,mlp.gate_proj,8,128,8,128,0.6975723505020142,56.888671875,58720256 +10,mlp.up_proj,2,32,8,128,34.16447830200195,17.5546875,58720256 +10,mlp.up_proj,2,64,8,128,39.9440803527832,15.77734375,58720256 +10,mlp.up_proj,2,128,8,128,45.22474670410156,14.888671875,58720256 +10,mlp.up_proj,3,32,8,128,14.615551948547363,24.5546875,58720256 +10,mlp.up_proj,3,64,8,128,17.10788917541504,22.77734375,58720256 +10,mlp.up_proj,3,128,8,128,19.388578414916992,21.888671875,58720256 +10,mlp.up_proj,4,32,8,128,6.816666126251221,31.5546875,58720256 +10,mlp.up_proj,4,64,8,128,7.987757205963135,29.77734375,58720256 +10,mlp.up_proj,4,128,8,128,9.048367500305176,28.888671875,58720256 +10,mlp.up_proj,8,32,8,128,0.41394659876823425,59.5546875,58720256 +10,mlp.up_proj,8,64,8,128,0.48397162556648254,57.77734375,58720256 +10,mlp.up_proj,8,128,8,128,0.5471081733703613,56.888671875,58720256 +10,mlp.down_proj,2,32,8,128,34.060115814208984,17.5546875,58720256 +10,mlp.down_proj,2,64,8,128,39.859745025634766,15.77734375,58720256 +10,mlp.down_proj,2,128,8,128,45.1495361328125,14.888671875,58720256 +10,mlp.down_proj,3,32,8,128,14.572885513305664,24.5546875,58720256 +10,mlp.down_proj,3,64,8,128,17.06893539428711,22.77734375,58720256 +10,mlp.down_proj,3,128,8,128,19.359783172607422,21.888671875,58720256 +10,mlp.down_proj,4,32,8,128,6.798254489898682,31.5546875,58720256 +10,mlp.down_proj,4,64,8,128,7.973402500152588,29.77734375,58720256 +10,mlp.down_proj,4,128,8,128,9.042780876159668,28.888671875,58720256 +10,mlp.down_proj,8,32,8,128,0.4128105640411377,59.5546875,58720256 +10,mlp.down_proj,8,64,8,128,0.4831791818141937,57.77734375,58720256 +10,mlp.down_proj,8,128,8,128,0.5467498302459717,56.888671875,58720256 +11,self_attn.q_proj,2,32,8,128,28.12038803100586,5.015625,16777216 +11,self_attn.q_proj,2,64,8,128,33.77008056640625,4.5078125,16777216 +11,self_attn.q_proj,2,128,8,128,39.85329818725586,4.25390625,16777216 +11,self_attn.q_proj,3,32,8,128,12.058806419372559,7.015625,16777216 +11,self_attn.q_proj,3,64,8,128,14.521921157836914,6.5078125,16777216 +11,self_attn.q_proj,3,128,8,128,17.422571182250977,6.25390625,16777216 +11,self_attn.q_proj,4,32,8,128,5.6265668869018555,9.015625,16777216 +11,self_attn.q_proj,4,64,8,128,6.777266979217529,8.5078125,16777216 +11,self_attn.q_proj,4,128,8,128,8.135604858398438,8.25390625,16777216 +11,self_attn.q_proj,8,32,8,128,0.34055307507514954,17.015625,16777216 +11,self_attn.q_proj,8,64,8,128,0.4099378287792206,16.5078125,16777216 +11,self_attn.q_proj,8,128,8,128,0.49041903018951416,16.25390625,16777216 +11,self_attn.v_proj,2,32,8,128,6.708601474761963,1.25390625,4194304 +11,self_attn.v_proj,2,64,8,128,7.860424041748047,1.126953125,4194304 +11,self_attn.v_proj,2,128,8,128,8.91523551940918,1.0634765625,4194304 +11,self_attn.v_proj,3,32,8,128,2.8698890209198,1.75390625,4194304 +11,self_attn.v_proj,3,64,8,128,3.367091178894043,1.626953125,4194304 +11,self_attn.v_proj,3,128,8,128,3.8249270915985107,1.5634765625,4194304 +11,self_attn.v_proj,4,32,8,128,1.3387746810913086,2.25390625,4194304 +11,self_attn.v_proj,4,64,8,128,1.571775197982788,2.126953125,4194304 +11,self_attn.v_proj,4,128,8,128,1.7860945463180542,2.0634765625,4194304 +11,self_attn.v_proj,8,32,8,128,0.0810854509472847,4.25390625,4194304 +11,self_attn.v_proj,8,64,8,128,0.09501796960830688,4.126953125,4194304 +11,self_attn.v_proj,8,128,8,128,0.10759054869413376,4.0634765625,4194304 +11,self_attn.k_proj,2,32,8,128,22.46952247619629,1.25390625,4194304 +11,self_attn.k_proj,2,64,8,128,26.8824520111084,1.126953125,4194304 +11,self_attn.k_proj,2,128,8,128,30.769580841064453,1.0634765625,4194304 +11,self_attn.k_proj,3,32,8,128,9.66723918914795,1.75390625,4194304 +11,self_attn.k_proj,3,64,8,128,11.600849151611328,1.626953125,4194304 +11,self_attn.k_proj,3,128,8,128,13.43286418914795,1.5634765625,4194304 +11,self_attn.k_proj,4,32,8,128,4.508543491363525,2.25390625,4194304 +11,self_attn.k_proj,4,64,8,128,5.4175310134887695,2.126953125,4194304 +11,self_attn.k_proj,4,128,8,128,6.271651744842529,2.0634765625,4194304 +11,self_attn.k_proj,8,32,8,128,0.27317366003990173,4.25390625,4194304 +11,self_attn.k_proj,8,64,8,128,0.32760030031204224,4.126953125,4194304 +11,self_attn.k_proj,8,128,8,128,0.3778547942638397,4.0634765625,4194304 +11,self_attn.o_proj,2,32,8,128,16.131656646728516,5.015625,16777216 +11,self_attn.o_proj,2,64,8,128,18.922481536865234,4.5078125,16777216 +11,self_attn.o_proj,2,128,8,128,21.499622344970703,4.25390625,16777216 +11,self_attn.o_proj,3,32,8,128,6.906184673309326,7.015625,16777216 +11,self_attn.o_proj,3,64,8,128,8.114346504211426,6.5078125,16777216 +11,self_attn.o_proj,3,128,8,128,9.238365173339844,6.25390625,16777216 +11,self_attn.o_proj,4,32,8,128,3.222871780395508,9.015625,16777216 +11,self_attn.o_proj,4,64,8,128,3.7875704765319824,8.5078125,16777216 +11,self_attn.o_proj,4,128,8,128,4.315328598022461,8.25390625,16777216 +11,self_attn.o_proj,8,32,8,128,0.19500353932380676,17.015625,16777216 +11,self_attn.o_proj,8,64,8,128,0.2289765477180481,16.5078125,16777216 +11,self_attn.o_proj,8,128,8,128,0.2600376605987549,16.25390625,16777216 +11,mlp.gate_proj,2,32,8,128,41.81515121459961,17.5546875,58720256 +11,mlp.gate_proj,2,64,8,128,49.63838577270508,15.77734375,58720256 +11,mlp.gate_proj,2,128,8,128,57.089256286621094,14.888671875,58720256 +11,mlp.gate_proj,3,32,8,128,17.907394409179688,24.5546875,58720256 +11,mlp.gate_proj,3,64,8,128,21.34540557861328,22.77734375,58720256 +11,mlp.gate_proj,3,128,8,128,24.66131591796875,21.888671875,58720256 +11,mlp.gate_proj,4,32,8,128,8.359386444091797,31.5546875,58720256 +11,mlp.gate_proj,4,64,8,128,9.95639705657959,29.77734375,58720256 +11,mlp.gate_proj,4,128,8,128,11.521852493286133,28.888671875,58720256 +11,mlp.gate_proj,8,32,8,128,0.5070455074310303,59.5546875,58720256 +11,mlp.gate_proj,8,64,8,128,0.6044442653656006,57.77734375,58720256 +11,mlp.gate_proj,8,128,8,128,0.6974939703941345,56.888671875,58720256 +11,mlp.up_proj,2,32,8,128,34.43723678588867,17.5546875,58720256 +11,mlp.up_proj,2,64,8,128,40.302310943603516,15.77734375,58720256 +11,mlp.up_proj,2,128,8,128,45.65860366821289,14.888671875,58720256 +11,mlp.up_proj,3,32,8,128,14.736382484436035,24.5546875,58720256 +11,mlp.up_proj,3,64,8,128,17.26561737060547,22.77734375,58720256 +11,mlp.up_proj,3,128,8,128,19.60649871826172,21.888671875,58720256 +11,mlp.up_proj,4,32,8,128,6.871672630310059,31.5546875,58720256 +11,mlp.up_proj,4,64,8,128,8.060824394226074,29.77734375,58720256 +11,mlp.up_proj,4,128,8,128,9.152819633483887,28.888671875,58720256 +11,mlp.up_proj,8,32,8,128,0.4172927737236023,59.5546875,58720256 +11,mlp.up_proj,8,64,8,128,0.4885105788707733,57.77734375,58720256 +11,mlp.up_proj,8,128,8,128,0.5530235171318054,56.888671875,58720256 +11,mlp.down_proj,2,32,8,128,34.319190979003906,17.5546875,58720256 +11,mlp.down_proj,2,64,8,128,40.198219299316406,15.77734375,58720256 +11,mlp.down_proj,2,128,8,128,45.58870315551758,14.888671875,58720256 +11,mlp.down_proj,3,32,8,128,14.695785522460938,24.5546875,58720256 +11,mlp.down_proj,3,64,8,128,17.235937118530273,22.77734375,58720256 +11,mlp.down_proj,3,128,8,128,19.604066848754883,21.888671875,58720256 +11,mlp.down_proj,4,32,8,128,6.855496406555176,31.5546875,58720256 +11,mlp.down_proj,4,64,8,128,8.051091194152832,29.77734375,58720256 +11,mlp.down_proj,4,128,8,128,9.154850006103516,28.888671875,58720256 +11,mlp.down_proj,8,32,8,128,0.4161611795425415,59.5546875,58720256 +11,mlp.down_proj,8,64,8,128,0.48788949847221375,57.77734375,58720256 +11,mlp.down_proj,8,128,8,128,0.5536084175109863,56.888671875,58720256 +12,self_attn.q_proj,2,32,8,128,27.797035217285156,5.015625,16777216 +12,self_attn.q_proj,2,64,8,128,33.1560173034668,4.5078125,16777216 +12,self_attn.q_proj,2,128,8,128,38.72084045410156,4.25390625,16777216 +12,self_attn.q_proj,3,32,8,128,11.901005744934082,7.015625,16777216 +12,self_attn.q_proj,3,64,8,128,14.220873832702637,6.5078125,16777216 +12,self_attn.q_proj,3,128,8,128,16.708192825317383,6.25390625,16777216 +12,self_attn.q_proj,4,32,8,128,5.552829742431641,9.015625,16777216 +12,self_attn.q_proj,4,64,8,128,6.633696556091309,8.5078125,16777216 +12,self_attn.q_proj,4,128,8,128,7.798645496368408,8.25390625,16777216 +12,self_attn.q_proj,8,32,8,128,0.3362867534160614,17.015625,16777216 +12,self_attn.q_proj,8,64,8,128,0.40128186345100403,16.5078125,16777216 +12,self_attn.q_proj,8,128,8,128,0.46997424960136414,16.25390625,16777216 +12,self_attn.v_proj,2,32,8,128,7.917884826660156,1.25390625,4194304 +12,self_attn.v_proj,2,64,8,128,9.306337356567383,1.126953125,4194304 +12,self_attn.v_proj,2,128,8,128,10.597631454467773,1.0634765625,4194304 +12,self_attn.v_proj,3,32,8,128,3.388002872467041,1.75390625,4194304 +12,self_attn.v_proj,3,64,8,128,3.9839751720428467,1.626953125,4194304 +12,self_attn.v_proj,3,128,8,128,4.541279315948486,1.5634765625,4194304 +12,self_attn.v_proj,4,32,8,128,1.5801403522491455,2.25390625,4194304 +12,self_attn.v_proj,4,64,8,128,1.8590441942214966,2.126953125,4194304 +12,self_attn.v_proj,4,128,8,128,2.1197032928466797,2.0634765625,4194304 +12,self_attn.v_proj,8,32,8,128,0.09574701637029648,4.25390625,4194304 +12,self_attn.v_proj,8,64,8,128,0.11245526373386383,4.126953125,4194304 +12,self_attn.v_proj,8,128,8,128,0.12775303423404694,4.0634765625,4194304 +12,self_attn.k_proj,2,32,8,128,21.361074447631836,1.25390625,4194304 +12,self_attn.k_proj,2,64,8,128,25.365234375,1.126953125,4194304 +12,self_attn.k_proj,2,128,8,128,29.014301300048828,1.0634765625,4194304 +12,self_attn.k_proj,3,32,8,128,9.150544166564941,1.75390625,4194304 +12,self_attn.k_proj,3,64,8,128,10.876232147216797,1.626953125,4194304 +12,self_attn.k_proj,3,128,8,128,12.47270393371582,1.5634765625,4194304 +12,self_attn.k_proj,4,32,8,128,4.267239570617676,2.25390625,4194304 +12,self_attn.k_proj,4,64,8,128,5.073766708374023,2.126953125,4194304 +12,self_attn.k_proj,4,128,8,128,5.820850372314453,2.0634765625,4194304 +12,self_attn.k_proj,8,32,8,128,0.25835976004600525,4.25390625,4194304 +12,self_attn.k_proj,8,64,8,128,0.30696287751197815,4.126953125,4194304 +12,self_attn.k_proj,8,128,8,128,0.3508041501045227,4.0634765625,4194304 +12,self_attn.o_proj,2,32,8,128,17.01788330078125,5.015625,16777216 +12,self_attn.o_proj,2,64,8,128,19.974430084228516,4.5078125,16777216 +12,self_attn.o_proj,2,128,8,128,22.706222534179688,4.25390625,16777216 +12,self_attn.o_proj,3,32,8,128,7.286149978637695,7.015625,16777216 +12,self_attn.o_proj,3,64,8,128,8.566222190856934,6.5078125,16777216 +12,self_attn.o_proj,3,128,8,128,9.770204544067383,6.25390625,16777216 +12,self_attn.o_proj,4,32,8,128,3.401235818862915,9.015625,16777216 +12,self_attn.o_proj,4,64,8,128,4.003439426422119,8.5078125,16777216 +12,self_attn.o_proj,4,128,8,128,4.568297863006592,8.25390625,16777216 +12,self_attn.o_proj,8,32,8,128,0.20581938326358795,17.015625,16777216 +12,self_attn.o_proj,8,64,8,128,0.2419353723526001,16.5078125,16777216 +12,self_attn.o_proj,8,128,8,128,0.27537795901298523,16.25390625,16777216 +12,mlp.gate_proj,2,32,8,128,41.53411865234375,17.5546875,58720256 +12,mlp.gate_proj,2,64,8,128,49.370887756347656,15.77734375,58720256 +12,mlp.gate_proj,2,128,8,128,56.79405975341797,14.888671875,58720256 +12,mlp.gate_proj,3,32,8,128,17.814899444580078,24.5546875,58720256 +12,mlp.gate_proj,3,64,8,128,21.28843116760254,22.77734375,58720256 +12,mlp.gate_proj,3,128,8,128,24.656173706054688,21.888671875,58720256 +12,mlp.gate_proj,4,32,8,128,8.316516876220703,31.5546875,58720256 +12,mlp.gate_proj,4,64,8,128,9.935540199279785,29.77734375,58720256 +12,mlp.gate_proj,4,128,8,128,11.533153533935547,28.888671875,58720256 +12,mlp.gate_proj,8,32,8,128,0.504429280757904,59.5546875,58720256 +12,mlp.gate_proj,8,64,8,128,0.6030895709991455,57.77734375,58720256 +12,mlp.gate_proj,8,128,8,128,0.698143482208252,56.888671875,58720256 +12,mlp.up_proj,2,32,8,128,35.204017639160156,17.5546875,58720256 +12,mlp.up_proj,2,64,8,128,41.254249572753906,15.77734375,58720256 +12,mlp.up_proj,2,128,8,128,46.80075454711914,14.888671875,58720256 +12,mlp.up_proj,3,32,8,128,15.07442855834961,24.5546875,58720256 +12,mlp.up_proj,3,64,8,128,17.694063186645508,22.77734375,58720256 +12,mlp.up_proj,3,128,8,128,20.13836097717285,21.888671875,58720256 +12,mlp.up_proj,4,32,8,128,7.030857086181641,31.5546875,58720256 +12,mlp.up_proj,4,64,8,128,8.259942054748535,29.77734375,58720256 +12,mlp.up_proj,4,128,8,128,9.398458480834961,28.888671875,58720256 +12,mlp.up_proj,8,32,8,128,0.4267740547657013,59.5546875,58720256 +12,mlp.up_proj,8,64,8,128,0.500659704208374,57.77734375,58720256 +12,mlp.up_proj,8,128,8,128,0.5684226751327515,56.888671875,58720256 +12,mlp.down_proj,2,32,8,128,34.783531188964844,17.5546875,58720256 +12,mlp.down_proj,2,64,8,128,40.73786926269531,15.77734375,58720256 +12,mlp.down_proj,2,128,8,128,46.18748092651367,14.888671875,58720256 +12,mlp.down_proj,3,32,8,128,14.893275260925293,24.5546875,58720256 +12,mlp.down_proj,3,64,8,128,17.4713077545166,22.77734375,58720256 +12,mlp.down_proj,3,128,8,128,19.884532928466797,21.888671875,58720256 +12,mlp.down_proj,4,32,8,128,6.949922561645508,31.5546875,58720256 +12,mlp.down_proj,4,64,8,128,8.162311553955078,29.77734375,58720256 +12,mlp.down_proj,4,128,8,128,9.291436195373535,28.888671875,58720256 +12,mlp.down_proj,8,32,8,128,0.42190849781036377,59.5546875,58720256 +12,mlp.down_proj,8,64,8,128,0.4948751628398895,57.77734375,58720256 +12,mlp.down_proj,8,128,8,128,0.5620514750480652,56.888671875,58720256 +13,self_attn.q_proj,2,32,8,128,28.455102920532227,5.015625,16777216 +13,self_attn.q_proj,2,64,8,128,34.15635299682617,4.5078125,16777216 +13,self_attn.q_proj,2,128,8,128,40.503334045410156,4.25390625,16777216 +13,self_attn.q_proj,3,32,8,128,12.193653106689453,7.015625,16777216 +13,self_attn.q_proj,3,64,8,128,14.67825984954834,6.5078125,16777216 +13,self_attn.q_proj,3,128,8,128,17.746578216552734,6.25390625,16777216 +13,self_attn.q_proj,4,32,8,128,5.6908440589904785,9.015625,16777216 +13,self_attn.q_proj,4,64,8,128,6.849457740783691,8.5078125,16777216 +13,self_attn.q_proj,4,128,8,128,8.289314270019531,8.25390625,16777216 +13,self_attn.q_proj,8,32,8,128,0.34462979435920715,17.015625,16777216 +13,self_attn.q_proj,8,64,8,128,0.41416385769844055,16.5078125,16777216 +13,self_attn.q_proj,8,128,8,128,0.4993613660335541,16.25390625,16777216 +13,self_attn.v_proj,2,32,8,128,7.356478691101074,1.25390625,4194304 +13,self_attn.v_proj,2,64,8,128,8.620710372924805,1.126953125,4194304 +13,self_attn.v_proj,2,128,8,128,9.774435043334961,1.0634765625,4194304 +13,self_attn.v_proj,3,32,8,128,3.1462562084198,1.75390625,4194304 +13,self_attn.v_proj,3,64,8,128,3.6927409172058105,1.626953125,4194304 +13,self_attn.v_proj,3,128,8,128,4.191192150115967,1.5634765625,4194304 +13,self_attn.v_proj,4,32,8,128,1.4680594205856323,2.25390625,4194304 +13,self_attn.v_proj,4,64,8,128,1.7234028577804565,2.126953125,4194304 +13,self_attn.v_proj,4,128,8,128,1.9566049575805664,2.0634765625,4194304 +13,self_attn.v_proj,8,32,8,128,0.08892128616571426,4.25390625,4194304 +13,self_attn.v_proj,8,64,8,128,0.10421504825353622,4.126953125,4194304 +13,self_attn.v_proj,8,128,8,128,0.11794047057628632,4.0634765625,4194304 +13,self_attn.k_proj,2,32,8,128,22.65964698791504,1.25390625,4194304 +13,self_attn.k_proj,2,64,8,128,27.125751495361328,1.126953125,4194304 +13,self_attn.k_proj,2,128,8,128,31.02312469482422,1.0634765625,4194304 +13,self_attn.k_proj,3,32,8,128,9.764394760131836,1.75390625,4194304 +13,self_attn.k_proj,3,64,8,128,11.75832748413086,1.626953125,4194304 +13,self_attn.k_proj,3,128,8,128,13.608409881591797,1.5634765625,4194304 +13,self_attn.k_proj,4,32,8,128,4.558618545532227,2.25390625,4194304 +13,self_attn.k_proj,4,64,8,128,5.489178657531738,2.126953125,4194304 +13,self_attn.k_proj,4,128,8,128,6.357184410095215,2.0634765625,4194304 +13,self_attn.k_proj,8,32,8,128,0.276089608669281,4.25390625,4194304 +13,self_attn.k_proj,8,64,8,128,0.33191001415252686,4.126953125,4194304 +13,self_attn.k_proj,8,128,8,128,0.38306576013565063,4.0634765625,4194304 +13,self_attn.o_proj,2,32,8,128,16.59221839904785,5.015625,16777216 +13,self_attn.o_proj,2,64,8,128,19.419004440307617,4.5078125,16777216 +13,self_attn.o_proj,2,128,8,128,21.98301887512207,4.25390625,16777216 +13,self_attn.o_proj,3,32,8,128,7.103479862213135,7.015625,16777216 +13,self_attn.o_proj,3,64,8,128,8.322955131530762,6.5078125,16777216 +13,self_attn.o_proj,3,128,8,128,9.439129829406738,6.25390625,16777216 +13,self_attn.o_proj,4,32,8,128,3.3159475326538086,9.015625,16777216 +13,self_attn.o_proj,4,64,8,128,3.888988733291626,8.5078125,16777216 +13,self_attn.o_proj,4,128,8,128,4.416804313659668,8.25390625,16777216 +13,self_attn.o_proj,8,32,8,128,0.20073331892490387,17.015625,16777216 +13,self_attn.o_proj,8,64,8,128,0.23519814014434814,16.5078125,16777216 +13,self_attn.o_proj,8,128,8,128,0.2663339674472809,16.25390625,16777216 +13,mlp.gate_proj,2,32,8,128,41.41061782836914,17.5546875,58720256 +13,mlp.gate_proj,2,64,8,128,49.192222595214844,15.77734375,58720256 +13,mlp.gate_proj,2,128,8,128,56.58549118041992,14.888671875,58720256 +13,mlp.gate_proj,3,32,8,128,17.774005889892578,24.5546875,58720256 +13,mlp.gate_proj,3,64,8,128,21.244884490966797,22.77734375,58720256 +13,mlp.gate_proj,3,128,8,128,24.634387969970703,21.888671875,58720256 +13,mlp.gate_proj,4,32,8,128,8.300552368164062,31.5546875,58720256 +13,mlp.gate_proj,4,64,8,128,9.920539855957031,29.77734375,58720256 +13,mlp.gate_proj,4,128,8,128,11.531843185424805,28.888671875,58720256 +13,mlp.gate_proj,8,32,8,128,0.5033843517303467,59.5546875,58720256 +13,mlp.gate_proj,8,64,8,128,0.6022018790245056,57.77734375,58720256 +13,mlp.gate_proj,8,128,8,128,0.6981767416000366,56.888671875,58720256 +13,mlp.up_proj,2,32,8,128,35.32726287841797,17.5546875,58720256 +13,mlp.up_proj,2,64,8,128,41.40066909790039,15.77734375,58720256 +13,mlp.up_proj,2,128,8,128,46.914676666259766,14.888671875,58720256 +13,mlp.up_proj,3,32,8,128,15.125348091125488,24.5546875,58720256 +13,mlp.up_proj,3,64,8,128,17.746788024902344,22.77734375,58720256 +13,mlp.up_proj,3,128,8,128,20.175251007080078,21.888671875,58720256 +13,mlp.up_proj,4,32,8,128,7.054731845855713,31.5546875,58720256 +13,mlp.up_proj,4,64,8,128,8.281837463378906,29.77734375,58720256 +13,mlp.up_proj,4,128,8,128,9.415802955627441,28.888671875,58720256 +13,mlp.up_proj,8,32,8,128,0.4282049834728241,59.5546875,58720256 +13,mlp.up_proj,8,64,8,128,0.5020537972450256,57.77734375,58720256 +13,mlp.up_proj,8,128,8,128,0.5695896148681641,56.888671875,58720256 +13,mlp.down_proj,2,32,8,128,34.76760482788086,17.5546875,58720256 +13,mlp.down_proj,2,64,8,128,40.688507080078125,15.77734375,58720256 +13,mlp.down_proj,2,128,8,128,46.097007751464844,14.888671875,58720256 +13,mlp.down_proj,3,32,8,128,14.875645637512207,24.5546875,58720256 +13,mlp.down_proj,3,64,8,128,17.419208526611328,22.77734375,58720256 +13,mlp.down_proj,3,128,8,128,19.770586013793945,21.888671875,58720256 +13,mlp.down_proj,4,32,8,128,6.938654899597168,31.5546875,58720256 +13,mlp.down_proj,4,64,8,128,8.134733200073242,29.77734375,58720256 +13,mlp.down_proj,4,128,8,128,9.237409591674805,28.888671875,58720256 +13,mlp.down_proj,8,32,8,128,0.4213395118713379,59.5546875,58720256 +13,mlp.down_proj,8,64,8,128,0.4931293725967407,57.77734375,58720256 +13,mlp.down_proj,8,128,8,128,0.5583329200744629,56.888671875,58720256 +14,self_attn.q_proj,2,32,8,128,27.587013244628906,5.015625,16777216 +14,self_attn.q_proj,2,64,8,128,32.99413299560547,4.5078125,16777216 +14,self_attn.q_proj,2,128,8,128,39.064212799072266,4.25390625,16777216 +14,self_attn.q_proj,3,32,8,128,11.845685958862305,7.015625,16777216 +14,self_attn.q_proj,3,64,8,128,14.2334566116333,6.5078125,16777216 +14,self_attn.q_proj,3,128,8,128,17.1420955657959,6.25390625,16777216 +14,self_attn.q_proj,4,32,8,128,5.529016017913818,9.015625,16777216 +14,self_attn.q_proj,4,64,8,128,6.6553168296813965,8.5078125,16777216 +14,self_attn.q_proj,4,128,8,128,8.030698776245117,8.25390625,16777216 +14,self_attn.q_proj,8,32,8,128,0.3348241448402405,17.015625,16777216 +14,self_attn.q_proj,8,64,8,128,0.40260857343673706,16.5078125,16777216 +14,self_attn.q_proj,8,128,8,128,0.4838525056838989,16.25390625,16777216 +14,self_attn.v_proj,2,32,8,128,7.427489757537842,1.25390625,4194304 +14,self_attn.v_proj,2,64,8,128,8.76595687866211,1.126953125,4194304 +14,self_attn.v_proj,2,128,8,128,9.996440887451172,1.0634765625,4194304 +14,self_attn.v_proj,3,32,8,128,3.1831064224243164,1.75390625,4194304 +14,self_attn.v_proj,3,64,8,128,3.7666611671447754,1.626953125,4194304 +14,self_attn.v_proj,3,128,8,128,4.310457706451416,1.5634765625,4194304 +14,self_attn.v_proj,4,32,8,128,1.485533595085144,2.25390625,4194304 +14,self_attn.v_proj,4,64,8,128,1.7582796812057495,2.126953125,4194304 +14,self_attn.v_proj,4,128,8,128,2.0138096809387207,2.0634765625,4194304 +14,self_attn.v_proj,8,32,8,128,0.09001454710960388,4.25390625,4194304 +14,self_attn.v_proj,8,64,8,128,0.10635726153850555,4.126953125,4194304 +14,self_attn.v_proj,8,128,8,128,0.12134986370801926,4.0634765625,4194304 +14,self_attn.k_proj,2,32,8,128,22.756763458251953,1.25390625,4194304 +14,self_attn.k_proj,2,64,8,128,27.260150909423828,1.126953125,4194304 +14,self_attn.k_proj,2,128,8,128,31.308923721313477,1.0634765625,4194304 +14,self_attn.k_proj,3,32,8,128,9.844118118286133,1.75390625,4194304 +14,self_attn.k_proj,3,64,8,128,11.942048072814941,1.626953125,4194304 +14,self_attn.k_proj,3,128,8,128,13.979451179504395,1.5634765625,4194304 +14,self_attn.k_proj,4,32,8,128,4.595693588256836,2.25390625,4194304 +14,self_attn.k_proj,4,64,8,128,5.579875946044922,2.126953125,4194304 +14,self_attn.k_proj,4,128,8,128,6.5451788902282715,2.0634765625,4194304 +14,self_attn.k_proj,8,32,8,128,0.2780858278274536,4.25390625,4194304 +14,self_attn.k_proj,8,64,8,128,0.33744677901268005,4.126953125,4194304 +14,self_attn.k_proj,8,128,8,128,0.3945960998535156,4.0634765625,4194304 +14,self_attn.o_proj,2,32,8,128,16.59701156616211,5.015625,16777216 +14,self_attn.o_proj,2,64,8,128,19.49690055847168,4.5078125,16777216 +14,self_attn.o_proj,2,128,8,128,22.176523208618164,4.25390625,16777216 +14,self_attn.o_proj,3,32,8,128,7.107360363006592,7.015625,16777216 +14,self_attn.o_proj,3,64,8,128,8.370301246643066,6.5078125,16777216 +14,self_attn.o_proj,3,128,8,128,9.571660995483398,6.25390625,16777216 +14,self_attn.o_proj,4,32,8,128,3.3155407905578613,9.015625,16777216 +14,self_attn.o_proj,4,64,8,128,3.9058568477630615,8.5078125,16777216 +14,self_attn.o_proj,4,128,8,128,4.466561794281006,8.25390625,16777216 +14,self_attn.o_proj,8,32,8,128,0.2006497085094452,17.015625,16777216 +14,self_attn.o_proj,8,64,8,128,0.23609234392642975,16.5078125,16777216 +14,self_attn.o_proj,8,128,8,128,0.26913222670555115,16.25390625,16777216 +14,mlp.gate_proj,2,32,8,128,42.471717834472656,17.5546875,58720256 +14,mlp.gate_proj,2,64,8,128,50.52969741821289,15.77734375,58720256 +14,mlp.gate_proj,2,128,8,128,58.2453498840332,14.888671875,58720256 +14,mlp.gate_proj,3,32,8,128,18.246932983398438,24.5546875,58720256 +14,mlp.gate_proj,3,64,8,128,21.87453842163086,22.77734375,58720256 +14,mlp.gate_proj,3,128,8,128,25.458263397216797,21.888671875,58720256 +14,mlp.gate_proj,4,32,8,128,8.51923656463623,31.5546875,58720256 +14,mlp.gate_proj,4,64,8,128,10.221134185791016,29.77734375,58720256 +14,mlp.gate_proj,4,128,8,128,11.91765022277832,28.888671875,58720256 +14,mlp.gate_proj,8,32,8,128,0.5168219804763794,59.5546875,58720256 +14,mlp.gate_proj,8,64,8,128,0.6202598214149475,57.77734375,58720256 +14,mlp.gate_proj,8,128,8,128,0.7216306924819946,56.888671875,58720256 +14,mlp.up_proj,2,32,8,128,35.258724212646484,17.5546875,58720256 +14,mlp.up_proj,2,64,8,128,41.309757232666016,15.77734375,58720256 +14,mlp.up_proj,2,128,8,128,46.802860260009766,14.888671875,58720256 +14,mlp.up_proj,3,32,8,128,15.092498779296875,24.5546875,58720256 +14,mlp.up_proj,3,64,8,128,17.700422286987305,22.77734375,58720256 +14,mlp.up_proj,3,128,8,128,20.120548248291016,21.888671875,58720256 +14,mlp.up_proj,4,32,8,128,7.039735317230225,31.5546875,58720256 +14,mlp.up_proj,4,64,8,128,8.264071464538574,29.77734375,58720256 +14,mlp.up_proj,4,128,8,128,9.390637397766113,28.888671875,58720256 +14,mlp.up_proj,8,32,8,128,0.42733946442604065,59.5546875,58720256 +14,mlp.up_proj,8,64,8,128,0.5009170174598694,57.77734375,58720256 +14,mlp.up_proj,8,128,8,128,0.5678375363349915,56.888671875,58720256 +14,mlp.down_proj,2,32,8,128,34.672943115234375,17.5546875,58720256 +14,mlp.down_proj,2,64,8,128,40.55575180053711,15.77734375,58720256 +14,mlp.down_proj,2,128,8,128,45.92131042480469,14.888671875,58720256 +14,mlp.down_proj,3,32,8,128,14.835634231567383,24.5546875,58720256 +14,mlp.down_proj,3,64,8,128,17.362468719482422,22.77734375,58720256 +14,mlp.down_proj,3,128,8,128,19.68979263305664,21.888671875,58720256 +14,mlp.down_proj,4,32,8,128,6.918953895568848,31.5546875,58720256 +14,mlp.down_proj,4,64,8,128,8.107073783874512,29.77734375,58720256 +14,mlp.down_proj,4,128,8,128,9.191179275512695,28.888671875,58720256 +14,mlp.down_proj,8,32,8,128,0.42015886306762695,59.5546875,58720256 +14,mlp.down_proj,8,64,8,128,0.4913758337497711,57.77734375,58720256 +14,mlp.down_proj,8,128,8,128,0.5558993220329285,56.888671875,58720256 +15,self_attn.q_proj,2,32,8,128,31.22846031188965,5.015625,16777216 +15,self_attn.q_proj,2,64,8,128,37.545265197753906,4.5078125,16777216 +15,self_attn.q_proj,2,128,8,128,44.28113555908203,4.25390625,16777216 +15,self_attn.q_proj,3,32,8,128,13.450045585632324,7.015625,16777216 +15,self_attn.q_proj,3,64,8,128,16.308645248413086,6.5078125,16777216 +15,self_attn.q_proj,3,128,8,128,19.853605270385742,6.25390625,16777216 +15,self_attn.q_proj,4,32,8,128,6.278322219848633,9.015625,16777216 +15,self_attn.q_proj,4,64,8,128,7.619574069976807,8.5078125,16777216 +15,self_attn.q_proj,4,128,8,128,9.286111831665039,8.25390625,16777216 +15,self_attn.q_proj,8,32,8,128,0.3800128698348999,17.015625,16777216 +15,self_attn.q_proj,8,64,8,128,0.4606950283050537,16.5078125,16777216 +15,self_attn.q_proj,8,128,8,128,0.5600678324699402,16.25390625,16777216 +15,self_attn.v_proj,2,32,8,128,8.195611000061035,1.25390625,4194304 +15,self_attn.v_proj,2,64,8,128,9.712187767028809,1.126953125,4194304 +15,self_attn.v_proj,2,128,8,128,11.115867614746094,1.0634765625,4194304 +15,self_attn.v_proj,3,32,8,128,3.513174533843994,1.75390625,4194304 +15,self_attn.v_proj,3,64,8,128,4.172936916351318,1.626953125,4194304 +15,self_attn.v_proj,3,128,8,128,4.791966438293457,1.5634765625,4194304 +15,self_attn.v_proj,4,32,8,128,1.6392368078231812,2.25390625,4194304 +15,self_attn.v_proj,4,64,8,128,1.9486216306686401,2.126953125,4194304 +15,self_attn.v_proj,4,128,8,128,2.2363219261169434,2.0634765625,4194304 +15,self_attn.v_proj,8,32,8,128,0.09922745078802109,4.25390625,4194304 +15,self_attn.v_proj,8,64,8,128,0.1178261861205101,4.126953125,4194304 +15,self_attn.v_proj,8,128,8,128,0.13481451570987701,4.0634765625,4194304 +15,self_attn.k_proj,2,32,8,128,21.68100357055664,1.25390625,4194304 +15,self_attn.k_proj,2,64,8,128,25.818817138671875,1.126953125,4194304 +15,self_attn.k_proj,2,128,8,128,29.628772735595703,1.0634765625,4194304 +15,self_attn.k_proj,3,32,8,128,9.294938087463379,1.75390625,4194304 +15,self_attn.k_proj,3,64,8,128,11.114538192749023,1.626953125,4194304 +15,self_attn.k_proj,3,128,8,128,12.832574844360352,1.5634765625,4194304 +15,self_attn.k_proj,4,32,8,128,4.334258079528809,2.25390625,4194304 +15,self_attn.k_proj,4,64,8,128,5.184545993804932,2.126953125,4194304 +15,self_attn.k_proj,4,128,8,128,5.986537456512451,2.0634765625,4194304 +15,self_attn.k_proj,8,32,8,128,0.26252487301826477,4.25390625,4194304 +15,self_attn.k_proj,8,64,8,128,0.3136964738368988,4.126953125,4194304 +15,self_attn.k_proj,8,128,8,128,0.3608015477657318,4.0634765625,4194304 +15,self_attn.o_proj,2,32,8,128,17.10445213317871,5.015625,16777216 +15,self_attn.o_proj,2,64,8,128,20.03852653503418,4.5078125,16777216 +15,self_attn.o_proj,2,128,8,128,22.755416870117188,4.25390625,16777216 +15,self_attn.o_proj,3,32,8,128,7.328201770782471,7.015625,16777216 +15,self_attn.o_proj,3,64,8,128,8.604158401489258,6.5078125,16777216 +15,self_attn.o_proj,3,128,8,128,9.800718307495117,6.25390625,16777216 +15,self_attn.o_proj,4,32,8,128,3.4218642711639404,9.015625,16777216 +15,self_attn.o_proj,4,64,8,128,4.019123554229736,8.5078125,16777216 +15,self_attn.o_proj,4,128,8,128,4.583128452301025,8.25390625,16777216 +15,self_attn.o_proj,8,32,8,128,0.20704436302185059,17.015625,16777216 +15,self_attn.o_proj,8,64,8,128,0.24297377467155457,16.5078125,16777216 +15,self_attn.o_proj,8,128,8,128,0.27623143792152405,16.25390625,16777216 +15,mlp.gate_proj,2,32,8,128,43.30869674682617,17.5546875,58720256 +15,mlp.gate_proj,2,64,8,128,51.51576232910156,15.77734375,58720256 +15,mlp.gate_proj,2,128,8,128,59.417781829833984,14.888671875,58720256 +15,mlp.gate_proj,3,32,8,128,18.58449935913086,24.5546875,58720256 +15,mlp.gate_proj,3,64,8,128,22.265291213989258,22.77734375,58720256 +15,mlp.gate_proj,3,128,8,128,25.90117645263672,21.888671875,58720256 +15,mlp.gate_proj,4,32,8,128,8.673805236816406,31.5546875,58720256 +15,mlp.gate_proj,4,64,8,128,10.391359329223633,29.77734375,58720256 +15,mlp.gate_proj,4,128,8,128,12.11255168914795,28.888671875,58720256 +15,mlp.gate_proj,8,32,8,128,0.5262082815170288,59.5546875,58720256 +15,mlp.gate_proj,8,64,8,128,0.6309165954589844,57.77734375,58720256 +15,mlp.gate_proj,8,128,8,128,0.7334462404251099,56.888671875,58720256 +15,mlp.up_proj,2,32,8,128,34.962894439697266,17.5546875,58720256 +15,mlp.up_proj,2,64,8,128,40.91743469238281,15.77734375,58720256 +15,mlp.up_proj,2,128,8,128,46.34127426147461,14.888671875,58720256 +15,mlp.up_proj,3,32,8,128,14.958983421325684,24.5546875,58720256 +15,mlp.up_proj,3,64,8,128,17.522083282470703,22.77734375,58720256 +15,mlp.up_proj,3,128,8,128,19.887866973876953,21.888671875,58720256 +15,mlp.up_proj,4,32,8,128,6.9775309562683105,31.5546875,58720256 +15,mlp.up_proj,4,64,8,128,8.180379867553711,29.77734375,58720256 +15,mlp.up_proj,4,128,8,128,9.279431343078613,28.888671875,58720256 +15,mlp.up_proj,8,32,8,128,0.42360246181488037,59.5546875,58720256 +15,mlp.up_proj,8,64,8,128,0.4957735240459442,57.77734375,58720256 +15,mlp.up_proj,8,128,8,128,0.5611969828605652,56.888671875,58720256 +15,mlp.down_proj,2,32,8,128,34.55253219604492,17.5546875,58720256 +15,mlp.down_proj,2,64,8,128,40.40397644042969,15.77734375,58720256 +15,mlp.down_proj,2,128,8,128,45.74128723144531,14.888671875,58720256 +15,mlp.down_proj,3,32,8,128,14.785499572753906,24.5546875,58720256 +15,mlp.down_proj,3,64,8,128,17.305683135986328,22.77734375,58720256 +15,mlp.down_proj,3,128,8,128,19.62993812561035,21.888671875,58720256 +15,mlp.down_proj,4,32,8,128,6.896656513214111,31.5546875,58720256 +15,mlp.down_proj,4,64,8,128,8.082080841064453,29.77734375,58720256 +15,mlp.down_proj,4,128,8,128,9.17086410522461,28.888671875,58720256 +15,mlp.down_proj,8,32,8,128,0.4186405837535858,59.5546875,58720256 +15,mlp.down_proj,8,64,8,128,0.48986107110977173,57.77734375,58720256 +15,mlp.down_proj,8,128,8,128,0.5544942021369934,56.888671875,58720256 +16,self_attn.q_proj,2,32,8,128,29.474246978759766,5.015625,16777216 +16,self_attn.q_proj,2,64,8,128,35.29362106323242,4.5078125,16777216 +16,self_attn.q_proj,2,128,8,128,41.4552116394043,4.25390625,16777216 +16,self_attn.q_proj,3,32,8,128,12.64678955078125,7.015625,16777216 +16,self_attn.q_proj,3,64,8,128,15.21742057800293,6.5078125,16777216 +16,self_attn.q_proj,3,128,8,128,18.16267967224121,6.25390625,16777216 +16,self_attn.q_proj,4,32,8,128,5.903792381286621,9.015625,16777216 +16,self_attn.q_proj,4,64,8,128,7.106362342834473,8.5078125,16777216 +16,self_attn.q_proj,4,128,8,128,8.49470043182373,8.25390625,16777216 +16,self_attn.q_proj,8,32,8,128,0.35732704401016235,17.015625,16777216 +16,self_attn.q_proj,8,64,8,128,0.42970114946365356,16.5078125,16777216 +16,self_attn.q_proj,8,128,8,128,0.5121365785598755,16.25390625,16777216 +16,self_attn.v_proj,2,32,8,128,7.619043350219727,1.25390625,4194304 +16,self_attn.v_proj,2,64,8,128,8.975646018981934,1.126953125,4194304 +16,self_attn.v_proj,2,128,8,128,10.22091293334961,1.0634765625,4194304 +16,self_attn.v_proj,3,32,8,128,3.2660443782806396,1.75390625,4194304 +16,self_attn.v_proj,3,64,8,128,3.863450288772583,1.626953125,4194304 +16,self_attn.v_proj,3,128,8,128,4.413819313049316,1.5634765625,4194304 +16,self_attn.v_proj,4,32,8,128,1.5243834257125854,2.25390625,4194304 +16,self_attn.v_proj,4,64,8,128,1.8025342226028442,2.126953125,4194304 +16,self_attn.v_proj,4,128,8,128,2.0616419315338135,2.0634765625,4194304 +16,self_attn.v_proj,8,32,8,128,0.09225548058748245,4.25390625,4194304 +16,self_attn.v_proj,8,64,8,128,0.10906665027141571,4.126953125,4194304 +16,self_attn.v_proj,8,128,8,128,0.12427056580781937,4.0634765625,4194304 +16,self_attn.k_proj,2,32,8,128,22.233821868896484,1.25390625,4194304 +16,self_attn.k_proj,2,64,8,128,26.606412887573242,1.126953125,4194304 +16,self_attn.k_proj,2,128,8,128,30.6308536529541,1.0634765625,4194304 +16,self_attn.k_proj,3,32,8,128,9.550228118896484,1.75390625,4194304 +16,self_attn.k_proj,3,64,8,128,11.499176979064941,1.626953125,4194304 +16,self_attn.k_proj,3,128,8,128,13.366073608398438,1.5634765625,4194304 +16,self_attn.k_proj,4,32,8,128,4.457359790802002,2.25390625,4194304 +16,self_attn.k_proj,4,64,8,128,5.366395473480225,2.126953125,4194304 +16,self_attn.k_proj,4,128,8,128,6.241940975189209,2.0634765625,4194304 +16,self_attn.k_proj,8,32,8,128,0.26972928643226624,4.25390625,4194304 +16,self_attn.k_proj,8,64,8,128,0.3245899975299835,4.126953125,4194304 +16,self_attn.k_proj,8,128,8,128,0.37611255049705505,4.0634765625,4194304 +16,self_attn.o_proj,2,32,8,128,16.676380157470703,5.015625,16777216 +16,self_attn.o_proj,2,64,8,128,19.54994010925293,4.5078125,16777216 +16,self_attn.o_proj,2,128,8,128,22.219879150390625,4.25390625,16777216 +16,self_attn.o_proj,3,32,8,128,7.153149127960205,7.015625,16777216 +16,self_attn.o_proj,3,64,8,128,8.417523384094238,6.5078125,16777216 +16,self_attn.o_proj,3,128,8,128,9.619704246520996,6.25390625,16777216 +16,self_attn.o_proj,4,32,8,128,3.341057538986206,9.015625,16777216 +16,self_attn.o_proj,4,64,8,128,3.9357497692108154,8.5078125,16777216 +16,self_attn.o_proj,4,128,8,128,4.503844261169434,8.25390625,16777216 +16,self_attn.o_proj,8,32,8,128,0.20220522582530975,17.015625,16777216 +16,self_attn.o_proj,8,64,8,128,0.2378915548324585,16.5078125,16777216 +16,self_attn.o_proj,8,128,8,128,0.27148276567459106,16.25390625,16777216 +16,mlp.gate_proj,2,32,8,128,44.23975372314453,17.5546875,58720256 +16,mlp.gate_proj,2,64,8,128,52.61695098876953,15.77734375,58720256 +16,mlp.gate_proj,2,128,8,128,60.71185302734375,14.888671875,58720256 +16,mlp.gate_proj,3,32,8,128,18.9517822265625,24.5546875,58720256 +16,mlp.gate_proj,3,64,8,128,22.661422729492188,22.77734375,58720256 +16,mlp.gate_proj,3,128,8,128,26.288793563842773,21.888671875,58720256 +16,mlp.gate_proj,4,32,8,128,8.841670989990234,31.5546875,58720256 +16,mlp.gate_proj,4,64,8,128,10.569990158081055,29.77734375,58720256 +16,mlp.gate_proj,4,128,8,128,12.27625846862793,28.888671875,58720256 +16,mlp.gate_proj,8,32,8,128,0.5365695953369141,59.5546875,58720256 +16,mlp.gate_proj,8,64,8,128,0.6417803168296814,57.77734375,58720256 +16,mlp.gate_proj,8,128,8,128,0.7433431148529053,56.888671875,58720256 +16,mlp.up_proj,2,32,8,128,34.45537185668945,17.5546875,58720256 +16,mlp.up_proj,2,64,8,128,40.281959533691406,15.77734375,58720256 +16,mlp.up_proj,2,128,8,128,45.5876579284668,14.888671875,58720256 +16,mlp.up_proj,3,32,8,128,14.73884105682373,24.5546875,58720256 +16,mlp.up_proj,3,64,8,128,17.248424530029297,22.77734375,58720256 +16,mlp.up_proj,3,128,8,128,19.545612335205078,21.888671875,58720256 +16,mlp.up_proj,4,32,8,128,6.873931407928467,31.5546875,58720256 +16,mlp.up_proj,4,64,8,128,8.05165958404541,29.77734375,58720256 +16,mlp.up_proj,4,128,8,128,9.124467849731445,28.888671875,58720256 +16,mlp.up_proj,8,32,8,128,0.4174143075942993,59.5546875,58720256 +16,mlp.up_proj,8,64,8,128,0.4879724681377411,57.77734375,58720256 +16,mlp.up_proj,8,128,8,128,0.5514988899230957,56.888671875,58720256 +16,mlp.down_proj,2,32,8,128,34.087093353271484,17.5546875,58720256 +16,mlp.down_proj,2,64,8,128,39.82257843017578,15.77734375,58720256 +16,mlp.down_proj,2,128,8,128,45.07321548461914,14.888671875,58720256 +16,mlp.down_proj,3,32,8,128,14.582074165344238,24.5546875,58720256 +16,mlp.down_proj,3,64,8,128,17.055126190185547,22.77734375,58720256 +16,mlp.down_proj,3,128,8,128,19.3234806060791,21.888671875,58720256 +16,mlp.down_proj,4,32,8,128,6.801875591278076,31.5546875,58720256 +16,mlp.down_proj,4,64,8,128,7.966272354125977,29.77734375,58720256 +16,mlp.down_proj,4,128,8,128,9.022490501403809,28.888671875,58720256 +16,mlp.down_proj,8,32,8,128,0.41310203075408936,59.5546875,58720256 +16,mlp.down_proj,8,64,8,128,0.48285144567489624,57.77734375,58720256 +16,mlp.down_proj,8,128,8,128,0.5457636117935181,56.888671875,58720256 +17,self_attn.q_proj,2,32,8,128,29.95086669921875,5.015625,16777216 +17,self_attn.q_proj,2,64,8,128,35.915184020996094,4.5078125,16777216 +17,self_attn.q_proj,2,128,8,128,42.061954498291016,4.25390625,16777216 +17,self_attn.q_proj,3,32,8,128,12.843481063842773,7.015625,16777216 +17,self_attn.q_proj,3,64,8,128,15.46299934387207,6.5078125,16777216 +17,self_attn.q_proj,3,128,8,128,18.46314811706543,6.25390625,16777216 +17,self_attn.q_proj,4,32,8,128,5.9944353103637695,9.015625,16777216 +17,self_attn.q_proj,4,64,8,128,7.216658592224121,8.5078125,16777216 +17,self_attn.q_proj,4,128,8,128,8.6205415725708,8.25390625,16777216 +17,self_attn.q_proj,8,32,8,128,0.36293044686317444,17.015625,16777216 +17,self_attn.q_proj,8,64,8,128,0.43635016679763794,16.5078125,16777216 +17,self_attn.q_proj,8,128,8,128,0.5194459557533264,16.25390625,16777216 +17,self_attn.v_proj,2,32,8,128,8.283431053161621,1.25390625,4194304 +17,self_attn.v_proj,2,64,8,128,9.74553108215332,1.126953125,4194304 +17,self_attn.v_proj,2,128,8,128,11.077119827270508,1.0634765625,4194304 +17,self_attn.v_proj,3,32,8,128,3.543118715286255,1.75390625,4194304 +17,self_attn.v_proj,3,64,8,128,4.172053813934326,1.626953125,4194304 +17,self_attn.v_proj,3,128,8,128,4.753373146057129,1.5634765625,4194304 +17,self_attn.v_proj,4,32,8,128,1.6532543897628784,2.25390625,4194304 +17,self_attn.v_proj,4,64,8,128,1.9463982582092285,2.126953125,4194304 +17,self_attn.v_proj,4,128,8,128,2.2174108028411865,2.0634765625,4194304 +17,self_attn.v_proj,8,32,8,128,0.10011625289916992,4.25390625,4194304 +17,self_attn.v_proj,8,64,8,128,0.1177472323179245,4.126953125,4194304 +17,self_attn.v_proj,8,128,8,128,0.1336275041103363,4.0634765625,4194304 +17,self_attn.k_proj,2,32,8,128,22.041820526123047,1.25390625,4194304 +17,self_attn.k_proj,2,64,8,128,26.287086486816406,1.126953125,4194304 +17,self_attn.k_proj,2,128,8,128,30.17961883544922,1.0634765625,4194304 +17,self_attn.k_proj,3,32,8,128,9.469921112060547,1.75390625,4194304 +17,self_attn.k_proj,3,64,8,128,11.347302436828613,1.626953125,4194304 +17,self_attn.k_proj,3,128,8,128,13.12747859954834,1.5634765625,4194304 +17,self_attn.k_proj,4,32,8,128,4.418277740478516,2.25390625,4194304 +17,self_attn.k_proj,4,64,8,128,5.2947678565979,2.126953125,4194304 +17,self_attn.k_proj,4,128,8,128,6.12764835357666,2.0634765625,4194304 +17,self_attn.k_proj,8,32,8,128,0.2674914300441742,4.25390625,4194304 +17,self_attn.k_proj,8,64,8,128,0.32020363211631775,4.126953125,4194304 +17,self_attn.k_proj,8,128,8,128,0.36931097507476807,4.0634765625,4194304 +17,self_attn.o_proj,2,32,8,128,17.242305755615234,5.015625,16777216 +17,self_attn.o_proj,2,64,8,128,20.16529083251953,4.5078125,16777216 +17,self_attn.o_proj,2,128,8,128,22.855175018310547,4.25390625,16777216 +17,self_attn.o_proj,3,32,8,128,7.386338710784912,7.015625,16777216 +17,self_attn.o_proj,3,64,8,128,8.655227661132812,6.5078125,16777216 +17,self_attn.o_proj,3,128,8,128,9.840927124023438,6.25390625,16777216 +17,self_attn.o_proj,4,32,8,128,3.448436975479126,9.015625,16777216 +17,self_attn.o_proj,4,64,8,128,4.043917655944824,8.5078125,16777216 +17,self_attn.o_proj,4,128,8,128,4.603419780731201,8.25390625,16777216 +17,self_attn.o_proj,8,32,8,128,0.20867519080638885,17.015625,16777216 +17,self_attn.o_proj,8,64,8,128,0.24436865746974945,16.5078125,16777216 +17,self_attn.o_proj,8,128,8,128,0.2774476110935211,16.25390625,16777216 +17,mlp.gate_proj,2,32,8,128,44.704383850097656,17.5546875,58720256 +17,mlp.gate_proj,2,64,8,128,53.177764892578125,15.77734375,58720256 +17,mlp.gate_proj,2,128,8,128,61.37278747558594,14.888671875,58720256 +17,mlp.gate_proj,3,32,8,128,19.149038314819336,24.5546875,58720256 +17,mlp.gate_proj,3,64,8,128,22.900238037109375,22.77734375,58720256 +17,mlp.gate_proj,3,128,8,128,26.586654663085938,21.888671875,58720256 +17,mlp.gate_proj,4,32,8,128,8.933100700378418,31.5546875,58720256 +17,mlp.gate_proj,4,64,8,128,10.684648513793945,29.77734375,58720256 +17,mlp.gate_proj,4,128,8,128,12.409584045410156,28.888671875,58720256 +17,mlp.gate_proj,8,32,8,128,0.5422821640968323,59.5546875,58720256 +17,mlp.gate_proj,8,64,8,128,0.648876428604126,57.77734375,58720256 +17,mlp.gate_proj,8,128,8,128,0.7515956163406372,56.888671875,58720256 +17,mlp.up_proj,2,32,8,128,34.31968307495117,17.5546875,58720256 +17,mlp.up_proj,2,64,8,128,40.0992317199707,15.77734375,58720256 +17,mlp.up_proj,2,128,8,128,45.37278747558594,14.888671875,58720256 +17,mlp.up_proj,3,32,8,128,14.68588924407959,24.5546875,58720256 +17,mlp.up_proj,3,64,8,128,17.172910690307617,22.77734375,58720256 +17,mlp.up_proj,3,128,8,128,19.44542694091797,21.888671875,58720256 +17,mlp.up_proj,4,32,8,128,6.848670959472656,31.5546875,58720256 +17,mlp.up_proj,4,64,8,128,8.01949691772461,29.77734375,58720256 +17,mlp.up_proj,4,128,8,128,9.078986167907715,28.888671875,58720256 +17,mlp.up_proj,8,32,8,128,0.41583073139190674,59.5546875,58720256 +17,mlp.up_proj,8,64,8,128,0.48593294620513916,57.77734375,58720256 +17,mlp.up_proj,8,128,8,128,0.5488820672035217,56.888671875,58720256 +17,mlp.down_proj,2,32,8,128,34.046875,17.5546875,58720256 +17,mlp.down_proj,2,64,8,128,39.77503967285156,15.77734375,58720256 +17,mlp.down_proj,2,128,8,128,45.01134490966797,14.888671875,58720256 +17,mlp.down_proj,3,32,8,128,14.575090408325195,24.5546875,58720256 +17,mlp.down_proj,3,64,8,128,17.057998657226562,22.77734375,58720256 +17,mlp.down_proj,3,128,8,128,19.339128494262695,21.888671875,58720256 +17,mlp.down_proj,4,32,8,128,6.8003458976745605,31.5546875,58720256 +17,mlp.down_proj,4,64,8,128,7.969241619110107,29.77734375,58720256 +17,mlp.down_proj,4,128,8,128,9.040688514709473,28.888671875,58720256 +17,mlp.down_proj,8,32,8,128,0.4129701852798462,59.5546875,58720256 +17,mlp.down_proj,8,64,8,128,0.48303285241127014,57.77734375,58720256 +17,mlp.down_proj,8,128,8,128,0.5467910170555115,56.888671875,58720256 +18,self_attn.q_proj,2,32,8,128,29.042482376098633,5.015625,16777216 +18,self_attn.q_proj,2,64,8,128,34.55720520019531,4.5078125,16777216 +18,self_attn.q_proj,2,128,8,128,40.68512725830078,4.25390625,16777216 +18,self_attn.q_proj,3,32,8,128,12.433228492736816,7.015625,16777216 +18,self_attn.q_proj,3,64,8,128,14.817197799682617,6.5078125,16777216 +18,self_attn.q_proj,3,128,8,128,17.61870574951172,6.25390625,16777216 +18,self_attn.q_proj,4,32,8,128,5.8041887283325195,9.015625,16777216 +18,self_attn.q_proj,4,64,8,128,6.9159955978393555,8.5078125,16777216 +18,self_attn.q_proj,4,128,8,128,8.22462272644043,8.25390625,16777216 +18,self_attn.q_proj,8,32,8,128,0.3514246344566345,17.015625,16777216 +18,self_attn.q_proj,8,64,8,128,0.4182252585887909,16.5078125,16777216 +18,self_attn.q_proj,8,128,8,128,0.49554330110549927,16.25390625,16777216 +18,self_attn.v_proj,2,32,8,128,7.6135687828063965,1.25390625,4194304 +18,self_attn.v_proj,2,64,8,128,8.968615531921387,1.126953125,4194304 +18,self_attn.v_proj,2,128,8,128,10.222861289978027,1.0634765625,4194304 +18,self_attn.v_proj,3,32,8,128,3.261977434158325,1.75390625,4194304 +18,self_attn.v_proj,3,64,8,128,3.853959560394287,1.626953125,4194304 +18,self_attn.v_proj,3,128,8,128,4.402278900146484,1.5634765625,4194304 +18,self_attn.v_proj,4,32,8,128,1.522200107574463,2.25390625,4194304 +18,self_attn.v_proj,4,64,8,128,1.7977567911148071,2.126953125,4194304 +18,self_attn.v_proj,4,128,8,128,2.054875612258911,2.0634765625,4194304 +18,self_attn.v_proj,8,32,8,128,0.09221041947603226,4.25390625,4194304 +18,self_attn.v_proj,8,64,8,128,0.1087983027100563,4.126953125,4194304 +18,self_attn.v_proj,8,128,8,128,0.12390248477458954,4.0634765625,4194304 +18,self_attn.k_proj,2,32,8,128,22.43804359436035,1.25390625,4194304 +18,self_attn.k_proj,2,64,8,128,26.65841293334961,1.126953125,4194304 +18,self_attn.k_proj,2,128,8,128,30.496511459350586,1.0634765625,4194304 +18,self_attn.k_proj,3,32,8,128,9.61676025390625,1.75390625,4194304 +18,self_attn.k_proj,3,64,8,128,11.462735176086426,1.626953125,4194304 +18,self_attn.k_proj,3,128,8,128,13.183043479919434,1.5634765625,4194304 +18,self_attn.k_proj,4,32,8,128,4.487565040588379,2.25390625,4194304 +18,self_attn.k_proj,4,64,8,128,5.347821235656738,2.126953125,4194304 +18,self_attn.k_proj,4,128,8,128,6.151449203491211,2.0634765625,4194304 +18,self_attn.k_proj,8,32,8,128,0.27152422070503235,4.25390625,4194304 +18,self_attn.k_proj,8,64,8,128,0.32342252135276794,4.126953125,4194304 +18,self_attn.k_proj,8,128,8,128,0.3708813488483429,4.0634765625,4194304 +18,self_attn.o_proj,2,32,8,128,16.851301193237305,5.015625,16777216 +18,self_attn.o_proj,2,64,8,128,19.73322296142578,4.5078125,16777216 +18,self_attn.o_proj,2,128,8,128,22.40922737121582,4.25390625,16777216 +18,self_attn.o_proj,3,32,8,128,7.212166786193848,7.015625,16777216 +18,self_attn.o_proj,3,64,8,128,8.4592924118042,6.5078125,16777216 +18,self_attn.o_proj,3,128,8,128,9.63355541229248,6.25390625,16777216 +18,self_attn.o_proj,4,32,8,128,3.366203546524048,9.015625,16777216 +18,self_attn.o_proj,4,64,8,128,3.949575424194336,8.5078125,16777216 +18,self_attn.o_proj,4,128,8,128,4.502011775970459,8.25390625,16777216 +18,self_attn.o_proj,8,32,8,128,0.20374850928783417,17.015625,16777216 +18,self_attn.o_proj,8,64,8,128,0.2388961762189865,16.5078125,16777216 +18,self_attn.o_proj,8,128,8,128,0.2713231146335602,16.25390625,16777216 +18,mlp.gate_proj,2,32,8,128,44.47488784790039,17.5546875,58720256 +18,mlp.gate_proj,2,64,8,128,52.77484130859375,15.77734375,58720256 +18,mlp.gate_proj,2,128,8,128,60.76519012451172,14.888671875,58720256 +18,mlp.gate_proj,3,32,8,128,19.03122329711914,24.5546875,58720256 +18,mlp.gate_proj,3,64,8,128,22.68387794494629,22.77734375,58720256 +18,mlp.gate_proj,3,128,8,128,26.212928771972656,21.888671875,58720256 +18,mlp.gate_proj,4,32,8,128,8.879648208618164,31.5546875,58720256 +18,mlp.gate_proj,4,64,8,128,10.581196784973145,29.77734375,58720256 +18,mlp.gate_proj,4,128,8,128,12.240177154541016,28.888671875,58720256 +18,mlp.gate_proj,8,32,8,128,0.5389745831489563,59.5546875,58720256 +18,mlp.gate_proj,8,64,8,128,0.6424528360366821,57.77734375,58720256 +18,mlp.gate_proj,8,128,8,128,0.7411465048789978,56.888671875,58720256 +18,mlp.up_proj,2,32,8,128,34.06605529785156,17.5546875,58720256 +18,mlp.up_proj,2,64,8,128,39.76256561279297,15.77734375,58720256 +18,mlp.up_proj,2,128,8,128,44.90513610839844,14.888671875,58720256 +18,mlp.up_proj,3,32,8,128,14.57112979888916,24.5546875,58720256 +18,mlp.up_proj,3,64,8,128,17.01555633544922,22.77734375,58720256 +18,mlp.up_proj,3,128,8,128,19.23248863220215,21.888671875,58720256 +18,mlp.up_proj,4,32,8,128,6.794175624847412,31.5546875,58720256 +18,mlp.up_proj,4,64,8,128,7.944119930267334,29.77734375,58720256 +18,mlp.up_proj,4,128,8,128,8.977556228637695,28.888671875,58720256 +18,mlp.up_proj,8,32,8,128,0.4126243591308594,59.5546875,58720256 +18,mlp.up_proj,8,64,8,128,0.48134008049964905,57.77734375,58720256 +18,mlp.up_proj,8,128,8,128,0.5426444411277771,56.888671875,58720256 +18,mlp.down_proj,2,32,8,128,33.907447814941406,17.5546875,58720256 +18,mlp.down_proj,2,64,8,128,39.57309341430664,15.77734375,58720256 +18,mlp.down_proj,2,128,8,128,44.72465896606445,14.888671875,58720256 +18,mlp.down_proj,3,32,8,128,14.502281188964844,24.5546875,58720256 +18,mlp.down_proj,3,64,8,128,16.94414710998535,22.77734375,58720256 +18,mlp.down_proj,3,128,8,128,19.16938018798828,21.888671875,58720256 +18,mlp.down_proj,4,32,8,128,6.764858245849609,31.5546875,58720256 +18,mlp.down_proj,4,64,8,128,7.911740779876709,29.77734375,58720256 +18,mlp.down_proj,4,128,8,128,8.949458122253418,28.888671875,58720256 +18,mlp.down_proj,8,32,8,128,0.41078734397888184,59.5546875,58720256 +18,mlp.down_proj,8,64,8,128,0.47953930497169495,57.77734375,58720256 +18,mlp.down_proj,8,128,8,128,0.5410352349281311,56.888671875,58720256 +19,self_attn.q_proj,2,32,8,128,29.170923233032227,5.015625,16777216 +19,self_attn.q_proj,2,64,8,128,34.81711196899414,4.5078125,16777216 +19,self_attn.q_proj,2,128,8,128,40.933223724365234,4.25390625,16777216 +19,self_attn.q_proj,3,32,8,128,12.49448013305664,7.015625,16777216 +19,self_attn.q_proj,3,64,8,128,14.94357681274414,6.5078125,16777216 +19,self_attn.q_proj,3,128,8,128,17.729894638061523,6.25390625,16777216 +19,self_attn.q_proj,4,32,8,128,5.830387115478516,9.015625,16777216 +19,self_attn.q_proj,4,64,8,128,6.9715094566345215,8.5078125,16777216 +19,self_attn.q_proj,4,128,8,128,8.276711463928223,8.25390625,16777216 +19,self_attn.q_proj,8,32,8,128,0.3530619740486145,17.015625,16777216 +19,self_attn.q_proj,8,64,8,128,0.4215770959854126,16.5078125,16777216 +19,self_attn.q_proj,8,128,8,128,0.49877867102622986,16.25390625,16777216 +19,self_attn.v_proj,2,32,8,128,8.066346168518066,1.25390625,4194304 +19,self_attn.v_proj,2,64,8,128,9.516999244689941,1.126953125,4194304 +19,self_attn.v_proj,2,128,8,128,10.859237670898438,1.0634765625,4194304 +19,self_attn.v_proj,3,32,8,128,3.4501121044158936,1.75390625,4194304 +19,self_attn.v_proj,3,64,8,128,4.0755743980407715,1.626953125,4194304 +19,self_attn.v_proj,3,128,8,128,4.657936096191406,1.5634765625,4194304 +19,self_attn.v_proj,4,32,8,128,1.6106960773468018,2.25390625,4194304 +19,self_attn.v_proj,4,64,8,128,1.9023813009262085,2.126953125,4194304 +19,self_attn.v_proj,4,128,8,128,2.1746299266815186,2.0634765625,4194304 +19,self_attn.v_proj,8,32,8,128,0.09752566367387772,4.25390625,4194304 +19,self_attn.v_proj,8,64,8,128,0.11502981930971146,4.126953125,4194304 +19,self_attn.v_proj,8,128,8,128,0.13096517324447632,4.0634765625,4194304 +19,self_attn.k_proj,2,32,8,128,21.51323127746582,1.25390625,4194304 +19,self_attn.k_proj,2,64,8,128,25.620941162109375,1.126953125,4194304 +19,self_attn.k_proj,2,128,8,128,29.378190994262695,1.0634765625,4194304 +19,self_attn.k_proj,3,32,8,128,9.21500015258789,1.75390625,4194304 +19,self_attn.k_proj,3,64,8,128,11.010863304138184,1.626953125,4194304 +19,self_attn.k_proj,3,128,8,128,12.70950698852539,1.5634765625,4194304 +19,self_attn.k_proj,4,32,8,128,4.299954414367676,2.25390625,4194304 +19,self_attn.k_proj,4,64,8,128,5.138056755065918,2.126953125,4194304 +19,self_attn.k_proj,4,128,8,128,5.9284844398498535,2.0634765625,4194304 +19,self_attn.k_proj,8,32,8,128,0.26019731163978577,4.25390625,4194304 +19,self_attn.k_proj,8,64,8,128,0.31067076325416565,4.126953125,4194304 +19,self_attn.k_proj,8,128,8,128,0.3574235737323761,4.0634765625,4194304 +19,self_attn.o_proj,2,32,8,128,17.063081741333008,5.015625,16777216 +19,self_attn.o_proj,2,64,8,128,19.96478843688965,4.5078125,16777216 +19,self_attn.o_proj,2,128,8,128,22.66409683227539,4.25390625,16777216 +19,self_attn.o_proj,3,32,8,128,7.30420446395874,7.015625,16777216 +19,self_attn.o_proj,3,64,8,128,8.560049057006836,6.5078125,16777216 +19,self_attn.o_proj,3,128,8,128,9.740911483764648,6.25390625,16777216 +19,self_attn.o_proj,4,32,8,128,3.407874345779419,9.015625,16777216 +19,self_attn.o_proj,4,64,8,128,3.9970288276672363,8.5078125,16777216 +19,self_attn.o_proj,4,128,8,128,4.549777984619141,8.25390625,16777216 +19,self_attn.o_proj,8,32,8,128,0.20627987384796143,17.015625,16777216 +19,self_attn.o_proj,8,64,8,128,0.24169477820396423,16.5078125,16777216 +19,self_attn.o_proj,8,128,8,128,0.2743333876132965,16.25390625,16777216 +19,mlp.gate_proj,2,32,8,128,44.57541275024414,17.5546875,58720256 +19,mlp.gate_proj,2,64,8,128,52.804534912109375,15.77734375,58720256 +19,mlp.gate_proj,2,128,8,128,60.777374267578125,14.888671875,58720256 +19,mlp.gate_proj,3,32,8,128,19.075414657592773,24.5546875,58720256 +19,mlp.gate_proj,3,64,8,128,22.691755294799805,22.77734375,58720256 +19,mlp.gate_proj,3,128,8,128,26.194007873535156,21.888671875,58720256 +19,mlp.gate_proj,4,32,8,128,8.899396896362305,31.5546875,58720256 +19,mlp.gate_proj,4,64,8,128,10.585711479187012,29.77734375,58720256 +19,mlp.gate_proj,4,128,8,128,12.235103607177734,28.888671875,58720256 +19,mlp.gate_proj,8,32,8,128,0.5401046276092529,59.5546875,58720256 +19,mlp.gate_proj,8,64,8,128,0.6427481174468994,57.77734375,58720256 +19,mlp.gate_proj,8,128,8,128,0.7405589818954468,56.888671875,58720256 +19,mlp.up_proj,2,32,8,128,33.90513229370117,17.5546875,58720256 +19,mlp.up_proj,2,64,8,128,39.545799255371094,15.77734375,58720256 +19,mlp.up_proj,2,128,8,128,44.661834716796875,14.888671875,58720256 +19,mlp.up_proj,3,32,8,128,14.499414443969727,24.5546875,58720256 +19,mlp.up_proj,3,64,8,128,16.921518325805664,22.77734375,58720256 +19,mlp.up_proj,3,128,8,128,19.11308479309082,21.888671875,58720256 +19,mlp.up_proj,4,32,8,128,6.762147903442383,31.5546875,58720256 +19,mlp.up_proj,4,64,8,128,7.899803638458252,29.77734375,58720256 +19,mlp.up_proj,4,128,8,128,8.923233032226562,28.888671875,58720256 +19,mlp.up_proj,8,32,8,128,0.41069695353507996,59.5546875,58720256 +19,mlp.up_proj,8,64,8,128,0.47870492935180664,57.77734375,58720256 +19,mlp.up_proj,8,128,8,128,0.5392631888389587,56.888671875,58720256 +19,mlp.down_proj,2,32,8,128,33.828460693359375,17.5546875,58720256 +19,mlp.down_proj,2,64,8,128,39.4703483581543,15.77734375,58720256 +19,mlp.down_proj,2,128,8,128,44.577667236328125,14.888671875,58720256 +19,mlp.down_proj,3,32,8,128,14.469304084777832,24.5546875,58720256 +19,mlp.down_proj,3,64,8,128,16.894807815551758,22.77734375,58720256 +19,mlp.down_proj,3,128,8,128,19.1041259765625,21.888671875,58720256 +19,mlp.down_proj,4,32,8,128,6.7488603591918945,31.5546875,58720256 +19,mlp.down_proj,4,64,8,128,7.887848377227783,29.77734375,58720256 +19,mlp.down_proj,4,128,8,128,8.919732093811035,28.888671875,58720256 +19,mlp.down_proj,8,32,8,128,0.40985897183418274,59.5546875,58720256 +19,mlp.down_proj,8,64,8,128,0.478181928396225,57.77734375,58720256 +19,mlp.down_proj,8,128,8,128,0.5393068194389343,56.888671875,58720256 +20,self_attn.q_proj,2,32,8,128,28.962358474731445,5.015625,16777216 +20,self_attn.q_proj,2,64,8,128,34.52019119262695,4.5078125,16777216 +20,self_attn.q_proj,2,128,8,128,40.505245208740234,4.25390625,16777216 +20,self_attn.q_proj,3,32,8,128,12.405719757080078,7.015625,16777216 +20,self_attn.q_proj,3,64,8,128,14.814352989196777,6.5078125,16777216 +20,self_attn.q_proj,3,128,8,128,17.588022232055664,6.25390625,16777216 +20,self_attn.q_proj,4,32,8,128,5.788156509399414,9.015625,16777216 +20,self_attn.q_proj,4,64,8,128,6.91267728805542,8.5078125,16777216 +20,self_attn.q_proj,4,128,8,128,8.212377548217773,8.25390625,16777216 +20,self_attn.q_proj,8,32,8,128,0.35043856501579285,17.015625,16777216 +20,self_attn.q_proj,8,64,8,128,0.41793254017829895,16.5078125,16777216 +20,self_attn.q_proj,8,128,8,128,0.49477124214172363,16.25390625,16777216 +20,self_attn.v_proj,2,32,8,128,8.473453521728516,1.25390625,4194304 +20,self_attn.v_proj,2,64,8,128,9.995060920715332,1.126953125,4194304 +20,self_attn.v_proj,2,128,8,128,11.41641616821289,1.0634765625,4194304 +20,self_attn.v_proj,3,32,8,128,3.62433123588562,1.75390625,4194304 +20,self_attn.v_proj,3,64,8,128,4.283699989318848,1.626953125,4194304 +20,self_attn.v_proj,3,128,8,128,4.89935302734375,1.5634765625,4194304 +20,self_attn.v_proj,4,32,8,128,1.690201759338379,2.25390625,4194304 +20,self_attn.v_proj,4,64,8,128,1.9987983703613281,2.126953125,4194304 +20,self_attn.v_proj,4,128,8,128,2.285043716430664,2.0634765625,4194304 +20,self_attn.v_proj,8,32,8,128,0.10244591534137726,4.25390625,4194304 +20,self_attn.v_proj,8,64,8,128,0.1208517774939537,4.126953125,4194304 +20,self_attn.v_proj,8,128,8,128,0.13784193992614746,4.0634765625,4194304 +20,self_attn.k_proj,2,32,8,128,21.273357391357422,1.25390625,4194304 +20,self_attn.k_proj,2,64,8,128,25.34518051147461,1.126953125,4194304 +20,self_attn.k_proj,2,128,8,128,29.10357093811035,1.0634765625,4194304 +20,self_attn.k_proj,3,32,8,128,9.117815017700195,1.75390625,4194304 +20,self_attn.k_proj,3,64,8,128,10.91283893585205,1.626953125,4194304 +20,self_attn.k_proj,3,128,8,128,12.614142417907715,1.5634765625,4194304 +20,self_attn.k_proj,4,32,8,128,4.257753849029541,2.25390625,4194304 +20,self_attn.k_proj,4,64,8,128,5.089752197265625,2.126953125,4194304 +20,self_attn.k_proj,4,128,8,128,5.888340473175049,2.0634765625,4194304 +20,self_attn.k_proj,8,32,8,128,0.25785672664642334,4.25390625,4194304 +20,self_attn.k_proj,8,64,8,128,0.3080633282661438,4.126953125,4194304 +20,self_attn.k_proj,8,128,8,128,0.3550806939601898,4.0634765625,4194304 +20,self_attn.o_proj,2,32,8,128,16.998645782470703,5.015625,16777216 +20,self_attn.o_proj,2,64,8,128,19.96787452697754,4.5078125,16777216 +20,self_attn.o_proj,2,128,8,128,22.734037399291992,4.25390625,16777216 +20,self_attn.o_proj,3,32,8,128,7.284494400024414,7.015625,16777216 +20,self_attn.o_proj,3,64,8,128,8.58784008026123,6.5078125,16777216 +20,self_attn.o_proj,3,128,8,128,9.84182357788086,6.25390625,16777216 +20,self_attn.o_proj,4,32,8,128,3.4014012813568115,9.015625,16777216 +20,self_attn.o_proj,4,64,8,128,4.010610103607178,8.5078125,16777216 +20,self_attn.o_proj,4,128,8,128,4.599480152130127,8.25390625,16777216 +20,self_attn.o_proj,8,32,8,128,0.2057718187570572,17.015625,16777216 +20,self_attn.o_proj,8,64,8,128,0.24243779480457306,16.5078125,16777216 +20,self_attn.o_proj,8,128,8,128,0.27724531292915344,16.25390625,16777216 +20,mlp.gate_proj,2,32,8,128,44.38024139404297,17.5546875,58720256 +20,mlp.gate_proj,2,64,8,128,52.49317932128906,15.77734375,58720256 +20,mlp.gate_proj,2,128,8,128,60.3219108581543,14.888671875,58720256 +20,mlp.gate_proj,3,32,8,128,18.985774993896484,24.5546875,58720256 +20,mlp.gate_proj,3,64,8,128,22.542360305786133,22.77734375,58720256 +20,mlp.gate_proj,3,128,8,128,25.963619232177734,21.888671875,58720256 +20,mlp.gate_proj,4,32,8,128,8.858451843261719,31.5546875,58720256 +20,mlp.gate_proj,4,64,8,128,10.51746654510498,29.77734375,58720256 +20,mlp.gate_proj,4,128,8,128,12.120981216430664,28.888671875,58720256 +20,mlp.gate_proj,8,32,8,128,0.5376598238945007,59.5546875,58720256 +20,mlp.gate_proj,8,64,8,128,0.6385093331336975,57.77734375,58720256 +20,mlp.gate_proj,8,128,8,128,0.733942985534668,56.888671875,58720256 +20,mlp.up_proj,2,32,8,128,33.99882125854492,17.5546875,58720256 +20,mlp.up_proj,2,64,8,128,39.63611602783203,15.77734375,58720256 +20,mlp.up_proj,2,128,8,128,44.74638366699219,14.888671875,58720256 +20,mlp.up_proj,3,32,8,128,14.53717041015625,24.5546875,58720256 +20,mlp.up_proj,3,64,8,128,16.961830139160156,22.77734375,58720256 +20,mlp.up_proj,3,128,8,128,19.155508041381836,21.888671875,58720256 +20,mlp.up_proj,4,32,8,128,6.7814178466796875,31.5546875,58720256 +20,mlp.up_proj,4,64,8,128,7.917990684509277,29.77734375,58720256 +20,mlp.up_proj,4,128,8,128,8.939305305480957,28.888671875,58720256 +20,mlp.up_proj,8,32,8,128,0.41178014874458313,59.5546875,58720256 +20,mlp.up_proj,8,64,8,128,0.47986772656440735,57.77734375,58720256 +20,mlp.up_proj,8,128,8,128,0.540381133556366,56.888671875,58720256 +20,mlp.down_proj,2,32,8,128,33.9361572265625,17.5546875,58720256 +20,mlp.down_proj,2,64,8,128,39.58932876586914,15.77734375,58720256 +20,mlp.down_proj,2,128,8,128,44.727333068847656,14.888671875,58720256 +20,mlp.down_proj,3,32,8,128,14.516053199768066,24.5546875,58720256 +20,mlp.down_proj,3,64,8,128,16.944902420043945,22.77734375,58720256 +20,mlp.down_proj,3,128,8,128,19.157201766967773,21.888671875,58720256 +20,mlp.down_proj,4,32,8,128,6.771024227142334,31.5546875,58720256 +20,mlp.down_proj,4,64,8,128,7.913134574890137,29.77734375,58720256 +20,mlp.down_proj,4,128,8,128,8.944794654846191,28.888671875,58720256 +20,mlp.down_proj,8,32,8,128,0.41114339232444763,59.5546875,58720256 +20,mlp.down_proj,8,64,8,128,0.47959455847740173,57.77734375,58720256 +20,mlp.down_proj,8,128,8,128,0.5406232476234436,56.888671875,58720256 +21,self_attn.q_proj,2,32,8,128,28.18620491027832,5.015625,16777216 +21,self_attn.q_proj,2,64,8,128,33.565284729003906,4.5078125,16777216 +21,self_attn.q_proj,2,128,8,128,39.381866455078125,4.25390625,16777216 +21,self_attn.q_proj,3,32,8,128,12.067045211791992,7.015625,16777216 +21,self_attn.q_proj,3,64,8,128,14.399670600891113,6.5078125,16777216 +21,self_attn.q_proj,3,128,8,128,16.998525619506836,6.25390625,16777216 +21,self_attn.q_proj,4,32,8,128,5.631928443908691,9.015625,16777216 +21,self_attn.q_proj,4,64,8,128,6.719544410705566,8.5078125,16777216 +21,self_attn.q_proj,4,128,8,128,7.9383063316345215,8.25390625,16777216 +21,self_attn.q_proj,8,32,8,128,0.3409528136253357,17.015625,16777216 +21,self_attn.q_proj,8,64,8,128,0.4061949849128723,16.5078125,16777216 +21,self_attn.q_proj,8,128,8,128,0.4781816601753235,16.25390625,16777216 +21,self_attn.v_proj,2,32,8,128,8.663697242736816,1.25390625,4194304 +21,self_attn.v_proj,2,64,8,128,10.200803756713867,1.126953125,4194304 +21,self_attn.v_proj,2,128,8,128,11.58244800567627,1.0634765625,4194304 +21,self_attn.v_proj,3,32,8,128,3.724097967147827,1.75390625,4194304 +21,self_attn.v_proj,3,64,8,128,4.398136615753174,1.626953125,4194304 +21,self_attn.v_proj,3,128,8,128,5.01934814453125,1.5634765625,4194304 +21,self_attn.v_proj,4,32,8,128,1.7358039617538452,2.25390625,4194304 +21,self_attn.v_proj,4,64,8,128,2.0530893802642822,2.126953125,4194304 +21,self_attn.v_proj,4,128,8,128,2.3439788818359375,2.0634765625,4194304 +21,self_attn.v_proj,8,32,8,128,0.10516036301851273,4.25390625,4194304 +21,self_attn.v_proj,8,64,8,128,0.12414537370204926,4.126953125,4194304 +21,self_attn.v_proj,8,128,8,128,0.14124320447444916,4.0634765625,4194304 +21,self_attn.k_proj,2,32,8,128,21.128854751586914,1.25390625,4194304 +21,self_attn.k_proj,2,64,8,128,25.067174911499023,1.126953125,4194304 +21,self_attn.k_proj,2,128,8,128,28.654985427856445,1.0634765625,4194304 +21,self_attn.k_proj,3,32,8,128,9.048358917236328,1.75390625,4194304 +21,self_attn.k_proj,3,64,8,128,10.765707969665527,1.626953125,4194304 +21,self_attn.k_proj,3,128,8,128,12.375262260437012,1.5634765625,4194304 +21,self_attn.k_proj,4,32,8,128,4.224000930786133,2.25390625,4194304 +21,self_attn.k_proj,4,64,8,128,5.025003910064697,2.126953125,4194304 +21,self_attn.k_proj,4,128,8,128,5.767529010772705,2.0634765625,4194304 +21,self_attn.k_proj,8,32,8,128,0.25570952892303467,4.25390625,4194304 +21,self_attn.k_proj,8,64,8,128,0.30379584431648254,4.126953125,4194304 +21,self_attn.k_proj,8,128,8,128,0.34786343574523926,4.0634765625,4194304 +21,self_attn.o_proj,2,32,8,128,17.319395065307617,5.015625,16777216 +21,self_attn.o_proj,2,64,8,128,20.300491333007812,4.5078125,16777216 +21,self_attn.o_proj,2,128,8,128,23.046751022338867,4.25390625,16777216 +21,self_attn.o_proj,3,32,8,128,7.457870960235596,7.015625,16777216 +21,self_attn.o_proj,3,64,8,128,8.804133415222168,6.5078125,16777216 +21,self_attn.o_proj,3,128,8,128,10.111490249633789,6.25390625,16777216 +21,self_attn.o_proj,4,32,8,128,3.4839088916778564,9.015625,16777216 +21,self_attn.o_proj,4,64,8,128,4.121822834014893,8.5078125,16777216 +21,self_attn.o_proj,4,128,8,128,4.753020763397217,8.25390625,16777216 +21,self_attn.o_proj,8,32,8,128,0.21076828241348267,17.015625,16777216 +21,self_attn.o_proj,8,64,8,128,0.24902115762233734,16.5078125,16777216 +21,self_attn.o_proj,8,128,8,128,0.28656089305877686,16.25390625,16777216 +21,mlp.gate_proj,2,32,8,128,44.668758392333984,17.5546875,58720256 +21,mlp.gate_proj,2,64,8,128,52.827735900878906,15.77734375,58720256 +21,mlp.gate_proj,2,128,8,128,60.69220733642578,14.888671875,58720256 +21,mlp.gate_proj,3,32,8,128,19.11691665649414,24.5546875,58720256 +21,mlp.gate_proj,3,64,8,128,22.70735740661621,22.77734375,58720256 +21,mlp.gate_proj,3,128,8,128,26.170879364013672,21.888671875,58720256 +21,mlp.gate_proj,4,32,8,128,8.920905113220215,31.5546875,58720256 +21,mlp.gate_proj,4,64,8,128,10.594423294067383,29.77734375,58720256 +21,mlp.gate_proj,4,128,8,128,12.221921920776367,28.888671875,58720256 +21,mlp.gate_proj,8,32,8,128,0.5413225293159485,59.5546875,58720256 +21,mlp.gate_proj,8,64,8,128,0.643252968788147,57.77734375,58720256 +21,mlp.gate_proj,8,128,8,128,0.7401362061500549,56.888671875,58720256 +21,mlp.up_proj,2,32,8,128,34.112152099609375,17.5546875,58720256 +21,mlp.up_proj,2,64,8,128,39.77892303466797,15.77734375,58720256 +21,mlp.up_proj,2,128,8,128,44.90371322631836,14.888671875,58720256 +21,mlp.up_proj,3,32,8,128,14.588909149169922,24.5546875,58720256 +21,mlp.up_proj,3,64,8,128,17.024507522583008,22.77734375,58720256 +21,mlp.up_proj,3,128,8,128,19.232635498046875,21.888671875,58720256 +21,mlp.up_proj,4,32,8,128,6.804201602935791,31.5546875,58720256 +21,mlp.up_proj,4,64,8,128,7.950302600860596,29.77734375,58720256 +21,mlp.up_proj,4,128,8,128,8.97830867767334,28.888671875,58720256 +21,mlp.up_proj,8,32,8,128,0.4131689965724945,59.5546875,58720256 +21,mlp.up_proj,8,64,8,128,0.4816552400588989,57.77734375,58720256 +21,mlp.up_proj,8,128,8,128,0.5427266359329224,56.888671875,58720256 +21,mlp.down_proj,2,32,8,128,34.055049896240234,17.5546875,58720256 +21,mlp.down_proj,2,64,8,128,39.72943115234375,15.77734375,58720256 +21,mlp.down_proj,2,128,8,128,44.87776184082031,14.888671875,58720256 +21,mlp.down_proj,3,32,8,128,14.565619468688965,24.5546875,58720256 +21,mlp.down_proj,3,64,8,128,17.009538650512695,22.77734375,58720256 +21,mlp.down_proj,3,128,8,128,19.234281539916992,21.888671875,58720256 +21,mlp.down_proj,4,32,8,128,6.793951511383057,31.5546875,58720256 +21,mlp.down_proj,4,64,8,128,7.944419860839844,29.77734375,58720256 +21,mlp.down_proj,4,128,8,128,8.981793403625488,28.888671875,58720256 +21,mlp.down_proj,8,32,8,128,0.4126131534576416,59.5546875,58720256 +21,mlp.down_proj,8,64,8,128,0.48147591948509216,57.77734375,58720256 +21,mlp.down_proj,8,128,8,128,0.5430736541748047,56.888671875,58720256 +22,self_attn.q_proj,2,32,8,128,27.34174346923828,5.015625,16777216 +22,self_attn.q_proj,2,64,8,128,32.466552734375,4.5078125,16777216 +22,self_attn.q_proj,2,128,8,128,37.7900390625,4.25390625,16777216 +22,self_attn.q_proj,3,32,8,128,11.713033676147461,7.015625,16777216 +22,self_attn.q_proj,3,64,8,128,13.933253288269043,6.5078125,16777216 +22,self_attn.q_proj,3,128,8,128,16.28204345703125,6.25390625,16777216 +22,self_attn.q_proj,4,32,8,128,5.46622371673584,9.015625,16777216 +22,self_attn.q_proj,4,64,8,128,6.504093170166016,8.5078125,16777216 +22,self_attn.q_proj,4,128,8,128,7.601945877075195,8.25390625,16777216 +22,self_attn.q_proj,8,32,8,128,0.3310038447380066,17.015625,16777216 +22,self_attn.q_proj,8,64,8,128,0.39333489537239075,16.5078125,16777216 +22,self_attn.q_proj,8,128,8,128,0.4580498933792114,16.25390625,16777216 +22,self_attn.v_proj,2,32,8,128,9.040300369262695,1.25390625,4194304 +22,self_attn.v_proj,2,64,8,128,10.587621688842773,1.126953125,4194304 +22,self_attn.v_proj,2,128,8,128,11.98330307006836,1.0634765625,4194304 +22,self_attn.v_proj,3,32,8,128,3.8699820041656494,1.75390625,4194304 +22,self_attn.v_proj,3,64,8,128,4.540487766265869,1.626953125,4194304 +22,self_attn.v_proj,3,128,8,128,5.156569004058838,1.5634765625,4194304 +22,self_attn.v_proj,4,32,8,128,1.8059133291244507,2.25390625,4194304 +22,self_attn.v_proj,4,64,8,128,2.1198692321777344,2.126953125,4194304 +22,self_attn.v_proj,4,128,8,128,2.409508466720581,2.0634765625,4194304 +22,self_attn.v_proj,8,32,8,128,0.10936756432056427,4.25390625,4194304 +22,self_attn.v_proj,8,64,8,128,0.12818443775177002,4.126953125,4194304 +22,self_attn.v_proj,8,128,8,128,0.14507843554019928,4.0634765625,4194304 +22,self_attn.k_proj,2,32,8,128,20.743793487548828,1.25390625,4194304 +22,self_attn.k_proj,2,64,8,128,24.672283172607422,1.126953125,4194304 +22,self_attn.k_proj,2,128,8,128,28.2346134185791,1.0634765625,4194304 +22,self_attn.k_proj,3,32,8,128,8.88916301727295,1.75390625,4194304 +22,self_attn.k_proj,3,64,8,128,10.609160423278809,1.626953125,4194304 +22,self_attn.k_proj,3,128,8,128,12.206852912902832,1.5634765625,4194304 +22,self_attn.k_proj,4,32,8,128,4.150344371795654,2.25390625,4194304 +22,self_attn.k_proj,4,64,8,128,4.950047969818115,2.126953125,4194304 +22,self_attn.k_proj,4,128,8,128,5.696535587310791,2.0634765625,4194304 +22,self_attn.k_proj,8,32,8,128,0.25118499994277954,4.25390625,4194304 +22,self_attn.k_proj,8,64,8,128,0.2992619276046753,4.126953125,4194304 +22,self_attn.k_proj,8,128,8,128,0.3433564305305481,4.0634765625,4194304 +22,self_attn.o_proj,2,32,8,128,17.593524932861328,5.015625,16777216 +22,self_attn.o_proj,2,64,8,128,20.551952362060547,4.5078125,16777216 +22,self_attn.o_proj,2,128,8,128,23.2547664642334,4.25390625,16777216 +22,self_attn.o_proj,3,32,8,128,7.536215782165527,7.015625,16777216 +22,self_attn.o_proj,3,64,8,128,8.820917129516602,6.5078125,16777216 +22,self_attn.o_proj,3,128,8,128,10.020706176757812,6.25390625,16777216 +22,self_attn.o_proj,4,32,8,128,3.515662670135498,9.015625,16777216 +22,self_attn.o_proj,4,64,8,128,4.119492053985596,8.5078125,16777216 +22,self_attn.o_proj,4,128,8,128,4.677903175354004,8.25390625,16777216 +22,self_attn.o_proj,8,32,8,128,0.21269546449184418,17.015625,16777216 +22,self_attn.o_proj,8,64,8,128,0.24892358481884003,16.5078125,16777216 +22,self_attn.o_proj,8,128,8,128,0.2820328176021576,16.25390625,16777216 +22,mlp.gate_proj,2,32,8,128,44.38797378540039,17.5546875,58720256 +22,mlp.gate_proj,2,64,8,128,52.3579216003418,15.77734375,58720256 +22,mlp.gate_proj,2,128,8,128,59.97238540649414,14.888671875,58720256 +22,mlp.gate_proj,3,32,8,128,18.98455047607422,24.5546875,58720256 +22,mlp.gate_proj,3,64,8,128,22.469825744628906,22.77734375,58720256 +22,mlp.gate_proj,3,128,8,128,25.789480209350586,21.888671875,58720256 +22,mlp.gate_proj,4,32,8,128,8.857576370239258,31.5546875,58720256 +22,mlp.gate_proj,4,64,8,128,10.48155689239502,29.77734375,58720256 +22,mlp.gate_proj,4,128,8,128,12.041306495666504,28.888671875,58720256 +22,mlp.gate_proj,8,32,8,128,0.5375619530677795,59.5546875,58720256 +22,mlp.gate_proj,8,64,8,128,0.6365211606025696,57.77734375,58720256 +22,mlp.gate_proj,8,128,8,128,0.7291144132614136,56.888671875,58720256 +22,mlp.up_proj,2,32,8,128,34.172943115234375,17.5546875,58720256 +22,mlp.up_proj,2,64,8,128,39.816776275634766,15.77734375,58720256 +22,mlp.up_proj,2,128,8,128,44.92223358154297,14.888671875,58720256 +22,mlp.up_proj,3,32,8,128,14.61396312713623,24.5546875,58720256 +22,mlp.up_proj,3,64,8,128,17.03720474243164,22.77734375,58720256 +22,mlp.up_proj,3,128,8,128,19.226682662963867,21.888671875,58720256 +22,mlp.up_proj,4,32,8,128,6.814948081970215,31.5546875,58720256 +22,mlp.up_proj,4,64,8,128,7.95427942276001,29.77734375,58720256 +22,mlp.up_proj,4,128,8,128,8.974326133728027,28.888671875,58720256 +22,mlp.up_proj,8,32,8,128,0.4138053357601166,59.5546875,58720256 +22,mlp.up_proj,8,64,8,128,0.48194193840026855,57.77734375,58720256 +22,mlp.up_proj,8,128,8,128,0.5424661636352539,56.888671875,58720256 +22,mlp.down_proj,2,32,8,128,34.15678787231445,17.5546875,58720256 +22,mlp.down_proj,2,64,8,128,39.825294494628906,15.77734375,58720256 +22,mlp.down_proj,2,128,8,128,44.97071838378906,14.888671875,58720256 +22,mlp.down_proj,3,32,8,128,14.605439186096191,24.5546875,58720256 +22,mlp.down_proj,3,64,8,128,17.038633346557617,22.77734375,58720256 +22,mlp.down_proj,3,128,8,128,19.239704132080078,21.888671875,58720256 +22,mlp.down_proj,4,32,8,128,6.812772274017334,31.5546875,58720256 +22,mlp.down_proj,4,64,8,128,7.957311153411865,29.77734375,58720256 +22,mlp.down_proj,4,128,8,128,8.982463836669922,28.888671875,58720256 +22,mlp.down_proj,8,32,8,128,0.41363024711608887,59.5546875,58720256 +22,mlp.down_proj,8,64,8,128,0.4821682870388031,57.77734375,58720256 +22,mlp.down_proj,8,128,8,128,0.5430733561515808,56.888671875,58720256 +23,self_attn.q_proj,2,32,8,128,27.55084228515625,5.015625,16777216 +23,self_attn.q_proj,2,64,8,128,32.65092468261719,4.5078125,16777216 +23,self_attn.q_proj,2,128,8,128,37.7667236328125,4.25390625,16777216 +23,self_attn.q_proj,3,32,8,128,11.803522109985352,7.015625,16777216 +23,self_attn.q_proj,3,64,8,128,14.021941184997559,6.5078125,16777216 +23,self_attn.q_proj,3,128,8,128,16.277530670166016,6.25390625,16777216 +23,self_attn.q_proj,4,32,8,128,5.50836181640625,9.015625,16777216 +23,self_attn.q_proj,4,64,8,128,6.544461727142334,8.5078125,16777216 +23,self_attn.q_proj,4,128,8,128,7.598830699920654,8.25390625,16777216 +23,self_attn.q_proj,8,32,8,128,0.33343085646629333,17.015625,16777216 +23,self_attn.q_proj,8,64,8,128,0.39569005370140076,16.5078125,16777216 +23,self_attn.q_proj,8,128,8,128,0.4577195644378662,16.25390625,16777216 +23,self_attn.v_proj,2,32,8,128,9.462922096252441,1.25390625,4194304 +23,self_attn.v_proj,2,64,8,128,11.069671630859375,1.126953125,4194304 +23,self_attn.v_proj,2,128,8,128,12.524882316589355,1.0634765625,4194304 +23,self_attn.v_proj,3,32,8,128,4.051651954650879,1.75390625,4194304 +23,self_attn.v_proj,3,64,8,128,4.744959354400635,1.626953125,4194304 +23,self_attn.v_proj,3,128,8,128,5.383528709411621,1.5634765625,4194304 +23,self_attn.v_proj,4,32,8,128,1.8884917497634888,2.25390625,4194304 +23,self_attn.v_proj,4,64,8,128,2.213996410369873,2.126953125,4194304 +23,self_attn.v_proj,4,128,8,128,2.510345935821533,2.0634765625,4194304 +23,self_attn.v_proj,8,32,8,128,0.11442040652036667,4.25390625,4194304 +23,self_attn.v_proj,8,64,8,128,0.134017214179039,4.126953125,4194304 +23,self_attn.v_proj,8,128,8,128,0.15136580169200897,4.0634765625,4194304 +23,self_attn.k_proj,2,32,8,128,20.51055145263672,1.25390625,4194304 +23,self_attn.k_proj,2,64,8,128,24.34478759765625,1.126953125,4194304 +23,self_attn.k_proj,2,128,8,128,27.829917907714844,1.0634765625,4194304 +23,self_attn.k_proj,3,32,8,128,8.79004192352295,1.75390625,4194304 +23,self_attn.k_proj,3,64,8,128,10.464144706726074,1.626953125,4194304 +23,self_attn.k_proj,3,128,8,128,12.02108097076416,1.5634765625,4194304 +23,self_attn.k_proj,4,32,8,128,4.104809761047363,2.25390625,4194304 +23,self_attn.k_proj,4,64,8,128,4.885592460632324,2.126953125,4194304 +23,self_attn.k_proj,4,128,8,128,5.615811347961426,2.0634765625,4194304 +23,self_attn.k_proj,8,32,8,128,0.2482151836156845,4.25390625,4194304 +23,self_attn.k_proj,8,64,8,128,0.2952117621898651,4.126953125,4194304 +23,self_attn.k_proj,8,128,8,128,0.33816438913345337,4.0634765625,4194304 +23,self_attn.o_proj,2,32,8,128,17.876432418823242,5.015625,16777216 +23,self_attn.o_proj,2,64,8,128,20.859375,4.5078125,16777216 +23,self_attn.o_proj,2,128,8,128,23.567546844482422,4.25390625,16777216 +23,self_attn.o_proj,3,32,8,128,7.6454267501831055,7.015625,16777216 +23,self_attn.o_proj,3,64,8,128,8.930878639221191,6.5078125,16777216 +23,self_attn.o_proj,3,128,8,128,10.111210823059082,6.25390625,16777216 +23,self_attn.o_proj,4,32,8,128,3.567565441131592,9.015625,16777216 +23,self_attn.o_proj,4,64,8,128,4.168977737426758,8.5078125,16777216 +23,self_attn.o_proj,4,128,8,128,4.72023344039917,8.25390625,16777216 +23,self_attn.o_proj,8,32,8,128,0.21589195728302002,17.015625,16777216 +23,self_attn.o_proj,8,64,8,128,0.2520052492618561,16.5078125,16777216 +23,self_attn.o_proj,8,128,8,128,0.2844109833240509,16.25390625,16777216 +23,mlp.gate_proj,2,32,8,128,44.39439392089844,17.5546875,58720256 +23,mlp.gate_proj,2,64,8,128,52.3275032043457,15.77734375,58720256 +23,mlp.gate_proj,2,128,8,128,59.90223693847656,14.888671875,58720256 +23,mlp.gate_proj,3,32,8,128,18.98650360107422,24.5546875,58720256 +23,mlp.gate_proj,3,64,8,128,22.452014923095703,22.77734375,58720256 +23,mlp.gate_proj,3,128,8,128,25.742658615112305,21.888671875,58720256 +23,mlp.gate_proj,4,32,8,128,8.85911750793457,31.5546875,58720256 +23,mlp.gate_proj,4,64,8,128,10.474272727966309,29.77734375,58720256 +23,mlp.gate_proj,4,128,8,128,12.019917488098145,28.888671875,58720256 +23,mlp.gate_proj,8,32,8,128,0.5376695394515991,59.5546875,58720256 +23,mlp.gate_proj,8,64,8,128,0.6359887719154358,57.77734375,58720256 +23,mlp.gate_proj,8,128,8,128,0.7278110980987549,56.888671875,58720256 +23,mlp.up_proj,2,32,8,128,34.269554138183594,17.5546875,58720256 +23,mlp.up_proj,2,64,8,128,39.93233108520508,15.77734375,58720256 +23,mlp.up_proj,2,128,8,128,45.04486083984375,14.888671875,58720256 +23,mlp.up_proj,3,32,8,128,14.653284072875977,24.5546875,58720256 +23,mlp.up_proj,3,64,8,128,17.082435607910156,22.77734375,58720256 +23,mlp.up_proj,3,128,8,128,19.2884578704834,21.888671875,58720256 +23,mlp.up_proj,4,32,8,128,6.834258556365967,31.5546875,58720256 +23,mlp.up_proj,4,64,8,128,7.976307392120361,29.77734375,58720256 +23,mlp.up_proj,4,128,8,128,9.000700950622559,28.888671875,58720256 +23,mlp.up_proj,8,32,8,128,0.4149945080280304,59.5546875,58720256 +23,mlp.up_proj,8,64,8,128,0.4832878112792969,57.77734375,58720256 +23,mlp.up_proj,8,128,8,128,0.5440319180488586,56.888671875,58720256 +23,mlp.down_proj,2,32,8,128,34.28191375732422,17.5546875,58720256 +23,mlp.down_proj,2,64,8,128,39.987186431884766,15.77734375,58720256 +23,mlp.down_proj,2,128,8,128,45.136817932128906,14.888671875,58720256 +23,mlp.down_proj,3,32,8,128,14.659160614013672,24.5546875,58720256 +23,mlp.down_proj,3,64,8,128,17.100849151611328,22.77734375,58720256 +23,mlp.down_proj,3,128,8,128,19.305492401123047,21.888671875,58720256 +23,mlp.down_proj,4,32,8,128,6.836849212646484,31.5546875,58720256 +23,mlp.down_proj,4,64,8,128,7.985709190368652,29.77734375,58720256 +23,mlp.down_proj,4,128,8,128,9.01148509979248,28.888671875,58720256 +23,mlp.down_proj,8,32,8,128,0.4151988923549652,59.5546875,58720256 +23,mlp.down_proj,8,64,8,128,0.48389530181884766,57.77734375,58720256 +23,mlp.down_proj,8,128,8,128,0.5449154376983643,56.888671875,58720256 +24,self_attn.q_proj,2,32,8,128,26.967443466186523,5.015625,16777216 +24,self_attn.q_proj,2,64,8,128,32.025665283203125,4.5078125,16777216 +24,self_attn.q_proj,2,128,8,128,37.257843017578125,4.25390625,16777216 +24,self_attn.q_proj,3,32,8,128,11.565925598144531,7.015625,16777216 +24,self_attn.q_proj,3,64,8,128,13.765057563781738,6.5078125,16777216 +24,self_attn.q_proj,3,128,8,128,16.10650634765625,6.25390625,16777216 +24,self_attn.q_proj,4,32,8,128,5.398556709289551,9.015625,16777216 +24,self_attn.q_proj,4,64,8,128,6.424530506134033,8.5078125,16777216 +24,self_attn.q_proj,4,128,8,128,7.52303409576416,8.25390625,16777216 +24,self_attn.q_proj,8,32,8,128,0.3268450200557709,17.015625,16777216 +24,self_attn.q_proj,8,64,8,128,0.38868993520736694,16.5078125,16777216 +24,self_attn.q_proj,8,128,8,128,0.4533500075340271,16.25390625,16777216 +24,self_attn.v_proj,2,32,8,128,10.52883243560791,1.25390625,4194304 +24,self_attn.v_proj,2,64,8,128,12.395832061767578,1.126953125,4194304 +24,self_attn.v_proj,2,128,8,128,14.100050926208496,1.0634765625,4194304 +24,self_attn.v_proj,3,32,8,128,4.512310981750488,1.75390625,4194304 +24,self_attn.v_proj,3,64,8,128,5.323564529418945,1.626953125,4194304 +24,self_attn.v_proj,3,128,8,128,6.074887275695801,1.5634765625,4194304 +24,self_attn.v_proj,4,32,8,128,2.1045491695404053,2.25390625,4194304 +24,self_attn.v_proj,4,64,8,128,2.4842498302459717,2.126953125,4194304 +24,self_attn.v_proj,4,128,8,128,2.835069417953491,2.0634765625,4194304 +24,self_attn.v_proj,8,32,8,128,0.12746655941009521,4.25390625,4194304 +24,self_attn.v_proj,8,64,8,128,0.15020856261253357,4.126953125,4194304 +24,self_attn.v_proj,8,128,8,128,0.17083439230918884,4.0634765625,4194304 +24,self_attn.k_proj,2,32,8,128,20.00737953186035,1.25390625,4194304 +24,self_attn.k_proj,2,64,8,128,23.888967514038086,1.126953125,4194304 +24,self_attn.k_proj,2,128,8,128,27.38829803466797,1.0634765625,4194304 +24,self_attn.k_proj,3,32,8,128,8.595484733581543,1.75390625,4194304 +24,self_attn.k_proj,3,64,8,128,10.321264266967773,1.626953125,4194304 +24,self_attn.k_proj,3,128,8,128,11.948944091796875,1.5634765625,4194304 +24,self_attn.k_proj,4,32,8,128,4.010400295257568,2.25390625,4194304 +24,self_attn.k_proj,4,64,8,128,4.813996315002441,2.126953125,4194304 +24,self_attn.k_proj,4,128,8,128,5.581087589263916,2.0634765625,4194304 +24,self_attn.k_proj,8,32,8,128,0.24281682074069977,4.25390625,4194304 +24,self_attn.k_proj,8,64,8,128,0.2914222478866577,4.126953125,4194304 +24,self_attn.k_proj,8,128,8,128,0.3362238109111786,4.0634765625,4194304 +24,self_attn.o_proj,2,32,8,128,18.659494400024414,5.015625,16777216 +24,self_attn.o_proj,2,64,8,128,21.793188095092773,4.5078125,16777216 +24,self_attn.o_proj,2,128,8,128,24.634496688842773,4.25390625,16777216 +24,self_attn.o_proj,3,32,8,128,7.98239278793335,7.015625,16777216 +24,self_attn.o_proj,3,64,8,128,9.327285766601562,6.5078125,16777216 +24,self_attn.o_proj,3,128,8,128,10.565587043762207,6.25390625,16777216 +24,self_attn.o_proj,4,32,8,128,3.72469425201416,9.015625,16777216 +24,self_attn.o_proj,4,64,8,128,4.352910041809082,8.5078125,16777216 +24,self_attn.o_proj,4,128,8,128,4.931228160858154,8.25390625,16777216 +24,self_attn.o_proj,8,32,8,128,0.22532835602760315,17.015625,16777216 +24,self_attn.o_proj,8,64,8,128,0.2631794214248657,16.5078125,16777216 +24,self_attn.o_proj,8,128,8,128,0.2970396876335144,16.25390625,16777216 +24,mlp.gate_proj,2,32,8,128,44.542537689208984,17.5546875,58720256 +24,mlp.gate_proj,2,64,8,128,52.47532653808594,15.77734375,58720256 +24,mlp.gate_proj,2,128,8,128,60.034664154052734,14.888671875,58720256 +24,mlp.gate_proj,3,32,8,128,19.050371170043945,24.5546875,58720256 +24,mlp.gate_proj,3,64,8,128,22.520366668701172,22.77734375,58720256 +24,mlp.gate_proj,3,128,8,128,25.805326461791992,21.888671875,58720256 +24,mlp.gate_proj,4,32,8,128,8.887596130371094,31.5546875,58720256 +24,mlp.gate_proj,4,64,8,128,10.505033493041992,29.77734375,58720256 +24,mlp.gate_proj,4,128,8,128,12.053984642028809,28.888671875,58720256 +24,mlp.gate_proj,8,32,8,128,0.5394554138183594,59.5546875,58720256 +24,mlp.gate_proj,8,64,8,128,0.6379772424697876,57.77734375,58720256 +24,mlp.gate_proj,8,128,8,128,0.7298521399497986,56.888671875,58720256 +24,mlp.up_proj,2,32,8,128,34.384300231933594,17.5546875,58720256 +24,mlp.up_proj,2,64,8,128,40.07299041748047,15.77734375,58720256 +24,mlp.up_proj,2,128,8,128,45.20325469970703,14.888671875,58720256 +24,mlp.up_proj,3,32,8,128,14.705096244812012,24.5546875,58720256 +24,mlp.up_proj,3,64,8,128,17.147953033447266,22.77734375,58720256 +24,mlp.up_proj,3,128,8,128,19.36118507385254,21.888671875,58720256 +24,mlp.up_proj,4,32,8,128,6.858816146850586,31.5546875,58720256 +24,mlp.up_proj,4,64,8,128,8.007072448730469,29.77734375,58720256 +24,mlp.up_proj,4,128,8,128,9.034786224365234,28.888671875,58720256 +24,mlp.up_proj,8,32,8,128,0.41639018058776855,59.5546875,58720256 +24,mlp.up_proj,8,64,8,128,0.48512476682662964,57.77734375,58720256 +24,mlp.up_proj,8,128,8,128,0.546180009841919,56.888671875,58720256 +24,mlp.down_proj,2,32,8,128,34.407413482666016,17.5546875,58720256 +24,mlp.down_proj,2,64,8,128,40.12506866455078,15.77734375,58720256 +24,mlp.down_proj,2,128,8,128,45.28114318847656,14.888671875,58720256 +24,mlp.down_proj,3,32,8,128,14.7113037109375,24.5546875,58720256 +24,mlp.down_proj,3,64,8,128,17.15890884399414,22.77734375,58720256 +24,mlp.down_proj,3,128,8,128,19.36607551574707,21.888671875,58720256 +24,mlp.down_proj,4,32,8,128,6.860785961151123,31.5546875,58720256 +24,mlp.down_proj,4,64,8,128,8.012078285217285,29.77734375,58720256 +24,mlp.down_proj,4,128,8,128,9.043488502502441,28.888671875,58720256 +24,mlp.down_proj,8,32,8,128,0.41663017868995667,59.5546875,58720256 +24,mlp.down_proj,8,64,8,128,0.48549625277519226,57.77734375,58720256 +24,mlp.down_proj,8,128,8,128,0.5465244650840759,56.888671875,58720256 +25,self_attn.q_proj,2,32,8,128,26.38517951965332,5.015625,16777216 +25,self_attn.q_proj,2,64,8,128,31.34227752685547,4.5078125,16777216 +25,self_attn.q_proj,2,128,8,128,36.3751106262207,4.25390625,16777216 +25,self_attn.q_proj,3,32,8,128,11.322434425354004,7.015625,16777216 +25,self_attn.q_proj,3,64,8,128,13.470715522766113,6.5078125,16777216 +25,self_attn.q_proj,3,128,8,128,15.702095031738281,6.25390625,16777216 +25,self_attn.q_proj,4,32,8,128,5.28386116027832,9.015625,16777216 +25,self_attn.q_proj,4,64,8,128,6.288248062133789,8.5078125,16777216 +25,self_attn.q_proj,4,128,8,128,7.33522891998291,8.25390625,16777216 +25,self_attn.q_proj,8,32,8,128,0.3199435770511627,17.015625,16777216 +25,self_attn.q_proj,8,64,8,128,0.38032475113868713,16.5078125,16777216 +25,self_attn.q_proj,8,128,8,128,0.4418511390686035,16.25390625,16777216 +25,self_attn.v_proj,2,32,8,128,10.45468807220459,1.25390625,4194304 +25,self_attn.v_proj,2,64,8,128,12.246832847595215,1.126953125,4194304 +25,self_attn.v_proj,2,128,8,128,13.8378267288208,1.0634765625,4194304 +25,self_attn.v_proj,3,32,8,128,4.489150524139404,1.75390625,4194304 +25,self_attn.v_proj,3,64,8,128,5.279685020446777,1.626953125,4194304 +25,self_attn.v_proj,3,128,8,128,6.0023980140686035,1.5634765625,4194304 +25,self_attn.v_proj,4,32,8,128,2.0939674377441406,2.25390625,4194304 +25,self_attn.v_proj,4,64,8,128,2.464479446411133,2.126953125,4194304 +25,self_attn.v_proj,4,128,8,128,2.801419734954834,2.0634765625,4194304 +25,self_attn.v_proj,8,32,8,128,0.12672945857048035,4.25390625,4194304 +25,self_attn.v_proj,8,64,8,128,0.14913631975650787,4.126953125,4194304 +25,self_attn.v_proj,8,128,8,128,0.16890200972557068,4.0634765625,4194304 +25,self_attn.k_proj,2,32,8,128,19.84722137451172,1.25390625,4194304 +25,self_attn.k_proj,2,64,8,128,23.568706512451172,1.126953125,4194304 +25,self_attn.k_proj,2,128,8,128,26.862586975097656,1.0634765625,4194304 +25,self_attn.k_proj,3,32,8,128,8.53041934967041,1.75390625,4194304 +25,self_attn.k_proj,3,64,8,128,10.198302268981934,1.626953125,4194304 +25,self_attn.k_proj,3,128,8,128,11.756684303283691,1.5634765625,4194304 +25,self_attn.k_proj,4,32,8,128,3.982346773147583,2.25390625,4194304 +25,self_attn.k_proj,4,64,8,128,4.7629523277282715,2.126953125,4194304 +25,self_attn.k_proj,4,128,8,128,5.4980902671813965,2.0634765625,4194304 +25,self_attn.k_proj,8,32,8,128,0.24093852937221527,4.25390625,4194304 +25,self_attn.k_proj,8,64,8,128,0.2880815267562866,4.126953125,4194304 +25,self_attn.k_proj,8,128,8,128,0.33134716749191284,4.0634765625,4194304 +25,self_attn.o_proj,2,32,8,128,18.802457809448242,5.015625,16777216 +25,self_attn.o_proj,2,64,8,128,22.004423141479492,4.5078125,16777216 +25,self_attn.o_proj,2,128,8,128,24.925765991210938,4.25390625,16777216 +25,self_attn.o_proj,3,32,8,128,8.053543090820312,7.015625,16777216 +25,self_attn.o_proj,3,64,8,128,9.452680587768555,6.5078125,16777216 +25,self_attn.o_proj,3,128,8,128,10.770747184753418,6.25390625,16777216 +25,self_attn.o_proj,4,32,8,128,3.758390188217163,9.015625,16777216 +25,self_attn.o_proj,4,64,8,128,4.411430835723877,8.5078125,16777216 +25,self_attn.o_proj,4,128,8,128,5.031032562255859,8.25390625,16777216 +25,self_attn.o_proj,8,32,8,128,0.22738772630691528,17.015625,16777216 +25,self_attn.o_proj,8,64,8,128,0.2668178081512451,16.5078125,16777216 +25,self_attn.o_proj,8,128,8,128,0.3032481372356415,16.25390625,16777216 +25,mlp.gate_proj,2,32,8,128,44.94672775268555,17.5546875,58720256 +25,mlp.gate_proj,2,64,8,128,53.014739990234375,15.77734375,58720256 +25,mlp.gate_proj,2,128,8,128,60.73751449584961,14.888671875,58720256 +25,mlp.gate_proj,3,32,8,128,19.224218368530273,24.5546875,58720256 +25,mlp.gate_proj,3,64,8,128,22.757991790771484,22.77734375,58720256 +25,mlp.gate_proj,3,128,8,128,26.134876251220703,21.888671875,58720256 +25,mlp.gate_proj,4,32,8,128,8.96870231628418,31.5546875,58720256 +25,mlp.gate_proj,4,64,8,128,10.616081237792969,29.77734375,58720256 +25,mlp.gate_proj,4,128,8,128,12.203603744506836,28.888671875,58720256 +25,mlp.gate_proj,8,32,8,128,0.5443291068077087,59.5546875,58720256 +25,mlp.gate_proj,8,64,8,128,0.6446169018745422,57.77734375,58720256 +25,mlp.gate_proj,8,128,8,128,0.7388179302215576,56.888671875,58720256 +25,mlp.up_proj,2,32,8,128,34.63493347167969,17.5546875,58720256 +25,mlp.up_proj,2,64,8,128,40.37312698364258,15.77734375,58720256 +25,mlp.up_proj,2,128,8,128,45.538658142089844,14.888671875,58720256 +25,mlp.up_proj,3,32,8,128,14.812726974487305,24.5546875,58720256 +25,mlp.up_proj,3,64,8,128,17.273929595947266,22.77734375,58720256 +25,mlp.up_proj,3,128,8,128,19.50259017944336,21.888671875,58720256 +25,mlp.up_proj,4,32,8,128,6.907599449157715,31.5546875,58720256 +25,mlp.up_proj,4,64,8,128,8.063788414001465,29.77734375,58720256 +25,mlp.up_proj,4,128,8,128,9.102322578430176,28.888671875,58720256 +25,mlp.up_proj,8,32,8,128,0.41935890913009644,59.5546875,58720256 +25,mlp.up_proj,8,64,8,128,0.488639771938324,57.77734375,58720256 +25,mlp.up_proj,8,128,8,128,0.5504123568534851,56.888671875,58720256 +25,mlp.down_proj,2,32,8,128,34.643646240234375,17.5546875,58720256 +25,mlp.down_proj,2,64,8,128,40.40605926513672,15.77734375,58720256 +25,mlp.down_proj,2,128,8,128,45.60557174682617,14.888671875,58720256 +25,mlp.down_proj,3,32,8,128,14.815231323242188,24.5546875,58720256 +25,mlp.down_proj,3,64,8,128,17.277801513671875,22.77734375,58720256 +25,mlp.down_proj,3,128,8,128,19.511585235595703,21.888671875,58720256 +25,mlp.down_proj,4,32,8,128,6.909078598022461,31.5546875,58720256 +25,mlp.down_proj,4,64,8,128,8.065701484680176,29.77734375,58720256 +25,mlp.down_proj,4,128,8,128,9.099312782287598,28.888671875,58720256 +25,mlp.down_proj,8,32,8,128,0.4195059835910797,59.5546875,58720256 +25,mlp.down_proj,8,64,8,128,0.4888390004634857,57.77734375,58720256 +25,mlp.down_proj,8,128,8,128,0.5503782033920288,56.888671875,58720256 +26,self_attn.q_proj,2,32,8,128,26.72275161743164,5.015625,16777216 +26,self_attn.q_proj,2,64,8,128,31.7291316986084,4.5078125,16777216 +26,self_attn.q_proj,2,128,8,128,36.99882125854492,4.25390625,16777216 +26,self_attn.q_proj,3,32,8,128,11.446874618530273,7.015625,16777216 +26,self_attn.q_proj,3,64,8,128,13.60999584197998,6.5078125,16777216 +26,self_attn.q_proj,3,128,8,128,15.96279525756836,6.25390625,16777216 +26,self_attn.q_proj,4,32,8,128,5.342832565307617,9.015625,16777216 +26,self_attn.q_proj,4,64,8,128,6.353458404541016,8.5078125,16777216 +26,self_attn.q_proj,4,128,8,128,7.452822685241699,8.25390625,16777216 +26,self_attn.q_proj,8,32,8,128,0.32345959544181824,17.015625,16777216 +26,self_attn.q_proj,8,64,8,128,0.38420626521110535,16.5078125,16777216 +26,self_attn.q_proj,8,128,8,128,0.44896960258483887,16.25390625,16777216 +26,self_attn.v_proj,2,32,8,128,10.8711576461792,1.25390625,4194304 +26,self_attn.v_proj,2,64,8,128,12.73658275604248,1.126953125,4194304 +26,self_attn.v_proj,2,128,8,128,14.415816307067871,1.0634765625,4194304 +26,self_attn.v_proj,3,32,8,128,4.669249057769775,1.75390625,4194304 +26,self_attn.v_proj,3,64,8,128,5.484409332275391,1.626953125,4194304 +26,self_attn.v_proj,3,128,8,128,6.233079433441162,1.5634765625,4194304 +26,self_attn.v_proj,4,32,8,128,2.1775381565093994,2.25390625,4194304 +26,self_attn.v_proj,4,64,8,128,2.560832977294922,2.126953125,4194304 +26,self_attn.v_proj,4,128,8,128,2.914492607116699,2.0634765625,4194304 +26,self_attn.v_proj,8,32,8,128,0.13187748193740845,4.25390625,4194304 +26,self_attn.v_proj,8,64,8,128,0.1549156904220581,4.126953125,4194304 +26,self_attn.v_proj,8,128,8,128,0.17574281990528107,4.0634765625,4194304 +26,self_attn.k_proj,2,32,8,128,20.522905349731445,1.25390625,4194304 +26,self_attn.k_proj,2,64,8,128,24.529216766357422,1.126953125,4194304 +26,self_attn.k_proj,2,128,8,128,28.11309814453125,1.0634765625,4194304 +26,self_attn.k_proj,3,32,8,128,8.804459571838379,1.75390625,4194304 +26,self_attn.k_proj,3,64,8,128,10.569839477539062,1.626953125,4194304 +26,self_attn.k_proj,3,128,8,128,12.245599746704102,1.5634765625,4194304 +26,self_attn.k_proj,4,32,8,128,4.108364582061768,2.25390625,4194304 +26,self_attn.k_proj,4,64,8,128,4.934474945068359,2.126953125,4194304 +26,self_attn.k_proj,4,128,8,128,5.715339183807373,2.0634765625,4194304 +26,self_attn.k_proj,8,32,8,128,0.24878785014152527,4.25390625,4194304 +26,self_attn.k_proj,8,64,8,128,0.2984074652194977,4.126953125,4194304 +26,self_attn.k_proj,8,128,8,128,0.3443288505077362,4.0634765625,4194304 +26,self_attn.o_proj,2,32,8,128,19.067707061767578,5.015625,16777216 +26,self_attn.o_proj,2,64,8,128,22.26862144470215,4.5078125,16777216 +26,self_attn.o_proj,2,128,8,128,25.18828582763672,4.25390625,16777216 +26,self_attn.o_proj,3,32,8,128,8.164194107055664,7.015625,16777216 +26,self_attn.o_proj,3,64,8,128,9.552496910095215,6.5078125,16777216 +26,self_attn.o_proj,3,128,8,128,10.836480140686035,6.25390625,16777216 +26,self_attn.o_proj,4,32,8,128,3.8104045391082764,9.015625,16777216 +26,self_attn.o_proj,4,64,8,128,4.458364486694336,8.5078125,16777216 +26,self_attn.o_proj,4,128,8,128,5.061059474945068,8.25390625,16777216 +26,self_attn.o_proj,8,32,8,128,0.2305389791727066,17.015625,16777216 +26,self_attn.o_proj,8,64,8,128,0.26954787969589233,16.5078125,16777216 +26,self_attn.o_proj,8,128,8,128,0.3051247000694275,16.25390625,16777216 +26,mlp.gate_proj,2,32,8,128,45.60263442993164,17.5546875,58720256 +26,mlp.gate_proj,2,64,8,128,54.03689956665039,15.77734375,58720256 +26,mlp.gate_proj,2,128,8,128,62.17963409423828,14.888671875,58720256 +26,mlp.gate_proj,3,32,8,128,19.51502227783203,24.5546875,58720256 +26,mlp.gate_proj,3,64,8,128,23.204566955566406,22.77734375,58720256 +26,mlp.gate_proj,3,128,8,128,26.806968688964844,21.888671875,58720256 +26,mlp.gate_proj,4,32,8,128,9.103170394897461,31.5546875,58720256 +26,mlp.gate_proj,4,64,8,128,10.829730033874512,29.77734375,58720256 +26,mlp.gate_proj,4,128,8,128,12.511475563049316,28.888671875,58720256 +26,mlp.gate_proj,8,32,8,128,0.5526251792907715,59.5546875,58720256 +26,mlp.gate_proj,8,64,8,128,0.6576411128044128,57.77734375,58720256 +26,mlp.gate_proj,8,128,8,128,0.7575823664665222,56.888671875,58720256 +26,mlp.up_proj,2,32,8,128,34.87777328491211,17.5546875,58720256 +26,mlp.up_proj,2,64,8,128,40.6598014831543,15.77734375,58720256 +26,mlp.up_proj,2,128,8,128,45.885169982910156,14.888671875,58720256 +26,mlp.up_proj,3,32,8,128,14.919743537902832,24.5546875,58720256 +26,mlp.up_proj,3,64,8,128,17.397361755371094,22.77734375,58720256 +26,mlp.up_proj,3,128,8,128,19.64816665649414,21.888671875,58720256 +26,mlp.up_proj,4,32,8,128,6.959071159362793,31.5546875,58720256 +26,mlp.up_proj,4,64,8,128,8.120895385742188,29.77734375,58720256 +26,mlp.up_proj,4,128,8,128,9.169035911560059,28.888671875,58720256 +26,mlp.up_proj,8,32,8,128,0.4224383234977722,59.5546875,58720256 +26,mlp.up_proj,8,64,8,128,0.49225977063179016,57.77734375,58720256 +26,mlp.up_proj,8,128,8,128,0.5544010400772095,56.888671875,58720256 +26,mlp.down_proj,2,32,8,128,34.90117645263672,17.5546875,58720256 +26,mlp.down_proj,2,64,8,128,40.71025466918945,15.77734375,58720256 +26,mlp.down_proj,2,128,8,128,45.93507766723633,14.888671875,58720256 +26,mlp.down_proj,3,32,8,128,14.926594734191895,24.5546875,58720256 +26,mlp.down_proj,3,64,8,128,17.40699577331543,22.77734375,58720256 +26,mlp.down_proj,3,128,8,128,19.647537231445312,21.888671875,58720256 +26,mlp.down_proj,4,32,8,128,6.962404251098633,31.5546875,58720256 +26,mlp.down_proj,4,64,8,128,8.126923561096191,29.77734375,58720256 +26,mlp.down_proj,4,128,8,128,9.171987533569336,28.888671875,58720256 +26,mlp.down_proj,8,32,8,128,0.4226948022842407,59.5546875,58720256 +26,mlp.down_proj,8,64,8,128,0.49252015352249146,57.77734375,58720256 +26,mlp.down_proj,8,128,8,128,0.5545075535774231,56.888671875,58720256 +27,self_attn.q_proj,2,32,8,128,26.230398178100586,5.015625,16777216 +27,self_attn.q_proj,2,64,8,128,31.18988037109375,4.5078125,16777216 +27,self_attn.q_proj,2,128,8,128,36.19734191894531,4.25390625,16777216 +27,self_attn.q_proj,3,32,8,128,11.255108833312988,7.015625,16777216 +27,self_attn.q_proj,3,64,8,128,13.404680252075195,6.5078125,16777216 +27,self_attn.q_proj,3,128,8,128,15.6549654006958,6.25390625,16777216 +27,self_attn.q_proj,4,32,8,128,5.252902984619141,9.015625,16777216 +27,self_attn.q_proj,4,64,8,128,6.257029056549072,8.5078125,16777216 +27,self_attn.q_proj,4,128,8,128,7.3085408210754395,8.25390625,16777216 +27,self_attn.q_proj,8,32,8,128,0.31807106733322144,17.015625,16777216 +27,self_attn.q_proj,8,64,8,128,0.3785006105899811,16.5078125,16777216 +27,self_attn.q_proj,8,128,8,128,0.44038981199264526,16.25390625,16777216 +27,self_attn.v_proj,2,32,8,128,11.793103218078613,1.25390625,4194304 +27,self_attn.v_proj,2,64,8,128,13.850370407104492,1.126953125,4194304 +27,self_attn.v_proj,2,128,8,128,15.705147743225098,1.0634765625,4194304 +27,self_attn.v_proj,3,32,8,128,5.086644172668457,1.75390625,4194304 +27,self_attn.v_proj,3,64,8,128,6.016627311706543,1.626953125,4194304 +27,self_attn.v_proj,3,128,8,128,6.8689422607421875,1.5634765625,4194304 +27,self_attn.v_proj,4,32,8,128,2.3751840591430664,2.25390625,4194304 +27,self_attn.v_proj,4,64,8,128,2.8110477924346924,2.126953125,4194304 +27,self_attn.v_proj,4,128,8,128,3.2160511016845703,2.0634765625,4194304 +27,self_attn.v_proj,8,32,8,128,0.14380308985710144,4.25390625,4194304 +27,self_attn.v_proj,8,64,8,128,0.170087993144989,4.126953125,4194304 +27,self_attn.v_proj,8,128,8,128,0.19379420578479767,4.0634765625,4194304 +27,self_attn.k_proj,2,32,8,128,20.30565643310547,1.25390625,4194304 +27,self_attn.k_proj,2,64,8,128,24.19519805908203,1.126953125,4194304 +27,self_attn.k_proj,2,128,8,128,27.621440887451172,1.0634765625,4194304 +27,self_attn.k_proj,3,32,8,128,8.728718757629395,1.75390625,4194304 +27,self_attn.k_proj,3,64,8,128,10.473807334899902,1.626953125,4194304 +27,self_attn.k_proj,3,128,8,128,12.109381675720215,1.5634765625,4194304 +27,self_attn.k_proj,4,32,8,128,4.074509143829346,2.25390625,4194304 +27,self_attn.k_proj,4,64,8,128,4.886240005493164,2.126953125,4194304 +27,self_attn.k_proj,4,128,8,128,5.654834270477295,2.0634765625,4194304 +27,self_attn.k_proj,8,32,8,128,0.2465667426586151,4.25390625,4194304 +27,self_attn.k_proj,8,64,8,128,0.29543790221214294,4.126953125,4194304 +27,self_attn.k_proj,8,128,8,128,0.3407815992832184,4.0634765625,4194304 +27,self_attn.o_proj,2,32,8,128,19.551572799682617,5.015625,16777216 +27,self_attn.o_proj,2,64,8,128,22.85969352722168,4.5078125,16777216 +27,self_attn.o_proj,2,128,8,128,25.89215087890625,4.25390625,16777216 +27,self_attn.o_proj,3,32,8,128,8.389416694641113,7.015625,16777216 +27,self_attn.o_proj,3,64,8,128,9.840544700622559,6.5078125,16777216 +27,self_attn.o_proj,3,128,8,128,11.198309898376465,6.25390625,16777216 +27,self_attn.o_proj,4,32,8,128,3.914794921875,9.015625,16777216 +27,self_attn.o_proj,4,64,8,128,4.593313694000244,8.5078125,16777216 +27,self_attn.o_proj,4,128,8,128,5.237447738647461,8.25390625,16777216 +27,self_attn.o_proj,8,32,8,128,0.2368350476026535,17.015625,16777216 +27,self_attn.o_proj,8,64,8,128,0.27773967385292053,16.5078125,16777216 +27,self_attn.o_proj,8,128,8,128,0.31544139981269836,16.25390625,16777216 +27,mlp.gate_proj,2,32,8,128,46.196205139160156,17.5546875,58720256 +27,mlp.gate_proj,2,64,8,128,54.9052734375,15.77734375,58720256 +27,mlp.gate_proj,2,128,8,128,63.390052795410156,14.888671875,58720256 +27,mlp.gate_proj,3,32,8,128,19.77214241027832,24.5546875,58720256 +27,mlp.gate_proj,3,64,8,128,23.58213233947754,22.77734375,58720256 +27,mlp.gate_proj,3,128,8,128,27.315258026123047,21.888671875,58720256 +27,mlp.gate_proj,4,32,8,128,9.223694801330566,31.5546875,58720256 +27,mlp.gate_proj,4,64,8,128,11.00752067565918,29.77734375,58720256 +27,mlp.gate_proj,4,128,8,128,12.751036643981934,28.888671875,58720256 +27,mlp.gate_proj,8,32,8,128,0.5600003600120544,59.5546875,58720256 +27,mlp.gate_proj,8,64,8,128,0.6684249043464661,57.77734375,58720256 +27,mlp.gate_proj,8,128,8,128,0.772031307220459,56.888671875,58720256 +27,mlp.up_proj,2,32,8,128,35.28170394897461,17.5546875,58720256 +27,mlp.up_proj,2,64,8,128,41.144649505615234,15.77734375,58720256 +27,mlp.up_proj,2,128,8,128,46.43455505371094,14.888671875,58720256 +27,mlp.up_proj,3,32,8,128,15.093939781188965,24.5546875,58720256 +27,mlp.up_proj,3,64,8,128,17.60397720336914,22.77734375,58720256 +27,mlp.up_proj,3,128,8,128,19.90220069885254,21.888671875,58720256 +27,mlp.up_proj,4,32,8,128,7.039376735687256,31.5546875,58720256 +27,mlp.up_proj,4,64,8,128,8.218472480773926,29.77734375,58720256 +27,mlp.up_proj,4,128,8,128,9.283465385437012,28.888671875,58720256 +27,mlp.up_proj,8,32,8,128,0.42727065086364746,59.5546875,58720256 +27,mlp.up_proj,8,64,8,128,0.4980669915676117,57.77734375,58720256 +27,mlp.up_proj,8,128,8,128,0.5613685846328735,56.888671875,58720256 +27,mlp.down_proj,2,32,8,128,35.187835693359375,17.5546875,58720256 +27,mlp.down_proj,2,64,8,128,41.034236907958984,15.77734375,58720256 +27,mlp.down_proj,2,128,8,128,46.29372787475586,14.888671875,58720256 +27,mlp.down_proj,3,32,8,128,15.048783302307129,24.5546875,58720256 +27,mlp.down_proj,3,64,8,128,17.546730041503906,22.77734375,58720256 +27,mlp.down_proj,3,128,8,128,19.809404373168945,21.888671875,58720256 +27,mlp.down_proj,4,32,8,128,7.018793106079102,31.5546875,58720256 +27,mlp.down_proj,4,64,8,128,8.191673278808594,29.77734375,58720256 +27,mlp.down_proj,4,128,8,128,9.246954917907715,28.888671875,58720256 +27,mlp.down_proj,8,32,8,128,0.42602190375328064,59.5546875,58720256 +27,mlp.down_proj,8,64,8,128,0.49651089310646057,57.77734375,58720256 +27,mlp.down_proj,8,128,8,128,0.5591133236885071,56.888671875,58720256 +28,self_attn.q_proj,2,32,8,128,26.428686141967773,5.015625,16777216 +28,self_attn.q_proj,2,64,8,128,31.29534339904785,4.5078125,16777216 +28,self_attn.q_proj,2,128,8,128,36.132442474365234,4.25390625,16777216 +28,self_attn.q_proj,3,32,8,128,11.316201210021973,7.015625,16777216 +28,self_attn.q_proj,3,64,8,128,13.427571296691895,6.5078125,16777216 +28,self_attn.q_proj,3,128,8,128,15.576738357543945,6.25390625,16777216 +28,self_attn.q_proj,4,32,8,128,5.28058385848999,9.015625,16777216 +28,self_attn.q_proj,4,64,8,128,6.2669677734375,8.5078125,16777216 +28,self_attn.q_proj,4,128,8,128,7.273374080657959,8.25390625,16777216 +28,self_attn.q_proj,8,32,8,128,0.3196749985218048,17.015625,16777216 +28,self_attn.q_proj,8,64,8,128,0.3790017366409302,16.5078125,16777216 +28,self_attn.q_proj,8,128,8,128,0.43822887539863586,16.25390625,16777216 +28,self_attn.v_proj,2,32,8,128,12.090920448303223,1.25390625,4194304 +28,self_attn.v_proj,2,64,8,128,14.199821472167969,1.126953125,4194304 +28,self_attn.v_proj,2,128,8,128,16.107200622558594,1.0634765625,4194304 +28,self_attn.v_proj,3,32,8,128,5.192327499389648,1.75390625,4194304 +28,self_attn.v_proj,3,64,8,128,6.12355375289917,1.626953125,4194304 +28,self_attn.v_proj,3,128,8,128,6.98307466506958,1.5634765625,4194304 +28,self_attn.v_proj,4,32,8,128,2.4239346981048584,2.25390625,4194304 +28,self_attn.v_proj,4,64,8,128,2.858705520629883,2.126953125,4194304 +28,self_attn.v_proj,4,128,8,128,3.2625157833099365,2.0634765625,4194304 +28,self_attn.v_proj,8,32,8,128,0.14679282903671265,4.25390625,4194304 +28,self_attn.v_proj,8,64,8,128,0.1729339212179184,4.126953125,4194304 +28,self_attn.v_proj,8,128,8,128,0.196629136800766,4.0634765625,4194304 +28,self_attn.k_proj,2,32,8,128,18.697195053100586,1.25390625,4194304 +28,self_attn.k_proj,2,64,8,128,22.12912368774414,1.126953125,4194304 +28,self_attn.k_proj,2,128,8,128,25.26163101196289,1.0634765625,4194304 +28,self_attn.k_proj,3,32,8,128,8.010294914245605,1.75390625,4194304 +28,self_attn.k_proj,3,64,8,128,9.501382827758789,1.626953125,4194304 +28,self_attn.k_proj,3,128,8,128,10.899209976196289,1.5634765625,4194304 +28,self_attn.k_proj,4,32,8,128,3.738710641860962,2.25390625,4194304 +28,self_attn.k_proj,4,64,8,128,4.435556888580322,2.126953125,4194304 +28,self_attn.k_proj,4,128,8,128,5.081577777862549,2.0634765625,4194304 +28,self_attn.k_proj,8,32,8,128,0.22635699808597565,4.25390625,4194304 +28,self_attn.k_proj,8,64,8,128,0.26823511719703674,4.126953125,4194304 +28,self_attn.k_proj,8,128,8,128,0.3064662516117096,4.0634765625,4194304 +28,self_attn.o_proj,2,32,8,128,20.0314998626709,5.015625,16777216 +28,self_attn.o_proj,2,64,8,128,23.39760971069336,4.5078125,16777216 +28,self_attn.o_proj,2,128,8,128,26.459760665893555,4.25390625,16777216 +28,self_attn.o_proj,3,32,8,128,8.583024024963379,7.015625,16777216 +28,self_attn.o_proj,3,64,8,128,10.0523681640625,6.5078125,16777216 +28,self_attn.o_proj,3,128,8,128,11.400594711303711,6.25390625,16777216 +28,self_attn.o_proj,4,32,8,128,4.0085320472717285,9.015625,16777216 +28,self_attn.o_proj,4,64,8,128,4.696210861206055,8.5078125,16777216 +28,self_attn.o_proj,4,128,8,128,5.339324951171875,8.25390625,16777216 +28,self_attn.o_proj,8,32,8,128,0.2425086945295334,17.015625,16777216 +28,self_attn.o_proj,8,64,8,128,0.2840512692928314,16.5078125,16777216 +28,self_attn.o_proj,8,128,8,128,0.32190805673599243,16.25390625,16777216 +28,mlp.gate_proj,2,32,8,128,46.221500396728516,17.5546875,58720256 +28,mlp.gate_proj,2,64,8,128,55.105674743652344,15.77734375,58720256 +28,mlp.gate_proj,2,128,8,128,63.75630569458008,14.888671875,58720256 +28,mlp.gate_proj,3,32,8,128,19.793174743652344,24.5546875,58720256 +28,mlp.gate_proj,3,64,8,128,23.681921005249023,22.77734375,58720256 +28,mlp.gate_proj,3,128,8,128,27.533466339111328,21.888671875,58720256 +28,mlp.gate_proj,4,32,8,128,9.23330307006836,31.5546875,58720256 +28,mlp.gate_proj,4,64,8,128,11.05634880065918,29.77734375,58720256 +28,mlp.gate_proj,4,128,8,128,12.848603248596191,28.888671875,58720256 +28,mlp.gate_proj,8,32,8,128,0.5605830550193787,59.5546875,58720256 +28,mlp.gate_proj,8,64,8,128,0.6713996529579163,57.77734375,58720256 +28,mlp.gate_proj,8,128,8,128,0.77810138463974,56.888671875,58720256 +28,mlp.up_proj,2,32,8,128,35.966766357421875,17.5546875,58720256 +28,mlp.up_proj,2,64,8,128,41.97301483154297,15.77734375,58720256 +28,mlp.up_proj,2,128,8,128,47.394840240478516,14.888671875,58720256 +28,mlp.up_proj,3,32,8,128,15.388667106628418,24.5546875,58720256 +28,mlp.up_proj,3,64,8,128,17.970359802246094,22.77734375,58720256 +28,mlp.up_proj,3,128,8,128,20.318269729614258,21.888671875,58720256 +28,mlp.up_proj,4,32,8,128,7.176243305206299,31.5546875,58720256 +28,mlp.up_proj,4,64,8,128,8.382697105407715,29.77734375,58720256 +28,mlp.up_proj,4,128,8,128,9.479881286621094,28.888671875,58720256 +28,mlp.up_proj,8,32,8,128,0.43556901812553406,59.5546875,58720256 +28,mlp.up_proj,8,64,8,128,0.508101224899292,57.77734375,58720256 +28,mlp.up_proj,8,128,8,128,0.5732479095458984,56.888671875,58720256 +28,mlp.down_proj,2,32,8,128,35.6870231628418,17.5546875,58720256 +28,mlp.down_proj,2,64,8,128,41.62888717651367,15.77734375,58720256 +28,mlp.down_proj,2,128,8,128,46.97878646850586,14.888671875,58720256 +28,mlp.down_proj,3,32,8,128,15.263729095458984,24.5546875,58720256 +28,mlp.down_proj,3,64,8,128,17.802682876586914,22.77734375,58720256 +28,mlp.down_proj,3,128,8,128,20.10279655456543,21.888671875,58720256 +28,mlp.down_proj,4,32,8,128,7.119475364685059,31.5546875,58720256 +28,mlp.down_proj,4,64,8,128,8.31109619140625,29.77734375,58720256 +28,mlp.down_proj,4,128,8,128,9.380497932434082,28.888671875,58720256 +28,mlp.down_proj,8,32,8,128,0.4321163296699524,59.5546875,58720256 +28,mlp.down_proj,8,64,8,128,0.5037019848823547,57.77734375,58720256 +28,mlp.down_proj,8,128,8,128,0.5675499439239502,56.888671875,58720256 +29,self_attn.q_proj,2,32,8,128,26.00552749633789,5.015625,16777216 +29,self_attn.q_proj,2,64,8,128,30.80367660522461,4.5078125,16777216 +29,self_attn.q_proj,2,128,8,128,35.81844711303711,4.25390625,16777216 +29,self_attn.q_proj,3,32,8,128,11.145312309265137,7.015625,16777216 +29,self_attn.q_proj,3,64,8,128,13.226649284362793,6.5078125,16777216 +29,self_attn.q_proj,3,128,8,128,15.45749282836914,6.25390625,16777216 +29,self_attn.q_proj,4,32,8,128,5.200691223144531,9.015625,16777216 +29,self_attn.q_proj,4,64,8,128,6.171305179595947,8.5078125,16777216 +29,self_attn.q_proj,4,128,8,128,7.21592378616333,8.25390625,16777216 +29,self_attn.q_proj,8,32,8,128,0.3148595988750458,17.015625,16777216 +29,self_attn.q_proj,8,64,8,128,0.37337467074394226,16.5078125,16777216 +29,self_attn.q_proj,8,128,8,128,0.4348812401294708,16.25390625,16777216 +29,self_attn.v_proj,2,32,8,128,12.523959159851074,1.25390625,4194304 +29,self_attn.v_proj,2,64,8,128,14.690808296203613,1.126953125,4194304 +29,self_attn.v_proj,2,128,8,128,16.632190704345703,1.0634765625,4194304 +29,self_attn.v_proj,3,32,8,128,5.392115592956543,1.75390625,4194304 +29,self_attn.v_proj,3,64,8,128,6.368161201477051,1.626953125,4194304 +29,self_attn.v_proj,3,128,8,128,7.268193244934082,1.5634765625,4194304 +29,self_attn.v_proj,4,32,8,128,2.5151612758636475,2.25390625,4194304 +29,self_attn.v_proj,4,64,8,128,2.9734504222869873,2.126953125,4194304 +29,self_attn.v_proj,4,128,8,128,3.403775215148926,2.0634765625,4194304 +29,self_attn.v_proj,8,32,8,128,0.15232786536216736,4.25390625,4194304 +29,self_attn.v_proj,8,64,8,128,0.17989321053028107,4.126953125,4194304 +29,self_attn.v_proj,8,128,8,128,0.205173522233963,4.0634765625,4194304 +29,self_attn.k_proj,2,32,8,128,19.7725830078125,1.25390625,4194304 +29,self_attn.k_proj,2,64,8,128,23.471574783325195,1.126953125,4194304 +29,self_attn.k_proj,2,128,8,128,26.8720760345459,1.0634765625,4194304 +29,self_attn.k_proj,3,32,8,128,8.48144245147705,1.75390625,4194304 +29,self_attn.k_proj,3,64,8,128,10.10207462310791,1.626953125,4194304 +29,self_attn.k_proj,3,128,8,128,11.622701644897461,1.5634765625,4194304 +29,self_attn.k_proj,4,32,8,128,3.9573285579681396,2.25390625,4194304 +29,self_attn.k_proj,4,64,8,128,4.716329574584961,2.126953125,4194304 +29,self_attn.k_proj,4,128,8,128,5.431185245513916,2.0634765625,4194304 +29,self_attn.k_proj,8,32,8,128,0.23949669301509857,4.25390625,4194304 +29,self_attn.k_proj,8,64,8,128,0.28515973687171936,4.126953125,4194304 +29,self_attn.k_proj,8,128,8,128,0.327343612909317,4.0634765625,4194304 +29,self_attn.o_proj,2,32,8,128,20.647443771362305,5.015625,16777216 +29,self_attn.o_proj,2,64,8,128,24.1241397857666,4.5078125,16777216 +29,self_attn.o_proj,2,128,8,128,27.31800079345703,4.25390625,16777216 +29,self_attn.o_proj,3,32,8,128,8.838510513305664,7.015625,16777216 +29,self_attn.o_proj,3,64,8,128,10.343005180358887,6.5078125,16777216 +29,self_attn.o_proj,3,128,8,128,11.744430541992188,6.25390625,16777216 +29,self_attn.o_proj,4,32,8,128,4.123586654663086,9.015625,16777216 +29,self_attn.o_proj,4,64,8,128,4.828439235687256,8.5078125,16777216 +29,self_attn.o_proj,4,128,8,128,5.48347806930542,8.25390625,16777216 +29,self_attn.o_proj,8,32,8,128,0.24951261281967163,17.015625,16777216 +29,self_attn.o_proj,8,64,8,128,0.29185888171195984,16.5078125,16777216 +29,self_attn.o_proj,8,128,8,128,0.33035680651664734,16.25390625,16777216 +29,mlp.gate_proj,2,32,8,128,46.53538513183594,17.5546875,58720256 +29,mlp.gate_proj,2,64,8,128,55.688201904296875,15.77734375,58720256 +29,mlp.gate_proj,2,128,8,128,64.65599822998047,14.888671875,58720256 +29,mlp.gate_proj,3,32,8,128,19.936012268066406,24.5546875,58720256 +29,mlp.gate_proj,3,64,8,128,23.951101303100586,22.77734375,58720256 +29,mlp.gate_proj,3,128,8,128,27.972835540771484,21.888671875,58720256 +29,mlp.gate_proj,4,32,8,128,9.299413681030273,31.5546875,58720256 +29,mlp.gate_proj,4,64,8,128,11.182499885559082,29.77734375,58720256 +29,mlp.gate_proj,4,128,8,128,13.054163932800293,28.888671875,58720256 +29,mlp.gate_proj,8,32,8,128,0.5645514726638794,59.5546875,58720256 +29,mlp.gate_proj,8,64,8,128,0.6790639758110046,57.77734375,58720256 +29,mlp.gate_proj,8,128,8,128,0.7904157042503357,56.888671875,58720256 +29,mlp.up_proj,2,32,8,128,36.85078430175781,17.5546875,58720256 +29,mlp.up_proj,2,64,8,128,43.05399703979492,15.77734375,58720256 +29,mlp.up_proj,2,128,8,128,48.64891815185547,14.888671875,58720256 +29,mlp.up_proj,3,32,8,128,15.770960807800293,24.5546875,58720256 +29,mlp.up_proj,3,64,8,128,18.416637420654297,22.77734375,58720256 +29,mlp.up_proj,3,128,8,128,20.84731101989746,21.888671875,58720256 +29,mlp.up_proj,4,32,8,128,7.353880882263184,31.5546875,58720256 +29,mlp.up_proj,4,64,8,128,8.594264030456543,29.77734375,58720256 +29,mlp.up_proj,4,128,8,128,9.726106643676758,28.888671875,58720256 +29,mlp.up_proj,8,32,8,128,0.4461195766925812,59.5546875,58720256 +29,mlp.up_proj,8,64,8,128,0.5211239457130432,57.77734375,58720256 +29,mlp.up_proj,8,128,8,128,0.5885738730430603,56.888671875,58720256 +29,mlp.down_proj,2,32,8,128,36.1666145324707,17.5546875,58720256 +29,mlp.down_proj,2,64,8,128,42.19386291503906,15.77734375,58720256 +29,mlp.down_proj,2,128,8,128,47.618167877197266,14.888671875,58720256 +29,mlp.down_proj,3,32,8,128,15.470270156860352,24.5546875,58720256 +29,mlp.down_proj,3,64,8,128,18.048030853271484,22.77734375,58720256 +29,mlp.down_proj,3,128,8,128,20.406352996826172,21.888671875,58720256 +29,mlp.down_proj,4,32,8,128,7.214730262756348,31.5546875,58720256 +29,mlp.down_proj,4,64,8,128,8.423977851867676,29.77734375,58720256 +29,mlp.down_proj,4,128,8,128,9.518974304199219,28.888671875,58720256 +29,mlp.down_proj,8,32,8,128,0.43782734870910645,59.5546875,58720256 +29,mlp.down_proj,8,64,8,128,0.5107836723327637,57.77734375,58720256 +29,mlp.down_proj,8,128,8,128,0.5759627223014832,56.888671875,58720256 +30,self_attn.q_proj,2,32,8,128,25.196308135986328,5.015625,16777216 +30,self_attn.q_proj,2,64,8,128,30.050796508789062,4.5078125,16777216 +30,self_attn.q_proj,2,128,8,128,34.93693542480469,4.25390625,16777216 +30,self_attn.q_proj,3,32,8,128,10.820404052734375,7.015625,16777216 +30,self_attn.q_proj,3,64,8,128,12.94897747039795,6.5078125,16777216 +30,self_attn.q_proj,3,128,8,128,15.237585067749023,6.25390625,16777216 +30,self_attn.q_proj,4,32,8,128,5.0505194664001465,9.015625,16777216 +30,self_attn.q_proj,4,64,8,128,6.043151378631592,8.5078125,16777216 +30,self_attn.q_proj,4,128,8,128,7.112979412078857,8.25390625,16777216 +30,self_attn.q_proj,8,32,8,128,0.30580830574035645,17.015625,16777216 +30,self_attn.q_proj,8,64,8,128,0.36565133929252625,16.5078125,16777216 +30,self_attn.q_proj,8,128,8,128,0.4289668798446655,16.25390625,16777216 +30,self_attn.v_proj,2,32,8,128,15.02125072479248,1.25390625,4194304 +30,self_attn.v_proj,2,64,8,128,17.631624221801758,1.126953125,4194304 +30,self_attn.v_proj,2,128,8,128,19.994483947753906,1.0634765625,4194304 +30,self_attn.v_proj,3,32,8,128,6.480691909790039,1.75390625,4194304 +30,self_attn.v_proj,3,64,8,128,7.6596879959106445,1.626953125,4194304 +30,self_attn.v_proj,3,128,8,128,8.729578971862793,1.5634765625,4194304 +30,self_attn.v_proj,4,32,8,128,3.0260024070739746,2.25390625,4194304 +30,self_attn.v_proj,4,64,8,128,3.579402208328247,2.126953125,4194304 +30,self_attn.v_proj,4,128,8,128,4.09032678604126,2.0634765625,4194304 +30,self_attn.v_proj,8,32,8,128,0.18331295251846313,4.25390625,4194304 +30,self_attn.v_proj,8,64,8,128,0.21650968492031097,4.126953125,4194304 +30,self_attn.v_proj,8,128,8,128,0.24649925529956818,4.0634765625,4194304 +30,self_attn.k_proj,2,32,8,128,17.458175659179688,1.25390625,4194304 +30,self_attn.k_proj,2,64,8,128,20.79300308227539,1.126953125,4194304 +30,self_attn.k_proj,2,128,8,128,23.82550621032715,1.0634765625,4194304 +30,self_attn.k_proj,3,32,8,128,7.502180099487305,1.75390625,4194304 +30,self_attn.k_proj,3,64,8,128,8.99234676361084,1.626953125,4194304 +30,self_attn.k_proj,3,128,8,128,10.396160125732422,1.5634765625,4194304 +30,self_attn.k_proj,4,32,8,128,3.498915672302246,2.25390625,4194304 +30,self_attn.k_proj,4,64,8,128,4.198184013366699,2.126953125,4194304 +30,self_attn.k_proj,4,128,8,128,4.856632232666016,2.0634765625,4194304 +30,self_attn.k_proj,8,32,8,128,0.2118491381406784,4.25390625,4194304 +30,self_attn.k_proj,8,64,8,128,0.25399449467658997,4.126953125,4194304 +30,self_attn.k_proj,8,128,8,128,0.29271388053894043,4.0634765625,4194304 +30,self_attn.o_proj,2,32,8,128,22.028724670410156,5.015625,16777216 +30,self_attn.o_proj,2,64,8,128,25.77220916748047,4.5078125,16777216 +30,self_attn.o_proj,2,128,8,128,29.217456817626953,4.25390625,16777216 +30,self_attn.o_proj,3,32,8,128,9.431700706481934,7.015625,16777216 +30,self_attn.o_proj,3,64,8,128,11.060696601867676,6.5078125,16777216 +30,self_attn.o_proj,3,128,8,128,12.583657264709473,6.25390625,16777216 +30,self_attn.o_proj,4,32,8,128,4.399063587188721,9.015625,16777216 +30,self_attn.o_proj,4,64,8,128,5.1628499031066895,8.5078125,16777216 +30,self_attn.o_proj,4,128,8,128,5.875484943389893,8.25390625,16777216 +30,self_attn.o_proj,8,32,8,128,0.26631367206573486,17.015625,16777216 +30,self_attn.o_proj,8,64,8,128,0.31211280822753906,16.5078125,16777216 +30,self_attn.o_proj,8,128,8,128,0.353950172662735,16.25390625,16777216 +30,mlp.gate_proj,2,32,8,128,48.54814147949219,17.5546875,58720256 +30,mlp.gate_proj,2,64,8,128,58.27192306518555,15.77734375,58720256 +30,mlp.gate_proj,2,128,8,128,67.65760803222656,14.888671875,58720256 +30,mlp.gate_proj,3,32,8,128,20.802818298339844,24.5546875,58720256 +30,mlp.gate_proj,3,64,8,128,25.045467376708984,22.77734375,58720256 +30,mlp.gate_proj,3,128,8,128,29.281566619873047,21.888671875,58720256 +30,mlp.gate_proj,4,32,8,128,9.702558517456055,31.5546875,58720256 +30,mlp.gate_proj,4,64,8,128,11.69450569152832,29.77734375,58720256 +30,mlp.gate_proj,4,128,8,128,13.667594909667969,28.888671875,58720256 +30,mlp.gate_proj,8,32,8,128,0.589381217956543,59.5546875,58720256 +30,mlp.gate_proj,8,64,8,128,0.7104009389877319,57.77734375,58720256 +30,mlp.gate_proj,8,128,8,128,0.8273317813873291,56.888671875,58720256 +30,mlp.up_proj,2,32,8,128,37.81081008911133,17.5546875,58720256 +30,mlp.up_proj,2,64,8,128,44.27680587768555,15.77734375,58720256 +30,mlp.up_proj,2,128,8,128,50.141212463378906,14.888671875,58720256 +30,mlp.up_proj,3,32,8,128,16.179561614990234,24.5546875,58720256 +30,mlp.up_proj,3,64,8,128,18.94173240661621,22.77734375,58720256 +30,mlp.up_proj,3,128,8,128,21.498088836669922,21.888671875,58720256 +30,mlp.up_proj,4,32,8,128,7.546878337860107,31.5546875,58720256 +30,mlp.up_proj,4,64,8,128,8.837961196899414,29.77734375,58720256 +30,mlp.up_proj,4,128,8,128,10.029991149902344,28.888671875,58720256 +30,mlp.up_proj,8,32,8,128,0.4577595591545105,59.5546875,58720256 +30,mlp.up_proj,8,64,8,128,0.5360674858093262,57.77734375,58720256 +30,mlp.up_proj,8,128,8,128,0.606948971748352,56.888671875,58720256 +30,mlp.down_proj,2,32,8,128,36.228599548339844,17.5546875,58720256 +30,mlp.down_proj,2,64,8,128,42.291839599609375,15.77734375,58720256 +30,mlp.down_proj,2,128,8,128,47.77397155761719,14.888671875,58720256 +30,mlp.down_proj,3,32,8,128,15.508816719055176,24.5546875,58720256 +30,mlp.down_proj,3,64,8,128,18.11805534362793,22.77734375,58720256 +30,mlp.down_proj,3,128,8,128,20.528676986694336,21.888671875,58720256 +30,mlp.down_proj,4,32,8,128,7.232919216156006,31.5546875,58720256 +30,mlp.down_proj,4,64,8,128,8.457255363464355,29.77734375,58720256 +30,mlp.down_proj,4,128,8,128,9.585814476013184,28.888671875,58720256 +30,mlp.down_proj,8,32,8,128,0.4388901889324188,59.5546875,58720256 +30,mlp.down_proj,8,64,8,128,0.512734055519104,57.77734375,58720256 +30,mlp.down_proj,8,128,8,128,0.5796442627906799,56.888671875,58720256 +31,self_attn.q_proj,2,32,8,128,27.7853946685791,5.015625,16777216 +31,self_attn.q_proj,2,64,8,128,33.145084381103516,4.5078125,16777216 +31,self_attn.q_proj,2,128,8,128,38.488712310791016,4.25390625,16777216 +31,self_attn.q_proj,3,32,8,128,11.928686141967773,7.015625,16777216 +31,self_attn.q_proj,3,64,8,128,14.285934448242188,6.5078125,16777216 +31,self_attn.q_proj,3,128,8,128,16.74061393737793,6.25390625,16777216 +31,self_attn.q_proj,4,32,8,128,5.565404891967773,9.015625,16777216 +31,self_attn.q_proj,4,64,8,128,6.667606830596924,8.5078125,16777216 +31,self_attn.q_proj,4,128,8,128,7.818569660186768,8.25390625,16777216 +31,self_attn.q_proj,8,32,8,128,0.3369702696800232,17.015625,16777216 +31,self_attn.q_proj,8,64,8,128,0.403188556432724,16.5078125,16777216 +31,self_attn.q_proj,8,128,8,128,0.47114652395248413,16.25390625,16777216 +31,self_attn.v_proj,2,32,8,128,13.460245132446289,1.25390625,4194304 +31,self_attn.v_proj,2,64,8,128,15.933310508728027,1.126953125,4194304 +31,self_attn.v_proj,2,128,8,128,18.187244415283203,1.0634765625,4194304 +31,self_attn.v_proj,3,32,8,128,5.793933391571045,1.75390625,4194304 +31,self_attn.v_proj,3,64,8,128,6.89697265625,1.626953125,4194304 +31,self_attn.v_proj,3,128,8,128,7.944997787475586,1.5634765625,4194304 +31,self_attn.v_proj,4,32,8,128,2.703720808029175,2.25390625,4194304 +31,self_attn.v_proj,4,64,8,128,3.2235920429229736,2.126953125,4194304 +31,self_attn.v_proj,4,128,8,128,3.717491388320923,2.0634765625,4194304 +31,self_attn.v_proj,8,32,8,128,0.16373677551746368,4.25390625,4194304 +31,self_attn.v_proj,8,64,8,128,0.1949443519115448,4.126953125,4194304 +31,self_attn.v_proj,8,128,8,128,0.22399581968784332,4.0634765625,4194304 +31,self_attn.k_proj,2,32,8,128,18.943012237548828,1.25390625,4194304 +31,self_attn.k_proj,2,64,8,128,22.429168701171875,1.126953125,4194304 +31,self_attn.k_proj,2,128,8,128,25.610763549804688,1.0634765625,4194304 +31,self_attn.k_proj,3,32,8,128,8.12702465057373,1.75390625,4194304 +31,self_attn.k_proj,3,64,8,128,9.663982391357422,1.626953125,4194304 +31,self_attn.k_proj,3,128,8,128,11.10566234588623,1.5634765625,4194304 +31,self_attn.k_proj,4,32,8,128,3.791293144226074,2.25390625,4194304 +31,self_attn.k_proj,4,64,8,128,4.509145259857178,2.126953125,4194304 +31,self_attn.k_proj,4,128,8,128,5.187222957611084,2.0634765625,4194304 +31,self_attn.k_proj,8,32,8,128,0.22956740856170654,4.25390625,4194304 +31,self_attn.k_proj,8,64,8,128,0.27280306816101074,4.126953125,4194304 +31,self_attn.k_proj,8,128,8,128,0.31249260902404785,4.0634765625,4194304 +31,self_attn.o_proj,2,32,8,128,21.93998146057129,5.015625,16777216 +31,self_attn.o_proj,2,64,8,128,25.865713119506836,4.5078125,16777216 +31,self_attn.o_proj,2,128,8,128,29.529573440551758,4.25390625,16777216 +31,self_attn.o_proj,3,32,8,128,9.455382347106934,7.015625,16777216 +31,self_attn.o_proj,3,64,8,128,11.25583553314209,6.5078125,16777216 +31,self_attn.o_proj,3,128,8,128,13.053057670593262,6.25390625,16777216 +31,self_attn.o_proj,4,32,8,128,4.415661334991455,9.015625,16777216 +31,self_attn.o_proj,4,64,8,128,5.26674747467041,8.5078125,16777216 +31,self_attn.o_proj,4,128,8,128,6.131242752075195,8.25390625,16777216 +31,self_attn.o_proj,8,32,8,128,0.2670601010322571,17.015625,16777216 +31,self_attn.o_proj,8,64,8,128,0.3184683918952942,16.5078125,16777216 +31,self_attn.o_proj,8,128,8,128,0.3694242537021637,16.25390625,16777216 +31,mlp.gate_proj,2,32,8,128,55.132991790771484,17.5546875,58720256 +31,mlp.gate_proj,2,64,8,128,66.1739501953125,15.77734375,58720256 +31,mlp.gate_proj,2,128,8,128,76.86265563964844,14.888671875,58720256 +31,mlp.gate_proj,3,32,8,128,23.65062141418457,24.5546875,58720256 +31,mlp.gate_proj,3,64,8,128,28.497373580932617,22.77734375,58720256 +31,mlp.gate_proj,3,128,8,128,33.36634826660156,21.888671875,58720256 +31,mlp.gate_proj,4,32,8,128,11.035598754882812,31.5546875,58720256 +31,mlp.gate_proj,4,64,8,128,13.294682502746582,29.77734375,58720256 +31,mlp.gate_proj,4,128,8,128,15.581149101257324,28.888671875,58720256 +31,mlp.gate_proj,8,32,8,128,0.6704089045524597,59.5546875,58720256 +31,mlp.gate_proj,8,64,8,128,0.8073409199714661,57.77734375,58720256 +31,mlp.gate_proj,8,128,8,128,0.9419750571250916,56.888671875,58720256 +31,mlp.up_proj,2,32,8,128,43.796512603759766,17.5546875,58720256 +31,mlp.up_proj,2,64,8,128,51.55417251586914,15.77734375,58720256 +31,mlp.up_proj,2,128,8,128,58.87849044799805,14.888671875,58720256 +31,mlp.up_proj,3,32,8,128,18.743408203125,24.5546875,58720256 +31,mlp.up_proj,3,64,8,128,22.15771484375,22.77734375,58720256 +31,mlp.up_proj,3,128,8,128,25.394575119018555,21.888671875,58720256 +31,mlp.up_proj,4,32,8,128,8.745729446411133,31.5546875,58720256 +31,mlp.up_proj,4,64,8,128,10.336227416992188,29.77734375,58720256 +31,mlp.up_proj,4,128,8,128,11.863717079162598,28.888671875,58720256 +31,mlp.up_proj,8,32,8,128,0.5307725071907043,59.5546875,58720256 +31,mlp.up_proj,8,64,8,128,0.6275953054428101,57.77734375,58720256 +31,mlp.up_proj,8,128,8,128,0.7182684540748596,56.888671875,58720256 +31,mlp.down_proj,2,32,8,128,37.130252838134766,17.5546875,58720256 +31,mlp.down_proj,2,64,8,128,43.456199645996094,15.77734375,58720256 +31,mlp.down_proj,2,128,8,128,49.1611328125,14.888671875,58720256 +31,mlp.down_proj,3,32,8,128,15.918204307556152,24.5546875,58720256 +31,mlp.down_proj,3,64,8,128,18.65615463256836,22.77734375,58720256 +31,mlp.down_proj,3,128,8,128,21.233890533447266,21.888671875,58720256 +31,mlp.down_proj,4,32,8,128,7.429920196533203,31.5546875,58720256 +31,mlp.down_proj,4,64,8,128,8.717047691345215,29.77734375,58720256 +31,mlp.down_proj,4,128,8,128,9.929636001586914,28.888671875,58720256 +31,mlp.down_proj,8,32,8,128,0.4507500231266022,59.5546875,58720256 +31,mlp.down_proj,8,64,8,128,0.5288225412368774,57.77734375,58720256 +31,mlp.down_proj,8,128,8,128,0.6011541485786438,56.888671875,58720256 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Meta-Llama-3-8B.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Meta-Llama-3-8B.csv new file mode 100644 index 0000000000000000000000000000000000000000..7a1a7dbafacc9b68d366d789e84cd734ee26c2e9 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-Meta-Llama-3-8B.csv @@ -0,0 +1,2689 @@ +layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params,kurtosis,cost_sensi,sensitivity +0,self_attn.q_proj,2,32,8,128,29.71926498413086,5.015625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,2,64,8,128,36.01884460449219,4.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,2,128,8,128,41.75963973999024,4.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,3,32,8,128,12.962769508361816,7.015625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,3,64,8,128,16.056236267089844,6.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,3,128,8,128,19.37236785888672,6.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,4,32,8,128,6.056663036346436,9.015625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,4,64,8,128,7.511712551116943,8.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,4,128,8,128,9.0902862548828125,8.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,8,32,8,128,0.3672587275505066,17.015625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,8,64,8,128,0.4547851383686065,16.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.q_proj,8,128,8,128,0.5482342839241028,16.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7 +0,self_attn.v_proj,2,32,8,128,5.1504902839660645,1.25390625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,2,64,8,128,6.072350025177002,1.126953125,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,2,128,8,128,6.909933567047119,1.0634765625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,3,32,8,128,2.2044689655303955,1.75390625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,3,64,8,128,2.6054704189300537,1.626953125,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,3,128,8,128,2.9787304401397705,1.5634765625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,4,32,8,128,1.0286246538162231,2.25390625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,4,64,8,128,1.2157593965530396,2.126953125,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,4,128,8,128,1.390173077583313,2.0634765625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,8,32,8,128,0.0632590651512146,4.25390625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,8,64,8,128,0.074266493320465,4.126953125,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.v_proj,8,128,8,128,0.0843224003911018,4.0634765625,4194304,5.243808587901302,NA,2.350435579501209e-7 +0,self_attn.k_proj,2,32,8,128,21.851238250732425,1.25390625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,2,64,8,128,26.34428596496582,1.126953125,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,2,128,8,128,30.26717185974121,1.0634765625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,3,32,8,128,9.60117244720459,1.75390625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,3,64,8,128,11.86343479156494,1.626953125,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,3,128,8,128,14.062352180480955,1.5634765625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,4,32,8,128,4.484797477722168,2.25390625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,4,64,8,128,5.547629356384277,2.126953125,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,4,128,8,128,6.592917919158936,2.0634765625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,8,32,8,128,0.2718508839607239,4.25390625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,8,64,8,128,0.3354998826980591,4.126953125,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,self_attn.k_proj,8,128,8,128,0.3979018926620483,4.0634765625,4194304,19.91079330638038,NA,2.350435579501209e-7 +0,mlp.gate_proj,2,32,8,128,33.444522857666016,17.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,2,64,8,128,39.36700820922852,15.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,2,128,8,128,44.94590759277344,14.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,3,32,8,128,14.31833267211914,24.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,3,64,8,128,16.907611846923828,22.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,3,128,8,128,19.3914794921875,21.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,4,32,8,128,6.679895401000977,31.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,4,64,8,128,7.8970112800598145,29.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,4,128,8,128,9.0552396774292,28.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,8,32,8,128,0.40571129322052,59.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,8,64,8,128,0.4786460995674133,57.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.gate_proj,8,128,8,128,0.5475429892539978,56.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7 +0,mlp.up_proj,2,32,8,128,30.09317207336425,17.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,2,64,8,128,35.18780517578125,15.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,2,128,8,128,39.822261810302734,14.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,3,32,8,128,12.867423057556152,24.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,3,64,8,128,15.071465492248535,22.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,3,128,8,128,17.074993133544922,21.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,4,32,8,128,6.00413179397583,31.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,4,64,8,128,7.029482841491699,29.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,4,128,8,128,7.972204685211182,28.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,8,32,8,128,0.3648918867111206,59.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,8,64,8,128,0.4266665875911712,57.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.up_proj,8,128,8,128,0.4817230999469757,56.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7 +0,mlp.down_proj,2,32,8,128,30.121797561645508,17.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,2,64,8,128,35.37422180175781,15.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,2,128,8,128,40.296199798583984,14.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,3,32,8,128,12.907261848449709,24.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,3,64,8,128,15.229618072509766,22.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,3,128,8,128,17.438817977905273,21.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,4,32,8,128,6.028163909912109,31.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,4,64,8,128,7.115785121917725,29.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,4,128,8,128,8.164895057678223,28.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,8,32,8,128,0.3665659725666046,59.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,8,64,8,128,0.4323022961616516,57.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6 +0,mlp.down_proj,8,128,8,128,0.4949232637882232,56.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6 +1,self_attn.q_proj,2,32,8,128,30.526927947998047,5.015625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,2,64,8,128,36.505821228027344,4.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,2,128,8,128,42.13641357421875,4.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,3,32,8,128,13.311883926391602,7.015625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,3,64,8,128,16.21119499206543,6.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,3,128,8,128,19.35423851013184,6.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,4,32,8,128,6.221262454986572,9.015625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,4,64,8,128,7.588432788848877,8.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,4,128,8,128,9.07388687133789,8.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,8,32,8,128,0.3765309751033783,17.015625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,8,64,8,128,0.4591035544872284,16.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.q_proj,8,128,8,128,0.5472977161407471,16.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7 +1,self_attn.v_proj,2,32,8,128,6.054126739501953,1.25390625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,2,64,8,128,7.120873928070068,1.126953125,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,2,128,8,128,8.102333068847656,1.0634765625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,3,32,8,128,2.589040279388428,1.75390625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,3,64,8,128,3.047174453735352,1.626953125,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,3,128,8,128,3.4693777561187744,1.5634765625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,4,32,8,128,1.207473874092102,2.25390625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,4,64,8,128,1.4214165210723877,2.126953125,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,4,128,8,128,1.6196296215057373,2.0634765625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,8,32,8,128,0.0731801688671112,4.25390625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,8,64,8,128,0.086038313806056907,4.126953125,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.v_proj,8,128,8,128,0.0976540595293045,4.0634765625,4194304,3.511953896727226,NA,3.939759949389554e-7 +1,self_attn.k_proj,2,32,8,128,22.087326049804688,1.25390625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,2,64,8,128,26.22841453552246,1.126953125,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,2,128,8,128,29.570714950561523,1.0634765625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,3,32,8,128,9.688775062561035,1.75390625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,3,64,8,128,11.815810203552246,1.626953125,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,3,128,8,128,13.7361421585083,1.5634765625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,4,32,8,128,4.534592151641846,2.25390625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,4,64,8,128,5.555020809173584,2.126953125,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,4,128,8,128,6.490731239318848,2.0634765625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,8,32,8,128,0.2743088901042938,4.25390625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,8,64,8,128,0.3359671831130981,4.126953125,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,self_attn.k_proj,8,128,8,128,0.3915407955646515,4.0634765625,4194304,6.762759900412172,NA,3.939759949389554e-7 +1,mlp.gate_proj,2,32,8,128,33.596656799316406,17.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,2,64,8,128,39.39640045166016,15.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,2,128,8,128,44.80052947998047,14.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,3,32,8,128,14.38129425048828,24.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,3,64,8,128,16.90346336364746,22.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,3,128,8,128,19.27972412109375,21.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,4,32,8,128,6.708982944488525,31.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,4,64,8,128,7.894912719726562,29.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,4,128,8,128,9.00425910949707,28.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,8,32,8,128,0.407469242811203,59.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,8,64,8,128,0.4784776568412781,57.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.gate_proj,8,128,8,128,0.544376790523529,56.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6 +1,mlp.up_proj,2,32,8,128,30.42632484436035,17.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,2,64,8,128,35.52070617675781,15.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,2,128,8,128,40.1715087890625,14.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,3,32,8,128,13.010882377624512,24.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,3,64,8,128,15.230850219726562,22.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,3,128,8,128,17.25382423400879,21.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,4,32,8,128,6.071969032287598,31.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,4,64,8,128,7.105710029602051,29.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,4,128,8,128,8.0638580322265625,28.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,8,32,8,128,0.368994653224945,59.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,8,64,8,128,0.4312596619129181,57.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.up_proj,8,128,8,128,0.4871818125247955,56.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6 +1,mlp.down_proj,2,32,8,128,30.320220947265625,17.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,2,64,8,128,35.40483474731445,15.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,2,128,8,128,39.99724960327149,14.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,3,32,8,128,12.976301193237305,24.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,3,64,8,128,15.205132484436035,22.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,3,128,8,128,17.241455078125,21.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,4,32,8,128,6.056885719299316,31.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,4,64,8,128,7.096342086791992,29.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,4,128,8,128,8.062268257141113,28.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,8,32,8,128,0.3681756258010864,59.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,8,64,8,128,0.4309409260749817,57.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5 +1,mlp.down_proj,8,128,8,128,0.4879005849361419,56.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5 +2,self_attn.q_proj,2,32,8,128,28.29009246826172,5.015625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,2,64,8,128,33.812381744384766,4.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,2,128,8,128,39.00408935546875,4.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,3,32,8,128,12.197437286376951,7.015625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,3,64,8,128,14.689261436462402,6.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,3,128,8,128,17.301185607910156,6.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,4,32,8,128,5.6953887939453125,9.015625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,4,64,8,128,6.862927913665772,8.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,4,128,8,128,8.08891773223877,8.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,8,32,8,128,0.3447518348693847,17.015625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,8,64,8,128,0.4150609374046325,16.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.q_proj,8,128,8,128,0.4876388013362884,16.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7 +2,self_attn.v_proj,2,32,8,128,5.159294128417969,1.25390625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,2,64,8,128,6.081952095031738,1.126953125,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,2,128,8,128,6.926016807556152,1.0634765625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,3,32,8,128,2.206528902053833,1.75390625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,3,64,8,128,2.6030614376068115,1.626953125,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,3,128,8,128,2.9701642990112305,1.5634765625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,4,32,8,128,1.028939962387085,2.25390625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,4,64,8,128,1.2139078378677368,2.126953125,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,4,128,8,128,1.3857122659683228,2.0634765625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,8,32,8,128,0.0623427294194698,4.25390625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,8,64,8,128,0.0734634175896644,4.126953125,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.v_proj,8,128,8,128,0.0835602283477783,4.0634765625,4194304,3.581136549491772,NA,9.326020631306164e-7 +2,self_attn.k_proj,2,32,8,128,22.59001541137696,1.25390625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,2,64,8,128,26.932594299316406,1.126953125,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,2,128,8,128,30.45580291748047,1.0634765625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,3,32,8,128,9.799921035766602,1.75390625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,3,64,8,128,11.85670566558838,1.626953125,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,3,128,8,128,13.769078254699709,1.5634765625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,4,32,8,128,4.577732563018799,2.25390625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,4,64,8,128,5.542038440704346,2.126953125,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,4,128,8,128,6.441653728485107,2.0634765625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,8,32,8,128,0.276971161365509,4.25390625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,8,64,8,128,0.3351361751556396,4.126953125,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,self_attn.k_proj,8,128,8,128,0.3882458806037903,4.0634765625,4194304,5.358498552438096,NA,9.326020631306164e-7 +2,mlp.gate_proj,2,32,8,128,34.53382873535156,17.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,2,64,8,128,40.52213287353516,15.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,2,128,8,128,46.107120513916016,14.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,3,32,8,128,14.783581733703612,24.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,3,64,8,128,17.375429153442383,22.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,3,128,8,128,19.83789253234864,21.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,4,32,8,128,6.895365238189697,31.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,4,64,8,128,8.111905097961426,29.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,4,128,8,128,9.259538650512695,28.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,8,32,8,128,0.4186978340148926,59.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,8,64,8,128,0.4917322695255279,57.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.gate_proj,8,128,8,128,0.5599081516265869,56.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6 +2,mlp.up_proj,2,32,8,128,30.22202110290528,17.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,2,64,8,128,35.24127197265625,15.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,2,128,8,128,39.81227493286133,14.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,3,32,8,128,12.916703224182127,24.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,3,64,8,128,15.095866203308104,22.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,3,128,8,128,17.067392349243164,21.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,4,32,8,128,6.027431488037109,31.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,4,64,8,128,7.041491508483887,29.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,4,128,8,128,7.969945907592773,28.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,8,32,8,128,0.3662784397602081,59.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,8,64,8,128,0.4273211061954498,57.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.up_proj,8,128,8,128,0.481521874666214,56.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6 +2,mlp.down_proj,2,32,8,128,30.312685012817383,17.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,2,64,8,128,35.36705780029297,15.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,2,128,8,128,39.947853088378906,14.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,3,32,8,128,12.969926834106444,24.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,3,64,8,128,15.17494010925293,22.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,3,128,8,128,17.175376892089844,21.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,4,32,8,128,6.054165363311768,31.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,4,64,8,128,7.08501672744751,29.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,4,128,8,128,8.038103103637695,28.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,8,32,8,128,0.36796528100967407,59.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,8,64,8,128,0.4299937486648559,57.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6 +2,mlp.down_proj,8,128,8,128,0.4857692122459411,56.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6 +3,self_attn.q_proj,2,32,8,128,28.09393882751465,5.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,2,64,8,128,33.566856384277344,4.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,2,128,8,128,39.03070831298828,4.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,3,32,8,128,12.062705039978027,7.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,3,64,8,128,14.456032752990724,6.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,3,128,8,128,16.985721588134766,6.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,4,32,8,128,5.631606578826904,9.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,4,64,8,128,6.747907638549805,8.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,4,128,8,128,7.93041467666626,8.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,8,32,8,128,0.3408825695514679,17.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,8,64,8,128,0.4081771075725555,16.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.q_proj,8,128,8,128,0.4779151082038879,16.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6 +3,self_attn.v_proj,2,32,8,128,6.054136276245117,1.25390625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,2,64,8,128,7.143193244934082,1.126953125,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,2,128,8,128,8.157306671142578,1.0634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,3,32,8,128,2.591524362564087,1.75390625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,3,64,8,128,3.0607292652130127,1.626953125,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,3,128,8,128,3.500352144241333,1.5634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,4,32,8,128,1.2083368301391602,2.25390625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,4,64,8,128,1.4283732175827026,2.126953125,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,4,128,8,128,1.6336627006530762,2.0634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,8,32,8,128,0.0732447206974029,4.25390625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,8,64,8,128,0.0864056944847107,4.126953125,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.v_proj,8,128,8,128,0.0984654575586319,4.0634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6 +3,self_attn.k_proj,2,32,8,128,22.24028968811035,1.25390625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,2,64,8,128,26.65947151184082,1.126953125,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,2,128,8,128,30.53901100158692,1.0634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,3,32,8,128,9.575121879577637,1.75390625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,3,64,8,128,11.551334381103516,1.626953125,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,3,128,8,128,13.431496620178224,1.5634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,4,32,8,128,4.46926212310791,2.25390625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,4,64,8,128,5.3901801109313965,2.126953125,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,4,128,8,128,6.272538185119629,2.0634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,8,32,8,128,0.2705325782299042,4.25390625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,8,64,8,128,0.3260603249073028,4.126953125,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,self_attn.k_proj,8,128,8,128,0.3779822885990143,4.0634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6 +3,mlp.gate_proj,2,32,8,128,37.02172470092773,17.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,2,64,8,128,43.68269729614258,15.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,2,128,8,128,49.92171859741211,14.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,3,32,8,128,15.858477592468262,24.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,3,64,8,128,18.73029518127441,22.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,3,128,8,128,21.516189575195312,21.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,4,32,8,128,7.397336006164551,31.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,4,64,8,128,8.7405548095703125,29.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,4,128,8,128,10.045246124267578,28.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,8,32,8,128,0.448901891708374,59.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,8,64,8,128,0.5302746891975403,57.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.gate_proj,8,128,8,128,0.6077975630760193,56.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6 +3,mlp.up_proj,2,32,8,128,29.67856216430664,17.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,2,64,8,128,34.637977600097656,15.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,2,128,8,128,39.1311149597168,14.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,3,32,8,128,12.684122085571287,24.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,3,64,8,128,14.836777687072754,22.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,3,128,8,128,16.790557861328125,21.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,4,32,8,128,5.920221328735352,31.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,4,64,8,128,6.921647548675537,29.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,4,128,8,128,7.839502334594727,28.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,8,32,8,128,0.3598023951053619,59.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,8,64,8,128,0.4200921356678009,57.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.up_proj,8,128,8,128,0.4737437069416046,56.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6 +3,mlp.down_proj,2,32,8,128,29.65584182739257,17.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,2,64,8,128,34.60835647583008,15.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,2,128,8,128,39.098388671875,14.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,3,32,8,128,12.679429054260254,24.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,3,64,8,128,14.82733917236328,22.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,3,128,8,128,16.778474807739258,21.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,4,32,8,128,5.918444633483887,31.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,4,64,8,128,6.918429374694824,29.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,4,128,8,128,7.836794376373291,28.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,8,32,8,128,0.3596154749393463,59.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,8,64,8,128,0.4199441969394684,57.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6 +3,mlp.down_proj,8,128,8,128,0.4735199809074402,56.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6 +4,self_attn.q_proj,2,32,8,128,28.03318214416504,5.015625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,2,64,8,128,33.52735900878906,4.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,2,128,8,128,39.10169219970703,4.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,3,32,8,128,12.06218719482422,7.015625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,3,64,8,128,14.514254570007324,6.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,3,128,8,128,17.2142276763916,6.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,4,32,8,128,5.631128311157227,9.015625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,4,64,8,128,6.779860973358154,8.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,4,128,8,128,8.049135208129883,8.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,8,32,8,128,0.3407697677612304,17.015625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,8,64,8,128,0.4098406732082367,16.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.q_proj,8,128,8,128,0.4849676191806793,16.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6 +4,self_attn.v_proj,2,32,8,128,6.615170001983643,1.25390625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,2,64,8,128,7.8174052238464355,1.126953125,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,2,128,8,128,8.9237060546875,1.0634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,3,32,8,128,2.828190565109253,1.75390625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,3,64,8,128,3.345294237136841,1.626953125,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,3,128,8,128,3.823539733886719,1.5634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,4,32,8,128,1.3191213607788086,2.25390625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,4,64,8,128,1.560590147972107,2.126953125,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,4,128,8,128,1.7844257354736328,2.0634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,8,32,8,128,0.079975239932537,4.25390625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,8,64,8,128,0.0944518074393272,4.126953125,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.v_proj,8,128,8,128,0.1075403913855552,4.0634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6 +4,self_attn.k_proj,2,32,8,128,21.67329216003418,1.25390625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,2,64,8,128,25.89285469055176,1.126953125,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,2,128,8,128,29.59549331665039,1.0634765625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,3,32,8,128,9.311653137207031,1.75390625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,3,64,8,128,11.197571754455566,1.626953125,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,3,128,8,128,12.988943099975586,1.5634765625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,4,32,8,128,4.346807479858398,2.25390625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,4,64,8,128,5.22780179977417,2.126953125,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,4,128,8,128,6.067966938018799,2.0634765625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,8,32,8,128,0.2630780041217804,4.25390625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,8,64,8,128,0.3159684240818023,4.126953125,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,self_attn.k_proj,8,128,8,128,0.365718811750412,4.0634765625,4194304,4.655811376588373,NA,3.732759296326549e-6 +4,mlp.gate_proj,2,32,8,128,39.353912353515625,17.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,2,64,8,128,46.50521469116211,15.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,2,128,8,128,53.20646667480469,14.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,3,32,8,128,16.850141525268555,24.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,3,64,8,128,19.94430923461914,22.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,3,128,8,128,22.93217086791992,21.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,4,32,8,128,7.865922927856445,31.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,4,64,8,128,9.306957244873049,29.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,4,128,8,128,10.701563835144045,28.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,8,32,8,128,0.4769289195537567,59.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,8,64,8,128,0.5647693276405334,57.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.gate_proj,8,128,8,128,0.6474993228912354,56.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6 +4,mlp.up_proj,2,32,8,128,29.0410213470459,17.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,2,64,8,128,33.95344543457031,15.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,2,128,8,128,38.41080093383789,14.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,3,32,8,128,12.422975540161133,24.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,3,64,8,128,14.561521530151367,22.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,3,128,8,128,16.53119659423828,21.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,4,32,8,128,5.799415588378906,31.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,4,64,8,128,6.79381275177002,29.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,4,128,8,128,7.719454765319824,28.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,8,32,8,128,0.3524595499038696,59.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,8,64,8,128,0.4124566912651062,57.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.up_proj,8,128,8,128,0.4665692746639251,56.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6 +4,mlp.down_proj,2,32,8,128,28.969560623168945,17.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,2,64,8,128,33.845821380615234,15.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,2,128,8,128,38.26799011230469,14.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,3,32,8,128,12.39323902130127,24.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,3,64,8,128,14.50938892364502,22.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,3,128,8,128,16.448408126831055,21.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,4,32,8,128,5.784383296966553,31.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,4,64,8,128,6.770299911499023,29.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,4,128,8,128,7.685372829437256,28.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,8,32,8,128,0.351535826921463,59.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,8,64,8,128,0.4111148416996002,57.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6 +4,mlp.down_proj,8,128,8,128,0.464570552110672,56.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6 +5,self_attn.q_proj,2,32,8,128,27.14842796325684,5.015625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,2,64,8,128,32.56990051269531,4.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,2,128,8,128,38.27643585205078,4.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,3,32,8,128,11.68593692779541,7.015625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,3,64,8,128,14.071290969848633,6.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,3,128,8,128,16.836549758911133,6.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,4,32,8,128,5.460404872894287,9.015625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,4,64,8,128,6.574841499328613,8.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,4,128,8,128,7.871114253997803,8.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,8,32,8,128,0.3304733335971832,17.015625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,8,64,8,128,0.3976515531539917,16.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.q_proj,8,128,8,128,0.4741759598255157,16.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6 +5,self_attn.v_proj,2,32,8,128,5.379310131072998,1.25390625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,2,64,8,128,6.315425872802734,1.126953125,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,2,128,8,128,7.180819988250732,1.0634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,3,32,8,128,2.30245590209961,1.75390625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,3,64,8,128,2.706099510192871,1.626953125,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,3,128,8,128,3.081350803375244,1.5634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,4,32,8,128,1.074188232421875,2.25390625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,4,64,8,128,1.2629473209381104,2.126953125,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,4,128,8,128,1.4385771751403809,2.0634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,8,32,8,128,0.0650263726711273,4.25390625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,8,64,8,128,0.0763984844088554,4.126953125,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.v_proj,8,128,8,128,0.0866768211126327,4.0634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6 +5,self_attn.k_proj,2,32,8,128,21.81302261352539,1.25390625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,2,64,8,128,26.0170955657959,1.126953125,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,2,128,8,128,29.667430877685547,1.0634765625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,3,32,8,128,9.45390796661377,1.75390625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,3,64,8,128,11.400733947753906,1.626953125,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,3,128,8,128,13.265999794006348,1.5634765625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,4,32,8,128,4.411849498748779,2.25390625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,4,64,8,128,5.332928657531738,2.126953125,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,4,128,8,128,6.2343430519104,2.0634765625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,8,32,8,128,0.2672629654407501,4.25390625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,8,64,8,128,0.3226409554481506,4.126953125,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,self_attn.k_proj,8,128,8,128,0.3757884502410888,4.0634765625,4194304,6.048360546833415,NA,5.492578566190787e-6 +5,mlp.gate_proj,2,32,8,128,39.130516052246094,17.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,2,64,8,128,46.13655471801758,15.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,2,128,8,128,52.64202499389648,14.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,3,32,8,128,16.74797821044922,24.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,3,64,8,128,19.76881980895996,22.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,3,128,8,128,22.633087158203125,21.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,4,32,8,128,7.815927505493164,31.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,4,64,8,128,9.223048210144045,29.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,4,128,8,128,10.564090728759766,28.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,8,32,8,128,0.4739591479301452,59.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,8,64,8,128,0.5594997406005859,57.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.gate_proj,8,128,8,128,0.6392660140991211,56.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5 +5,mlp.up_proj,2,32,8,128,29.15910530090332,17.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,2,64,8,128,34.057796478271484,15.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,2,128,8,128,38.49320602416992,14.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,3,32,8,128,12.469191551208496,24.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,3,64,8,128,14.595012664794922,22.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,3,128,8,128,16.53561782836914,21.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,4,32,8,128,5.82036018371582,31.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,4,64,8,128,6.808206081390381,29.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,4,128,8,128,7.719542503356934,28.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,8,32,8,128,0.353719025850296,59.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,8,64,8,128,0.4134189784526825,57.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.up_proj,8,128,8,128,0.4666213691234588,56.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5 +5,mlp.down_proj,2,32,8,128,29.086917877197266,17.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,2,64,8,128,33.97077560424805,15.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,2,128,8,128,38.39598846435547,14.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,3,32,8,128,12.442087173461914,24.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,3,64,8,128,14.556851387023926,22.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,3,128,8,128,16.480491638183594,21.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,4,32,8,128,5.809384822845459,31.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,4,64,8,128,6.79417085647583,29.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,4,128,8,128,7.705014228820801,28.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,8,32,8,128,0.3530448377132416,59.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,8,64,8,128,0.412678062915802,57.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6 +5,mlp.down_proj,8,128,8,128,0.4659603834152221,56.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6 +6,self_attn.q_proj,2,32,8,128,28.26160430908203,5.015625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,2,64,8,128,33.862728118896484,4.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,2,128,8,128,39.84922409057617,4.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,3,32,8,128,12.133743286132812,7.015625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,3,64,8,128,14.60372543334961,6.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,3,128,8,128,17.438888549804688,6.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,4,32,8,128,5.66416072845459,9.015625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,4,64,8,128,6.8173508644104,8.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,4,128,8,128,8.145289421081543,8.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,8,32,8,128,0.3429482877254486,17.015625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,8,64,8,128,0.4121284186840057,16.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.q_proj,8,128,8,128,0.4907255172729492,16.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6 +6,self_attn.v_proj,2,32,8,128,5.851216793060303,1.25390625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,2,64,8,128,6.8759050369262695,1.126953125,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,2,128,8,128,7.827797889709473,1.0634765625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,3,32,8,128,2.5037496089935303,1.75390625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,3,64,8,128,2.9412851333618164,1.626953125,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,3,128,8,128,3.351409435272217,1.5634765625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,4,32,8,128,1.1683902740478516,2.25390625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,4,64,8,128,1.3734513521194458,2.126953125,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,4,128,8,128,1.5643017292022705,2.0634765625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,8,32,8,128,0.0707335993647575,4.25390625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,8,64,8,128,0.0831061825156211,4.126953125,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.v_proj,8,128,8,128,0.0942878797650337,4.0634765625,4194304,3.376907655091106,NA,6.826936896686675e-6 +6,self_attn.k_proj,2,32,8,128,21.420391082763672,1.25390625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,2,64,8,128,25.570756912231445,1.126953125,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,2,128,8,128,29.383703231811523,1.0634765625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,3,32,8,128,9.189220428466797,1.75390625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,3,64,8,128,10.99506378173828,1.626953125,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,3,128,8,128,12.71745777130127,1.5634765625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,4,32,8,128,4.287354946136475,2.25390625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,4,64,8,128,5.131492137908936,2.126953125,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,4,128,8,128,5.935476303100586,2.0634765625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,8,32,8,128,0.2594151198863983,4.25390625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,8,64,8,128,0.3101383447647095,4.126953125,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,self_attn.k_proj,8,128,8,128,0.3574665784835815,4.0634765625,4194304,4.107301027533515,NA,6.826936896686675e-6 +6,mlp.gate_proj,2,32,8,128,39.572059631347656,17.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,2,64,8,128,46.76994323730469,15.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,2,128,8,128,53.50589370727539,14.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,3,32,8,128,16.936744689941406,24.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,3,64,8,128,20.04867935180664,22.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,3,128,8,128,23.027267456054688,21.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,4,32,8,128,7.904539108276367,31.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,4,64,8,128,9.356647491455078,29.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,4,128,8,128,10.750324249267578,28.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,8,32,8,128,0.4793758690357208,59.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,8,64,8,128,0.5675809383392334,57.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.gate_proj,8,128,8,128,0.650355339050293,56.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5 +6,mlp.up_proj,2,32,8,128,29.13815116882324,17.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,2,64,8,128,34.03101348876953,15.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,2,128,8,128,38.463661193847656,14.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,3,32,8,128,12.457345008850098,24.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,3,64,8,128,14.575265884399414,22.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,3,128,8,128,16.508161544799805,21.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,4,32,8,128,5.814871788024902,31.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,4,64,8,128,6.801267147064209,29.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,4,128,8,128,7.705705165863037,28.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,8,32,8,128,0.3534092903137207,59.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,8,64,8,128,0.4128694534301758,57.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.up_proj,8,128,8,128,0.4657552242279053,56.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5 +6,mlp.down_proj,2,32,8,128,29.1191635131836,17.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,2,64,8,128,34.0229606628418,15.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,2,128,8,128,38.480712890625,14.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,3,32,8,128,12.451114654541016,24.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,3,64,8,128,14.576369285583496,22.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,3,128,8,128,16.521066665649414,21.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,4,32,8,128,5.813693523406982,31.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,4,64,8,128,6.803946971893311,29.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,4,128,8,128,7.718326091766357,28.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,8,32,8,128,0.353316456079483,59.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,8,64,8,128,0.4130769073963165,57.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5 +6,mlp.down_proj,8,128,8,128,0.4666117131710052,56.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5 +7,self_attn.q_proj,2,32,8,128,26.976608276367188,5.015625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,2,64,8,128,32.303985595703125,4.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,2,128,8,128,37.8601188659668,4.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,3,32,8,128,11.569588661193848,7.015625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,3,64,8,128,13.894417762756348,6.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,3,128,8,128,16.53858757019043,6.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,4,32,8,128,5.400040626525879,9.015625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,4,64,8,128,6.484575271606445,8.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,4,128,8,128,7.722560405731201,8.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,8,32,8,128,0.327002465724945,17.015625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,8,64,8,128,0.3922163546085357,16.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.q_proj,8,128,8,128,0.4653173089027405,16.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6 +7,self_attn.v_proj,2,32,8,128,5.812392711639404,1.25390625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,2,64,8,128,6.83394193649292,1.126953125,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,2,128,8,128,7.764544486999512,1.0634765625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,3,32,8,128,2.4866905212402344,1.75390625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,3,64,8,128,2.92472505569458,1.626953125,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,3,128,8,128,3.3279316425323486,1.5634765625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,4,32,8,128,1.160157561302185,2.25390625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,4,64,8,128,1.364603877067566,2.126953125,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,4,128,8,128,1.5533496141433716,2.0634765625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,8,32,8,128,0.0702754035592079,4.25390625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,8,64,8,128,0.0825935527682304,4.126953125,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.v_proj,8,128,8,128,0.0936173573136329,4.0634765625,4194304,3.364970783290237,NA,8.94385993888136e-6 +7,self_attn.k_proj,2,32,8,128,21.49588966369629,1.25390625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,2,64,8,128,25.60579299926757,1.126953125,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,2,128,8,128,29.29781913757324,1.0634765625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,3,32,8,128,9.206317901611328,1.75390625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,3,64,8,128,10.975910186767578,1.626953125,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,3,128,8,128,12.625019073486328,1.5634765625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,4,32,8,128,4.297920227050781,2.25390625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,4,64,8,128,5.124042987823486,2.126953125,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,4,128,8,128,5.891855716705322,2.0634765625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,8,32,8,128,0.2601531744003296,4.25390625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,8,64,8,128,0.3099227845668793,4.126953125,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,self_attn.k_proj,8,128,8,128,0.3550952076911926,4.0634765625,4194304,3.850992348386023,NA,8.94385993888136e-6 +7,mlp.gate_proj,2,32,8,128,39.03369522094727,17.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,2,64,8,128,46.20256423950195,15.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,2,128,8,128,52.90453720092773,14.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,3,32,8,128,16.709978103637695,24.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,3,64,8,128,19.80340576171875,22.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,3,128,8,128,22.763891220092773,21.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,4,32,8,128,7.798851013183594,31.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,4,64,8,128,9.24035930633545,29.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,4,128,8,128,10.62183952331543,28.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,8,32,8,128,0.4729699194431305,59.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,8,64,8,128,0.5606297850608826,57.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.gate_proj,8,128,8,128,0.6429332494735718,56.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5 +7,mlp.up_proj,2,32,8,128,29.60834503173828,17.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,2,64,8,128,34.610897064208984,15.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,2,128,8,128,39.16851806640625,14.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,3,32,8,128,12.663779258728027,24.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,3,64,8,128,14.840734481811523,22.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,3,128,8,128,16.8330020904541,21.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,4,32,8,128,5.910112857818604,31.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,4,64,8,128,6.922947883605957,29.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,4,128,8,128,7.86088228225708,28.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,8,32,8,128,0.3591862618923187,59.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,8,64,8,128,0.4202658236026764,57.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.up_proj,8,128,8,128,0.4751189947128296,56.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5 +7,mlp.down_proj,2,32,8,128,29.65262031555176,17.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,2,64,8,128,34.703914642333984,15.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,2,128,8,128,39.31455993652344,14.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,3,32,8,128,12.686455726623535,24.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,3,64,8,128,14.885343551635742,22.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,3,128,8,128,16.9116268157959,21.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,4,32,8,128,5.921884536743164,31.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,4,64,8,128,6.947753429412842,29.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,4,128,8,128,7.906002998352051,28.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,8,32,8,128,0.3598684668540954,59.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,8,64,8,128,0.4216315448284149,57.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5 +7,mlp.down_proj,8,128,8,128,0.4777296781539917,56.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5 +8,self_attn.q_proj,2,32,8,128,27.061317443847656,5.015625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,2,64,8,128,32.52377700805664,4.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,2,128,8,128,38.69818115234375,4.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,3,32,8,128,11.656363487243652,7.015625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,3,64,8,128,14.124595642089844,6.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,3,128,8,128,17.215373992919922,6.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,4,32,8,128,5.444422245025635,9.015625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,4,64,8,128,6.6116204261779785,8.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,4,128,8,128,8.074461936950684,8.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,8,32,8,128,0.3296016752719879,17.015625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,8,64,8,128,0.3997628390789032,16.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.q_proj,8,128,8,128,0.4867331981658935,16.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5 +8,self_attn.v_proj,2,32,8,128,6.304591178894043,1.25390625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,2,64,8,128,7.507236957550049,1.126953125,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,2,128,8,128,8.639371871948242,1.0634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,3,32,8,128,2.69922947883606,1.75390625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,3,64,8,128,3.2172539234161377,1.626953125,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,3,128,8,128,3.715339422225952,1.5634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,4,32,8,128,1.2599207162857056,2.25390625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,4,64,8,128,1.501517653465271,2.126953125,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,4,128,8,128,1.7340031862258911,2.0634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,8,32,8,128,0.0762663707137107,4.25390625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,8,64,8,128,0.09086574614048,4.126953125,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.v_proj,8,128,8,128,0.1045241877436637,4.0634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5 +8,self_attn.k_proj,2,32,8,128,22.023910522460938,1.25390625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,2,64,8,128,26.45915031433105,1.126953125,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,2,128,8,128,30.406747817993164,1.0634765625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,3,32,8,128,9.50877857208252,1.75390625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,3,64,8,128,11.526528358459473,1.626953125,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,3,128,8,128,13.486717224121094,1.5634765625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,4,32,8,128,4.436545372009277,2.25390625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,4,64,8,128,5.3773512840271,2.126953125,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,4,128,8,128,6.301447868347168,2.0634765625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,8,32,8,128,0.2684425115585327,4.25390625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,8,64,8,128,0.3253226578235626,4.126953125,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,self_attn.k_proj,8,128,8,128,0.3797986507415771,4.0634765625,4194304,5.208863520397107,NA,1.259053260582732e-5 +8,mlp.gate_proj,2,32,8,128,39.39292144775391,17.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,2,64,8,128,46.70141220092773,15.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,2,128,8,128,53.55084228515625,14.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,3,32,8,128,16.86261749267578,24.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,3,64,8,128,20.022985458374023,22.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,3,128,8,128,23.060375213623047,21.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,4,32,8,128,7.87108850479126,31.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,4,64,8,128,9.34132194519043,29.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,4,128,8,128,10.763914108276367,28.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,8,32,8,128,0.4773235917091369,59.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,8,64,8,128,0.5668628811836243,57.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.gate_proj,8,128,8,128,0.6513233780860901,56.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5 +8,mlp.up_proj,2,32,8,128,29.44819641113281,17.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,2,64,8,128,34.424102783203125,15.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,2,128,8,128,38.94149017333984,14.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,3,32,8,128,12.594765663146973,24.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,3,64,8,128,14.757983207702637,22.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,3,128,8,128,16.737844467163086,21.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,4,32,8,128,5.8780717849731445,31.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,4,64,8,128,6.88339376449585,29.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,4,128,8,128,7.817975044250488,28.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,8,32,8,128,0.3572321534156799,59.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,8,64,8,128,0.4178987145423889,57.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.up_proj,8,128,8,128,0.4723220765590668,56.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5 +8,mlp.down_proj,2,32,8,128,29.567283630371097,17.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,2,64,8,128,34.5979118347168,15.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,2,128,8,128,39.1610221862793,14.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,3,32,8,128,12.648090362548828,24.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,3,64,8,128,14.835460662841797,22.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,3,128,8,128,16.83810043334961,21.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,4,32,8,128,5.904628753662109,31.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,4,64,8,128,6.924561023712158,29.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,4,128,8,128,7.871026992797852,28.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,8,32,8,128,0.3588110208511352,59.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,8,64,8,128,0.4203183948993683,57.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5 +8,mlp.down_proj,8,128,8,128,0.4758267402648926,56.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5 +9,self_attn.q_proj,2,32,8,128,26.9770565032959,5.015625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,2,64,8,128,32.41333770751953,4.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,2,128,8,128,38.673099517822266,4.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,3,32,8,128,11.569629669189451,7.015625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,3,64,8,128,13.95190143585205,6.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,3,128,8,128,16.984012603759766,6.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,4,32,8,128,5.3989787101745605,9.015625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,4,64,8,128,6.50968074798584,8.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,4,128,8,128,7.928092002868652,8.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,8,32,8,128,0.3267835080623626,17.015625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,8,64,8,128,0.3938423097133636,16.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.q_proj,8,128,8,128,0.4779276549816131,16.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5 +9,self_attn.v_proj,2,32,8,128,7.524624824523926,1.25390625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,2,64,8,128,9.065919876098633,1.126953125,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,2,128,8,128,10.531023979187012,1.0634765625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,3,32,8,128,3.2294089794158936,1.75390625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,3,64,8,128,3.907343626022339,1.626953125,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,3,128,8,128,4.580047607421875,1.5634765625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,4,32,8,128,1.5074481964111328,2.25390625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,4,64,8,128,1.824461340904236,2.126953125,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,4,128,8,128,2.1387925148010254,2.0634765625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,8,32,8,128,0.0912665277719497,4.25390625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,8,64,8,128,0.1103613376617431,4.126953125,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.v_proj,8,128,8,128,0.128845676779747,4.0634765625,4194304,4.470621777443523,NA,1.545005761727225e-5 +9,self_attn.k_proj,2,32,8,128,21.76741981506348,1.25390625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,2,64,8,128,26.06185340881348,1.126953125,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,2,128,8,128,29.87959861755371,1.0634765625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,3,32,8,128,9.348188400268556,1.75390625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,3,64,8,128,11.24947452545166,1.626953125,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,3,128,8,128,13.032777786254885,1.5634765625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,4,32,8,128,4.36350679397583,2.25390625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,4,64,8,128,5.2510175704956055,2.126953125,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,4,128,8,128,6.081087112426758,2.0634765625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,8,32,8,128,0.264181911945343,4.25390625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,8,64,8,128,0.3175808787345886,4.126953125,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,self_attn.k_proj,8,128,8,128,0.3666176497936249,4.0634765625,4194304,4.353797933169363,NA,1.545005761727225e-5 +9,mlp.gate_proj,2,32,8,128,40.1043701171875,17.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,2,64,8,128,47.65932083129883,15.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,2,128,8,128,54.80207824707031,14.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,3,32,8,128,17.172544479370117,24.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,3,64,8,128,20.46648406982422,22.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,3,128,8,128,23.66988945007324,21.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,4,32,8,128,8.018026351928711,31.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,4,64,8,128,9.550697326660156,29.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,4,128,8,128,11.0488920211792,28.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,8,32,8,128,0.4861500263214111,59.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,8,64,8,128,0.5795626640319824,57.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.gate_proj,8,128,8,128,0.6688878536224365,56.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5 +9,mlp.up_proj,2,32,8,128,29.79354286193848,17.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,2,64,8,128,34.850040435791016,15.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,2,128,8,128,39.46410369873047,14.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,3,32,8,128,12.743324279785156,24.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,3,64,8,128,14.948670387268066,22.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,3,128,8,128,16.972240447998047,21.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,4,32,8,128,5.946471691131592,31.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,4,64,8,128,6.972580909729004,29.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,4,128,8,128,7.927926540374756,28.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,8,32,8,128,0.3614477813243866,59.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,8,64,8,128,0.4233103394508362,57.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.up_proj,8,128,8,128,0.4790314733982086,56.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5 +9,mlp.down_proj,2,32,8,128,29.70336151123047,17.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,2,64,8,128,34.76505279541016,15.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,2,128,8,128,39.38078689575195,14.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,3,32,8,128,12.70130729675293,24.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,3,64,8,128,14.899825096130373,22.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,3,128,8,128,16.9156436920166,21.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,4,32,8,128,5.929380416870117,31.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,4,64,8,128,6.953718662261963,29.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,4,128,8,128,7.905187606811523,28.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,8,32,8,128,0.3603951930999756,59.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,8,64,8,128,0.4221662580966949,57.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5 +9,mlp.down_proj,8,128,8,128,0.4778989255428314,56.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5 +10,self_attn.q_proj,2,32,8,128,27.73469352722168,5.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,2,64,8,128,33.35016632080078,4.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,2,128,8,128,39.38727951049805,4.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,3,32,8,128,11.962119102478027,7.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,3,64,8,128,14.490964889526367,6.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,3,128,8,128,17.632457733154297,6.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,4,32,8,128,5.58771562576294,9.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,4,64,8,128,6.779984474182129,8.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,4,128,8,128,8.269221305847168,8.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,8,32,8,128,0.3381999135017395,17.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,8,64,8,128,0.4100817441940307,16.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.q_proj,8,128,8,128,0.4986184537410736,16.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5 +10,self_attn.v_proj,2,32,8,128,6.229307651519775,1.25390625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,2,64,8,128,7.42263126373291,1.126953125,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,2,128,8,128,8.537671089172363,1.0634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,3,32,8,128,2.6669013500213623,1.75390625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,3,64,8,128,3.182350873947144,1.626953125,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,3,128,8,128,3.6750028133392334,1.5634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,4,32,8,128,1.2443753480911257,2.25390625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,4,64,8,128,1.4845033884048462,2.126953125,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,4,128,8,128,1.7152011394500732,2.0634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,8,32,8,128,0.0753431543707847,4.25390625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,8,64,8,128,0.0898873060941696,4.126953125,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.v_proj,8,128,8,128,0.1033799722790718,4.0634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5 +10,self_attn.k_proj,2,32,8,128,22.137977600097656,1.25390625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,2,64,8,128,26.48815155029297,1.126953125,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,2,128,8,128,30.33242416381836,1.0634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,3,32,8,128,9.56244659423828,1.75390625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,3,64,8,128,11.564157485961914,1.626953125,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,3,128,8,128,13.468461990356444,1.5634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,4,32,8,128,4.46476936340332,2.25390625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,4,64,8,128,5.40137243270874,2.126953125,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,4,128,8,128,6.312881946563721,2.0634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,8,32,8,128,0.2701395153999328,4.25390625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,8,64,8,128,0.3267165720462799,4.126953125,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,self_attn.k_proj,8,128,8,128,0.3802925944328308,4.0634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5 +10,mlp.gate_proj,2,32,8,128,38.944854736328125,17.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,2,64,8,128,46.25450897216797,15.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,2,128,8,128,53.101993560791016,14.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,3,32,8,128,16.674837112426758,24.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,3,64,8,128,19.83896255493164,22.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,3,128,8,128,22.89117431640625,21.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,4,32,8,128,7.783293724060059,31.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,4,64,8,128,9.25632667541504,29.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,4,128,8,128,10.690305709838867,28.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,8,32,8,128,0.4720402657985687,59.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,8,64,8,128,0.5616547465324402,57.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.gate_proj,8,128,8,128,0.6467053294181824,56.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5 +10,mlp.up_proj,2,32,8,128,30.180099487304688,17.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,2,64,8,128,35.26427459716797,15.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,2,128,8,128,39.9040412902832,14.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,3,32,8,128,12.90137767791748,24.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,3,64,8,128,15.10992431640625,22.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,3,128,8,128,17.116329193115234,21.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,4,32,8,128,6.021190643310547,31.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,4,64,8,128,7.0478057861328125,29.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,4,128,8,128,7.995498657226562,28.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,8,32,8,128,0.36592498421669,59.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,8,64,8,128,0.4277883470058441,57.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.up_proj,8,128,8,128,0.4828921258449554,56.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5 +10,mlp.down_proj,2,32,8,128,30.11432456970215,17.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,2,64,8,128,35.22907257080078,15.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,2,128,8,128,39.86897659301758,14.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,3,32,8,128,12.874347686767578,24.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,3,64,8,128,15.087624549865724,22.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,3,128,8,128,17.10296630859375,21.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,4,32,8,128,6.009179592132568,31.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,4,64,8,128,7.041721343994141,29.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,4,128,8,128,7.99387788772583,28.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,8,32,8,128,0.3652148842811584,59.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,8,64,8,128,0.4273668527603149,57.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5 +10,mlp.down_proj,8,128,8,128,0.482973575592041,56.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5 +11,self_attn.q_proj,2,32,8,128,26.12057113647461,5.015625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,2,64,8,128,31.403221130371097,4.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,2,128,8,128,37.03877639770508,4.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,3,32,8,128,11.208246231079102,7.015625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,3,64,8,128,13.515785217285156,6.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,3,128,8,128,16.240028381347656,6.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,4,32,8,128,5.228873252868652,9.015625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,4,64,8,128,6.310160636901856,8.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,4,128,8,128,7.584232330322266,8.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,8,32,8,128,0.3165485560894012,17.015625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,8,64,8,128,0.3815952241420746,16.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.q_proj,8,128,8,128,0.4569851756095886,16.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5 +11,self_attn.v_proj,2,32,8,128,5.915624618530273,1.25390625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,2,64,8,128,6.933813095092773,1.126953125,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,2,128,8,128,7.871473789215088,1.0634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,3,32,8,128,2.531064510345459,1.75390625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,3,64,8,128,2.9708845615386963,1.626953125,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,3,128,8,128,3.3750290870666504,1.5634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,4,32,8,128,1.1811060905456543,2.25390625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,4,64,8,128,1.3862783908843994,2.126953125,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,4,128,8,128,1.576763391494751,2.0634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,8,32,8,128,0.0715062394738197,4.25390625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,8,64,8,128,0.0838671326637268,4.126953125,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.v_proj,8,128,8,128,0.0949677526950836,4.0634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5 +11,self_attn.k_proj,2,32,8,128,21.53693389892578,1.25390625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,2,64,8,128,25.738405227661133,1.126953125,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,2,128,8,128,29.473670959472656,1.0634765625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,3,32,8,128,9.25912094116211,1.75390625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,3,64,8,128,11.105602264404297,1.626953125,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,3,128,8,128,12.856239318847656,1.5634765625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,4,32,8,128,4.320185661315918,2.25390625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,4,64,8,128,5.183795928955078,2.126953125,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,4,128,8,128,6.003633499145508,2.0634765625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,8,32,8,128,0.2614936530590057,4.25390625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,8,64,8,128,0.3133919239044189,4.126953125,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,self_attn.k_proj,8,128,8,128,0.361666202545166,4.0634765625,4194304,4.226251359421093,NA,1.598008384462446e-5 +11,mlp.gate_proj,2,32,8,128,38.63543701171875,17.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,2,64,8,128,45.97889709472656,15.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,2,128,8,128,52.91830062866211,14.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,3,32,8,128,16.557937622070312,24.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,3,64,8,128,19.7586612701416,22.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,3,128,8,128,22.90111541748047,21.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,4,32,8,128,7.72847318649292,31.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,4,64,8,128,9.2195281982421875,29.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,4,128,8,128,10.691786766052246,28.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,8,32,8,128,0.4687098860740661,59.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,8,64,8,128,0.5594026446342468,57.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.gate_proj,8,128,8,128,0.6470131874084473,56.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5 +11,mlp.up_proj,2,32,8,128,30.451913833618164,17.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,2,64,8,128,35.6109504699707,15.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,2,128,8,128,40.33466339111328,14.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,3,32,8,128,13.019288063049316,24.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,3,64,8,128,15.262990951538086,22.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,3,128,8,128,17.307527542114258,21.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,4,32,8,128,6.074724197387695,31.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,4,64,8,128,7.120927333831787,29.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,4,128,8,128,8.082134246826172,28.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,8,32,8,128,0.3691640496253967,59.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,8,64,8,128,0.4320685863494873,57.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.up_proj,8,128,8,128,0.4883591830730438,56.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5 +11,mlp.down_proj,2,32,8,128,30.37547492980957,17.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,2,64,8,128,35.56100845336914,15.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,2,128,8,128,40.31380081176758,14.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,3,32,8,128,12.995153427124023,24.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,3,64,8,128,15.258899688720703,22.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,3,128,8,128,17.339706420898438,21.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,4,32,8,128,6.06576681137085,31.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,4,64,8,128,7.121227741241455,29.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,4,128,8,128,8.104607582092285,28.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,8,32,8,128,0.3685401678085327,59.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,8,64,8,128,0.4321089386940002,57.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5 +11,mlp.down_proj,8,128,8,128,0.4897418320178985,56.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5 +12,self_attn.q_proj,2,32,8,128,25.861541748046875,5.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,2,64,8,128,30.882308959960938,4.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,2,128,8,128,36.07247924804688,4.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,3,32,8,128,11.073997497558594,7.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,3,64,8,128,13.24992847442627,6.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,3,128,8,128,15.60166072845459,6.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,4,32,8,128,5.16636323928833,9.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,4,64,8,128,6.183772087097168,8.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,4,128,8,128,7.281061172485352,8.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,8,32,8,128,0.3127245008945465,17.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,8,64,8,128,0.3739984035491943,16.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.q_proj,8,128,8,128,0.4387511610984802,16.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5 +12,self_attn.v_proj,2,32,8,128,7.040297985076904,1.25390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,2,64,8,128,8.279337882995605,1.126953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,2,128,8,128,9.429340362548828,1.0634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,3,32,8,128,3.011828899383545,1.75390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,3,64,8,128,3.543575525283813,1.626953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,3,128,8,128,4.041069984436035,1.5634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,4,32,8,128,1.406035542488098,2.25390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,4,64,8,128,1.6543664932250977,2.126953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,4,128,8,128,1.8858826160430908,2.0634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,8,32,8,128,0.0851678997278213,4.25390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,8,64,8,128,0.1000915542244911,4.126953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.v_proj,8,128,8,128,0.1137010008096695,4.0634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5 +12,self_attn.k_proj,2,32,8,128,20.48916244506836,1.25390625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,2,64,8,128,24.324567794799805,1.126953125,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,2,128,8,128,27.846071243286133,1.0634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,3,32,8,128,8.775659561157227,1.75390625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,3,64,8,128,10.430098533630373,1.626953125,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,3,128,8,128,11.960859298706056,1.5634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,4,32,8,128,4.094570159912109,2.25390625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,4,64,8,128,4.867015838623047,2.126953125,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,4,128,8,128,5.583600997924805,2.0634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,8,32,8,128,0.2478196173906326,4.25390625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,8,64,8,128,0.2942624986171722,4.126953125,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,self_attn.k_proj,8,128,8,128,0.3365941643714905,4.0634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5 +12,mlp.gate_proj,2,32,8,128,38.39822769165039,17.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,2,64,8,128,45.75447082519531,15.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,2,128,8,128,52.67757415771485,14.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,3,32,8,128,16.485942840576172,24.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,3,64,8,128,19.719900131225582,22.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,3,128,8,128,22.919330596923828,21.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,4,32,8,128,7.694408416748047,31.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,4,64,8,128,9.2075653076171875,29.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,4,128,8,128,10.714046478271484,28.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,8,32,8,128,0.4667559862136841,59.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,8,64,8,128,0.5586689114570618,57.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.gate_proj,8,128,8,128,0.6483145952224731,56.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5 +12,mlp.up_proj,2,32,8,128,31.15839958190918,17.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,2,64,8,128,36.47867584228516,15.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,2,128,8,128,41.35436630249024,14.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,3,32,8,128,13.327926635742188,24.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,3,64,8,128,15.65520191192627,22.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,3,128,8,128,17.792837142944336,21.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,4,32,8,128,6.218276500701904,31.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,4,64,8,128,7.304477214813232,29.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,4,128,8,128,8.313840866088867,28.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,8,32,8,128,0.3778644204139709,59.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,8,64,8,128,0.4431090354919433,57.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.up_proj,8,128,8,128,0.5022327303886414,56.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5 +12,mlp.down_proj,2,32,8,128,30.80364799499512,17.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,2,64,8,128,36.04652404785156,15.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,2,128,8,128,40.88020706176758,14.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,3,32,8,128,13.178635597229004,24.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,3,64,8,128,15.473698616027832,22.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,3,128,8,128,17.591928482055664,21.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,4,32,8,128,6.151588439941406,31.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,4,64,8,128,7.222397804260254,29.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,4,128,8,128,8.224127769470215,28.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,8,32,8,128,0.3738064467906952,59.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,8,64,8,128,0.4383414089679718,57.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5 +12,mlp.down_proj,8,128,8,128,0.4972971975803375,56.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5 +13,self_attn.q_proj,2,32,8,128,26.5046501159668,5.015625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,2,64,8,128,31.847789764404297,4.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,2,128,8,128,37.72823715209961,4.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,3,32,8,128,11.362935066223145,7.015625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,3,64,8,128,13.68783950805664,6.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,3,128,8,128,16.56133270263672,6.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,4,32,8,128,5.300324440002441,9.015625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,4,64,8,128,6.38405704498291,8.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,4,128,8,128,7.732006549835205,8.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,8,32,8,128,0.3209334313869476,17.015625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,8,64,8,128,0.3862396776676178,16.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.q_proj,8,128,8,128,0.4659466743469238,16.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5 +13,self_attn.v_proj,2,32,8,128,6.478818416595459,1.25390625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,2,64,8,128,7.596816062927246,1.126953125,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,2,128,8,128,8.60972785949707,1.0634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,3,32,8,128,2.772458791732788,1.75390625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,3,64,8,128,3.2537105083465576,1.626953125,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,3,128,8,128,3.693915843963623,1.5634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,4,32,8,128,1.293779730796814,2.25390625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,4,64,8,128,1.5194296836853027,2.126953125,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,4,128,8,128,1.724428415298462,2.0634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,8,32,8,128,0.0783509612083435,4.25390625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,8,64,8,128,0.0918719694018364,4.126953125,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.v_proj,8,128,8,128,0.1039059609174728,4.0634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5 +13,self_attn.k_proj,2,32,8,128,21.89759635925293,1.25390625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,2,64,8,128,26.193098068237305,1.126953125,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,2,128,8,128,29.959367752075195,1.0634765625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,3,32,8,128,9.432870864868164,1.75390625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,3,64,8,128,11.339022636413574,1.626953125,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,3,128,8,128,13.113142013549805,1.5634765625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,4,32,8,128,4.4028754234313965,2.25390625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,4,64,8,128,5.293444633483887,2.126953125,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,4,128,8,128,6.125287055969238,2.0634765625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,8,32,8,128,0.2664675414562225,4.25390625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,8,64,8,128,0.3202824890613556,4.126953125,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,self_attn.k_proj,8,128,8,128,0.3691720366477966,4.0634765625,4194304,4.501807253157103,NA,2.202295399911236e-5 +13,mlp.gate_proj,2,32,8,128,38.32114410400391,17.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,2,64,8,128,45.63823699951172,15.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,2,128,8,128,52.51383590698242,14.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,3,32,8,128,16.46413803100586,24.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,3,64,8,128,19.693836212158203,22.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,3,128,8,128,22.910493850708008,21.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,4,32,8,128,7.686895847320557,31.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,4,64,8,128,9.201266288757324,29.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,4,128,8,128,10.720735549926758,28.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,8,32,8,128,0.4662733376026153,59.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,8,64,8,128,0.5582589507102966,57.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.gate_proj,8,128,8,128,0.6486979722976685,56.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5 +13,mlp.up_proj,2,32,8,128,31.239431381225582,17.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,2,64,8,128,36.55526351928711,15.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,2,128,8,128,41.43852996826172,14.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,3,32,8,128,13.356857299804688,24.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,3,64,8,128,15.68277359008789,22.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,3,128,8,128,17.81464958190918,21.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,4,32,8,128,6.231928825378418,31.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,4,64,8,128,7.315294742584228,29.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,4,128,8,128,8.315678596496582,28.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,8,32,8,128,0.3787055909633636,59.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,8,64,8,128,0.4438227415084839,57.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.up_proj,8,128,8,128,0.50248783826828,56.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5 +13,mlp.down_proj,2,32,8,128,30.7618350982666,17.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,2,64,8,128,35.97294616699219,15.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,2,128,8,128,40.74884414672852,14.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,3,32,8,128,13.150542259216309,24.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,3,64,8,128,15.41523551940918,22.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,3,128,8,128,17.47993278503418,21.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,4,32,8,128,6.136257171630859,31.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,4,64,8,128,7.191862106323242,29.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,4,128,8,128,8.164878845214844,28.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,8,32,8,128,0.3729257583618164,59.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,8,64,8,128,0.4364360272884369,57.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5 +13,mlp.down_proj,8,128,8,128,0.4934674501419067,56.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5 +14,self_attn.q_proj,2,32,8,128,25.61806869506836,5.015625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,2,64,8,128,30.653871536254883,4.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,2,128,8,128,36.28984451293945,4.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,3,32,8,128,11.005311965942385,7.015625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,3,64,8,128,13.233725547790527,6.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,3,128,8,128,15.956876754760742,6.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,4,32,8,128,5.137575149536133,9.015625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,4,64,8,128,6.189054489135742,8.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,4,128,8,128,7.473674297332764,8.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,8,32,8,128,0.3108876943588257,17.015625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,8,64,8,128,0.3742972612380981,16.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.q_proj,8,128,8,128,0.4504112601280212,16.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5 +14,self_attn.v_proj,2,32,8,128,6.5562052726745605,1.25390625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,2,64,8,128,7.741369724273682,1.126953125,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,2,128,8,128,8.832467079162598,1.0634765625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,3,32,8,128,2.809292316436768,1.75390625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,3,64,8,128,3.325889587402344,1.626953125,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,3,128,8,128,3.810516119003296,1.5634765625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,4,32,8,128,1.31143057346344,2.25390625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,4,64,8,128,1.5532509088516235,2.126953125,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,4,128,8,128,1.7792961597442627,2.0634765625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,8,32,8,128,0.0793827325105667,4.25390625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,8,64,8,128,0.0939697101712226,4.126953125,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.v_proj,8,128,8,128,0.1072141602635383,4.0634765625,4194304,3.625143634568319,NA,2.678215787454974e-5 +14,self_attn.k_proj,2,32,8,128,21.997194290161133,1.25390625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,2,64,8,128,26.364633560180664,1.126953125,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,2,128,8,128,30.29219055175781,1.0634765625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,3,32,8,128,9.51581573486328,1.75390625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,3,64,8,128,11.549477577209473,1.626953125,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,3,128,8,128,13.514533996582031,1.5634765625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,4,32,8,128,4.443383693695068,2.25390625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,4,64,8,128,5.3974151611328125,2.126953125,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,4,128,8,128,6.329847812652588,2.0634765625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,8,32,8,128,0.2690210342407226,4.25390625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,8,64,8,128,0.3263379633426666,4.126953125,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,self_attn.k_proj,8,128,8,128,0.3813648819923401,4.0634765625,4194304,5.444123532860034,NA,2.678215787454974e-5 +14,mlp.gate_proj,2,32,8,128,39.41016387939453,17.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,2,64,8,128,46.97098922729492,15.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,2,128,8,128,54.20206451416016,14.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,3,32,8,128,16.946481704711914,24.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,3,64,8,128,20.342782974243164,22.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,3,128,8,128,23.739776611328125,21.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,4,32,8,128,7.914947509765625,31.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,4,64,8,128,9.501712799072266,29.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,4,128,8,128,11.119906425476074,28.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,8,32,8,128,0.4799664616584778,59.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,8,64,8,128,0.5766294598579407,57.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.gate_proj,8,128,8,128,0.6727944016456604,56.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5 +14,mlp.up_proj,2,32,8,128,31.120824813842773,17.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,2,64,8,128,36.41718673706055,15.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,2,128,8,128,41.25801849365234,14.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,3,32,8,128,13.307998657226562,24.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,3,64,8,128,15.620094299316406,22.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,3,128,8,128,17.724742889404297,21.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,4,32,8,128,6.208417892456055,31.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,4,64,8,128,7.287253856658935,29.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,4,128,8,128,8.278099060058594,28.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,8,32,8,128,0.377309650182724,59.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,8,64,8,128,0.442139208316803,57.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.up_proj,8,128,8,128,0.5002653002738953,56.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5 +14,mlp.down_proj,2,32,8,128,30.64486122131348,17.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,2,64,8,128,35.82240676879883,15.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,2,128,8,128,40.55413055419922,14.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,3,32,8,128,13.101051330566406,24.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,3,64,8,128,15.34364128112793,22.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,3,128,8,128,17.384769439697266,21.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,4,32,8,128,6.1127753257751465,31.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,4,64,8,128,7.158343315124512,29.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,4,128,8,128,8.121088981628418,28.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,8,32,8,128,0.371524840593338,59.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,8,64,8,128,0.4344290196895599,57.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5 +14,mlp.down_proj,8,128,8,128,0.4907621443271637,56.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5 +15,self_attn.q_proj,2,32,8,128,29.381113052368164,5.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,2,64,8,128,35.35570526123047,4.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,2,128,8,128,41.67421722412109,4.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,3,32,8,128,12.663312911987305,7.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,3,64,8,128,15.373734474182127,6.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,3,128,8,128,18.746606826782227,6.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,4,32,8,128,5.9120683670043945,9.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,4,64,8,128,7.184608936309815,8.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,4,128,8,128,8.775914192199707,8.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,8,32,8,128,0.3578397035598755,17.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,8,64,8,128,0.4344322681427002,16.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.q_proj,8,128,8,128,0.5290461778640747,16.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5 +15,self_attn.v_proj,2,32,8,128,7.238279819488525,1.25390625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,2,64,8,128,8.587432861328125,1.126953125,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,2,128,8,128,9.83323097229004,1.0634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,3,32,8,128,3.1004669666290283,1.75390625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,3,64,8,128,3.6899452209472656,1.626953125,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,3,128,8,128,4.236072540283203,1.5634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,4,32,8,128,1.447650671005249,2.25390625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,4,64,8,128,1.7214912176132202,2.126953125,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,4,128,8,128,1.9767045974731443,2.0634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,8,32,8,128,0.087690532207489,4.25390625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,8,64,8,128,0.1041000336408615,4.126953125,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.v_proj,8,128,8,128,0.1191994249820709,4.0634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5 +15,self_attn.k_proj,2,32,8,128,21.04084205627441,1.25390625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,2,64,8,128,25.06307411193848,1.126953125,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,2,128,8,128,28.77065658569336,1.0634765625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,3,32,8,128,9.0236663818359375,1.75390625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,3,64,8,128,10.787558555603027,1.626953125,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,3,128,8,128,12.454113006591797,1.5634765625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,4,32,8,128,4.210366249084473,2.25390625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,4,64,8,128,5.031194686889648,2.126953125,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,4,128,8,128,5.814916133880615,2.0634765625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,8,32,8,128,0.2549245059490204,4.25390625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,8,64,8,128,0.3044169843196869,4.126953125,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,self_attn.k_proj,8,128,8,128,0.3503837883472442,4.0634765625,4194304,4.194555335325779,NA,2.328735718037933e-5 +15,mlp.gate_proj,2,32,8,128,40.27034378051758,17.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,2,64,8,128,48.0027961730957,15.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,2,128,8,128,55.40787887573242,14.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,3,32,8,128,17.291048049926758,24.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,3,64,8,128,20.744365692138672,22.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,3,128,8,128,24.19746971130371,21.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,4,32,8,128,8.074305534362793,31.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,4,64,8,128,9.683999061584473,29.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,4,128,8,128,11.320701599121094,28.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,8,32,8,128,0.4895641803741455,59.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,8,64,8,128,0.5876905918121338,57.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.gate_proj,8,128,8,128,0.68519526720047,56.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5 +15,mlp.up_proj,2,32,8,128,30.786827087402344,17.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,2,64,8,128,35.99784469604492,15.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,2,128,8,128,40.7634391784668,14.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,3,32,8,128,13.163134574890137,24.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,3,64,8,128,15.429580688476562,22.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,3,128,8,128,17.48986053466797,21.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,4,32,8,128,6.141441822052002,31.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,4,64,8,128,7.195915222167969,29.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,4,128,8,128,8.165613174438477,28.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,8,32,8,128,0.3731957972049713,59.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,8,64,8,128,0.4366535246372223,57.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.up_proj,8,128,8,128,0.4933501482009887,56.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5 +15,mlp.down_proj,2,32,8,128,30.46190452575684,17.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,2,64,8,128,35.60262680053711,15.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,2,128,8,128,40.28842544555664,14.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,3,32,8,128,13.020027160644531,24.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,3,64,8,128,15.25452709197998,22.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,3,128,8,128,17.293224334716797,21.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,4,32,8,128,6.07711935043335,31.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,4,64,8,128,7.117532253265381,29.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,4,128,8,128,8.078948974609375,28.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,8,32,8,128,0.3692785799503326,59.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,8,64,8,128,0.4319080412387848,57.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5 +15,mlp.down_proj,8,128,8,128,0.4883196055889129,56.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5 +16,self_attn.q_proj,2,32,8,128,27.64369010925293,5.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,2,64,8,128,33.14297103881836,4.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,2,128,8,128,38.94926834106445,4.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,3,32,8,128,11.870041847229004,7.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,3,64,8,128,14.29415798187256,6.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,3,128,8,128,17.090879440307617,6.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,4,32,8,128,5.538924217224121,9.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,4,64,8,128,6.677006721496582,8.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,4,128,8,128,7.995426177978516,8.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,8,32,8,128,0.3354693651199341,17.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,8,64,8,128,0.4038237333297729,16.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.q_proj,8,128,8,128,0.4819633960723877,16.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5 +16,self_attn.v_proj,2,32,8,128,6.684208869934082,1.25390625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,2,64,8,128,7.885555267333984,1.126953125,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,2,128,8,128,8.979830741882324,1.0634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,3,32,8,128,2.866643190383911,1.75390625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,3,64,8,128,3.392282724380493,1.626953125,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,3,128,8,128,3.8811380863189697,1.5634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,4,32,8,128,1.3368695974349976,2.25390625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,4,64,8,128,1.5830122232437134,2.126953125,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,4,128,8,128,1.8121806383132928,2.0634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,8,32,8,128,0.080979585647583,4.25390625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,8,64,8,128,0.0957918316125869,4.126953125,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.v_proj,8,128,8,128,0.1093247085809707,4.0634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5 +16,self_attn.k_proj,2,32,8,128,21.50868034362793,1.25390625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,2,64,8,128,25.74004554748535,1.126953125,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,2,128,8,128,29.626441955566406,1.0634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,3,32,8,128,9.239008903503418,1.75390625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,3,64,8,128,11.116647720336914,1.626953125,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,3,128,8,128,12.926946640014648,1.5634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,4,32,8,128,4.311690807342529,2.25390625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,4,64,8,128,5.187531471252441,2.126953125,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,4,128,8,128,6.0370965003967285,2.0634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,8,32,8,128,0.261018842458725,4.25390625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,8,64,8,128,0.3137146830558777,4.126953125,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,self_attn.k_proj,8,128,8,128,0.3635620474815368,4.0634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5 +16,mlp.gate_proj,2,32,8,128,41.20309066772461,17.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,2,64,8,128,49.11593246459961,15.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,2,128,8,128,56.7407341003418,14.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,3,32,8,128,17.65849494934082,24.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,3,64,8,128,21.15079498291016,22.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,3,128,8,128,24.60658264160156,21.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,4,32,8,128,8.242731094360352,31.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,4,64,8,128,9.868480682373049,29.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,4,128,8,128,11.499907493591309,28.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,8,32,8,128,0.499927282333374,59.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,8,64,8,128,0.5988888740539551,57.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.gate_proj,8,128,8,128,0.6959277391433716,56.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5 +16,mlp.up_proj,2,32,8,128,30.287851333618164,17.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,2,64,8,128,35.38748550415039,15.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,2,128,8,128,40.03034591674805,14.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,3,32,8,128,12.94804573059082,24.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,3,64,8,128,15.161388397216797,22.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,3,128,8,128,17.161760330200195,21.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,4,32,8,128,6.040839195251465,31.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,4,64,8,128,7.070522785186768,29.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,4,128,8,128,8.018121719360352,28.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,8,32,8,128,0.3671192526817322,59.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,8,64,8,128,0.4291189908981323,57.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.up_proj,8,128,8,128,0.4842276573181152,56.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5 +16,mlp.down_proj,2,32,8,128,29.9908447265625,17.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,2,64,8,128,35.022098541259766,15.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,2,128,8,128,39.60578155517578,14.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,3,32,8,128,12.824692726135254,24.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,3,64,8,128,15.006463050842283,22.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,3,128,8,128,16.995845794677734,21.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,4,32,8,128,5.985208034515381,31.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,4,64,8,128,7.0030622482299805,29.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,4,128,8,128,7.940154075622559,28.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,8,32,8,128,0.3637315034866333,59.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,8,64,8,128,0.4250793755054474,57.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5 +16,mlp.down_proj,8,128,8,128,0.4799773395061493,56.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5 +17,self_attn.q_proj,2,32,8,128,28.243900299072266,5.015625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,2,64,8,128,33.9013671875,4.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,2,128,8,128,39.67461013793945,4.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,3,32,8,128,12.116202354431152,7.015625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,3,64,8,128,14.606330871582031,6.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,3,128,8,128,17.453651428222656,6.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,4,32,8,128,5.654839515686035,9.015625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,4,64,8,128,6.818025588989258,8.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,4,128,8,128,8.15267562866211,8.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,8,32,8,128,0.3423314392566681,17.015625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,8,64,8,128,0.4120905995368957,16.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.q_proj,8,128,8,128,0.491163969039917,16.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5 +17,self_attn.v_proj,2,32,8,128,7.274535655975342,1.25390625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,2,64,8,128,8.557072639465332,1.126953125,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,2,128,8,128,9.730944633483888,1.0634765625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,3,32,8,128,3.112748146057129,1.75390625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,3,64,8,128,3.664875745773315,1.626953125,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,3,128,8,128,4.174811363220215,1.5634765625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,4,32,8,128,1.4522428512573242,2.25390625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,4,64,8,128,1.7108559608459473,2.126953125,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,4,128,8,128,1.9488672018051147,2.0634765625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,8,32,8,128,0.0879412665963172,4.25390625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,8,64,8,128,0.1034611463546752,4.126953125,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.v_proj,8,128,8,128,0.1174446046352386,4.0634765625,4194304,3.41969594431638,NA,2.093178591167089e-5 +17,self_attn.k_proj,2,32,8,128,21.450056076049805,1.25390625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,2,64,8,128,25.589683532714844,1.126953125,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,2,128,8,128,29.36025619506836,1.0634765625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,3,32,8,128,9.210640907287598,1.75390625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,3,64,8,128,11.032175064086914,1.626953125,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,3,128,8,128,12.762304306030272,1.5634765625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,4,32,8,128,4.297592639923096,2.25390625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,4,64,8,128,5.15066385269165,2.126953125,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,4,128,8,128,5.959950923919678,2.0634765625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,8,32,8,128,0.2601512372493744,4.25390625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,8,64,8,128,0.3113722205162048,4.126953125,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,self_attn.k_proj,8,128,8,128,0.3592627048492431,4.0634765625,4194304,4.530582257065935,NA,2.093178591167089e-5 +17,mlp.gate_proj,2,32,8,128,41.69158172607422,17.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,2,64,8,128,49.69167709350586,15.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,2,128,8,128,57.42879104614258,14.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,3,32,8,128,17.86820411682129,24.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,3,64,8,128,21.40752601623535,22.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,3,128,8,128,24.90820503234864,21.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,4,32,8,128,8.338918685913086,31.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,4,64,8,128,9.987314224243164,29.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,4,128,8,128,11.63386344909668,28.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,8,32,8,128,0.5058061480522156,59.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,8,64,8,128,0.6063711047172546,57.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.gate_proj,8,128,8,128,0.7045861482620239,56.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5 +17,mlp.up_proj,2,32,8,128,30.13922119140625,17.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,2,64,8,128,35.19236755371094,15.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,2,128,8,128,39.78667068481445,14.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,3,32,8,128,12.884716033935549,24.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,3,64,8,128,15.078737258911133,22.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,3,128,8,128,17.062938690185547,21.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,4,32,8,128,6.011682033538818,31.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,4,64,8,128,7.033123970031738,29.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,4,128,8,128,7.97081708908081,28.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,8,32,8,128,0.3654176890850067,59.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,8,64,8,128,0.4268718361854553,57.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.up_proj,8,128,8,128,0.4815108478069305,56.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5 +17,mlp.down_proj,2,32,8,128,29.92352867126465,17.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,2,64,8,128,34.939910888671875,15.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,2,128,8,128,39.52711868286133,14.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,3,32,8,128,12.802757263183594,24.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,3,64,8,128,14.989115715026855,22.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,3,128,8,128,16.984521865844727,21.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,4,32,8,128,5.975630760192871,31.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,4,64,8,128,6.997411727905273,29.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,4,128,8,128,7.946293830871582,28.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,8,32,8,128,0.3631942868232727,59.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,8,64,8,128,0.4247721433639526,57.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5 +17,mlp.down_proj,8,128,8,128,0.4803014993667602,56.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5 +18,self_attn.q_proj,2,32,8,128,27.30513381958008,5.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,2,64,8,128,32.50996017456055,4.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,2,128,8,128,38.30387878417969,4.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,3,32,8,128,11.690193176269531,7.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,3,64,8,128,13.941518783569336,6.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,3,128,8,128,16.589494705200195,6.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,4,32,8,128,5.45534086227417,9.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,4,64,8,128,6.506054401397705,8.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,4,128,8,128,7.744174957275391,8.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,8,32,8,128,0.3303576409816742,17.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,8,64,8,128,0.3934611678123474,16.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.q_proj,8,128,8,128,0.4666202068328857,16.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5 +18,self_attn.v_proj,2,32,8,128,6.611334323883057,1.25390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,2,64,8,128,7.799045085906982,1.126953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,2,128,8,128,8.893885612487793,1.0634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,3,32,8,128,2.832592725753784,1.75390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,3,64,8,128,3.348790168762207,1.626953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,3,128,8,128,3.829371929168701,1.5634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,4,32,8,128,1.3221431970596311,2.25390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,4,64,8,128,1.5629265308380127,2.126953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,4,128,8,128,1.788077473640442,2.0634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,8,32,8,128,0.0800507664680481,4.25390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,8,64,8,128,0.0945601016283035,4.126953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.v_proj,8,128,8,128,0.107764333486557,4.0634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5 +18,self_attn.k_proj,2,32,8,128,21.83268165588379,1.25390625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,2,64,8,128,25.942413330078125,1.126953125,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,2,128,8,128,29.68288993835449,1.0634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,3,32,8,128,9.358898162841797,1.75390625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,3,64,8,128,11.15013599395752,1.626953125,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,3,128,8,128,12.824159622192385,1.5634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,4,32,8,128,4.364651203155518,2.25390625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,4,64,8,128,5.203103542327881,2.126953125,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,4,128,8,128,5.986807823181152,2.0634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,8,32,8,128,0.264183759689331,4.25390625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,8,64,8,128,0.3146951198577881,4.126953125,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,self_attn.k_proj,8,128,8,128,0.3608006238937378,4.0634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5 +18,mlp.gate_proj,2,32,8,128,41.45539855957031,17.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,2,64,8,128,49.28993988037109,15.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,2,128,8,128,56.8601188659668,14.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,3,32,8,128,17.751710891723633,24.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,3,64,8,128,21.1910343170166,22.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,3,128,8,128,24.55527877807617,21.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,4,32,8,128,8.285884857177734,31.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,4,64,8,128,9.88524055480957,29.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,4,128,8,128,11.470847129821776,28.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,8,32,8,128,0.5024477243423462,59.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,8,64,8,128,0.5998983979225159,57.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.gate_proj,8,128,8,128,0.6943526268005371,56.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5 +18,mlp.up_proj,2,32,8,128,29.890323638916016,17.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,2,64,8,128,34.86293029785156,15.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,2,128,8,128,39.36655426025391,14.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,3,32,8,128,12.7736177444458,24.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,3,64,8,128,14.925217628479004,22.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,3,128,8,128,16.863479614257812,21.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,4,32,8,128,5.9599432945251465,31.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,4,64,8,128,6.961752891540527,29.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,4,128,8,128,7.874423027038574,28.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,8,32,8,128,0.3622696101665497,59.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,8,64,8,128,0.4225781559944153,57.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.up_proj,8,128,8,128,0.4757608473300934,56.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5 +18,mlp.down_proj,2,32,8,128,29.774751663208008,17.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,2,64,8,128,34.73874282836914,15.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,2,128,8,128,39.24348831176758,14.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,3,32,8,128,12.72853183746338,24.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,3,64,8,128,14.876253128051758,22.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,3,128,8,128,16.82144546508789,21.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,4,32,8,128,5.939826488494873,31.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,4,64,8,128,6.940883636474609,29.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,4,128,8,128,7.857932090759277,28.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,8,32,8,128,0.3610455095767975,59.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,8,64,8,128,0.4213594198226928,57.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5 +18,mlp.down_proj,8,128,8,128,0.4748960435390472,56.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5 +19,self_attn.q_proj,2,32,8,128,27.51787567138672,5.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,2,64,8,128,32.85622024536133,4.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,2,128,8,128,38.62761688232422,4.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,3,32,8,128,11.786128997802734,7.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,3,64,8,128,14.105334281921388,6.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,3,128,8,128,16.762516021728516,6.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,4,32,8,128,5.501605987548828,9.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,4,64,8,128,6.584290027618408,8.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,4,128,8,128,7.82921838760376,8.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,8,32,8,128,0.333046406507492,17.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,8,64,8,128,0.3981630206108093,16.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.q_proj,8,128,8,128,0.4716943204402923,16.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5 +19,self_attn.v_proj,2,32,8,128,7.030470371246338,1.25390625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,2,64,8,128,8.304581642150879,1.126953125,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,2,128,8,128,9.479228973388672,1.0634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,3,32,8,128,3.008863925933838,1.75390625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,3,64,8,128,3.5585362911224365,1.626953125,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,3,128,8,128,4.066886425018311,1.5634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,4,32,8,128,1.404317021369934,2.25390625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,4,64,8,128,1.6601585149765017,2.126953125,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,4,128,8,128,1.8971848487854004,2.0634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,8,32,8,128,0.0850284621119499,4.25390625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,8,64,8,128,0.1004456281661987,4.126953125,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.v_proj,8,128,8,128,0.1143800020217895,4.0634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5 +19,self_attn.k_proj,2,32,8,128,21.01363182067871,1.25390625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,2,64,8,128,24.98497200012207,1.126953125,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,2,128,8,128,28.669771194458008,1.0634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,3,32,8,128,8.993736267089844,1.75390625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,3,64,8,128,10.7388277053833,1.626953125,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,3,128,8,128,12.39926528930664,1.5634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,4,32,8,128,4.197534561157227,2.25390625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,4,64,8,128,5.010700225830078,2.126953125,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,4,128,8,128,5.782396793365479,2.0634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,8,32,8,128,0.2539751529693603,4.25390625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,8,64,8,128,0.3030913770198822,4.126953125,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,self_attn.k_proj,8,128,8,128,0.3485586941242218,4.0634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5 +19,mlp.gate_proj,2,32,8,128,41.56752014160156,17.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,2,64,8,128,49.3490104675293,15.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,2,128,8,128,56.8859748840332,14.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,3,32,8,128,17.795673370361328,24.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,3,64,8,128,21.20685195922852,22.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,3,128,8,128,24.54552459716797,21.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,4,32,8,128,8.306742668151855,31.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,4,64,8,128,9.892951965332031,29.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,4,128,8,128,11.465137481689451,28.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,8,32,8,128,0.5038571357727051,59.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,8,64,8,128,0.6006820201873779,57.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.gate_proj,8,128,8,128,0.6943200826644897,56.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5 +19,mlp.up_proj,2,32,8,128,29.724327087402344,17.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,2,64,8,128,34.64796829223633,15.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,2,128,8,128,39.09792709350586,14.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,3,32,8,128,12.704089164733888,24.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,3,64,8,128,14.834226608276367,22.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,3,128,8,128,16.749486923217773,21.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,4,32,8,128,5.928028583526611,31.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,4,64,8,128,6.918476581573486,29.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,4,128,8,128,7.819451332092285,28.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,8,32,8,128,0.3603351414203644,59.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,8,64,8,128,0.4199293255805969,57.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.up_proj,8,128,8,128,0.4725494384765625,56.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5 +19,mlp.down_proj,2,32,8,128,29.674684524536133,17.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,2,64,8,128,34.607879638671875,15.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,2,128,8,128,39.08812713623047,14.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,3,32,8,128,12.686997413635254,24.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,3,64,8,128,14.819061279296877,22.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,3,128,8,128,16.75285530090332,21.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,4,32,8,128,5.920626640319824,31.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,4,64,8,128,6.914460182189941,29.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,4,128,8,128,7.822765827178955,28.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,8,32,8,128,0.3599112629890442,59.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,8,64,8,128,0.4198633134365082,57.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5 +19,mlp.down_proj,8,128,8,128,0.4729660749435425,56.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5 +20,self_attn.q_proj,2,32,8,128,27.31611633300781,5.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,2,64,8,128,32.593658447265625,4.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,2,128,8,128,38.2834587097168,4.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,3,32,8,128,11.700910568237305,7.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,3,64,8,128,13.989947319030762,6.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,3,128,8,128,16.647871017456055,6.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,4,32,8,128,5.460706233978272,9.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,4,64,8,128,6.527448654174805,8.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,4,128,8,128,7.773484230041504,8.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,8,32,8,128,0.3306195735931396,17.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,8,64,8,128,0.3947963714599609,16.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.q_proj,8,128,8,128,0.4683714509010315,16.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5 +20,self_attn.v_proj,2,32,8,128,7.3568525314331055,1.25390625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,2,64,8,128,8.683716773986816,1.126953125,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,2,128,8,128,9.915620803833008,1.0634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,3,32,8,128,3.148019313812256,1.75390625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,3,64,8,128,3.718897819519043,1.626953125,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,3,128,8,128,4.255496978759766,1.5634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,4,32,8,128,1.4689884185791016,2.25390625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,4,64,8,128,1.7367335557937622,2.126953125,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,4,128,8,128,1.987268686294556,2.0634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,8,32,8,128,0.0889342427253723,4.25390625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,8,64,8,128,0.1050533503293991,4.126953125,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.v_proj,8,128,8,128,0.1197745874524116,4.0634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5 +20,self_attn.k_proj,2,32,8,128,20.810379028320312,1.25390625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,2,64,8,128,24.79256057739257,1.126953125,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,2,128,8,128,28.46826171875,1.0634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,3,32,8,128,8.924111366271973,1.75390625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,3,64,8,128,10.67470359802246,1.626953125,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,3,128,8,128,12.336373329162598,1.5634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,4,32,8,128,4.164855480194092,2.25390625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,4,64,8,128,4.982593536376953,2.126953125,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,4,128,8,128,5.7602858543396,2.0634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,8,32,8,128,0.2521738708019256,4.25390625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,8,64,8,128,0.301372081041336,4.126953125,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,self_attn.k_proj,8,128,8,128,0.3471800684928894,4.0634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5 +20,mlp.gate_proj,2,32,8,128,41.40095901489258,17.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,2,64,8,128,49.0767936706543,15.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,2,128,8,128,56.47530364990234,14.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,3,32,8,128,17.720191955566406,24.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,3,64,8,128,21.07521247863769,22.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,3,128,8,128,24.336673736572266,21.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,4,32,8,128,8.272833824157715,31.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,4,64,8,128,9.831510543823242,29.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,4,128,8,128,11.366402626037598,28.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,8,32,8,128,0.5016068816184998,59.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,8,64,8,128,0.5967675447463989,57.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.gate_proj,8,128,8,128,0.6880964636802673,56.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5 +20,mlp.up_proj,2,32,8,128,29.80850410461425,17.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,2,64,8,128,34.74061584472656,15.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,2,128,8,128,39.19353103637695,14.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,3,32,8,128,12.740331649780272,24.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,3,64,8,128,14.869338989257812,22.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,3,128,8,128,16.785167694091797,21.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,4,32,8,128,5.9447431564331055,31.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,4,64,8,128,6.93602991104126,29.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,4,128,8,128,7.835127353668213,28.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,8,32,8,128,0.3612865805625915,59.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,8,64,8,128,0.4209855198860168,57.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.up_proj,8,128,8,128,0.4734995067119598,56.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5 +20,mlp.down_proj,2,32,8,128,29.76346969604492,17.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,2,64,8,128,34.7084846496582,15.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,2,128,8,128,39.18354797363281,14.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,3,32,8,128,12.723167419433594,24.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,3,64,8,128,14.86019515991211,22.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,3,128,8,128,16.789318084716797,21.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,4,32,8,128,5.938924789428711,31.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,4,64,8,128,6.934266090393066,29.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,4,128,8,128,7.8432393074035645,28.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,8,32,8,128,0.3608524799346924,59.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,8,64,8,128,0.4208720326423645,57.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5 +20,mlp.down_proj,8,128,8,128,0.473922461271286,56.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5 +21,self_attn.q_proj,2,32,8,128,26.613637924194336,5.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,2,64,8,128,31.72728157043457,4.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,2,128,8,128,37.28963851928711,4.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,3,32,8,128,11.398756980895996,7.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,3,64,8,128,13.612385749816896,6.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,3,128,8,128,16.119693756103516,6.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,4,32,8,128,5.320000171661377,9.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,4,64,8,128,6.354912757873535,8.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,4,128,8,128,7.528256893157959,8.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,8,32,8,128,0.3220490217208862,17.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,8,64,8,128,0.3843535780906677,16.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.q_proj,8,128,8,128,0.4536494314670563,16.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5 +21,self_attn.v_proj,2,32,8,128,7.535129547119141,1.25390625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,2,64,8,128,8.869972229003906,1.126953125,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,2,128,8,128,10.069236755371094,1.0634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,3,32,8,128,3.236624956130981,1.75390625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,3,64,8,128,3.824875831604004,1.626953125,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,3,128,8,128,4.362272262573242,1.5634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,4,32,8,128,1.511125922203064,2.25390625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,4,64,8,128,1.7847682237625122,2.126953125,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,4,128,8,128,2.038510322570801,2.0634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,8,32,8,128,0.091513954102993,4.25390625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,8,64,8,128,0.1080395504832267,4.126953125,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.v_proj,8,128,8,128,0.122831143438816,4.0634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5 +21,self_attn.k_proj,2,32,8,128,20.66949462890625,1.25390625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,2,64,8,128,24.51624298095703,1.126953125,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,2,128,8,128,28.033615112304688,1.0634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,3,32,8,128,8.851082801818848,1.75390625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,3,64,8,128,10.525378227233888,1.626953125,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,3,128,8,128,12.100557327270508,1.5634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,4,32,8,128,4.1329874992370605,2.25390625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,4,64,8,128,4.914510726928711,2.126953125,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,4,128,8,128,5.648586273193359,2.0634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,8,32,8,128,0.2501528859138489,4.25390625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,8,64,8,128,0.2973358035087585,4.126953125,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,self_attn.k_proj,8,128,8,128,0.3402147889137268,4.0634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5 +21,mlp.gate_proj,2,32,8,128,41.70660781860352,17.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,2,64,8,128,49.44601440429688,15.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,2,128,8,128,56.88743591308594,14.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,3,32,8,128,17.864152908325195,24.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,3,64,8,128,21.24855232238769,22.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,3,128,8,128,24.55762481689453,21.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,4,32,8,128,8.337279319763184,31.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,4,64,8,128,9.915610313415527,29.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,4,128,8,128,11.472938537597656,28.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,8,32,8,128,0.5056385397911072,59.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,8,64,8,128,0.6018403172492981,57.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.gate_proj,8,128,8,128,0.6946837902069092,56.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5 +21,mlp.up_proj,2,32,8,128,29.90048789978028,17.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,2,64,8,128,34.84867858886719,15.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,2,128,8,128,39.3271598815918,14.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,3,32,8,128,12.779494285583496,24.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,3,64,8,128,14.922487258911133,22.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,3,128,8,128,16.849123001098633,21.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,4,32,8,128,5.964380264282227,31.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,4,64,8,128,6.9611639976501465,29.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,4,128,8,128,7.867311954498291,28.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,8,32,8,128,0.3624645173549652,59.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,8,64,8,128,0.4223872125148773,57.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.up_proj,8,128,8,128,0.4753336608409881,56.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5 +21,mlp.down_proj,2,32,8,128,29.85926628112793,17.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,2,64,8,128,34.81978225708008,15.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,2,128,8,128,39.30513381958008,14.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,3,32,8,128,12.764520645141602,24.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,3,64,8,128,14.916038513183594,22.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,3,128,8,128,16.857704162597656,21.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,4,32,8,128,5.956398010253906,31.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,4,64,8,128,6.958872318267822,29.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,4,128,8,128,7.873658180236816,28.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,8,32,8,128,0.3620364665985107,59.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,8,64,8,128,0.4224471151828766,57.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5 +21,mlp.down_proj,8,128,8,128,0.4759640395641327,56.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5 +22,self_attn.q_proj,2,32,8,128,25.80609130859375,5.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,2,64,8,128,30.682270050048828,4.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,2,128,8,128,35.724220275878906,4.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,3,32,8,128,11.056154251098633,7.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,3,64,8,128,13.17056941986084,6.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,3,128,8,128,15.422249794006348,6.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,4,32,8,128,5.15937089920044,9.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,4,64,8,128,6.1462082862854,8.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,4,128,8,128,7.201276302337647,8.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,8,32,8,128,0.3124086260795593,17.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,8,64,8,128,0.3716919422149658,16.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.q_proj,8,128,8,128,0.4336889386177063,16.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5 +22,self_attn.v_proj,2,32,8,128,7.859790325164795,1.25390625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,2,64,8,128,9.203939437866213,1.126953125,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,2,128,8,128,10.420348167419434,1.0634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,3,32,8,128,3.3680005073547363,1.75390625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,3,64,8,128,3.952582597732544,1.626953125,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,3,128,8,128,4.486354351043701,1.5634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,4,32,8,128,1.5703846216201782,2.25390625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,4,64,8,128,1.8439210653305047,2.126953125,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,4,128,8,128,2.0944957733154297,2.0634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,8,32,8,128,0.0951443091034889,4.25390625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,8,64,8,128,0.1115146800875663,4.126953125,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.v_proj,8,128,8,128,0.1261604875326156,4.0634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5 +22,self_attn.k_proj,2,32,8,128,20.348981857299805,1.25390625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,2,64,8,128,24.19514274597168,1.126953125,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,2,128,8,128,27.700634002685547,1.0634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,3,32,8,128,8.726444244384766,1.75390625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,3,64,8,128,10.408102989196776,1.626953125,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,3,128,8,128,11.971535682678224,1.5634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,4,32,8,128,4.073057651519775,2.25390625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,4,64,8,128,4.855081081390381,2.126953125,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,4,128,8,128,5.589670181274414,2.0634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,8,32,8,128,0.2463237047195434,4.25390625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,8,64,8,128,0.2936039268970489,4.126953125,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,self_attn.k_proj,8,128,8,128,0.3368735909461975,4.0634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5 +22,mlp.gate_proj,2,32,8,128,41.4613151550293,17.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,2,64,8,128,49.01024627685547,15.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,2,128,8,128,56.24254989624024,14.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,3,32,8,128,17.741302490234375,24.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,3,64,8,128,21.031667709350582,22.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,3,128,8,128,24.197364807128903,21.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,4,32,8,128,8.280515670776367,31.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,4,64,8,128,9.813997268676758,29.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,4,128,8,128,11.302498817443848,28.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,8,32,8,128,0.502230167388916,59.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,8,64,8,128,0.595484733581543,57.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.gate_proj,8,128,8,128,0.6840186715126038,56.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5 +22,mlp.up_proj,2,32,8,128,29.95798110961914,17.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,2,64,8,128,34.88895797729492,15.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,2,128,8,128,39.34118270874024,14.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,3,32,8,128,12.801081657409668,24.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,3,64,8,128,14.933656692504885,22.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,3,128,8,128,16.854429244995117,21.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,4,32,8,128,5.973466396331787,31.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,4,64,8,128,6.9661478996276855,29.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,4,128,8,128,7.867184638977051,28.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,8,32,8,128,0.3629900217056274,59.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,8,64,8,128,0.4228070378303528,57.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.up_proj,8,128,8,128,0.4752513766288757,56.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5 +22,mlp.down_proj,2,32,8,128,29.94915580749512,17.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,2,64,8,128,34.90878677368164,15.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,2,128,8,128,39.39927291870117,14.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,3,32,8,128,12.799161911010742,24.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,3,64,8,128,14.940771102905272,22.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,3,128,8,128,16.866147994995117,21.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,4,32,8,128,5.972724437713623,31.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,4,64,8,128,6.97109317779541,29.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,4,128,8,128,7.876532077789307,28.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,8,32,8,128,0.3630459606647491,59.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,8,64,8,128,0.4231007993221283,57.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5 +22,mlp.down_proj,8,128,8,128,0.4760325253009796,56.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5 +23,self_attn.q_proj,2,32,8,128,26.0787353515625,5.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,2,64,8,128,30.93896484375,4.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,2,128,8,128,35.81895446777344,4.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,3,32,8,128,11.175576210021973,7.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,3,64,8,128,13.291476249694824,6.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,3,128,8,128,15.451822280883787,6.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,4,32,8,128,5.21599006652832,9.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,4,64,8,128,6.204405307769775,8.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,4,128,8,128,7.213257789611816,8.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,8,32,8,128,0.3157745599746704,17.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,8,64,8,128,0.3752713799476623,16.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.q_proj,8,128,8,128,0.43470299243927,16.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5 +23,self_attn.v_proj,2,32,8,128,8.22785472869873,1.25390625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,2,64,8,128,9.627814292907717,1.126953125,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,2,128,8,128,10.899560928344728,1.0634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,3,32,8,128,3.5217771530151367,1.75390625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,3,64,8,128,4.127471446990967,1.626953125,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,3,128,8,128,4.680270195007324,1.5634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,4,32,8,128,1.6431550979614258,2.25390625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,4,64,8,128,1.925819635391236,2.126953125,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,4,128,8,128,2.184685230255127,2.0634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,8,32,8,128,0.0995269864797592,4.25390625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,8,64,8,128,0.1165014430880546,4.126953125,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.v_proj,8,128,8,128,0.131690114736557,4.0634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5 +23,self_attn.k_proj,2,32,8,128,20.1629695892334,1.25390625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,2,64,8,128,23.915668487548828,1.126953125,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,2,128,8,128,27.350526809692383,1.0634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,3,32,8,128,8.644986152648926,1.75390625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,3,64,8,128,10.28492259979248,1.626953125,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,3,128,8,128,11.814826011657717,1.5634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,4,32,8,128,4.0297722816467285,2.25390625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,4,64,8,128,4.7978715896606445,2.126953125,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,4,128,8,128,5.514894485473633,2.0634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,8,32,8,128,0.2441458851099014,4.25390625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,8,64,8,128,0.2903361618518829,4.126953125,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,self_attn.k_proj,8,128,8,128,0.3323166370391845,4.0634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5 +23,mlp.gate_proj,2,32,8,128,41.48234558105469,17.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,2,64,8,128,48.99182891845703,15.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,2,128,8,128,56.17173767089844,14.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,3,32,8,128,17.746910095214844,24.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,3,64,8,128,21.01276206970215,22.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,3,128,8,128,24.165817260742188,21.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,4,32,8,128,8.283446311950684,31.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,4,64,8,128,9.805842399597168,29.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,4,128,8,128,11.291529655456545,28.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,8,32,8,128,0.5024151802062988,59.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,8,64,8,128,0.5952130556106567,57.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.gate_proj,8,128,8,128,0.6831754446029663,56.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5 +23,mlp.up_proj,2,32,8,128,30.061702728271484,17.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,2,64,8,128,34.997066497802734,15.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,2,128,8,128,39.485137939453125,14.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,3,32,8,128,12.845873832702637,24.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,3,64,8,128,14.98180866241455,22.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,3,128,8,128,16.903629302978516,21.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,4,32,8,128,5.993279457092285,31.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,4,64,8,128,6.989481925964356,29.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,4,128,8,128,7.890745162963867,28.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,8,32,8,128,0.3642669320106506,59.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,8,64,8,128,0.4241582751274109,57.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.up_proj,8,128,8,128,0.476793110370636,56.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5 +23,mlp.down_proj,2,32,8,128,30.07884788513184,17.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,2,64,8,128,35.05538558959961,15.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,2,128,8,128,39.556678771972656,14.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,3,32,8,128,12.851511001586914,24.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,3,64,8,128,15.004534721374512,22.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,3,128,8,128,16.92758560180664,21.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,4,32,8,128,5.996975898742676,31.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,4,64,8,128,6.999776363372803,29.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,4,128,8,128,7.905321598052978,28.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,8,32,8,128,0.3644680082798004,59.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,8,64,8,128,0.4247871041297912,57.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5 +23,mlp.down_proj,8,128,8,128,0.4777131676673889,56.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5 +24,self_attn.q_proj,2,32,8,128,25.547338485717773,5.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,2,64,8,128,30.37944602966309,4.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,2,128,8,128,35.39955139160156,4.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,3,32,8,128,10.960057258605955,7.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,3,64,8,128,13.067266464233398,6.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,3,128,8,128,15.332708358764648,6.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,4,32,8,128,5.114231109619141,9.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,4,64,8,128,6.100057125091553,8.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,4,128,8,128,7.159446716308594,8.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,8,32,8,128,0.309750497341156,17.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,8,64,8,128,0.3689931631088257,16.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.q_proj,8,128,8,128,0.4316083788871765,16.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5 +24,self_attn.v_proj,2,32,8,128,9.21454620361328,1.25390625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,2,64,8,128,10.856494903564451,1.126953125,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,2,128,8,128,12.344929695129396,1.0634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,3,32,8,128,3.948546886444092,1.75390625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,3,64,8,128,4.659796237945557,1.626953125,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,3,128,8,128,5.323941707611084,1.5634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,4,32,8,128,1.8415682315826416,2.25390625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,4,64,8,128,2.1758296489715576,2.126953125,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,4,128,8,128,2.484900951385498,2.0634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,8,32,8,128,0.1115508377552032,4.25390625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,8,64,8,128,0.1316062808036804,4.126953125,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.v_proj,8,128,8,128,0.1497008502483368,4.0634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5 +24,self_attn.k_proj,2,32,8,128,19.65592384338379,1.25390625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,2,64,8,128,23.45724868774414,1.126953125,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,2,128,8,128,26.901391983032227,1.0634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,3,32,8,128,8.4442777633667,1.75390625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,3,64,8,128,10.13780689239502,1.626953125,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,3,128,8,128,11.737333297729492,1.5634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,4,32,8,128,3.938342332839966,2.25390625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,4,64,8,128,4.7286696434021,2.126953125,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,4,128,8,128,5.481230735778809,2.0634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,8,32,8,128,0.2383520901203155,4.25390625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,8,64,8,128,0.2860680222511291,4.126953125,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,self_attn.k_proj,8,128,8,128,0.3303963541984558,4.0634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5 +24,mlp.gate_proj,2,32,8,128,41.63678359985352,17.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,2,64,8,128,49.15891647338867,15.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,2,128,8,128,56.340293884277344,14.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,3,32,8,128,17.812793731689453,24.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,3,64,8,128,21.092559814453125,22.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,3,128,8,128,24.236173629760746,21.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,4,32,8,128,8.314607620239258,31.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,4,64,8,128,9.837584495544434,29.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,4,128,8,128,11.327216148376465,28.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,8,32,8,128,0.5042733550071716,59.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,8,64,8,128,0.5973262190818787,57.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.gate_proj,8,128,8,128,0.6853898763656616,56.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5 +24,mlp.up_proj,2,32,8,128,30.197975158691406,17.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,2,64,8,128,35.16866683959961,15.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,2,128,8,128,39.64540481567383,14.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,3,32,8,128,12.90445613861084,24.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,3,64,8,128,15.05442237854004,22.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,3,128,8,128,16.983179092407227,21.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,4,32,8,128,6.020968437194824,31.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,4,64,8,128,7.023828983306885,29.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,4,128,8,128,7.930030822753906,28.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,8,32,8,128,0.3659334182739258,59.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,8,64,8,128,0.4261747896671295,57.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.up_proj,8,128,8,128,0.4791154861450195,56.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5 +24,mlp.down_proj,2,32,8,128,30.21818351745605,17.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,2,64,8,128,35.216644287109375,15.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,2,128,8,128,39.71371078491211,14.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,3,32,8,128,12.912989616394045,24.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,3,64,8,128,15.066468238830566,22.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,3,128,8,128,16.998876571655273,21.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,4,32,8,128,6.024109840393066,31.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,4,64,8,128,7.027546405792236,29.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,4,128,8,128,7.935858726501465,28.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,8,32,8,128,0.3661464154720306,59.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,8,64,8,128,0.4265152513980865,57.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5 +24,mlp.down_proj,8,128,8,128,0.4795728325843811,56.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5 +25,self_attn.q_proj,2,32,8,128,24.947967529296875,5.015625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,2,64,8,128,29.67112159729004,4.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,2,128,8,128,34.52857208251953,4.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,3,32,8,128,10.708093643188477,7.015625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,3,64,8,128,12.765064239501951,6.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,3,128,8,128,14.922027587890623,6.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,4,32,8,128,4.997904300689697,9.015625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,4,64,8,128,5.958414077758789,8.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,4,128,8,128,6.966994285583496,8.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,8,32,8,128,0.302553117275238,17.015625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,8,64,8,128,0.3603058755397796,16.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.q_proj,8,128,8,128,0.4198490381240845,16.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5 +25,self_attn.v_proj,2,32,8,128,9.151209831237791,1.25390625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,2,64,8,128,10.719162940979004,1.126953125,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,2,128,8,128,12.100696563720703,1.0634765625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,3,32,8,128,3.927787780761719,1.75390625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,3,64,8,128,4.620433330535889,1.626953125,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,3,128,8,128,5.249710559844971,1.5634765625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,4,32,8,128,1.8322116136550903,2.25390625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,4,64,8,128,2.155165195465088,2.126953125,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,4,128,8,128,2.4497387409210205,2.0634765625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,8,32,8,128,0.1109356805682182,4.25390625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,8,64,8,128,0.1303529143333435,4.126953125,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.v_proj,8,128,8,128,0.1476238965988159,4.0634765625,4194304,3.679973980730717,NA,3.281717363279313e-5 +25,self_attn.k_proj,2,32,8,128,19.44907188415528,1.25390625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,2,64,8,128,23.125280380249023,1.126953125,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,2,128,8,128,26.34911727905273,1.0634765625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,3,32,8,128,8.36483097076416,1.75390625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,3,64,8,128,10.003403663635254,1.626953125,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,3,128,8,128,11.534846305847168,1.5634765625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,4,32,8,128,3.904937505722046,2.25390625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,4,64,8,128,4.670424938201904,2.126953125,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,4,128,8,128,5.3935546875,2.0634765625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,8,32,8,128,0.2363821864128112,4.25390625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,8,64,8,128,0.2823867499828338,4.126953125,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,self_attn.k_proj,8,128,8,128,0.3251522481441498,4.0634765625,4194304,4.834827145491792,NA,3.281717363279313e-5 +25,mlp.gate_proj,2,32,8,128,42.0462760925293,17.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,2,64,8,128,49.69237518310547,15.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,2,128,8,128,57.04249572753906,14.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,3,32,8,128,17.991498947143555,24.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,3,64,8,128,21.332902908325195,22.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,3,128,8,128,24.555967330932617,21.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,4,32,8,128,8.395861625671387,31.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,4,64,8,128,9.950685501098633,29.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,4,128,8,128,11.46721076965332,28.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,8,32,8,128,0.5092549324035645,59.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,8,64,8,128,0.6039854884147644,57.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.gate_proj,8,128,8,128,0.6943060755729675,56.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5 +25,mlp.up_proj,2,32,8,128,30.46546936035156,17.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,2,64,8,128,35.48342514038086,15.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,2,128,8,128,40.01642608642578,14.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,3,32,8,128,13.018470764160156,24.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,3,64,8,128,15.19490909576416,22.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,3,128,8,128,17.145286560058594,21.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,4,32,8,128,6.074680805206299,31.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,4,64,8,128,7.086403846740723,29.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,4,128,8,128,8.004955291748047,28.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,8,32,8,128,0.3691857159137726,59.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,8,64,8,128,0.4300290644168854,57.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.up_proj,8,128,8,128,0.4836242198944092,56.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5 +25,mlp.down_proj,2,32,8,128,30.48154258728028,17.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,2,64,8,128,35.51686096191406,15.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,2,128,8,128,40.075382232666016,14.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,3,32,8,128,13.021214485168455,24.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,3,64,8,128,15.201196670532228,22.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,3,128,8,128,17.143184661865234,21.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,4,32,8,128,6.07676887512207,31.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,4,64,8,128,7.08850622177124,29.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,4,128,8,128,8.005151748657227,28.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,8,32,8,128,0.3692609965801239,59.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,8,64,8,128,0.4302140176296234,57.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5 +25,mlp.down_proj,8,128,8,128,0.4836136102676391,56.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5 +26,self_attn.q_proj,2,32,8,128,25.341272354125977,5.015625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,2,64,8,128,30.107913970947266,4.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,2,128,8,128,35.163570404052734,4.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,3,32,8,128,10.86099338531494,7.015625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,3,64,8,128,12.921805381774902,6.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,3,128,8,128,15.189332008361816,6.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,4,32,8,128,5.066875457763672,9.015625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,4,64,8,128,6.032660484313965,8.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,4,128,8,128,7.090411186218262,8.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,8,32,8,128,0.3067868947982788,17.015625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,8,64,8,128,0.3648423254489898,16.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.q_proj,8,128,8,128,0.4275660812854767,16.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5 +26,self_attn.v_proj,2,32,8,128,9.58985996246338,1.25390625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,2,64,8,128,11.23852825164795,1.126953125,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,2,128,8,128,12.71847438812256,1.0634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,3,32,8,128,4.117676734924316,1.75390625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,3,64,8,128,4.841552257537842,1.626953125,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,3,128,8,128,5.503371238708496,1.5634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,4,32,8,128,1.9206961393356323,2.25390625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,4,64,8,128,2.258723497390747,2.126953125,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,4,128,8,128,2.570795774459839,2.0634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,8,32,8,128,0.116376981139183,4.25390625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,8,64,8,128,0.1366570144891739,4.126953125,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.v_proj,8,128,8,128,0.1549541652202606,4.0634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5 +26,self_attn.k_proj,2,32,8,128,20.219913482666016,1.25390625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,2,64,8,128,24.16564559936523,1.126953125,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,2,128,8,128,27.7011833190918,1.0634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,3,32,8,128,8.676671981811523,1.75390625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,3,64,8,128,10.416951179504396,1.626953125,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,3,128,8,128,12.064611434936523,1.5634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,4,32,8,128,4.051470756530762,2.25390625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,4,64,8,128,4.862581253051758,2.126953125,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,4,128,8,128,5.62973690032959,2.0634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,8,32,8,128,0.2450635582208633,4.25390625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,8,64,8,128,0.2940589189529419,4.126953125,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,self_attn.k_proj,8,128,8,128,0.3393743634223938,4.0634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5 +26,mlp.gate_proj,2,32,8,128,42.71823120117188,17.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,2,64,8,128,50.7114143371582,15.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,2,128,8,128,58.47052001953125,14.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,3,32,8,128,18.284013748168945,24.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,3,64,8,128,21.78992080688477,22.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,3,128,8,128,25.228498458862305,21.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,4,32,8,128,8.531761169433594,31.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,4,64,8,128,10.16638469696045,29.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,4,128,8,128,11.780739784240724,28.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,8,32,8,128,0.5176286697387695,59.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,8,64,8,128,0.6171294450759888,57.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.gate_proj,8,128,8,128,0.71336430311203,56.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4 +26,mlp.up_proj,2,32,8,128,30.723989486694336,17.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,2,64,8,128,35.78413772583008,15.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,2,128,8,128,40.35905456542969,14.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,3,32,8,128,13.128897666931152,24.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,3,64,8,128,15.32298755645752,22.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,3,128,8,128,17.285682678222656,21.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,4,32,8,128,6.12621021270752,31.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,4,64,8,128,7.1481451988220215,29.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,4,128,8,128,8.072976112365723,28.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,8,32,8,128,0.3722635507583618,59.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,8,64,8,128,0.4336202144622803,57.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.up_proj,8,128,8,128,0.4876210391521454,56.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4 +26,mlp.down_proj,2,32,8,128,30.7542724609375,17.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,2,64,8,128,35.83990478515625,15.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,2,128,8,128,40.42461395263672,14.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,3,32,8,128,13.135388374328612,24.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,3,64,8,128,15.3322172164917,22.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,3,128,8,128,17.29229736328125,21.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,4,32,8,128,6.129529476165772,31.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,4,64,8,128,7.152489185333252,29.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,4,128,8,128,8.074078559875488,28.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,8,32,8,128,0.3724960386753082,59.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,8,64,8,128,0.4339540898799896,57.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5 +26,mlp.down_proj,8,128,8,128,0.4879628419876098,56.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5 +27,self_attn.q_proj,2,32,8,128,24.8642635345459,5.015625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,2,64,8,128,29.59623336791992,4.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,2,128,8,128,34.41663360595703,4.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,3,32,8,128,10.677159309387209,7.015625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,3,64,8,128,12.733939170837402,6.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,3,128,8,128,14.900238037109377,6.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,4,32,8,128,4.981285572052002,9.015625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,4,64,8,128,5.944271564483643,8.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,4,128,8,128,6.957520961761475,8.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,8,32,8,128,0.3015709519386291,17.015625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,8,64,8,128,0.3596012890338897,16.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.q_proj,8,128,8,128,0.4192833602428436,16.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5 +27,self_attn.v_proj,2,32,8,128,10.465078353881836,1.25390625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,2,64,8,128,12.289191246032717,1.126953125,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,2,128,8,128,13.937743186950684,1.0634765625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,3,32,8,128,4.516843318939209,1.75390625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,3,64,8,128,5.34511137008667,1.626953125,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,3,128,8,128,6.108231067657471,1.5634765625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,4,32,8,128,2.109103918075561,2.25390625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,4,64,8,128,2.497126817703247,2.126953125,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,4,128,8,128,2.859825134277344,2.0634765625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,8,32,8,128,0.1276155263185501,4.25390625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,8,64,8,128,0.1510124653577804,4.126953125,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.v_proj,8,128,8,128,0.1722709089517593,4.0634765625,4194304,4.277643019761399,NA,5.681679249391891e-5 +27,self_attn.k_proj,2,32,8,128,20.050647735595703,1.25390625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,2,64,8,128,23.887086868286133,1.126953125,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,2,128,8,128,27.287803649902344,1.0634765625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,3,32,8,128,8.62087345123291,1.75390625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,3,64,8,128,10.342302322387695,1.626953125,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,3,128,8,128,11.975882530212402,1.5634765625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,4,32,8,128,4.024779319763184,2.25390625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,4,64,8,128,4.828305244445801,2.126953125,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,4,128,8,128,5.590905666351318,2.0634765625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,8,32,8,128,0.2436296641826629,4.25390625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,8,64,8,128,0.2918936610221863,4.126953125,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,self_attn.k_proj,8,128,8,128,0.3370570540428161,4.0634765625,4194304,5.417876234887497,NA,5.681679249391891e-5 +27,mlp.gate_proj,2,32,8,128,43.32671737670898,17.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,2,64,8,128,51.58456802368164,15.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,2,128,8,128,59.66764450073242,14.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,3,32,8,128,18.5451774597168,24.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,3,64,8,128,22.17163276672364,22.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,3,128,8,128,25.74264335632324,21.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,4,32,8,128,8.65266227722168,31.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,4,64,8,128,10.34215259552002,29.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,4,128,8,128,12.019047737121582,28.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,8,32,8,128,0.5250110030174255,59.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,8,64,8,128,0.6281165480613708,57.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.gate_proj,8,128,8,128,0.727878987789154,56.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4 +27,mlp.up_proj,2,32,8,128,31.14664077758789,17.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,2,64,8,128,36.28898239135742,15.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,2,128,8,128,40.94233322143555,14.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,3,32,8,128,13.308853149414062,24.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,3,64,8,128,15.538352966308594,22.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,3,128,8,128,17.550668716430664,21.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,4,32,8,128,6.208175659179688,31.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,4,64,8,128,7.247622489929199,29.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,4,128,8,128,8.18896770477295,28.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,8,32,8,128,0.3773232996463775,59.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,8,64,8,128,0.439748078584671,57.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.up_proj,8,128,8,128,0.4947958886623382,56.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4 +27,mlp.down_proj,2,32,8,128,31.081233978271484,17.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,2,64,8,128,36.21077728271485,15.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,2,128,8,128,40.85276412963867,14.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,3,32,8,128,13.277050018310549,24.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,3,64,8,128,15.495223999023438,22.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,3,128,8,128,17.479244232177734,21.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,4,32,8,128,6.194422245025635,31.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,4,64,8,128,7.227962017059326,29.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,4,128,8,128,8.158595085144043,28.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,8,32,8,128,0.3764027953147888,59.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,8,64,8,128,0.4385375380516052,57.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4 +27,mlp.down_proj,8,128,8,128,0.4930305778980255,56.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4 +28,self_attn.q_proj,2,32,8,128,25.128877639770508,5.015625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,2,64,8,128,29.8079776763916,4.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,2,128,8,128,34.4616813659668,4.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,3,32,8,128,10.761868476867676,7.015625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,3,64,8,128,12.79036808013916,6.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,3,128,8,128,14.881439208984377,6.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,4,32,8,128,5.023092746734619,9.015625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,4,64,8,128,5.970032691955566,8.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,4,128,8,128,6.946596622467041,8.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,8,32,8,128,0.3040515482425689,17.015625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,8,64,8,128,0.3610921204090118,16.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.q_proj,8,128,8,128,0.4184679388999939,16.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5 +28,self_attn.v_proj,2,32,8,128,10.774648666381836,1.25390625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,2,64,8,128,12.643866539001465,1.126953125,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,2,128,8,128,14.361855506896973,1.0634765625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,3,32,8,128,4.6230010986328125,1.75390625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,3,64,8,128,5.453924655914307,1.626953125,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,3,128,8,128,6.219938278198242,1.5634765625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,4,32,8,128,2.158798694610596,2.25390625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,4,64,8,128,2.5453004837036133,2.126953125,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,4,128,8,128,2.905359029769897,2.0634765625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,8,32,8,128,0.1306397467851638,4.25390625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,8,64,8,128,0.153981015086174,4.126953125,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.v_proj,8,128,8,128,0.1752317398786544,4.0634765625,4194304,3.530892919231584,NA,6.343707354972139e-5 +28,self_attn.k_proj,2,32,8,128,18.45705986022949,1.25390625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,2,64,8,128,21.84394264221192,1.126953125,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,2,128,8,128,24.92411422729492,1.0634765625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,3,32,8,128,7.911220073699951,1.75390625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,3,64,8,128,9.379950523376465,1.626953125,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,3,128,8,128,10.762051582336426,1.5634765625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,4,32,8,128,3.691862344741821,2.25390625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,4,64,8,128,4.379453659057617,2.126953125,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,4,128,8,128,5.02256441116333,2.0634765625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,8,32,8,128,0.2233436703681945,4.25390625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,8,64,8,128,0.2647968828678131,4.126953125,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,self_attn.k_proj,8,128,8,128,0.3025440573692322,4.0634765625,4194304,4.394677532187053,NA,6.343707354972139e-5 +28,mlp.gate_proj,2,32,8,128,43.369659423828125,17.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,2,64,8,128,51.801700592041016,15.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,2,128,8,128,60.04250717163086,14.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,3,32,8,128,18.57156181335449,24.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,3,64,8,128,22.285978317260746,22.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,3,128,8,128,25.955814361572266,21.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,4,32,8,128,8.665281295776367,31.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,4,64,8,128,10.39654541015625,29.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,4,128,8,128,12.125450134277344,28.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,8,32,8,128,0.5258702635765076,59.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,8,64,8,128,0.6312415599822998,57.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.gate_proj,8,128,8,128,0.7340298295021057,56.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4 +28,mlp.up_proj,2,32,8,128,31.849252700805664,17.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,2,64,8,128,37.12252426147461,15.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,2,128,8,128,41.938255310058594,14.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,3,32,8,128,13.61239528656006,24.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,3,64,8,128,15.902270317077637,22.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,3,128,8,128,17.96623420715332,21.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,4,32,8,128,6.348927974700928,31.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,4,64,8,128,7.418264865875244,29.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,4,128,8,128,8.385152816772461,28.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,8,32,8,128,0.385782241821289,59.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,8,64,8,128,0.4498964548110962,57.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.up_proj,8,128,8,128,0.5066384673118591,56.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4 +28,mlp.down_proj,2,32,8,128,31.62061309814453,17.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,2,64,8,128,36.8379135131836,15.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,2,128,8,128,41.56753158569336,14.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,3,32,8,128,13.507688522338867,24.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,3,64,8,128,15.76890468597412,22.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,3,128,8,128,17.79062843322754,21.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,4,32,8,128,6.302467346191406,31.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,4,64,8,128,7.356359004974365,29.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,4,128,8,128,8.305274963378906,28.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,8,32,8,128,0.3828789293766022,59.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,8,64,8,128,0.4461589753627777,57.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4 +28,mlp.down_proj,8,128,8,128,0.5018621683120728,56.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4 +29,self_attn.q_proj,2,32,8,128,24.61381721496582,5.015625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,2,64,8,128,29.170211791992188,4.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,2,128,8,128,33.989593505859375,4.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,3,32,8,128,10.543471336364746,7.015625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,3,64,8,128,12.524491310119627,6.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,3,128,8,128,14.67572784423828,6.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,4,32,8,128,4.920046806335449,9.015625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,4,64,8,128,5.845945358276367,8.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,4,128,8,128,6.854568004608154,8.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,8,32,8,128,0.2980112135410309,17.015625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,8,64,8,128,0.3536786735057831,16.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.q_proj,8,128,8,128,0.4130273759365082,16.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5 +29,self_attn.v_proj,2,32,8,128,11.208635330200195,1.25390625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,2,64,8,128,13.142824172973633,1.126953125,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,2,128,8,128,14.882481575012209,1.0634765625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,3,32,8,128,4.8240885734558105,1.75390625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,3,64,8,128,5.69713830947876,1.626953125,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,3,128,8,128,6.506002426147461,1.5634765625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,4,32,8,128,2.253066301345825,2.25390625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,4,64,8,128,2.6614325046539307,2.126953125,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,4,128,8,128,3.0452027320861816,2.0634765625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,8,32,8,128,0.136377677321434,4.25390625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,8,64,8,128,0.1609858870506286,4.126953125,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.v_proj,8,128,8,128,0.1835995316505432,4.0634765625,4194304,3.864623471658886,NA,9.894864342641084e-5 +29,self_attn.k_proj,2,32,8,128,19.53223991394043,1.25390625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,2,64,8,128,23.18816947937012,1.126953125,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,2,128,8,128,26.52897071838379,1.0634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,3,32,8,128,8.37331771850586,1.75390625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,3,64,8,128,9.976702690124512,1.626953125,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,3,128,8,128,11.493720054626465,1.5634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,4,32,8,128,3.9071877002716064,2.25390625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,4,64,8,128,4.657023906707764,2.126953125,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,4,128,8,128,5.365540981292725,2.0634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,8,32,8,128,0.2366570085287094,4.25390625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,8,64,8,128,0.2816894948482513,4.126953125,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,self_attn.k_proj,8,128,8,128,0.323504775762558,4.0634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5 +29,mlp.gate_proj,2,32,8,128,43.68278884887695,17.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,2,64,8,128,52.37386703491211,15.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,2,128,8,128,60.91967010498047,14.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,3,32,8,128,18.71257972717285,24.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,3,64,8,128,22.55941390991211,22.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,3,128,8,128,26.394655227661133,21.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,4,32,8,128,8.729564666748047,31.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,4,64,8,128,10.525504112243652,29.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,4,128,8,128,12.324248313903809,28.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,8,32,8,128,0.5298333168029785,59.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,8,64,8,128,0.6389312744140625,57.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.gate_proj,8,128,8,128,0.7463177442550659,56.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4 +29,mlp.up_proj,2,32,8,128,32.798038482666016,17.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,2,64,8,128,38.2857666015625,15.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,2,128,8,128,43.2891960144043,14.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,3,32,8,128,14.019954681396484,24.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,3,64,8,128,16.389312744140625,22.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,3,128,8,128,18.53764533996582,21.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,4,32,8,128,6.540040493011475,31.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,4,64,8,128,7.649489879608154,29.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,4,128,8,128,8.652259826660156,28.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,8,32,8,128,0.3972397446632385,59.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,8,64,8,128,0.4637281000614166,57.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.up_proj,8,128,8,128,0.5229818820953369,56.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4 +29,mlp.down_proj,2,32,8,128,32.18458938598633,17.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,2,64,8,128,37.50214767456055,15.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,2,128,8,128,42.336612701416016,14.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,3,32,8,128,13.748483657836914,24.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,3,64,8,128,16.058168411254883,22.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,3,128,8,128,18.12397766113281,21.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,4,32,8,128,6.413782119750977,31.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,4,64,8,128,7.490348815917969,29.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,4,128,8,128,8.46006965637207,28.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,8,32,8,128,0.3897242546081543,59.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,8,64,8,128,0.4543188214302063,57.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4 +29,mlp.down_proj,8,128,8,128,0.5115240216255188,56.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4 +30,self_attn.q_proj,2,32,8,128,23.932138442993164,5.015625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,2,64,8,128,28.549251556396484,4.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,2,128,8,128,33.23263168334961,4.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,3,32,8,128,10.276080131530762,7.015625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,3,64,8,128,12.305699348449709,6.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,3,128,8,128,14.515734672546388,6.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,4,32,8,128,4.796483993530273,9.015625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,4,64,8,128,5.746490955352783,8.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,4,128,8,128,6.780810832977295,8.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,8,32,8,128,0.2903332710266113,17.015625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,8,64,8,128,0.3475486040115356,16.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.q_proj,8,128,8,128,0.4086917042732239,16.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4 +30,self_attn.v_proj,2,32,8,128,13.593299865722656,1.25390625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,2,64,8,128,15.973933219909668,1.126953125,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,2,128,8,128,18.097976684570312,1.0634765625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,3,32,8,128,5.866329669952393,1.75390625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,3,64,8,128,6.936065196990967,1.626953125,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,3,128,8,128,7.914339065551758,1.5634765625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,4,32,8,128,2.7383298873901367,2.25390625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,4,64,8,128,3.242396831512451,2.126953125,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,4,128,8,128,3.705921411514282,2.0634765625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,8,32,8,128,0.1659533679485321,4.25390625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,8,64,8,128,0.1961722820997238,4.126953125,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.v_proj,8,128,8,128,0.2233032137155532,4.0634765625,4194304,4.031878953855771,NA,2.0991287601646e-4 +30,self_attn.k_proj,2,32,8,128,17.307981491088867,1.25390625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,2,64,8,128,20.61306953430176,1.126953125,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,2,128,8,128,23.60615921020508,1.0634765625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,3,32,8,128,7.435863971710205,1.75390625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,3,64,8,128,8.913440704345703,1.626953125,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,3,128,8,128,10.305980682373049,1.5634765625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,4,32,8,128,3.468794345855713,2.25390625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,4,64,8,128,4.161543369293213,2.126953125,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,4,128,8,128,4.810798645019531,2.0634765625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,8,32,8,128,0.2100029587745666,4.25390625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,8,64,8,128,0.2517099976539612,4.126953125,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,self_attn.k_proj,8,128,8,128,0.2900921702384949,4.0634765625,4194304,6.817786100262908,NA,2.0991287601646e-4 +30,mlp.gate_proj,2,32,8,128,45.67716979980469,17.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,2,64,8,128,54.90836715698242,15.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,2,128,8,128,63.91456985473633,14.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,3,32,8,128,19.56645774841309,24.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,3,64,8,128,23.636524200439453,22.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,3,128,8,128,27.691688537597656,21.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,4,32,8,128,9.128087043762209,31.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,4,64,8,128,11.03372859954834,29.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,4,128,8,128,12.926897048950195,28.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,8,32,8,128,0.5540670156478882,59.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,8,64,8,128,0.670048713684082,57.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.gate_proj,8,128,8,128,0.7827392220497131,56.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4 +30,mlp.up_proj,2,32,8,128,33.815696716308594,17.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,2,64,8,128,39.57140350341797,15.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,2,128,8,128,44.87066650390625,14.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,3,32,8,128,14.462481498718262,24.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,3,64,8,128,16.943021774291992,22.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,3,128,8,128,19.22920227050781,21.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,4,32,8,128,6.745366096496582,31.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,4,64,8,128,7.909011363983154,29.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,4,128,8,128,8.969450950622559,28.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,8,32,8,128,0.4095952808856964,59.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,8,64,8,128,0.4794001579284668,57.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.up_proj,8,128,8,128,0.5422195196151733,56.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4 +30,mlp.down_proj,2,32,8,128,32.36918640136719,17.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,2,64,8,128,37.74430847167969,15.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,2,128,8,128,42.64178085327149,14.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,3,32,8,128,13.840448379516602,24.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,3,64,8,128,16.18268394470215,22.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,3,128,8,128,18.312593460083008,21.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,4,32,8,128,6.456140041351318,31.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,4,64,8,128,7.550130367279053,29.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,4,128,8,128,8.548364639282227,28.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,8,32,8,128,0.3921487033367157,59.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,8,64,8,128,0.457781434059143,57.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4 +30,mlp.down_proj,8,128,8,128,0.5167698860168457,56.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4 +31,self_attn.q_proj,2,32,8,128,26.472570419311523,5.015625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,2,64,8,128,31.61062431335449,4.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,2,128,8,128,36.75197982788086,4.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,3,32,8,128,11.368563652038574,7.015625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,3,64,8,128,13.6343412399292,6.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,3,128,8,128,16.04696273803711,6.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,4,32,8,128,5.305564403533936,9.015625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,4,64,8,128,6.367289066314697,8.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,4,128,8,128,7.496603488922119,8.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,8,32,8,128,0.3211942613124847,17.015625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,8,64,8,128,0.3851563930511474,16.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.q_proj,8,128,8,128,0.4517876803874969,16.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4 +31,self_attn.v_proj,2,32,8,128,12.27296543121338,1.25390625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,2,64,8,128,14.527003288269045,1.126953125,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,2,128,8,128,16.583271026611328,1.0634765625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,3,32,8,128,5.2820329666137695,1.75390625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,3,64,8,128,6.289672374725342,1.626953125,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,3,128,8,128,7.240473747253418,1.5634765625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,4,32,8,128,2.46514892578125,2.25390625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,4,64,8,128,2.937155246734619,2.126953125,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,4,128,8,128,3.386962413787842,2.0634765625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,8,32,8,128,0.1492158472537994,4.25390625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,8,64,8,128,0.1775472611188888,4.126953125,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.v_proj,8,128,8,128,0.2040145844221115,4.0634765625,4194304,4.308375838130611,NA,2.6171901845373e-4 +31,self_attn.k_proj,2,32,8,128,18.52001571655273,1.25390625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,2,64,8,128,21.92378807067871,1.126953125,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,2,128,8,128,25.032684326171875,1.0634765625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,3,32,8,128,7.956022262573242,1.75390625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,3,64,8,128,9.463165283203123,1.626953125,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,3,128,8,128,10.881024360656738,1.5634765625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,4,32,8,128,3.711784601211548,2.25390625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,4,64,8,128,4.416527271270752,2.126953125,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,4,128,8,128,5.085290431976318,2.0634765625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,8,32,8,128,0.2247177511453628,4.25390625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,8,64,8,128,0.2671961486339569,4.126953125,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,self_attn.k_proj,8,128,8,128,0.3062677085399627,4.0634765625,4194304,5.780387350146194,NA,2.6171901845373e-4 +31,mlp.gate_proj,2,32,8,128,51.96072769165039,17.5546875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,2,64,8,128,62.50411605834961,15.77734375,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,2,128,8,128,72.632171630859375,14.888671875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,3,32,8,128,22.3032455444336,24.5546875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,3,64,8,128,26.91626167297364,22.77734375,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,3,128,8,128,31.600067138671875,21.888671875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,4,32,8,128,10.4017972946167,31.5546875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,4,64,8,128,12.561721801757812,29.77734375,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,4,128,8,128,14.746207237243652,28.888671875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,8,32,8,128,0.6317282915115356,59.5546875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,8,64,8,128,0.7629415392875671,57.77734375,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.gate_proj,8,128,8,128,0.8918254375457764,56.888671875,58720256,4.928933565096192,NA,0.00210070470348 +31,mlp.up_proj,2,32,8,128,39.715599060058594,17.5546875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,2,64,8,128,46.80196762084961,15.77734375,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,2,128,8,128,53.45743942260742,14.888671875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,3,32,8,128,17.01172637939453,24.5546875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,3,64,8,128,20.10135078430176,22.77734375,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,3,128,8,128,23.08347511291504,21.888671875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,4,32,8,128,7.942127704620361,31.5546875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,4,64,8,128,9.382147789001465,29.77734375,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,4,128,8,128,10.781952857971191,28.888671875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,8,32,8,128,0.4815772175788879,59.5546875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,8,64,8,128,0.5693203806877136,57.77734375,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.up_proj,8,128,8,128,0.6523986458778381,56.888671875,58720256,4.221088632679399,NA,0.00210070470348 +31,mlp.down_proj,2,32,8,128,33.46739196777344,17.5546875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,2,64,8,128,39.11680221557617,15.77734375,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,2,128,8,128,44.33309173583984,14.888671875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,3,32,8,128,14.33449649810791,24.5546875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,3,64,8,128,16.806303024291992,22.77734375,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,3,128,8,128,19.115291595458984,21.888671875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,4,32,8,128,6.690278053283691,31.5546875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,4,64,8,128,7.854846954345703,29.77734375,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,4,128,8,128,8.939277648925781,28.888671875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,8,32,8,128,0.4062495231628418,59.5546875,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,8,64,8,128,0.4761776328086853,57.77734375,58720256,4.448183420812677,NA,0.10236936062574388 +31,mlp.down_proj,8,128,8,128,0.5405846834182739,56.888671875,58720256,4.448183420812677,NA,0.10236936062574388 +0,self_attn.o_proj,2,32,8,128,11.62565803527832,5.015625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,2,64,8,128,13.760478973388672,4.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,2,128,8,128,15.775431632995604,4.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,3,32,8,128,5.015730857849121,7.015625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,3,64,8,128,5.997352600097656,6.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,3,128,8,128,6.988894939422607,6.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,4,32,8,128,2.347808599472046,9.015625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,4,64,8,128,2.814517021179199,8.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,4,128,8,128,3.2919631004333496,8.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,8,32,8,128,0.1421578079462051,17.015625,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,8,64,8,128,0.1704837083816528,16.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7 +0,self_attn.o_proj,8,128,8,128,0.1990951299667358,16.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7 +1,self_attn.o_proj,2,32,8,128,13.320256233215332,5.015625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,2,64,8,128,15.85250186920166,4.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,2,128,8,128,18.25271224975586,4.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,3,32,8,128,5.814366817474365,7.015625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,3,64,8,128,7.047060966491699,6.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,3,128,8,128,8.325507164001465,6.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,4,32,8,128,2.7238576412200928,9.015625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,4,64,8,128,3.32004714012146,8.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,4,128,8,128,3.962393045425415,8.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,8,32,8,128,0.1648536026477813,17.015625,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,8,64,8,128,0.2008322775363922,16.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7 +1,self_attn.o_proj,8,128,8,128,0.2392399460077285,16.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7 +2,self_attn.o_proj,2,32,8,128,12.128545761108398,5.015625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,2,64,8,128,14.357359886169434,4.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,2,128,8,128,16.48350715637207,4.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,3,32,8,128,5.209362030029297,7.015625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,3,64,8,128,6.204757690429688,6.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,3,128,8,128,7.207065582275391,6.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,4,32,8,128,2.432673931121826,9.015625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,4,64,8,128,2.898388624191284,8.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,4,128,8,128,3.370327472686768,8.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,8,32,8,128,0.1472485661506652,17.015625,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,8,64,8,128,0.1753260344266891,16.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7 +2,self_attn.o_proj,8,128,8,128,0.2031807005405426,16.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7 +3,self_attn.o_proj,2,32,8,128,13.677197456359863,5.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,2,64,8,128,16.046985626220703,4.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,2,128,8,128,18.24164581298828,4.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,3,32,8,128,5.862359046936035,7.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,3,64,8,128,6.901886463165283,6.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,3,128,8,128,7.893731594085693,6.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,4,32,8,128,2.7355918884277344,9.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,4,64,8,128,3.2214348316192627,8.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,4,128,8,128,3.687077522277832,8.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,8,32,8,128,0.1656187325716018,17.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,8,64,8,128,0.1948099285364151,16.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6 +3,self_attn.o_proj,8,128,8,128,0.2220923602581024,16.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6 +4,self_attn.o_proj,2,32,8,128,14.107036590576172,5.015625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,2,64,8,128,16.614194869995117,4.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,2,128,8,128,18.963024139404297,4.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,3,32,8,128,6.076632499694824,7.015625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,3,64,8,128,7.223157405853271,6.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,3,128,8,128,8.366185188293457,6.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,4,32,8,128,2.836582660675049,9.015625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,4,64,8,128,3.375967264175415,8.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,4,128,8,128,3.922086238861084,8.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,8,32,8,128,0.1717239171266555,17.015625,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,8,64,8,128,0.2041394561529159,16.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6 +4,self_attn.o_proj,8,128,8,128,0.236321359872818,16.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6 +5,self_attn.o_proj,2,32,8,128,12.935108184814451,5.015625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,2,64,8,128,15.15129852294922,4.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,2,128,8,128,17.19681167602539,4.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,3,32,8,128,5.535101413726807,7.015625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,3,64,8,128,6.496115684509277,6.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,3,128,8,128,7.393651008605957,6.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,4,32,8,128,2.583302974700928,9.015625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,4,64,8,128,3.0311734676361084,8.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,4,128,8,128,3.4520888328552246,8.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,8,32,8,128,0.1563338637351989,17.015625,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,8,64,8,128,0.1833816468715667,16.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6 +5,self_attn.o_proj,8,128,8,128,0.2080183327198028,16.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6 +6,self_attn.o_proj,2,32,8,128,13.487937927246094,5.015625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,2,64,8,128,15.787430763244627,4.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,2,128,8,128,17.891263961791992,4.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,3,32,8,128,5.77061653137207,7.015625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,3,64,8,128,6.7621049880981445,6.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,3,128,8,128,7.691638946533203,6.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,4,32,8,128,2.694453716278076,9.015625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,4,64,8,128,3.1594767570495605,8.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,4,128,8,128,3.593095302581787,8.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,8,32,8,128,0.1630697697401046,17.015625,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,8,64,8,128,0.1910372674465179,16.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6 +6,self_attn.o_proj,8,128,8,128,0.2165930122137069,16.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6 +7,self_attn.o_proj,2,32,8,128,13.807162284851074,5.015625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,2,64,8,128,16.211566925048828,4.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,2,128,8,128,18.44451904296875,4.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,3,32,8,128,5.908563613891602,7.015625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,3,64,8,128,6.952080726623535,6.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,3,128,8,128,7.942596912384033,6.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,4,32,8,128,2.758080244064331,9.015625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,4,64,8,128,3.245979070663452,8.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,4,128,8,128,3.710340023040772,8.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,8,32,8,128,0.1669350117444992,17.015625,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,8,64,8,128,0.1963274776935577,16.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6 +7,self_attn.o_proj,8,128,8,128,0.2235273271799087,16.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6 +8,self_attn.o_proj,2,32,8,128,14.093366622924805,5.015625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,2,64,8,128,16.5416202545166,4.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,2,128,8,128,18.809246063232425,4.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,3,32,8,128,6.036656856536865,7.015625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,3,64,8,128,7.104803085327148,6.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,3,128,8,128,8.118633270263672,6.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,4,32,8,128,2.819232702255249,9.015625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,4,64,8,128,3.32045841217041,8.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,4,128,8,128,3.797840833663941,8.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,8,32,8,128,0.1706764101982116,17.015625,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,8,64,8,128,0.2007231414318084,16.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5 +8,self_attn.o_proj,8,128,8,128,0.2288123369216919,16.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5 +9,self_attn.o_proj,2,32,8,128,15.094277381896973,5.015625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,2,64,8,128,17.70538902282715,4.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,2,128,8,128,20.105255126953125,4.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,3,32,8,128,6.4606523513793945,7.015625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,3,64,8,128,7.59069013595581,6.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,3,128,8,128,8.645902633666992,6.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,4,32,8,128,3.014559030532837,9.015625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,4,64,8,128,3.5421299934387207,8.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,4,128,8,128,4.038563251495361,8.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,8,32,8,128,0.1824861317873001,17.015625,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,8,64,8,128,0.2141808569431305,16.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5 +9,self_attn.o_proj,8,128,8,128,0.2432269304990768,16.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5 +10,self_attn.o_proj,2,32,8,128,13.89360523223877,5.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,2,64,8,128,16.27947998046875,4.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,2,128,8,128,18.45605659484864,4.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,3,32,8,128,5.949365139007568,7.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,3,64,8,128,6.983646392822266,6.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,3,128,8,128,7.942622184753418,6.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,4,32,8,128,2.7777304649353027,9.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,4,64,8,128,3.262352705001831,8.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,4,128,8,128,3.7130494117736816,8.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,8,32,8,128,0.1681594401597976,17.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,8,64,8,128,0.1972410529851913,16.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5 +10,self_attn.o_proj,8,128,8,128,0.2237548381090164,16.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5 +11,self_attn.o_proj,2,32,8,128,14.244765281677246,5.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,2,64,8,128,16.716487884521484,4.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,2,128,8,128,18.98614883422852,4.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,3,32,8,128,6.097167015075684,7.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,3,64,8,128,7.168283939361572,6.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,3,128,8,128,8.164941787719727,6.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,4,32,8,128,2.84542179107666,9.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,4,64,8,128,3.345279693603516,8.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,4,128,8,128,3.812110424041748,8.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,8,32,8,128,0.1722400337457656,17.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,8,64,8,128,0.2022191286087036,16.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5 +11,self_attn.o_proj,8,128,8,128,0.2296272665262222,16.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5 +12,self_attn.o_proj,2,32,8,128,15.051139831542969,5.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,2,64,8,128,17.665584564208984,4.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,2,128,8,128,20.06760025024414,4.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,3,32,8,128,6.443262100219727,7.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,3,64,8,128,7.575900077819824,6.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,3,128,8,128,8.629935264587402,6.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,4,32,8,128,3.008702754974365,9.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,4,64,8,128,3.538581371307373,8.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,4,128,8,128,4.036471843719482,8.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,8,32,8,128,0.1820550709962844,17.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,8,64,8,128,0.2139701545238495,16.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5 +12,self_attn.o_proj,8,128,8,128,0.243266299366951,16.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5 +13,self_attn.o_proj,2,32,8,128,14.635335922241213,5.015625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,2,64,8,128,17.123037338256836,4.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,2,128,8,128,19.390033721923828,4.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,3,32,8,128,6.264823913574219,7.015625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,3,64,8,128,7.340836524963379,6.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,3,128,8,128,8.326494216918945,6.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,4,32,8,128,2.9257869720458984,9.015625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,4,64,8,128,3.4308698177337646,8.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,4,128,8,128,3.8957786560058594,8.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,8,32,8,128,0.1771737784147262,17.015625,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,8,64,8,128,0.2075408399105072,16.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5 +13,self_attn.o_proj,8,128,8,128,0.2349102646112442,16.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5 +14,self_attn.o_proj,2,32,8,128,14.63621425628662,5.015625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,2,64,8,128,17.19491958618164,4.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,2,128,8,128,19.55340003967285,4.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,3,32,8,128,6.268218994140625,7.015625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,3,64,8,128,7.380595684051514,6.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,3,128,8,128,8.436356544494629,6.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,4,32,8,128,2.9249253273010254,9.015625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,4,64,8,128,3.444727659225464,8.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,4,128,8,128,3.938780069351196,8.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,8,32,8,128,0.1770697385072708,17.015625,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,8,64,8,128,0.208248421549797,16.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5 +14,self_attn.o_proj,8,128,8,128,0.2372278124094009,16.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5 +15,self_attn.o_proj,2,32,8,128,15.051722526550291,5.015625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,2,64,8,128,17.624523162841797,4.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,2,128,8,128,20.003849029541016,4.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,3,32,8,128,6.447397708892822,7.015625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,3,64,8,128,7.569221019744873,6.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,3,128,8,128,8.61890983581543,6.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,4,32,8,128,3.008628368377685,9.015625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,4,64,8,128,3.5341615676879883,8.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,4,128,8,128,4.029312610626221,8.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,8,32,8,128,0.1822375059127807,17.015625,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,8,64,8,128,0.2137708812952041,16.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5 +15,self_attn.o_proj,8,128,8,128,0.2428483068943023,16.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5 +16,self_attn.o_proj,2,32,8,128,14.671588897705078,5.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,2,64,8,128,17.20387840270996,4.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,2,128,8,128,19.533960342407227,4.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,3,32,8,128,6.29518985748291,7.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,3,64,8,128,7.406421661376953,6.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,3,128,8,128,8.455559730529785,6.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,4,32,8,128,2.938764333724976,9.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,4,64,8,128,3.4616949558258057,8.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,4,128,8,128,3.9625065326690674,8.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,8,32,8,128,0.1780003607273101,17.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,8,64,8,128,0.2093121707439422,16.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5 +16,self_attn.o_proj,8,128,8,128,0.2387027740478515,16.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5 +17,self_attn.o_proj,2,32,8,128,15.159659385681152,5.015625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,2,64,8,128,17.726320266723633,4.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,2,128,8,128,20.09735679626465,4.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,3,32,8,128,6.492598056793213,7.015625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,3,64,8,128,7.610992431640625,6.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,3,128,8,128,8.64831256866455,6.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,4,32,8,128,3.031862258911133,9.015625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,4,64,8,128,3.556722640991211,8.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,4,128,8,128,4.0480780601501465,8.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,8,32,8,128,0.1835595369338989,17.015625,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,8,64,8,128,0.2150303274393081,16.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5 +17,self_attn.o_proj,8,128,8,128,0.2438358813524246,16.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5 +18,self_attn.o_proj,2,32,8,128,14.78054141998291,5.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,2,64,8,128,17.31252098083496,4.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,2,128,8,128,19.65506362915039,4.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,3,32,8,128,6.328427791595459,7.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,3,64,8,128,7.425669193267822,6.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,3,128,8,128,8.456360816955566,6.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,4,32,8,128,2.95471715927124,9.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,4,64,8,128,3.467207908630371,8.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,4,128,8,128,3.9520959854125977,8.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,8,32,8,128,0.1788134425878524,17.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,8,64,8,128,0.2096264213323593,16.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5 +18,self_attn.o_proj,8,128,8,128,0.2381150126457214,16.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5 +19,self_attn.o_proj,2,32,8,128,14.957073211669922,5.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,2,64,8,128,17.497554779052734,4.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,2,128,8,128,19.854581832885746,4.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,3,32,8,128,6.4000678062438965,7.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,3,64,8,128,7.502575874328613,6.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,3,128,8,128,8.5342864990234375,6.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,4,32,8,128,2.9878108501434326,9.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,4,64,8,128,3.504570245742798,8.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,4,128,8,128,3.99132251739502,8.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,8,32,8,128,0.1809410005807876,17.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,8,64,8,128,0.2119685560464859,16.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5 +19,self_attn.o_proj,8,128,8,128,0.2405126243829727,16.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5 +20,self_attn.o_proj,2,32,8,128,14.878053665161133,5.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,2,64,8,128,17.47359275817871,4.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,2,128,8,128,19.893165588378903,4.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,3,32,8,128,6.376826763153076,7.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,3,64,8,128,7.518867492675781,6.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,3,128,8,128,8.61892318725586,6.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,4,32,8,128,2.976961612701416,9.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,4,64,8,128,3.511537551879883,8.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,4,128,8,128,4.029510498046875,8.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,8,32,8,128,0.1802045851945877,17.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,8,64,8,128,0.2123738974332809,16.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +20,self_attn.o_proj,8,128,8,128,0.2428355813026428,16.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5 +21,self_attn.o_proj,2,32,8,128,15.174367904663086,5.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,2,64,8,128,17.784578323364258,4.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,2,128,8,128,20.190414428710938,4.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,3,32,8,128,6.5373382568359375,7.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,3,64,8,128,7.720632553100586,6.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,3,128,8,128,8.863565444946289,6.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,4,32,8,128,3.054070234298706,9.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,4,64,8,128,3.612856864929199,8.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,4,128,8,128,4.1695098876953125,8.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,8,32,8,128,0.1848398596048355,17.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,8,64,8,128,0.2183932512998581,16.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5 +21,self_attn.o_proj,8,128,8,128,0.2512237429618835,16.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5 +22,self_attn.o_proj,2,32,8,128,15.403858184814451,5.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,2,64,8,128,17.991758346557617,4.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,2,128,8,128,20.357152938842773,4.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,3,32,8,128,6.59965705871582,7.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,3,64,8,128,7.729466915130615,6.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,3,128,8,128,8.768677711486816,6.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,4,32,8,128,3.078189849853516,9.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,4,64,8,128,3.607653617858887,8.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,4,128,8,128,4.097971439361572,8.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,8,32,8,128,0.1864184588193893,17.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,8,64,8,128,0.2181044220924377,16.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5 +22,self_attn.o_proj,8,128,8,128,0.2469657659530639,16.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5 +23,self_attn.o_proj,2,32,8,128,15.661473274230955,5.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,2,64,8,128,18.266996383666992,4.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,2,128,8,128,20.64474296569824,4.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,3,32,8,128,6.695975303649902,7.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,3,64,8,128,7.823517322540283,6.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,3,128,8,128,8.849101066589355,6.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,4,32,8,128,3.1255362033843994,9.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,4,64,8,128,3.651583194732666,8.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,4,128,8,128,4.133560180664063,8.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,8,32,8,128,0.1891371309757232,17.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,8,64,8,128,0.2207475155591964,16.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +23,self_attn.o_proj,8,128,8,128,0.2490492910146713,16.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5 +24,self_attn.o_proj,2,32,8,128,16.394994735717773,5.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,2,64,8,128,19.14152336120605,4.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,2,128,8,128,21.64337539672852,4.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,3,32,8,128,7.011787414550781,7.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,3,64,8,128,8.197203636169434,6.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,3,128,8,128,9.272076606750488,6.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,4,32,8,128,3.272226572036743,9.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,4,64,8,128,3.825253486633301,8.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,4,128,8,128,4.328807830810547,8.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,8,32,8,128,0.1980541795492172,17.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,8,64,8,128,0.2311242371797561,16.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +24,self_attn.o_proj,8,128,8,128,0.2609146535396576,16.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5 +25,self_attn.o_proj,2,32,8,128,16.53202247619629,5.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,2,64,8,128,19.339879989624023,4.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,2,128,8,128,21.89888572692871,4.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,3,32,8,128,7.078935623168945,7.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,3,64,8,128,8.309752464294434,6.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,3,128,8,128,9.464712142944336,6.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,4,32,8,128,3.3040571212768555,9.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,4,64,8,128,3.879425287246704,8.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,4,128,8,128,4.418932914733887,8.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,8,32,8,128,0.1999684274196624,17.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,8,64,8,128,0.2344671934843063,16.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5 +25,self_attn.o_proj,8,128,8,128,0.2664281129837036,16.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5 +26,self_attn.o_proj,2,32,8,128,16.810836791992188,5.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,2,64,8,128,19.624425888061523,4.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,2,128,8,128,22.199460983276367,4.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,3,32,8,128,7.196712493896484,7.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,3,64,8,128,8.41883659362793,6.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,3,128,8,128,9.55011749267578,6.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,4,32,8,128,3.3594369888305664,9.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,4,64,8,128,3.9312922954559326,8.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,4,128,8,128,4.46165943145752,8.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,8,32,8,128,0.2032840251922607,17.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,8,64,8,128,0.237574964761734,16.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5 +26,self_attn.o_proj,8,128,8,128,0.2688663601875305,16.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5 +27,self_attn.o_proj,2,32,8,128,17.266559600830078,5.015625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,2,64,8,128,20.1784782409668,4.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,2,128,8,128,22.853853225708008,4.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,3,32,8,128,7.411194324493408,7.015625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,3,64,8,128,8.691489219665527,6.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,3,128,8,128,9.894498825073242,6.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,4,32,8,128,3.457504749298096,9.015625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,4,64,8,128,4.058493137359619,8.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,4,128,8,128,4.6237874031066895,8.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,8,32,8,128,0.2092214375734329,17.015625,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,8,64,8,128,0.2453083097934723,16.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5 +27,self_attn.o_proj,8,128,8,128,0.2786425650119781,16.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5 +28,self_attn.o_proj,2,32,8,128,17.700225830078125,5.015625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,2,64,8,128,20.673864364624023,4.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,2,128,8,128,23.37679672241211,4.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,3,32,8,128,7.58986759185791,7.015625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,3,64,8,128,8.88255786895752,6.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,3,128,8,128,10.08473014831543,6.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,4,32,8,128,3.5435383319854736,9.015625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,4,64,8,128,4.151835441589356,8.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,4,128,8,128,4.720759391784668,8.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,8,32,8,128,0.2144432812929153,17.015625,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,8,64,8,128,0.250990092754364,16.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5 +28,self_attn.o_proj,8,128,8,128,0.2844915986061096,16.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5 +29,self_attn.o_proj,2,32,8,128,18.33413887023925,5.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,2,64,8,128,21.427490234375,4.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,2,128,8,128,24.24564552307129,4.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,3,32,8,128,7.850170612335205,7.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,3,64,8,128,9.1857271194458,6.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,3,128,8,128,10.42802619934082,6.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,4,32,8,128,3.662515878677368,9.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,4,64,8,128,4.287515640258789,8.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,4,128,8,128,4.868528366088867,8.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,8,32,8,128,0.2216504961252212,17.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,8,64,8,128,0.2591716349124908,16.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5 +29,self_attn.o_proj,8,128,8,128,0.2935357689857483,16.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5 +30,self_attn.o_proj,2,32,8,128,19.64671897888184,5.015625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,2,64,8,128,22.98756217956543,4.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,2,128,8,128,26.05941200256348,4.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,3,32,8,128,8.41417121887207,7.015625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,3,64,8,128,9.869516372680664,6.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,3,128,8,128,11.22982120513916,6.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,4,32,8,128,3.9267284870147705,9.015625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,4,64,8,128,4.603414535522461,8.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,4,128,8,128,5.240401744842529,8.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,8,32,8,128,0.2375096082687378,17.015625,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,8,64,8,128,0.2784024477005005,16.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4 +30,self_attn.o_proj,8,128,8,128,0.3158771097660064,16.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4 +31,self_attn.o_proj,2,32,8,128,19.6539363861084,5.015625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,2,64,8,128,23.17051887512207,4.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,2,128,8,128,26.426538467407227,4.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,3,32,8,128,8.46374797821045,7.015625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,3,64,8,128,10.066619873046877,6.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,3,128,8,128,11.667180061340332,6.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,4,32,8,128,3.952760696411133,9.015625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,4,64,8,128,4.707706928253174,8.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,4,128,8,128,5.475996017456055,8.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,8,32,8,128,0.2390454709529876,17.015625,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,8,64,8,128,0.2846476435661316,16.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4 +31,self_attn.o_proj,8,128,8,128,0.3298292756080627,16.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-H-14-laion2B-s32B-b79K.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-H-14-laion2B-s32B-b79K.csv new file mode 100644 index 0000000000000000000000000000000000000000..b008d12f3b16a5b462bed1cc1c22caa840349c30 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-H-14-laion2B-s32B-b79K.csv @@ -0,0 +1,113 @@ +module,layer,kurtosis +vision.mlp.c_fc,0,40.90237464363011 +vision.mlp.c_fc,1,8.623074922686055 +vision.mlp.c_fc,2,9.97244388122774 +vision.mlp.c_fc,3,8.84226648033505 +vision.mlp.c_fc,4,6.679282327315599 +vision.mlp.c_fc,5,5.079880661110845 +vision.mlp.c_fc,6,4.880932310379808 +vision.mlp.c_fc,7,4.627381829171479 +vision.mlp.c_fc,8,3.9064125305929482 +vision.mlp.c_fc,9,3.7876465036865037 +vision.mlp.c_fc,10,3.7124408015905463 +vision.mlp.c_fc,11,3.5936330266701173 +vision.mlp.c_fc,12,3.532952137305315 +vision.mlp.c_fc,13,3.3851543523342795 +vision.mlp.c_fc,14,3.2873419131054304 +vision.mlp.c_fc,15,3.21782809316773 +vision.mlp.c_fc,16,3.193747931233614 +vision.mlp.c_fc,17,3.1630314377596305 +vision.mlp.c_fc,18,3.111462971801169 +vision.mlp.c_fc,19,3.105595856069085 +vision.mlp.c_fc,20,3.125886942463749 +vision.mlp.c_fc,21,3.112450361245664 +vision.mlp.c_fc,22,3.079523772985348 +vision.mlp.c_fc,23,3.0821569476632615 +vision.mlp.c_fc,24,3.060682454627166 +vision.mlp.c_fc,25,3.0582903694192485 +vision.mlp.c_fc,26,3.066707948588762 +vision.mlp.c_fc,27,3.098921361428591 +vision.mlp.c_fc,28,3.115322580545271 +vision.mlp.c_fc,29,3.1341133294881063 +vision.mlp.c_fc,30,3.0777676137143355 +vision.mlp.c_fc,31,3.1157306945381347 +vision.mlp.c_proj,0,42.596327918914184 +vision.mlp.c_proj,1,5.700569954788923 +vision.mlp.c_proj,2,13.632761342438107 +vision.mlp.c_proj,3,16.268235060436744 +vision.mlp.c_proj,4,6.3570446531375 +vision.mlp.c_proj,5,4.809992151850216 +vision.mlp.c_proj,6,5.053291310517847 +vision.mlp.c_proj,7,7.712895831700571 +vision.mlp.c_proj,8,3.6717534300900505 +vision.mlp.c_proj,9,3.8858509415381075 +vision.mlp.c_proj,10,4.615226983169325 +vision.mlp.c_proj,11,3.9462090594902732 +vision.mlp.c_proj,12,4.290422057573261 +vision.mlp.c_proj,13,4.200667942302044 +vision.mlp.c_proj,14,3.5894479766105856 +vision.mlp.c_proj,15,3.295239725306919 +vision.mlp.c_proj,16,3.258246320341865 +vision.mlp.c_proj,17,3.287998430108839 +vision.mlp.c_proj,18,3.106840184971173 +vision.mlp.c_proj,19,3.1621437963814087 +vision.mlp.c_proj,20,3.168454537760785 +vision.mlp.c_proj,21,3.213436382602411 +vision.mlp.c_proj,22,3.173853870505496 +vision.mlp.c_proj,23,3.178954522388751 +vision.mlp.c_proj,24,3.110805792772743 +vision.mlp.c_proj,25,3.114617319926587 +vision.mlp.c_proj,26,3.1252555548204226 +vision.mlp.c_proj,27,3.2274298263451326 +vision.mlp.c_proj,28,3.4402281120641667 +vision.mlp.c_proj,29,3.4039192043631497 +vision.mlp.c_proj,30,3.235419305882699 +vision.mlp.c_proj,31,3.2103127283743196 +text.mlp.c_fc,0,3.130876328110002 +text.mlp.c_fc,1,3.522946431618204 +text.mlp.c_fc,2,3.0665727137671914 +text.mlp.c_fc,3,3.07939282139385 +text.mlp.c_fc,4,3.073907675889281 +text.mlp.c_fc,5,3.0760243182327653 +text.mlp.c_fc,6,3.074341006878764 +text.mlp.c_fc,7,3.070467308878268 +text.mlp.c_fc,8,3.071675866527997 +text.mlp.c_fc,9,3.0676553119556074 +text.mlp.c_fc,10,3.0588525342169723 +text.mlp.c_fc,11,3.050926459548459 +text.mlp.c_fc,12,3.0518055370811794 +text.mlp.c_fc,13,3.0420668373340813 +text.mlp.c_fc,14,3.0318151874070547 +text.mlp.c_fc,15,3.039579656917142 +text.mlp.c_fc,16,3.030832331612468 +text.mlp.c_fc,17,3.0464599870244147 +text.mlp.c_fc,18,3.0314247283166527 +text.mlp.c_fc,19,3.0366579505365423 +text.mlp.c_fc,20,3.035431457627752 +text.mlp.c_fc,21,3.035719259091557 +text.mlp.c_fc,22,3.0609612140958817 +text.mlp.c_fc,23,3.0913598384430627 +text.mlp.c_proj,0,7.6155345275418025 +text.mlp.c_proj,1,10.138321188679202 +text.mlp.c_proj,2,3.0448966809401976 +text.mlp.c_proj,3,3.111820890605678 +text.mlp.c_proj,4,3.0514440139756176 +text.mlp.c_proj,5,3.7893358959560755 +text.mlp.c_proj,6,3.0707829989536948 +text.mlp.c_proj,7,3.065090920136809 +text.mlp.c_proj,8,3.0726887739189013 +text.mlp.c_proj,9,3.3939847588778926 +text.mlp.c_proj,10,3.1983749033206665 +text.mlp.c_proj,11,3.1750157970807344 +text.mlp.c_proj,12,3.0569032145916935 +text.mlp.c_proj,13,3.352986917766462 +text.mlp.c_proj,14,3.040870872900523 +text.mlp.c_proj,15,3.084122174777231 +text.mlp.c_proj,16,3.083968401193075 +text.mlp.c_proj,17,3.057870708712376 +text.mlp.c_proj,18,3.154006715549809 +text.mlp.c_proj,19,3.099173814869719 +text.mlp.c_proj,20,3.1643892434809398 +text.mlp.c_proj,21,3.1143910580456344 +text.mlp.c_proj,22,3.2050609846862375 +text.mlp.c_proj,23,3.1967177502646615 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv new file mode 100644 index 0000000000000000000000000000000000000000..0c6f7f4fb28f74c490ed94826434e6150eeaab8a --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv @@ -0,0 +1,73 @@ +module,layer,kurtosis +vision.mlp.c_fc,0,82.60432030937712 +vision.mlp.c_fc,1,12.176342485174775 +vision.mlp.c_fc,2,9.386186464371631 +vision.mlp.c_fc,3,6.0384986521099115 +vision.mlp.c_fc,4,5.121874436272312 +vision.mlp.c_fc,5,4.184969377709542 +vision.mlp.c_fc,6,4.256117717855523 +vision.mlp.c_fc,7,3.953278547908379 +vision.mlp.c_fc,8,3.923302040046366 +vision.mlp.c_fc,9,3.640527020435186 +vision.mlp.c_fc,10,3.8766116638521972 +vision.mlp.c_fc,11,3.364467358103123 +vision.mlp.c_fc,12,3.2501048983174172 +vision.mlp.c_fc,13,3.144904609090658 +vision.mlp.c_fc,14,3.1176022019209833 +vision.mlp.c_fc,15,3.170790033142059 +vision.mlp.c_fc,16,3.1901952176386157 +vision.mlp.c_fc,17,3.1728736833856566 +vision.mlp.c_fc,18,3.1092420753715064 +vision.mlp.c_fc,19,3.1289105323616995 +vision.mlp.c_fc,20,3.1633778054509616 +vision.mlp.c_fc,21,3.1948778928908625 +vision.mlp.c_fc,22,3.153364851937131 +vision.mlp.c_fc,23,3.1481862277653847 +vision.mlp.c_proj,0,48.91602226406029 +vision.mlp.c_proj,1,7.994586159177927 +vision.mlp.c_proj,2,9.259903305960076 +vision.mlp.c_proj,3,12.161504492865403 +vision.mlp.c_proj,4,11.137208824577082 +vision.mlp.c_proj,5,4.50118105140538 +vision.mlp.c_proj,6,3.9747365034192206 +vision.mlp.c_proj,7,4.846873977315018 +vision.mlp.c_proj,8,4.914742513787588 +vision.mlp.c_proj,9,5.0506453475209065 +vision.mlp.c_proj,10,13.838509154283804 +vision.mlp.c_proj,11,4.258906842824289 +vision.mlp.c_proj,12,3.4482411935554316 +vision.mlp.c_proj,13,3.4563538182296156 +vision.mlp.c_proj,14,3.1506796697623156 +vision.mlp.c_proj,15,3.140821870180245 +vision.mlp.c_proj,16,3.1870137578290736 +vision.mlp.c_proj,17,3.1622263243928037 +vision.mlp.c_proj,18,3.1727003318990294 +vision.mlp.c_proj,19,3.126420350320576 +vision.mlp.c_proj,20,3.214422564736524 +vision.mlp.c_proj,21,3.5410765909033515 +vision.mlp.c_proj,22,3.885907825953595 +vision.mlp.c_proj,23,3.241221353196137 +text.mlp.c_fc,0,3.111321629333239 +text.mlp.c_fc,1,3.1040094243226832 +text.mlp.c_fc,2,3.0476798189406455 +text.mlp.c_fc,3,3.0567396405641647 +text.mlp.c_fc,4,3.072398311749821 +text.mlp.c_fc,5,3.0581450022889753 +text.mlp.c_fc,6,3.053470998692992 +text.mlp.c_fc,7,3.0531601497538126 +text.mlp.c_fc,8,3.0696775698776992 +text.mlp.c_fc,9,3.065453985089178 +text.mlp.c_fc,10,3.067657384247557 +text.mlp.c_fc,11,3.086129858871849 +text.mlp.c_proj,0,16.411416535361464 +text.mlp.c_proj,1,10.636934397271798 +text.mlp.c_proj,2,5.467879831695406 +text.mlp.c_proj,3,3.117101945617834 +text.mlp.c_proj,4,3.162125318135728 +text.mlp.c_proj,5,3.114353715482094 +text.mlp.c_proj,6,3.1560610360682673 +text.mlp.c_proj,7,3.230989551532942 +text.mlp.c_proj,8,3.273957420333387 +text.mlp.c_proj,9,3.322224898837691 +text.mlp.c_proj,10,3.252371370612472 +text.mlp.c_proj,11,3.2207147468979427 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py new file mode 100644 index 0000000000000000000000000000000000000000..601128e6acedc080764ca766c4d8b10f0872ce2c --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py @@ -0,0 +1,66 @@ +import os +import time + +import torch +import transformers +from awq import AutoAWQForCausalLM +from transformers import AutoConfig + +from lm_quant_toolkit.adapter.common import get_model_storage_size + + +def create_autoawq_model(model_id, quant_config, config_id, load_quantized, save_dir): + model_file_size = 0 + quantized = False + quant_path = f"{save_dir}/{model_id}-{config_id}-awq" + + config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) + # To avoid OOM after huggingface transformers 4.36.2 + config.use_cache = False + if load_quantized and os.path.exists(quant_path): + model = AutoAWQForCausalLM.from_quantized( + quant_path, + device_map="auto", + offload_state_dict=False, + config=config, + ) + tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) + quantized = True + model_file_size = get_model_storage_size(quant_path) + model = model.cuda() + else: + tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) + # max_memory={0: "18GiB", "cpu": "60GiB"}, + # ) + model = AutoAWQForCausalLM.from_pretrained( + model_id, + device_map="auto", + offload_state_dict=False, + torch_dtype=torch.float16, + max_memory={0: "18GiB", "cpu": "60GiB"}, + config=config, + ) + return model, tokenizer, quantized, model_file_size + + +def quantize_autoawq_model( + model, tokenizer, quant_config, model_id, config_id, save_dir +): + t1 = time.time() + model.quantize(tokenizer, quant_config=quant_config) + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with AutoAWQ") + quant_path = f"{save_dir}/{model_id}-{config_id}-awq" + model.save_quantized(quant_path) + tokenizer.save_pretrained(quant_path) + # persistent the quantized model + os.sync() + return model, t2 - t1, _get_model_file_size(quant_path) + + +def _get_model_file_size(quant_path): + quant_fp_pt = os.path.join(quant_path, "qmodel.pth") + if os.path.exists(quant_fp_pt): + return os.path.getsize(quant_fp_pt) + else: + return get_model_storage_size(quant_path) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py new file mode 100644 index 0000000000000000000000000000000000000000..6a2822005935cc164ad3e02693f0e12d99b5a2bb --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py @@ -0,0 +1,85 @@ +import gc +import os +import random +import time + +import torch +import transformers +from auto_gptq import AutoGPTQForCausalLM +from datasets import load_dataset +from tqdm import tqdm + + +# Adapted from: https://towardsdatascience.com/4-bit-quantization-with-gptq-36b0f4f02c34 +def prepare_model(model, tokenizer, n_samples=1024, max_tokens=512, use_triton=False): + # Load data and tokenize examples + data = load_dataset( + "allenai/c4", + data_files="en/c4-train.00001-of-01024.json.gz", + split=f"train[:{n_samples}]", + ) + # ~536K tokens + tokenized_data = torch.cat( + [ + tokenizer(data[i]["text"], return_tensors="pt").input_ids + for i in tqdm(range(len(data))) + ], + axis=-1, + ) + + # Format tokenized examples + random.seed(1) + examples_ids = [] + for _ in range(n_samples): + i = random.randint(0, tokenized_data.shape[1] - max_tokens - 1) + j = i + max_tokens + input_ids = tokenized_data[:, i:j] + attention_mask = torch.ones_like(input_ids) + examples_ids.append({"input_ids": input_ids, "attention_mask": attention_mask}) + + print("Using " + str(len(examples_ids)) + " samples for calibration.") + model.quantize(examples_ids, batch_size=1, use_triton=use_triton) + # model = model.cuda() + # with torch.no_grad(): + # x = model(input_ids.to('cuda')) + # del examples_ids, x + del examples_ids + torch.cuda.empty_cache() + gc.collect() + return model + + +def create_autogptq_model(model_id, quant_config, config_id, load_quantized, save_dir): + model_file_size = 0 + quantized = False + quant_path = f"{save_dir}/{model_id}-{config_id}-gptq" + if load_quantized and os.path.exists(quant_path): + model = AutoGPTQForCausalLM.from_quantized(quant_path, device="cuda:0") + tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) + quantized = True + model_file_size = _get_model_file_size(quant_path, quant_config) + else: + tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) + model = AutoGPTQForCausalLM.from_pretrained(model_id, quant_config) + return model, tokenizer, quantized, model_file_size + + +def quantize_autogptq_model( + model, tokenizer, quant_config, model_id, config_id, save_dir +): + t1 = time.time() + model = prepare_model(model, tokenizer) + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with AutoGPTQ") + quant_path = f"{save_dir}/{model_id}-{config_id}-gptq" + model.save_quantized(quant_path, use_safetensors=True) + # persistent the quantized model + os.sync() + return model, t2 - t1, _get_model_file_size(quant_path, quant_config) + + +def _get_model_file_size(quant_path, quant_config): + b = quant_config.bits + g = quant_config.group_size + fp = os.path.join(quant_path, f"gptq_model-{b}bit-{g}g.safetensors") + return os.path.getsize(fp) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py new file mode 100644 index 0000000000000000000000000000000000000000..d40e8a962118a79dbeac89860282482590eb9642 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py @@ -0,0 +1,91 @@ +import os +import time + +import torch +from accelerate import ( + infer_auto_device_map, + init_empty_weights, + load_checkpoint_in_model, +) +from awq.quantize.pre_quant import apply_awq, run_awq +from awq.quantize.quantizer import real_quantize_model_weight +from awq.utils.utils import simple_dispatch_model +from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer + + +def create_awq_model(model_id, quant_config, config_id, load_quantized, save_dir): + quantized = False + quant_path = f"{save_dir}/{model_id}-{config_id}-awq" + + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=False, trust_remote_code=True + ) + config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) + # Note (Haotian): To avoid OOM after huggingface transformers 4.36.2 + config.use_cache = False + if load_quantized and os.path.exists(f"{quant_path}/qmodel.pth"): + with init_empty_weights(): + model = AutoModelForCausalLM.from_config( + config=config, torch_dtype=torch.float16, trust_remote_code=True + ) + max_memory = {0: "20GiB", "cpu": "60GiB"} + # Infer device map + kwargs = {"max_memory": max_memory} if len(max_memory) else {} + device_map = infer_auto_device_map( + model, + no_split_module_classes=[ + "OPTDecoderLayer", + "LlamaDecoderLayer", + "BloomBlock", + "MPTBlock", + "DecoderLayer", + ], + **kwargs, + ) + # Load checkpoint in the model + load_checkpoint_in_model( + model, + checkpoint=quant_path, + device_map=device_map, + offload_state_dict=False, + ) + # Dispatch model + model = simple_dispatch_model(model, device_map=device_map) + quantized = True + model.eval() + else: + kwargs = {"torch_dtype": torch.float16, "low_cpu_mem_usage": True} + model = AutoModelForCausalLM.from_pretrained( + model_id, config=config, trust_remote_code=True, **kwargs + ) + return model, tokenizer, quantized, 0 + + +def quantize_awq_model(model, tokenizer, quant_config, model_id, config_id, save_dir): + t1 = time.time() + nbits = quant_config.pop("w_bit") + awq_results = run_awq( + model, + tokenizer, + w_bit=nbits, + q_config=quant_config, + n_samples=128, + seqlen=512, + ) + intermediate_fp = f"{save_dir}/{model_id}-{config_id}-awq/intermediate.pth" + dirpath = os.path.dirname(intermediate_fp) + os.makedirs(dirpath, exist_ok=True) + torch.save(awq_results, intermediate_fp) + awq_results = torch.load(intermediate_fp, map_location="cpu") + apply_awq(model, awq_results) + real_quantize_model_weight(model, w_bit=nbits, q_config=quant_config) + + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with AWQ") + quant_path = f"{save_dir}/{model_id}-{config_id}-awq" + quant_fp = os.path.join(quant_path, "qmodel.pth") + torch.save(model.cpu().state_dict(), quant_fp) + tokenizer.save_pretrained(quant_path) + + model_file_size = os.path.getsize(quant_fp) + return model, t2 - t1, model_file_size diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py new file mode 100644 index 0000000000000000000000000000000000000000..71a6234bd9fc1834630b015212eea46e7fcf6038 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py @@ -0,0 +1,37 @@ +import os +import time + +from transformers import AutoModelForCausalLM, AutoTokenizer + +from lm_quant_toolkit.adapter.common import get_model_storage_size + + +def create_bnb_model(model_id, quant_config, config_id, load_quantized, save_dir): + quantized = False + model_file_size = 0 + quant_path = f"{save_dir}/{model_id}-{config_id}-bnb" + if load_quantized and os.path.exists(quant_path): + model = AutoModelForCausalLM.from_pretrained(quant_path) + tokenizer = AutoTokenizer.from_pretrained(model_id) + quantized = True + model_file_size = get_model_storage_size(quant_path) + else: + model = None + tokenizer = AutoTokenizer.from_pretrained(model_id) + return model, tokenizer, quantized, model_file_size + + +def quantize_bnb_model(model, tokenizer, quant_config, model_id, config_id, save_dir): + model_file_size = 0 + t1 = time.time() + model = AutoModelForCausalLM.from_pretrained( + model_id, quantization_config=quant_config + ) + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with BnB") + quant_path = f"{save_dir}/{model_id}-{config_id}-bnb" + model.save_pretrained(quant_path) + # persistent the quantized model + os.sync() + model_file_size = get_model_storage_size(quant_path) + return model, t2 - t1, model_file_size diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py new file mode 100644 index 0000000000000000000000000000000000000000..9a323216de087a42049a2c903f89eaca5ab81cd9 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py @@ -0,0 +1,20 @@ +import json +import os + + +def get_model_storage_size( + base_dir, + index_file="model.safetensors.index.json", + model_file="model.safetensors", +): + size = 0 + index_file = os.path.join(base_dir, index_file) + if os.path.exists(index_file): + # model is split into shards + with open(index_file, "r") as f: + index = json.load(f) + for shard in set(index["weight_map"].values()): + size += os.path.getsize(os.path.join(base_dir, shard)) + else: + size = os.path.getsize(os.path.join(base_dir, model_file)) + return size diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py new file mode 100644 index 0000000000000000000000000000000000000000..7ba54fff22b16a191deda2520ce680eb0b3eb8bb --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py @@ -0,0 +1,17 @@ +import torch +import transformers +from transformers import AutoModelForCausalLM + +from lm_quant_toolkit.adapter.common import get_model_storage_size +from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir + + +def create_fp16_model(model_id, quant_config, config_id, load_quantized, save_dir): + model_file_size = 0 + model = AutoModelForCausalLM.from_pretrained( + model_id, device_map="auto", torch_dtype=torch.float16 + ) + tokenizer = transformers.AutoTokenizer.from_pretrained(model_id) + base_dir = get_hf_model_storge_base_dir(model_id) + model_file_size = get_model_storage_size(base_dir) + return model, tokenizer, False, model_file_size diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py new file mode 100644 index 0000000000000000000000000000000000000000..5c8a09e699f843376071c8430aa3e86419b2aab4 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py @@ -0,0 +1,34 @@ +import os +import time + +from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer +from hqq.engine.hf import HQQModelForCausalLM + + +def create_hqq_model(model_id, quant_config, config_id, load_quantized, save_dir): + quantized = False + model_file_size = 0 + quant_path = f"{save_dir}/{model_id}-{config_id}-hqq" + if load_quantized and os.path.exists(quant_path): + model = HQQModelForCausalLM.from_quantized(quant_path) + tokenizer = hggAutoTokenizer.from_pretrained(model_id) + quantized = True + model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt")) + else: + model = HQQModelForCausalLM.from_pretrained(model_id) + tokenizer = hggAutoTokenizer.from_pretrained(model_id) + return model, tokenizer, quantized, model_file_size + + +def quantize_hqq_model(model, tokenizer, quant_config, model_id, config_id, save_dir): + model_file_size = 0 + t1 = time.time() + model.quantize_model(quant_config=quant_config) + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with HQQ") + quant_path = f"{save_dir}/{model_id}-{config_id}-hqq" + model.save_quantized(quant_path) + # persistent the quantized model + os.sync() + model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt")) + return model, t2 - t1, model_file_size diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py new file mode 100644 index 0000000000000000000000000000000000000000..3baebbf79d6078c0ed293d50fbc87d0acc4296ee --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py @@ -0,0 +1,34 @@ +import os +import time + +from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer +from hqq.engine.hf import HQQModelForCausalLM + + +def create_mxq_model(model_id, quant_config, config_id, load_quantized, save_dir): + quantized = False + model_file_size = 0 + quant_path = f"{save_dir}/{model_id}-{config_id}-mxq" + if load_quantized and os.path.exists(quant_path): + model = HQQModelForCausalLM.from_quantized(quant_path) + tokenizer = hggAutoTokenizer.from_pretrained(model_id) + quantized = True + model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt")) + else: + model = HQQModelForCausalLM.from_pretrained(model_id) + tokenizer = hggAutoTokenizer.from_pretrained(model_id) + return model, tokenizer, quantized, model_file_size + + +def quantize_mxq_model(model, tokenizer, quant_config, model_id, config_id, save_dir): + model_file_size = 0 + t1 = time.time() + model.quantize_model(quant_config=quant_config) + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with MXQ") + quant_path = f"{save_dir}/{model_id}-{config_id}-mxq" + model.save_quantized(quant_path) + # persistent the quantized model + os.sync() + model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt")) + return model, t2 - t1, model_file_size diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py new file mode 100644 index 0000000000000000000000000000000000000000..7e3d0887d92ea80c12df01bcdc61ca2d60f9686c --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py @@ -0,0 +1,27 @@ +import os +import time +from pathlib import Path + +from hqq.engine.open_clip import HQQOpenCLIP + + +def create_hqq_model(model_id, quant_config, config_id, load_quantized, save_dir): + quantized = False + quant_path = f"{save_dir}/{model_id}-{config_id}-hqq" + if load_quantized and os.path.exists(quant_path): + model = HQQOpenCLIP.from_quantized(quant_path) + quantized = True + else: + model = HQQOpenCLIP.create_model(model_id, device="cpu") + return model, quantized + + +def quantize_hqq_model(model, quant_config, model_id, config_id, save_dir): + t1 = time.time() + model.quantize_model(quant_config=quant_config) + t2 = time.time() + print("Took " + str(t2 - t1) + " seconds to quantize the model with HQQ") + quant_path = f"{save_dir}/{model_id}-{config_id}-hqq" + Path(quant_path).mkdir(parents=True, exist_ok=True) + model.save_quantized(save_dir=quant_path) + return model, t2 - t1 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/__init__.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py new file mode 100644 index 0000000000000000000000000000000000000000..0a10102ef0afe1a16dbfb1fbad1240d1b71faeeb --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py @@ -0,0 +1,759 @@ +import copy +import logging +import os + +# from adapter.awq import create_awq_model +# from adapter.awq import quantize_awq_model +from datetime import datetime +from pathlib import Path + +import pandas as pd +import torch +from auto_gptq import BaseQuantizeConfig as GPTQQuantConfig +from hqq.core.quantize import BaseQuantizeConfig as HQQQuantConfig +from transformers import BitsAndBytesConfig + +from lm_quant_toolkit.adapter.autoawq import ( + create_autoawq_model, + quantize_autoawq_model, +) +from lm_quant_toolkit.adapter.autogptq import ( + create_autogptq_model, + quantize_autogptq_model, +) +from lm_quant_toolkit.adapter.bnb import create_bnb_model, quantize_bnb_model +from lm_quant_toolkit.adapter.fp16 import create_fp16_model +from lm_quant_toolkit.adapter.hqq import create_hqq_model, quantize_hqq_model +from lm_quant_toolkit.adapter.mxq import create_mxq_model, quantize_mxq_model +from lm_quant_toolkit.eval.common import ( + HQQ_CONFIGS, + _dump_cuda_mem_snapshot, + _reset_peak_memory_stats, + cleanup, + combine_metrics, + get_memory_metrics, + get_mxq_quant_meta_data_file, + persist_progress, + save_partial_metric, +) +from lm_quant_toolkit.eval.leaderboard import eval_llm_leaderboard +from lm_quant_toolkit.eval.perplexity import eval_ppls + +ALL_MODELS = [ + "meta-llama/Llama-2-7b-hf", + "meta-llama/Llama-2-13b-hf", + "meta-llama/Meta-Llama-3-8B", + "meta-llama/Llama-3.1-8B", +] + +MXQ_CONFIGS = [ + ( + f"{bits:.2f}".replace(".", "_"), + HQQQuantConfig(mixed=True, budget=bits, quant_scale=True), + ) + for bits in [5.00, 4.75, 4.50, 4.25, 4.01, 3.76, 3.50, 3.00, 2.75, 2.48] +] + +BNB_CONFIGS = [ + ( + "b4g64", + BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype="float16", + bnb_4bit_use_double_quant=True, + ), + ), + ( + "b8g128", + BitsAndBytesConfig( + load_in_8bit=True, + bnb_4bit_compute_dtype="float16", + ), + ), +] + + +AUTOAWQ_CONFIGS = [ + ("b4g32", {"w_bit": 4, "q_group_size": 32, "zero_point": True, "version": "GEMM"}), + ("b4g64", {"w_bit": 4, "q_group_size": 64, "zero_point": True, "version": "GEMM"}), + ( + "b4g128", + {"w_bit": 4, "q_group_size": 128, "zero_point": True, "version": "GEMM"}, + ), + # 3-bit not supported by AutoAWQ right now + # ("b3g64", {"w_bit": 3, "q_group_size": 64, "zero_point": True, 'version':'gemv_fast'}), + # ("b3g128", {"w_bit": 3, "q_group_size": 128, "zero_point": True, 'version':'gemv_fast'}), +] + +GPTQ_CONFIGS = [ + ( + "b8g32", + GPTQQuantConfig(bits=8, group_size=32, damp_percent=0.01, desc_act=False), + ), + ( + "b8g64", + GPTQQuantConfig(bits=8, group_size=64, damp_percent=0.01, desc_act=False), + ), + ( + "b8g128", + GPTQQuantConfig(bits=8, group_size=128, damp_percent=0.01, desc_act=False), + ), + ( + "b4g32", + GPTQQuantConfig(bits=4, group_size=32, damp_percent=0.01, desc_act=False), + ), + ( + "b4g64", + GPTQQuantConfig(bits=4, group_size=64, damp_percent=0.01, desc_act=False), + ), + ( + "b4g128", + GPTQQuantConfig(bits=4, group_size=128, damp_percent=0.01, desc_act=False), + ), + ( + "b3g32", + GPTQQuantConfig(bits=3, group_size=32, damp_percent=0.01, desc_act=False), + ), + ( + "b3g64", + GPTQQuantConfig(bits=3, group_size=64, damp_percent=0.01, desc_act=False), + ), + ( + "b3g128", + GPTQQuantConfig(bits=3, group_size=128, damp_percent=0.01, desc_act=False), + ), +] + + +def gen_experiment_items(models, tasks): + dikts = [] + for algo, spec in tasks.items(): + configs = spec["configs"] + for config in configs: + for model_id in models: + dikts.append( + { + "model": model_id, + "cfg": config[0], + "task_type": spec["type"], + "algo": algo, + } + ) + return pd.DataFrame(dikts) + + +def _setup_fn(algo, spec): + match algo: + case "fp16": + spec["create_fn"] = create_fp16_model + spec["quantize_fn"] = None + case "awq": + spec["create_fn"] = create_autoawq_model + spec["quantize_fn"] = quantize_autoawq_model + case "gptq": + spec["create_fn"] = create_autogptq_model + spec["quantize_fn"] = quantize_autogptq_model + case "hqq": + spec["create_fn"] = create_hqq_model + spec["quantize_fn"] = quantize_hqq_model + case "bnb": + spec["create_fn"] = create_bnb_model + spec["quantize_fn"] = quantize_bnb_model + case "mxq": + spec["create_fn"] = create_mxq_model + spec["quantize_fn"] = quantize_mxq_model + case _: + raise ValueError(f"Invalid algo: {algo}") + + +def do_expermient( + experiment_name, + models, + tasks, + quant_dir="snapshots", + result_dir="results", + log_dir="logs", + track_cuda_memory=False, + **kwargs, +): + df_all = gen_experiment_items(models, tasks) + progress_path = os.path.join(result_dir, experiment_name, "progress.csv") + if Path(progress_path).exists(): + df_saved = pd.read_csv(progress_path) + df_all = df_all.merge( + df_saved, how="left", on=["model", "cfg", "task_type", "algo"] + ) + # filter already processed repos, equivalent to SQL is null + df_todo = df_all.query("status != status or status != 1") + else: + df_all["status"] = 0 + df_all["completion_time"] = "" + df_todo = df_all + print("*" * 72) + print("Sub-task list:") + print(df_all) + cnt_todo, cnt_tot = len(df_todo), len(df_all) + print(f"Todo:{cnt_todo}, Done: {cnt_tot - cnt_todo}, Total: {cnt_tot}") + if cnt_todo == 0: + print("Tasks completed!") + print("*" * 72) + if cnt_todo == 0: + return + + df_todo = df_todo.sort_values(by=["model", "cfg"], ascending=False) + for idx, row in df_todo.iterrows(): + model_id = row["model"] + algo = row["algo"] + task_type = row["task_type"] + cfg = row["cfg"] + spec = tasks[algo] + _setup_fn(algo, spec) + config = [c for c in spec["configs"] if c[0] == cfg][0] + quant_fn = spec["quantize_fn"] + metric = _init_metrics(model_id, algo, config) + print("*" * 72) + if task_type == "quant": + print(f"Quantizing {algo} on {model_id} w/ config: {cfg}...") + elif task_type == "eval_ppl": + print(f"Evaluating {algo} PPL on {model_id} w/ config: {cfg}...") + elif task_type == "eval_leaderboard": + print( + f"Evaluating {algo} LLM Leaderboard benchmarks on {model_id} w/ config: {cfg}..." + ) + else: + print( + f"Evaluating {algo} model storage metrics on {model_id} w/ config: {cfg}..." + ) + print("*" * 72) + + if track_cuda_memory: + torch.cuda.memory._record_memory_history() + _reset_peak_memory_stats() + if task_type != "eval_leaderboard": + create_fn = spec["create_fn"] + model, tokenizer, quantized, model_file_size = create_fn( + model_id, config[1], cfg, quant_fn is not None, quant_dir + ) + + if not quantized and quant_fn: + # avoid interventions between models + quant_config = copy.deepcopy(config[1]) + if algo == "mxq": + ok, metric_fp = get_mxq_quant_meta_data_file(model_id) + if not ok: + print( + f"Quantization meta data file: {metric_fp} doesn't exists!" + ) + return + quant_config["quant_metrics_file"] = metric_fp + quant_config["weight_algo"] = kwargs.get("weight_algo", None) + quant_config["boost_layers"] = kwargs.get("boost_layers", None) + quant_config["decline_layers"] = kwargs.get("decline_layers", None) + quant_config["boost_stop"] = kwargs.get("boost_stop", None) + quant_config["decline_stop"] = kwargs.get("decline_stop", None) + quant_config["ablation"] = kwargs.get("ablation", None) + quant_config["top_m_layer"] = kwargs.get("top_m_layer", None) + quant_config["factor"] = kwargs.get("factor", None) + model, duration, model_file_size = quant_fn( + model, + tokenizer, + quant_config, + model_id, + cfg, + quant_dir, + ) + metric["quant_duration"] = duration + if task_type == "eval_model_storage": + allot, reserved = get_memory_metrics() + metric["load_mem_allot"] = allot + metric["load_mem_reserved"] = reserved + metric["model_storage_size"] = model_file_size + + elif task_type == "eval_ppl": + # Evaluate the quantized model + metric = eval_ppls(model, tokenizer, metric) + metric["ppl_mem_allot"], metric["ppl_mem_reserved"] = ( + get_memory_metrics() + ) + if track_cuda_memory: + _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir) + cleanup(model) + else: + metric = eval_llm_leaderboard( + experiment_name, + model_id, + algo, + cfg, + quant_fn is not None, + metric, + quant_dir, + result_dir, + ) + metric["leaderboard_mem_allot"], metric["leaderboard_mem_reserved"] = ( + get_memory_metrics() + ) + if track_cuda_memory: + _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir) + save_partial_metric(experiment_name, algo, model_id, cfg, metric, result_dir) + df_all.loc[ + (df_all["model"] == model_id) + & (df_all["cfg"] == cfg) + & (df_all["algo"] == algo) + & (df_all["task_type"] == task_type), + ["status", "completion_time"], + ] = 1, datetime.now().strftime("%Y-%m-%d %H:%M:%S") + persist_progress(df_all, progress_path) + # combine metrics + combine_metrics(experiment_name, result_dir) + + +def _init_metrics(model_id, algo, config): + return { + "model": model_id.split("/")[1], + "algo": algo, + "config": config[0], + "config_detail": str(config[1]).replace("\n", ""), + "quant_duration": 0, + "model_storage_size": 0, + "load_mem_allot": 0, + "load_mem_reserved": 0, + "ppl_mem_allot": 0, + "ppl_mem_reserved": 0, + "leaderboard_mem_allot": 0, + "leaderboard_mem_reserved": 0, + "quant_duration": 0, + "ppl_wikitext": 0, + "ppl_c4": 0, + "duration_wikitext": 0, + "duration_c4": 0, + "duration_leaderboard": 0, + "ifeval": 0, + "bbh": 0, + "mathlevel5": 0, + "gpqa": 0, + "musr": 0, + "mmlupro": 0, + } + + +def do_expermient_fdata( + experiment_name, + models, + tasks, + track_cuda_memory=False, +): + do_expermient( + experiment_name, + models, + tasks, + quant_dir="/fdata/llm/mxq/snapshots", + result_dir="/fdata/llm/mxq/results", + log_dir="/fdata/llm/mxq/logs", + track_cuda_memory=track_cuda_memory, + ) + + +######################################################################## +# Quantization experiments +######################################################################## + + +def experiment_quant_hqq(): + models = ALL_MODELS + tasks = { + "hqq": { + "type": "quant", + "configs": HQQ_CONFIGS, + }, + } + do_expermient_fdata( + "quant_hqq", + models, + tasks, + ) + + +def experiment_quant_mxq(): + models = ALL_MODELS + type = "quant" + algo = "mxq" + tasks = { + algo: { + "type": type, + "configs": MXQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}_mxq", models, tasks) + + +def experiment_quant_awq(): + # models = [ALL_MODELS[0], ALL_MODELS[2]] + models = [ALL_MODELS[1]] + type = "quant" + algo = "awq" + tasks = { + algo: { + "type": type, + "configs": AUTOAWQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_quant_gptq(): + models = ALL_MODELS + type = "quant" + algo = "gptq" + tasks = { + algo: { + "type": type, + "configs": GPTQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_quantize_405B(): + models = [ + "meta-llama/Meta-Llama-3.1-405B-Instruct", + ] + + tasks = { + "hqq": { + "type": "quant", + "configs": HQQ_CONFIGS[1:2], + }, + } + do_expermient( + "quant_hqq_405B", + models, + tasks, + quant_dir="/data/gqq-eval/snapshots/", + ) + + +######################################################################## +# Perplexity evaluation experiments +######################################################################## + + +def experiment_ppl_eval_fp16(): + models = ALL_MODELS + type = "eval_ppl" + algo = "fp16" + tasks = { + algo: { + "type": type, + "configs": [ + ("base", {}), + ], + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_ppl_eval_awq(): + models = [ALL_MODELS[0], ALL_MODELS[2]] + type = "eval_ppl" + algo = "awq" + tasks = { + algo: { + "type": type, + "configs": AUTOAWQ_CONFIGS[1:2], + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_ppl_eval_gptq(): + models = ALL_MODELS + type = "eval_ppl" + algo = "gptq" + tasks = { + algo: { + "type": type, + "configs": GPTQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_ppl_eval_hqq(): + models = ALL_MODELS + type = "eval_ppl" + algo = "hqq" + tasks = { + algo: { + "type": type, + "configs": HQQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +######################################################################## +# Open LLM Leaderboard evaluation experiments +######################################################################## + + +def experiment_llm_leaderboard_fp16(): + models = [ALL_MODELS[0], ALL_MODELS[2]] + type = "eval_leaderboard" + algo = "fp16" + tasks = { + algo: { + "type": type, + "configs": [ + ("base", {}), + ], + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_llm_leaderboard_autogptq(): + models = ALL_MODELS + type = "eval_leaderboard" + algo = "gptq" + tasks = { + algo: { + "type": type, + "configs": GPTQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_llm_leaderboard_hqq(): + models = ALL_MODELS + type = "eval_leaderboard" + algo = "hqq" + tasks = { + algo: { + "type": type, + "configs": HQQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_llm_leaderboard_mxq(): + models = ALL_MODELS + type = "eval_leaderboard" + algo = "mxq" + tasks = { + algo: { + "type": type, + "configs": MXQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_llm_leaderboard_autoawq(): + models = ALL_MODELS[0:1] + type = "eval_leaderboard" + algo = "awq" + tasks = { + algo: { + "type": type, + "configs": AUTOAWQ_CONFIGS[0:2], + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +######################################################################## +# Mixed Quant Eval experiments +######################################################################## + + +def experiment_quant_ppl_eval_mxq_comprise(): + models = ALL_MODELS + equiv_mxq_configs = [] + nbits = [4.06, 4.10, 4.15, 4.19, 4.24, 4.28, 4.33] + for bits in nbits: + cfg_name = f"mxq-{str(bits).replace('.', '_')}" + equiv_mxq_configs.append( + (cfg_name, HQQQuantConfig(mixed=True, budget=bits, quant_scale=True)) + ) + quant_tasks = { + "hqq": { + "type": "quant", + "configs": equiv_mxq_configs, + }, + } + ppl_tasks = { + "hqq": { + "type": "eval_ppl", + "configs": equiv_mxq_configs, + }, + } + do_expermient_fdata("quant_mxq_compromise", models, quant_tasks) + do_expermient_fdata("eval_mxq_compromise", models, ppl_tasks) + + +######################################################################## +# Misc experiments +######################################################################## + + +def experiment_fp16_llama3_8B_OOM(): + models = ALL_MODELS[-1:] + type = "eval_ppl" + algo = "fp16" + tasks = { + algo: { + "type": type, + "configs": [ + ("base", {}), + ], + }, + } + do_expermient_fdata( + f"{type}_llama3_8B_OOM_{algo}", + models, + tasks, + track_cuda_memory=True, + ) + + +def experiment_fp16_vs_hqq_eval_gpu_mem(): + models = ALL_MODELS[-1:] + type = "eval_ppl" + algo = "fp16" + tasks = { + algo: { + "type": type, + "configs": [ + ("base", {}), + ], + }, + } + do_expermient_fdata("experiment_fp16_vs_hqq_eval_gpu_mem", models, tasks) + algo = "hqq" + tasks = { + algo: { + "type": type, + "configs": HQQ_CONFIGS[1:2], + }, + } + do_expermient_fdata("experiment_fp16_vs_hqq_eval_gpu_mem", models, tasks) + + +def experiment_eval_model_storage(): + models = ALL_MODELS + type = "eval_model_storage" + tasks = { + "fp16": { + "type": type, + "configs": [ + ("base", {}), + ], + }, + "mxq": { + "type": type, + "configs": MXQ_CONFIGS, + }, + "hqq": { + "type": type, + "configs": HQQ_CONFIGS, + }, + "awq": { + "type": type, + "configs": AUTOAWQ_CONFIGS, + }, + "gptq": {"type": type, "configs": GPTQ_CONFIGS}, + } + for i in range(5): + do_expermient_fdata(f"eval_model_storge_{i}", models, tasks) + + +def experiment_eval_ppl_all(): + models = ALL_MODELS + type = "eval_ppl" + tasks = { + "fp16": { + "type": type, + "configs": [ + ("base", {}), + ], + }, + "mxq": { + "type": type, + "configs": MXQ_CONFIGS, + }, + "hqq": { + "type": type, + "configs": HQQ_CONFIGS, + }, + "awq": { + "type": type, + "configs": AUTOAWQ_CONFIGS, + }, + "gptq": {"type": type, "configs": GPTQ_CONFIGS}, + } + do_expermient_fdata("experiment_eval_ppl_all", models, tasks) + + +def experiment_debug_quant_hqq(): + models = [ALL_MODELS[1]] + type = "eval_model_storage" + algo = "hqq" + tasks = { + algo: { + "type": type, + "configs": HQQ_CONFIGS[1:2], + }, + } + + do_expermient( + f"debug_{type}_{algo}", + models, + tasks, + quant_dir="/fdata/llm/mxq/snapshots-debug", + result_dir="/fdata/llm/mxq/results", + ) + + +def main(): + logging.basicConfig( + format="%(asctime)s %(levelname)s [%(name)s] %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", + ) + + # experiment_llm_leaderboard_autogptq() + # experiment_llm_leaderboard_fp16() + # experiment_llm_leaderboard_hqq() + # experiment_llm_leaderboard_autoawq() + # experiment_quant_hqq() + # experiment_quant_mxq() + # experiment_quant_awq() + # experiment_quant_gptq() + # experiment_ppl_eval_fp16() + # experiment_ppl_eval_hqq() + # experiment_ppl_eval_gptq() + # experiment_ppl_eval_awq() + # experiment_fp16_llama3_8B_OOM() + # experiment_fp16_vs_hqq_eval_gpu_mem() + # experiment_debug_quant_hqq() + # experiment_eval_model_storage() + # experiment_eval_ppl_all() + experiment_debug_quant_hqq() + + +if __name__ == "__main__": + # os.environ['HF_DATASETS_OFFLINE'] = '1' + + max_threads = str(min(8, os.cpu_count())) + os.environ["OMP_NUM_THREADS"] = max_threads + os.environ["OPENBLAS_NUM_THREADS"] = max_threads + os.environ["MKL_NUM_THREADS"] = max_threads + os.environ["VECLIB_MAXIMUM_THREADS"] = max_threads + os.environ["NUMEXPR_NUM_THREADS"] = max_threads + os.environ["NUMEXPR_MAX_THREADS"] = max_threads + os.environ["HF_HOME"] = "/data/hugginface/" + + main() diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench_vit.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..2a30d5efc33f98cb994d46acd57c79028e7b7fa4 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench_vit.py @@ -0,0 +1,451 @@ +import copy +import logging +import os +from datetime import datetime +from pathlib import Path + +import pandas as pd +import torch +from hqq.core.quantize import BaseQuantizeConfig as HQQQuantConfig + +from lm_quant_toolkit.eval.clipbenchmark import ( + eval_linear_probe, + eval_zeroshot_classification, +) +from lm_quant_toolkit.eval.common import ( + HQQ_CONFIGS, + _dump_cuda_mem_snapshot, + _reset_peak_memory_stats, + calc_bits, + combine_metrics, + get_memory_metrics, + get_mxq_quant_meta_data_file, + persist_progress, + save_partial_metric, +) + +ALL_MODELS = [ + "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", + "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", +] + +MXQ_CONFIGS = [ + ( + f"{bits:.2f}".replace(".", "_"), + HQQQuantConfig(mixed=True, budget=bits, quant_scale=True), + ) + for bits in [ + 7.80, + 7.72, + 7.64, + 7.56, + 7.48, + 7.40, + 7.32, + 5.00, + 4.95, + 4.90, + 4.86, + 4.82, + 4.78, + 4.73, + 4.00, + 3.96, + 3.92, + 3.00, + ] +] + + +def gen_experiment_items(models, tasks): + dikts = [] + for algo, spec in tasks.items(): + configs = spec["configs"] + for config in configs: + for model_id in models: + dikts.append( + { + "model": model_id, + "cfg": config[0], + "task_type": spec["type"], + "algo": algo, + } + ) + return pd.DataFrame(dikts) + + +def do_expermient( + experiment_name, + models, + tasks, + quant_dir="snapshots", + result_dir="results", + track_cuda_memory=False, + **kwargs, +): + df_all = gen_experiment_items(models, tasks) + progress_path = os.path.join(result_dir, experiment_name, "progress.csv") + if Path(progress_path).exists(): + df_saved = pd.read_csv(progress_path) + df_all = df_all.merge( + df_saved, how="left", on=["model", "cfg", "task_type", "algo"] + ) + # filter already processed repos, equivalent to SQL is null + df_todo = df_all.query("status != status or status != 1") + else: + df_all["status"] = 0 + df_all["completion_time"] = "" + df_todo = df_all + print("*" * 72) + print("Sub-task list:") + print(df_all) + cnt_todo, cnt_tot = len(df_todo), len(df_all) + print(f"Todo:{cnt_todo}, Done: {cnt_tot - cnt_todo}, Total: {cnt_tot}") + if cnt_todo == 0: + print("Tasks completed!") + print("*" * 72) + if cnt_todo == 0: + return + + df_todo = df_todo.sort_values(by=["model", "cfg"], ascending=False) + for idx, row in df_todo.iterrows(): + model_id = row["model"] + algo = row["algo"] + task_type = row["task_type"] + cfg = row["cfg"] + spec = tasks[algo] + config = [c for c in spec["configs"] if c[0] == cfg][0] + metric = _init_metrics(model_id, algo, config) + print("*" * 72) + # if task_type == "quant": + # print(f"Quantizing {algo} on {model_id} w/ config: {cfg}...") + # elif task_type == "eval_zeroshot_cls": + if task_type == "eval_zeroshot_cls": + print( + f"Evaluating {algo} zero-shot classification on {model_id} w/ config: {cfg}..." + ) + else: + print(f"Evaluating {algo} linear-probe on {model_id} w/ config: {cfg}...") + print("*" * 72) + + if track_cuda_memory: + torch.cuda.memory._record_memory_history() + _reset_peak_memory_stats() + + quant_config = None + if algo != "fp16": + quant_config = copy.deepcopy(config[1]) + if algo == "mxq": + ok, metric_fp = get_mxq_quant_meta_data_file(model_id) + if not ok: + print(f"Quantization meta data file: {metric_fp} doesn't exists!") + return + quant_config["quant_metrics_file"] = metric_fp + quant_config["weight_algo"] = kwargs.get("weight_algo", None) + quant_config["boost_stop"] = kwargs.get("boost_stop", None) + quant_config["decline_stop"] = kwargs.get("decline_stop", None) + quant_config["top_m_layer"] = kwargs.get("top_m_layer", None) + if task_type == "eval_zeroshot_cls": + # avoid interventions between models + metric = eval_zeroshot_classification( + metric, + model_id, + result_dir, + quant_dir, + quant_config, + ) + ( + metric["zeroshot_mem_allot"], + metric["zeroshot_mem_reserved"], + ) = get_memory_metrics() + else: + # Make model with different quantization configs don't share the + # pre-calated features vectors, thus they are evaluated + # separately. + feature_root = os.path.join(quant_dir, "features", cfg) + Path(feature_root).mkdir(parents=True, exist_ok=True) + metric = eval_linear_probe( + metric, + model_id, + result_dir, + quant_dir, + quant_config, + feature_root, + ) + ( + metric["linear_probe_mem_allot"], + metric["linear_probe_mem_reserved"], + ) = get_memory_metrics() + + if track_cuda_memory: + _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir) + save_partial_metric(experiment_name, algo, model_id, cfg, metric, result_dir) + df_all.loc[ + (df_all["model"] == model_id) + & (df_all["cfg"] == cfg) + & (df_all["algo"] == algo) + & (df_all["task_type"] == task_type), + ["status", "completion_time"], + ] = 1, datetime.now().strftime("%Y-%m-%d %H:%M:%S") + persist_progress(df_all, progress_path) + # combine metrics + combine_metrics(experiment_name, result_dir) + + +def _init_metrics(model_id, algo, config): + quant_config = copy.deepcopy(config[1]) + mixed = quant_config.get("mixed", False) + if mixed: + quant_config.pop("weight_quant_params", None) + quant_config.pop("scale_quant_params", None) + quant_config.pop("zero_quant_params", None) + + return { + "model": model_id.split("/")[1], + "algo": algo, + "config": config[0], + "config_detail": quant_config, + "quant_duration": 0, + "load_mem_allot": 0, + "load_mem_reserved": 0, + "zeroshot_mem_allot": 0, + "zeroshot_mem_reserved": 0, + "linear_probe_mem_allot": 0, + "linear_probe_mem_reserved": 0, + "acc1_zeroshot_cls": 0, + "acc5_zeroshot_cls": 0, + "recall_zeroshot_cls": 0, + "duration_zeroshot_cls": 0, + "acc1_linear_probe": 0, + "acc5_linear_probe": 0, + "recall_linear_probe": 0, + "duration_linear_probe": 0, + } + + +def get_mxq_bits(reduction_pcts=[3, 5, 8]): + nbits = [] + for cfg in HQQ_CONFIGS: + bpp = calc_bits( + cfg[1]["weight_quant_params"]["nbits"], + cfg[1]["weight_quant_params"]["group_size"], + 8, + 128, + ) + nbits.extend([round(bpp * (1 - pct / 100), 2) for pct in reduction_pcts]) + return sorted(list(set(nbits)), reverse=True) + + +def do_expermient_fdata( + experiment_name, + models, + tasks, + track_cuda_memory=False, +): + do_expermient( + experiment_name, + models, + tasks, + quant_dir="/fdata/llm/mxq/snapshots", + result_dir="/fdata/llm/mxq/results", + track_cuda_memory=track_cuda_memory, + ) + + +######################################################################## +# Quantization experiments +######################################################################## + + +def experiment_quant_hqq(): + models = ALL_MODELS + tasks = { + "hqq": { + "type": "quant", + "configs": HQQ_CONFIGS, + }, + } + do_expermient_fdata( + "quant_hqq", + models, + tasks, + ) + + +def experiment_quant_mxq(): + models = ALL_MODELS + type = "quant" + algo = "hqq" + tasks = { + algo: { + "type": type, + "configs": HQQ_CONFIGS[9:], + }, + } + do_expermient_fdata(f"{type}_{algo}_vit", models, tasks) + + +######################################################################## +# zero-shot image classification evaluation experiments +######################################################################## + + +def experiment_zeroshot_eval_fp16(): + models = ALL_MODELS + type = "eval_zeroshot_cls" + algo = "fp16" + tasks = { + algo: { + "type": type, + "configs": [ + ("base", {}), + ], + }, + } + do_expermient_fdata(f"{type}_{algo}", models, tasks) + + +def experiment_eval_fp16_combined(): + models = ALL_MODELS + algo = "fp16" + tasks = { + algo: { + "type": "eval_zeroshot_cls", + "configs": [ + ("base", {}), + ], + }, + } + linear_probe_tasks = { + algo: { + "type": "eval_linear_probe", + "configs": [ + ("base", {}), + ], + }, + } + do_expermient_fdata(f"eval_linear_probe_{algo}2", models, linear_probe_tasks) + do_expermient_fdata(f"eval_zeroshot_cls_{algo}2", models, tasks) + + +def experiment_eval_hqq_comprehensive(): + models = ALL_MODELS + tasks = { + "hqq": { + "type": "eval_zeroshot_cls", + "configs": HQQ_CONFIGS, + } + } + linear_probe_tasks = { + "hqq": { + "type": "eval_linear_probe", + "configs": HQQ_CONFIGS, + }, + } + do_expermient_fdata("eval_lp_hqq_comprehensive4", models, linear_probe_tasks) + do_expermient_fdata("eval_zs_hqq_comprehensive5", models, tasks) + + +def experiment_eval_mxq_358_memory_saving(): + nbits = get_mxq_bits() + # budget 2.07 is infeasible + nbits = [bit for bit in nbits if bit > 2.07] + mxq_configs = [ + ( + f"{bits:.2f}".replace(".", "_"), + HQQQuantConfig(mixed=True, budget=bits, quant_scale=True), + ) + for bits in nbits + ] + models = ALL_MODELS + tasks = { + "mxq": { + "type": "eval_zeroshot_cls", + "configs": mxq_configs, + } + } + linear_probe_tasks = { + "mxq": { + "type": "eval_linear_probe", + "configs": mxq_configs, + }, + } + do_expermient_fdata("eval_lp_mxq_358_memory_saving4", models, linear_probe_tasks) + do_expermient_fdata("eval_zs_mxq_358_memory_saving5", models, tasks) + + +def experiment_eval_mxq_comprehensive(): + models = ALL_MODELS + zeroshot_tasks = { + "mxq": { + "type": "eval_zeroshot_cls", + "configs": MXQ_CONFIGS, + }, + } + linear_probe_tasks = { + "mxq": { + "type": "eval_linear_probe", + "configs": MXQ_CONFIGS, + }, + } + do_expermient_fdata("eval_zs_mxq_comprehensive2", models, zeroshot_tasks) + do_expermient_fdata("eval_lp_mxq_comprehensive2", models, linear_probe_tasks) + + +def experiment_zeroshot_eval_mxq(): + models = ALL_MODELS + type = "eval_zeroshot_cls" + tasks = { + "mxq": { + "type": type, + "configs": MXQ_CONFIGS, + }, + } + do_expermient_fdata(f"{type}_mxq", models, tasks) + + +def experiment_eval_mxq_combined(): + models = ALL_MODELS + equiv_mxq_configs = [ + ( + f"{bits:.2f}".replace(".", "_"), + HQQQuantConfig(mixed=True, budget=bits, quant_scale=True), + ) + for bits in [4.06, 4.10, 4.15, 4.19, 4.24, 4.28, 4.33] + ] + zeroshot_tasks = { + "mxq": { + "type": "eval_zeroshot_cls", + "configs": equiv_mxq_configs, + }, + } + linear_probe_tasks = { + "mxq": { + "type": "eval_linear_probe", + "configs": equiv_mxq_configs, + }, + } + do_expermient_fdata("eval_linear_probe_mxq2", models, linear_probe_tasks) + do_expermient_fdata("eval_zeroshot_cls_mxq2", models, zeroshot_tasks) + + +def main(): + logging.basicConfig( + format="%(asctime)s %(levelname)s [%(name)s] %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", + ) + + # experiment_quant_hqq() + # experiment_quant_mxq() + # experiment_zeroshot_eval_fp16() + # experiment_zeroshot_eval_mxq() + # experiment_eval_mxq_combined() + # experiment_eval_fp16_combined() + experiment_eval_hqq_comprehensive() + experiment_eval_mxq_358_memory_saving() + + +if __name__ == "__main__": + main() diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/clipbenchmark.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/clipbenchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..a54f52c164ae0a9d1396c8d971037cd90dea5dfe --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/clipbenchmark.py @@ -0,0 +1,109 @@ +import os +import sys +import time +from pathlib import Path + +from clip_benchmark.cli import get_parser_args, run + + +def eval_clip_benchmark( + task, + model_id, + result_dir, + quant_dir, + quant_config, + additional_args=[], +): + # build args for clip_benchmark + model_type = "open_clip_hqq" if quant_config else "open_clip" + comps = model_id.split("/") + elems = comps[1].split("-") + model_name = "-".join(elems[1:4]) + pretrained = "-".join(elems[4:]) + extra_args = "" + if quant_config: + mixed = quant_config.get("mixed", None) + if mixed: + budget = quant_config["budget"] + qmf = quant_config["quant_metrics_file"] + weight_algo = quant_config["weight_algo"] + boost_stop = quant_config["boost_stop"] + top_m_layer = quant_config["top_m_layer"] + extra_args = f"budget={budget},quant_metrics_file={qmf},weight_algo={weight_algo},boost_stop={boost_stop},top_m_layer={top_m_layer}" + else: + b = quant_config["weight_quant_params"]["nbits"] + g = quant_config["weight_quant_params"]["group_size"] + extra_args = f"nbits={b},group_size={g}" + wds_cache_dir = os.path.join(quant_dir, "wds-dataset") + Path(wds_cache_dir).mkdir(parents=True, exist_ok=True) + out_file = f"benchmark_{{dataset}}_{{pretrained}}_{{model}}_{{language}}_{{task}}-{model_type}.json" + out_fp = os.path.join(result_dir, out_file) + args_str = [ + "eval", + f"--task={task}", + f"--model_type={model_type}", + "--dataset=wds/imagenet1k", + "--dataset_root=https://huggingface.co/datasets/clip-benchmark/wds_{dataset_cleaned}/tree/main", + f"--output={out_fp}", + f"--wds_cache_dir={wds_cache_dir}", + f"--extra_args={extra_args}", + ] + if len(additional_args) > 0: + args_str.extend(additional_args) + t1 = time.time() + + # clear the command line args for main python program + # so that they are not passed to clip_benchmark CLI + sys.argv = [sys.argv[0]] + parser, _ = get_parser_args() + cli_args = parser.parse_args(args_str) + cli_args.dataset = cli_args.dataset[0] + if any(["--train_split" in arg for arg in additional_args]): + cli_args.train_split = cli_args.train_split[0] + cli_args.model = model_name + cli_args.pretrained = pretrained + result_dict = run(cli_args) + t2 = time.time() + result_dict["__lm_quant_tk_duration"] = t2 - t1 + return result_dict + + +def eval_zeroshot_classification(metric, model_id, result_dir, quant_dir, quant_config): + result_dict = eval_clip_benchmark( + "zeroshot_classification", + model_id, + result_dir, + quant_dir, + quant_config, + ) + metric["acc1_zeroshot_cls"] = result_dict["metrics"]["acc1"] + metric["acc5_zeroshot_cls"] = result_dict["metrics"]["acc5"] + metric["recall_zeroshot_cls"] = result_dict["metrics"]["mean_per_class_recall"] + metric["duration_zeroshot_cls"] = result_dict["__lm_quant_tk_duration"] + return metric + + +def eval_linear_probe( + metric, model_id, result_dir, quant_dir, quant_config, feature_root +): + additional_args = [ + "--batch_size=512", + "--fewshot_lr=0.1", + "--fewshot_epochs=20", + "--train_split=train", + "--test_split=test", + f"--feature_root={feature_root}", + ] + result_dict = eval_clip_benchmark( + "linear_probe", + model_id, + result_dir, + quant_dir, + quant_config, + additional_args, + ) + metric["acc1_linear_probe"] = result_dict["metrics"]["lp_acc1"] + metric["acc5_linear_probe"] = result_dict["metrics"]["lp_acc5"] + metric["recall_linear_probe"] = result_dict["metrics"]["lp_mean_per_class_recall"] + metric["duration_linear_probe"] = result_dict["__lm_quant_tk_duration"] + return metric diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/common.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/common.py new file mode 100644 index 0000000000000000000000000000000000000000..315e2f4030be4c08a887b5fdb7e3f05da05ab23a --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/common.py @@ -0,0 +1,239 @@ +import gc +import glob +import os +from datetime import datetime +from pathlib import Path + +import pandas as pd +import torch +from hqq.core.quantize import BaseQuantizeConfig as HQQQuantConfig +from hqq.utils.optimizer import find_optimal_configs + +from lm_quant_toolkit.utils.hub import LLAMA_MODELS, VIT_OPENCLIP_MODELS + +HQQ_CONFIGS = [ + ("b8g32", HQQQuantConfig(nbits=8, group_size=32, quant_scale=True)), + ("b8g64", HQQQuantConfig(nbits=8, group_size=64, quant_scale=True)), + ("b8g128", HQQQuantConfig(nbits=8, group_size=128, quant_scale=True)), + ("b4g32", HQQQuantConfig(nbits=4, group_size=32, quant_scale=True)), + ("b4g64", HQQQuantConfig(nbits=4, group_size=64, quant_scale=True)), + ("b4g128", HQQQuantConfig(nbits=4, group_size=128, quant_scale=True)), + ("b3g32", HQQQuantConfig(nbits=3, group_size=32, quant_scale=True)), + ("b3g64", HQQQuantConfig(nbits=3, group_size=64, quant_scale=True)), + ("b3g128", HQQQuantConfig(nbits=3, group_size=128, quant_scale=True)), + ("b2g16", HQQQuantConfig(nbits=2, group_size=16, quant_scale=True)), + ("b2g32", HQQQuantConfig(nbits=2, group_size=32, quant_scale=True)), + ("b2g64", HQQQuantConfig(nbits=2, group_size=64, quant_scale=True)), +] + + +def get_mxq_quant_meta_data_file(model_id): + short_id = model_id.split("/")[1] + data_dir = os.path.join(os.path.dirname(__file__), "..", "..", "data") + fp = os.path.join(data_dir, f"fnorm-{short_id}.csv") + return os.path.exists(fp), os.path.abspath(fp) + + +def persist_progress(df, progress_path): + """Save the progress of experiment for resumption.""" + Path(progress_path).parent.mkdir(parents=True, exist_ok=True) + df.to_csv(progress_path, index=False) + + +def save_partial_metric(experiment_name, algo, model_id, config, metric, result_dir): + metrics = [metric] + df = pd.DataFrame(metrics) + result_dir = os.path.join(result_dir, experiment_name) + os.makedirs(result_dir, exist_ok=True) + model_short_id = model_id.split("/")[1] + file_name = f"{result_dir}/partial-{algo}-{model_short_id}-{config}.csv" + df.to_csv(file_name, index=False) + + +def _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir): + mem_fp = f"{result_dir}/{experiment_name}/mem-snapshot-{algo}-{model_id.split('/')[1]}.pickle" + torch.cuda.memory._dump_snapshot(mem_fp) + + +def combine_metrics(experiment_name, result_dir): + dfs = [] + iters = glob.iglob(f"{result_dir}/{experiment_name}/partial-*.csv") + for it in iters: + df = pd.read_csv(it) + dfs.append(df) + combined = pd.concat(dfs) + ts_str = datetime.now().strftime("%Y%m%d%H%M%S") + file_name = f"{result_dir}/{experiment_name}/result-{experiment_name}-{ts_str}.csv" + Path(file_name).parent.mkdir(parents=True, exist_ok=True) + combined.to_csv(file_name, index=False) + + +def cleanup(model): + del model + torch.cuda.empty_cache() + gc.collect() + + +def calc_bits(b1, g1, b2=8, g2=128): + return b1 + 2 * b2 / g1 + 32 / g1 / g2 + + +def get_mxq_bits(reduction_pcts=[3, 5, 8]): + nbits = [] + for cfg in HQQ_CONFIGS: + bpp = calc_bits( + cfg[1]["weight_quant_params"]["nbits"], + cfg[1]["weight_quant_params"]["group_size"], + 8, + 128, + ) + nbits.extend([round(bpp * (1 - pct / 100), 2) for pct in reduction_pcts]) + return sorted(list(set(nbits)), reverse=True) + + +def _reset_peak_memory_stats(): + return torch.cuda.reset_peak_memory_stats() + + +def get_memory_metrics(): + return torch.cuda.max_memory_allocated(), torch.cuda.max_memory_reserved() + + +def plan_eval_bit_budgets( + model_arch="ViT", + points=5, + step=1, + bases=[4.51], + include_base=False, +): + for base in bases: + ideals, solvables = get_eval_plan(model_arch, base, points, step, include_base) + print("*" * 72) + print(f"base: {base}") + for t in zip(ideals, solvables): + print(f"ideal: {t[0]:.2f}, solvable: {t[1]:.2f}") + print("*" * 72) + + +def get_eval_plan(model_arch, base, points, step, include_base): + ideals = [] + solvables = [] + start = 0 if include_base else 1 + for point in range(start, points + 1): + tentative = round(base + point * step, 2) + ideals.append(tentative) + ret = try_solvable(model_arch, tentative, step) + if ret is not None: + solvables.append(ret) + else: + solvables.append(0.0) + return ideals, solvables + + +def try_solvable(model_arch, bit_budget, step): + if model_arch == "ViT": + model_ids = VIT_OPENCLIP_MODELS.keys() + else: + model_ids = [ + key for key in LLAMA_MODELS if LLAMA_MODELS[key].get("experiment", False) + ] + + dikt = {} + feasible_budget = round(bit_budget, 2) + for model_id in model_ids: + attempts = 1 + _, fp = get_mxq_quant_meta_data_file(model_id) + while True: + try: + # find_optimal_configs(fp, feasible_budget, time_limit=200) + find_optimal_configs( + fp, + feasible_budget, + time_limit=200, + weight_algo="sensi-milp", + ) + dikt[model_id] = feasible_budget + break + except ValueError: + print(f"Warning: {feasible_budget:.2f} unsolvable for model {model_id}") + if attempts > 3: + return None + feasible_budget += -0.01 if step < 0 else 0.01 + attempts += 1 + if len(set(dikt.values())) > 1: + for model_id, budget in dikt.items(): + if abs(budget - feasible_budget) > 0.01: + try: + fp = get_mxq_quant_meta_data_file(model_id) + find_optimal_configs(fp, feasible_budget, time_limit=200) + except ValueError: + print( + f"Warning: {feasible_budget:.2f} unsolvable for model {model_id}" + ) + return None + return feasible_budget + + +def debug_milp_solvable(model_id, bit_budget): + _, fp = get_mxq_quant_meta_data_file(model_id) + configs = find_optimal_configs(fp, bit_budget, time_limit=200) + print(configs) + + +def plan_432_bits(): + bases = [4.51, 4.25, 4.13] + plan_eval_bit_budgets( + model_arch="llm", points=5, step=0.02, bases=bases, include_base=True + ) + plan_eval_bit_budgets( + model_arch="llm", points=5, step=-0.02, bases=bases, include_base=True + ) + bases = [3.51, 3.25, 3.13, 2.51, 2.25, 2.13] + plan_eval_bit_budgets( + model_arch="llm", points=3, step=0.02, bases=bases, include_base=True + ) + plan_eval_bit_budgets( + model_arch="llm", points=3, step=-0.02, bases=bases, include_base=True + ) + + +def plan_567_bits(): + best_bit_budget = calc_bits(8, 32, 8, 128) + save_objs = [10, 20, 30, 40] + bases = [best_bit_budget * (100 - obj) / 100 for obj in save_objs] + plan_eval_bit_budgets( + model_arch="llm", points=5, step=0.02, bases=bases, include_base=True + ) + plan_eval_bit_budgets( + model_arch="llm", points=5, step=-0.02, bases=bases, include_base=True + ) + + +def fill_budget_gap(start, stop, step=0.02): + result = [] + d = start + step + while d < stop: + result.append(round(d, 2)) + d += step + return result + + +def fill_gaps(): + gaps = [] + # gaps.extend(fill_budget_gap(3.57, 4.03)) + # gaps.extend(fill_budget_gap(3.29, 3.45)) + # gaps.extend(fill_budget_gap(6.92, 7.56)) + # gaps.extend(fill_budget_gap(4.61, 5.00)) + # gaps.extend(fill_budget_gap(5.20, 5.86)) + gaps.extend(fill_budget_gap(6.01, 6.70)) + # gaps.extend([4.39, 4.37]) + print(gaps) + plan_eval_bit_budgets( + model_arch="llm", points=0, step=0.02, bases=gaps, include_base=True + ) + + +if __name__ == "__main__": + # fill_gaps() + plan_432_bits() + plan_567_bits() diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/leaderboard.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/leaderboard.py new file mode 100644 index 0000000000000000000000000000000000000000..7ce24adcbeaae001784397c0335eabd63d30314b --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/leaderboard.py @@ -0,0 +1,304 @@ +import json +import os +import shutil +import time +from pathlib import Path + +import numpy as np +from lm_eval import evaluator +from lm_eval.tasks import TaskManager + +from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir + +HIGHER_IS_BETTER_SYMBOLS = { + True: "↑", + False: "↓", +} + + +def make_table(result_dict, column: str = "results", sort_results: bool = False): + """Generate table of results.""" + from pytablewriter import LatexTableWriter, MarkdownTableWriter + + if column == "results": + column_name = "Tasks" + elif column == "groups": + column_name = "Groups" + + all_headers = [ + column_name, + "Version", + "Filter", + "n-shot", + "Metric", + "", + "Value", + "", + "Stderr", + ] + + md_writer = MarkdownTableWriter() + latex_writer = LatexTableWriter() + md_writer.headers = all_headers + latex_writer.headers = all_headers + + values = [] + + keys = result_dict[column].keys() + if sort_results: + # sort entries alphabetically by task or group name. + # NOTE: we default here to false, because order matters + # for multi-level table printing a la mmlu. + # sorting here would mess that up + keys = sorted(keys) + for k in keys: + dic = result_dict[column][k] + version = result_dict["versions"].get(k, " N/A") + n = str(result_dict.get("n-shot", " ").get(k, " ")) + higher_is_better = result_dict.get("higher_is_better", {}).get(k, {}) + + if "alias" in dic: + k = dic.pop("alias") + + metric_items = dic.items() + metric_items = sorted(metric_items) + + for (mf), v in metric_items: + m, _, f = mf.partition(",") + if m.endswith("_stderr"): + continue + + hib = HIGHER_IS_BETTER_SYMBOLS.get(higher_is_better.get(m), "") + + v = "%.4f" % v if isinstance(v, float) else v + + if m + "_stderr" + "," + f in dic: + se = dic[m + "_stderr" + "," + f] + se = " N/A" if se == "N/A" else "%.4f" % se + values.append([k, version, f, n, m, hib, v, "±", se]) + else: + values.append([k, version, f, n, m, hib, v, "", ""]) + k = "" + version = "" + md_writer.value_matrix = values + latex_writer.value_matrix = values + + # todo: make latex table look good + # print(latex_writer.dumps()) + + return md_writer.dumps() + + +def handle_non_serializable(o): + if isinstance(o, np.int64) or isinstance(o, np.int32): + return int(o) + elif isinstance(o, set): + return list(o) + else: + return str(o) + + +def eval_llm_leaderboard( + experiment_name, + model_id, + quant_method, + confg_name, + quantized, + metric, + quant_base_dir, + result_dir, + verbosity="INFO", +): + if quantized: + quant_dir = os.path.join( + quant_base_dir, f"{model_id}-{confg_name}-{quant_method.lower()}" + ) + # prepare the quantized model by copying tokenizer files + _prepare_tokenizer_files(model_id, quant_dir) + model_args = f"pretrained={quant_dir},quant_method={quant_method}" + else: + model_args = f"pretrained={model_id}" + + t1 = time.time() + task_manager = TaskManager(verbosity) + results = evaluator.simple_evaluate( + model="hf", + model_args=model_args, + tasks="leaderboard", + # num_fewshot=args.num_fewshot, + batch_size="auto:16", + max_batch_size=16, + device="cuda:0", + # use_cache=True, + # check_integrity=True, + write_out=False, + log_samples=True, + system_instruction=None, + # apply_chat_template=args.apply_chat_template, + fewshot_as_multiturn=False, + # gen_kwargs=args.gen_kwargs, + task_manager=task_manager, + verbosity=verbosity, + predict_only=False, + random_seed=0, + numpy_random_seed=1234, + torch_random_seed=1234, + fewshot_random_seed=1234, + ) + t2 = time.time() + + if results is not None: + lm_eval_result_fp = os.path.join( + result_dir, + experiment_name, + f"{model_id.split('/')[1]}-{confg_name}-{quant_method.lower()}-results.json", + ) + Path(lm_eval_result_fp).parent.mkdir(parents=True, exist_ok=True) + + results.pop("samples", None) + with open(lm_eval_result_fp, "w") as fh: + json.dump( + results, + fh, + indent=2, + default=handle_non_serializable, + ensure_ascii=False, + ) + print(make_table(results)) + + ifeval, bbh, mathlevel5, gpqa, musr, mmlupro = _cal_leaderboard_score(results) + metric["ifeval"] = ifeval + metric["bbh"] = bbh + metric["mathlevel5"] = mathlevel5 + metric["gpqa"] = gpqa + metric["musr"] = musr + metric["mmlupro"] = mmlupro + metric["duration_leaderboard"] = t2 - t1 + return metric + + +def _cal_leaderboard_score(results): + ifeval = _cal_leaderboard_ifeval_score(results) + bbh = _cal_leaderboard_bbh(results) + mathlevel5 = _cal_leaderboard_mathlevel5(results) + gpqa = _cal_leaderboard_gpqa(results) + musr = _cal_leaderboard_musr(results) + mmlupro = _cal_leaderboard_mmlu_pro(results) + return ifeval, bbh, mathlevel5, gpqa, musr, mmlupro + + +def _cal_leaderboard_gpqa(results): + res_gpqa = results["results"]["leaderboard_gpqa"] + value = res_gpqa.get("acc_norm,none", None) + # aggregate data from sub tasks + if value is None: + subtask_names = results["group_subtasks"]["leaderboard_gpqa"] + metrics = {} + for name in subtask_names: + metrics[name] = len(results["configs"][name]["doc_to_choice"]) + scores = [] + for metric, choices in metrics.items(): + value = results["results"][metric]["acc_norm,none"] + scores.append(_cal_normalized_score(value, 1 / choices, 1.0)) + return sum(scores) / len(scores) + + else: + return _cal_normalized_score(value, 1 / 4, 1.0) + + +def _cal_leaderboard_mathlevel5(results): + res_math_hard = results["results"]["leaderboard_math_hard"] + value = res_math_hard.get("exact_match,none", None) + # aggregate data from sub tasks + if value is None: + subtask_names = results["group_subtasks"]["leaderboard_math_hard"] + scores = [] + for metric in subtask_names: + value = results["results"][metric]["exact_match,none"] + scores.append(_cal_normalized_score(value, 0, 1.0)) + return sum(scores) / len(scores) + else: + return _cal_normalized_score(value, 0, 1.0) + + +def _cal_leaderboard_bbh(results): + subtask_names = results["group_subtasks"]["leaderboard_bbh"] + metrics = {} + for name in subtask_names: + metrics[name] = len(results["configs"][name]["doc_to_choice"]) + scores = [] + for metric, choices in metrics.items(): + value = results["results"][metric]["acc_norm,none"] + scores.append(_cal_normalized_score(value, 1 / choices, 1.0)) + return sum(scores) / len(scores) + + +# noqa refer to: https://huggingface.co/docs/leaderboards/open_llm_leaderboard/normalization#example-normalizing-musr-scores +def _cal_leaderboard_musr(results): + metrics = { + "leaderboard_musr_murder_mysteries": 2, + "leaderboard_musr_object_placements": 5, + "leaderboard_musr_team_allocation": 3, + } + scores = [] + for metric, choices in metrics.items(): + value = results["results"][metric]["acc_norm,none"] + scores.append(_cal_normalized_score(value, 1 / choices, 1.0)) + return sum(scores) / len(scores) + + +def _cal_leaderboard_mmlu_pro(results): + value = results["results"]["leaderboard_mmlu_pro"]["acc,none"] + return _cal_normalized_score(value, 0.1, 1.0) + + +def _cal_leaderboard_ifeval_score(results): + scores = [] + value1 = results["results"]["leaderboard_ifeval"]["inst_level_strict_acc,none"] + value2 = results["results"]["leaderboard_ifeval"]["prompt_level_strict_acc,none"] + scores.append(_cal_normalized_score(value1, 0, 1.0)) + scores.append(_cal_normalized_score(value2, 0, 1.0)) + return sum(scores) / len(scores) + + +def _cal_normalized_score(value, lower_bound, higher_bound=1.0): + if value < lower_bound: + return 0 + return 100 * (value - lower_bound) / (higher_bound - lower_bound) + + +def _prepare_tokenizer_files(model_id, quant_dir): + files = [ + "config.json", + "tokenizer.model", + "tokenizer.json", + "special_tokens_map.json", + "tokenizer_config.json", + ] + + base_dir = get_hf_model_storge_base_dir(model_id) + for f in files: + src_fp = os.path.join(base_dir, f) + if os.path.exists(src_fp): + dst_fp = os.path.join(quant_dir, f) + shutil.copyfile(src_fp, dst_fp, follow_symlinks=True) + + +def test_run(fp): + with open(fp) as fh: + results = json.load(fh) + ifeval, bbh, mathlevel5, gpqa, musr, mmlupro = _cal_leaderboard_score(results) + avg = (ifeval + bbh + mathlevel5 + gpqa + musr + mmlupro) / 6 + print(f"avg={avg:.2f}") + print(f"ifeval={ifeval:.2f}") + print(f"bbh={bbh:.2f}") + print(f"mathlevel5={mathlevel5:.2f}") + print(f"gpqa={gpqa:.2f}") + print(f"musr={musr:.2f}") + print(f"mmlupro={mmlupro:.2f}") + + +if __name__ == "__main__": + # noqa result3.json is a copy of https://huggingface.co/datasets/open-llm-leaderboard/meta-llama__Llama-2-7b-hf-details/raw/main/meta-llama__Llama-2-7b-hf/results_2024-06-16T18-52-55.970021.json + test_run("results/fp16_leaderboard/Llama-2-7b-hf-base-fp16-results.json") + test_run("logs/result3.json") diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/perplexity.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..0a4b9b955d7d76e1753fcef1176ba67d45ef1b99 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/eval/perplexity.py @@ -0,0 +1,125 @@ +import gc +import time + +import numpy as np +import torch +from datasets import load_dataset +from tqdm import tqdm + + +def cleanup(): + torch.cuda.empty_cache() + gc.collect() + + +def eval_ptb(model, tokenizer, max_length=1024, stride=512, verbose=True): + dataset = load_dataset("ptb_text_only", "penn_treebank", split="test") + return eval_ppl( + "ptb", + model, + tokenizer, + dataset, + text_column="sentence", + max_length=max_length, + stride=stride, + verbose=verbose, + ) + + +def eval_c4(model, tokenizer, max_length=1024, stride=512, verbose=True): + dataset = load_dataset( + "allenai/c4", + data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"}, + split="validation", + download_mode="reuse_dataset_if_exists", + ) + # pick first 1100 + dataset = dataset[:1100] + return eval_ppl( + "C4", + model, + tokenizer, + dataset, + text_column="text", + max_length=max_length, + stride=stride, + verbose=verbose, + ) + + +def eval_wikitext2(model, tokenizer, max_length=1024, stride=512, verbose=True): + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="test") + return eval_ppl( + "wikitext", + model, + tokenizer, + dataset, + text_column="text", + max_length=max_length, + stride=stride, + verbose=verbose, + ) + + +# Adapted from https://huggingface.co/transformers/v4.2.2/perplexity.html +def eval_ppl( + ds_type, + model, + tokenizer, + dataset, + text_column="text", + max_length=1024, + stride=512, + verbose=True, +): + model.eval() + tokenizer.pad_token = tokenizer.eos_token + tokenizer.padding_side = "right" + tokenizer.add_eos_token = False + + encodings = tokenizer("\n\n".join(dataset[text_column]), return_tensors="pt") + + encodings["input_ids"] = encodings["input_ids"].to("cuda") + + lls, t = [], [] + for i in tqdm( + range(0, encodings["input_ids"].size(1), stride), + desc=ds_type, + disable=not verbose, + ): + begin_loc = max(i + stride - max_length, 0) + end_loc = min(i + stride, encodings["input_ids"].size(1)) + trg_len = end_loc - i + input_ids = encodings["input_ids"][:, begin_loc:end_loc] + target_ids = input_ids.clone() + target_ids[:, :-trg_len] = -100 # ignore context + + t1 = time.time() + with torch.no_grad(): + log_likelihood = model(input_ids, labels=target_ids).loss * trg_len + torch.cuda.synchronize() + t2 = time.time() + t.append((t2 - t1)) + lls.append(log_likelihood) + + del input_ids, target_ids + + ppl = np.round(float(torch.exp(torch.stack(lls).sum() / end_loc)), 4) + pred_time = np.round(np.mean(t), 3) + if verbose: + print(f"{ds_type} perplexity: {ppl}, time: {pred_time} sec") + + del encodings + cleanup() + + return ppl, pred_time + + +def eval_ppls(model, tokenizer, metric): + ppl_wikitext, duration_wikitext = eval_wikitext2(model, tokenizer, verbose=True) + ppl_c4, duration_c4 = eval_c4(model, tokenizer, verbose=True) + metric["ppl_wikitext"] = ppl_wikitext + metric["ppl_c4"] = ppl_c4 + metric["duration_wikitext"] = duration_wikitext + metric["duration_c4"] = duration_c4 + return metric diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/__init__.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/lp_norm.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/lp_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..abc89cdf9fd83e44ac6bfeb44eda7acb4b0ba75d --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/lp_norm.py @@ -0,0 +1,51 @@ +import numpy as np +import pandas as pd +import torch +from hqq.core.quantize import Quantizer as hQuant + +from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir +from lm_quant_toolkit.utils.safetensors import get_tensor + + +def quant_hqq(tensor, nbits, group_size=64, lp_norm=0.7, optimize=True): + opt_params = { + "lp_norm": lp_norm, + "iters": 100, + "early_stop": False, + } + + wq, meta = hQuant.quantize( + tensor, nbits=nbits, group_size=group_size, optimize=optimize, **opt_params + ) + return hQuant.dequantize(wq, meta) + + +if __name__ == "__main__": + model_id = "meta-llama/Llama-2-7b-hf" + base_dir = get_hf_model_storge_base_dir(model_id) + + layer = 31 + module = "self_attn.o_proj" + matrix_name = f"model.layers.{layer}.{module}.weight" + w = get_tensor(matrix_name, base_dir) + + dick = [] + for lp_norm in np.linspace(0.1, 0.9, 9): + for b in [3, 4, 8]: + for g in [32, 64, 128]: + wq_hqq = quant_hqq(w, nbits=b, lp_norm=lp_norm, group_size=g) + norm_hqq = torch.norm(w - wq_hqq) + dick.append( + { + "model": model_id.split("/")[1], + "layer": layer, + "module": module, + "lp_norm": lp_norm, + "b": b, + "g": g, + "fnorm": norm_hqq.item(), + } + ) + print(f"FNorm HQQ(lp_norm={lp_norm:.2f}): {norm_hqq}") + df = pd.DataFrame(dick) + df.to_csv("lp_norm_tuning.csv", index=False) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/quant_sim.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/quant_sim.py new file mode 100644 index 0000000000000000000000000000000000000000..f13699aace4854ed8d014d7899f2e050a007e572 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/quant_sim.py @@ -0,0 +1,64 @@ +import pandas as pd +from hqq.utils.optimizer import find_optimal_configs + +from lm_quant_toolkit.eval.common import get_mxq_quant_meta_data_file + + +def dump_mxq_objectives(model_ids, bit_budgets, csv_fp="mxq-objectives.csv"): + dikt = [] + for model_id in model_ids: + short_id = model_id.split("/")[1] + _, fp = get_mxq_quant_meta_data_file(model_id) + for bit_budget in bit_budgets: + _, objective = find_optimal_configs(fp, bit_budget, time_limit=200) + dikt.append( + { + "model": short_id, + "bpp": bit_budget, + "fnorm": objective, + } + ) + + df = pd.DataFrame(dikt) + df.to_csv(csv_fp, index=False) + + +def dump_mxq_configs(model_ids, bit_budgets, csv_fp, weight_algo, factor): + dikt = [] + for model_id in model_ids: + short_id = model_id.split("/")[1] + for bit_budget in bit_budgets: + try: + _, fp = get_mxq_quant_meta_data_file(model_id) + kwargs = {"weight_algo": weight_algo, "factor": factor} + configs, objective = find_optimal_configs( + fp, + bit_budget, + time_limit=200, + **kwargs, + ) + for k, v in configs.items(): + comps = k.split(".", 1) + layer, module = comps[0], comps[1] + dikt.append( + { + "model": short_id, + "module": module, + "layer": layer, + "memmb": 0, # to work with the plot program + "bit_budget": bit_budget, + "b1": v[0], + "g1": v[1], + "b2": v[2], + "g2": v[3], + } + ) + except ValueError: + print(f"Warning: {bit_budget:.2f} unsolvable for model {model_id}") + df = pd.DataFrame(dikt) + df.to_csv(csv_fp, index=False) + + +if __name__ == "__main__": + bit_budgets = [4.51, 4.25, 4.13] + dump_mxq_configs(["meta-llama/Llama-2-7b-hf"], bit_budgets) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py new file mode 100644 index 0000000000000000000000000000000000000000..87cbfc89495b18c18e7ad4a4289acf6b23eaf2c8 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 + + +import re + +import pandas as pd +import torch +from hqq.core.quantize import Quantizer as hQuant +from safetensors.torch import save_file as safe_save +from scipy.stats import kurtosis +from torch import uint8 + +from lm_quant_toolkit.eval.common import calc_bits +from lm_quant_toolkit.utils.safetensors import get_tensor, get_tensor_dual + + +def load_weight(matrix_name, base_dir): + m = f"{matrix_name}.weight" + return get_tensor(m, base_dir) + + +def load_weight_dual(prefix, base_dir, st_file): + o = f"{prefix}.weight" + q = f"{prefix}.qweight" + return get_tensor_dual(o, q, base_dir, st_file) + + +def dequantize(wq, meta): + # Zero/Scale packed together + if "zero_scale" in meta: + zero_scale = meta["zero_scale"] + + if zero_scale.dtype == uint8: + meta["zero_q"], meta["scale_q"] = zero_scale[0], zero_scale[1] + else: + meta["zero"], meta["scale"] = zero_scale[0], zero_scale[1] + + if meta["quant_zero"]: + meta["zero"] = hQuant.dequantize(meta["zero_q"], meta["meta_zero"]) + + if meta["quant_scale"]: + meta["scale"] = hQuant.dequantize(meta["scale_q"], meta["meta_scale"]) + return hQuant.dequantize(wq, meta) + + +def restore_weight(matrix, state_dict): + key = matrix + if key in state_dict: + m_dikt = state_dict[key] + if "meta" in m_dikt: + meta_dict = m_dikt["meta"] + meta_scale_dict = meta_dict.get("meta_scale", None) + b1 = meta_dict["nbits"] + g1 = meta_dict["group_size"] + b2 = meta_scale_dict["nbits"] if meta_scale_dict else 8 + g2 = meta_scale_dict["group_size"] if meta_scale_dict else 128 + quant_config = { + "b1": b1, + "g1": g1, + "b2": b2, + "g2": g2, + } + wq = dequantize(m_dikt["W_q"], meta_dict) + return wq, quant_config + else: + return None, None + else: + return None, None + + +def save_compare_pair( + base_dir, quant_base_dir, quant_cfg, model_id, layers, output_dir +): + file_path = f"{quant_base_dir}/{model_id}-{quant_cfg}-hqq/qmodel.pt" + state_dict = torch.load(file_path, map_location="cpu") + + tensors = {} + metadata = {} + # walk thru the linear layers + # for each layer + # for each linear module + # load the original weight + # load the quantized weight and dequantized + # save the two matrix into a combined safetensors + for layer in range(layers): + matricies = [ + f"model.layers.{layer}.mlp.down_proj", + f"model.layers.{layer}.mlp.gate_proj", + f"model.layers.{layer}.mlp.up_proj", + f"model.layers.{layer}.self_attn.k_proj", + f"model.layers.{layer}.self_attn.o_proj", + f"model.layers.{layer}.self_attn.q_proj", + f"model.layers.{layer}.self_attn.v_proj", + ] + for matrix in matricies: + wq, quant_cfg = restore_weight(matrix, state_dict) + if wq is None: + # skip unquantized matrix + continue + wo = load_weight(matrix, base_dir) + tensors[f"{matrix}.weight"] = wo + tensors[f"{matrix}.qweight"] = wq + metadata[f"{matrix}.quant_cfg.b1"] = str(quant_cfg["b1"]) + metadata[f"{matrix}.quant_cfg.b2"] = str(quant_cfg["b2"]) + metadata[f"{matrix}.quant_cfg.g1"] = str(quant_cfg["g1"]) + metadata[f"{matrix}.quant_cfg.g2"] = str(quant_cfg["g2"]) + + output_fp = f"{output_dir}/{model_id}-cmp.safetensors" + safe_save(tensors, output_fp, metadata=metadata) + + +def compare_pair(model_id, layers, output_dir): + st_file = f"{output_dir}/{model_id}-cmp.safetensors" + for layer in range(layers): + matricies = [ + f"model.layers.{layer}.mlp.down_proj", + f"model.layers.{layer}.mlp.gate_proj", + f"model.layers.{layer}.mlp.up_proj", + f"model.layers.{layer}.self_attn.k_proj", + f"model.layers.{layer}.self_attn.o_proj", + f"model.layers.{layer}.self_attn.q_proj", + f"model.layers.{layer}.self_attn.v_proj", + ] + for matrix in matricies: + wo, wq = load_weight_dual(matrix, output_dir, st_file) + diff = torch.norm(wo - wq).item() + kurt_peason = kurtosis( + wo.numpy(), axis=None, fisher=False, bias=True, nan_policy="omit" + ) + kurt_fisher = kurtosis( + wo.numpy(), axis=None, fisher=True, bias=True, nan_policy="omit" + ) + # print(f"{matrix} FNorm Diff: {diff:.5f} Kurtosis: {kurt:.2f}") + print(f"{matrix},{diff:.5f},{kurt_fisher:.3f},{kurt_peason:.3f}") + + +def is_linear_module(key): + self_attns = ["q_proj", "v_proj", "k_proj", "o_proj"] + mlps = ["gate_proj", "up_proj", "down_proj"] + modules = self_attns + mlps + for module in modules: + if module in key: + return True + return False + + +def extract_quant_config(base_dir, model_id, config, algo="hqq"): + file_path = f"{base_dir}/{model_id}-{config}-{algo}/qmodel.pt" + dikt = torch.load(file_path, map_location="cpu") + quant_configs = {} + mem_fp16_all_total = 0 + mem_all_total = 0 + mem_quant_total = 0 + param_quant_total = 0 + param_all_total = 0 + # search quantized linear module with meta + for key in dikt.keys(): + m_dikt = dikt[key] + if is_linear_module(key): + if "meta" in m_dikt: + meta_dict = m_dikt["meta"] + meta_scale_dict = meta_dict.get("meta_scale", None) + shape = meta_dict["shape"] + b1 = meta_dict["nbits"] + g1 = meta_dict["group_size"] + b2 = meta_scale_dict["nbits"] if meta_scale_dict else 8 + g2 = meta_scale_dict["group_size"] if meta_scale_dict else 128 + param_count = shape[0] * shape[1] + param_quant_total += param_count + param_all_total += param_count + memmb = ( + (b1 + 2 * b2 / g1 + 32 / (g1 * g2)) * param_count / 8 / 1024 / 1024 + ) + mem_fp16_all_total += param_count * 2 / 1024 / 1024 + mem_quant_total += memmb + mem_all_total += memmb + quant_configs[key] = { + "b1": b1, + "g1": g1, + "b2": b2, + "g2": g2, + "memmb": memmb, + "param_cnt": param_count, + } + else: + w = m_dikt["weight"] + mem_all_total += w.numel() * 2 / 1024 / 1024 + mem_fp16_all_total += w.numel() * 2 / 1024 / 1024 + param_all_total += w.numel() + return ( + quant_configs, + mem_quant_total, + mem_all_total, + mem_fp16_all_total, + param_quant_total, + param_all_total, + ) + + +def get_mem_usage_df(model_ids, confs, base_dir): + dikts = [] + for model_id in model_ids: + for conf in confs: + configs, mem_quant_total, mem_all_total, mem_fp16_all_total, _, _ = ( + extract_quant_config(base_dir, model_id, conf) + ) + dikt = { + "model": model_id.split("/")[1], + "config": conf, + "mem_quant_total": mem_quant_total, + "mem_all_total": mem_all_total, + "mem_fp16_all_total": mem_fp16_all_total, + } + dikts.append(dikt) + df = pd.DataFrame(dikts) + return df + + +def dump_quant_allocation( + quant_dir, model_ids, confs, csv_fp="mxq-allot.csv", attempts=None, algo="mxq" +): + dikt = [] + pat = re.compile(r"model\.layers\.(\d+)\.(.+)") + pat2 = re.compile(r"b(\d+)g(\d+)") + loops = ["mxq1"] if attempts is None else attempts + for attempt in loops: + snapshot_dir = f"{quant_dir}/{attempt}" + for model_id in model_ids: + for conf in confs: + m2 = re.match(pat2, conf) + if m2: + b1 = int(m2.group(1)) + g1 = int(m2.group(2)) + bit_budget = round(calc_bits(b1, g1), 2) + else: + bit_budget = conf.replace("_", ".") + + ( + configs, + mem_quant_total, + mem_all_total, + mem_fp16_all_total, + param_quant_total, + param_all_total, + ) = extract_quant_config(snapshot_dir, model_id, conf, algo=algo) + for key, val in configs.items(): + matcher = re.match(pat, key) + if matcher: + layer = matcher.group(1) + module = matcher.group(2) + val["model"] = model_id.split("/")[1] + val["layer"] = layer + val["module"] = module + val["bit_budget"] = bit_budget + val["params_quant_tot"] = param_quant_total + val["params_all_tot"] = param_all_total + dikt.append(val) + columns = [ + "model", + "module", + "layer", + "memmb", + "param_cnt", + "bit_budget", + "b1", + "g1", + "b2", + "g2", + "params_quant_tot", + "params_all_tot", + ] + df = pd.DataFrame(dikt) + df.to_csv(csv_fp, index=False, columns=columns) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/__init__.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/fnorm.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/fnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..75d7abd215d4339a0ca5ac9444cf30027417b406 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/fnorm.py @@ -0,0 +1,298 @@ +import os +from timeit import default_timer as timer + +import pandas as pd +import torch +from hqq.core.quantize import Quantizer as hQuant + +from lm_quant_toolkit.utils.hub import ( + LLAMA_MODELS, + VIT_OPENCLIP_MODELS, + get_hf_model_storge_base_dir, +) +from lm_quant_toolkit.utils.safetensors import get_tensor + + +def quant_hqq(tensor, nbits, group_size=64, optimize=True): + wq, meta = hQuant.quantize( + tensor, nbits=nbits, group_size=group_size, optimize=optimize + ) + return hQuant.dequantize(wq, meta) + + +def calc_fnorm_vit( + state_dict, + model_type, + prefix, + layer, + module, + suffix, + nbits1, + gsizes1, + nbits2, + gsizes2, +): + dikts = [] + matrix_name = f"{prefix}.{layer}.{module}.{suffix}" + w = state_dict[matrix_name] + params = w.numel() + for nbit1 in nbits1: + for gsize1 in gsizes1: + for nbit2 in nbits2: + for gsize2 in gsizes2: + wq_hqq = quant_hqq(w, nbits=nbit1, group_size=gsize1, optimize=True) + norm_hqq = torch.norm(w - wq_hqq).item() + bpp = nbit1 + 2 * nbit2 / gsize1 + 32 / (gsize1 * gsize2) + memmb = bpp * params / 8 / (1024**2) + dikt = { + "layer": layer, + "module": f"{model_type}.{module}", + "nbit1": nbit1, + "gsize1": gsize1, + "nbit2": nbit2, + "gsize2": gsize2, + "fnorm": norm_hqq, + "memmb": memmb, + "params": params, + } + dikts.append(dikt) + return dikts + + +def calc_fnorm( + base_dir, prefix, layer, module, suffix, nbits1, gsizes1, nbits2, gsizes2 +): + dikts = [] + matrix_name = f"{prefix}.{layer}.{module}.{suffix}" + w = get_tensor(matrix_name, base_dir) + params = w.numel() + for nbit1 in nbits1: + for gsize1 in gsizes1: + for nbit2 in nbits2: + for gsize2 in gsizes2: + wq_hqq = quant_hqq(w, nbits=nbit1, group_size=gsize1, optimize=True) + norm_hqq = torch.norm(w - wq_hqq).item() + bpp = nbit1 + 2 * nbit2 / gsize1 + 32 / (gsize1 * gsize2) + memmb = bpp * params / 8 / (1024**2) + dikt = { + "layer": layer, + "module": module, + "nbit1": nbit1, + "gsize1": gsize1, + "nbit2": nbit2, + "gsize2": gsize2, + "fnorm": norm_hqq, + "memmb": memmb, + "params": params, + } + dikts.append(dikt) + return dikts + + +def calc_fnorm_for_vit_model(model_id, base_dir, layer_cfg): + # self_attns = ["out_proj"] + self_attns = [] + mlps = ["c_fc", "c_proj"] + nbits1 = [2, 3, 4, 8] + gsizes1 = [32, 64, 128] + nbits2 = [8] + gsizes2 = [128] + prefixes = [ + "visual.transformer.resblocks", + "transformer.resblocks", + ] + suffix = "weight" + dikts = [] + state_dict = torch.load( + os.path.join(base_dir, "open_clip_pytorch_model.bin"), + weights_only=True, + ) + for prefix in prefixes: + if "visual" in prefix: + model_type = "vision" + layers = layer_cfg["vlayers"] + else: + model_type = "text" + layers = layer_cfg["tlayers"] + for layer in range(layers): + for attn in self_attns: + ds = calc_fnorm_vit( + state_dict, + model_type, + prefix, + layer, + f"attn.{attn}", + suffix, + nbits1, + gsizes1, + nbits2, + gsizes2, + ) + dikts.extend(ds) + for mlp in mlps: + ds = calc_fnorm_vit( + state_dict, + model_type, + prefix, + layer, + f"mlp.{mlp}", + suffix, + nbits1, + gsizes1, + nbits2, + gsizes2, + ) + dikts.extend(ds) + df = pd.DataFrame(dikts) + file_name = f"data/fnorm-{model_id.split('/')[1]}.csv" + df.to_csv( + file_name, + columns=[ + "layer", + "module", + "nbit1", + "gsize1", + "nbit2", + "gsize2", + "fnorm", + "memmb", + "params", + ], + index=False, + ) + + +def calc_fnorm_for_model(model_id, base_dir, layers, output_dir="data"): + self_attns = ["q_proj", "v_proj", "k_proj", "o_proj"] + mlps = ["gate_proj", "up_proj", "down_proj"] + nbits1 = [2, 3, 4, 8] + gsizes1 = [32, 64, 128] + nbits2 = [8] + gsizes2 = [128] + prefix = "model.layers" + suffix = "weight" + dikts = [] + for layer in range(layers): + for attn in self_attns: + ds = calc_fnorm( + base_dir, + prefix, + layer, + f"self_attn.{attn}", + suffix, + nbits1, + gsizes1, + nbits2, + gsizes2, + ) + dikts.extend(ds) + for mlp in mlps: + ds = calc_fnorm( + base_dir, + prefix, + layer, + f"mlp.{mlp}", + suffix, + nbits1, + gsizes1, + nbits2, + gsizes2, + ) + dikts.extend(ds) + + df = pd.DataFrame(dikts) + file_name = f"{output_dir}/fnorm-{model_id.split('/')[1]}.csv" + df.to_csv( + file_name, + columns=[ + "layer", + "module", + "nbit1", + "gsize1", + "nbit2", + "gsize2", + "fnorm", + "memmb", + "params", + ], + index=False, + ) + + +def main_vit(): + for model_id, layer_cfg in VIT_OPENCLIP_MODELS.items(): + t1 = timer() + base_dir = layer_cfg.get("base_dir", None) + model_base_dir = get_hf_model_storge_base_dir(model_id, base_dir) + calc_fnorm_for_vit_model(model_id, model_base_dir, layer_cfg) + t2 = timer() + print(f"Finished {model_id} metrics calc in {t2 - t1} seconds") + + +def main(): + for model_id, model in LLAMA_MODELS.items(): + if model_id != "meta-llama/Llama-3.1-8B": + continue + t1 = timer() + base_dir = model.get("base_dir", None) + model_base_dir = get_hf_model_storge_base_dir(model_id, base_dir) + calc_fnorm_for_model(model_id, model_base_dir, model["layers"]) + t2 = timer() + print(f"Finished {model_id} metrics calc in {t2 - t1} seconds") + + +def join_kurtosis(): + for model_id, model in LLAMA_MODELS.items(): + exp = model.get("experiment", False) + if not exp or model_id == "meta-llama/Llama-3.1-8B": + continue + t1 = timer() + model_short_id = model_id.split("/")[1] + df_fnorm = pd.read_csv(f"data/fnorm-{model_short_id}.csv") + df_wdist = pd.read_csv(f"data/wdist-{model_short_id}.csv") + # calculate scaled kurtosis + df_kurt_agg = df_wdist.groupby("module").agg( + kurt_max=pd.NamedAgg(column="kurtosis", aggfunc="max"), + kurt_min=pd.NamedAgg(column="kurtosis", aggfunc="min"), + ) + df_wdist = df_wdist.merge(df_kurt_agg, how="left", on="module") + df_wdist["kurtosis_scaled"] = (df_wdist["kurtosis"] - df_wdist["kurt_min"]) / ( + df_wdist["kurt_max"] - df_wdist["kurt_min"] + ) + df_fnorm = df_fnorm[ + [ + "layer", + "module", + "nbit1", + "gsize1", + "nbit2", + "gsize2", + "fnorm", + "memmb", + "params", + ] + ] + df_fnorm = pd.merge(df_fnorm, df_wdist, how="inner", on=["module", "layer"]) + df_fnorm = df_fnorm[ + [ + "layer", + "module", + "nbit1", + "gsize1", + "nbit2", + "gsize2", + "fnorm", + "memmb", + "params", + "kurtosis", + "kurtosis_scaled", + ] + ] + + df_fnorm.to_csv(f"fnorm-{model_short_id}.csv", index=False) + t2 = timer() + print(f"Finished {model_id} Kurtosis data join in {t2 - t1} seconds") + + +if __name__ == "__main__": + join_kurtosis() diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py new file mode 100644 index 0000000000000000000000000000000000000000..d9f4630edb02ddceff9f21b96b1fd08ece69360d --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py @@ -0,0 +1,453 @@ +import functools +import gc +import inspect +import os +import re +import time +from collections import defaultdict +from typing import List + +import pandas as pd +import torch +import torch.nn as nn +from bitsandbytes.functional import dequantize_nf4 +from bitsandbytes.nn import Params4bit +from datasets import load_dataset +from hqq.core.quantize import Quantizer as hQuant +from tqdm import tqdm +from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers.models.llama.modeling_llama import LlamaDecoderLayer + + +# nbits is defined as placeholder to be consistent with other quant methods +def quant_nf4(tensor, nbits=4, group_size=64): + qw = Params4bit(tensor, blocksize=group_size, quant_type="nf4").to(tensor.device) + return dequantize_nf4(qw.data, qw.quant_state) + + +def quant_hqq(tensor, nbits, group_size=64, optimize=True): + wq, meta = hQuant.quantize( + tensor, nbits=nbits, group_size=group_size, optimize=optimize + ) + return hQuant.dequantize(wq, meta) + + +def get_named_linears(module): + return {name: m for name, m in module.named_modules() if isinstance(m, nn.Linear)} + + +def get_layers_for_scaling(module: LlamaDecoderLayer, input_feat, module_kwargs): + layers = [] + + # attention input + layers.append( + dict( + part="attn_in", + prev_op=module.input_layernorm, + layers=[ + module.self_attn.q_proj, + module.self_attn.k_proj, + module.self_attn.v_proj, + ], + inp=input_feat["self_attn.q_proj"], + module2inspect=module.self_attn, + kwargs=module_kwargs, + ) + ) + + # attention out + # Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696 + if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape: + layers.append( + dict( + part="attn_out", + prev_op=module.self_attn.v_proj, + layers=[module.self_attn.o_proj], + inp=input_feat["self_attn.o_proj"], + ) + ) + + # linear 1 + layers.append( + dict( + part="mlp_gate", + prev_op=module.post_attention_layernorm, + layers=[module.mlp.gate_proj, module.mlp.up_proj], + inp=input_feat["mlp.gate_proj"], + module2inspect=module.mlp, + ) + ) + + # linear 2 + layers.append( + dict( + part="mlp_down", + prev_op=module.mlp.up_proj, + layers=[module.mlp.down_proj], + inp=input_feat["mlp.down_proj"], + ) + ) + + return layers + + +def clear_memory(weight=None): + if weight is not None: + del weight + gc.collect() + torch.cuda.empty_cache() + + +def get_best_device(idx=None): + if os.environ.get("USE_CPU_FOR_SENSITIVITY", None) == "1": + return "cpu" + if torch.cuda.is_available(): + if idx is None: + return "cuda:0" + else: + return "cuda:" + str(idx % torch.cuda.device_count()) + elif torch.backends.mps.is_available(): + return "mps" + else: + return "cpu" + + +def get_calib_dataset( + data: str = "pileval", + tokenizer=None, + n_samples=512, + block_size=512, + split="train", + text_column="text", +): + if data == "pileval": + dataset = load_dataset("mit-han-lab/pile-val-backup", split="validation") + elif data == "wikitext": + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="validation") + elif data == "bos": + dataset = load_dataset("schnell18/branch-of-science", split="train") + elif data == "c4": + dataset = load_dataset( + "allenai/c4", + data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"}, + split="validation", + download_mode="reuse_dataset_if_exists", + ) + else: + dataset = load_dataset(data, split=split) + + dataset = dataset.shuffle(seed=42) + + samples = [] + n_run = 0 + for data in dataset: + if isinstance(data, list): + line_encoded = data + else: + line = data[text_column] + line = line.strip() + line_encoded = tokenizer.encode(line) + if len(line_encoded) > 512: + continue + sample = torch.tensor([line_encoded]) + if sample.numel() == 0: + continue + samples.append(sample) + n_run += 1 + if n_run == n_samples: + break + # now concatenate all samples and split according to block size + cat_samples = torch.cat(samples, dim=1) + n_split = cat_samples.shape[1] // block_size + return [ + cat_samples[:, i * block_size : (i + 1) * block_size] for i in range(n_split) + ] + + +class SensitiveLayerFinder: + def __init__( + self, + model, + model_name, + w_bit, + group_size, + tokenizer, + calib_data="pileval", + split="train", + text_column="text", + quant_method="hqq", + ) -> None: + self.model = model + self.model_name = model_name + self.tokenizer = tokenizer + self.w_bit = w_bit + self.group_size = group_size + self.calib_data = calib_data + self.split = split + self.text_column = text_column + self.quant_method = quant_method + self.layers, self.module_kwargs, self.inps = self.init_quant() + if self.quant_method == "rtn": + self.quant_func = functools.partial(quant_hqq, optimize=False) + elif self.quant_method == "hqq": + self.quant_func = functools.partial(quant_hqq, optimize=True) + elif self.quant_method == "bnb": + self.quant_func = quant_nf4 + + @torch.no_grad() + def measure(self, csv_fp): + dikts = [] + cfg = f"b{self.w_bit}g{self.group_size}" + for i in tqdm( + range(len(self.layers)), desc=f"{self.model_name}-{cfg}-{self.calib_data}" + ): + # Move module and inputs to correct device + common_device = next(self.layers[i].parameters()).device + if common_device is None or str(common_device) == "cpu": + best_device = get_best_device(i) + self.layers[i] = self.layers[i].to(best_device) + common_device = next(self.layers[i].parameters()).device + + if self.module_kwargs.get("position_ids") is not None: + self.module_kwargs["position_ids"] = self.module_kwargs[ + "position_ids" + ].to(common_device) + + if self.module_kwargs.get("attention_mask") is not None: + self.module_kwargs["attention_mask"] = self.module_kwargs[ + "attention_mask" + ].to(common_device) + + self.inps = self.inps.to(common_device) + named_linears = get_named_linears(self.layers[i]) + input_feat = self._get_input_feat(self.layers[i], named_linears) + clear_memory() + + module_config = get_layers_for_scaling( + self.layers[i], input_feat, self.module_kwargs + ) + + for layer in module_config: + part = layer.pop("part", "Unknown") + mse = self._measure_layer_sensitivity(self.layers[i], **layer) + dikts.append( + { + "dataset": self.calib_data, + "part": part, + "model": self.model_name, + "nbits": self.w_bit, + "group_size": self.group_size, + "layer": i, + "sensitivity": mse, + } + ) + + del module_config + del input_feat + clear_memory() + return dikts + + def _measure_layer_sensitivity( + self, + module, + prev_op, + layers: List[nn.Linear], + inp: torch.Tensor, + module2inspect=None, + kwargs={}, + ): + if module2inspect is None: + assert len(layers) == 1 + module2inspect = layers[0] + + # Put x on the right device + inp = inp.to(next(module2inspect.parameters()).device) + + module_kwargs = self._sanitize_kwargs(kwargs, module2inspect) + + fp16_output = module2inspect(inp, **module_kwargs) + if isinstance(fp16_output, tuple): + fp16_output = fp16_output[0] + + # Quantize the weights + for fc in layers: + # call quantization function + fc.weight.data = self.quant_func( + fc.weight.data, self.w_bit, self.group_size + ) + + # W * X + int_w_output = module2inspect(inp, **module_kwargs) + if isinstance(int_w_output, tuple): + int_w_output = int_w_output[0] + + # compute mean squared error (L2 norm) + mse = (fp16_output - int_w_output).float().pow(2).mean().item() + del fp16_output + del int_w_output + clear_memory() + return mse + + def init_quant(self, n_samples=128, seqlen=512): + modules = self.model.model.layers + samples = get_calib_dataset( + data=self.calib_data, + tokenizer=self.tokenizer, + n_samples=n_samples, + block_size=seqlen, + split=self.split, + text_column=self.text_column, + ) + samples = torch.cat(samples, dim=0) + + inps = [] + layer_kwargs = {} + + best_device = get_best_device() + modules[0] = modules[0].to(best_device) + self.model.model.embed_tokens = self.model.model.embed_tokens.to(best_device) + + # get input and kwargs to layer 0 + # with_kwargs is only supported in PyTorch 2.0 + # use this Catcher hack for now + class Catcher(nn.Module): + def __init__(self, module): + super().__init__() + self.module = module + + def forward(self, *args, **kwargs): + # assume first input to forward is hidden states + if len(args) > 0: + hidden_states = args[0] + del args + else: + first_key = list(kwargs.keys())[0] + hidden_states = kwargs.pop(first_key) + + inps.append(hidden_states) + layer_kwargs.update(kwargs) + raise ValueError # early exit to break later inference + + # patch layer 0 to catch input and kwargs + modules[0] = Catcher(modules[0]) + try: + self.model(samples.to(next(self.model.parameters()).device)) + except ValueError: # work with early exit + pass + modules[0] = modules[0].module # restore + + # Update the layer kwargs with `prepare_inputs_for_generation` method + # that takes care of everything to avoid unexpected errors. + layer_kwargs = self.model.prepare_inputs_for_generation(samples, **layer_kwargs) + # Pop the input_ids as they are not needed at all. + layer_kwargs.pop("input_ids") + + del samples + inps = inps[0] + + modules[0] = modules[0].cpu() + self.model.model.embed_tokens = self.model.model.embed_tokens.to("cpu") + + clear_memory() + + if layer_kwargs.get("attention_mask") is not None: + layer_kwargs["attention_mask"] = layer_kwargs["attention_mask"].to( + best_device + ) + + return modules, layer_kwargs, inps + + def _get_input_feat(self, layer, named_linears): + # firstly, get input features of all linear layers + def cache_input_hook(m, x, y, name, feat_dict): + x = x[0] + x = x.detach().cpu() + feat_dict[name].append(x) + + input_feat = defaultdict(list) + handles = [] + + for name in named_linears: + handles.append( + named_linears[name].register_forward_hook( + functools.partial(cache_input_hook, name=name, feat_dict=input_feat) + ) + ) + self.inps = self.inps.to(next(layer.parameters()).device) # in case multi-gpu + # get output as next layer's input + + # Sanitize the kwargs in case we use transformers version that contains + # kwargs that are not handled by the module. + # Useful for trust_remote_code models. + module_kwargs = self._sanitize_kwargs(self.module_kwargs, layer) + + self.inps = layer(self.inps, **module_kwargs)[0] + for h in handles: + h.remove() + # now solve for scaling and clipping + input_feat = {k: torch.cat(v, dim=0) for k, v in input_feat.items()} + + return input_feat + + def _sanitize_kwargs(self, inputs_kwargs, module): + """ + Remove the arguments that are not supported in the module's + forward pass to avoid breaking behaviour between different versions + of transformers. + + Args: + inputs_kwargs (`dict`): + The input dictionary to pass to the model layer + module (`torch.nn.Module`): + Target module to quantize. + """ + module_signature = inspect.signature(module.forward).parameters + sanitized_kwargs = {} + for k, v in inputs_kwargs.items(): + if k in module_signature: + sanitized_kwargs[k] = v + return sanitized_kwargs + + +def measure_sensitivity(models, quant_method, cfgs, calib_datasets, csv_fp): + pat = re.compile(r"b(\d)g(\d+)") + bgs = [] + for cfg in cfgs: + m = re.match(pat, cfg) + if m: + bgs.append((int(m.group(1)), int(m.group(2)))) + dikts = [] + for ds in calib_datasets: + for bg in bgs: + for model_path in models: + short_name = model_path.split("/")[1] + model = AutoModelForCausalLM.from_pretrained( + model_path, + torch_dtype=torch.float16, + device_map="auto", + offload_state_dict=False, + max_memory={0: "18GiB", "cpu": "60GiB"}, + ) + tokenizer = AutoTokenizer.from_pretrained(model_path, legacy=False) + finder = SensitiveLayerFinder( + model, + short_name, + bg[0], + bg[1], + tokenizer, + ds, + quant_method=quant_method, + ) + dikts.extend(finder.measure(csv_fp)) + clear_memory() + time.sleep(2) + + df = pd.DataFrame(dikts) + df.to_csv(csv_fp, index=False) + + +# with profile( +# activities=[ProfilerActivity.CPU], profile_memory=True, record_shapes=True +# ) as prof: +# finder.identify(csv_fp) +# print(prof.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=10)) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/wdist.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/wdist.py new file mode 100644 index 0000000000000000000000000000000000000000..38daabc635a10376bc6e09b5f3b9349bf0665232 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/prep/wdist.py @@ -0,0 +1,291 @@ +import re + +import numpy as np +import pandas as pd +from scipy.stats import kurtosis + +from lm_quant_toolkit.utils.hub import ( + LLAMA_MODELS, + VIT_OPENCLIP_MODELS, + get_hf_model_storge_base_dir, +) +from lm_quant_toolkit.utils.pickle import load_state_dict +from lm_quant_toolkit.utils.safetensors import get_tensor + + +def calculate_kurtosis_llm(model_id, base_dir, layers, output_dir): + modules = [ + "self_attn.q_proj", + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.o_proj", + "mlp.gate_proj", + "mlp.down_proj", + "mlp.up_proj", + ] + dikts = [] + for module in modules: + for layer in range(layers): + full_name = f"model.layers.{layer}.{module}.weight" + w = get_tensor(full_name, base_dir) + param_count = w.numel() + w = w.flatten().float().numpy() + kurt_pearson = kurtosis( + w, axis=None, fisher=False, bias=True, nan_policy="omit" + ) + dikt = { + "module": module, + "layer": layer, + "param_count": param_count, + "kurtosis": kurt_pearson, + } + dikts.append(dikt) + df = pd.DataFrame(dikts) + short_id = model_id.split("/")[1] + csv_fp = f"{output_dir}/kurtosis-{short_id}.csv" + df.to_csv(csv_fp, index=False) + + +def calculate_kurtosis_vit(model_id, model_cfg, output_dir): + modules = [ + "mlp.c_fc", + "mlp.c_proj", + ] + dikts = [] + state_dict = load_state_dict(model_id) + for layer_type, layers in model_cfg.items(): + if layer_type == "vlayers": + model_type = "vision" + prefix = "visual.transformer" + else: + model_type = "text" + prefix = "transformer" + for i, module in enumerate(modules): + for layer in range(layers): + full_name = f"{prefix}.resblocks.{layer}.{module}.weight" + w = state_dict[full_name] + w = w.flatten().float().numpy() + kurt_pearson = kurtosis( + w, axis=None, fisher=False, bias=True, nan_policy="omit" + ) + dikt = { + "module": f"{model_type}.{module}", + "layer": layer, + "kurtosis": kurt_pearson, + } + dikts.append(dikt) + df = pd.DataFrame(dikts) + short_id = model_id.split("/")[1] + csv_fp = f"{output_dir}/kurtosis-{short_id}.csv" + df.to_csv(csv_fp, index=False) + + +def summarize_vit_percentiles( + model_id, + model_cfg, + csv_fp, + wt_file="open_clip_pytorch_model.bin", +): + modules = [ + "attn.in_proj_bias", + "attn.in_proj_weight", + "attn.out_proj.bias", + "attn.out_proj.weight", + "ln_1.bias", + "ln_1.weight", + "ln_2.bias", + "ln_2.weight", + "mlp.c_fc.bias", + "mlp.c_fc.weight", + "mlp.c_proj.bias", + "mlp.c_proj.weight", + ] + dikts = [] + state_dict = load_state_dict(model_id) + for layer_type, layers in model_cfg.items(): + if layer_type == "vlayers": + model_type = "vision" + prefix = "visual.transformer" + else: + model_type = "text" + prefix = "transformer" + for i, module in enumerate(modules): + for layer in range(layers): + full_name = f"{prefix}.resblocks.{layer}.{module}" + w = state_dict[full_name] + param_count = w.numel() + w = w.flatten().float().numpy() + percentiles = np.percentile(np.abs(w), [0, 99, 99.9, 99.99, 100]) + kurt_pearson = kurtosis( + w, axis=None, fisher=False, bias=True, nan_policy="omit" + ) + dikt = { + "type": model_type, + "module": module, + "param_count": param_count, + "layer": layer, + "percentile_0": percentiles[0], + "percentile_99": percentiles[1], + "percentile_999": percentiles[2], + "percentile_9999": percentiles[3], + "percentile_100": percentiles[4], + "kurtosis": kurt_pearson, + } + dikts.append(dikt) + df = pd.DataFrame(dikts) + df.to_csv(csv_fp, index=False) + + +def summarize_llama_quantable_params(base_dir, layers, fp, llama2=True): + modules = { + "norm": {"layerwise": False, "quant": False}, + "lm_head": {"layerwise": False, "quant": False, "prefix": ""}, + "embed_tokens": {"layerwise": False, "quant": False}, + "input_layernorm": {"layerwise": True, "quant": False}, + "post_attention_layernorm": {"layerwise": True, "quant": False}, + "mlp.down_proj": {"layerwise": True, "quant": True}, + "mlp.gate_proj": {"layerwise": True, "quant": True}, + "mlp.up_proj": {"layerwise": True, "quant": True}, + "self_attn.k_proj": {"layerwise": True, "quant": True}, + "self_attn.o_proj": {"layerwise": True, "quant": True}, + "self_attn.q_proj": {"layerwise": True, "quant": True}, + "self_attn.v_proj": {"layerwise": True, "quant": True}, + } + if llama2: + modules["self_attn.rotary_emb"] = { + "layerwise": True, + "quant": False, + "suffix": "inv_freq", + } + dikts = [] + for module, data in modules.items(): + suffix = data.get("suffix", "weight") + if data["layerwise"]: + for layer in range(layers): + full_name = f"model.layers.{layer}.{module}.{suffix}" + w = get_tensor(full_name, base_dir) + param_count = w.numel() + dikt = { + "module": module, + "layer": layer, + "param_count": param_count, + "quant_count": param_count if data["quant"] else 0, + } + dikts.append(dikt) + else: + prefix = data.get("prefix", "model") + if prefix == "": + full_name = f"{module}.{suffix}" + else: + full_name = f"{prefix}.{module}.{suffix}" + w = get_tensor(full_name, base_dir) + param_count = w.numel() + dikt = { + "module": module, + "layer": 0, + "param_count": param_count, + "quant_count": param_count if data["quant"] else 0, + } + dikts.append(dikt) + df = pd.DataFrame(dikts) + df.to_csv(fp, index=False) + + +def summarize_vit_quantable_params( + model_id, + csv_fp, + wt_file="open_clip_pytorch_model.bin", +): + quantables = [ + "mlp.c_fc.weight", + "mlp.c_proj.weight", + ] + dikts = [] + state_dict = load_state_dict(model_id) + pat = r"(.*)\.resblocks\.(\d+)\.(.*)" + for key in state_dict: + m = re.match(pat, key) + if m is not None: + layer = int(m.group(2)) + if "visual" in m.group(1): + type = "vision" + elif "transformer" in m.group(1): + type = "text" + else: + type = "" + module = m.group(3) + else: + layer = 0 + module = key + if "visual" in key: + type = "vision" + elif "transformer" in key: + type = "text" + else: + type = "" + quantable = any([qnt in key for qnt in quantables]) + w = state_dict[key] + param_count = w.numel() + dikt = { + "type": type, + "module": module, + "layer": layer, + "param_count": param_count, + "quant_count": param_count if quantable else 0, + } + dikts.append(dikt) + df = pd.DataFrame(dikts) + df.to_csv(csv_fp, index=False) + + +def aggregate_quantable_parameters(models, base_dir="data", unit=1_000_000_000): + ret = {} + for model in models: + csv_file = f"{base_dir}/quantable-{model}.csv" + df = pd.read_csv(csv_file) + total = df["param_count"].sum() + quant = df["quant_count"].sum() + pct = quant / total + ret[model] = (total / unit, quant / unit, pct * 100) + return ret + + +def summarize_known_llama_quantable_params(base_dir="data", skip_models=None): + for model_id, cfg in LLAMA_MODELS.items(): + if skip_models is not None and len(skip_models) > 0: + if any([skip_model in model_id for skip_model in skip_models]): + continue + model_base_dir = get_hf_model_storge_base_dir( + model_id, cfg.get("base_dir", None) + ) + layers = cfg["layers"] + llama2 = cfg.get("llama2", True) + csv_file = f"{base_dir}/quantable-{model_id.split('/')[1]}.csv" + summarize_llama_quantable_params( + model_base_dir, layers, csv_file, llama2=llama2 + ) + + +def summarize_known_vit_quantable_params(base_dir="data", skip_models=None): + for model_id, cfg in VIT_OPENCLIP_MODELS.items(): + if skip_models is not None and len(skip_models) > 0: + if any([skip_model in model_id for skip_model in skip_models]): + continue + csv_file = f"{base_dir}/quantable-{model_id.split('/')[1]}.csv" + summarize_vit_quantable_params(model_id, csv_file) + + +def summarize_vit_wdist(base_dir="data"): + for model_id, cfg in VIT_OPENCLIP_MODELS.items(): + csv_file = f"data/wdist-{model_id.split('/')[1]}.csv" + summarize_vit_percentiles(model_id, cfg, csv_file) + + +def calculate_vit_kurtosis(): + for model_id, cfg in VIT_OPENCLIP_MODELS.items(): + calculate_kurtosis_vit(model_id, cfg, "data") + + +if __name__ == "__main__": + calculate_vit_kurtosis() + # summarize_known_vit_quantable_params() diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/__init__.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/hub.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..414e444f0331ceb53b35a8cf0a659fbdeacf180d --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/hub.py @@ -0,0 +1,67 @@ +import os + +from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE + +LLAMA_MODELS = { + "meta-llama/Llama-2-7b-hf": { + "layers": 32, + "experiment": True, + }, + "meta-llama/Llama-2-13b-hf": { + "layers": 40, + "experiment": True, + }, + "meta-llama/Meta-Llama-3-8B": { + "layers": 32, + "llama2": False, + "experiment": True, + }, + "meta-llama/Llama-3.1-8B": { + "layers": 32, + "llama2": False, + "experiment": False, + }, + "meta-llama/Meta-Llama-3-70B": { + "layers": 80, + "llama2": False, + "base_dir": "/data/hugginface/hub", + }, + "meta-llama/Llama-2-70b-hf": { + "layers": 80, + "base_dir": "/data/hugginface/hub", + }, + "meta-llama/Meta-Llama-3-70B-Instruct": { + "layers": 80, + "base_dir": "/data/hugginface/hub", + "llama2": False, + }, + "meta-llama/Meta-Llama-3.1-405B-Instruct": { + "layers": 126, + "base_dir": "/data/hugginface/hub", + "llama2": False, + }, +} + +VIT_OPENCLIP_MODELS = { + "laion/CLIP-ViT-B-32-laion2B-s34B-b79K": {"vlayers": 12, "tlayers": 12}, + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K": {"vlayers": 32, "tlayers": 24}, + "laion/CLIP-ViT-L-14-laion2B-s32B-b82K": {"vlayers": 24, "tlayers": 12}, +} + +SENSITIVITY_MODELS = [ + "meta-llama/Llama-2-7b-hf", + "meta-llama/Llama-2-7b-chat-hf", + "epfl-llm/meditron-7b", + "meta-llama/Llama-3.1-8B", + "meta-llama/Llama-3.1-8B-Instruct", +] + + +def get_hf_model_storge_base_dir(model_id, hf_hub_dir=None): + model_id_x = model_id.replace("/", "--") + cache_dir = hf_hub_dir if hf_hub_dir else HUGGINGFACE_HUB_CACHE + hf_model_dir = os.path.join(cache_dir, f"models--{model_id_x}") + ref_main_fp = os.path.join(hf_model_dir, "refs", "main") + with open(ref_main_fp) as fh: + commit_sha = fh.read().strip() + return os.path.join(hf_model_dir, "snapshots", commit_sha) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/pickle.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/pickle.py new file mode 100644 index 0000000000000000000000000000000000000000..67408bb639b04573c1d76af0156bae1d571b7140 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/pickle.py @@ -0,0 +1,18 @@ +import os + +import torch + +from .hub import get_hf_model_storge_base_dir + + +def dump_vit_weight_names(model_id, wt_file="open_clip_pytorch_model.bin"): + base_dir = get_hf_model_storge_base_dir(model_id) + state_dict = torch.load(os.path.join(base_dir, wt_file), weights_only=True) + for name in state_dict: + print(name) + + +def load_state_dict(model_id, wt_file="open_clip_pytorch_model.bin"): + base_dir = get_hf_model_storge_base_dir(model_id) + wt_file_fp = os.path.join(base_dir, wt_file) + return torch.load(wt_file_fp, weights_only=True) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/safetensors.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/safetensors.py new file mode 100644 index 0000000000000000000000000000000000000000..0c20b1a028418162b9e7311c881ff78fa1bb3ce0 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/utils/safetensors.py @@ -0,0 +1,43 @@ +import json +import os + +from safetensors import safe_open + + +def get_tensor(matrix_name, base_dir, index_json="model.safetensors.index.json"): + fp = os.path.join(base_dir, index_json) + with open(fp, "r") as fh: + index = json.load(fh) + try: + st_file = index["weight_map"][matrix_name] + mp = os.path.join(base_dir, st_file) + with safe_open(mp, framework="pt", device="cpu") as f: + return f.get_tensor(matrix_name) + except Exception: + raise ValueError(f"Invalid key {matrix_name}") + + +def get_tensor2(matrix_name, st_file_fp): + try: + with safe_open(st_file_fp, framework="pt", device="cpu") as f: + return f.get_tensor(matrix_name) + except Exception: + raise ValueError("Fail to retrieve tensor keys") + + +def get_tensor_keys(base_dir, st_file): + try: + mp = os.path.join(base_dir, st_file) + with safe_open(mp, framework="pt", device="cpu") as f: + return f.keys() + except Exception: + raise ValueError("Fail to retrieve tensor keys") + + +def get_tensor_dual(matrix1, matrix2, base_dir, st_file): + try: + mp = os.path.join(base_dir, st_file) + with safe_open(mp, framework="pt", device="cpu") as f: + return f.get_tensor(matrix1), f.get_tensor(matrix2) + except Exception: + raise ValueError(f"Invalid key {matrix1}") diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/combine-fnorm-llama.R b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/combine-fnorm-llama.R new file mode 100644 index 0000000000000000000000000000000000000000..ffeca2579759586fb6c8e59040b381382d547d42 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/combine-fnorm-llama.R @@ -0,0 +1,41 @@ +library(dplyr) +library(readr) + +strip_name <- function(name) { + start <- nchar("fnorm-") + 1 + stop <- nchar(name) - 4 + return(substr(name, start, stop)) +} + +fnorm_data_dir <- "../src/data/tmp" +fnorm_dir <- normalizePath(fnorm_data_dir) +fnorm_fps <- dir( + path = fnorm_dir, + pattern = paste0("fnorm-.*", "\\.csv$"), + full.names = TRUE +) +names(fnorm_fps) <- sapply((basename(fnorm_fps)), strip_name) +df_fnorm <- plyr::ldply( + fnorm_fps, + read.csv, + stringsAsFactors = FALSE, + .id = "model" +) + +k_cols <- c( + "model", + "module", + "layer", + "cfg", + "memmb", + "params", + "fnorm", + "sensitivity", + "kurtosis" +) +df_fnorm <- df_fnorm |> + dplyr::mutate( + cfg = paste0("b", nbit1, "g", gsize1) + ) |> + select(all_of(k_cols)) +write_csv(df_fnorm, "llama-fnorm.csv") diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/combine-wdist-llama.R b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/combine-wdist-llama.R new file mode 100644 index 0000000000000000000000000000000000000000..3c2c4b31e1d0a8a68ed21c1a3000d043cf9486ba --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/combine-wdist-llama.R @@ -0,0 +1,45 @@ +library(dplyr) +library(readr) + +strip_name <- function(name) { + start <- nchar("wdist-") + 1 + stop <- nchar(name) - 4 + return(substr(name, + start, + stop)) +} + +wdist_data_dir <- "../data-vis/data/wdist/tmp" +wdist_dir <- normalizePath(wdist_data_dir) +wdist_fps <- dir( + path = wdist_dir, + + pattern = paste0("wdist-.*", + "\\.csv$"), + + full.names = TRUE +) +names(wdist_fps) <- sapply((basename(wdist_fps)), + strip_name) +df_wdist <- plyr::ldply( + wdist_fps, + read.csv, + stringsAsFactors = FALSE, + .id = "model" +) + +k_cols <- c( + "model", + "module", + "param_count", + "layer", + "percentile_0", + "percentile_99", + "percentile_999", + "percentile_9999", + "percentile_100", + "kurtosis" +) +df_wdist <- df_wdist |> + select(all_of(k_cols)) +write_csv(df_wdist, "llama-wdist.csv") diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/dense.txt b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/dense.txt new file mode 100644 index 0000000000000000000000000000000000000000..ec991a0c6025dbce52d7883cf2b086a3276c944e --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/dense.txt @@ -0,0 +1,259 @@ +8.51 +8.25 +8.13 +7.76 +7.74 +7.72 +7.7 +7.68 +7.66 +7.63 +7.62 +7.6 +7.57 +7.56 +7.54 +7.53 +7.5 +7.48 +7.47 +7.44 +7.42 +7.41 +7.38 +7.36 +7.34 +7.32 +7.3 +7.29 +7.26 +7.24 +7.22 +7.2 +7.19 +7.16 +7.14 +7.13 +7.1 +7.08 +7.06 +7.04 +7.02 +7.01 +6.98 +6.96 +6.95 +6.92 +6.89 +6.87 +6.86 +6.83 +6.81 +6.78 +6.77 +6.75 +6.72 +6.71 +6.69 +6.68 +6.65 +6.63 +6.61 +6.59 +6.57 +6.55 +6.53 +6.51 +6.49 +6.47 +6.45 +6.43 +6.41 +6.39 +6.37 +6.35 +6.33 +6.31 +6.29 +6.27 +6.25 +6.23 +6.21 +6.19 +6.17 +6.15 +6.13 +6.11 +6.09 +6.07 +6.06 +6.05 +6.04 +6.03 +6.02 +6 +5.98 +5.96 +5.94 +5.92 +5.9 +5.88 +5.86 +5.84 +5.82 +5.8 +5.78 +5.76 +5.74 +5.72 +5.7 +5.68 +5.66 +5.64 +5.62 +5.6 +5.58 +5.56 +5.54 +5.52 +5.5 +5.48 +5.46 +5.44 +5.42 +5.4 +5.38 +5.36 +5.34 +5.32 +5.3 +5.28 +5.26 +5.24 +5.22 +5.2 +5.18 +5.16 +5.14 +5.12 +5.1 +5.08 +5.06 +5.04 +5.02 +5 +4.99 +4.97 +4.95 +4.93 +4.91 +4.89 +4.87 +4.85 +4.83 +4.81 +4.79 +4.77 +4.75 +4.73 +4.71 +4.69 +4.67 +4.65 +4.63 +4.61 +4.59 +4.57 +4.55 +4.53 +4.51 +4.49 +4.47 +4.45 +4.43 +4.41 +4.39 +4.37 +4.35 +4.33 +4.31 +4.29 +4.27 +4.25 +4.23 +4.21 +4.19 +4.17 +4.15 +4.13 +4.11 +4.09 +4.07 +4.05 +4.03 +4.01 +3.99 +3.97 +3.95 +3.93 +3.91 +3.89 +3.87 +3.85 +3.83 +3.81 +3.79 +3.77 +3.75 +3.73 +3.71 +3.69 +3.67 +3.65 +3.63 +3.61 +3.59 +3.57 +3.55 +3.53 +3.51 +3.49 +3.47 +3.45 +3.43 +3.42 +3.41 +3.39 +3.37 +3.35 +3.33 +3.31 +3.29 +3.27 +3.25 +3.23 +3.21 +3.19 +3.17 +3.15 +3.13 +3.11 +3.09 +3.07 +2.57 +2.55 +2.53 +2.51 +2.49 +2.47 +2.45 +2.31 +2.29 +2.27 +2.25 +2.23 +2.21 +2.19 +2.17 +2.15 +2.13 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/.gitignore b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..75d4de92603b2b16a801530e715126ff8744d5fc --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/.gitignore @@ -0,0 +1 @@ +settings.ini diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/.pdbrc b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/.pdbrc new file mode 100644 index 0000000000000000000000000000000000000000..79fbfe37d75d4a05ea1dbb2954eb2550d011d9e7 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/.pdbrc @@ -0,0 +1,2 @@ +# b generate.py:96 +b generate.py:108 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/debug-llama2-7b.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/debug-llama2-7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..db309055a0a9ef8d503b10a9968c4c726aebcc35 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/debug-llama2-7b.sh @@ -0,0 +1,5 @@ +python -m pdb generate.py \ + --model="meta-llama/Llama-2-7b-chat-hf" \ + --variant="vLLM" \ + --topic-file=topics-l1.txt \ + --trace diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/generate.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b92a3f7f49627e8ebd53ef010ef10d65c073d5 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/generate.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +import re +from argparse import ArgumentParser +from datetime import datetime +from timeit import default_timer as timer + +import pandas as pd +from langchain.prompts import PromptTemplate +from langchain.schema.output_parser import StrOutputParser +from langchain_community.llms import VLLMOpenAI +from langchain_core.output_parsers.list import CommaSeparatedListOutputParser + + +def generate_level1_topic(model, variant, topic, count=10): + output_parser = CommaSeparatedListOutputParser() + format_instructions = output_parser.get_format_instructions() + template = """ +Please divide the {topic} into {count} most important sub-topics. +Number each sub-topic. Give answers without any extra description +or explanation. {format_instructions} +""" + prompt = PromptTemplate( + template=template, + input_variables=["topic", "count"], + partial_variables={"format_instructions": format_instructions}, + ) + + llm = _get_llm_model(model, variant) + + chain = prompt | llm | output_parser + result = chain.invoke({"count": count, "topic": topic}) + return result + + +def generate_level2_topic(model, variant, topic): + template = """ +Please explain {topic} concisely in around 100 to 120 words. +Give the answer directly without any extra words such as +"sure", "certainly", "of course" etc. Don't begin the answer with +words such as "system", "expert" etc to indicate role. +""" + prompt = PromptTemplate( + template=template, + input_variables=["topic"], + ) + + llm = _get_llm_model(model, variant) + + chain = prompt | llm | StrOutputParser() + result = chain.invoke({"topic": topic}) + + return result + + +def parse_level1_output(topic, output): + pat = re.compile(r"^\d+\.\s+") + ret = [] + for topic in output: + m = re.match(pat, topic) + if m: + ret.append(topic[m.end(0) :]) + return ret + + +def parse_output(model, topic, text): + instructions = text.split("\n") + dict_list = [] + for inst in instructions: + if len(inst) < 60: + continue + ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + new_task = { + "topic": topic, + "timestamp": ts, + "text": inst, + } + dict_list.append(new_task) + return dict_list + + +def _get_llm_model(model, variant): + return VLLMOpenAI( + model_name=model, + openai_api_key="token-abc123", + openai_api_base="http://localhost:8000/v1", + temperature=0.6, + ) + + +def save_llm_raw_output(llm_output_dir, model, variant, topic, text): + short_id = model.split("/")[1] + topic_str = topic.replace(" ", "-").replace("/", "-") + file_name = f"{llm_output_dir}/output-{topic_str}-{short_id}-{variant}.log" + with open(file_name, "w") as writer: + writer.write(text) + + +def generate(plans, llm_output_dir, model="llama", variant="7b", trace=True): + t0 = timer() + dict_list = [] + for plan in plans: + ti0 = timer() + topic = plan["topic"] + output = generate_level1_topic(model, variant, topic, count=plan["count"]) + ti1 = timer() + sub_topics = parse_level1_output(topic, output) + for sub_topic in sub_topics: + output = generate_level2_topic(model, variant, sub_topic) + if trace: + save_llm_raw_output( + llm_output_dir, + model, + variant, + sub_topic, + output, + ) + print(f"{sub_topic} generation took {(ti1 - ti0):.2f} seconds!") + records = parse_output(model, sub_topic, output) + dict_list.extend(records) + df = pd.DataFrame(dict_list) + short_id = model.split("/")[1] + file_name = f"data-{short_id}-{variant}.csv" + df.to_csv( + file_name, columns=["topic", "text", "timestamp"], index=False, doublequote=True + ) + + t1 = timer() + if trace: + print(f"All instructions generation took {t1 - t0} seconds") + + +def _parse_args(): + # Create the parser + parser = ArgumentParser(description="Kubernetes SFT Dataset Generator") + + # Add arguments + parser.add_argument( + "-m", + "--model", + required=True, + help="LLM model name, eg: claude, gpt, mistral, llama2", + ) + parser.add_argument( + "-v", "--variant", required=True, help="model variant such as opus, sonnet" + ) + parser.add_argument( + "-d", "--trace", action="store_true", default=False, help="Print trace messages" + ) + parser.add_argument( + "--topic-file", + default="topics.txt", + help="The Kubernetes topics for generated instructions", + ) + parser.add_argument( + "--llm-output-dir", + default="results", + help="The directory to save raw LLM output", + ) + + # Parse the arguments + args = parser.parse_args() + return args + + +def _load_topics(file): + plans = [] + with open(file, "r") as f: + for line in f.readlines(): + topic, count = line.split(":") + topic = topic.replace('"', "") + plans.append( + { + "topic": topic, + "count": int(count.rstrip()), + } + ) + return plans + + +if __name__ == "__main__": + args = _parse_args() + print(args) + + model = args.model + variant = args.variant + trace = args.trace + llm_output_dir = args.llm_output_dir + plans = _load_topics(args.topic_file) + + generate(plans, llm_output_dir, model, variant, trace) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/llama2-7b.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/llama2-7b.sh new file mode 100644 index 0000000000000000000000000000000000000000..bce8f6635751fc04dbd531419568a99f2d33a642 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/llama2-7b.sh @@ -0,0 +1,5 @@ +./generate.py \ + --model="meta-llama/Llama-2-7b-chat-hf" \ + --variant="vLLM" \ + --topic-file=topics-l1.txt \ + --trace diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/split.sh b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/split.sh new file mode 100644 index 0000000000000000000000000000000000000000..74db5607f9959b0ac339cf72bf329450d9102d09 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/split.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +split -a 1 -n 5 data-Llama-2-7b-chat-hf-vLLM.csv data- +cat data-a data-b data-c data-d > train.csv +head -1 data-Llama-2-7b-chat-hf-vLLM.csv > test.csv +cat data-e >> test.csv +rm -f data-a data-b data-c data-d data-e diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/topics-l1.txt b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/topics-l1.txt new file mode 100644 index 0000000000000000000000000000000000000000..f173e6a19440ae883c9ba5568afc559fc229268b --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/gen-ds-bos/topics-l1.txt @@ -0,0 +1,25 @@ +"Math":15 +"Engineering":15 +"Computer Science":15 +"Chemstry":15 +"Physics":15 +"Geography":10 +"History":7 +"English":9 +"Psychology":8 +"Politics":8 +"Law":10 +"Art":10 +"Music":10 +"Photography":5 +"Medical":10 +"Biology":10 +"Business":10 +"Agiculture":10 +"Architecture":10 +"Journalism":10 +"Oceanography":5 +"Social Science":5 +"Ecology":5 +"Astronomy":5 +"Life Sciences":5 diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/join.R b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/join.R new file mode 100644 index 0000000000000000000000000000000000000000..0344fdfc11e1f00b90df8fa1955a84e6733228d4 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/join.R @@ -0,0 +1,19 @@ +#!/usr/bin/env Rscript + +library(tidyverse) +library(dplyr) +library(readr) + +join_kurt <- function(model_id) { + fnorm_csv_fp <- paste0("fnorm-", model_id, ".csv") + kurt_csv_fp <- paste0("kurtosis-", model_id, ".csv") + df1 <- read_csv(fnorm_csv_fp) + df2 <- read_csv(kurt_csv_fp) + df <- df1 |> + left_join(df2, by = c("module", "layer")) + write_csv(df, fnorm_csv_fp) +} + +join_kurt("CLIP-ViT-B-32-laion2B-s34B-b79K") +join_kurt("CLIP-ViT-L-14-laion2B-s32B-b82K") +join_kurt("CLIP-ViT-H-14-laion2B-s32B-b79K") diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/pre-commit b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/pre-commit new file mode 100644 index 0000000000000000000000000000000000000000..38cdc1a3b54613ef0544f8f662292e5febd55cd9 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/pre-commit @@ -0,0 +1,11 @@ +#!/bin/sh -e + +rootdir=`git rev-parse --show-toplevel` +( + cd $rootdir + python -m venv .pre-commit-venv + .pre-commit-venv/bin/python -m pip install tox + rm -rf dist .cache *.egg-info _build + find ./ -name '*.pyc' -delete + .pre-commit-venv/bin/tox $@ +) diff --git a/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/typeset-bit-budget.py b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/typeset-bit-budget.py new file mode 100644 index 0000000000000000000000000000000000000000..db3cc4b538812d95b34ccab1bb47e550dccb5e15 --- /dev/null +++ b/lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/utils/typeset-bit-budget.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + + +def to_latex(bits, nbit, cols=10): + v = len(bits) // cols + rows = v if len(bits) % cols == 0 else v + 1 + for i, bit in enumerate(bits): + if i == 0: + print("\\midrule") + print( + "\\multirowcell{" + str(rows) + "}{\\textbf{" + str(nbit) + "-bit}} ", + end="", + ) + print(f"& {bit:.2f} ", end="") + if (i + 1) % cols == 0: + print(r" \\") + elif i == len(bits) - 1: + padding = " &" * (cols - (i + 1) % cols) + print(f"{padding} \\\\") + + +def main(ncols=10): + head = f""" +\\begin{{table*}} +\\centering +\\caption[SensiMiLP/KurtMiLP Experiment Settings]{{ +The SensiMiLP/KurtMiLP dense experiment bit budget settings. +}} +\\label{{tab:sensi-kurt-milp-settings}} +\\begin{{tabular}}{{r{'l' * ncols}}} +\\toprule +""" + + heading = r"\textbf{Group} & " + fields = [f" \\textbf{{V{i}}}" for i in range(1, ncols + 1)] + heading += "&".join(fields) + heading += r" \\" + + tail = r""" +\bottomrule +\end{tabular} +\end{table*} +""" + + with open("dense.txt", "r") as fh: + bits = [float(f) for f in fh.readlines()] + bits8 = [f for f in bits if f >= 8.0] + bits7 = [f for f in bits if f >= 7.0 and f < 8.0] + bits6 = [f for f in bits if f >= 6.0 and f < 7.0] + bits5 = [f for f in bits if f >= 5.0 and f < 6.0] + bits4 = [f for f in bits if f >= 4.0 and f < 5.0] + bits3 = [f for f in bits if f >= 3.0 and f < 4.0] + bits2 = [f for f in bits if f >= 2.0 and f < 3.0] + + print(head) + print(heading) + to_latex(bits8, 8, ncols) + to_latex(bits7, 7, ncols) + to_latex(bits6, 6, ncols) + to_latex(bits5, 5, ncols) + to_latex(bits4, 4, ncols) + to_latex(bits3, 3, ncols) + to_latex(bits2, 2, ncols) + print(tail) + + +if __name__ == "__main__": + main(12) diff --git a/lm-quant-toolkit/scripts/.pdbrc b/lm-quant-toolkit/scripts/.pdbrc new file mode 100644 index 0000000000000000000000000000000000000000..3688ced3e7ae9c062a40d0b0e9b9c03e822fcbe5 --- /dev/null +++ b/lm-quant-toolkit/scripts/.pdbrc @@ -0,0 +1,43 @@ +# AutoAWQ 0.2.5 two-device error break points +# b /home/justin/work/AutoAWQ/awq/models/base.py:170 +# b /home/justin/work/AutoAWQ/awq/quantize/quantizer.py:61 + +# debug GPU memory measuring issue +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py:242 + +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py:39 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py:46 +# b /home/justin/miniconda3/envs/quant-eval/lib/python3.11/site-packages/transformers/modeling_utils.py:955, param_device == 'cpu' +# b /home/justin/miniconda3/envs/quant-eval/lib/python3.11/site-packages/transformers/modeling_utils.py:973, param_device == 'cpu' +# b /home/justin/work/transformers/modeling_utils.py:989 +# b /home/justin/work/transformers/src/transformers/modeling_utils.py:4795 +# b /home/justin/work/transformers/src/transformers/modeling_utils.py:4213 +# b /home/justin/work/AutoAWQ/awq/models/base.py:338 +# b /home/justin/work/accelerate/src/accelerate/big_modeling.py:420 +# b /home/justin/work/accelerate/src/accelerate/hooks.py:579, module_name == 'model.layers.28.mlp.gate_proj' +# b /home/justin/work/accelerate/src/accelerate/hooks.py:579, module_name == 'model.layers.28' +# b /home/justin/work/accelerate/src/accelerate/hooks.py:288 +# b /home/justin/work/accelerate/src/accelerate/hooks.py:308 +# b /home/justin/work/accelerate/src/accelerate/hooks.py:327 +# b /home/justin/miniconda3/envs/quant-eval/lib/python3.11/site-packages/transformers/modeling_utils.py:4194 +# b /home/justin/miniconda3/envs/quant-eval/lib/python3.11/site-packages/transformers/quantizers/quantizer_bnb_4bit.py:190 +# b /home/justin/work/transformers/src/transformers/modeling_utils.py:4194 +# b /home/justin/work/hqq/hqq/utils/optimizer.py:21 + +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py:216 +# b /home/justin/work/lm-quant-toolkit/src/cli.py:436 +# b /home/justin/work/hqq/hqq/utils/optimizer.py:19 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py:206, i == 25 +# b /home/justin/work/hqq/hqq/models/base.py:262 +# b /home/justin/work/hqq/hqq/utils/optimizer.py:26 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py:395 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py:413 +# b /home/justin/work/hqq/hqq/utils/optimizer.py:26 +# b /home/justin/work/hqq/hqq/core/quantize.py:116 +# b /home/justin/work/hqq/hqq/core/quantize.py:175 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py:24 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py:256 +# b /home/justin/work/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py:270 +# b /home/justin/work/hqq/hqq/utils/optimizer.py:157 +b /home/justin/work/CLIP_benchmark/clip_benchmark/models/open_clip_hqq.py:13 +b /home/justin/work/hqq/hqq/models/open_clip/base.py:63 diff --git a/lm-quant-toolkit/scripts/debug-dump-sensi-rtn.sh b/lm-quant-toolkit/scripts/debug-dump-sensi-rtn.sh new file mode 100644 index 0000000000000000000000000000000000000000..8457d1bd602349244b2f348db379c9010df62bbd --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-dump-sensi-rtn.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +QUANT_METHODS="rtn" +# QUANT_METHODS="hqq" +CALIB_DATASETS="pileval" +CONFIGS="b4g64" +MODELS="meta-llama/Llama-2-7b-hf" + +RESULT_BASE_DIR="/fdata/llm/mxq/results" +EXP_NAME=debug_hqq_vs_rtn_sensi_llama +RESULT_DIR=$RESULT_BASE_DIR/$EXP_NAME +mkdir -p $RESULT_DIR/data + +for QNT_MTD in $QUANT_METHODS; do + for DS in $CALIB_DATASETS; do + for CFG in $CONFIGS; do + for MODEL in $MODELS; do + SHORT_ID=$(echo $MODEL | cut -d/ -f2) + OUT_FILE="$RESULT_DIR/${QNT_MTD}-${SHORT_ID}-${CFG}-${DS}.csv" + python -m pdb ../src/dump.py sensi \ + --model $MODEL \ + --quant-method $QNT_MTD \ + --config $CFG \ + --calib-dataset $DS \ + --output-file $OUT_FILE + done + done + done +done diff --git a/lm-quant-toolkit/scripts/debug-llama-hqq-baseline.sh b/lm-quant-toolkit/scripts/debug-llama-hqq-baseline.sh new file mode 100644 index 0000000000000000000000000000000000000000..55eef8e622099f4ca240ac63a91c6132d5614844 --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llama-hqq-baseline.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +BUDGETS="3.51" +CFGS="b3g32" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +ATTEMPT="debug-hqq-memory" +EXP_BASE_NAME=$ATTEMPT +EXP_NAME="${ATTEMPT}" +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +MODELS="0" + +echo "=========Run memory evaluation on batch ${EXP_NAME}=========" +algo=hqq +model_ids=$MODELS +for m in $model_ids; do + for cfg in ${CFGS}; do + python -m pdb ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done +done diff --git a/lm-quant-toolkit/scripts/debug-llama-sensi-7b-cfg-wise-5.sh b/lm-quant-toolkit/scripts/debug-llama-sensi-7b-cfg-wise-5.sh new file mode 100644 index 0000000000000000000000000000000000000000..163d9cd01211c5a417eebdb7eddfaa5cdcc7f76b --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llama-sensi-7b-cfg-wise-5.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +ATTEMPT="debug-sensi-cfg-wise-7b5" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-milp + +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +MODELS="0" +EXP_NAME="${ATTEMPT}" +echo "=========Run perplexity evaluation on Llama-2-7b=========" +python -m pdb ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file diff --git a/lm-quant-toolkit/scripts/debug-llama-sensi-boost.sh b/lm-quant-toolkit/scripts/debug-llama-sensi-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..04f3bda7bba414ff9aede15fe042fdbcfbcc8af1 --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llama-sensi-boost.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +BOOST_STOP=2 +ATTEMPT="sensi-boost-debug" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-boost + +log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +MODELS="0 1 2" +# MODELS="0" +EXP_NAME="${ATTEMPT}" +echo "=========Run perplexity evaluation=========" +python -m pdb ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-stop $BOOST_STOP \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi diff --git a/lm-quant-toolkit/scripts/debug-llama-sensi-milp-abl.sh b/lm-quant-toolkit/scripts/debug-llama-sensi-milp-abl.sh new file mode 100644 index 0000000000000000000000000000000000000000..7db083b66b2a46af62f8fb367e40cbfcda10142c --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llama-sensi-milp-abl.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +BUDGETS="6.89" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-milp +MODELS="2" +MODEL_NAMES="Meta-Llama-3-8B" + + +BOOST_TOP_MS="2" + +for BOOST_TOP_M in $BOOST_TOP_MS; do + ATTEMPT="sensi-milp-abl-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python -m pdb ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --top-m-layer $BOOST_TOP_M \ + --ablation \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs ]; then + mkdir pdfs + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + cd pdfs + pdflatex table.tex + cd .. + + # plot configuration allocations for 3 * 12 MXQ combinations + MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B" + BUDGETS=${MXQ_BATCH[@]:1} + for model in $MODELS; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --attempt1 mxq1 \ + --attempt2 $ATTEMPT \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR + +done + diff --git a/lm-quant-toolkit/scripts/debug-llama-sensi-milp.sh b/lm-quant-toolkit/scripts/debug-llama-sensi-milp.sh new file mode 100644 index 0000000000000000000000000000000000000000..a61e701d1e6446f8c5c310a57e04eb0047cd3f0f --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llama-sensi-milp.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +BUDGETS="6.89" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 +weight_algo=sensi-milp +MODELS="0" +MODEL_NAMES="Llama-2-7b-hf" + + +BOOST_TOP_MS="2" + +for BOOST_TOP_M in $BOOST_TOP_MS; do + ATTEMPT="debug2-sm-${BOOST_TOP_M}" + EXP_BASE_NAME=$ATTEMPT + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + + log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + + mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT + mkdir -p $RESULT_DIR/${EXP_NAME}_ppl + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + + # MODELS="0" + EXP_NAME="${ATTEMPT}" + echo "=========Run perplexity evaluation=========" + python ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --top-m-layer $BOOST_TOP_M \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE + fi + echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_ppl \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT + + echo "=========Dump quantization configs on batch ${EXP_NAME}=========" + mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT + python ../src/cli.py dump \ + --type quant_config \ + --model $MODELS \ + --budget ${BUDGETS} \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \ + 2>&1 \ + | tee -a $log_file + + echo "=========Run memory evaluation on batch ${EXP_NAME}=========" + algo=mxq + model_ids=$MODELS + for m in $model_ids; do + for cfg in ${BUDGETS}; do + python ../src/cli.py llm \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name "${EXP_NAME}_stor" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file + done + done + echo "=========Collect memory evaluation result on batch ${EXP_NAME}=========" + find $RESULT_DIR/${EXP_NAME}_stor \ + -name "result-*.csv" \ + -printf '%T@ %p\n' \ + | sort -n \ + | tail -1 \ + | cut -d' ' -f2 \ + | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT + + # echo "=========Delete quantized models of batch ${batch_name}=========" + # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr + + OLD_DIR=$(pwd) + cd $RESULT_DIR/$EXP_BASE_NAME + if [ ! -d pdfs/allot ]; then + mkdir -p pdfs/allot + fi + $OLD_DIR/../data-vis/combine.R \ + --baseline_data_dir $OLD_DIR/../data-vis/data \ + --mxq_data_dir data + $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv + $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv + $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv + $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT + pdflatex table.tex + + # plot configuration allocations for 3 * 12 MXQ combinations + for model in $MODEL_NAMES; do + for budget in $BUDGETS; do + $OLD_DIR/../data-vis/plot-mxq-allocation.R \ + -m $model \ + -b $budget \ + --attempt1 $ATTEMPT \ + --attempt2 mxq1 \ + --fnorm_data_dir $OLD_DIR/../src/data \ + --quant_cfg_allot_file data/quant-cfg-allocation.csv + done + done + cd $OLD_DIR +done + diff --git a/lm-quant-toolkit/scripts/debug-llama2-13b-boost.sh b/lm-quant-toolkit/scripts/debug-llama2-13b-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..5b8255008cba7b92e15c108a38f611ce950ee094 --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llama2-13b-boost.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +BUDGETS="4.13 4.25 4.51" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +MODELS="1" +ATTEMPT="debug-Llama2-13B-boost" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-directive +boost_layers="3 39" +EXP_NAME="${ATTEMPT}" +log_file="logs/bench-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +python -m pdb ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --boost-layer $boost_layers \ + --boost-stop 2 \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file diff --git a/lm-quant-toolkit/scripts/debug-llm-ppl-13b-awq.sh b/lm-quant-toolkit/scripts/debug-llm-ppl-13b-awq.sh new file mode 100644 index 0000000000000000000000000000000000000000..7daa036c2a8e20d7f32da00a1a88ea5b0273af9d --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llm-ppl-13b-awq.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python -m pdb ../src/cli.py llm \ + --task eval_ppl \ + --model 1 \ + --algo awq \ + --config b4g32 b4g64 b4g128 \ + --experiment-name eval_ppl_13b-awq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log diff --git a/lm-quant-toolkit/scripts/debug-llm-stor-bnb.sh b/lm-quant-toolkit/scripts/debug-llm-stor-bnb.sh new file mode 100644 index 0000000000000000000000000000000000000000..ec23d86977f19381c258e5eeeb60a6e21c506e75 --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-llm-stor-bnb.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +algo=bnb +model_ids="1" +cfgs="b4g64" +for m in $model_ids; do + for cfg in $cfgs ; do + python -m pdb ../src/cli.py llm \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + --model $m \ + --algo ${algo} \ + --config ${cfg} \ + --task eval_model_storage \ + --experiment-name debug_model_storge-bnb2 \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log + done +done + diff --git a/lm-quant-toolkit/scripts/debug-milp-low-bit.sh b/lm-quant-toolkit/scripts/debug-milp-low-bit.sh new file mode 100644 index 0000000000000000000000000000000000000000..a5f8813a854218639889229de0a4bbb49769f3b3 --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-milp-low-bit.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51" +# BUDGETS="4.17 4.29 4.56 4.21 4.34 4.60" +# BUDGETS="4.09 4.46" +# BUDGETS="4.25" +# BUDGETS="3.12 3.25 3.51" +BUDGETS="4.25" +RESULT_DIR="/fdata/llm/mxq/results" +QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots" + +FACTOR=2 +ATTEMPT="ts-lowbit-mix-sensi" +EXP_BASE_NAME=$ATTEMPT +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor} + +# Use cached dataset to speedup wikitext, c4 ppl evaluation +export HF_DATASETS_OFFLINE=1 + +weight_algo=sensi-milp + +log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log" + +mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT +mkdir -p $RESULT_DIR/${EXP_NAME}_ppl +mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT + +# MODELS="0 1 2" +MODELS="0" +EXP_NAME="${ATTEMPT}" +echo "=========Run perplexity evaluation=========" +# python ../src/cli.py llm \ +python -m pdb ../src/cli.py llm \ + --task eval_ppl \ + --model $MODELS \ + --algo mxq \ + --weight-algo $weight_algo \ + --factor $FACTOR \ + --config ${BUDGETS} \ + --experiment-name "${EXP_NAME}_ppl" \ + --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \ + --result-dir=$RESULT_DIR \ + 2>&1 \ + | tee -a $log_file +EXIT_CODE=$? +if [ $EXIT_CODE -ne 0 ]; then + echo "Perplexity evaluation failed!" + exit $EXIT_CODE +fi diff --git a/lm-quant-toolkit/scripts/debug-mistral-sensi-metrics.sh b/lm-quant-toolkit/scripts/debug-mistral-sensi-metrics.sh new file mode 100644 index 0000000000000000000000000000000000000000..00e510fc3c52dd4569448c1e6aa30fb35672ff7d --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-mistral-sensi-metrics.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +RESULT_BASE_DIR="/fdata/llm/mxq/results" +CALIB_DATASETS="pileval" +CONFIGS="b4g64" +MODELS="mistralai/Ministral-8B-Instruct-2410" + +EXP_NAME=sensi_mistral +RESULT_DIR=$RESULT_BASE_DIR/$EXP_NAME +mkdir -p $RESULT_DIR/data + +for DS in $CALIB_DATASETS; do + for CFG in $CONFIGS; do + for MODEL in $MODELS; do + SHORT_ID=$(echo $MODEL | cut -d/ -f2) + OUT_FILE="${RESULT_DIR}/data/qwen25-sensi-${SHORT_ID}-${CFG}-${DS}.csv" + python -m pdb ../src/dump.py sensi \ + --model $MODEL \ + --config $CFG \ + --calib-dataset $DS \ + --output-file $OUT_FILE + done + done +done + + +OLD_DIR=$(pwd) +cd $RESULT_DIR +if [ ! -d pdfs ]; then + mkdir -p pdfs +fi +$OLD_DIR/../data-vis/plot-variant-sensi.R data/ diff --git a/lm-quant-toolkit/scripts/debug-sens-dump-13b.sh b/lm-quant-toolkit/scripts/debug-sens-dump-13b.sh new file mode 100644 index 0000000000000000000000000000000000000000..c4482c49bef4911356b6d3b8d40820cf260005c9 --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-sens-dump-13b.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +CFG=b4g64 +DS=pileval +MODEL=1 +OUT_FILE="debug-13b.csv" + +python -m pdb ../src/cli.py dump \ + --type sensitivity \ + --model $MODEL \ + --config $CFG \ + --calib-dataset $DS \ + --output-file $OUT_FILE diff --git a/lm-quant-toolkit/scripts/debug-vit-zs-mxq-kurt-boost.sh b/lm-quant-toolkit/scripts/debug-vit-zs-mxq-kurt-boost.sh new file mode 100644 index 0000000000000000000000000000000000000000..2227e89d5d005ffb7504d371919f6a6cad661d5d --- /dev/null +++ b/lm-quant-toolkit/scripts/debug-vit-zs-mxq-kurt-boost.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ ! -d logs ]; then + mkdir logs +fi + +python -m pdb ../src/cli.py vit \ + --task eval_zeroshot_cls \ + --model 0 1 \ + --config 4.51 4.25 4.13 3.51 3.25 3.13 \ + --algo mxq \ + --weight-algo kurt-boost \ + --boost-stop 2 \ + --top-m-layer 1 \ + --experiment-name eval_zs_BH_mxq_kurt_boost \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/debug-bench-vit-$(date +%Y%m%d%H%M%S).log + diff --git a/lm-quant-toolkit/scripts/dump-cfgs-for-check.sh b/lm-quant-toolkit/scripts/dump-cfgs-for-check.sh new file mode 100644 index 0000000000000000000000000000000000000000..68c411560814a797dd3f7d70b63e9dccebbafa6c --- /dev/null +++ b/lm-quant-toolkit/scripts/dump-cfgs-for-check.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python ../src/cli.py dump \ + --type quant_config \ + --model 0 1 2 \ + --attempt mxq1 kurt-global kurt-scaled pct5 pct6 \ + --budget 3.51 3.25 3.13 4.51 4.25 4.13 \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --output_file mxq-mem-bound-check.csv diff --git a/lm-quant-toolkit/scripts/dump-mxq-obj.sh b/lm-quant-toolkit/scripts/dump-mxq-obj.sh new file mode 100644 index 0000000000000000000000000000000000000000..a6803e02b442b54203abfae60d99c16583915830 --- /dev/null +++ b/lm-quant-toolkit/scripts/dump-mxq-obj.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +python ../src/cli.py dump \ + --type objective \ + --model 0 1 2 \ + --budget 6.03 6.05 6.07 6.09 6.11 6.13 6.15 6.17 6.19 6.21 6.23 6.25 6.27 6.29 6.31 6.33 6.35 6.37 6.39 6.41 6.43 6.45 6.47 6.49 6.51 6.53 6.55 6.57 6.59 6.61 6.63 6.65 6.68 6.69 4.61 4.59 4.57 4.55 4.53 4.51 4.49 4.47 4.45 4.43 4.41 4.35 4.33 4.31 4.29 4.27 4.25 4.23 4.21 4.19 4.17 4.15 4.13 4.11 4.09 4.07 4.05 4.03 3.57 3.55 3.53 3.51 3.49 3.47 3.45 3.31 3.29 3.27 3.25 3.23 3.21 3.19 3.17 3.15 3.13 3.11 3.09 3.07 2.57 2.55 2.53 2.51 2.49 2.47 2.45 2.31 2.29 2.27 2.25 2.23 2.21 2.19 2.17 2.15 2.13 7.76 7.74 7.72 7.70 7.68 7.66 7.63 7.62 7.60 7.57 7.56 6.92 6.89 6.87 6.86 6.83 6.81 6.78 6.77 6.75 6.72 6.71 6.06 6.04 6.02 6.00 5.98 5.96 5.94 5.92 5.90 5.88 5.86 5.20 5.18 5.16 5.14 5.12 5.10 5.08 5.06 5.04 5.02 5.00 6.98 6.96 6.95 5.86 5.84 5.82 5.80 5.78 5.76 5.74 5.72 5.70 5.68 5.66 5.64 5.62 5.60 5.58 5.56 5.54 5.52 5.50 5.48 5.46 5.44 5.42 5.40 5.38 5.36 5.34 5.32 5.30 5.28 5.26 5.24 5.22 4.99 4.97 4.95 4.93 4.91 4.89 4.87 4.85 4.83 4.81 4.79 4.77 4.75 4.73 4.71 4.69 4.67 4.65 4.63 4.39 4.37 4.03 4.01 3.99 3.97 3.95 3.93 3.91 3.89 3.87 3.85 3.83 3.81 3.79 3.77 3.75 3.73 3.71 3.69 3.67 3.65 3.63 3.61 3.59 3.43 3.41 3.39 3.37 3.35 3.33 3.31 7.56 7.54 7.53 7.50 7.48 7.47 7.44 7.42 7.41 7.38 7.36 7.34 7.32 7.30 7.29 7.26 7.24 7.22 7.20 7.19 7.16 7.14 7.13 7.10 7.08 7.06 7.04 7.02 7.01 8.13 8.25 8.51 3.42 \ + --output_file mxq-objectives.csv diff --git a/lm-quant-toolkit/scripts/dump-quant-cfg-allot.sh b/lm-quant-toolkit/scripts/dump-quant-cfg-allot.sh new file mode 100644 index 0000000000000000000000000000000000000000..dae9d94d55c2b7dee4be11d5fb0068f942825302 --- /dev/null +++ b/lm-quant-toolkit/scripts/dump-quant-cfg-allot.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +CONFIGS="2.13 2.25 2.51 3.51 3.25 3.13 4.51 4.25 4.13" +QUANT_SNAPSHOT_BASE_DIR="/fdata/llm/mxq/snapshots" + +ATTEMPT="mxq1" +QUANT_SNAPSHOT_DIR="$QUANT_SNAPSHOT_BASE_DIR/$ATTEMPT" +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --config $CONFIGS \ + --experiment-name quant-mxq-allot-$ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log + +python ../src/cli.py dump \ + --type quant_config \ + --model 0 1 2 \ + --budget $CONFIGS \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_BASE_DIR \ + --output_file quant-cfg-allot-$ATTEMPT.csv + +ATTEMPT="kurt-scaled" +QUANT_SNAPSHOT_DIR="$QUANT_SNAPSHOT_BASE_DIR/$ATTEMPT" +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo mxq \ + --weight-algo $ATTEMPT \ + --config $CONFIGS \ + --experiment-name quant-mxq-allot-$ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log + + +python ../src/cli.py dump \ + --type quant_config \ + --model 0 1 2 \ + --budget $CONFIGS \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_BASE_DIR \ + --output_file quant-cfg-allot-$ATTEMPT.csv + + +CONFIGS="b2g64 b2g32 b3g128 b3g64 b3g32 b4g128 b4g64 b4g32" +QUANT_SNAPSHOT_DIR="$QUANT_SNAPSHOT_BASE_DIR/hqq" +python ../src/cli.py llm \ + --task quant \ + --model 0 1 2 \ + --algo hqq \ + --config $CONFIGS \ + --experiment-name quant-hqq-allot \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log + + +ATTEMPT=hqq +python ../src/cli.py dump \ + --type quant_config \ + --model 0 1 2 \ + --budget $CONFIGS \ + --attempt $ATTEMPT \ + --quant-snapshot-dir=$QUANT_SNAPSHOT_BASE_DIR \ + --output_file quant-cfg-allot-hqq.csv diff --git a/lm-quant-toolkit/scripts/eval-llm-ppl-2bit-dense1-mxq.sh b/lm-quant-toolkit/scripts/eval-llm-ppl-2bit-dense1-mxq.sh new file mode 100644 index 0000000000000000000000000000000000000000..dc064eeff48cbcfc49441ac7fc9f040d9d13045b --- /dev/null +++ b/lm-quant-toolkit/scripts/eval-llm-ppl-2bit-dense1-mxq.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# export HF_HOME=/data/hugginface +# conda activate quant-eval + +if [ ! -d logs ]; then + mkdir logs +fi + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +python ../src/cli.py llm \ + --task eval_ppl \ + --model 0 1 2 \ + --algo mxq \ + --config 2.57 2.55 2.53 2.51 2.49 2.47 2.45 2.31 2.29 2.27 2.25 2.23 2.21 2.19 2.17 2.15 2.13 \ + --experiment-name eval_ppl_2bit_dense1-mxq \ + --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \ + --result-dir="/fdata/llm/mxq/results" \ + 2>&1 \ + | tee logs/bench-$(date +%Y%m%d%H%M%S).log + diff --git a/models/Llama-3.1-8B-quantization-baselines/model-00001-of-00004.safetensors b/models/Llama-3.1-8B-quantization-baselines/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f11a94d08e8b093fa47af0049a5793be51adea46 --- /dev/null +++ b/models/Llama-3.1-8B-quantization-baselines/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e90c0528fee933b6868c1fc47a94c74f7db9e16bf0ddb591a95365554005fa2e +size 4976698592 diff --git a/models/Llama-3.1-8B-quantization-baselines/model-00002-of-00004.safetensors b/models/Llama-3.1-8B-quantization-baselines/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c97fdcee11d95cb568547259c4395fe68c71cd49 --- /dev/null +++ b/models/Llama-3.1-8B-quantization-baselines/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f42a20e22477309d2fe63e5a8296375fc139343f0f2bea399d3cfde96028a139 +size 4999802616 diff --git a/models/Llama-3.1-8B-quantization-baselines/model-00003-of-00004.safetensors b/models/Llama-3.1-8B-quantization-baselines/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2f51502ade2140724d4c918d0b2d7bbb8714627f --- /dev/null +++ b/models/Llama-3.1-8B-quantization-baselines/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1585d0147642972431e77f9ac8b8c9e50f7d2ced73183419f87e1b1ed7f1c4be +size 4915916080 diff --git a/models/Qwen/Qwen2.5-14B/model-00001-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00001-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0504bafae1ddcbc85d48285c87c8321cbaff57fc --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00001-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e015e2bc9a26b4e46d77913d8c667608ae7e48aa1eca04af5786c2408f4bc0fa +size 3885134104 diff --git a/models/Qwen/Qwen2.5-14B/model-00002-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00002-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c9a026a826f7bcb4b1206bf4953c9690e20c2875 --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00002-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7efa1adc8257218813dcb494bb2a3d5775fa268735ab39e5b8119e233c21462 +size 3995327992 diff --git a/models/Qwen/Qwen2.5-14B/model-00003-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00003-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..383698ee64c0b8e0ec66f9ffc0f75af212855bee --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00003-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68127c9bc4fab170f7aaf63d5c7ac9e182afd10b74a1c6bb8025afefc11447cb +size 3995328064 diff --git a/models/Qwen/Qwen2.5-14B/model-00004-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00004-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f52854c5a074379894d35c1a1ba7fc3c95b852e4 --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00004-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7250708a789da850ff40a4a5be335971dfa0d2bd7cba2e9905916dab06744d75 +size 3995348776 diff --git a/models/Qwen/Qwen2.5-14B/model-00005-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00005-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..885c5514c7a9ebb84bfecbccb6f7281ef4592b6d --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00005-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2754167c1cbaf94b9af9c7eb646a2286a596f9ded5e2e3c4c5e6a4464352c9e +size 3979614448 diff --git a/models/Qwen/Qwen2.5-14B/model-00006-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00006-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..03bbc4b3b2b35894d533efe6514b73480926ea27 --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00006-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7045cf78c68178b626546982d12b6e9c8e289f1bf1e65c42225ed13e07847180 +size 3995328064 diff --git a/models/Qwen/Qwen2.5-14B/model-00007-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00007-of-00008.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e238a7a4dec4d38d336806d198ab64df5b042d76 --- /dev/null +++ b/models/Qwen/Qwen2.5-14B/model-00007-of-00008.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d870a72c656404ee7524163ba996bf55050fff252dfe639a90715a9e2c47dba +size 3995328064 diff --git a/models/Qwen/Qwen2.5-32B/model-00001-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00001-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f9bc9a9dbb04341fa4bcb2649cc3541c89159efa --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00001-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c569737aa83e2e4fcc6a1ca9d33941008bc9e7571cb5816646bbb57a17ecaf62 +size 3916519128 diff --git a/models/Qwen/Qwen2.5-32B/model-00002-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00002-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3bcaa0fd0e60874abffb663c1418276d4450c525 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00002-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc884500edb17d9eda00ce41ba97775f1f0ab3bbf33d17256f8c647cdb4be952 +size 3900847488 diff --git a/models/Qwen/Qwen2.5-32B/model-00003-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00003-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..683625e43bdd7865e3322276a5bde94e991be1da --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00003-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e03d773a356510d40d38f25d1d36179feeff7b14a92dea3e11913820d9a37b9c +size 3900847488 diff --git a/models/Qwen/Qwen2.5-32B/model-00004-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00004-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6b934e031859a0204512a4ff786046f3497ede0f --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00004-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa15074d9d26f4a9fb98e63bd3e1ca3437ca0e3de016d815c548341b9561ea45 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00005-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00005-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..97bdbee4543c6554cbb5f00fcfbced79bf21ca89 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00005-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a022b584cab2278b488c5ed49e7e12cbd79ef4a6bf108fc22df056fae2e79d +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00006-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00006-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d83ff751c6fbd0ee4cec735c2605d8d0dfb50cda --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00006-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a18d92eb21c30e887ec52d6105094607b393e56adace973470877cc5ffa14f14 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00007-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00007-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c54e0a6e94b5f5de0dc8766892bd3a5d2aede687 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00007-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:810803fe0312a192fedb8f2028143357df963149ba2a430b971b6c10f7606b50 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00008-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00008-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d1c4e77124345a73df3290a80e7107a06219f958 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00008-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b896e75cb3b4d2cee8bf1de2ffb52e41d706afc597a8f7d83c5b1682e6b8c003 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00009-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00009-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..51d59b68d12d81251958b056eced014b86ce47f1 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00009-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273fc11a86b09ae342c9beb6441022b2a4c60333305332d21b6e894460c0bd2b +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00010-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00010-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70db4e808fc0e45b32fb13094b6aa2d8fc0c98b7 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00010-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d890c08b9200ce0001e7cefd9367f65662675432fccc08def619b96659714e +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00011-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00011-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d4664efac363796e4b3a9151e8624233321a4141 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00011-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:054170d4fb6f0b2c2a4522c092d3fe382294db4676250be2c1d2c03acf269fa6 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00012-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00012-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8767127923fa10f67f23ef904a44e743b7b1b6e8 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00012-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866a2697d65ccf7c55e885433fd43e210ea05df7f6f557b016a536f26eed84ac +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00013-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00013-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..af5103a0aa747f2760826276466d481b21cdb049 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00013-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b013155496808ff156c3a41e0bfa30608213d1f627d14703ef84fac04e0c2412 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00014-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00014-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0b061cc128dbb61eb18de65a4df5b61041bede1c --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00014-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a71915970636dc19252e7c31229cf8747234bc9c3c79e0470154f43d519fb80 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00015-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00015-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..374bbe23a8fdfcf7294ccd9ef1568694ad8cc52d --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00015-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d5f13f8ebcfcb447bde163addab2032a5b9d6335a1d0194728e56990cfe077 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00016-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00016-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b2d14ffc508e89664bd4aff6f956e61fb2dd95a9 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00016-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f99af3960f7f2fa195b40ccb1b5851be08c4191cacd64e1fb8bcf5e818160f25 +size 3900847536 diff --git a/models/Qwen/Qwen2.5-32B/model-00017-of-00017.safetensors b/models/Qwen/Qwen2.5-32B/model-00017-of-00017.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..52a3457de2fc81ab8ddd6ffb0d5b0224e1b78ff6 --- /dev/null +++ b/models/Qwen/Qwen2.5-32B/model-00017-of-00017.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af67ca1cd45025ca8311da7972e83a6975c6ad7d936863b51278cdf6d25782e +size 3098609680 diff --git a/models/Qwen/Qwen2.5-7B/model-00001-of-00004.safetensors b/models/Qwen/Qwen2.5-7B/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..873d16421a58490774a0360e80f6a34813b847d5 --- /dev/null +++ b/models/Qwen/Qwen2.5-7B/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6d4b6e881d17e9235934419b1c17a8bb0f1108579404e0a468a6afdeae6e868 +size 3945441440 diff --git a/models/Qwen/Qwen2.5-7B/model-00002-of-00004.safetensors b/models/Qwen/Qwen2.5-7B/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5cecce2723293ac7163668886ae117ae896b26e4 --- /dev/null +++ b/models/Qwen/Qwen2.5-7B/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:687e8d08dc82b5f7d69322f1d8691d49b59db9039b07e7ea71010fb6434d5274 +size 3864726352 diff --git a/models/Qwen/Qwen2.5-7B/model-00003-of-00004.safetensors b/models/Qwen/Qwen2.5-7B/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7a9530b162e56d82189a816993c74aee4fdea0d7 --- /dev/null +++ b/models/Qwen/Qwen2.5-7B/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32652eb23537597cca8c7a56ec85b5d5f1c7245b16269a904d57e85aa1aff30e +size 3864726424 diff --git a/models/Qwen/Qwen2.5-7B/model-00004-of-00004.safetensors b/models/Qwen/Qwen2.5-7B/model-00004-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bb387551c5920ec4ebd68f2bb59e9c6b781160e8 --- /dev/null +++ b/models/Qwen/Qwen2.5-7B/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5a2298dddcf228129975a9a271912a9f8dc817957deecde523e3154481ec3fb +size 3556377672 diff --git a/models/Qwen2.5-7B-quantization-baselines/model-00001-of-00004.safetensors b/models/Qwen2.5-7B-quantization-baselines/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f843febd0fbcef5ddab52a84a7769bafa8d33345 --- /dev/null +++ b/models/Qwen2.5-7B-quantization-baselines/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8618820530d90db767c5e331d44908f3d123fadabb00bf35d3f9cde75786e20b +size 4877660672 diff --git a/models/Qwen2.5-7B-quantization-baselines/model-00002-of-00004.safetensors b/models/Qwen2.5-7B-quantization-baselines/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95690ad9476bb0ca1b54e080d9a4fe83f7d7b7a1 --- /dev/null +++ b/models/Qwen2.5-7B-quantization-baselines/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c20389514b98d285932f472b3407e9ca9df402211b67133bca43be394418dfc7 +size 4932750888 diff --git a/models/Qwen2.5-7B-quantization-baselines/model-00003-of-00004.safetensors b/models/Qwen2.5-7B-quantization-baselines/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cca4b3abb430051553e9783ba9c00cd004e541fb --- /dev/null +++ b/models/Qwen2.5-7B-quantization-baselines/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a50b2bf1d1d26e1cfa05095a9a13b0f88f81643adca06989be1ed0351e5c62 +size 4330865088 diff --git a/models/Qwen3-8B/model-00001-of-00005.safetensors b/models/Qwen3-8B/model-00001-of-00005.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8d46cf470aff053c27fd6d956d4d69af9460bd2a --- /dev/null +++ b/models/Qwen3-8B/model-00001-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31d6a825ae35f11fb85b195b4c42c146c051e446433125a215336abdf95cbf5f +size 3996250744 diff --git a/models/Qwen3-8B/model-00002-of-00005.safetensors b/models/Qwen3-8B/model-00002-of-00005.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e726a2bcb3b100cba7ae899689cc029d8a08b20c --- /dev/null +++ b/models/Qwen3-8B/model-00002-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5991236cea6fe21f3d43cab0f0e84448734fbbe0789816202989f2ddc9d18282 +size 3993160032 diff --git a/models/Qwen3-8B/model-00003-of-00005.safetensors b/models/Qwen3-8B/model-00003-of-00005.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c94db38adbcb837246c797986365b8e4603ef3ac --- /dev/null +++ b/models/Qwen3-8B/model-00003-of-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5185c4794be2d8a9784d5753c9922db38df478ce11f9ed0b415b7304d896836 +size 3959604768 diff --git a/models/meta-llama/Llama-2-7b-hf/model-00001-of-00002.safetensors b/models/meta-llama/Llama-2-7b-hf/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..10be3c8544a06c6f2c54d5588ff10c62a8f01c75 --- /dev/null +++ b/models/meta-llama/Llama-2-7b-hf/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec71fd53e99766de38f24753b30c9e8942630e9e576a1ba27b0ec531e87be41 +size 9976578928 diff --git a/models/meta-llama/Llama-2-7b-hf/model-00002-of-00002.safetensors b/models/meta-llama/Llama-2-7b-hf/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e400a9b4c7a358bdc744d34ac4bab91f4743fe4c --- /dev/null +++ b/models/meta-llama/Llama-2-7b-hf/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41780b5dac322ac35598737e99208d90bdc632a1ba3389ebedbb46a1d8385a7f +size 3500297344 diff --git a/models/meta-llama/Llama-2-7b-hf/pytorch_model-00001-of-00003.bin b/models/meta-llama/Llama-2-7b-hf/pytorch_model-00001-of-00003.bin new file mode 100644 index 0000000000000000000000000000000000000000..35d777ea0a8f18c99f5d4250b45c3f50674519d4 --- /dev/null +++ b/models/meta-llama/Llama-2-7b-hf/pytorch_model-00001-of-00003.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa574ee4fa536cc1dfc1c1baf622144fa037a5259967112eeb1a40c7cc00484a +size 9877989586 diff --git a/models/meta-llama/Llama-2-7b-hf/pytorch_model-00002-of-00003.bin b/models/meta-llama/Llama-2-7b-hf/pytorch_model-00002-of-00003.bin new file mode 100644 index 0000000000000000000000000000000000000000..507c9b9b930c92febcb710a4b319de0de8bfc8bf --- /dev/null +++ b/models/meta-llama/Llama-2-7b-hf/pytorch_model-00002-of-00003.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60632e7f2530e713d57cfe2152019a01b1018070ce497e1f0c7fae4bf1340314 +size 9894801014 diff --git a/models/meta-llama/Llama-2-7b-hf/pytorch_model-00003-of-00003.bin b/models/meta-llama/Llama-2-7b-hf/pytorch_model-00003-of-00003.bin new file mode 100644 index 0000000000000000000000000000000000000000..c041886f6a5c3d4ed85a543c62c28114bcafaf62 --- /dev/null +++ b/models/meta-llama/Llama-2-7b-hf/pytorch_model-00003-of-00003.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd491dd3738e74dd4e18b964b9f8a15f785d487ab2d0c03c7a7b22fdac7bce1a +size 7180990649 diff --git a/models/meta-llama/Llama-3.1-8B/model-00001-of-00004.safetensors b/models/meta-llama/Llama-3.1-8B/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a70cf544ea82b645a9084d345203e3b855c474af --- /dev/null +++ b/models/meta-llama/Llama-3.1-8B/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b9704ab09cdeb097aa4a0a24bca96f906eec36bad63ab495bc21475058601b +size 4976698672 diff --git a/models/meta-llama/Llama-3.1-8B/model-00002-of-00004.safetensors b/models/meta-llama/Llama-3.1-8B/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..440f54a28d3e777128cca95168dd93017e99d8ef --- /dev/null +++ b/models/meta-llama/Llama-3.1-8B/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c28b25e7541751056ee126627e007f8d4288319733285e9f7b17b9ff6eb313f0 +size 4999802720 diff --git a/models/meta-llama/Llama-3.1-8B/model-00003-of-00004.safetensors b/models/meta-llama/Llama-3.1-8B/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b42588cf2677760fc6f6c39b19e55a31fca75e1c --- /dev/null +++ b/models/meta-llama/Llama-3.1-8B/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8e9504dd4e4a146d484c52a97584ec14dac92237c46b064934af67a85e7d383 +size 4915916176 diff --git a/models/meta-llama/Llama-3.1-8B/original/consolidated.00.pth b/models/meta-llama/Llama-3.1-8B/original/consolidated.00.pth new file mode 100644 index 0000000000000000000000000000000000000000..38431bc1d77985fa7f3a14653046c9a2916a8d90 --- /dev/null +++ b/models/meta-llama/Llama-3.1-8B/original/consolidated.00.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4bf93805847a87006dfc036ec518dc1ff94605f3a0bcd819916477ae9dcafbc +size 16060621688 diff --git a/quantization_metric/ZD/ZD_Llama-2-7b-hf.json b/quantization_metric/ZD/ZD_Llama-2-7b-hf.json new file mode 100644 index 0000000000000000000000000000000000000000..a8f0100a5691e8b9e7e6c7a5fc0599f92a416968 --- /dev/null +++ b/quantization_metric/ZD/ZD_Llama-2-7b-hf.json @@ -0,0 +1,34 @@ +[ + 0.12127955257892609, + 0.13089843094348907, + 0.14821207523345947, + 0.15012630820274353, + 0.15147793292999268, + 0.15166506171226501, + 0.1496262401342392, + 0.14946621656417847, + 0.1508074551820755, + 0.1513732522726059, + 0.15072329342365265, + 0.14903846383094788, + 0.1507146805524826, + 0.15127427875995636, + 0.15130800008773804, + 0.15193095803260803, + 0.1514924019575119, + 0.15300948917865753, + 0.15337951481342316, + 0.15288618206977844, + 0.15325477719306946, + 0.15330111980438232, + 0.15372204780578613, + 0.15461428463459015, + 0.1528640240430832, + 0.1542043685913086, + 0.15291397273540497, + 0.1548752337694168, + 0.15376469492912292, + 0.15261045098304749, + 0.15288527309894562, + 0.15156638622283936 +] \ No newline at end of file diff --git a/quantization_metric/__pycache__/expert_number.cpython-310.pyc b/quantization_metric/__pycache__/expert_number.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..47d52d65b7c5d4cace295421cd95ebeab16020d3 Binary files /dev/null and b/quantization_metric/__pycache__/expert_number.cpython-310.pyc differ diff --git a/quantization_metric/__pycache__/expert_number.cpython-312.pyc b/quantization_metric/__pycache__/expert_number.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6c0dae39f3b15b965b0501b3dc334c665925177 Binary files /dev/null and b/quantization_metric/__pycache__/expert_number.cpython-312.pyc differ diff --git a/quantization_metric/__pycache__/lsaq_quant.cpython-310.pyc b/quantization_metric/__pycache__/lsaq_quant.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1822613dab7a69e91bcaf5d3fb657db0668451d Binary files /dev/null and b/quantization_metric/__pycache__/lsaq_quant.cpython-310.pyc differ diff --git a/quantization_metric/__pycache__/lsaq_quant.cpython-311.pyc b/quantization_metric/__pycache__/lsaq_quant.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e4e30d0e9dccfe0a4f68020bf98e4c5c1779392b Binary files /dev/null and b/quantization_metric/__pycache__/lsaq_quant.cpython-311.pyc differ diff --git a/quantization_metric/__pycache__/lsaq_quant_assign.cpython-310.pyc b/quantization_metric/__pycache__/lsaq_quant_assign.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36a4bbb938062a1daef3c5a680ce94e2960a9161 Binary files /dev/null and b/quantization_metric/__pycache__/lsaq_quant_assign.cpython-310.pyc differ diff --git a/quantization_metric/__pycache__/lsaq_quant_assign.cpython-311.pyc b/quantization_metric/__pycache__/lsaq_quant_assign.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..efbea4a48e4444cc0d5ea720924df858655d1bc5 Binary files /dev/null and b/quantization_metric/__pycache__/lsaq_quant_assign.cpython-311.pyc differ diff --git a/quantization_metric/alpha_hat/alpha_values_Llama-2-7b-hf.json b/quantization_metric/alpha_hat/alpha_values_Llama-2-7b-hf.json new file mode 100644 index 0000000000000000000000000000000000000000..d246fa5b46c03309fe39c62fb59d575e319cd36d --- /dev/null +++ b/quantization_metric/alpha_hat/alpha_values_Llama-2-7b-hf.json @@ -0,0 +1,34 @@ +[ + 9.349713325500488, + 10.815018653869629, + 11.897616386413574, + 12.029354095458984, + 12.309849739074707, + 11.946270942687988, + 11.481233596801758, + 11.721869468688965, + 11.327689170837402, + 11.53799057006836, + 10.726588249206543, + 10.718080520629883, + 11.553681373596191, + 11.870050430297852, + 11.590679168701172, + 12.22303295135498, + 13.55667781829834, + 14.409969329833984, + 15.17994499206543, + 14.177068710327148, + 16.08867073059082, + 15.446638107299805, + 16.46088981628418, + 15.665449142456055, + 15.563199996948242, + 15.74089241027832, + 15.062054634094238, + 16.634864807128906, + 16.235889434814453, + 17.409889221191406, + 19.63990020751953, + 16.72626304626465 +] \ No newline at end of file diff --git a/quantization_metric/alpha_hat/alpha_values_Mistral-7B-Instruct-v0.3.json b/quantization_metric/alpha_hat/alpha_values_Mistral-7B-Instruct-v0.3.json new file mode 100644 index 0000000000000000000000000000000000000000..dc0503fc29504a736fc5241f154c2469bfdb6277 --- /dev/null +++ b/quantization_metric/alpha_hat/alpha_values_Mistral-7B-Instruct-v0.3.json @@ -0,0 +1,34 @@ +[ + 1.1970752477645874, + 0.2429095059633255, + -1.1510264873504639, + -1.022344708442688, + -1.3006900548934937, + -1.8991762399673462, + -0.9122233390808105, + -1.2367451190948486, + -0.3070283830165863, + -0.7728005647659302, + -0.23463645577430725, + -0.06571756303310394, + -0.41424596309661865, + -0.13831397891044617, + -0.5175665020942688, + -1.3752782344818115, + -1.2438682317733765, + -0.0604880228638649, + -1.688420057296753, + -0.1004464328289032, + -10.284761428833008, + -0.31737300753593445, + -0.016758646816015244, + 0.26234492659568787, + -0.4036133885383606, + 0.3988102674484253, + 1.0909702777862549, + 1.8093615770339966, + 2.014542818069458, + 1.0202614068984985, + 3.161954641342163, + 4.37812614440918 +] \ No newline at end of file diff --git a/quantization_metric/alpha_hat/alpha_values_Qwen3-4B.json b/quantization_metric/alpha_hat/alpha_values_Qwen3-4B.json new file mode 100644 index 0000000000000000000000000000000000000000..fdb5dc3632f34ef79dc58945c1dfa2cec5101815 --- /dev/null +++ b/quantization_metric/alpha_hat/alpha_values_Qwen3-4B.json @@ -0,0 +1,38 @@ +[ + 18.473791122436523, + 12.412610054016113, + 11.746052742004395, + 12.59785270690918, + 14.191147804260254, + 14.61119556427002, + 14.40878677368164, + 13.916247367858887, + 13.724929809570312, + 14.568377494812012, + 14.460932731628418, + 14.532349586486816, + 12.853517532348633, + 13.398237228393555, + 13.056329727172852, + 11.42686939239502, + 13.749275207519531, + 12.086167335510254, + 12.459146499633789, + 11.623270034790039, + 10.636680603027344, + 12.55407428741455, + 13.503619194030762, + 13.36585807800293, + 14.470367431640625, + 14.47610092163086, + 15.77418327331543, + 16.20818328857422, + 16.98804473876953, + 16.581771850585938, + 17.51226806640625, + 15.744329452514648, + 14.348668098449707, + 15.260462760925293, + 17.142507553100586, + 18.759679794311523 +] \ No newline at end of file diff --git a/quantization_metric/alpha_hat/alpha_values_Qwen3-8B.json b/quantization_metric/alpha_hat/alpha_values_Qwen3-8B.json new file mode 100644 index 0000000000000000000000000000000000000000..1c421357e59c24ba87c7a3328d69bc1256248f6b --- /dev/null +++ b/quantization_metric/alpha_hat/alpha_values_Qwen3-8B.json @@ -0,0 +1,38 @@ +[ + 17.414451599121094, + 13.166569709777832, + 13.165433883666992, + 12.927250862121582, + 17.36843490600586, + 16.749385833740234, + 15.934614181518555, + 15.633746147155762, + 16.226032257080078, + 16.654895782470703, + 15.740387916564941, + 16.90516471862793, + 16.932762145996094, + 14.34117317199707, + 15.148332595825195, + 14.41555404663086, + 16.522960662841797, + 14.885143280029297, + 13.803291320800781, + 15.464813232421875, + 12.883759498596191, + 13.800076484680176, + 15.136292457580566, + 17.452375411987305, + 15.968628883361816, + 17.6728572845459, + 18.218076705932617, + 18.50853729248047, + 19.06429100036621, + 19.157466888427734, + 21.419404983520508, + 19.745914459228516, + 17.979087829589844, + 18.474262237548828, + 21.041194915771484, + 22.842403411865234 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Llama-2-13b-hf.json b/quantization_metric/alpha_values/alpha_values_Llama-2-13b-hf.json new file mode 100644 index 0000000000000000000000000000000000000000..10ff96c9ab74a2b2df9bbbae025efd9a790e4330 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Llama-2-13b-hf.json @@ -0,0 +1,42 @@ +[ + 1.9240748882293701, + 2.3745579719543457, + 2.724717617034912, + 2.835265636444092, + 2.9549429416656494, + 2.929586172103882, + 2.8934645652770996, + 2.9061501026153564, + 2.8172874450683594, + 2.8394970893859863, + 2.732104778289795, + 2.6958770751953125, + 2.810821056365967, + 2.7806975841522217, + 2.91890025138855, + 2.9019577503204346, + 3.065373659133911, + 3.453108549118042, + 3.488546848297119, + 3.520150899887085, + 3.5741539001464844, + 3.7289161682128906, + 4.226119518280029, + 4.061612606048584, + 4.175902843475342, + 4.32330846786499, + 4.3027567863464355, + 4.224728107452393, + 4.069504261016846, + 4.380784034729004, + 4.033390522003174, + 4.307707786560059, + 4.171357154846191, + 4.100185871124268, + 3.9821338653564453, + 3.991210460662842, + 3.8532230854034424, + 3.5658786296844482, + 3.6418871879577637, + 3.113738536834717 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Llama-2-7b-hf.json b/quantization_metric/alpha_values/alpha_values_Llama-2-7b-hf.json new file mode 100644 index 0000000000000000000000000000000000000000..3a0f77d226137799570ffa7f8906aa29049cca53 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Llama-2-7b-hf.json @@ -0,0 +1,34 @@ +[ + 2.075355052947998, + 2.7356295585632324, + 3.041445016860962, + 3.1171741485595703, + 3.1137518882751465, + 3.123548746109009, + 3.014518976211548, + 3.044421911239624, + 2.786062479019165, + 2.8967933654785156, + 2.7134411334991455, + 2.6317508220672607, + 2.9590649604797363, + 2.999225616455078, + 3.030318021774292, + 3.153773069381714, + 3.4680943489074707, + 3.683506488800049, + 4.001385688781738, + 3.840238571166992, + 4.345433235168457, + 4.5532355308532715, + 4.32323694229126, + 4.144618988037109, + 4.2521443367004395, + 4.0785441398620605, + 3.7366340160369873, + 4.243911266326904, + 4.220475196838379, + 4.087046146392822, + 4.347034454345703, + 3.43670916557312 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Llama-3.1-70B-Instruct.json b/quantization_metric/alpha_values/alpha_values_Llama-3.1-70B-Instruct.json new file mode 100644 index 0000000000000000000000000000000000000000..bcf7bef1dbd0c3b025a933cbe34a91b1868bcda9 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Llama-3.1-70B-Instruct.json @@ -0,0 +1,82 @@ +[ + 1.6929006576538086, + 1.9539729356765747, + 2.3376071453094482, + 2.409193277359009, + 2.3469371795654297, + 2.5984151363372803, + 2.5063364505767822, + 2.449045181274414, + 2.3525421619415283, + 2.6812832355499268, + 2.746723175048828, + 3.3517818450927734, + 2.3949036598205566, + 2.7501437664031982, + 2.5470402240753174, + 2.497628927230835, + 2.7421603202819824, + 2.9180281162261963, + 2.955772876739502, + 2.8964169025421143, + 3.0606186389923096, + 3.3243038654327393, + 3.288444995880127, + 3.0300278663635254, + 2.9185283184051514, + 3.2030508518218994, + 4.052588939666748, + 3.6225106716156006, + 3.572335720062256, + 3.2277333736419678, + 2.869872808456421, + 2.871173858642578, + 3.428236722946167, + 3.333185911178589, + 4.026333808898926, + 3.583235025405884, + 3.1893064975738525, + 3.655658483505249, + 4.54026985168457, + 4.551355838775635, + 4.140618801116943, + 4.175935745239258, + 4.186201572418213, + 4.025412559509277, + 4.102836608886719, + 4.936075687408447, + 3.666147232055664, + 4.007832050323486, + 3.9864513874053955, + 4.1600728034973145, + 4.089629173278809, + 4.177455425262451, + 3.8165247440338135, + 4.339850902557373, + 4.424893856048584, + 4.5735368728637695, + 3.785533905029297, + 3.9309983253479004, + 4.061143398284912, + 4.309998512268066, + 3.763291835784912, + 3.8677618503570557, + 5.505738258361816, + 4.192714691162109, + 3.559302806854248, + 4.617941379547119, + 4.0876970291137695, + 4.151554584503174, + 3.9038517475128174, + 4.164182186126709, + 4.404284477233887, + 4.043477535247803, + 3.9881432056427, + 3.537508487701416, + 4.639923572540283, + 4.931902885437012, + 3.6128978729248047, + 3.2964489459991455, + 3.179161548614502, + 5.565837383270264 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Llama-3.2-3B-Instruct.json b/quantization_metric/alpha_values/alpha_values_Llama-3.2-3B-Instruct.json new file mode 100644 index 0000000000000000000000000000000000000000..dae45a3240606d557c616d4608da969cf775eb66 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Llama-3.2-3B-Instruct.json @@ -0,0 +1,30 @@ +[ + 3.4369940757751465, + 3.930715799331665, + 4.339383602142334, + 4.037461757659912, + 4.26265287399292, + 3.9877302646636963, + 3.9612863063812256, + 3.120554208755493, + 3.408421277999878, + 2.807912588119507, + 2.855315685272217, + 3.238760232925415, + 3.0182764530181885, + 3.225879669189453, + 4.079075336456299, + 4.196334362030029, + 4.043272495269775, + 4.20671272277832, + 3.9570248126983643, + 4.40455436706543, + 4.692604064941406, + 4.675817012786865, + 4.317934989929199, + 4.598446846008301, + 4.055878162384033, + 4.15275239944458, + 3.248081684112549, + 3.1271440982818604 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Mistral-7B-Instruct-v0.3.json b/quantization_metric/alpha_values/alpha_values_Mistral-7B-Instruct-v0.3.json new file mode 100644 index 0000000000000000000000000000000000000000..a07b5fa297e532e5c9164f28c6d6d3dfce5ba8cc --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Mistral-7B-Instruct-v0.3.json @@ -0,0 +1,34 @@ +[ + 2.4255356788635254, + 2.818350315093994, + 3.6927058696746826, + 3.426816940307617, + 3.7591593265533447, + 3.9702260494232178, + 3.624544858932495, + 3.421255350112915, + 3.01892352104187, + 3.4116899967193604, + 2.9628560543060303, + 2.7914068698883057, + 2.8085458278656006, + 2.9254696369171143, + 3.159223794937134, + 3.481060028076172, + 3.6433298587799072, + 3.4767093658447266, + 4.326750755310059, + 4.060551643371582, + 8.307598114013672, + 4.394644737243652, + 4.439133644104004, + 4.324174404144287, + 4.803303241729736, + 4.131540298461914, + 4.190897464752197, + 4.455796718597412, + 3.928572177886963, + 4.2314066886901855, + 3.484334945678711, + 2.9920871257781982 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Qwen3-4B.json b/quantization_metric/alpha_values/alpha_values_Qwen3-4B.json new file mode 100644 index 0000000000000000000000000000000000000000..2a18411b17c183fc2fe48ed99e4feccd9d27e1c7 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Qwen3-4B.json @@ -0,0 +1,38 @@ +[ + 4.457682132720947, + 3.0654776096343994, + 3.045346736907959, + 3.191307783126831, + 3.8381669521331787, + 4.069720268249512, + 4.3133225440979, + 4.257547855377197, + 4.053768634796143, + 4.491023540496826, + 4.205219268798828, + 4.255455493927002, + 3.5994584560394287, + 3.6606783866882324, + 3.826634645462036, + 3.1331303119659424, + 3.507666826248169, + 3.257235288619995, + 3.444204807281494, + 3.5552237033843994, + 3.09621000289917, + 3.7987544536590576, + 4.17490816116333, + 4.095727443695068, + 4.156641960144043, + 4.385300636291504, + 4.660891532897949, + 4.94378662109375, + 5.0574951171875, + 4.685359954833984, + 4.999693870544434, + 4.295413494110107, + 4.022119045257568, + 3.9893815517425537, + 4.358809471130371, + 4.655974864959717 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_Qwen3-8B.json b/quantization_metric/alpha_values/alpha_values_Qwen3-8B.json new file mode 100644 index 0000000000000000000000000000000000000000..0e2c3b11e756fcbe65bf27f37131bf3f9a6bd840 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_Qwen3-8B.json @@ -0,0 +1,38 @@ +[ + 3.5419270992279053, + 2.924048900604248, + 2.972186326980591, + 3.0156102180480957, + 4.216619968414307, + 3.9915831089019775, + 4.183230876922607, + 4.117135047912598, + 4.165277004241943, + 4.146717071533203, + 3.8434388637542725, + 4.332492351531982, + 4.381091117858887, + 3.3219542503356934, + 3.765162229537964, + 3.4117016792297363, + 3.7673075199127197, + 3.589017152786255, + 3.178703546524048, + 4.176419258117676, + 3.148484468460083, + 3.43060040473938, + 4.067776679992676, + 4.740360736846924, + 3.8485159873962402, + 4.422205448150635, + 4.484000205993652, + 4.661281585693359, + 4.704173564910889, + 4.565606117248535, + 5.161460876464844, + 4.608503818511963, + 4.278461933135986, + 4.127822399139404, + 4.576713562011719, + 4.683767795562744 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint2.json b/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint2.json new file mode 100644 index 0000000000000000000000000000000000000000..079a257bea7f297b4e78e87c6faf31a554943088 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint2.json @@ -0,0 +1,34 @@ +[ + 2.68857741355896, + 2.6641123294830322, + 3.7894043922424316, + 4.127062797546387, + 4.204006195068359, + 3.9363315105438232, + 4.048568248748779, + 3.2329509258270264, + 3.923393964767456, + 4.119979381561279, + 4.197810649871826, + 4.110360145568848, + 3.811523914337158, + 3.819208860397339, + 4.224693775177002, + 4.35741662979126, + 4.461002826690674, + 4.876941204071045, + 4.9134297370910645, + 5.207923889160156, + 5.245199680328369, + 4.932727813720703, + 5.113242149353027, + 5.1742119789123535, + 5.166106224060059, + 4.805064678192139, + 4.393592834472656, + 5.668731689453125, + 4.576906681060791, + 4.095530986785889, + 4.571784019470215, + 3.785792827606201 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint4.json b/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint4.json new file mode 100644 index 0000000000000000000000000000000000000000..cd6501683f9dfe5668cbeb2e3f3a32107e9e033e --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint4.json @@ -0,0 +1,34 @@ +[ + 2.315829038619995, + 2.7345669269561768, + 3.087406635284424, + 3.1560237407684326, + 3.245877981185913, + 3.091688871383667, + 3.0850579738616943, + 3.0631680488586426, + 2.815711498260498, + 2.890636682510376, + 2.737410306930542, + 2.7631287574768066, + 2.881362199783325, + 3.0447804927825928, + 2.9720280170440674, + 3.234403610229492, + 3.5771870613098145, + 3.7242681980133057, + 3.8575851917266846, + 3.7989277839660645, + 4.186004161834717, + 4.3772149085998535, + 4.543514251708984, + 4.044042110443115, + 4.262796878814697, + 4.192296981811523, + 3.987403392791748, + 4.351071357727051, + 4.23954439163208, + 4.062229633331299, + 4.557445049285889, + 3.6800825595855713 +] \ No newline at end of file diff --git a/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint8.json b/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint8.json new file mode 100644 index 0000000000000000000000000000000000000000..e284643c8ac67411b154ca3e948042dd73c91297 --- /dev/null +++ b/quantization_metric/alpha_values/alpha_values_llama-2-7b-qint8.json @@ -0,0 +1,34 @@ +[ + 2.131688356399536, + 2.7337377071380615, + 3.078596591949463, + 3.205598831176758, + 3.1117091178894043, + 3.1127636432647705, + 3.0389673709869385, + 2.976348638534546, + 2.832753896713257, + 2.901033878326416, + 2.7390427589416504, + 2.6919305324554443, + 2.9270668029785156, + 3.0459907054901123, + 3.0026676654815674, + 3.137556314468384, + 3.4077694416046143, + 3.8087637424468994, + 4.167283535003662, + 3.7401230335235596, + 4.127479076385498, + 4.326955318450928, + 4.120416164398193, + 4.082348823547363, + 4.115578651428223, + 3.93860125541687, + 3.9005298614501953, + 4.379550457000732, + 4.309576511383057, + 4.215869426727295, + 4.466194152832031, + 3.471622943878174 +] \ No newline at end of file diff --git a/quantization_metric/alphalora/.gitignore b/quantization_metric/alphalora/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..c18dd8d83ceed1806b50b0aaa46beb7e335fff13 --- /dev/null +++ b/quantization_metric/alphalora/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/quantization_metric/alphalora/README.md b/quantization_metric/alphalora/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ba34ca68cbe1691c74ea99e386838eaa777c566f --- /dev/null +++ b/quantization_metric/alphalora/README.md @@ -0,0 +1,71 @@ +# AlphaLoRA: Assigning LoRA Experts Based on Layer Training Quality + +Welcome to the official GitHub repository for our paper, "[AlphaLoRA: Assigning LoRA Experts Based on Layer Training Quality](https://arxiv.org/html/2410.10054v1)." + + + + + + + + + + +1. **Clone the repository** + + ```bash + git clone https://github.com/peijunallin/alphalora.git + cd alphalora + ``` + +2. **Install dependencies** + + ```bash + conda create -n alphalora python=3.10 -y + conda activate alphalora + pip install -r requirements.txt + ``` + +3. **Determine number of experts and Top K** + + Run the `expert_number.py` script to get the number of experts and the top_k parameters: + + ```bash + CUDA_VISIBLE_DEVICES=3 python expert_number.py \ + --model "mistralai/Mistral-7B-v0.1" \ + --target_sum 160 \ + --beta 2.5 + ``` + +4. **Train on six datasets** + + + ```bash + bash run_all.sh + ``` + + Before running the script, ensure to adjust the following hyperparameters in `run_all.sh`: + + | Hyperparameters | Description | + |--------------------------|-------------------------------------------------------------------| + | `base_model` | The path to the base model. | + | `root_data_path` | The path to the six datasets. | + | `number_experts` | The number of experts for each layer (32 numbers). | + | `top_k` | The top K value for each layer (32 numbers). | + | `output_dir` | The directory path to save the LoRA experts' weights. | + +5. **Evaluate on six datasets** + + Ensure that `mola_weights` corresponds to the `output_dir` used during training, and keep the expert number and top_K settings consistent. + + ```bash + bash eval_all.sh + ``` + + +## Acknowlegements + +Our code is based on [MoLA](https://github.com/gcyzsl/mola) and [TempBalance](https://github.com/yefanzhou/tempbalance). + + + diff --git a/quantization_metric/alphalora/__pycache__/expert_number.cpython-310.pyc b/quantization_metric/alphalora/__pycache__/expert_number.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..117270efae6d984ba00e39ccf56cfb9b8e7069ee Binary files /dev/null and b/quantization_metric/alphalora/__pycache__/expert_number.cpython-310.pyc differ diff --git a/quantization_metric/alphalora/__pycache__/expert_number.cpython-311.pyc b/quantization_metric/alphalora/__pycache__/expert_number.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a10e8e95f26ab39ca60d164b5e4794f1ee474e0 Binary files /dev/null and b/quantization_metric/alphalora/__pycache__/expert_number.cpython-311.pyc differ diff --git a/quantization_metric/alphalora/__pycache__/expert_number.cpython-312.pyc b/quantization_metric/alphalora/__pycache__/expert_number.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5bc1af9cf14d2e40ccc9f6f410deead912839af0 Binary files /dev/null and b/quantization_metric/alphalora/__pycache__/expert_number.cpython-312.pyc differ diff --git a/quantization_metric/alphalora/eval_all.sh b/quantization_metric/alphalora/eval_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..c7fc694f51b3831f4dc660260f83f1ef462704f1 --- /dev/null +++ b/quantization_metric/alphalora/eval_all.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Set base paths for data and weights + +base_weights_path="" + +#The weights path should be the same with the output_dir in run_all.sh +mola_weights=( + "$base_weights_path/mistral_alpha_glue_rte_all" + "$base_weights_path/mistral_alpha_glue_mrpc_all" + "$base_weights_path/mistral_alpha_glue_cola_all" + "$base_weights_path/mistral_alpha_qa_text_scienceq_all" + "$base_weights_path/mistral_alpha_qa_commonq_all" + "$base_weights_path/mistral_alpha_qa_openbook_all" + +) + +test_data=( + "$base_weights_path/datasets/glue_rte_test.json" + "$base_weights_path/datasets/glue_mrpc_test.json" + "$base_weights_path/datasets/glue_cola_test.json" + "$base_weights_path/datasets/qa_text_scienceq_test_all.json" + "$base_weights_path/datasets/qa_commonq_test_all.json" + "$base_weights_path/datasets/qa_openbook_test_all.json" +) + +checkpoints=("checkpoint-10" "checkpoint-15" "checkpoint-20") + +# Loop through the arrays and run evaluations +# + +for (( i=0; i<${#mola_weights[@]}; i++ )); do + for checkpoint in "${checkpoints[@]}"; do + if [[ $checkpoint == "checkpoint-20" ]]; then + checkpoint_dir="${mola_weights[i]}" + else + checkpoint_dir="${mola_weights[i]}/${checkpoint}" + fi + + if [ -d "$checkpoint_dir" ]; then + echo "Evaluating $(basename ${mola_weights[i]}) - $checkpoint" + CUDA_VISIBLE_DEVICES=3 python evaluation.py \ + --test_dataset "${test_data[i]}" \ + --base_model "mistralai/Mistral-7B-v0.1" \ + --mola_weights "$checkpoint_dir" \ + --batch_size 8 \ + --lora_target_modules "q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj" \ + --number_experts "1,3,5,4,4,5,4,4,3,3,2,2,2,2,3,4,9,4,8,7,8,7,8,7,9,8,6,10,6,7,3,2" \ + --top_k "1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2" \ + --save_path "$base_weights_path/results/mistral_alpha/$(basename ${mola_weights[i]})_${checkpoint}.json" + else + echo "Checkpoint directory $checkpoint_dir does not exist." + fi + done +done diff --git a/quantization_metric/alphalora/evaluation.py b/quantization_metric/alphalora/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..ce259891702857de00ddcd18ee24439cd91ce509 --- /dev/null +++ b/quantization_metric/alphalora/evaluation.py @@ -0,0 +1,296 @@ +import os +import re +import json +import argparse +import random +from tqdm import tqdm +from datasets import load_from_disk +import torch +from src.mola_peft_model_hacked import PeftModel +from transformers import GenerationConfig, LlamaTokenizer, AutoConfig +import sys +from typing import Union +from src.mola_modeling_llama_hacked import LlamaForCausalLM_d +from transformers import AutoTokenizer +from src.mola_modeling_mistral_hacked import MistralForCausalLM_d +from src.mistralconfig import MistralConfig +from src.gemma_config import GemmaConfig +from src.mola_modeling_gemma import GemmaForCausalLM_d + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + +try: + if torch.backends.mps.is_available(): + device = "mps" +except: # noqa: E722 + pass + +seed = 10 +random.seed(seed) +torch.manual_seed(0) + + +class Prompter(object): + __slots__ = ("template", "_verbose") + + def __init__(self, template_name: str = "", verbose: bool = False): + self._verbose = verbose + self.template = { + "description": "Template used by Alpaca-LoRA.", + "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n", + "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n", + "response_split": "### Response:" + } + + if self._verbose: + print( + f"Using prompt template {template_name}: {self.template['description']}" + ) + + def generate_prompt( + self, + instruction: str, + input: Union[None, str] = None, + label: Union[None, str] = None, + ) -> str: + # returns the full prompt from instruction and optional input + # if a label (=response, =output) is provided, it's also appended. + if input: + res = self.template["prompt_input"].format( + instruction=instruction, input=input + ) + else: + res = self.template["prompt_no_input"].format( + instruction=instruction + ) + if label: + res = f"{res}{label}" + if self._verbose: + print(res) + return res + + def get_response(self, output: str) -> str: + return output.split(self.template["response_split"])[1].strip() + + +def main(): + parser = argparse.ArgumentParser(description='Evaluation') + # Defining arguments + parser.add_argument('--test_dataset', type=str, default="./scienceqa/scienceq_test.json", + help='test_dataset') + parser.add_argument('--base_model', type=str, default="NousResearch/Llama-2-7b-hf", help='base_model') + parser.add_argument('--mola_weights', type=str, default="./scienceqa_mola", + help='mola_model') + parser.add_argument('--number_experts', type=str, + default="2,2,2,2,2,2,2,2,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8", + help='experts number') + parser.add_argument('--top_k', type=str, + default="2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2", + help='lora_model') + parser.add_argument('--save_path', type=str, + default="./results/mola_test_sciqa_seed_10.json", + help='lora_model') + parser.add_argument('--lora_target_modules', type=str, + default="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj", help='lora_target_modules') + parser.add_argument('--batch_size', type=int, default=8, help='base_model') + + # Parsing arguments + args = parser.parse_args() + if args.test_dataset.endswith(".json") or args.test_dataset.endswith(".jsonl"): + data_a = json.load(open(args.test_dataset)) + else: + data_aa = load_from_disk(args.test_dataset)["test"] + data_a = [d for d in data_aa] + + base_model = args.base_model + mola_weights = args.mola_weights + max_batch_size = args.batch_size + + lora_target_modules = args.lora_target_modules.split(",") + lora_target_modules = [str(lr) for lr in lora_target_modules] + print(lora_target_modules) + number_experts = args.number_experts.split(",") + number_experts = [int(lr) for lr in number_experts] + print(number_experts) + top_k = args.top_k.split(",") + top_k = [int(lr) for lr in top_k] + print(top_k) + + print(args.test_dataset) + print(args.base_model) + print(args.mola_weights) + + load_8bit = False + + if "Llama" in base_model or "llama" in base_model: + tokenizer = LlamaTokenizer.from_pretrained(base_model, padding_side='left') + config = AutoConfig.from_pretrained(base_model) + + config.lora_target_modules = lora_target_modules + if device == "cuda": + model = LlamaForCausalLM_d.from_pretrained( + base_model, + config=config, + load_in_8bit=load_8bit, + torch_dtype=torch.float16, + device_map="auto", + ) + model = PeftModel.from_pretrained( + model, + mola_weights, + torch_dtype=torch.float16, + number_experts=number_experts, + top_k=top_k, + ) + else: + model = LlamaForCausalLM_d.from_pretrained( + base_model, config=config, device_map={"": device}, low_cpu_mem_usage=True + ) + model = PeftModel.from_pretrained( + model, + mola_weights, + device_map={"": device}, + ) + elif "Mistral" in base_model: + tokenizer = AutoTokenizer.from_pretrained(base_model,padding_side='left') + config = MistralConfig.from_pretrained(base_model) + config.lora_target_modules = lora_target_modules + model = MistralForCausalLM_d.from_pretrained( + base_model, + config=config, + load_in_8bit=False, + torch_dtype=torch.float16, + device_map="auto" + ) + model = PeftModel.from_pretrained( + model, + mola_weights, + torch_dtype=torch.float16, + number_experts=number_experts, + top_k=top_k, + ) + else: + print("Not support") + + + obalance = False + model.get_new_parameters(number_experts, top_k, obalance) + + print(model.config.pad_token_id, tokenizer.pad_token_id) + print(model.config.bos_token_id, tokenizer.bos_token_id) + print(model.config.eos_token_id, tokenizer.eos_token_id) + # unwind broken decapoda-research config + model.config.pad_token_id = tokenizer.pad_token_id = 0 # unk + model.config.bos_token_id = 1 + model.config.eos_token_id = 2 + + if not load_8bit: + model.half() # seems to fix bugs for some users. + + model.eval() + if torch.__version__ >= "2" and sys.platform != "win32": + model = torch.compile(model) + + prompter = Prompter(template_name="alpaca") + + max_new_tokens = 128 + save_every = 200 + + correct = 0 + results = [] + outputs = [] + gt = [] + + for start_idx in tqdm(range(0, len(data_a), max_batch_size)): + end_idx = min(start_idx + max_batch_size, len(data_a)) + batch = data_a[start_idx:end_idx] + + answers = [str(example["answer"]) for example in batch] + + # generate prompt + prompts = [prompter.generate_prompt(example['instruction'], example['input']) for example in batch] + inputs = tokenizer(prompts, padding=True, return_tensors="pt") + input_ids = inputs["input_ids"].to(device) + + with torch.no_grad(): + generation_output = model.generate( + input_ids=input_ids, + return_dict_in_generate=True, + output_scores=True, + max_new_tokens=max_new_tokens, + ) + s = generation_output.sequences + output = tokenizer.batch_decode(s) + output = [prompter.get_response(otp) for otp in output] + # extract the answer + print(output) + + # TODO: Here we use different pattern to extract the answer for different datasets + # pattern = re.compile(r'The anwser to the question is (\d+):*') + if "cola" in args.test_dataset or "mrpc" in args.test_dataset or "rte" in args.test_dataset: + pattern = re.compile(r'Answer: ([\w\s]+)') + res = [pattern.search(otp).group(1) for otp in output] + + else: + pattern = re.compile(r'The answer is ([A-Z]).') + res = [pattern.findall(otp) for otp in output] + #------------------------------------------------------------------------------------------------------- + print(res) + pred = [] + for r_i in range(len(res)): + + #TODO: for diiferent datasets, we need to use different pattern to match the answer + if "rte" in args.test_dataset: + if res[r_i] == "entailment" or res[r_i] == "not entailment": + answer = res[r_i] # 'A', 'B', ... + else: + answer = "FAILED" + print(res[r_i]) + elif "mrpc" in args.test_dataset: + if res[r_i] == "equivalent" or res[r_i] == "not equivalent": + answer = res[r_i] # 'A', 'B', ... + else: + answer = "FAILED" + print(res[r_i]) + elif "cola" in args.test_dataset: + if res[r_i] == "acceptable" or res[r_i] == "unacceptable": + answer = res[r_i] # 'A', 'B', ... + else: + answer = "FAILED" + print(res[r_i]) + else: + if len(res[r_i]) == 1: + answer = res[r_i][0] + #------------------------------------------------------------------------------------------------------- + pred.append(answer) + results.append(res[r_i]) + outputs.append(output[r_i]) + gt.append(answers[r_i]) + + if str(answer) == str(answers[r_i]): + correct += 1 + print('correct:', str(answer), str(answers[r_i])) + else: + print('gt-ans:', str(answer), str(answers[r_i])) + + acc = correct / len(results) * 100 + + if end_idx % save_every == 0 or end_idx == len(data_a): + result_file = args.save_path + os.makedirs(result_file, exist_ok=True) + print(f"{len(results)}/{len(data_a)}, correct: {correct}, acc: {round(acc, 2)}%, saving to {result_file}") + data = {} + data['acc'] = acc + data['correct'] = correct + data['len'] = len(results) + data['results'] = results + data['outputs'] = outputs + with open(result_file, 'w') as f: + json.dump(data, f, indent=2, separators=(',', ': ')) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/quantization_metric/alphalora/expert_number.py b/quantization_metric/alphalora/expert_number.py new file mode 100644 index 0000000000000000000000000000000000000000..078a943612c9367aeda0a39da239417313d09e40 --- /dev/null +++ b/quantization_metric/alphalora/expert_number.py @@ -0,0 +1,471 @@ +import os + +os.environ['HF_HOME'] = "/home/yiren/new_ssd2/MoLA/huggingface_cache" +os.environ['HF_DATASETS_CACHE'] = "/home/yiren/new_ssd2/MoLA/huggingface_cache" +os.environ['TRANSFORMERS_CACHE'] = "/home/yiren/new_ssd2/MoLA/huggingface_cache" +import argparse +import numpy as np +import torch +import torch.nn as nn +from transformers import AutoTokenizer, AutoModelForCausalLM, LlamaTokenizer +import tqdm + + +def exponential_scaling(values, target_sum, exponent): + values = np.array(values) + scaled_values = np.power(values, exponent) + scaled_integers = np.round((scaled_values / scaled_values.sum()) * target_sum).astype(int) + + while scaled_integers.sum() != target_sum: + difference = target_sum - scaled_integers.sum() + if difference > 0: + scaled_integers[np.argmin(scaled_values - scaled_integers)] += 1 + else: + scaled_integers[np.argmax(scaled_values - scaled_integers)] -= 1 + + return scaled_integers + + +def fix_finger(w, bins=100, pl_fitting=True, EVALS_THRESH=1e-4, filter_zeros=False): + eigs = torch.square(torch.linalg.svdvals(w).flatten()) + eigs, _ = torch.sort(eigs, descending=False) + + if filter_zeros: + nz_eigs = eigs[eigs > EVALS_THRESH] + N = len(nz_eigs) + else: + # print(f"{name} Skip Filter Zero") + nz_eigs = eigs + N = len(nz_eigs) + + log_nz_eigs = torch.log(nz_eigs + 1e-8) + alphas = torch.zeros(N - 1) + Ds = torch.ones(N - 1) + if pl_fitting: + hist_nz_eigs = torch.log10(nz_eigs + 1e-8) + min_e, max_e = hist_nz_eigs.min(), hist_nz_eigs.max() + counts = torch.histc(hist_nz_eigs, bins, min=min_e, max=max_e) + if torch.any(torch.isnan(counts)) or torch.any(torch.isinf(counts)): + print(f"Warning: NaN or Inf in histogram counts. Adjusting range.") + counts = torch.zeros_like(counts) # 用 0 填充 + boundaries = torch.linspace(min_e, max_e, bins + 1) + h = counts, boundaries + ih = torch.argmax(h[0]) + xmin2 = 10 ** h[1][ih] + xmin_min = torch.log10(0.95 * xmin2) + xmin_max = 1.5 * xmin2 + + for i, xmin in enumerate(nz_eigs[:-1]): + if pl_fitting == True: + if xmin < xmin_min: + continue + if xmin > xmin_max: + break + + n = float(N - i) + #seq = torch.arange(n).cuda(nz_eigs.device) + alpha = 1 + n / (torch.sum(log_nz_eigs[i:]) - n * log_nz_eigs[i]) + alphas[i] = alpha + if alpha > 1: + seq = torch.arange(n, device=nz_eigs.device) + Ds[i] = torch.max(torch.abs( + 1 - (nz_eigs[i:] / xmin) ** (-alpha + 1) - seq / n + )) + + min_D_index = torch.argmin(Ds) + final_alpha = alphas[min_D_index] + + return final_alpha + + +def fix_finger_hat(w, bins=100, pl_fitting=True, EVALS_THRESH=1e-4, filter_zeros=False): + eigs = torch.square(torch.linalg.svdvals(w).flatten()) + eigs, _ = torch.sort(eigs, descending=False) + + if filter_zeros: + nz_eigs = eigs[eigs > EVALS_THRESH] + N = len(nz_eigs) + else: + nz_eigs = eigs + N = len(nz_eigs) + + log_nz_eigs = torch.log(nz_eigs + 1e-8) + alphas = torch.zeros(N - 1) + Ds = torch.ones(N - 1) + if pl_fitting: + hist_nz_eigs = torch.log10(nz_eigs + 1e-8) + min_e, max_e = hist_nz_eigs.min(), hist_nz_eigs.max() + counts = torch.histc(hist_nz_eigs, bins, min=min_e, max=max_e) + if torch.any(torch.isnan(counts)) or torch.any(torch.isinf(counts)): + print("Warning: NaN/Inf in histogram counts. Adjusting range.") + counts = torch.zeros_like(counts) + boundaries = torch.linspace(min_e, max_e, bins + 1) + h = counts, boundaries + ih = torch.argmax(h[0]) + xmin2 = 10 ** h[1][ih] + xmin_min = torch.log10(0.95 * xmin2) # 保持你原来的写法（如需一致尺度可改成 0.95 * xmin2） + xmin_max = 1.5 * xmin2 + + for i, xmin in enumerate(nz_eigs[:-1]): + if pl_fitting: + if xmin < xmin_min: + continue + if xmin > xmin_max: + break + + n = float(N - i) + alpha = 1 + n / (torch.sum(log_nz_eigs[i:]) - n * log_nz_eigs[i]) + alphas[i] = alpha + if alpha > 1: + seq = torch.arange(n, device=nz_eigs.device) + Ds[i] = torch.max(torch.abs( + 1 - (nz_eigs[i:] / xmin) ** (-alpha + 1) - seq / n + )) + + min_D_index = torch.argmin(Ds) + final_alpha = alphas[min_D_index] + + # === 仅此新增：按 Alpha_Hat 定义做缩放 === + lambda_max = torch.max(nz_eigs) # 最大特征值 = 最大奇异值^2 + final_alpha_hat = final_alpha * torch.log(lambda_max + 1e-8) # 自然对数 + + return final_alpha_hat + +def stable_rank(W): + # 计算 Frobenius 范数 + frobenius_norm = torch.norm(W, p='fro') # Frobenius norm + print('frobenius_norm', frobenius_norm) + # 计算谱范数 + spectral_norm = torch.linalg.norm(W, ord=2) # Spectral norm + print('spectral_norm', spectral_norm) + # 计算 Stable_Rank + stable_rank_value = frobenius_norm**2 / spectral_norm**2 + return stable_rank_value + +def calculate_coherence_score(W, transpose=False): + """ + 计算矩阵的 Average Coherence Score。 + + Args: + W: 权重矩阵 + transpose (bool): 是否需要转置。 + - 对于 MLP Up_proj, Q, K: 设为 False (比较行/神经元) + - 对于 MLP Down_proj: 建议设为 True (比较列/输入特征对应) + - 对于 MHA Output: 建议设为 True (比较列/Head拼接来源) + + Returns: + float: 分数越低 (接近0) -> 正交性越好 -> 重要性越高 -> High Bits + 分数越高 (接近1) -> 冗余度高 -> 重要性越低 -> Low Bits + """ + # 0. 基础处理 + W = W.detach().float() + if transpose: + W = W.t() + + n_vectors = W.shape[0] + + # 如果向量数量太多(如 >20000)，计算 Gram 矩阵可能会 OOM， + # 此时可以考虑采样或者分块，但对于一般 LLM (最大 ~11k) 通常没问题。 + + # 1. 行向量归一化 (L2 Norm) + # dim=1: 把每一行变成单位向量 + w_norm = F.normalize(W, p=2, dim=1) + + # 2. 计算 Gram Matrix + # shape: [n_vectors, n_vectors] + gram_matrix = torch.mm(w_norm, w_norm.t()) + + # 3. 提取上三角非对角元素 (优化性能) + # 只需要计算 i < j 的部分，不用全矩阵 mask + # triu_indices 会返回上三角的坐标，offset=1 去掉对角线 + triu_indices = torch.triu_indices(n_vectors, n_vectors, offset=1, device=W.device) + + # 提取这些元素 + off_diagonal_elements = gram_matrix[triu_indices[0], triu_indices[1]] + + # 4. 计算平均绝对值 + avg_coherence = off_diagonal_elements.abs().mean() + + return avg_coherence # 转为 float，防止显存泄露 + + +import torch.nn.functional as F + +def head_diversity(W, num_heads): + """ + 计算 Head Diversity Score (基于余弦相似度) + 输入 W: [d_model, d_model] (例如 Q 或 K 的权重) + 输入 num_heads: int (注意力头的数量) + """ + # import pdb; pdb.set_trace() + W = W.float().contiguous() + d_out, d_in = W.shape + head_dim = d_out // num_heads + + # 1. Reshape 成 [num_heads, head_dim, d_in] + # 假设权重排列是连续的 heads + w_heads = W.view(num_heads, head_dim, d_in) + + # 2. 将每个 Head 展平为一个向量 [num_heads, feature_dim] + w_flat = w_heads.reshape(num_heads, -1) + + # 3. 归一化 (为了算 Cosine Similarity) + w_norm = F.normalize(w_flat, p=2, dim=1) + + # 4. 计算 Gram 矩阵 (两两相似度) + gram_matrix = torch.mm(w_norm, w_norm.t()) # shape: [num_heads, num_heads] + + # 5. 计算非对角线元素的平均绝对相似度 + # torch.triu 取上三角，diagonal=1 去掉对角线(也就是去掉了自己和自己的相似度1) + mask = torch.triu(torch.ones_like(gram_matrix), diagonal=1).bool() + avg_similarity = gram_matrix[mask].abs().mean() + + # print('avg_head_similarity', avg_similarity) + + # 6. Diversity = 1 - Similarity (相似度越低，多样性越高) + diversity_score = 1.0 - avg_similarity + + return diversity_score + +def compute_head_diversity(weight: torch.Tensor, num_heads: int) -> torch.Tensor: + """ + 计算 Transformer Attention 层的 Head Diversity 指标，返回 torch.Tensor。 + + Args: + weight (torch.Tensor): 权重矩阵，形状通常为 [out_features, in_features]。 + num_heads (int): Attention Heads 的数量。 + + Returns: + torch.Tensor: Diversity Score (0.0 到 1.0 之间)，形状为 [1]。 + """ + # 强制连续性 (保留，防止转置等操作导致错误) + weight = weight.contiguous() + + # 1. 获取维度信息 + out_features, in_features = weight.shape + + # 确保维度能被 head 数整除 + assert out_features % num_heads == 0, "Weight output dim must be divisible by num_heads" + head_dim = out_features // num_heads + + # 2. Reshape & Flatten (使用 view 或 reshape 均可，因为我们已处理了 contiguous) + weight_reshaped = weight.view(num_heads, head_dim, in_features) + heads_flat = weight_reshaped.view(num_heads, -1) + + # 3. 计算余弦相似度矩阵 (Cosine Similarity Matrix) + heads_norm = F.normalize(heads_flat, p=2, dim=1) + cosine_sim_matrix = torch.mm(heads_norm, heads_norm.t()) + + # 4. 计算平均相似度 (忽略对角线) + # 确保 mask 在同一设备上 + mask = torch.eye(num_heads, device=weight.device, dtype=torch.bool) + + # 获取非对角元素 + off_diagonal_sims = cosine_sim_matrix.masked_select(~mask) + + # 计算平均相似度 (注意: 此时 avg_similarity 是一个 tensor) + avg_similarity = off_diagonal_sims.mean() + + # 5. 计算 Diversity Score + # 使用 torch.abs() 保持 tensor 类型 + diversity_score = 1.0 - torch.abs(avg_similarity) + + return diversity_score + +def TCR(Z, epsilon=1e-6): + # 获取特征维度 d 和批量大小 B + d, _ = Z.shape # d 是特征维度 + B = 10 + # 计算协方差矩阵 ZZ^T + ZZ_T = torch.matmul(Z, Z.T) + + # 计算调整后的矩阵 I_d + (d / (B * epsilon^2)) * ZZ_T + I_d = torch.eye(d).to(Z.device) # 单位矩阵 + term = I_d + (d / (B * epsilon**2)) * ZZ_T + # 计算行列式的对数 + log_det_term = torch.logdet(term) + + # 返回 TCR 值 + return 0.5 * log_det_term + +def effective_rank(A): + # 计算矩阵的奇异值 + singular_values = torch.linalg.svdvals(A) + + # 计算奇异值的归一化比例 + p = singular_values / singular_values.sum() + + # 计算香农熵 + entropy = -torch.sum(p * torch.log(p + 1e-10)) # 加上一个小常数避免 log(0) + + # 计算有效秩 + erank = torch.exp(entropy) + + return erank + +def ZD(weight_matrix: torch.Tensor, threshold: float = 1.0) -> float: + """ + 计算某个模块参数的 ZD 分数 + ZD = z-score > threshold 的权重比例 + """ + weights = weight_matrix.view(-1) # 展平 + mu = weights.mean() # 均值 μ + sigma = weights.std(unbiased=False) # 标准差 σ + if sigma.item() == 0: # 防止除零 + return 0.0 + z_scores = (weights - mu) / sigma # 计算 z-score + ratio = (z_scores > threshold).float().mean() # 超过阈值的比例 + return ratio + + +class WrappedGPT: + def __init__(self, layer, layer_id=0, layer_name="none"): + self.layer = layer + self.dev = layer.weight.device + self.rows, self.columns = layer.weight.data.shape + self.scaler_row = torch.zeros(self.columns, device=self.dev) + self.nsamples = 0 + + def add_batch(self, inp, out): + if len(inp.shape) == 2: + inp = inp.unsqueeze(0) + tmp = inp.shape[0] + if isinstance(self.layer, torch.nn.Linear) and len(inp.shape) == 3: + inp = inp.reshape((-1, inp.shape[-1])) + inp = inp.t() + + self.scaler_row *= self.nsamples / (self.nsamples + tmp) + self.nsamples += tmp + self.scaler_row += torch.norm(inp.float(), p=2, dim=1) ** 2 / self.nsamples + + +# from lsaq_quant import W8A16Linear +def find_layers(module, layers=[nn.Linear], name='', keyword=None): + + if type(module) in layers: + # 2. (新增) 过滤逻辑：检查名字里有没有我们想要的 keyword + if keyword is not None: + # 如果 keyword 是字符串，检查是否包含 + if isinstance(keyword, str): + if keyword not in name: + return {} + # 如果 keyword 是列表（例如 ['k_proj', 'q_proj']），检查是否包含其中任意一个 + elif isinstance(keyword, list): + if not any(k in name for k in keyword): + return {} + + return {name: module} + + res = {} + for name1, child in module.named_children(): + res.update(find_layers( + child, layers=layers, name=name + '.' + name1 if name != '' else name1, keyword=keyword + )) + return res + +def head_diversity_asssist(num_q_heads, num_kv_heads, subset): + ans = [] + wq, wk, wv, wo = 0.15, 0.35, 0.35, 0.15 + for subset_name in subset: + W = subset[subset_name].weight.data.float() + if "q_proj" in subset_name : + ans.append(wq * head_diversity(W, num_q_heads)) + elif "k_proj" in subset_name: + ans.append(wk * head_diversity(W, num_kv_heads)) + elif "v_proj" in subset_name: + ans.append(wv * head_diversity(W, num_kv_heads)) + elif "o_proj" in subset_name: + ans.append(wo * head_diversity(W.t(), num_q_heads)) + # import pdb; pdb.set_trace() + return ans + +func_call = { + 'alpha': fix_finger, # alpha_values 大的值对应层做 2bit + 'alpha_hat': fix_finger_hat, # 大的值对应层做 2bit + 'stable_rank': stable_rank, # 小的值对应层做 2bit + 'effective_rank': effective_rank, # 小的值对应层做 2bit + 'ZD': ZD, # 小的值对应层做 2bit + 'head_diversity': head_diversity_asssist, # 小的值对应层做 2bit + 'coherence': calculate_coherence_score, # 大的值对应层做 2bit + +} + + + +def calculate_expert(model, metric='alpha', keyword=None): + all_layer_alpha = [] + layers = model.model.layers + + config = model.config + print("config: \n", config) + if hasattr(config, "num_attention_heads"): + # 适用于: BERT, RoBERTa, LLaMA, DistilBERT, BART, Electra + num_q_heads = config.num_attention_heads + + if hasattr(config, "num_key_value_heads"): + num_kv_heads = config.num_key_value_heads + else: + num_kv_heads = num_q_heads + + # print("num_heads: ", num_heads) + for i, layer in enumerate(layers): + + subset = find_layers(layer, keyword=keyword) + print(f"Processing layer {i}--subset--{subset}") + + # layer_final_alpha = [fix_finger(subset[name].weight.data.float()) for name in subset] + # layer_final_alpha = [fix_finger_hat(subset[name].weight.data.float()) for name in subset] + # layer_final_alpha = [stable_rank(subset[name].weight.data.float()) for name in subset] + # layer_final_alpha = [effective_rank(subset[name].weight.data.float()) for name in subset] + # layer_final_alpha = [ZD(subset[name].weight.data.float()) for name in subset] + if metric == 'head_diversity': + layer_final_alpha = func_call[metric](num_q_heads, num_kv_heads, subset) + all_layer_alpha.append(torch.stack(layer_final_alpha).sum().item()) + else: + layer_final_alpha = [func_call[metric](subset[name].weight.data.float()) for name in subset] + # import pdb; pdb.set_trace() + all_layer_alpha.append(torch.stack(layer_final_alpha).mean().item()) + + print(f"{metric} value of layer {i} ---{all_layer_alpha[i]} ") + + torch.cuda.empty_cache() + return all_layer_alpha + + + +def get_llm(model_name): + return AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto" + ) + + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--model', default="mistralai/Mistral-7B-v0.1", type=str) + parser.add_argument('--seed', type=int, default=25) + parser.add_argument('--beta', type=float, default=2.5) + parser.add_argument('--target_sum', type=int, default=160) + + + args = parser.parse_args() + + np.random.seed(args.seed) + torch.random.manual_seed(args.seed) + + model = get_llm(args.model) + model.eval() + + distribution = calculate_expert(model) + + print("Distribution:", distribution) + quantized_vector = exponential_scaling(distribution, args.target_sum, args.beta) + print("Total expert number:", sum(quantized_vector)) + print("expert number: ", ','.join(map(str, quantized_vector))) + + topkk = [2 if n > 1 else 1 for n in quantized_vector] + topkk = ','.join(map(str, topkk)) + print("top_k: ", topkk) + +if __name__ == '__main__': + main() diff --git a/quantization_metric/alphalora/mola_training_mistral.py b/quantization_metric/alphalora/mola_training_mistral.py new file mode 100644 index 0000000000000000000000000000000000000000..b2e02fe9044e3dfc8f903ac7d42f4f9d56248a40 --- /dev/null +++ b/quantization_metric/alphalora/mola_training_mistral.py @@ -0,0 +1,327 @@ +import os +import sys +import fire +import torch +import transformers +from transformers import Trainer +from datasets import load_dataset +from datasets import load_from_disk + +from peft import ( + prepare_model_for_int8_training, +) +from src.mola_mapping_hacked import get_peft_model +from src.mola_lora_hacked import LoraConfig +from src.mola_peft_model_hacked import set_peft_model_state_dict_moe + +from transformers import LlamaTokenizer, AutoConfig +from transformers import AutoTokenizer +from src.mola_modeling_mistral_hacked import MistralForCausalLM_d +from src.mistralconfig import MistralConfig +from utils.prompter import Prompter +from transformers import TrainerCallback + + +os.environ["WANDB_DISABLED"] = "true" + +import random + +seed = 10 +random.seed(seed) +torch.manual_seed(0) + + + +def train( + # model/data params + base_model: str = "NousResearch/Llama-2-7b-hf", # the only required argument + data_path: str = "./sampled_data/sampled_scienceqa_train_all.hf", # ../datasets/CL_biology_scienceq_train_all.hf + output_dir: str = "./sampled_scienceqa_256r_8mbs_no8bit_1", + # training hyperparams + batch_size: int = 128, + micro_batch_size: int = 8, + num_epochs: int = 1, + learning_rate: float = 3e-4, + cutoff_len: int = 256, + val_set_size: int = 2, + # lora hyperparams + lora_r: str = "8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8", + lora_alpha: int = 16, + lora_dropout: float = 0.05, + lora_target_modules: str = "q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj", + # mola hyperparams + number_experts: str = "2,2,2,2,2,2,2,2,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8", + top_k: str = "2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2", + # llm hyperparams + train_on_inputs: bool = True, # if False, masks out inputs in loss + add_eos_token: bool = True, + group_by_length: bool = False, # faster, but produces an odd training loss curve + # wandb params + wandb_project: str = "", + wandb_run_name: str = "", + wandb_watch: str = "", # options: false | gradients | all + wandb_log_model: str = "", # options: false | true + resume_from_checkpoint: str = './step2_biology256r_8mbs_no8bit_scale10', # either training checkpoint or final adapter + prompt_template_name: str = "alpaca", # The prompt template to use, will default to alpaca. + obalance: bool = False, +): + if isinstance(lora_r, str): + lora_r = lora_r.split(",") + number_experts = number_experts.split(",") + top_k = top_k.split(",") + lora_target_modules = lora_target_modules.split(",") + + lora_r = [int(lr) for lr in lora_r] + number_experts = [int(lr) for lr in number_experts] + top_k = [int(lr) for lr in top_k] + lora_target_modules = [str(lr) for lr in lora_target_modules] + else: + lora_r = [int(lr) for lr in lora_r] + number_experts = [int(lr) for lr in number_experts] + top_k = [int(lr) for lr in top_k] + lora_target_modules = [str(lr) for lr in lora_target_modules] + + if int(os.environ.get("LOCAL_RANK", 0)) == 0: + print( + f"base_model: {base_model}\n" + f"data_path: {data_path}\n" + f"output_dir: {output_dir}\n" + f"batch_size: {batch_size}\n" + f"micro_batch_size: {micro_batch_size}\n" + f"num_epochs: {num_epochs}\n" + f"learning_rate: {learning_rate}\n" + f"cutoff_len: {cutoff_len}\n" + f"val_set_size: {val_set_size}\n" + f"lora_r: {lora_r}\n" + f"number_experts: {number_experts}\n" + f"top_k: {top_k}\n" + f"lora_alpha: {lora_alpha}\n" + f"lora_dropout: {lora_dropout}\n" + f"lora_target_modules: {lora_target_modules}\n" + f"train_on_inputs: {train_on_inputs}\n" + f"add_eos_token: {add_eos_token}\n" + f"group_by_length: {group_by_length}\n" + f"wandb_project: {wandb_project}\n" + f"wandb_run_name: {wandb_run_name}\n" + f"wandb_watch: {wandb_watch}\n" + f"wandb_log_model: {wandb_log_model}\n" + f"resume_from_checkpoint: {resume_from_checkpoint or False}\n" + f"prompt template: {prompt_template_name}\n" + f"obalance: {obalance}\n" + ) + assert ( + base_model + ), "Please specify a --base_model, e.g. --base_model='huggyllama/llama-7b'" + gradient_accumulation_steps = batch_size // micro_batch_size + + prompter = Prompter(prompt_template_name) + + device_map = "auto" + world_size = int(os.environ.get("WORLD_SIZE", 1)) + ddp = world_size != 1 + if ddp: + device_map = {"": int(os.environ.get("LOCAL_RANK") or 0)} + gradient_accumulation_steps = gradient_accumulation_steps // world_size + + # Check if parameter passed or if set within environ + use_wandb = len(wandb_project) > 0 or ( + "WANDB_PROJECT" in os.environ and len(os.environ["WANDB_PROJECT"]) > 0 + ) + # Only overwrite environ if wandb param passed + if len(wandb_project) > 0: + os.environ["WANDB_PROJECT"] = wandb_project + if len(wandb_watch) > 0: + os.environ["WANDB_WATCH"] = wandb_watch + if len(wandb_log_model) > 0: + os.environ["WANDB_LOG_MODEL"] = wandb_log_model + + + config = MistralConfig.from_pretrained(base_model) + config.lora_target_modules = lora_target_modules + model = MistralForCausalLM_d.from_pretrained( + base_model, + config=config, + load_in_8bit=False, + torch_dtype=torch.float16, + device_map=device_map + ) + + ##newmodified + model.get_new_parameters(number_experts, top_k, obalance) + + tokenizer = AutoTokenizer.from_pretrained(base_model) + + tokenizer.pad_token_id = ( + 0 # unk. we want this to be different from the eos token + ) + tokenizer.padding_side = "left" # Allow batched inference + + def tokenize(prompt, add_eos_token=True): + result = tokenizer( + prompt, + truncation=True, + max_length=cutoff_len, + padding=False, + return_tensors=None, + ) + if ( + result["input_ids"][-1] != tokenizer.eos_token_id + and len(result["input_ids"]) < cutoff_len + and add_eos_token + ): + result["input_ids"].append(tokenizer.eos_token_id) + result["attention_mask"].append(1) + + result["labels"] = result["input_ids"].copy() + + return result + + def generate_and_tokenize_prompt(data_point): + full_prompt = prompter.generate_prompt( + data_point["instruction"], + data_point["input"], + data_point["output"], + ) + tokenized_full_prompt = tokenize(full_prompt) + if not train_on_inputs: + user_prompt = prompter.generate_prompt( + data_point["instruction"], data_point["input"] + ) + tokenized_user_prompt = tokenize( + user_prompt, add_eos_token=add_eos_token + ) + user_prompt_len = len(tokenized_user_prompt["input_ids"]) + + if add_eos_token: + user_prompt_len -= 1 + + tokenized_full_prompt["labels"] = [ + -100 + ] * user_prompt_len + tokenized_full_prompt["labels"][ + user_prompt_len: + ] # could be sped up, probably + return tokenized_full_prompt + + model = prepare_model_for_int8_training(model) + + config = LoraConfig( + r=lora_r, + lora_alpha=lora_alpha, + target_modules=lora_target_modules, + lora_dropout=lora_dropout, + bias="none", + task_type="CAUSAL_LM", + ) + + model = get_peft_model(model, config, number_experts=number_experts, top_k=top_k) + + if data_path.endswith(".json") or data_path.endswith(".jsonl"): + data = load_dataset("json", data_files=data_path) + else: + data = load_from_disk(data_path) + # data = load_dataset(data_path) + + if resume_from_checkpoint: + # Check the available weights and load them + checkpoint_name = os.path.join( + resume_from_checkpoint, "pytorch_model.bin" + ) # Full checkpoint + if not os.path.exists(checkpoint_name): + checkpoint_name = os.path.join( + resume_from_checkpoint, "adapter_model.bin" + ) # only LoRA model - LoRA config above has to fit + resume_from_checkpoint = ( + False # So the trainer won't try loading its state + ) + # The two files above have a different name depending on how they were saved, but are actually the same. + if os.path.exists(checkpoint_name): + print(f"Restarting from {checkpoint_name}") + adapters_weights = torch.load(checkpoint_name) + set_peft_model_state_dict_moe(model, adapters_weights) + else: + print(f"Checkpoint {checkpoint_name} not found") + + if val_set_size > 0: + train_val = data["train"].train_test_split( + test_size=val_set_size, shuffle=True, seed=42 + ) + train_data = ( + train_val["train"].shuffle().map(generate_and_tokenize_prompt) + ) + val_data = ( + train_val["test"].shuffle().map(generate_and_tokenize_prompt) + ) + else: + train_data = data["train"].shuffle().map(generate_and_tokenize_prompt) + val_data = None + + + if not ddp and torch.cuda.device_count() > 1: + # keeps Trainer from trying its own DataParallelism when more than 1 gpu is available + model.is_parallelizable = True + model.model_parallel = True + + class SavePretrainedCallback(TrainerCallback): + + + def __init__(self, save_at_epochs): + super().__init__() + self.save_at_epochs = save_at_epochs + + def on_epoch_end(self, args, state, control, model=None, **kwargs): + current_epoch = int(round(state.epoch)) + print("current_epoch: ",current_epoch )# Round to nearest whole number + if model is not None and current_epoch in self.save_at_epochs: + output_dir = os.path.join(args.output_dir, f'checkpoint-{current_epoch}') + model.save_pretrained(output_dir) + print(f"Model saved at epoch {current_epoch} to {output_dir}") + + save_at_epoch = [10,15] # save at the end of training + + trainer = Trainer( + callbacks=[SavePretrainedCallback(save_at_epoch)], + model=model, + train_dataset=train_data, + eval_dataset=val_data, + args=transformers.TrainingArguments( + per_device_train_batch_size=micro_batch_size, + gradient_accumulation_steps=gradient_accumulation_steps, + warmup_steps=100, + num_train_epochs=num_epochs, + learning_rate=learning_rate, + fp16=True, + logging_steps=10, + optim="adamw_torch", + evaluation_strategy="steps" if val_set_size > 0 else "no", + save_strategy="steps", + eval_steps=None if val_set_size > 0 else None, + save_steps=20000, + output_dir=output_dir, + save_total_limit=6, + load_best_model_at_end=False, # True if val_set_size > 0 else False, + ddp_find_unused_parameters=False if ddp else None, + group_by_length=group_by_length, + report_to="wandb" if use_wandb else None, + run_name=wandb_run_name if use_wandb else None, + ), + data_collator=transformers.DataCollatorForSeq2Seq( + tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True + ), + ) + model.config.use_cache = False + + + if torch.__version__ >= "2" and sys.platform != "win32": + model = torch.compile(model) + + + trainer.train() + model.save_pretrained(output_dir) + + print( + "\n If there's a warning about missing keys above, please disregard :)" + ) + + +if __name__ == "__main__": + fire.Fire(train) diff --git a/quantization_metric/alphalora/requirements.txt b/quantization_metric/alphalora/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b1efc83115ecf3f0b393f8377c9c277018260800 --- /dev/null +++ b/quantization_metric/alphalora/requirements.txt @@ -0,0 +1,71 @@ +accelerate==0.27.2 +aiohttp==3.8.5 +aiosignal==1.3.1 +annotated-types==0.5.0 +anyio==3.7.1 +async-timeout==4.0.3 +attrs==23.1.0 +certifi==2023.7.22 +charset-normalizer==3.2.0 +cmake==3.27.5 +datasets==2.14.5 +dill==0.3.7 +exceptiongroup==1.1.3 +fastapi==0.103.2 +filelock==3.12.4 +fire==0.5.0 +Fraction==2.2.0 +frozenlist==1.4.0 +fsspec==2023.6.0 +huggingface-hub==0.24.2 +idna==3.4 +Jinja2==3.1.2 +jsonlines==4.0.0 +lit==17.0.1 +MarkupSafe==2.1.3 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +networkx==3.1 +numpy==1.26.0 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +packaging==23.1 +pandas==2.1.1 +peft==0.4.0 +pillow==10.2.0 +psutil==5.9.5 +pyarrow==13.0.0 +pydantic==2.4.2 +pydantic_core==2.10.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.8.8 +requests==2.31.0 +safetensors==0.4.3 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +starlette==0.27.0 +sympy==1.12 +termcolor==2.3.0 +tokenizers==0.19.1 +torch==1.13.1 +tqdm==4.66.1 +transformers==4.42.3 +triton==2.0.0 +typing_extensions==4.8.0 +tzdata==2023.3 +urllib3==2.0.5 +xxhash==3.3.0 +yarl==1.9.2 diff --git a/quantization_metric/alphalora/run_all.sh b/quantization_metric/alphalora/run_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..4a3b2dd752ffd863f03a249f2d7bcb4de684b097 --- /dev/null +++ b/quantization_metric/alphalora/run_all.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Set the root data path +root_data_path="" + +# Define data paths using the root path +data_paths=( + "$root_data_path/datasets/glue_rte_all.hf/" + "$root_data_path/datasets/glue_mrpc_all.hf/" + "$root_data_path/datasets/glue_cola_all.hf/" + "$root_data_path/datasets/qa_text_scienceq_all.hf/" + "$root_data_path/datasets/qa_commonq_all.hf/" + "$root_data_path/datasets/qa_openbook_all.hf/" +) + +# Loop through data paths and run experiments +for data_path in "${data_paths[@]}"; do + # Extract filename from data path + filename=$(basename "$data_path") + # Remove trailing slash if present + filename=${filename%/} + filename=${filename%.hf} + echo $filename + + # Run experiment + CUDA_VISIBLE_DEVICES=0,1,2 python mola_training_mistral.py \ + --base_model "mistralai/Mistral-7B-v0.1" \ + --data_path "$data_path" \ + --output_dir "$root_data_path/mistral_alpha_$filename" \ + --batch_size 128 \ + --micro_batch_size 8 \ + --num_epochs 20 \ + --learning_rate 3e-4 \ + --cutoff_len 256 \ + --val_set_size 1 \ + --lora_r "8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8" \ + --lora_alpha 16 \ + --lora_dropout 0.05 \ + --lora_target_modules "q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj" \ + --number_experts "1,3,5,4,4,5,4,4,3,3,2,2,2,2,3,4,9,4,8,7,8,7,8,7,9,8,6,10,6,7,3,2" \ + --top_k "1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2" \ + --train_on_inputs \ + --group_by_length \ + --add_eos_token +done \ No newline at end of file diff --git a/quantization_metric/alphalora/src/attn.py b/quantization_metric/alphalora/src/attn.py new file mode 100644 index 0000000000000000000000000000000000000000..aafd1ac2cc7bed788eabeef67ec1de813b83c7b0 --- /dev/null +++ b/quantization_metric/alphalora/src/attn.py @@ -0,0 +1,493 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch + + +@dataclass +class AttentionMaskConverter: + """ + A utility attention mask class that allows one to: + - Create a causal 4d mask + - Create a causal 4d mask with slided window + - Convert a 2d attention mask (batch_size, query_length) to a 4d attention mask (batch_size, 1, query_length, + key_value_length) that can be multiplied with attention scores + + Examples: + + ```python + >>> import torch + >>> from transformers.modeling_attn_mask_utils import AttentionMaskConverter + + >>> converter = AttentionMaskConverter(True) + >>> converter.to_4d(torch.tensor([[0, 0, 0, 1, 1]]), 5, key_value_length=5, dtype=torch.float32) + tensor([[[[-3.4028e+38, -3.4028e+38, -3.4028e+38, -3.4028e+38, -3.4028e+38], + [-3.4028e+38, -3.4028e+38, -3.4028e+38, -3.4028e+38, -3.4028e+38], + [-3.4028e+38, -3.4028e+38, -3.4028e+38, -3.4028e+38, -3.4028e+38], + [-3.4028e+38, -3.4028e+38, -3.4028e+38, 0.0000e+00, -3.4028e+38], + [-3.4028e+38, -3.4028e+38, -3.4028e+38, 0.0000e+00, 0.0000e+00]]]]) + ``` + + Parameters: + is_causal (`bool`): + Whether the attention mask should be a uni-directional (causal) or bi-directional mask. + + sliding_window (`int`, *optional*): + Optionally, the sliding window masks can be created if `sliding_window` is defined to a positive integer. + """ + + is_causal: bool + sliding_window: int + + def __init__(self, is_causal: bool, sliding_window: Optional[int] = None): + self.is_causal = is_causal + self.sliding_window = sliding_window + + if self.sliding_window is not None and self.sliding_window <= 0: + raise ValueError( + f"Make sure that when passing `sliding_window` that its value is a strictly positive integer, not `{self.sliding_window}`" + ) + + def to_causal_4d( + self, + batch_size: int, + query_length: int, + key_value_length: int, + dtype: torch.dtype, + device: Union[torch.device, "str"] = "cpu", + ) -> Optional[torch.Tensor]: + """ + Creates a causal 4D mask of (bsz, head_dim=1, query_length, key_value_length) shape and adds large negative + bias to upper right hand triangular matrix (causal mask). + """ + if not self.is_causal: + raise ValueError(f"Please use `to_causal_4d` only if {self.__class__} has `is_causal` set to True.") + + # If shape is not cached, create a new causal mask and cache it + input_shape = (batch_size, query_length) + past_key_values_length = key_value_length - query_length + + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + causal_4d_mask = None + if input_shape[-1] > 1 or self.sliding_window is not None: + causal_4d_mask = self._make_causal_mask( + input_shape, + dtype, + device=device, + past_key_values_length=past_key_values_length, + sliding_window=self.sliding_window, + ) + + return causal_4d_mask + + def to_4d( + self, + attention_mask_2d: torch.Tensor, + query_length: int, + dtype: torch.dtype, + key_value_length: Optional[int] = None, + ) -> torch.Tensor: + """ + Converts 2D attention mask to 4D attention mask by expanding mask to (bsz, head_dim=1, query_length, + key_value_length) shape and by adding a large negative bias to not-attended positions. If attention_mask is + causal, a causal mask will be added. + """ + input_shape = (attention_mask_2d.shape[0], query_length) + + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + causal_4d_mask = None + if (input_shape[-1] > 1 or self.sliding_window is not None) and self.is_causal: + if key_value_length is None: + raise ValueError( + "This attention mask converter is causal. Make sure to pass `key_value_length` to correctly create a causal mask." + ) + + past_key_values_length = key_value_length - query_length + causal_4d_mask = self._make_causal_mask( + input_shape, + dtype, + device=attention_mask_2d.device, + past_key_values_length=past_key_values_length, + sliding_window=self.sliding_window, + ) + elif self.sliding_window is not None: + raise NotImplementedError("Sliding window is currently only implemented for causal masking") + + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = self._expand_mask(attention_mask_2d, dtype, tgt_len=input_shape[-1]).to( + attention_mask_2d.device + ) + + if causal_4d_mask is not None: + expanded_attn_mask = causal_4d_mask.masked_fill(expanded_attn_mask.bool(), torch.finfo(dtype).min) + + # expanded_attn_mask + causal_4d_mask can cause some overflow + expanded_4d_mask = expanded_attn_mask + + return expanded_4d_mask + + @staticmethod + def _make_causal_mask( + input_ids_shape: torch.Size, + dtype: torch.dtype, + device: torch.device, + past_key_values_length: int = 0, + sliding_window: Optional[int] = None, + ): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) + + # add lower triangular sliding window mask if necessary + if sliding_window is not None: + diagonal = past_key_values_length - sliding_window - 1 + + context_mask = torch.tril(torch.ones_like(mask, dtype=torch.bool), diagonal=diagonal) + mask.masked_fill_(context_mask, torch.finfo(dtype).min) + + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + @staticmethod + def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + @staticmethod + def _unmask_unattended( + expanded_mask: torch.FloatTensor, + min_dtype: float, + ): + # fmt: off + """ + Attend to all tokens in masked rows from the expanded attention mask, for example the relevant first rows when + using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + Details: https://github.com/pytorch/pytorch/issues/110213 + + `expanded_mask` is [bsz, num_masks, tgt_seq_len, src_seq_len] or [bsz, tgt_seq_len, src_seq_len]. + `attention_mask` is [bsz, src_seq_len]. + + The dimension num_masks of `expanded_mask` is most often 1, but it can also be the number of heads in the case of alibi attention bias. + + For example, if `expanded_mask` is (e.g. here left-padding case) + ``` + [[[[0, 0, 0], + [0, 0, 0], + [0, 0, 1]]], + [[[1, 0, 0], + [1, 1, 0], + [1, 1, 1]]], + [[[0, 0, 0], + [0, 1, 0], + [0, 1, 1]]]] + ``` + then the modified `expanded_mask` will be + ``` + [[[[1, 1, 1], <-- modified + [1, 1, 1], <-- modified + [0, 0, 1]]], + [[[1, 0, 0], + [1, 1, 0], + [1, 1, 1]]], + [[[1, 1, 1], <-- modified + [0, 1, 0], + [0, 1, 1]]]] + ``` + """ + # fmt: on + if expanded_mask.dtype == torch.bool: + raise ValueError( + "AttentionMaskConverter._unmask_unattended expects a float `expanded_mask`, got a BoolTensor." + ) + + return expanded_mask.mul(~torch.all(expanded_mask == min_dtype, dim=-1, keepdim=True)) + + @staticmethod + def _ignore_causal_mask_sdpa( + attention_mask: Optional[torch.Tensor], + inputs_embeds: torch.Tensor, + past_key_values_length: int, + sliding_window: Optional[int] = None, + is_training: bool = False, + ) -> bool: + """ + Detects whether the optional user-specified attention_mask & the automatically created causal mask can be ignored in case PyTorch's SDPA is used, rather relying on SDPA's `is_causal` argument. + + In case no token is masked in the `attention_mask` argument, if `query_length == 1` or + `key_value_length == query_length`, we rather rely on SDPA `is_causal` argument to use causal/non-causal masks, + allowing to dispatch to the flash attention kernel (that can otherwise not be used if a custom `attn_mask` is passed). + """ + + _, query_length = inputs_embeds.shape[0], inputs_embeds.shape[1] + key_value_length = query_length + past_key_values_length + + is_tracing = ( + torch.jit.is_tracing() + or isinstance(inputs_embeds, torch.fx.Proxy) + or (hasattr(torch, "_dynamo") and torch._dynamo.is_compiling()) + ) + + ignore_causal_mask = False + + if attention_mask is None: + # TODO: When tracing with TorchDynamo with fullgraph=True, the model is recompiled depending on the input shape, thus SDPA's `is_causal` argument is rightfully updated (see https://gist.github.com/fxmarty/1313f39037fc1c112508989628c57363). However, when using `torch.export` or + # or `torch.onnx.dynamo_export`, we must pass an example input, and `is_causal` behavior is hard-coded. If a user exports a model with q_len > 1, the exported model will hard-code `is_causal=True` which is in general wrong (see https://github.com/pytorch/pytorch/issues/108108). + # Thus, we only set `ignore_causal_mask = True` if the model is set to training. + # + # Besides, jit.trace can not handle the `q_len > 1` condition for `is_causal` (`TypeError: scaled_dot_product_attention(): argument 'is_causal' must be bool, not Tensor`). + if ( + (is_training or not is_tracing) + and (query_length == 1 or key_value_length == query_length) + and (sliding_window is None or key_value_length < sliding_window) + ): + ignore_causal_mask = True + elif sliding_window is None or key_value_length < sliding_window: + if len(attention_mask.shape) == 4: + return False + elif (is_training or not is_tracing) and torch.all(attention_mask == 1): + if query_length == 1 or key_value_length == query_length: + # For query_length == 1, causal attention and bi-directional attention are the same. + ignore_causal_mask = True + + # Unfortunately, for query_length > 1 and key_value_length != query_length, we cannot generally ignore the attention mask, as SDPA causal mask generation + # may be wrong. We will set `is_causal=False` in SDPA and rely on Transformers attention_mask instead, hence not setting it to None here. + # Reference: https://github.com/pytorch/pytorch/issues/108108 + # TODO: maybe revisit this with https://github.com/pytorch/pytorch/pull/114823 in PyTorch 2.3. + + return ignore_causal_mask + + +def _prepare_4d_causal_attention_mask( + attention_mask: Optional[torch.Tensor], + input_shape: Union[torch.Size, Tuple, List], + inputs_embeds: torch.Tensor, + past_key_values_length: int, + sliding_window: Optional[int] = None, +): + """ + Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape + `(batch_size, key_value_length)` + + Args: + attention_mask (`torch.Tensor` or `None`): + A 2D attention mask of shape `(batch_size, key_value_length)` + input_shape (`tuple(int)` or `list(int)` or `torch.Size`): + The input shape should be a tuple that defines `(batch_size, query_length)`. + inputs_embeds (`torch.Tensor`): + The embedded inputs as a torch Tensor. + past_key_values_length (`int`): + The length of the key value cache. + sliding_window (`int`, *optional*): + If the model uses windowed attention, a sliding window should be passed. + """ + attn_mask_converter = AttentionMaskConverter(is_causal=True, sliding_window=sliding_window) + + key_value_length = input_shape[-1] + past_key_values_length + + # 4d mask is passed through the layers + if attention_mask is not None and len(attention_mask.shape) == 2: + attention_mask = attn_mask_converter.to_4d( + attention_mask, input_shape[-1], key_value_length=key_value_length, dtype=inputs_embeds.dtype + ) + elif attention_mask is not None and len(attention_mask.shape) == 4: + expected_shape = (input_shape[0], 1, input_shape[1], key_value_length) + if tuple(attention_mask.shape) != expected_shape: + raise ValueError( + f"Incorrect 4D attention_mask shape: {tuple(attention_mask.shape)}; expected: {expected_shape}." + ) + else: + # if the 4D mask has correct shape - invert it and fill with negative infinity + inverted_mask = 1.0 - attention_mask + attention_mask = inverted_mask.masked_fill( + inverted_mask.to(torch.bool), torch.finfo(inputs_embeds.dtype).min + ) + else: + attention_mask = attn_mask_converter.to_causal_4d( + input_shape[0], input_shape[-1], key_value_length, dtype=inputs_embeds.dtype, device=inputs_embeds.device + ) + + return attention_mask + + +# Adapted from _prepare_4d_causal_attention_mask +def _prepare_4d_causal_attention_mask_for_sdpa( + attention_mask: Optional[torch.Tensor], + input_shape: Union[torch.Size, Tuple, List], + inputs_embeds: torch.Tensor, + past_key_values_length: int, + sliding_window: Optional[int] = None, +): + """ + Prepares the correct `attn_mask` argument to be used by `torch.nn.functional.scaled_dot_product_attention`. + + In case no token is masked in the `attention_mask` argument, we simply set it to `None` for the cases `query_length == 1` and + `key_value_length == query_length`, and rely instead on SDPA `is_causal` argument to use causal/non-causal masks, + allowing to dispatch to the flash attention kernel (that can otherwise not be used if a custom `attn_mask` is passed). + """ + attn_mask_converter = AttentionMaskConverter(is_causal=True, sliding_window=sliding_window) + + key_value_length = input_shape[-1] + past_key_values_length + + # torch.jit.trace, symbolic_trace and torchdynamo with fullgraph=True are unable to capture the controlflow `is_causal=attention_mask is None and q_len > 1` + # used as an SDPA argument. We keep compatibility with these tracing tools by always using SDPA's `attn_mask` argument in case we are tracing. + # TODO: For dynamo, rather use a check on fullgraph=True once this is possible (https://github.com/pytorch/pytorch/pull/120400). + is_tracing = ( + torch.jit.is_tracing() + or isinstance(inputs_embeds, torch.fx.Proxy) + or (hasattr(torch, "_dynamo") and torch._dynamo.is_compiling()) + ) + + ignore_causal_mask = AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + sliding_window=sliding_window, + ) + + if ignore_causal_mask: + expanded_4d_mask = None + elif attention_mask is None: + expanded_4d_mask = attn_mask_converter.to_causal_4d( + input_shape[0], input_shape[-1], key_value_length, dtype=inputs_embeds.dtype, device=inputs_embeds.device + ) + else: + if attention_mask.dim() == 4: + # in this case we assume that the mask comes already in inverted form and requires no inversion or slicing + if attention_mask.max() != 0: + raise ValueError("Custom 4D attention mask should be passed in inverted form with max==0`") + expanded_4d_mask = attention_mask + else: + expanded_4d_mask = attn_mask_converter.to_4d( + attention_mask, + input_shape[-1], + dtype=inputs_embeds.dtype, + key_value_length=key_value_length, + ) + + # Attend to all tokens in masked rows from the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + if not is_tracing and expanded_4d_mask.device.type == "cuda": + expanded_4d_mask = AttentionMaskConverter._unmask_unattended( + expanded_4d_mask, min_dtype=torch.finfo(inputs_embeds.dtype).min + ) + + return expanded_4d_mask + + +def _prepare_4d_attention_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Creates a non-causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape + `(batch_size, key_value_length)` + + Args: + mask (`torch.Tensor` or `None`): + A 2D attention mask of shape `(batch_size, key_value_length)` + dtype (`torch.dtype`): + The torch dtype the created mask shall have. + tgt_len (`int`): + The target length or query length the created mask shall have. + """ + return AttentionMaskConverter._expand_mask(mask=mask, dtype=dtype, tgt_len=tgt_len) + + +def _prepare_4d_attention_mask_for_sdpa(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Creates a non-causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape + `(batch_size, key_value_length)` + + Args: + mask (`torch.Tensor` or `None`): + A 2D attention mask of shape `(batch_size, key_value_length)` + dtype (`torch.dtype`): + The torch dtype the created mask shall have. + tgt_len (`int`): + The target length or query length the created mask shall have. + """ + batch_size, key_value_length = mask.shape + tgt_len = tgt_len if tgt_len is not None else key_value_length + + # torch.jit.trace, symbolic_trace and torchdynamo with fullgraph=True are unable to capture the controlflow `is_causal=attention_mask is None and q_len > 1` + # used as an SDPA argument. We keep compatibility with these tracing tools by always using SDPA's `attn_mask` argument in case we are tracing. + # TODO: For dynamo, rather use a check on fullgraph=True once this is possible (https://github.com/pytorch/pytorch/pull/120400). + is_tracing = ( + torch.jit.is_tracing() + or isinstance(mask, torch.fx.Proxy) + or (hasattr(torch, "_dynamo") and torch._dynamo.is_compiling()) + ) + + if not is_tracing and torch.all(mask == 1): + if tgt_len == 1: + # For query_length == 1, causal attention and bi-directional attention are the same. + return None + elif key_value_length == tgt_len: + return None + else: + # Unfortunately, for query_length > 1 and key_value_length != query_length, we can not generally ignore the attention mask, as SDPA causal mask generation + # may be wrong. We will set is_causal=False in SDPA and rely on Transformers attention_mask instead, hence not setting it to None here. + # Reference: https://github.com/pytorch/pytorch/issues/108108 + return AttentionMaskConverter._expand_mask(mask=mask, dtype=dtype, tgt_len=tgt_len) + else: + return AttentionMaskConverter._expand_mask(mask=mask, dtype=dtype, tgt_len=tgt_len) + + +def _create_4d_causal_attention_mask( + input_shape: Union[torch.Size, Tuple, List], + dtype: torch.dtype, + device: torch.device, + past_key_values_length: int = 0, + sliding_window: Optional[int] = None, +) -> Optional[torch.Tensor]: + """ + Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` + + Args: + input_shape (`tuple(int)` or `list(int)` or `torch.Size`): + The input shape should be a tuple that defines `(batch_size, query_length)`. + dtype (`torch.dtype`): + The torch dtype the created mask shall have. + device (`int`): + The torch device the created mask shall have. + sliding_window (`int`, *optional*): + If the model uses windowed attention, a sliding window should be passed. + """ + attn_mask_converter = AttentionMaskConverter(is_causal=True, sliding_window=sliding_window) + + key_value_length = past_key_values_length + input_shape[-1] + attention_mask = attn_mask_converter.to_causal_4d( + input_shape[0], input_shape[-1], key_value_length, dtype=dtype, device=device + ) + + return attention_mask \ No newline at end of file diff --git a/quantization_metric/alphalora/src/cache_utils.py b/quantization_metric/alphalora/src/cache_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..171b91854d3e033e39599e5596af8802aacff675 --- /dev/null +++ b/quantization_metric/alphalora/src/cache_utils.py @@ -0,0 +1,450 @@ +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import torch + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + + +@dataclass +class Cache: + """ + Base, abstract class for all caches. The actual data structure is specific to each subclass. + """ + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. These are specific to each subclass and allow new types of + cache to be created. + + Return: + A tuple containing the updated key and value states. + """ + raise NotImplementedError("Make sure to implement `update` in a subclass.") + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + raise NotImplementedError("Make sure to implement `get_seq_length` in a subclass.") + + def get_max_length(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states, if there is any.""" + raise NotImplementedError("Make sure to implement `get_max_length` in a subclass.") + + def get_usable_length(self, new_seq_length: int, layer_idx: Optional[int] = 0) -> int: + """Given the sequence length of the new inputs, returns the usable length of the cache.""" + # Cache without size limit -> all cache is usable + # Cache with size limit -> if the length cache plus the length of the new inputs is larger the maximum cache + # length, we will need to evict part of the cache (and thus not all cache is usable) + max_length = self.get_max_length() + previous_seq_length = self.get_seq_length(layer_idx) + if max_length is not None and previous_seq_length + new_seq_length > max_length: + return max_length - new_seq_length + return previous_seq_length + + def reorder_cache(self, beam_idx: torch.LongTensor): + """Reorders the cache for beam search, given the selected beam indices.""" + for layer_idx in range(len(self.key_cache)): + device = self.key_cache[layer_idx].device + self.key_cache[layer_idx] = self.key_cache[layer_idx].index_select(0, beam_idx.to(device)) + device = self.value_cache[layer_idx].device + self.value_cache[layer_idx] = self.value_cache[layer_idx].index_select(0, beam_idx.to(device)) + + @property + def seen_tokens(self): + logger.warning_once( + "The `seen_tokens` attribute is deprecated and will be removed in v4.41. Use the `cache_position` " + "model input instead." + ) + if hasattr(self, "_seen_tokens"): + return self._seen_tokens + else: + return None + + +class DynamicCache(Cache): + """ + A cache that grows dynamically as more tokens are generated. This is the default for generative models. + + It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is + `[batch_size, num_heads, seq_len, head_dim]`. + """ + + def __init__(self) -> None: + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + self._seen_tokens = 0 # Used in `generate` to keep tally of how many tokens the cache has seen + + def __getitem__(self, layer_idx: int) -> List[Tuple[torch.Tensor]]: + """ + Support for backwards-compatible `past_key_value` indexing, e.g. `past_key_value[0][0].shape[2]` to get the + sequence length. + """ + if layer_idx < len(self): + return (self.key_cache[layer_idx], self.value_cache[layer_idx]) + else: + raise KeyError(f"Cache only has {len(self)} layers, attempted to access layer with index {layer_idx}") + + def __iter__(self): + """ + Support for backwards-compatible `past_key_value` iteration, e.g. `for x in past_key_value:` to iterate over + keys and values + """ + for layer_idx in range(len(self)): + yield (self.key_cache[layer_idx], self.value_cache[layer_idx]) + + def __len__(self): + """ + Support for backwards-compatible `past_key_value` length, e.g. `len(past_key_value)`. This value corresponds + to the number of layers in the model. + """ + return len(self.key_cache) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. No additional arguments are used in `DynamicCache`. + + Return: + A tuple containing the updated key and value states. + """ + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + # Update the cache + if len(self.key_cache) <= layer_idx: + self.key_cache.append(key_states) + self.value_cache.append(value_states) + else: + self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + if len(self.key_cache) <= layer_idx: + return 0 + return self.key_cache[layer_idx].shape[-2] + + def get_max_length(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states. DynamicCache does not have a maximum length.""" + return None + + def to_legacy_cache(self) -> Tuple[Tuple[torch.Tensor], Tuple[torch.Tensor]]: + """Converts the `DynamicCache` instance into the its equivalent in the legacy cache format.""" + legacy_cache = () + for layer_idx in range(len(self)): + legacy_cache += ((self.key_cache[layer_idx], self.value_cache[layer_idx]),) + return legacy_cache + + @classmethod + def from_legacy_cache(cls, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None) -> "DynamicCache": + """Converts a cache in the legacy cache format into an equivalent `DynamicCache`.""" + cache = cls() + if past_key_values is not None: + for layer_idx in range(len(past_key_values)): + key_states, value_states = past_key_values[layer_idx] + cache.update(key_states, value_states, layer_idx) + return cache + + +class SinkCache(Cache): + """ + A cache that as described in the [Attention Sinks paper](https://arxiv.org/abs/2309.17453). It allows the model to + generate beyond the length of its context window, without losing fluency in the conversation. As it discards past + tokens, the model will lose the ability to generate tokens that depend on the context that was discarded. + + It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is + `[batch_size, num_heads, seq_len, head_dim]`. + + Parameters: + window_length (`int`): + The length of the context window. + num_sink_tokens (`int`): + The number of sink tokens. See the original paper for more information. + """ + + def __init__(self, window_length: int, num_sink_tokens: int) -> None: + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + self.window_length = window_length + self.num_sink_tokens = num_sink_tokens + self.cos_sin_rerotation_cache = {} + self._cos_cache = None + self._sin_cache = None + self._seen_tokens = 0 # Used in `generate` to keep tally of how many tokens the cache has seen + + @staticmethod + def _rotate_half(x): + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def _apply_key_rotary_pos_emb( + self, key_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> torch.Tensor: + rotated_key_states = (key_states * cos) + (self._rotate_half(key_states) * sin) + return rotated_key_states + + def _get_rerotation_cos_sin( + self, key_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + if key_states.shape[-2] not in self.cos_sin_rerotation_cache: + # Upcast to float32 temporarily for better accuracy + cos = cos.to(torch.float32) + sin = sin.to(torch.float32) + + # Compute the cos and sin required for back- and forward-rotating to one position earlier in the sequence + original_cos = cos[self.num_sink_tokens + key_states.shape[-2] :] + shifted_cos = cos[self.num_sink_tokens : -key_states.shape[-2]] + original_sin = sin[self.num_sink_tokens + key_states.shape[-2] :] + shifted_sin = sin[self.num_sink_tokens : -key_states.shape[-2]] + rerotation_cos = original_cos * shifted_cos + original_sin * shifted_sin + rerotation_sin = -original_sin * shifted_cos + original_cos * shifted_sin + + self.cos_sin_rerotation_cache[key_states.shape[-2]] = ( + rerotation_cos.to(key_states.dtype).unsqueeze(0), + rerotation_sin.to(key_states.dtype).unsqueeze(0), + ) + return self.cos_sin_rerotation_cache[key_states.shape[-2]] + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states. A layer index can be optionally passed.""" + # TODO: deprecate this function in favor of `cache_position` + # Workaround to make 'key_states.shape[-2] + past_key_value.get_seq_length(self.layer_idx)' <= window_length + if len(self.key_cache) <= layer_idx: + return 0 + return self.key_cache[layer_idx].shape[-2] + + def get_max_length(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states.""" + return self.window_length + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. The following arguments can be used in `SinkCache`: `sin`, + `cos` and `partial_rotation_size`. These arguments are used with models using RoPE, to recompute the + rotation as the tokens are shifted. + + Return: + A tuple containing the updated key and value states. + """ + # Optional kwargs for `SinkCache` -- needed on models using RoPE. `partial_rotation_size` is used on models + # with partially rotated position embeddings, like Phi or Persimmon. + sin = cache_kwargs.get("sin") + cos = cache_kwargs.get("cos") + partial_rotation_size = cache_kwargs.get("partial_rotation_size") + using_rope = cos is not None and sin is not None + + # Update the number of seen tokens + if layer_idx == 0: + self._seen_tokens += key_states.shape[-2] + + # Update the sin/cos cache, which holds sin/cos values for all possible positions + if using_rope and layer_idx == 0: + # BC: some models still pass `sin`/`cos` with 2 dims. In those models, they are the full sin/cos. Remove + # after all RoPE models have a llama-like cache utilization. + if cos.dim() == 2: + self._cos_cache = cos + self._sin_cache = sin + else: + if self._cos_cache is None: + self._cos_cache = cos[0, ...] + self._sin_cache = sin[0, ...] + elif self._cos_cache.shape[0] < self.window_length: + self._cos_cache = torch.cat([self._cos_cache, cos[0, ...]], dim=0) + self._sin_cache = torch.cat([self._sin_cache, sin[0, ...]], dim=0) + + # [bsz, num_heads, seq_len, head_dim] + if len(self.key_cache) <= layer_idx: + # Empty cache + self.key_cache.append(key_states) + self.value_cache.append(value_states) + + elif key_states.shape[-2] + self.get_seq_length(layer_idx) < self.window_length: + # Growing cache + self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) + self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + + else: + # Shifting cache + keys_to_keep = self.key_cache[layer_idx][ + :, :, -self.window_length + self.num_sink_tokens + key_states.shape[-2] : + ] + + # On RoPE models, we need to recompute the Key rotation as the tokens are shifted + if using_rope: + rerotation_cos, rerotation_sin = self._get_rerotation_cos_sin( + key_states, self._cos_cache[: self.window_length], self._sin_cache[: self.window_length] + ) + if partial_rotation_size is not None: + keys_to_keep, keys_pass = ( + keys_to_keep[..., :partial_rotation_size], + keys_to_keep[..., partial_rotation_size:], + ) + keys_to_keep = self._apply_key_rotary_pos_emb(keys_to_keep, rerotation_cos, rerotation_sin) + if partial_rotation_size is not None: + keys_to_keep = torch.cat((keys_to_keep, keys_pass), dim=-1) + + # Concatenate sink tokens, shifted & rotated tokens (if needed), and new tokens + sink_keys = self.key_cache[layer_idx][:, :, : self.num_sink_tokens] + self.key_cache[layer_idx] = torch.cat([sink_keys, keys_to_keep, key_states], dim=-2) + + sink_values = self.value_cache[layer_idx][:, :, : self.num_sink_tokens] + values_to_keep = self.value_cache[layer_idx][ + :, :, -self.window_length + self.num_sink_tokens + value_states.shape[-2] : + ] + self.value_cache[layer_idx] = torch.cat([sink_values, values_to_keep, value_states], dim=-2) + + return self.key_cache[layer_idx], self.value_cache[layer_idx] + + +class StaticCache(Cache): + """ + Static Cache class to be used with `torch.compile(model)`. + + Parameters: + config (`PretrainedConfig): + The configuration file defining the shape-related attributes required to initialize the static cache. + max_batch_size (`int`): + The maximum batch size with which the model will be used. + max_cache_len (`int`): + The maximum sequence length with which the model will be used. + device (`torch.device`): + The device on which the cache should be initialized. Should be the same as the layer. + dtype (*optional*, defaults to `torch.float32`): + The default `dtype` to use when initializing the layer. + """ + + def __init__(self, config: PretrainedConfig, max_batch_size: int, max_cache_len: int, device, dtype=None) -> None: + super().__init__() + self.max_batch_size = max_batch_size + self.max_cache_len = config.max_position_embeddings if max_cache_len is None else max_cache_len + # Some model define a custom `head_dim` != config.hidden_size // config.num_attention_heads + self.head_dim = ( + config.head_dim if hasattr(config, "head_dim") else config.hidden_size // config.num_attention_heads + ) + + self.dtype = dtype if dtype is not None else torch.float32 + self.num_key_value_heads = ( + config.num_attention_heads if config.num_key_value_heads is None else config.num_key_value_heads + ) + + self.key_cache: List[torch.Tensor] = [] + self.value_cache: List[torch.Tensor] = [] + cache_shape = (max_batch_size, self.num_key_value_heads, self.max_cache_len, self.head_dim) + for _ in range(config.num_hidden_layers): + # Note: `mark_static_address` is used to tag the cache as an fixed data pointer, preventing cuda graph + # breaks when updating the cache. + new_layer_key_cache = torch.zeros(cache_shape, dtype=self.dtype, device=device) + new_layer_value_cache = torch.zeros(cache_shape, dtype=self.dtype, device=device) + torch._dynamo.mark_static_address(new_layer_key_cache) + torch._dynamo.mark_static_address(new_layer_value_cache) + self.key_cache.append(new_layer_key_cache) + self.value_cache.append(new_layer_value_cache) + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. + It is VERY important to index using a tensor, otherwise you introduce a copy to the device. + + Parameters: + key_states (`torch.Tensor`): + The new key states to cache. + value_states (`torch.Tensor`): + The new value states to cache. + layer_idx (`int`): + The index of the layer to cache the states for. + cache_kwargs (`Dict[str, Any]`, `optional`): + Additional arguments for the cache subclass. The `StaticCache` needs the `cache_position` input + to know how where to write in the cache. + + Return: + A tuple containing the updated key and value states. + """ + cache_position = cache_kwargs.get("cache_position") + k_out = self.key_cache[layer_idx] + v_out = self.value_cache[layer_idx] + + k_out[:, :, cache_position] = key_states + v_out[:, :, cache_position] = value_states + + return k_out, v_out + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns the sequence length of the cached states that were seen by the model.""" + # Occupied cache == any slot in the 3rd dim (sequence length) holds a non-zero value. To save on compute, let's + # limit the check to the first batch member and head dimension. + # TODO: deprecate this function in favor of `cache_position` + return (self.key_cache[layer_idx][0, 0].any(dim=-1)).sum() + + def get_max_length(self) -> Optional[int]: + """Returns the maximum sequence length of the cached states.""" + return self.max_cache_len + + def reset(self): + """Resets the cache values while preserving the objects""" + for layer_idx in range(len(self.key_cache)): + # In-place ops prevent breaking the static address + self.key_cache[layer_idx].zero_() + self.value_cache[layer_idx].zero_() \ No newline at end of file diff --git a/quantization_metric/alphalora/src/gemma_config.py b/quantization_metric/alphalora/src/gemma_config.py new file mode 100644 index 0000000000000000000000000000000000000000..b0f9448d8a0555db66929e8fc229e5fc97086870 --- /dev/null +++ b/quantization_metric/alphalora/src/gemma_config.py @@ -0,0 +1,155 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Gemma model configuration""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + + +#from ..deprecated._archive_maps import GEMMA_PRETRAINED_CONFIG_ARCHIVE_MAP # noqa: F401, E402 + + +class GemmaConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`GemmaModel`]. It is used to instantiate an Gemma + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the Gemma-7B. + + e.g. [google/gemma-7b](https://huggingface.co/google/gemma-7b) + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 256000): + Vocabulary size of the Gemma model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`GemmaModel`] + hidden_size (`int`, *optional*, defaults to 3072): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 24576): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 28): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*, defaults to 16): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + head_dim (`int`, *optional*, defaults to 256): + The attention head dimension. + hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`): + The legacy activation function. It is overwritten by the `hidden_activation`. + hidden_activation (`str` or `function`, *optional*): + The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"` + if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function. + max_position_embeddings (`int`, *optional*, defaults to 8192): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + pad_token_id (`int`, *optional*, defaults to 0): + Padding token id. + eos_token_id (`int`, *optional*, defaults to 1): + End of stream token id. + bos_token_id (`int`, *optional*, defaults to 2): + Beginning of stream token id. + tie_word_embeddings (`bool`, *optional*, defaults to `True`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + + ```python + >>> from transformers import GemmaModel, GemmaConfig + + >>> # Initializing a Gemma gemma-7b style configuration + >>> configuration = GemmaConfig() + + >>> # Initializing a model from the gemma-7b style configuration + >>> model = GemmaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "gemma" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + _attn_implementation="eager", + vocab_size=256000, + hidden_size=3072, + intermediate_size=24576, + num_hidden_layers=28, + num_attention_heads=16, + num_key_value_heads=16, + head_dim=256, + hidden_act="gelu_pytorch_tanh", + hidden_activation=None, + max_position_embeddings=8192, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + eos_token_id=1, + bos_token_id=2, + tie_word_embeddings=True, + rope_theta=10000.0, + attention_bias=False, + attention_dropout=0.0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self._attn_implementation = "eager" + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.head_dim = head_dim + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.hidden_activation = hidden_activation + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) \ No newline at end of file diff --git a/quantization_metric/alphalora/src/mistral_model.py b/quantization_metric/alphalora/src/mistral_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f9421dd0fe40116dbbe52bb4fae49e72f1ea5ad5 --- /dev/null +++ b/quantization_metric/alphalora/src/mistral_model.py @@ -0,0 +1,1387 @@ +# coding=utf-8 +# Copyright 2023 Mistral AI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch Mistral model.""" +import inspect +import math +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast +from ...modeling_utils import PreTrainedModel +from ...utils import ( + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_flash_attn_2_available, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from .configuration_mistral import MistralConfig + + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_func, flash_attn_varlen_func + from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters) + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "MistralConfig" + + +# Copied from transformers.models.llama.modeling_llama._get_unpad_data +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + + +# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Mistral +class MistralRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + MistralRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +# copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Mistral +# TODO @Arthur no longer copied from LLama after static cache +class MistralRotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + # Build here to make `torch.jit.trace` work. + self._set_cos_sin_cache( + seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype() + ) + + def _set_cos_sin_cache(self, seq_len, device, dtype): + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.int64).type_as(self.inv_freq) + + freqs = torch.outer(t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if seq_len > self.max_seq_len_cached: + self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) + + return ( + self.cos_cached[:seq_len].to(dtype=x.dtype), + self.sin_cached[:seq_len].to(dtype=x.dtype), + ) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +# copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb +# TODO @Arthur no longer copied from LLama after static cache +def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos[position_ids].unsqueeze(unsqueeze_dim) + sin = sin[position_ids].unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class MistralMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class MistralAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: MistralConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + self.attention_dropout = config.attention_dropout + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + self.rotary_emb = MistralRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + attn_weights = attn_weights + attention_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class MistralFlashAttention2(MistralAttention): + """ + Mistral flash attention module. This module inherits from `MistralAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + **kwargs, + ): + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + + # overwrite attention_mask with padding_mask + attention_mask = kwargs.pop("padding_mask") + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1 + cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + use_sliding_windows = ( + _flash_supports_window_size + and getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + ) + + if not _flash_supports_window_size: + logger.warning_once( + "The current flash attention version does not support sliding window attention, for a more memory efficient implementation" + " make sure to upgrade flash-attn library." + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + attn_output = self._flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + dropout=dropout_rate, + use_sliding_windows=use_sliding_windows, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + def _flash_attention_forward( + self, + query_states, + key_states, + value_states, + attention_mask, + query_length, + dropout=0.0, + softmax_scale=None, + use_sliding_windows=False, + ): + """ + Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Attention API + attention_mask (`torch.Tensor`): + The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + dropout (`float`): + Attention dropout + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) + use_sliding_windows (`bool`, *optional*): + Whether to activate sliding window attention. + """ + if not self._flash_attn_uses_top_left_mask: + causal = self.is_causal + else: + # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__. + causal = self.is_causal and query_length != 1 + + # Contains at least one padding token in the sequence + if attention_mask is not None: + batch_size = query_states.shape[0] + query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( + query_states, key_states, value_states, attention_mask, query_length + ) + + cu_seqlens_q, cu_seqlens_k = cu_seq_lens + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens + + if not use_sliding_windows: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output_unpad = flash_attn_varlen_func( + query_states, + key_states, + value_states, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_in_batch_q, + max_seqlen_k=max_seqlen_in_batch_k, + dropout_p=dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) + else: + if not use_sliding_windows: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + ) + else: + attn_output = flash_attn_func( + query_states, + key_states, + value_states, + dropout, + softmax_scale=softmax_scale, + causal=causal, + window_size=(self.config.sliding_window, self.config.sliding_window), + ) + + return attn_output + + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): + batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape + + # On the first iteration we need to properly re-create the padding mask + # by slicing it on the proper place + if kv_seq_len != attention_mask.shape[-1]: + attention_mask_num_tokens = attention_mask.shape[-1] + attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :] + + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) + + key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k) + + if query_length == kv_seq_len: + query_layer = index_first_axis( + query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k + ) + cu_seqlens_q = cu_seqlens_k + max_seqlen_in_batch_q = max_seqlen_in_batch_k + indices_q = indices_k + elif query_length == 1: + max_seqlen_in_batch_q = 1 + cu_seqlens_q = torch.arange( + batch_size + 1, dtype=torch.int32, device=query_layer.device + ) # There is a memcpy here, that is very bad. + indices_q = cu_seqlens_q[:-1] + query_layer = query_layer.squeeze(1) + else: + # The -q_len: slice assumes left padding. + attention_mask = attention_mask[:, -query_length:] + query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) + + return ( + query_layer, + key_layer, + value_layer, + indices_q, + (cu_seqlens_q, cu_seqlens_k), + (max_seqlen_in_batch_q, max_seqlen_in_batch_k), + ) + + +# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Mistral +# TODO @Arthur no longer copied from LLama after static cache +class MistralSdpaAttention(MistralAttention): + """ + Mistral attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `MistralAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from MistralAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "MistralModel is using MistralSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal=self.is_causal and attention_mask is None and q_len > 1, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + +MISTRAL_ATTENTION_CLASSES = { + "eager": MistralAttention, + "flash_attention_2": MistralFlashAttention2, + "sdpa": MistralSdpaAttention, +} + + +class MistralDecoderLayer(nn.Module): + def __init__(self, config: MistralConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = MISTRAL_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx) + + self.mlp = MistralMLP(config) + self.input_layernorm = MistralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = MistralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, sequence_length)` where padding elements are indicated by 0. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +MISTRAL_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`MistralConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Mistral Model outputting raw hidden-states without any specific head on top.", + MISTRAL_START_DOCSTRING, +) +class MistralPreTrainedModel(PreTrainedModel): + config_class = MistralConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["MistralDecoderLayer"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_cache_class = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +MISTRAL_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + Two formats are allowed: + - a [`~cache_utils.Cache`] instance; + - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy + cache format. + + The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the + legacy cache format will be returned. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare Mistral Model outputting raw hidden-states without any specific head on top.", + MISTRAL_START_DOCSTRING, +) +class MistralModel(MistralPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MistralDecoderLayer`] + + Args: + config: MistralConfig + """ + + def __init__(self, config: MistralConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [MistralDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self._attn_implementation = config._attn_implementation + self.norm = MistralRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + past_key_values_length = 0 + + if use_cache: + use_legacy_cache = not isinstance(past_key_values, Cache) + if use_legacy_cache: + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + past_key_values_length = past_key_values.get_usable_length(seq_length) + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache: + is_padding_right = attention_mask[:, -1].sum().item() != batch_size + if is_padding_right: + raise ValueError( + "You are attempting to perform batched generation with padding_side='right'" + " this may lead to unexpected behaviour for Flash Attention version of Mistral. Make sure to " + " call `tokenizer.padding_side = 'left'` before tokenizing the input. " + ) + + if self._attn_implementation == "flash_attention_2": + # 2d mask is passed through the layers + attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None + elif self._attn_implementation == "sdpa" and not output_attentions: + # output_attentions=True can not be supported when using SDPA, and we fall back on + # the manual implementation that requires a 4D causal mask in all cases. + attention_mask = _prepare_4d_causal_attention_mask_for_sdpa( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + sliding_window=self.config.sliding_window, + ) + else: + # 4d mask is passed through the layers + attention_mask = _prepare_4d_causal_attention_mask( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + sliding_window=self.config.sliding_window, + ) + + hidden_states = inputs_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + attention_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = None + if use_cache: + next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache + + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class MistralForCausalLM(MistralPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = MistralModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, MistralForCausalLM + + >>> model = MistralForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1") + >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Ensure tensors are on the same device + shift_labels = shift_labels.to(shift_logits.device) + loss_fct = CrossEntropyLoss() + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + # Omit tokens covered by past_key_values + if past_key_values is not None: + if isinstance(past_key_values, Cache): + cache_length = past_key_values.get_seq_length() + past_length = past_key_values.seen_tokens + max_cache_length = past_key_values.get_max_length() + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + +@add_start_docstrings( + """ + The Mistral Model transformer with a sequence classification head on top (linear layer). + + [`MistralForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + MISTRAL_START_DOCSTRING, +) +# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Mistral, LLAMA->MISTRAL +class MistralForSequenceClassification(MistralPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = MistralModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(MISTRAL_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility + sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1 + sequence_lengths = sequence_lengths % input_ids.shape[-1] + sequence_lengths = sequence_lengths.to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) \ No newline at end of file diff --git a/quantization_metric/alphalora/src/mistralconfig.py b/quantization_metric/alphalora/src/mistralconfig.py new file mode 100644 index 0000000000000000000000000000000000000000..74122ce8b05fecbffceb8c2ed7777aa39d24de97 --- /dev/null +++ b/quantization_metric/alphalora/src/mistralconfig.py @@ -0,0 +1,151 @@ +# coding=utf-8 +# Copyright 2023 Mistral AI and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Mistral model configuration""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + + +#from ..deprecated._archive_maps import MISTRAL_PRETRAINED_CONFIG_ARCHIVE_MAP # noqa: F401, E402 + + +class MistralConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`MistralModel`]. It is used to instantiate an + Mistral model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the Mistral-7B-v0.1 or Mistral-7B-Instruct-v0.1. + + [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) + [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the Mistral model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`MistralModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 14336): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 8): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to `4096*32`): + The maximum sequence length that this model might ever be used with. Mistral's sliding window attention + allows sequence of up to 4096*32 tokens. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + pad_token_id (`int`, *optional*): + The id of the padding token. + bos_token_id (`int`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`int`, *optional*, defaults to 2): + The id of the "end-of-sequence" token. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + sliding_window (`int`, *optional*, defaults to 4096): + Sliding window attention window size. If not specified, will default to `4096`. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + + ```python + >>> from transformers import MistralModel, MistralConfig + + >>> # Initializing a Mistral 7B style configuration + >>> configuration = MistralConfig() + + >>> # Initializing a model from the Mistral 7B style configuration + >>> model = MistralModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "mistral" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + _attn_implementation="eager", + vocab_size=32000, + hidden_size=4096, + intermediate_size=14336, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=4096 * 32, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + rope_theta=10000.0, + sliding_window=4096, + attention_dropout=0.0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.sliding_window = sliding_window + self._attn_implementation = "_attn_implementation" + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.attention_dropout = attention_dropout + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) \ No newline at end of file diff --git a/quantization_metric/alphalora/src/mola_lora_hacked.py b/quantization_metric/alphalora/src/mola_lora_hacked.py new file mode 100644 index 0000000000000000000000000000000000000000..101b5e66b580f2b50c33cd4fc7d454350d51c082 --- /dev/null +++ b/quantization_metric/alphalora/src/mola_lora_hacked.py @@ -0,0 +1,892 @@ +# coding=utf-8 +import math +import re +import warnings +from dataclasses import asdict, dataclass, field, replace +from enum import Enum +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers.pytorch_utils import Conv1D + +from peft.import_utils import is_bnb_4bit_available, is_bnb_available +from peft.utils import ( + COMMON_LAYERS_PATTERN, + TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING, + ModulesToSaveWrapper, + PeftConfig, + PeftType, + _freeze_adapter, + _get_submodules, + transpose, +) + +if is_bnb_available(): + import bitsandbytes as bnb + + +@dataclass +class LoraConfig(PeftConfig): + """ + This is the configuration class to store the configuration of a [`LoraModel`]. + + Args: + r (`int`): Lora attention dimension. + target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to. + lora_alpha (`int`): The alpha parameter for Lora scaling. + lora_dropout (`float`): The dropout probability for Lora layers. + fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out). + For example, gpt-2 uses `Conv1D` which stores weights like (fan_in, fan_out) and hence this should be set to `True`.: + bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only' + modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable + and saved in the final checkpoint. + layers_to_transform (`Union[List[int],int]`): + The layer indexes to transform, if this argument is specified, it will apply the LoRA transformations on + the layer indexes that are specified in this list. If a single integer is passed, it will apply the LoRA + transformations on the layer at this index. + layers_pattern (`str`): + The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer + pattern is not in the common layers pattern. + """ + + r: int = field(default=8, metadata={"help": "Lora attention dimension"}) + target_modules: Optional[Union[List[str], str]] = field( + default=None, + metadata={ + "help": "List of module names or regex expression of the module names to replace with Lora." + "For example, ['q', 'v'] or '.*decoder.*(SelfAttention|EncDecAttention).*(q|v)$' " + }, + ) + lora_alpha: int = field(default=8, metadata={"help": "Lora alpha"}) + lora_dropout: float = field(default=0.0, metadata={"help": "Lora dropout"}) + fan_in_fan_out: bool = field( + default=False, + metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"}, + ) + bias: str = field(default="none", metadata={"help": "Bias type for Lora. Can be 'none', 'all' or 'lora_only'"}) + modules_to_save: Optional[List[str]] = field( + default=None, + metadata={ + "help": "List of modules apart from LoRA layers to be set as trainable and saved in the final checkpoint. " + "For example, in Sequence Classification or Token Classification tasks, " + "the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved." + }, + ) + init_lora_weights: bool = field( + default=True, + metadata={"help": "Whether to initialize the weights of the Lora layers."}, + ) + layers_to_transform: Optional[Union[List, int]] = field( + default=None, + metadata={ + "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index." + }, + ) + layers_pattern: Optional[str] = field( + default=None, + metadata={ + "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." + }, + ) + + def __post_init__(self): + self.peft_type = PeftType.LORA + + +class LoraModel(torch.nn.Module): + """ + Creates Low Rank Adapter (Lora) model from a pretrained transformers model. + + Args: + model ([`~transformers.PreTrainedModel`]): The model to be adapted. + config ([`LoraConfig`]): The configuration of the Lora model. + + Returns: + `torch.nn.Module`: The Lora model. + + Example: + + ```py + >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig + >>> from peft import LoraModel, LoraConfig + + >>> config = LoraConfig( + ... peft_type="LORA", + ... task_type="SEQ_2_SEQ_LM", + ... r=8, + ... lora_alpha=32, + ... target_modules=["q", "v"], + ... lora_dropout=0.01, + ... ) + + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> lora_model = LoraModel(config, model) + ``` + + ```py + >>> import transformers + >>> from peft import LoraConfig, PeftModel, get_peft_model, prepare_model_for_int8_training + + >>> target_modules = ["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"] + >>> config = LoraConfig( + ... r=4, lora_alpha=16, target_modules=target_modules, lora_dropout=0.1, bias="none", task_type="CAUSAL_LM" + ... ) + + >>> model = transformers.GPTJForCausalLM.from_pretrained( + ... "kakaobrain/kogpt", + ... revision="KoGPT6B-ryan1.5b-float16", # or float32 version: revision=KoGPT6B-ryan1.5b + ... pad_token_id=tokenizer.eos_token_id, + ... use_cache=False, + ... device_map={"": rank}, + ... torch_dtype=torch.float16, + ... load_in_8bit=True, + ... ) + >>> model = prepare_model_for_int8_training(model) + >>> lora_model = get_peft_model(model, config) + ``` + + **Attributes**: + - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. + - **peft_config** ([`LoraConfig`]): The configuration of the Lora model. + """ + + def __init__(self, model, config, adapter_name, number_experts, top_k): + super().__init__() + self.model = model + self.forward = self.model.forward + self.peft_config = config + self.number_experts = number_experts + self.top_k = top_k + self.add_adapter(adapter_name, self.number_experts, self.top_k, self.peft_config[adapter_name]) + + def add_adapter(self, adapter_name, number_experts, top_k, config=None): + if config is not None: + model_config = self.model.config.to_dict() if hasattr(self.model.config, "to_dict") else self.model.config + config = self._prepare_lora_config(config, model_config) + self.peft_config[adapter_name] = config + self._find_and_replace(adapter_name) + if len(self.peft_config) > 1 and self.peft_config[adapter_name].bias != "none": + raise ValueError( + "LoraModel supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters." + ) + # mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) ##modified + mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) + print("TRAINING MOLA") + + if self.peft_config[adapter_name].inference_mode: + _freeze_adapter(self.model, adapter_name) + + def _check_quantization_dependency(self): + loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False) + loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False) + if (loaded_in_4bit or loaded_in_8bit) and not is_bnb_available(): + raise ImportError( + "To use Lora with 8-bit or 4-bit quantization, please install the `bitsandbytes` package. " + "You can install it with `pip install bitsandbytes`." + ) + + def _check_target_module_exists(self, lora_config, key): + if isinstance(lora_config.target_modules, str): + target_module_found = re.fullmatch(lora_config.target_modules, key) + else: + target_module_found = any(key.endswith(target_key) for target_key in lora_config.target_modules) + is_using_layer_indexes = getattr(lora_config, "layers_to_transform", None) is not None + layer_indexing_pattern = getattr(lora_config, "layers_pattern", None) + + if is_using_layer_indexes and target_module_found: + layers_pattern = COMMON_LAYERS_PATTERN if layer_indexing_pattern is None else layer_indexing_pattern + layers_pattern = [layers_pattern] if isinstance(layers_pattern, str) else layers_pattern + + for pattern in layers_pattern: + layer_index = re.match(f".*.{pattern}\.(\d+)\.*", key) + if layer_index is not None: + layer_index = int(layer_index.group(1)) + if isinstance(lora_config.layers_to_transform, int): + target_module_found = layer_index == lora_config.layers_to_transform + else: + target_module_found = layer_index in lora_config.layers_to_transform + + break + else: + target_module_found = False + return target_module_found + + def _create_new_module(self, lora_config, adapter_name, target, layer_index, number_experts, top_k): ### modified2 + bias = hasattr(target, "bias") and target.bias is not None + kwargs = { + "r": lora_config.r, + "lora_alpha": lora_config.lora_alpha, + "lora_dropout": lora_config.lora_dropout, + "fan_in_fan_out": lora_config.fan_in_fan_out, + "init_lora_weights": lora_config.init_lora_weights, + + } + loaded_in_4bit = getattr(self.model, "is_loaded_in_4bit", False) + loaded_in_8bit = getattr(self.model, "is_loaded_in_8bit", False) + + if loaded_in_8bit and isinstance(target, bnb.nn.Linear8bitLt): + eightbit_kwargs = kwargs.copy() + eightbit_kwargs.update( + { + "has_fp16_weights": target.state.has_fp16_weights, + "memory_efficient_backward": target.state.memory_efficient_backward, + "threshold": target.state.threshold, + "index": target.index, + } + ) + new_module = Linear8bitLt( + adapter_name, target.in_features, target.out_features, bias=bias, **eightbit_kwargs + ) + elif loaded_in_4bit and is_bnb_4bit_available() and isinstance(target, bnb.nn.Linear4bit): + fourbit_kwargs = kwargs.copy() + fourbit_kwargs.update( + { + "compute_dtype": target.compute_dtype, + "compress_statistics": target.weight.compress_statistics, + "quant_type": target.weight.quant_type, + } + ) + new_module = Linear4bit(adapter_name, target.in_features, target.out_features, bias=bias, **fourbit_kwargs) + elif isinstance(target, torch.nn.Embedding): + embedding_kwargs = kwargs.copy() + embedding_kwargs.pop("fan_in_fan_out", None) + in_features, out_features = target.num_embeddings, target.embedding_dim + new_module = Embedding(adapter_name, in_features, out_features, **embedding_kwargs) + elif isinstance(target, torch.nn.Conv2d): + out_channels, in_channels = target.weight.size()[:2] + kernel_size = target.weight.size()[2:] + stride = target.stride + padding = target.padding + new_module = Conv2d(adapter_name, in_channels, out_channels, kernel_size, stride, padding, **kwargs) + else: + if isinstance(target, torch.nn.Linear): + in_features, out_features = target.in_features, target.out_features + if kwargs["fan_in_fan_out"]: + warnings.warn( + "fan_in_fan_out is set to True but the target module is `torch.nn.Linear`. " + "Setting fan_in_fan_out to False." + ) + kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = False + elif isinstance(target, Conv1D): + in_features, out_features = ( + target.weight.ds_shape if hasattr(target.weight, "ds_shape") else target.weight.shape + ) + if not kwargs["fan_in_fan_out"]: + warnings.warn( + "fan_in_fan_out is set to False but the target module is `Conv1D`. " + "Setting fan_in_fan_out to True." + ) + kwargs["fan_in_fan_out"] = lora_config.fan_in_fan_out = True + else: + raise ValueError( + f"Target module {target} is not supported. " + f"Currently, only `torch.nn.Linear` and `Conv1D` are supported." + ) + new_module = Linear_MoE(adapter_name, in_features, out_features, bias=bias, layer_index=layer_index, + number_experts=number_experts, top_k=top_k, **kwargs) ### modified2 ##newmodified + + return new_module + + def _find_and_replace(self, adapter_name): + lora_config = self.peft_config[adapter_name] + self._check_quantization_dependency() + is_target_modules_in_base_model = False + key_list = [key for key, _ in self.model.named_modules()] + + for key in key_list: + if not self._check_target_module_exists(lora_config, key): + continue + + is_target_modules_in_base_model = True + parent, target, target_name = _get_submodules(self.model, key) + layer_index = int(key.split(".")[2]) ### modified2 + + if isinstance(target, LoraLayer) and isinstance(target, torch.nn.Conv2d): + target.update_layer_conv2d( + adapter_name, + lora_config.r, + lora_config.lora_alpha, + lora_config.lora_dropout, + lora_config.init_lora_weights, + ) + elif isinstance(target, LoraLayer): + target.update_layer( + adapter_name, + lora_config.r, + lora_config.lora_alpha, + lora_config.lora_dropout, + lora_config.init_lora_weights, + ) + else: + new_module = self._create_new_module(lora_config, adapter_name, target, layer_index, self.number_experts, self.top_k) ### modified2 + self._replace_module(parent, target_name, new_module, target, layer_index) ### modified2 + + if not is_target_modules_in_base_model: + raise ValueError( + f"Target modules {lora_config.target_modules} not found in the base model. " + f"Please check the target modules and try again." + ) + + def _replace_module(self, parent_module, child_name, new_module, old_module, layer_index): ### modified2 + setattr(parent_module, child_name, new_module) + new_module.weight = old_module.weight + if hasattr(old_module, "bias"): + if old_module.bias is not None: + new_module.bias = old_module.bias + + if getattr(old_module, "state", None) is not None: + new_module.state = old_module.state + new_module.to(old_module.weight.device) + + # dispatch to correct device + ##newmodified + for name, module in new_module.named_modules(): + if "lora_" in name: + module.to(old_module.weight.device) + if "router" in name: + module.to(old_module.weight.device) + if "ranknum" in name: + module.to(old_module.weight.device) + + def __getattr__(self, name: str): + """Forward missing attributes to the wrapped module.""" + try: + return super().__getattr__(name) # defer to nn.Module's logic + except AttributeError: + return getattr(self.model, name) + + def get_peft_config_as_dict(self, inference: bool = False): + config_dict = {} + for key, value in self.peft_config.items(): + config = {k: v.value if isinstance(v, Enum) else v for k, v in asdict(value).items()} + if inference: + config["inference_mode"] = True + config_dict[key] = config + return config + + def _set_adapter_layers(self, enabled=True): + for module in self.model.modules(): + if isinstance(module, LoraLayer): + module.disable_adapters = False if enabled else True + + def enable_adapter_layers(self): + self._set_adapter_layers(enabled=True) + + def disable_adapter_layers(self): + self._set_adapter_layers(enabled=False) + + def set_adapter(self, adapter_name): + for module in self.model.modules(): + if isinstance(module, LoraLayer): + if module.merged: + warnings.warn("Adapter cannot be set when the model is merged. Unmerging the model first.") + module.unmerge() + module.active_adapter = adapter_name + + def merge_adapter(self): + for module in self.model.modules(): + if isinstance(module, LoraLayer): + module.merge() + + def unmerge_adapter(self): + for module in self.model.modules(): + if isinstance(module, LoraLayer): + module.unmerge() + + @staticmethod + def _prepare_lora_config(peft_config, model_config): + if peft_config.target_modules is None: + if model_config["model_type"] not in TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING: + raise ValueError("Please specify `target_modules` in `peft_config`") + peft_config.target_modules = TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING[model_config["model_type"]] + return peft_config + + def merge_and_unload(self): + r""" + This method merges the LoRa layers into the base model. This is needed if someone wants to use the base model + as a standalone model. + + Example: + + ```py + >>> from transformers import AutoModelForCausalLM + >>> from peft import PeftModel + + >>> base_model = AutoModelForCausalLM.from_pretrained("tiiuae/falcon-40b") + >>> peft_model_id = "smangrul/falcon-40B-int4-peft-lora-sfttrainer-sample" + >>> model = PeftModel.from_pretrained(base_model, peft_model_id) + >>> merged_model = model.merge_and_unload() + ``` + """ + if getattr(self.config, "model_type", None) == "gpt2": + raise ValueError("GPT2 models are not supported for merging LORA layers") + + if getattr(self.model, "is_loaded_in_8bit", False) or getattr(self.model, "is_loaded_in_4bit", False): + raise ValueError("Cannot merge LORA layers when the model is loaded in 8-bit mode") + + key_list = [key for key, _ in self.model.named_modules() if "lora" not in key] + for key in key_list: + try: + parent, target, target_name = _get_submodules(self.model, key) + except AttributeError: + continue + if isinstance(target, LoraLayer): + if isinstance(target, nn.Embedding): + new_module = torch.nn.Embedding(target.in_features, target.out_features) + elif isinstance(target, nn.Conv2d): + new_module = torch.nn.Conv2d( + target.in_channels, + target.out_channels, + kernel_size=target.kernel_size, + stride=target.stride, + padding=target.padding, + dilation=target.dilation, + ) + else: + bias = target.bias is not None + new_module = torch.nn.Linear(target.in_features, target.out_features, bias=bias) + target.merge() + self._replace_module(parent, target_name, new_module, target) + + # save any additional trainable modules part of `modules_to_save` + if isinstance(target, ModulesToSaveWrapper): + setattr(parent, target_name, target.modules_to_save[target.active_adapter]) + + return self.model + + def add_weighted_adapter(self, adapters, weights, adapter_name): + if len({self.peft_config[adapter].r for adapter in adapters}) != 1: + raise ValueError("All adapters must have the same r value") + self.peft_config[adapter_name] = replace( + self.peft_config[adapters[0]], lora_alpha=self.peft_config[adapters[0]].r + ) + self._find_and_replace(adapter_name) + mark_only_lora_as_trainable(self.model, self.peft_config[adapter_name].bias) + _freeze_adapter(self.model, adapter_name) + key_list = [key for key, _ in self.model.named_modules() if "lora" not in key] + for key in key_list: + _, target, _ = _get_submodules(self.model, key) + if isinstance(target, LoraLayer): + if adapter_name in target.lora_A: + target.lora_A[adapter_name].weight.data = target.lora_A[adapter_name].weight.data * 0.0 + target.lora_B[adapter_name].weight.data = target.lora_B[adapter_name].weight.data * 0.0 + for adapter, weight in zip(adapters, weights): + if adapter not in target.lora_A: + continue + target.lora_A[adapter_name].weight.data += ( + target.lora_A[adapter].weight.data * weight * target.scaling[adapter] + ) + target.lora_B[adapter_name].weight.data += target.lora_B[adapter].weight.data * weight + + elif adapter_name in target.lora_embedding_A: + target.lora_embedding_A[adapter_name].data = target.lora_embedding_A[adapter_name].data * 0.0 + target.lora_embedding_B[adapter_name].data = target.lora_embedding_B[adapter_name].data * 0.0 + for adapter, weight in zip(adapters, weights): + if adapter not in target.lora_embedding_A: + continue + target.lora_embedding_A[adapter_name].data += ( + target.lora_embedding_A[adapter].data * weight * target.scaling[adapter] + ) + target.lora_embedding_B[adapter_name].data += target.lora_embedding_B[adapter].data * weight + + +# Below code is based on https://github.com/microsoft/LoRA/blob/main/loralib/layers.py +# and modified to work with PyTorch FSDP + + +# ------------------------------------------------------------------------------------------ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +# ------------------------------------------------------------------------------------------ + + +# had to adapt it for `lora_only` to work +##newmodified +def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None: + for n, p in model.named_parameters(): + if "lora_" not in n and 'router' not in n: + p.requires_grad = False + if bias == "none": + return + elif bias == "all": + for n, p in model.named_parameters(): + if "bias" in n: + p.requires_grad = True + elif bias == "lora_only": + for m in model.modules(): + if isinstance(m, LoraLayer) and hasattr(m, "bias") and m.bias is not None: + m.bias.requires_grad = True + else: + raise NotImplementedError + +##modified +def mark_only_lora_B_as_trainable(model: nn.Module, bias: str = "none") -> None: + print('*******************', 'ONLY TRAIN EXPERTS WITHOUT ROUTER', '*******************') + for n, p in model.named_parameters(): + if "lora_" not in n: + p.requires_grad = False + if bias == "none": + return + elif bias == "all": + for n, p in model.named_parameters(): + if "bias" in n: + p.requires_grad = True + elif bias == "lora_only": + for m in model.modules(): + if isinstance(m, LoraLayer) and hasattr(m, "bias") and m.bias is not None: + m.bias.requires_grad = True + else: + raise NotImplementedError + +class LoraLayer: + def __init__(self, in_features: int, out_features: int, **kwargs): + self.r = {} + self.lora_alpha = {} + self.scaling = {} + self.lora_dropout = nn.ModuleDict({}) + self.lora_A = nn.ModuleDict({}) + self.lora_B = nn.ModuleDict({}) + # For Embedding layer + self.lora_embedding_A = nn.ParameterDict({}) + self.lora_embedding_B = nn.ParameterDict({}) + # Mark the weight as unmerged + self.merged = False + self.disable_adapters = False + self.in_features = in_features + self.out_features = out_features + self.kwargs = kwargs + + def update_layer(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, layer_index): ### modified2 + self.r[adapter_name] = r + self.lora_alpha[adapter_name] = lora_alpha + if lora_dropout > 0.0: + lora_dropout_layer = nn.Dropout(p=lora_dropout) + else: + lora_dropout_layer = nn.Identity() + + self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer})) + # Actual trainable parameters + if isinstance(r, list): ### modified2 + self.lora_A.update(nn.ModuleDict({adapter_name: nn.Linear(self.in_features, r[layer_index], bias=False)})) + self.lora_B.update(nn.ModuleDict({adapter_name: nn.Linear(r[layer_index], self.out_features, bias=False)})) + self.scaling[adapter_name] = lora_alpha / r[layer_index] + else: + if r > 0: + self.lora_A.update(nn.ModuleDict({adapter_name: nn.Linear(self.in_features, r, bias=False)})) + self.lora_B.update(nn.ModuleDict({adapter_name: nn.Linear(r, self.out_features, bias=False)})) + self.scaling[adapter_name] = lora_alpha / r + if init_lora_weights: + self.reset_lora_parameters(adapter_name) + self.to(self.weight.device) + + def update_layer_conv2d(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights): + self.r[adapter_name] = r + self.lora_alpha[adapter_name] = lora_alpha + if lora_dropout > 0.0: + lora_dropout_layer = nn.Dropout(p=lora_dropout) + else: + lora_dropout_layer = nn.Identity() + + self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer})) + # Actual trainable parameters + if r > 0: + kernel_size = self.kwargs["kernel_size"] + stride = self.kwargs["stride"] + padding = self.kwargs["padding"] + self.lora_A.update( + nn.ModuleDict({adapter_name: nn.Conv2d(self.in_features, r, kernel_size, stride, padding, bias=False)}) + ) + self.lora_B.update( + nn.ModuleDict({adapter_name: nn.Conv2d(r, self.out_features, (1, 1), (1, 1), bias=False)}) + ) + self.scaling[adapter_name] = lora_alpha / r + if init_lora_weights: + self.reset_lora_parameters(adapter_name) + self.to(self.weight.device) + + def update_layer_embedding(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights): + self.r[adapter_name] = r + self.lora_alpha[adapter_name] = lora_alpha + if lora_dropout > 0.0: + lora_dropout_layer = nn.Dropout(p=lora_dropout) + else: + lora_dropout_layer = nn.Identity() + + self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer})) + # Actual trainable parameters + if r > 0: + self.lora_embedding_A.update( + nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((r, self.in_features)))}) + ) + self.lora_embedding_B.update( + nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((self.out_features, r)))}) + ) + self.scaling[adapter_name] = lora_alpha / r + if init_lora_weights: + self.reset_lora_parameters(adapter_name) + self.to(self.weight.device) + + def reset_lora_parameters(self, adapter_name): + if adapter_name in self.lora_A.keys(): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A[adapter_name].weight, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B[adapter_name].weight) + if adapter_name in self.lora_embedding_A.keys(): + # initialize a the same way as the default for nn.linear and b to zero + nn.init.zeros_(self.lora_embedding_A[adapter_name]) + nn.init.normal_(self.lora_embedding_B[adapter_name]) + +##newmodified +class LoraMoE_Layer: + def __init__(self, in_features: int, out_features: int, **kwargs): + self.r = {} + self.lora_alpha = {} + self.scaling = {} + self.lora_dropout = nn.ModuleDict({}) + self.lora_A = nn.ModuleDict({}) + self.lora_B = nn.ModuleDict({}) + self.router = nn.ModuleDict({}) + # For Embedding layer + self.lora_embedding_A = nn.ParameterDict({}) + self.lora_embedding_B = nn.ParameterDict({}) + # Mark the weight as unmerged + self.merged = False + self.disable_adapters = False + self.in_features = in_features + self.out_features = out_features + self.kwargs = kwargs + + def update_layer(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, layer_index): ### modified2 + self.r[adapter_name] = r + self.lora_alpha[adapter_name] = lora_alpha + if lora_dropout > 0.0: + lora_dropout_layer = nn.Dropout(p=lora_dropout) + else: + lora_dropout_layer = nn.Identity() + + self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer})) + # Actual trainable parameters + if isinstance(r, list): ### modified2 + ##newmodified + # print("LINEAR", self.number_experts) + self.router.update(nn.ModuleDict({'router': nn.Linear(self.in_features, self.number_experts, bias=False)})) + for i in range(self.number_experts): + adapter_name_moe = adapter_name + '_' + str(i) + self.lora_A.update(nn.ModuleDict({adapter_name_moe: nn.Linear(self.in_features, r[layer_index], bias=False)})) + self.lora_B.update(nn.ModuleDict({adapter_name_moe: nn.Linear(r[layer_index], self.out_features, bias=False)})) + self.scaling[adapter_name] = lora_alpha / r[layer_index] + else: + if r > 0: + self.lora_A.update(nn.ModuleDict({adapter_name: nn.Linear(self.in_features, r, bias=False)})) + self.lora_B.update(nn.ModuleDict({adapter_name: nn.Linear(r, self.out_features, bias=False)})) + self.scaling[adapter_name] = lora_alpha / r + ##newmodified + if init_lora_weights: + if isinstance(r, list): + self.reset_lora_parameters_MoE(adapter_name) + else: + self.reset_lora_parameters(adapter_name) + self.to(self.weight.device) + + def update_layer_conv2d(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights): + self.r[adapter_name] = r + self.lora_alpha[adapter_name] = lora_alpha + if lora_dropout > 0.0: + lora_dropout_layer = nn.Dropout(p=lora_dropout) + else: + lora_dropout_layer = nn.Identity() + + self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer})) + # Actual trainable parameters + if r > 0: + kernel_size = self.kwargs["kernel_size"] + stride = self.kwargs["stride"] + padding = self.kwargs["padding"] + self.lora_A.update( + nn.ModuleDict({adapter_name: nn.Conv2d(self.in_features, r, kernel_size, stride, padding, bias=False)}) + ) + self.lora_B.update( + nn.ModuleDict({adapter_name: nn.Conv2d(r, self.out_features, (1, 1), (1, 1), bias=False)}) + ) + self.scaling[adapter_name] = lora_alpha / r + if init_lora_weights: + self.reset_lora_parameters(adapter_name) + self.to(self.weight.device) + + def update_layer_embedding(self, adapter_name, r, lora_alpha, lora_dropout, init_lora_weights): + self.r[adapter_name] = r + self.lora_alpha[adapter_name] = lora_alpha + if lora_dropout > 0.0: + lora_dropout_layer = nn.Dropout(p=lora_dropout) + else: + lora_dropout_layer = nn.Identity() + + self.lora_dropout.update(nn.ModuleDict({adapter_name: lora_dropout_layer})) + # Actual trainable parameters + if r > 0: + self.lora_embedding_A.update( + nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((r, self.in_features)))}) + ) + self.lora_embedding_B.update( + nn.ParameterDict({adapter_name: nn.Parameter(self.weight.new_zeros((self.out_features, r)))}) + ) + self.scaling[adapter_name] = lora_alpha / r + if init_lora_weights: + self.reset_lora_parameters(adapter_name) + self.to(self.weight.device) + ##newmodified + def reset_lora_parameters_MoE(self, adapter_name): + nn.init.kaiming_uniform_(self.router['router'].weight, a=math.sqrt(5)) + for i in range(self.number_experts): + adapter_name_moe = adapter_name + '_' + str(i) + if adapter_name_moe in self.lora_A.keys(): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A[adapter_name_moe].weight, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B[adapter_name_moe].weight) + if adapter_name_moe in self.lora_embedding_A.keys(): + # initialize a the same way as the default for nn.linear and b to zero + nn.init.zeros_(self.lora_embedding_A[adapter_name_moe]) + nn.init.normal_(self.lora_embedding_B[adapter_name_moe]) + def reset_lora_parameters(self, adapter_name): + if adapter_name in self.lora_A.keys(): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A[adapter_name].weight, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B[adapter_name].weight) + if adapter_name in self.lora_embedding_A.keys(): + # initialize a the same way as the default for nn.linear and b to zero + nn.init.zeros_(self.lora_embedding_A[adapter_name]) + nn.init.normal_(self.lora_embedding_B[adapter_name]) + +##newmodified +class Linear_MoE(nn.Linear, LoraMoE_Layer): + # Lora implemented in a dense layer + def __init__( + self, + adapter_name: str, + in_features: int, + out_features: int, + r: int = 0, + lora_alpha: int = 1, + lora_dropout: float = 0.0, + fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) + layer_index: int = 0, ## modified2 + number_experts: list = [8] * 32, + top_k: list = [2] * 32, + **kwargs, + ): + init_lora_weights = kwargs.pop("init_lora_weights", True) + + nn.Linear.__init__(self, in_features, out_features, **kwargs) + ##newmodified + LoraMoE_Layer.__init__(self, in_features=in_features, out_features=out_features) + + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + self.number_experts = number_experts[layer_index] + self.top_k = top_k[layer_index] + self.fan_in_fan_out = fan_in_fan_out + if fan_in_fan_out: + self.weight.data = self.weight.data.T + + nn.Linear.reset_parameters(self) + self.update_layer(adapter_name, r, lora_alpha, lora_dropout, init_lora_weights, layer_index) ##modified2 + self.active_adapter = adapter_name + self.softmax = nn.Softmax(dim=-1) ##modified + self.sigmoid = nn.Sigmoid() + + def merge(self): + if self.active_adapter not in self.lora_A.keys(): + return + if self.merged: + warnings.warn("Already merged. Nothing to do.") + return + if self.r[self.active_adapter] > 0: + self.weight.data += ( + transpose( + self.lora_B[self.active_adapter].weight @ self.lora_A[self.active_adapter].weight, + self.fan_in_fan_out, + ) + * self.scaling[self.active_adapter] + ) + self.merged = True + + def unmerge(self): + if self.active_adapter not in self.lora_A.keys(): + return + if not self.merged: + warnings.warn("Already unmerged. Nothing to do.") + return + if self.r[self.active_adapter] > 0: + self.weight.data -= ( + transpose( + self.lora_B[self.active_adapter].weight @ self.lora_A[self.active_adapter].weight, + self.fan_in_fan_out, + ) + * self.scaling[self.active_adapter] + ) + self.merged = False + + def forward(self, x: torch.Tensor): + previous_dtype = x.dtype + + ##newmodified + ### modified2 elif self.r[self.active_adapter] > 0 and not self.merged: + + result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias) + batch_size, sequence_length, hidden_dim = x.shape + x = x.view(-1, hidden_dim) + router_logits = self.router['router'](x) + # router_probabilities = nn.functional.softmax(router_logits, dim=-1, dtype=self.dtype).to(self.input_dtype) + routing_weights_before = F.softmax(router_logits, dim=1, dtype=torch.float) + routing_weights, selected_experts = torch.topk(routing_weights_before, self.top_k, dim=-1) + routing_weights /= routing_weights.sum(dim=-1, keepdim=True) + routing_weights = routing_weights.to(x.dtype) + + expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.number_experts).permute(2, 1, 0) + + final_hidden_states = torch.zeros( + (batch_size * sequence_length, self.out_features), dtype=x.dtype, device=x.device + ) + + for expert_idx in range(self.number_experts): + adapter_name_moe = self.active_adapter + '_' + str(expert_idx) + idx, top_x = torch.where(expert_mask[expert_idx]) + + if top_x.shape[0] == 0: + continue + + # in torch it is faster to index using lists than torch tensors + top_x_list = top_x.tolist() + idx_list = idx.tolist() + + # Index the correct hidden states and compute the expert hidden state for + # the current expert. We need to make sure to multiply the output hidden + # states by `routing_weights` on the corresponding tokens (top-1 and top-2) + current_state = x[None, top_x_list].reshape(-1, hidden_dim) + expert_output = ( + self.lora_B[adapter_name_moe]( + self.lora_A[adapter_name_moe](self.lora_dropout[self.active_adapter](current_state)) + ) + * self.scaling[self.active_adapter] + ) + current_hidden_states = expert_output * routing_weights[top_x_list, idx_list, None] + + # However `index_add_` only support torch tensors for indexing so we'll use + # the `top_x` tensor here. + final_hidden_states.index_add_(0, top_x, current_hidden_states.to(x.dtype)) + + # x = x.to(self.lora_A[adapter_name_moe].weight.dtype) + + # expert_Gating = router_probabilities[:,:,i].unsqueeze(-1) + # expert_o = expert_Gating * expert_output + # result += router_probabilities[:,:,i].unsqueeze(-1) * expert_output + + final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, self.out_features) + result += final_hidden_states + + + result = result.to(previous_dtype) + + return result, router_logits diff --git a/quantization_metric/alphalora/src/mola_modeling_gemma.py b/quantization_metric/alphalora/src/mola_modeling_gemma.py new file mode 100644 index 0000000000000000000000000000000000000000..a70329fc5dc3c5b387039c79147999ce3bfabe11 --- /dev/null +++ b/quantization_metric/alphalora/src/mola_modeling_gemma.py @@ -0,0 +1,1812 @@ +# coding=utf-8 +# Copyright 2024 Google Inc. HuggingFace Inc. team. All rights reserved. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch Gemma model.""" + +import math +import warnings +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.activations import ACT2FN +from src.cache_utils import Cache, DynamicCache, StaticCache +from src.attn import ( + AttentionMaskConverter, + _prepare_4d_causal_attention_mask, +) +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast +from transformers.modeling_utils import PreTrainedModel + +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS #, is_torch_greater_or_equal_than_1_13 +from transformers.utils import ( + add_start_docstrings, + add_start_docstrings_to_model_forward, + # is_flash_attn_2_available, + # is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from transformers.utils.import_utils import is_torch_fx_available +from src.gemma_config import GemmaConfig + + +# if is_flash_attn_2_available(): +# from flash_attn import flash_attn_func, flash_attn_varlen_func +# from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa + + +# This makes `_prepare_4d_causal_attention_mask` a leaf function in the FX graph. +# It means that the function will not be traced through and simply appear as a node in the graph. +# if is_torch_fx_available(): +# if not is_torch_greater_or_equal_than_1_13: +# import torch.fx +# +# _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask) + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "GemmaConfig" + + +def load_balancing_loss_func(gate_logits: torch.Tensor, num_experts: torch.Tensor = None, top_k=2) -> float: + r""" + Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. + + See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss + function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between + experts is too unbalanced. + + Args: + gate_logits (Union[`torch.Tensor`, Tuple[torch.Tensor]): + Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of + shape [batch_size X sequence_length, num_experts]. + num_experts (`int`, *optional*): + Number of experts + + Returns: + The auxiliary loss. + """ + #print(gate_logits) + if gate_logits is None or not isinstance(gate_logits, tuple): + return 0 + + if isinstance(gate_logits, tuple): + compute_device = gate_logits[0].device + + overall_loss = torch.tensor(0, dtype=torch.float32).to(compute_device) + for layer_i in range(len(gate_logits)): + layer_gate = gate_logits[layer_i] + concatenated_gate_logits = layer_gate.to(compute_device) + + routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) + + _, selected_experts = torch.topk(routing_weights, top_k[layer_i], dim=-1) + + # treat `top_k` as tokens (shape is `top_k X [batch_size X sequence_length]`) + selected_experts = selected_experts.reshape(-1) + + expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts[layer_i]) + expert_mask_1 = torch.sum(expert_mask, dim=-2) + + tokens_per_expert = expert_mask_1 / torch.sum(expert_mask_1) + + # expert_mask = torch.max(expert_mask, dim=-2).values + # + # # Compute the percentage of tokens routed to each experts + # tokens_per_expert = torch.mean(expert_mask.float(), dim=0) + + # Compute the average probability of routing to these experts + router_prob_per_expert = torch.mean(routing_weights, dim=0) + + # overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(-1)) + layer_loss = torch.sum(tokens_per_expert * router_prob_per_expert) * num_experts[layer_i] + overall_loss += layer_loss + return overall_loss + + +def _get_unpad_data(attention_mask): + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0)) + return ( + indices, + cu_seqlens, + max_seqlen_in_batch, + ) + +def detect_nan_inf(var, ctx): + if torch.isnan(var).any().item(): + print(f"{ctx} has ans") + return True + if torch.isinf(var).any().item(): + print(f"{ctx} has inf") + return True + return False + +class GemmaRMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.zeros(dim)) + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + output = self._norm(x.float()) + # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16) + # See https://github.com/huggingface/transformers/pull/29402 + output = output * (1.0 + self.weight.float()) + return output.type_as(x) + + +ALL_LAYERNORM_LAYERS.append(GemmaRMSNorm) + + +class GemmaRotaryEmbedding(nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.register_buffer("inv_freq", None, persistent=False) + + @torch.no_grad() + def forward(self, x, position_ids, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if self.inv_freq is None: + self.inv_freq = 1.0 / ( + self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim) + ) + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + # Force float32 since bfloat16 loses precision on long contexts + # See https://github.com/huggingface/transformers/pull/29285 + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class GemmaMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + + self.lora_target_modules = config.lora_target_modules ##modified + if config.hidden_activation is None: + logger.warning_once( + "Gemma's activation function should be approximate GeLU and not exact GeLU.\n" + "Changing the activation function to `gelu_pytorch_tanh`." + f"if you want to use the legacy `{config.hidden_act}`, " + f"edit the `model.config` to set `hidden_activation={config.hidden_act}` " + " instead of `hidden_act`. See https://github.com/huggingface/transformers/pull/29402 for more details." + ) + hidden_activation = "gelu_pytorch_tanh" + else: + hidden_activation = config.hidden_activation + self.act_fn = ACT2FN[hidden_activation] + + def forward(self, x): + #return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + #modfied + router_logits = [] + if 'up_proj' in self.lora_target_modules: + up, r_l1 = self.up_proj(x) + router_logits.append(r_l1) + else: + up = self.up_proj(x) + + if 'gate_proj' in self.lora_target_modules: + gate, r_l2 = self.gate_proj(x) + router_logits.append(r_l2) + else: + gate = self.gate_proj(x) + + if 'down_proj' in self.lora_target_modules: + down_proj, r_l3 = self.down_proj(self.act_fn(gate) * up) + router_logits.append(r_l3) + else: + down_proj = self.down_proj(self.act_fn(gate) * up) + + if len(router_logits) != 0: + router_logits = torch.cat(router_logits, dim=0) + return down_proj, router_logits + else: + # return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class GemmaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + # Ignore copy + def __init__(self, config: GemmaConfig, layer_idx: Optional[int] = None): + super().__init__() + self.lora_target_modules = config.lora_target_modules ##modified + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = config.head_dim + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + + if self.hidden_size % self.num_heads != 0: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias) + self.rotary_emb = GemmaRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + # query_states = self.q_proj(hidden_states) + # key_states = self.k_proj(hidden_states) + # value_states = self.v_proj(hidden_states) + + # modfied-------------------------------------------------------- + router_logits = [] + if 'q_proj' in self.lora_target_modules: + query_states, r_l1 = self.q_proj(hidden_states) + router_logits.append(r_l1) + else: + query_states = self.q_proj(hidden_states) + + if 'v_proj' in self.lora_target_modules: + value_states, r_l2 = self.v_proj(hidden_states) + router_logits.append(r_l2) + else: + value_states = self.v_proj(hidden_states) + + if 'k_proj' in self.lora_target_modules: + key_states, r_l3 = self.k_proj(hidden_states) + router_logits.append(r_l3) + else: + key_states = self.k_proj(hidden_states) + # modfied-------------------------------------------------------- + + + + + + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + past_key_value = getattr(self, "past_key_value", past_key_value) + cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + + attn_output = attn_output.view(bsz, q_len, -1) + # attn_output = self.o_proj(attn_output) + # + # if not output_attentions: + # attn_weights = None + # + # return attn_output, attn_weights, past_key_value + # modfied-------------------------------------------------------- + if 'o_proj' in self.lora_target_modules: + # if True: + attn_output, r_l = self.o_proj(attn_output) + router_logits.append(r_l) + else: + attn_output = self.o_proj(attn_output) + if not output_attentions: + attn_weights = None + + if len(router_logits) != 0: + router_logits = torch.cat(router_logits, dim=0) + return attn_output, attn_weights, past_key_value, router_logits + # modfied-------------------------------------------------------- + + + + + +# Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2 with Llama->Gemma +# class GemmaFlashAttention2(GemmaAttention): +# """ +# Gemma flash attention module. This module inherits from `GemmaAttention` as the weights of the module stays +# untouched. The only required change would be on the forward pass where it needs to correctly call the public API of +# flash attention and deal with padding tokens in case the input contains any of them. +# """ +# +# def __init__(self, *args, **kwargs): +# super().__init__(*args, **kwargs) +# +# # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. +# # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. +# # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). +# self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() +# +# # Ignore copy +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.LongTensor] = None, +# position_ids: Optional[torch.LongTensor] = None, +# past_key_value: Optional[Cache] = None, +# output_attentions: bool = False, +# use_cache: bool = False, +# cache_position: Optional[torch.LongTensor] = None, +# **kwargs, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# output_attentions = False +# +# bsz, q_len, _ = hidden_states.size() +# +# query_states = self.q_proj(hidden_states) +# key_states = self.k_proj(hidden_states) +# value_states = self.v_proj(hidden_states) +# +# # Flash attention requires the input to have the shape +# # batch_size x seq_length x head_dim x hidden_dim +# # therefore we just need to keep the original shape +# query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) +# key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) +# value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) +# +# cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None) +# query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None) +# +# past_key_value = getattr(self, "past_key_value", past_key_value) +# +# if past_key_value is not None: +# # sin and cos are specific to RoPE models; cache_position needed for the static cache +# cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} +# key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) +# +# # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache +# # to be able to avoid many of these transpose/reshape/view. +# query_states = query_states.transpose(1, 2) +# key_states = key_states.transpose(1, 2) +# value_states = value_states.transpose(1, 2) +# +# dropout_rate = self.attention_dropout if self.training else 0.0 +# +# # In PEFT, usually we cast the layer norms in float32 for training stability reasons +# # therefore the input hidden states gets silently casted in float32. Hence, we need +# # cast them back in the correct dtype just to be sure everything works as expected. +# # This might slowdown training & inference so it is recommended to not cast the LayerNorms +# # in fp32. (GemmaRMSNorm handles it correctly) +# +# input_dtype = query_states.dtype +# if input_dtype == torch.float32: +# if torch.is_autocast_enabled(): +# target_dtype = torch.get_autocast_gpu_dtype() +# # Handle the case where the model is quantized +# elif hasattr(self.config, "_pre_quantization_dtype"): +# target_dtype = self.config._pre_quantization_dtype +# else: +# target_dtype = self.q_proj.weight.dtype +# +# logger.warning_once( +# f"The input hidden states seems to be silently casted in float32, this might be related to" +# f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" +# f" {target_dtype}." +# ) +# +# query_states = query_states.to(target_dtype) +# key_states = key_states.to(target_dtype) +# value_states = value_states.to(target_dtype) +# +# attn_output = self._flash_attention_forward( +# query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate +# ) +# +# attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() +# attn_output = self.o_proj(attn_output) +# +# if not output_attentions: +# attn_weights = None +# +# return attn_output, attn_weights, past_key_value +# +# def _flash_attention_forward( +# self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None +# ): +# """ +# Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token +# first unpad the input, then computes the attention scores and pad the final attention scores. +# +# Args: +# query_states (`torch.Tensor`): +# Input query states to be passed to Flash Attention API +# key_states (`torch.Tensor`): +# Input key states to be passed to Flash Attention API +# value_states (`torch.Tensor`): +# Input value states to be passed to Flash Attention API +# attention_mask (`torch.Tensor`): +# The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the +# position of padding tokens and 1 for the position of non-padding tokens. +# dropout (`float`): +# Attention dropout +# softmax_scale (`float`, *optional*): +# The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) +# """ +# if not self._flash_attn_uses_top_left_mask: +# causal = self.is_causal +# else: +# # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in GemmaFlashAttention2 __init__. +# causal = self.is_causal and query_length != 1 +# +# # Contains at least one padding token in the sequence +# if attention_mask is not None: +# batch_size = query_states.shape[0] +# query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input( +# query_states, key_states, value_states, attention_mask, query_length +# ) +# +# cu_seqlens_q, cu_seqlens_k = cu_seq_lens +# max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens +# +# attn_output_unpad = flash_attn_varlen_func( +# query_states, +# key_states, +# value_states, +# cu_seqlens_q=cu_seqlens_q, +# cu_seqlens_k=cu_seqlens_k, +# max_seqlen_q=max_seqlen_in_batch_q, +# max_seqlen_k=max_seqlen_in_batch_k, +# dropout_p=dropout, +# softmax_scale=softmax_scale, +# causal=causal, +# ) +# +# attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length) +# else: +# attn_output = flash_attn_func( +# query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal +# ) +# +# return attn_output +# +# def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length): +# indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask) +# batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape +# +# key_layer = index_first_axis( +# key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k +# ) +# value_layer = index_first_axis( +# value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k +# ) +# if query_length == kv_seq_len: +# query_layer = index_first_axis( +# query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k +# ) +# cu_seqlens_q = cu_seqlens_k +# max_seqlen_in_batch_q = max_seqlen_in_batch_k +# indices_q = indices_k +# elif query_length == 1: +# max_seqlen_in_batch_q = 1 +# cu_seqlens_q = torch.arange( +# batch_size + 1, dtype=torch.int32, device=query_layer.device +# ) # There is a memcpy here, that is very bad. +# indices_q = cu_seqlens_q[:-1] +# query_layer = query_layer.squeeze(1) +# else: +# # The -q_len: slice assumes left padding. +# attention_mask = attention_mask[:, -query_length:] +# query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask) +# +# return ( +# query_layer, +# key_layer, +# value_layer, +# indices_q, +# (cu_seqlens_q, cu_seqlens_k), +# (max_seqlen_in_batch_q, max_seqlen_in_batch_k), +# ) +# +# +# # Copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Gemma +# class GemmaSdpaAttention(GemmaAttention): +# """ +# Gemma attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from +# `GemmaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to +# SDPA API. +# """ +# +# # Ignore copy +# def forward( +# self, +# hidden_states: torch.Tensor, +# attention_mask: Optional[torch.Tensor] = None, +# position_ids: Optional[torch.LongTensor] = None, +# past_key_value: Optional[Cache] = None, +# output_attentions: bool = False, +# use_cache: bool = False, +# cache_position: Optional[torch.LongTensor] = None, +# ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: +# if output_attentions: +# # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. +# logger.warning_once( +# "GemmaModel is using GemmaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " +# 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' +# ) +# return super().forward( +# hidden_states=hidden_states, +# attention_mask=attention_mask, +# position_ids=position_ids, +# past_key_value=past_key_value, +# output_attentions=output_attentions, +# use_cache=use_cache, +# cache_position=cache_position, +# ) +# +# bsz, q_len, _ = hidden_states.size() +# +# query_states = self.q_proj(hidden_states) +# key_states = self.k_proj(hidden_states) +# value_states = self.v_proj(hidden_states) +# +# query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) +# key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) +# value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) +# +# cos, sin = self.rotary_emb(value_states, position_ids, seq_len=None) +# query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, None) +# +# past_key_value = getattr(self, "past_key_value", past_key_value) +# +# if past_key_value is not None: +# # sin and cos are specific to RoPE models; cache_position needed for the static cache +# cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} +# key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) +# +# key_states = repeat_kv(key_states, self.num_key_value_groups) +# value_states = repeat_kv(value_states, self.num_key_value_groups) +# +# causal_mask = attention_mask +# if attention_mask is not None: +# causal_mask = causal_mask[:, :, :, : key_states.shape[-2]] +# +# # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, +# # Reference: https://github.com/pytorch/pytorch/issues/112577. +# if query_states.device.type == "cuda" and causal_mask is not None: +# query_states = query_states.contiguous() +# key_states = key_states.contiguous() +# value_states = value_states.contiguous() +# +# # In case we are not compiling, we may set `causal_mask` to None, which is required to dispatch to SDPA's Flash Attention 2 backend, rather +# # relying on the `is_causal` argument. +# attn_output = torch.nn.functional.scaled_dot_product_attention( +# query_states, +# key_states, +# value_states, +# attn_mask=causal_mask, +# dropout_p=self.attention_dropout if self.training else 0.0, +# is_causal=causal_mask is None and q_len > 1, +# ) +# +# attn_output = attn_output.transpose(1, 2).contiguous() +# attn_output = attn_output.view(bsz, q_len, -1) +# +# attn_output = self.o_proj(attn_output) +# +# return attn_output, None, past_key_value + + +GEMMA_ATTENTION_CLASSES = { + "eager": GemmaAttention, + # "flash_attention_2": GemmaFlashAttention2, + # "sdpa": GemmaSdpaAttention, +} + + +# Copied from transformers.models.llama.modeling_llama.LlamaDecoderLayer with LLAMA->GEMMA,Llama->Gemma +class GemmaDecoderLayer(nn.Module): + def __init__(self, config: GemmaConfig, layer_idx: int): + super().__init__() + self.lora_target_modules = config.lora_target_modules ##modified + self.hidden_size = config.hidden_size + + #self.self_attn = GEMMA_ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx) + self.self_attn = GEMMA_ATTENTION_CLASSES["eager"](config=config, layer_idx=layer_idx) + self.mlp = GemmaMLP(config) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + output_router_logits: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): + attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, + query_sequence_length, key_sequence_length)` if default attention is used. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + if "padding_mask" in kwargs: + warnings.warn( + "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`" + ) + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + # hidden_states, self_attn_weights, present_key_value = self.self_attn( + # hidden_states=hidden_states, + # attention_mask=attention_mask, + # position_ids=position_ids, + # past_key_value=past_key_value, + # output_attentions=output_attentions, + # use_cache=use_cache, + # cache_position=cache_position, + # **kwargs, + # ) + + hidden_states, self_attn_weights, present_key_value, router_logits = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + + #hidden_states = self.mlp(hidden_states) + + if 'down_proj' in self.lora_target_modules or 'gate_proj' in self.lora_target_modules or 'up_proj' in self.lora_target_modules: + ##newmodified + hidden_states, r_l = self.mlp(hidden_states) + if len(router_logits) != 0: + router_logits = torch.cat([router_logits, r_l], dim=0) + else: + router_logits.append(r_l) + router_logits = torch.cat(router_logits, dim=0) + else: + hidden_states = self.mlp(hidden_states) + + #modfied-------------------------------------------------------- + + + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + if output_router_logits: ##newmodified + outputs += (router_logits,) + + return outputs + + +GEMMA_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`GemmaConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Gemma Model outputting raw hidden-states without any specific head on top.", + GEMMA_START_DOCSTRING, +) +class GemmaPreTrainedModel(PreTrainedModel): + config_class = GemmaConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _keep_in_fp32_modules = ["inv_freq", "rotary_emb", "cos_cached", "sin_cached"] + _no_split_modules = ["GemmaDecoderLayer"] + _skip_keys_device_placement = ["past_key_values", "causal_mask"] + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_cache_class = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _setup_cache(self, cache_cls, max_batch_size, max_cache_len: Optional[int] = None): + if self.config._attn_implementation == "flash_attention_2" and cache_cls == StaticCache: + raise ValueError( + "`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` " + "make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformers" + ) + + for layer in self.model.layers: + weights = layer.self_attn.o_proj.weight + layer.self_attn.past_key_value = cache_cls( + self.config, max_batch_size, max_cache_len, device=weights.device, dtype=weights.dtype + ) + + def _reset_cache(self): + for layer in self.model.layers: + layer.self_attn.past_key_value = None + + +GEMMA_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + Two formats are allowed: + - a [`~cache_utils.Cache`] instance; + - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy + cache format. + + The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the + legacy cache format will be returned. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, + this tensor is not affected by padding. It is used to update the cache in the correct position and to infer + the complete sequence length. +""" + + +@add_start_docstrings( + "The bare Gemma Model outputting raw hidden-states without any specific head on top.", + GEMMA_START_DOCSTRING, +) +# Copied from transformers.models.llama.modeling_llama.LlamaModel with LLAMA->GEMMA,Llama->Gemma +class GemmaModel(GemmaPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`GemmaDecoderLayer`] + + Args: + config: GemmaConfig + """ + + def __init__(self, config: GemmaConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [GemmaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) + # Ignore copy + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, ##newmodified + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + #modfied-------------------------------------------------------- + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You cannot specify both input_ids and inputs_embeds at the same time, and must specify either one" + ) + + if self.gradient_checkpointing and self.training and use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + use_cache = False + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + past_seen_tokens = 0 + if use_cache: # kept for BC (cache positions) + if not isinstance(past_key_values, StaticCache): + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + past_seen_tokens = past_key_values.get_seq_length() + + if cache_position is None: + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask(attention_mask, inputs_embeds, cache_position, past_seen_tokens) + + # embed positions + hidden_states = inputs_embeds + + # normalized + # Gemma downcasts the below to float16, causing sqrt(3072)=55.4256 to become 55.5 + # See https://github.com/huggingface/transformers/pull/29402 + normalizer = torch.tensor(self.config.hidden_size**0.5, dtype=hidden_states.dtype) + hidden_states = hidden_states * normalizer + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = None + + #modfied-------------------------------------------------------- + all_router_logits = () if output_router_logits else None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + #modfied-------------------------------------------------------- + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + output_router_logits, # newmodified + use_cache, + cache_position, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + output_router_logits=output_router_logits, # newmodified + use_cache=use_cache, + cache_position=cache_position, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + #modfied-------------------------------------------------------- + if output_router_logits: + all_router_logits += (layer_outputs[-1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = None + if use_cache: + next_cache = ( + next_decoder_cache.to_legacy_cache() if isinstance(next_decoder_cache, Cache) else next_decoder_cache + ) + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ), all_router_logits # newmodified + + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_seen_tokens: int, + ): + # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static + # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. + # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using + # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 + + # if self.config._attn_implementation == "flash_attention_2": + # if attention_mask is not None and 0.0 in attention_mask: + # return attention_mask + # return None + # + # if self.config._attn_implementation == "sdpa": + # # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, + # # in order to dispatch on Flash Attention 2. + # if AttentionMaskConverter._ignore_causal_mask_sdpa( + # attention_mask, inputs_embeds=input_tensor, past_key_values_length=past_seen_tokens + # ): + # return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + if hasattr(getattr(self.layers[0], "self_attn", {}), "past_key_value"): # static cache + target_length = self.config.max_position_embeddings + else: # dynamic cache + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device) + if sequence_length != 1: + causal_mask = torch.triu(causal_mask, diagonal=1) + causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + if attention_mask.dim() == 2: + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + elif attention_mask.dim() == 4: + # backwards compatibility: we allow passing a 4D attention mask shorter than the input length with + # cache. In that case, the 4D attention mask attends to the newest tokens only. + if attention_mask.shape[-2] < cache_position[0] + sequence_length: + offset = cache_position[0] + else: + offset = 0 + mask_shape = attention_mask.shape + mask_slice = (attention_mask.eq(0.0)).to(dtype=dtype) * min_dtype + causal_mask[ + : mask_shape[0], : mask_shape[1], offset : mask_shape[2] + offset, : mask_shape[3] + ] = mask_slice + + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type == "cuda" + ): + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + +# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM with LLAMA->GEMMA,Llama->Gemma,llama->gemma +class GemmaForCausalLM(GemmaPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = GemmaModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + # Ignore copy + @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, GemmaForCausalLM + + >>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b") + >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b") + + >>> prompt = "What is your favorite condiment?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "What is your favorite condiment?" + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, cache_position=None, **kwargs + ): + # With static cache, the `past_key_values` is None + # TODO joao: standardize interface for the different Cache classes and remove of this if + has_static_cache = False + if past_key_values is None: + past_key_values = getattr(getattr(self.model.layers[0], "self_attn", {}), "past_key_value", None) + has_static_cache = past_key_values is not None + + past_length = 0 + if past_key_values is not None: + if isinstance(past_key_values, Cache): + past_length = cache_position[0] if cache_position is not None else past_key_values.get_seq_length() + max_cache_length = ( + torch.tensor(past_key_values.get_max_length(), device=input_ids.device) + if past_key_values.get_max_length() is not None + else None + ) + cache_length = past_length if max_cache_length is None else torch.min(max_cache_length, past_length) + # TODO joao: remove this `else` after `generate` prioritizes `Cache` objects + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise + # recompiles graphs as the stride of the inputs is a guard. Ref: https://github.com/huggingface/transformers/pull/29114 + # TODO: use `next_tokens` directly instead. + model_inputs = {"input_ids": input_ids.contiguous()} + + input_length = position_ids.shape[-1] if position_ids is not None else input_ids.shape[-1] + if cache_position is None: + cache_position = torch.arange(past_length, past_length + input_length, device=input_ids.device) + else: + cache_position = cache_position[-input_length:] + + if has_static_cache: + past_key_values = None + + model_inputs.update( + { + "position_ids": position_ids, + "cache_position": cache_position, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + +class GemmaForCausalLM_d(GemmaPreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.router_aux_loss_coef = 0.001 + self.model = GemmaModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_new_parameters(self, number_experts, top_k, oblance): + self.obalance = oblance + self.number_experts = number_experts + self.top_k = top_k + print("NEW PARAMETERS", "obalance", self.obalance) + + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + # Ignore copy + @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, GemmaForCausalLM + + >>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b") + >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b") + + >>> prompt = "What is your favorite condiment?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "What is your favorite condiment?" + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + + #modfied-------------------------------------------------------- + output_router_logits = ( + output_router_logits if output_router_logits is not None else False + ) + + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs, router_logits = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, # newmodified + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + logits = logits.float() + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + # if not return_dict: + # output = (logits,) + outputs[1:] + # return (loss,) + output if loss is not None else output + ##newmodified + aux_loss = None + if output_router_logits: + aux_loss = load_balancing_loss_func( + router_logits if return_dict else outputs[-1], self.number_experts, self.top_k + ) + + # print('ce:', loss) ##modified + if self.obalance: + aux_loss = self.router_aux_loss_coef * aux_loss + loss = loss.cuda(0) + aux_loss.cuda(0) + + print('blance:', aux_loss.clone().detach().cpu()) + print('total loss:', loss.clone().detach().cpu()) + + ##newmodified + if not return_dict: + output = (logits,) + outputs[1:] + if output_router_logits: + output = (aux_loss,) + output + return (loss,) + output if loss is not None else output + + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, cache_position=None, **kwargs + ): + # With static cache, the `past_key_values` is None + # TODO joao: standardize interface for the different Cache classes and remove of this if + has_static_cache = False + if past_key_values is None: + past_key_values = getattr(getattr(self.model.layers[0], "self_attn", {}), "past_key_value", None) + has_static_cache = past_key_values is not None + + past_length = 0 + if past_key_values is not None: + if isinstance(past_key_values, Cache): + past_length = cache_position[0] if cache_position is not None else past_key_values.get_seq_length() + max_cache_length = ( + torch.tensor(past_key_values.get_max_length(), device=input_ids.device) + if past_key_values.get_max_length() is not None + else None + ) + cache_length = past_length if max_cache_length is None else torch.min(max_cache_length, past_length) + # TODO joao: remove this `else` after `generate` prioritizes `Cache` objects + else: + cache_length = past_length = past_key_values[0][0].shape[2] + max_cache_length = None + + # Keep only the unprocessed tokens: + # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where + # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as + # input) + if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]: + input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :] + # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard + # input_ids based on the past_length. + elif past_length < input_ids.shape[1]: + input_ids = input_ids[:, past_length:] + # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens. + + # If we are about to go beyond the maximum cache length, we need to crop the input attention mask. + if ( + max_cache_length is not None + and attention_mask is not None + and cache_length + input_ids.shape[1] > max_cache_length + ): + attention_mask = attention_mask[:, -max_cache_length:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + # The `contiguous()` here is necessary to have a static stride during decoding. torchdynamo otherwise + # recompiles graphs as the stride of the inputs is a guard. Ref: https://github.com/huggingface/transformers/pull/29114 + # TODO: use `next_tokens` directly instead. + model_inputs = {"input_ids": input_ids.contiguous()} + + input_length = position_ids.shape[-1] if position_ids is not None else input_ids.shape[-1] + if cache_position is None: + cache_position = torch.arange(past_length, past_length + input_length, device=input_ids.device) + else: + cache_position = cache_position[-input_length:] + + if has_static_cache: + past_key_values = None + + model_inputs.update( + { + "position_ids": position_ids, + "cache_position": cache_position, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past), + ) + return reordered_past + + + +@add_start_docstrings( + """ + The Gemma Model transformer with a sequence classification head on top (linear layer). + + [`GemmaForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + GEMMA_START_DOCSTRING, +) +# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with LLAMA->GEMMA,Llama->Gemma +class GemmaForSequenceClassification(GemmaPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = GemmaModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(GEMMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility + sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1 + sequence_lengths = sequence_lengths % input_ids.shape[-1] + sequence_lengths = sequence_lengths.to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + labels = labels.to(logits.device) + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) \ No newline at end of file diff --git a/quantization_metric/alphalora/src/mola_modeling_llama_hacked.py b/quantization_metric/alphalora/src/mola_modeling_llama_hacked.py new file mode 100644 index 0000000000000000000000000000000000000000..858db43476a807056a7133347ad82db9086e16e0 --- /dev/null +++ b/quantization_metric/alphalora/src/mola_modeling_llama_hacked.py @@ -0,0 +1,1488 @@ +# coding=utf-8 +""" PyTorch LLaMA model.""" +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.activations import ACT2FN +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from transformers.models.llama.configuration_llama import LlamaConfig + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "LlamaConfig" +#newmodified +def load_balancing_loss_func(gate_logits: torch.Tensor, num_experts: torch.Tensor = None, top_k=2) -> float: + r""" + Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. + + See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss + function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between + experts is too unbalanced. + + Args: + gate_logits (Union[`torch.Tensor`, Tuple[torch.Tensor]): + Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of + shape [batch_size X sequence_length, num_experts]. + num_experts (`int`, *optional*): + Number of experts + + Returns: + The auxiliary loss. + """ + if gate_logits is None or not isinstance(gate_logits, tuple): + return 0 + + if isinstance(gate_logits, tuple): + compute_device = gate_logits[0].device + + overall_loss = torch.tensor(0, dtype=torch.float32).to(compute_device) + for layer_i in range(len(gate_logits)): + layer_gate = gate_logits[layer_i] + concatenated_gate_logits = layer_gate.to(compute_device) + + routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) + + _, selected_experts = torch.topk(routing_weights, top_k[layer_i], dim=-1) + + # treat `top_k` as tokens (shape is `top_k X [batch_size X sequence_length]`) + selected_experts = selected_experts.reshape(-1) + + expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts[layer_i]) + expert_mask_1 = torch.sum(expert_mask, dim=-2) + + tokens_per_expert = expert_mask_1 / torch.sum(expert_mask_1) + + # expert_mask = torch.max(expert_mask, dim=-2).values + # + # # Compute the percentage of tokens routed to each experts + # tokens_per_expert = torch.mean(expert_mask.float(), dim=0) + + # Compute the average probability of routing to these experts + router_prob_per_expert = torch.mean(routing_weights, dim=0) + + # overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(-1)) + layer_loss = torch.sum(tokens_per_expert * router_prob_per_expert) * num_experts[layer_i] + overall_loss += layer_loss + return overall_loss + + +# def load_balancing_loss_func(gate_logits: torch.Tensor, num_experts: torch.Tensor = None, top_k=2) -> float: +# r""" +# Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. +# +# See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss +# function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between +# experts is too unbalanced. +# +# Args: +# gate_logits (Union[`torch.Tensor`, Tuple[torch.Tensor]): +# Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of +# shape [batch_size X sequence_length, num_experts]. +# num_experts (`int`, *optional*): +# Number of experts +# +# Returns: +# The auxiliary loss. +# """ +# if gate_logits is None or not isinstance(gate_logits, tuple): +# return 0 +# +# if isinstance(gate_logits, tuple): +# compute_device = gate_logits[0].device +# concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0) +# +# routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1) +# +# _, selected_experts = torch.topk(routing_weights, top_k, dim=-1) +# +# # treat `top_k` as tokens (shape is `top_k X [batch_size X sequence_length]`) +# selected_experts = selected_experts.reshape(-1) +# +# expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts) +# expert_mask_1 = torch.sum(expert_mask, dim=-2) +# +# tokens_per_expert = expert_mask_1 / torch.sum(expert_mask_1) +# +# # expert_mask = torch.max(expert_mask, dim=-2).values +# # +# # # Compute the percentage of tokens routed to each experts +# # tokens_per_expert = torch.mean(expert_mask.float(), dim=0) +# +# # Compute the average probability of routing to these experts +# router_prob_per_expert = torch.mean(routing_weights, dim=0) +# +# # overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(-1)) +# overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert) +# return overall_loss * num_experts + +def detect_nan_inf(var, ctx): + if torch.isnan(var).any().item(): + print(f"{ctx} has ans") + return True + if torch.isinf(var).any().item(): + print(f"{ctx} has inf") + return True + return False + +def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) + mask_cond = torch.arange(mask.size(-1)) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class LlamaRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + + +class LlamaRotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim)) + self.register_buffer("inv_freq", inv_freq) + + # Build here to make `torch.jit.trace` work. + self.max_seq_len_cached = max_position_embeddings + t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case. + if seq_len > self.max_seq_len_cached: + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1).to(x.device) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) + return ( + self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + ) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0): + cos = cos[..., offset : q.shape[-2] + offset, :] + sin = sin[..., offset : q.shape[-2] + offset, :] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class LlamaMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + config: LlamaConfig, ##modified + ): + super().__init__() + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.act_fn = ACT2FN[hidden_act] + self.lora_target_modules = config.lora_target_modules ##modified + + def forward(self, x): + ##modified + router_logits = [] + if 'up_proj' in self.lora_target_modules: + up, r_l1 = self.up_proj(x) + router_logits.append(r_l1) + else: + up = self.up_proj(x) + + if 'gate_proj' in self.lora_target_modules: + gate, r_l2 = self.gate_proj(x) + router_logits.append(r_l2) + else: + gate = self.gate_proj(x) + + if 'down_proj' in self.lora_target_modules: + down_proj, r_l3 = self.down_proj(self.act_fn(gate) * up) + router_logits.append(r_l3) + else: + down_proj = self.down_proj(self.act_fn(gate) * up) + + if len(router_logits) != 0: + router_logits = torch.cat(router_logits, dim=0) + return down_proj, router_logits + else: + # return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + + + +class LlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + hidden_size: int, + num_heads: int, + config: LlamaConfig, ##modified + ): + super().__init__() + self.lora_target_modules = config.lora_target_modules ##modified + + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + + if (self.head_dim * num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {num_heads})." + ) + self.q_proj = nn.Linear( + hidden_size, + num_heads * self.head_dim, + bias=False, + ) + self.k_proj = nn.Linear( + hidden_size, + num_heads * self.head_dim, + bias=False, + ) + self.v_proj = nn.Linear( + hidden_size, + num_heads * self.head_dim, + bias=False, + ) + self.o_proj = nn.Linear( + num_heads * self.head_dim, + hidden_size, + bias=False, + ) + self.rotary_emb = LlamaRotaryEmbedding(self.head_dim) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + detect_nan_inf(hidden_states , "hidden_states detect") + with torch.cuda.amp.autocast(enabled=False): + bsz, q_len, _ = hidden_states.size() + ##newmodified + router_logits = [] + if 'q_proj' in self.lora_target_modules: + query_states, r_l1 = self.q_proj(hidden_states) + router_logits.append(r_l1) + else: + query_states = self.q_proj(hidden_states) + + if 'v_proj' in self.lora_target_modules: + value_states, r_l2 = self.v_proj(hidden_states) + router_logits.append(r_l2) + else: + value_states = self.v_proj(hidden_states) + + if 'k_proj' in self.lora_target_modules: + key_states, r_l3 = self.k_proj(hidden_states) + router_logits.append(r_l3) + else: + key_states = self.k_proj(hidden_states) + + # if True: + # if 'q_proj' in self.lora_target_modules: + # query_states, r_l1 = self.q_proj(hidden_states) + # # query_states = query_states[0] + # if 'v_proj' in self.lora_target_modules: + # value_states, r_l2 = self.v_proj(hidden_states) + # # value_states = value_states[0] + # if 'k_proj' in self.lora_target_modules: + # key_states, r_l3 = self.k_proj(hidden_states) + # # key_states = key_states[0] + # router_logits = torch.cat([layer_gate for layer_gate in [r_l1, r_l2, r_l3]], dim=0) + # else: + # query_states = self.q_proj(hidden_states) + # value_states = self.v_proj(hidden_states) + # key_states = self.k_proj(hidden_states) + # query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + # key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + # value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + + kv_seq_len = key_states.shape[-2] + offset = 0 + if past_key_value is not None: + offset = past_key_value[0].shape[-2] + kv_seq_len += offset + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, offset=offset) + # [bsz, nh, t, hd] + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + if detect_nan_inf(attn_weights , "attn_weights matmul"): + attn_weights = torch.matmul(query_states.to(torch.float32), key_states.transpose(2, 3).to(torch.float32)).to(torch.float32) / math.sqrt(self.head_dim) + print("second:", torch.isnan(attn_weights).any().item(), torch.isinf(attn_weights).any().item()) + + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + ##modified + ##newmodified + if 'o_proj' in self.lora_target_modules: + # if True: + attn_output, r_l = self.o_proj(attn_output) + router_logits.append(r_l) + else: + attn_output = self.o_proj(attn_output) + if not output_attentions: + attn_weights = None + + if len(router_logits) != 0: + router_logits = torch.cat(router_logits, dim=0) + return attn_output, attn_weights, past_key_value, router_logits + + # def forward(self, + # hidden_states: torch.Tensor, + # past_key_value: Optional[Tuple[torch.Tensor]] = None, + # attention_mask: Optional[torch.Tensor] = None, + # output_attentions: bool = False, + # use_cache: bool = False): + # with torch.cuda.amp.autocast(enabled=False): + # return self._forward(hidden_states) + + +class LlamaDecoderLayer(nn.Module): + def __init__(self, config: LlamaConfig): + super().__init__() + + self.lora_target_modules = config.lora_target_modules ##modified + self.hidden_size = config.hidden_size + ##modified + self.self_attn = LlamaAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + config=config, + ) + self.mlp = LlamaMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + config=config, + ) + + self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + ##newmodified + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + output_router_logits: Optional[bool] = False, + use_cache: Optional[bool] = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + ##newmodified + hidden_states, self_attn_weights, present_key_value, router_logits= self.self_attn( + hidden_states=hidden_states, + past_key_value=past_key_value, + attention_mask=attention_mask, + output_attentions=output_attentions, + use_cache=use_cache, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + ##modified + ##newmodified + if 'down_proj' in self.lora_target_modules or 'gate_proj' in self.lora_target_modules or 'up_proj' in self.lora_target_modules: + ##newmodified + hidden_states, r_l = self.mlp(hidden_states) + if len(router_logits) != 0: + router_logits = torch.cat([router_logits, r_l], dim=0) + else: + router_logits.append(r_l) + router_logits = torch.cat(router_logits, dim=0) + else: + hidden_states = self.mlp(hidden_states) + + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + ##newmodified + if output_router_logits: + outputs += (router_logits,) + + return outputs + + +LLAMA_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`LlamaConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", + LLAMA_START_DOCSTRING, +) +class LlamaPreTrainedModel(PreTrainedModel): + config_class = LlamaConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["LlamaDecoderLayer"] + _keys_to_ignore_on_load_unexpected = [r"decoder\.version"] + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, LlamaModel): + module.gradient_checkpointing = value + + +LLAMA_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", + LLAMA_START_DOCSTRING, +) +class LlamaModel(LlamaPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] + + Args: + config: LlamaConfig + """ + ##modified + def __init__(self, config: LlamaConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + if hasattr(config, "mm_vision_tower"): + from transformers import CLIPVisionModel + self.vision_tower = [CLIPVisionModel.from_pretrained(config.mm_vision_tower)] + + if hasattr(config, "use_mm_proj"): + self.mm_projector = nn.Linear(config.mm_hidden_size, config.hidden_size) + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length + ).to(inputs_embeds.device) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + ##newmodified + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + images: Optional[torch.FloatTensor] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + ##newmodified + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + + # HACK: replace back original embeddings for pretraining + orig_embeds_params = getattr(self, 'orig_embeds_params', None) + if orig_embeds_params is not None: + orig_embeds_params = orig_embeds_params[0] + with torch.no_grad(): + self.get_input_embeddings().weight[:-2] = orig_embeds_params[:-2].data + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + vision_tower = getattr(self, 'vision_tower', None) + if vision_tower is not None and (input_ids.shape[1] != 1 or self.training) and images is not None: + # TODO: this is a modified multimodal LLM -- Haotian Liu + vision_tower = vision_tower[0] # HACK: for FSDP + with torch.no_grad(): + if type(images) is list: + # variable length images + image_features = [] + for image in images: + image_forward_out = vision_tower(image.unsqueeze(0), output_hidden_states=True) + select_hidden_state_layer = getattr(self.config, "mm_vision_select_layer", -1) + select_hidden_state = image_forward_out.hidden_states[select_hidden_state_layer] + image_feature = select_hidden_state[:, 1:] + image_features.append(image_feature) + else: + image_forward_outs = vision_tower(images, output_hidden_states=True) + select_hidden_state_layer = getattr(self.config, "mm_vision_select_layer", -1) + select_hidden_state = image_forward_outs.hidden_states[select_hidden_state_layer] + image_features = select_hidden_state[:, 1:] + if type(images) is list: + image_features = [self.mm_projector(image_feature)[0] for image_feature in image_features] + else: + image_features = self.mm_projector(image_features) + dummy_image_features = torch.zeros(256, 1024, device=inputs_embeds.device, dtype=inputs_embeds.dtype) + dummy_image_features = self.mm_projector(dummy_image_features) + + new_input_embeds = [] + cur_image_idx = 0 + for cur_input_ids, cur_input_embeds in zip(input_ids, inputs_embeds): + if (cur_input_ids == vision_tower.config.im_patch_token).sum() == 0: + # multimodal LLM, but the current sample is not multimodal + cur_input_embeds = cur_input_embeds + (0. * dummy_image_features).sum() + new_input_embeds.append(cur_input_embeds) + continue + if vision_tower.config.use_im_start_end: + cur_image_features = image_features[cur_image_idx] + num_patches = cur_image_features.shape[0] + if (cur_input_ids == vision_tower.config.im_start_token).sum() != (cur_input_ids == vision_tower.config.im_end_token).sum(): + raise ValueError("The number of im_start_token and im_end_token should be the same") + image_start_tokens = torch.where(cur_input_ids == vision_tower.config.im_start_token)[0] + for image_start_token_pos in image_start_tokens: + cur_image_features = image_features[cur_image_idx].to(device=cur_input_embeds.device) + num_patches = cur_image_features.shape[0] + if cur_input_ids[image_start_token_pos + num_patches + 1] != vision_tower.config.im_end_token: + raise ValueError("Seems that the image is cut.") + cur_new_input_embeds = torch.cat((cur_input_embeds[:image_start_token_pos+1], cur_image_features, cur_input_embeds[image_start_token_pos + num_patches + 1:]), dim=0) + cur_image_idx += 1 + new_input_embeds.append(cur_new_input_embeds) + else: + cur_image_features = image_features[cur_image_idx] + num_patches = cur_image_features.shape[0] + if (cur_input_ids == vision_tower.config.im_patch_token).sum() != num_patches: + raise ValueError("The number of im_patch_token should be the same as the number of patches") + masked_indices = torch.where(cur_input_ids == vision_tower.config.im_patch_token)[0] + mask_index_start = masked_indices[0] + if (masked_indices != torch.arange(mask_index_start, mask_index_start+num_patches, device=masked_indices.device, dtype=masked_indices.dtype)).any(): + raise ValueError("The im_patch_token should be continuous") + cur_new_input_embeds = torch.cat((cur_input_embeds[:mask_index_start], cur_image_features, cur_input_embeds[mask_index_start+num_patches:]), dim=0) + new_input_embeds.append(cur_new_input_embeds) + inputs_embeds = torch.stack(new_input_embeds, dim=0) + + # embed positions + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + ) + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + ) + + hidden_states = inputs_embeds + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + # decoder layers + ##newmodified + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + all_router_logits = () if output_router_logits else None + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, output_attentions, None) + + return custom_forward + ##newmodified + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + attention_mask, + output_router_logits, + None, + ) + ##modified + # layer_outputs = layer_outputs[:-1] + else: + ##newmodified + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + past_key_value=past_key_value, + output_attentions=output_attentions, + output_router_logits=output_router_logits, + use_cache=use_cache, + ) + # layer_outputs = layer_outputs[:-1] + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + ##newmodified + if output_router_logits: + all_router_logits += (layer_outputs[-1],) + + ##modified + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ), all_router_logits ##modified + + +class LlamaForCausalLM(LlamaPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = LlamaModel(config) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + images: Optional[torch.FloatTensor] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional + tensors are only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, LlamaForCausalLM + + >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS) + >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER) + + >>> prompt = "Hey, are you consciours? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + images=images + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model/pipeline parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values: + input_ids = input_ids[:, -1:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + "images": kwargs.get("images", None), + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past +##modified +class LlamaForCausalLM_d(LlamaPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + ##newmodified + self.router_aux_loss_coef = 0.001 + + self.model = LlamaModel(config) + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + ##modified + def get_new_parameters(self, number_experts, top_k, oblance): + self.obalance = oblance + self.number_experts = number_experts + self.top_k = top_k + print("NEW PARAMETERS", "obalance", self.obalance) + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + ##newmodified + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + images: Optional[torch.FloatTensor] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional + tensors are only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, LlamaForCausalLM + + >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS) + >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER) + + >>> prompt = "Hey, are you consciours? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + ##newmodified + output_router_logits = ( + output_router_logits if output_router_logits is not None else False + ) + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + ##modified + ##modified ##newmodified + outputs, router_logits = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + return_dict=return_dict, + images=images + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model/pipeline parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + # if not return_dict: + # output = (logits,) + outputs[1:] + # return (loss,) + output if loss is not None else output + + ##newmodified + aux_loss = None + if output_router_logits: + aux_loss = load_balancing_loss_func( + router_logits if return_dict else outputs[-1], self.number_experts, self.top_k + ) + + # print('ce:', loss) ##modified + if self.obalance: + aux_loss = self.router_aux_loss_coef * aux_loss + loss = loss.cuda(0) + aux_loss.cuda(0) + + print('blance:', aux_loss.clone().detach().cpu()) + print('total loss:', loss.clone().detach().cpu()) + + + ##newmodified + if not return_dict: + output = (logits,) + outputs[1:] + if output_router_logits: + output = (aux_loss,) + output + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values: + input_ids = input_ids[:, -1:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + "images": kwargs.get("images", None), + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past + +@add_start_docstrings( + """ + The LLaMa Model transformer with a sequence classification head on top (linear layer). + + [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + LLAMA_START_DOCSTRING, +) +class LlamaForSequenceClassification(LlamaPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = LlamaModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + sequence_lengths = (torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1).to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/quantization_metric/alphalora/templates/README.md b/quantization_metric/alphalora/templates/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9a942d0d86a303208781e08d69bbe6beac65d964 --- /dev/null +++ b/quantization_metric/alphalora/templates/README.md @@ -0,0 +1,46 @@ +# Prompt templates + +This directory contains template styles for the prompts used to finetune LoRA models. + +## Format + +A template is described via a JSON file with the following keys: + +- `prompt_input`: The template to use when input is not None. Uses `{instruction}` and `{input}` placeholders. +- `prompt_no_input`: The template to use when input is None. Uses `{instruction}` placeholders. +- `description`: A short description of the template, with possible use cases. +- `response_split`: The text to use as separator when cutting real response from the model output. + +No `{response}` placeholder was used, since the response is always the last element of the template and is just to be concatenated to the rest. + +## Example template + +The default template, used unless otherwise specified, is `alpaca.json` + +```json +{ + "description": "Template used by Alpaca-LoRA.", + "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n", + "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n", + "response_split": "### Response:" +} + +``` + +## Current templates + +### alpaca + +Default template used for generic LoRA fine tunes so far. + +### alpaca_legacy + +Legacy template used by the original alpaca repo, with no `\n` after the response field. Kept for reference and experiments. + +### alpaca_short + +A trimmed down alpaca template which seems to perform just as well and spare some tokens. Models created with the default template seem to be queryable by the short tempalte as well. More experiments are welcome. + +### vigogne + +The default alpaca template, translated to french. This template was used to train the "Vigogne" LoRA and is to be used to query it, or for extra fine tuning. diff --git a/quantization_metric/alphalora/templates/alpaca_legacy.json b/quantization_metric/alphalora/templates/alpaca_legacy.json new file mode 100644 index 0000000000000000000000000000000000000000..9b46cfa8fa06af0d7855a0d7b2896ecbd2281b1c --- /dev/null +++ b/quantization_metric/alphalora/templates/alpaca_legacy.json @@ -0,0 +1,6 @@ +{ + "description": "Legacy template, used by Original Alpaca repository.", + "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:", + "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:", + "response_split": "### Response:" +} diff --git a/quantization_metric/bit_layers/random_layers/configure_30.json b/quantization_metric/bit_layers/random_layers/configure_30.json new file mode 100644 index 0000000000000000000000000000000000000000..5b9066931fcb8fe3c46fd7de54df82a8a6237f7b --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_30.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 2, + 4, + 4, + 2, + 2, + 4, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 2, + 4, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_31.json b/quantization_metric/bit_layers/random_layers/configure_31.json new file mode 100644 index 0000000000000000000000000000000000000000..d5959fe8320797500eda03c785ff3589303daf39 --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_31.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 2, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_32.json b/quantization_metric/bit_layers/random_layers/configure_32.json new file mode 100644 index 0000000000000000000000000000000000000000..3231d4e54da961dc03ef3383b1fda3c444b956bf --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_32.json @@ -0,0 +1 @@ +[8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8] diff --git a/quantization_metric/bit_layers/random_layers/configure_33.json b/quantization_metric/bit_layers/random_layers/configure_33.json new file mode 100644 index 0000000000000000000000000000000000000000..45a102a9bd26cc0081212e30bcf803705514fc17 --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_33.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 2, + 2, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 2, + 4, + 2, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_5.json b/quantization_metric/bit_layers/random_layers/configure_5.json new file mode 100644 index 0000000000000000000000000000000000000000..7371147214d2e4914a9e0465d80fd3c0bf7a34a8 --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_5.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 2, + 2, + 2, + 2, + 4, + 4, + 2, + 4, + 2, + 2, + 2, + 2, + 4, + 4, + 2, + 4, + 2, + 2, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_6.json b/quantization_metric/bit_layers/random_layers/configure_6.json new file mode 100644 index 0000000000000000000000000000000000000000..543d59a9a8d8bd5c5b25dbb025fc1fcb05ea315f --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_6.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 2, + 2, + 2, + 4, + 4, + 4, + 2, + 4, + 4, + 2, + 4, + 2, + 2, + 4, + 2, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_7.json b/quantization_metric/bit_layers/random_layers/configure_7.json new file mode 100644 index 0000000000000000000000000000000000000000..a7cb88d515f7093907819c8b299fd5f9c93122a9 --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_7.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 2, + 2, + 2, + 4, + 2, + 2, + 2, + 2, + 4, + 4, + 2, + 2, + 2, + 4, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_8.json b/quantization_metric/bit_layers/random_layers/configure_8.json new file mode 100644 index 0000000000000000000000000000000000000000..94db1939918c052c516b10cc4f4ada9551b63078 --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_8.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 2, + 4, + 2, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/random_layers/configure_9.json b/quantization_metric/bit_layers/random_layers/configure_9.json new file mode 100644 index 0000000000000000000000000000000000000000..869f149e376cf1df8457a79dbb477017b20ab7de --- /dev/null +++ b/quantization_metric/bit_layers/random_layers/configure_9.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 4, + 4, + 2, + 4, + 2, + 4, + 2, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_10.json b/quantization_metric/bit_layers/sides2middle/configure_10.json new file mode 100644 index 0000000000000000000000000000000000000000..3a2a17cf000544dcb67eba19708bb2306e0a4012 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_10.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_11.json b/quantization_metric/bit_layers/sides2middle/configure_11.json new file mode 100644 index 0000000000000000000000000000000000000000..89f106bb38ca4b113e1f8d15fc2aa046c99b119b --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_11.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_12.json b/quantization_metric/bit_layers/sides2middle/configure_12.json new file mode 100644 index 0000000000000000000000000000000000000000..bb6cd1b6dc4fc7d304604b79c9095573540d5e09 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_12.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_14.json b/quantization_metric/bit_layers/sides2middle/configure_14.json new file mode 100644 index 0000000000000000000000000000000000000000..ba3e27b9ede344603c97a6d6f0abb37dfaf3ad26 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_14.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_15.json b/quantization_metric/bit_layers/sides2middle/configure_15.json new file mode 100644 index 0000000000000000000000000000000000000000..65c14f31d516fa20fb8c20624f6f0109f2ba35ed --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_15.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_16.json b/quantization_metric/bit_layers/sides2middle/configure_16.json new file mode 100644 index 0000000000000000000000000000000000000000..d5aa821d00f8a628bf14b2e0ecdce1edca1d16f6 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_16.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_17.json b/quantization_metric/bit_layers/sides2middle/configure_17.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0592046fe45475f8c05f7cb55dd7918b4cdbd7 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_17.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_18.json b/quantization_metric/bit_layers/sides2middle/configure_18.json new file mode 100644 index 0000000000000000000000000000000000000000..65f3eea857009b7adffe017ad4e1347e237682eb --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_18.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_19.json b/quantization_metric/bit_layers/sides2middle/configure_19.json new file mode 100644 index 0000000000000000000000000000000000000000..ea71e6b0a0d05543e11da413ebe989ac66c184fa --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_19.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_2.json b/quantization_metric/bit_layers/sides2middle/configure_2.json new file mode 100644 index 0000000000000000000000000000000000000000..70ae4319b26dea63d192c4cf7ea9cb3fd5a2502e --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_2.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_20.json b/quantization_metric/bit_layers/sides2middle/configure_20.json new file mode 100644 index 0000000000000000000000000000000000000000..8246fcda43ec6be7abbcdd58fdaa49ec10b11872 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_20.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_21.json b/quantization_metric/bit_layers/sides2middle/configure_21.json new file mode 100644 index 0000000000000000000000000000000000000000..672941e3e828c892f13c6a4c2ebc9918ecef3a9f --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_21.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_22.json b/quantization_metric/bit_layers/sides2middle/configure_22.json new file mode 100644 index 0000000000000000000000000000000000000000..fe876aaf2197ef1f89abe241c3f9a471b896f7e8 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_22.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_23.json b/quantization_metric/bit_layers/sides2middle/configure_23.json new file mode 100644 index 0000000000000000000000000000000000000000..3b8b9b7a00c8addf85f9273f845f4adb9f30ffc0 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_23.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 4, + 2, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 2, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_24.json b/quantization_metric/bit_layers/sides2middle/configure_24.json new file mode 100644 index 0000000000000000000000000000000000000000..ea71e6b0a0d05543e11da413ebe989ac66c184fa --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_24.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_3.json b/quantization_metric/bit_layers/sides2middle/configure_3.json new file mode 100644 index 0000000000000000000000000000000000000000..7fe5ffc5d77407c6c7fce21d48c431172facc565 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_3.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_4.json b/quantization_metric/bit_layers/sides2middle/configure_4.json new file mode 100644 index 0000000000000000000000000000000000000000..68dd982d64d4415423a67ab414cd7b01ac852c14 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_4.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_5.json b/quantization_metric/bit_layers/sides2middle/configure_5.json new file mode 100644 index 0000000000000000000000000000000000000000..02e5d549b77edcb1962e5f94f496084db5c5bd12 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_5.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_6.json b/quantization_metric/bit_layers/sides2middle/configure_6.json new file mode 100644 index 0000000000000000000000000000000000000000..397a613ff3c89daddd08ef4e7e480ca18a4fd756 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_6.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_7.json b/quantization_metric/bit_layers/sides2middle/configure_7.json new file mode 100644 index 0000000000000000000000000000000000000000..2ac83fa207fe506931048809ac401fe88d63cdf1 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_7.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_8.json b/quantization_metric/bit_layers/sides2middle/configure_8.json new file mode 100644 index 0000000000000000000000000000000000000000..9809807a3c44ed50350f859f00a566ff6f1ec22a --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_8.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/configure_9.json b/quantization_metric/bit_layers/sides2middle/configure_9.json new file mode 100644 index 0000000000000000000000000000000000000000..6a4a5cd8296ce0043023b96b2544df7d1b47a268 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/configure_9.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/sides2middle/test.py b/quantization_metric/bit_layers/sides2middle/test.py new file mode 100644 index 0000000000000000000000000000000000000000..dcba3b08acd1238a9975da7606cc12426320e019 --- /dev/null +++ b/quantization_metric/bit_layers/sides2middle/test.py @@ -0,0 +1,23 @@ +import json +import os + +data = [4] * 32 +for i in range(10, 29): + data[i] = 2 + +id = 2 +left = 10 +right = 28 +while sum(data) < 128: + data[left] = 4 + with open(f"configure_{id}.json", "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=4) + id += 1 + left += 1 + print(data) + data[right] = 4 + with open(f"configure_{id}.json", "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=4) + right -= 1 + id += 1 + print(data) diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_10.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_10.json new file mode 100644 index 0000000000000000000000000000000000000000..c07636b5dd47d0ce37473c826678ee203ac8aa36 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_10.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_11.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_11.json new file mode 100644 index 0000000000000000000000000000000000000000..1ad8fcb44d9b406054be00fd047c6dbd95608784 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_11.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_12.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_12.json new file mode 100644 index 0000000000000000000000000000000000000000..c9942bc5470c0accea269275be1d9575d442f697 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_12.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_13.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_13.json new file mode 100644 index 0000000000000000000000000000000000000000..a42e156c8b042266f18de8d3b2e4a9e0f4064ff9 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_13.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_14.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_14.json new file mode 100644 index 0000000000000000000000000000000000000000..8679a237c44c93e6474f477f63214b997d285ebb --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_14.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_15.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_15.json new file mode 100644 index 0000000000000000000000000000000000000000..219dd92735c9f553d076b573d2b6697a8f730436 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_15.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_16.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_16.json new file mode 100644 index 0000000000000000000000000000000000000000..a294c023fdb906f1df5fc4bf73dcf9349b36a8b3 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_16.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_17.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_17.json new file mode 100644 index 0000000000000000000000000000000000000000..daadd921791866bd7511780eaf713abae879a645 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_17.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_18.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_18.json new file mode 100644 index 0000000000000000000000000000000000000000..5e3f53ce54821f8c57d69c9d953747ba0f9f2673 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_18.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_19.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_19.json new file mode 100644 index 0000000000000000000000000000000000000000..36a9398df97f4b81e59e078e2521967ece9fb876 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_19.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_20.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_20.json new file mode 100644 index 0000000000000000000000000000000000000000..6c265b3a8304c91c1e0fe35f2dc93946ccde7923 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_20.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_21.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_21.json new file mode 100644 index 0000000000000000000000000000000000000000..217f830520e2c19ddf1953f244cda61630cef6b2 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_21.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_22.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_22.json new file mode 100644 index 0000000000000000000000000000000000000000..9cfead063589f3808393224a23970dabf42102c6 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_22.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_23.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_23.json new file mode 100644 index 0000000000000000000000000000000000000000..80cc9f7167267187ce80d3ea76b06e59f55cb223 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_23.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_24.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_24.json new file mode 100644 index 0000000000000000000000000000000000000000..83522c1ca7d7a10c21c8df0254117686dbe9d8e4 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_24.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_25.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_25.json new file mode 100644 index 0000000000000000000000000000000000000000..9bd4b08431b6a4d47125511cf17ef01edb60c08a --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_25.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_26.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_26.json new file mode 100644 index 0000000000000000000000000000000000000000..7611f1228f4b8664584151a9135f86e8b13d7c4e --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_26.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_27.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_27.json new file mode 100644 index 0000000000000000000000000000000000000000..e0423cfe278f19afa38778761c7dc75b43d45ba4 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_27.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_28.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_28.json new file mode 100644 index 0000000000000000000000000000000000000000..15a151374b2ff32cc0500c26bb69cf8ad50ae60f --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_28.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_29.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_29.json new file mode 100644 index 0000000000000000000000000000000000000000..d99cbcd5aecaec34d6d3892adbad0c6a8f990b58 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_29.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_3.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_3.json new file mode 100644 index 0000000000000000000000000000000000000000..c0ae777e4fb4b8e355e31828f5baee26cdb03f8d --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_3.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_4.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_4.json new file mode 100644 index 0000000000000000000000000000000000000000..caec5b2273a28390611da0757349bfdc506d3664 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_1/configure_4.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_10.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_10.json new file mode 100644 index 0000000000000000000000000000000000000000..0181d91c629653d84f4b029cfc4706e02f496b27 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_10.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_11.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_11.json new file mode 100644 index 0000000000000000000000000000000000000000..63a515b50c05fd14a40769de559e966865b4da2b --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_11.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_12.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_12.json new file mode 100644 index 0000000000000000000000000000000000000000..0a40fad325c87ae56594673181d732227148fec2 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_12.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_13.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_13.json new file mode 100644 index 0000000000000000000000000000000000000000..780f21961d3353fe58994aaa297a14ca5598b94a --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_13.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_14.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_14.json new file mode 100644 index 0000000000000000000000000000000000000000..831bbef104b31a2865a6acccc10bcfe1b1bb0df7 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_14.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_15.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_15.json new file mode 100644 index 0000000000000000000000000000000000000000..89ab8f9e3d1ea6ba4c27b2ce1c8445d5cd3d8b15 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_15.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_16.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_16.json new file mode 100644 index 0000000000000000000000000000000000000000..6200126cb06a110701e5db5572a736a2af3f28d7 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_16.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file diff --git a/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_18.json b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_18.json new file mode 100644 index 0000000000000000000000000000000000000000..46f493b16fa3301a4cf0f106c6ca1d838ca3d127 --- /dev/null +++ b/quantization_metric/bit_layers/singletask_arc_easy/bits_2/configure_18.json @@ -0,0 +1,34 @@ +[ + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 4 +] \ No newline at end of file