Add files using upload-large-folder tool

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-boost.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+EXP_BASE_NAME="sensi-boost"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+layers="31 1"
+ATTEMPT="${weight_algo}"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+  --task eval_ppl \
+  --model 0 \
+  --algo mxq \
+  --weight-algo $weight_algo \
+  --boost-layer $layers \
+  --config ${BUDGETS} \
+  --experiment-name "${EXP_NAME}_ppl" \
+  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+  --result-dir=$RESULT_DIR \
+  2>&1 \
+  | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+  echo "Perplexity evaluation failed!"
+  exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+  --type quant_config \
+  --model 0 \
+  --budget ${BUDGETS} \
+  --attempt $ATTEMPT \
+  --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+  --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+  2>&1 \
+  | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids="0"
+for m in $model_ids; do
+  for cfg in ${BUDGETS}; do
+      python ../src/cli.py llm \
+          --model $m \
+          --algo ${algo} \
+          --config ${cfg} \
+          --task eval_model_storage \
+          --experiment-name "${EXP_NAME}_stor" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+  done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+    for budget in $BGS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 $ATTEMPT \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-kurt-boost.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=kurt-boost
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# BOOST_STOPS="2 3"
+# BOOST_TOP_MS="1 2 3 0"
+BOOST_STOPS="2"
+BOOST_TOP_MS="1"
+
+
+for BOOST_STOP in $BOOST_STOPS; do
+    for BOOST_TOP_M in $BOOST_TOP_MS; do
+        ATTEMPT="kurt-boost-${BOOST_STOP}-${BOOST_TOP_M}"
+        EXP_BASE_NAME=$ATTEMPT
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+        log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+        mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+        mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+        # MODELS="0"
+        EXP_NAME="${ATTEMPT}"
+        echo "=========Run perplexity evaluation========="
+        python ../src/cli.py llm \
+          --task eval_ppl \
+          --model $MODELS \
+          --algo mxq \
+          --weight-algo $weight_algo \
+          --boost-stop $BOOST_STOP \
+          --top-m-layer $BOOST_TOP_M \
+          --config ${BUDGETS} \
+          --experiment-name "${EXP_NAME}_ppl" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+        EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+          echo "Perplexity evaluation failed!"
+          exit $EXIT_CODE
+        fi
+        echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_ppl \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+        echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+        python ../src/cli.py dump \
+          --type quant_config \
+          --model $MODELS \
+          --budget ${BUDGETS} \
+          --attempt $ATTEMPT \
+          --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+          --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+          2>&1 \
+          | tee -a $log_file
+
+        echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+        algo=mxq
+        model_ids=$MODELS
+        for m in $model_ids; do
+          for cfg in ${BUDGETS}; do
+              python ../src/cli.py llm \
+                  --model $m \
+                  --algo ${algo} \
+                  --config ${cfg} \
+                  --task eval_model_storage \
+                  --experiment-name "${EXP_NAME}_stor" \
+                  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+                  --result-dir=$RESULT_DIR \
+                  2>&1 \
+                  | tee -a $log_file
+          done
+        done
+        echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_stor \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+        # echo "=========Delete quantized models of batch ${batch_name}========="
+        # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+        OLD_DIR=$(pwd)
+        cd $RESULT_DIR/$EXP_BASE_NAME
+        if [ ! -d pdfs/allot ]; then
+            mkdir -p pdfs/allot
+        fi
+        $OLD_DIR/../data-vis/combine.R \
+            --baseline_data_dir $OLD_DIR/../data-vis/data \
+            --mxq_data_dir data
+        $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+        $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+        $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+        $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+        pdflatex table.tex
+
+        # plot configuration allocations for 3 * 12 MXQ combinations
+        for model in $MODEL_NAMES; do
+            for budget in $BUDGETS; do
+                $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+                  -m $model \
+                  -b $budget \
+                  --attempt1 $ATTEMPT \
+                  --attempt2 mxq1 \
+                  --fnorm_data_dir $OLD_DIR/../src/data \
+                  --quant_cfg_allot_file data/quant-cfg-allocation.csv
+            done
+        done
+        cd $OLD_DIR
+    done
+done
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-ablation.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-boost
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# BOOST_STOPS="2 3"
+# BOOST_TOP_MS="1 2 3 0"
+
+BOOST_STOPS="3"
+BOOST_TOP_MS="3 0"
+
+for BOOST_STOP in $BOOST_STOPS; do
+    for BOOST_TOP_M in $BOOST_TOP_MS; do
+        ATTEMPT="sensi-abl-${BOOST_STOP}-${BOOST_TOP_M}"
+        EXP_BASE_NAME=$ATTEMPT
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+        log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+        mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+        mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+        # MODELS="0"
+        EXP_NAME="${ATTEMPT}"
+        echo "=========Run perplexity evaluation========="
+        python ../src/cli.py llm \
+          --task eval_ppl \
+          --model $MODELS \
+          --algo mxq \
+          --weight-algo $weight_algo \
+          --boost-stop $BOOST_STOP \
+          --top-m-layer $BOOST_TOP_M \
+          --ablation \
+          --config ${BUDGETS} \
+          --experiment-name "${EXP_NAME}_ppl" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+        EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+          echo "Perplexity evaluation failed!"
+          exit $EXIT_CODE
+        fi
+        echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_ppl \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+        echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+        python ../src/cli.py dump \
+          --type quant_config \
+          --model $MODELS \
+          --budget ${BUDGETS} \
+          --attempt $ATTEMPT \
+          --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+          --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+          2>&1 \
+          | tee -a $log_file
+
+        echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+        algo=mxq
+        model_ids=$MODELS
+        for m in $model_ids; do
+          for cfg in ${BUDGETS}; do
+              python ../src/cli.py llm \
+                  --model $m \
+                  --algo ${algo} \
+                  --config ${cfg} \
+                  --task eval_model_storage \
+                  --experiment-name "${EXP_NAME}_stor" \
+                  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+                  --result-dir=$RESULT_DIR \
+                  2>&1 \
+                  | tee -a $log_file
+          done
+        done
+        echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_stor \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+        # echo "=========Delete quantized models of batch ${batch_name}========="
+        # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+        OLD_DIR=$(pwd)
+        cd $RESULT_DIR/$EXP_BASE_NAME
+        if [ ! -d pdfs/allot ]; then
+            mkdir -p pdfs/allot
+        fi
+        $OLD_DIR/../data-vis/combine.R \
+            --baseline_data_dir $OLD_DIR/../data-vis/data \
+            --mxq_data_dir data
+        $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+        $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+        $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+        $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+        pdflatex table.tex
+
+        # plot configuration allocations for 3 * 12 MXQ combinations
+        for model in $MODEL_NAMES; do
+            for budget in $BUDGETS; do
+                $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+                  -m $model \
+                  -b $budget \
+                  --attempt1 $ATTEMPT \
+                  --attempt2 mxq1 \
+                  --fnorm_data_dir $OLD_DIR/../src/data \
+                  --quant_cfg_allot_file data/quant-cfg-allocation.csv
+            done
+        done
+        cd $OLD_DIR
+    done
+done
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-boost
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# BOOST_STOPS="2 3"
+BOOST_STOPS="3"
+BOOST_TOP_MS="0"
+
+for BOOST_STOP in $BOOST_STOPS; do
+    for BOOST_TOP_M in $BOOST_TOP_MS; do
+        if [[ $BOOST_STOP -eq 2 && $BOOST_TOP_M -eq 1 ]]; then
+            continue
+        fi
+        ATTEMPT="sensi-boost-${BOOST_STOP}-${BOOST_TOP_M}"
+        EXP_BASE_NAME=$ATTEMPT
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+        log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+        mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+        mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+        # MODELS="0"
+        EXP_NAME="${ATTEMPT}"
+        echo "=========Run perplexity evaluation========="
+        python ../src/cli.py llm \
+          --task eval_ppl \
+          --model $MODELS \
+          --algo mxq \
+          --weight-algo $weight_algo \
+          --boost-stop $BOOST_STOP \
+          --top-m-layer $BOOST_TOP_M \
+          --config ${BUDGETS} \
+          --experiment-name "${EXP_NAME}_ppl" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+        EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+          echo "Perplexity evaluation failed!"
+          exit $EXIT_CODE
+        fi
+        echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_ppl \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+        echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+        python ../src/cli.py dump \
+          --type quant_config \
+          --model $MODELS \
+          --budget ${BUDGETS} \
+          --attempt $ATTEMPT \
+          --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+          --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+          2>&1 \
+          | tee -a $log_file
+
+        echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+        algo=mxq
+        model_ids=$MODELS
+        for m in $model_ids; do
+          for cfg in ${BUDGETS}; do
+              python ../src/cli.py llm \
+                  --model $m \
+                  --algo ${algo} \
+                  --config ${cfg} \
+                  --task eval_model_storage \
+                  --experiment-name "${EXP_NAME}_stor" \
+                  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+                  --result-dir=$RESULT_DIR \
+                  2>&1 \
+                  | tee -a $log_file
+          done
+        done
+        echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_stor \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+        # echo "=========Delete quantized models of batch ${batch_name}========="
+        # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+        OLD_DIR=$(pwd)
+        cd $RESULT_DIR/$EXP_BASE_NAME
+        if [ ! -d pdfs/allot ]; then
+            mkdir -p pdfs/allot
+        fi
+        $OLD_DIR/../data-vis/combine.R \
+            --baseline_data_dir $OLD_DIR/../data-vis/data \
+            --mxq_data_dir data
+        $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+        $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+        $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+        $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+        pdflatex table.tex
+
+        # plot configuration allocations for 3 * 12 MXQ combinations
+        for model in $MODEL_NAMES; do
+            for budget in $BUDGETS; do
+                $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+                  -m $model \
+                  -b $budget \
+                  --attempt1 $ATTEMPT \
+                  --attempt2 mxq1 \
+                  --fnorm_data_dir $OLD_DIR/../src/data \
+                  --quant_cfg_allot_file data/quant-cfg-allocation.csv
+            done
+        done
+        cd $OLD_DIR
+    done
+done
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="llama-sensi-milp-2x"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+MODELS="0 2"
+EXP_NAME="${ATTEMPT}"
+echo "=========Run perplexity evaluation on Llama-2-7b========="
+python ../src/cli.py llm \
+  --task eval_ppl \
+  --model $MODELS \
+  --algo mxq \
+  --weight-algo $weight_algo \
+  --config ${BUDGETS} \
+  --experiment-name "${EXP_NAME}_ppl" \
+  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+  --result-dir=$RESULT_DIR \
+  2>&1 \
+  | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+  echo "Perplexity evaluation failed!"
+  exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+  --type quant_config \
+  --model $MODELS \
+  --budget ${BUDGETS} \
+  --attempt $ATTEMPT \
+  --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+  --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+  2>&1 \
+  | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+  for cfg in ${BUDGETS}; do
+      python ../src/cli.py llm \
+          --model $m \
+          --algo ${algo} \
+          --config ${cfg} \
+          --task eval_model_storage \
+          --experiment-name "${EXP_NAME}_stor" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+  done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Meta-Llama-3-8B"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+    for budget in $BGS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 $ATTEMPT \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# for SensiMiLP ablation test, all topm values are equivalent
+
+ATTEMPT="sensi-milp-abl"
+EXP_BASE_NAME=$ATTEMPT
+EXP_NAME="${ATTEMPT}"
+
+log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation========="
+python ../src/cli.py llm \
+  --task eval_ppl \
+  --model $MODELS \
+  --algo mxq \
+  --weight-algo $weight_algo \
+  --ablation \
+  --config ${BUDGETS} \
+  --experiment-name "${EXP_NAME}_ppl" \
+  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+  --result-dir=$RESULT_DIR \
+  2>&1 \
+  | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+  echo "Perplexity evaluation failed!"
+  exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+  --type quant_config \
+  --model $MODELS \
+  --budget ${BUDGETS} \
+  --attempt $ATTEMPT \
+  --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+  --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+  2>&1 \
+  | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+  for cfg in ${BUDGETS}; do
+      python ../src/cli.py llm \
+          --model $m \
+          --algo ${algo} \
+          --config ${cfg} \
+          --task eval_model_storage \
+          --experiment-name "${EXP_NAME}_stor" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+  done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs ]; then
+    mkdir pdfs
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+cd pdfs
+pdflatex table.tex
+cd ..
+
+cd $OLD_DIR

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+MODELS="1"
+ATTEMPT="Llama2-13B-boost"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+boost_layers="3 39"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+  --task eval_ppl \
+  --model $MODELS \
+  --algo mxq \
+  --weight-algo $weight_algo \
+  --boost-layer $boost_layers \
+  --boost-stop 2 \
+  --config ${BUDGETS} \
+  --experiment-name "${EXP_NAME}_ppl" \
+  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+  --result-dir=$RESULT_DIR \
+  2>&1 \
+  | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+  echo "Perplexity evaluation failed!"
+  exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+  --type quant_config \
+  --model $MODELS \
+  --budget ${BUDGETS} \
+  --attempt $ATTEMPT \
+  --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+  --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+  2>&1 \
+  | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+  for cfg in ${BUDGETS}; do
+      python ../src/cli.py llm \
+          --model $m \
+          --algo ${algo} \
+          --config ${cfg} \
+          --task eval_model_storage \
+          --experiment-name "${EXP_NAME}_stor" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+  done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-13b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+    for budget in $BGS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 $ATTEMPT \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+
+# 100%, 99%, 98%, 97%, 96%, 95% of [3.51, 4.25]
+# MXQ2=(minibatch 3.51 3.47 3.44 3.40 3.37 3.33 4.25 4.21 4.17 4.12 4.08 4.04)
+MXQ2=(minibatch 6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07)
+MXQ_BATCHES=(MXQ2)
+declare -n MXQ_BATCH
+
+ATTEMPT="sensi-milp-mini2"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+EXP_BASE_NAME="${ATTEMPT}"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+# Setup data files directories for reporting
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+
+weight_algo=sensi-milp
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+  batch_name=${MXQ_BATCH[@]:0:1}
+  EXP_NAME="${EXP_BASE_NAME}-${batch_name}"
+  mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+  mkdir -p $RESULT_DIR/${EXP_NAME}_stor
+  log_file="logs/bench-${EXP_NAME}-$(date +%Y%m%d%H%M%S).log"
+
+  echo "=========Run perplexity evaluation on batch ${batch_name}========="
+  python ../src/cli.py llm \
+      --task eval_ppl \
+      --model 0 1 2 \
+      --algo mxq \
+      --weight-algo ${weight_algo} \
+      --config ${MXQ_BATCH[@]:1} \
+      --experiment-name "${EXP_NAME}_ppl" \
+      --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+      --result-dir=$RESULT_DIR \
+      2>&1 \
+      | tee -a $log_file
+  EXIT_CODE=$?
+  if [ $EXIT_CODE -ne 0 ]; then
+      echo "Perplexity evaluation failed!"
+      exit $EXIT_CODE
+  fi
+  echo "=========Collect perplexity evaluation result on batch ${batch_name}========="
+  find $RESULT_DIR/${EXP_NAME}_ppl \
+      -name "result-*.csv" \
+      -printf '%T@ %p\n' \
+      | sort -n \
+      | tail -1 \
+      | cut -d' '  -f2 \
+      | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+  echo "=========Dump quantization configs on batch ${batch_name}========="
+  mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+  python ../src/cli.py dump \
+      --type quant_config \
+      --model 0 1 2 \
+      --budget ${MXQ_BATCH[@]:1} \
+      --attempt $ATTEMPT \
+      --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+      --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+      2>&1 \
+      | tee -a $log_file
+
+  echo "=========Run memory evaluation on batch ${batch_name}========="
+  algo=mxq
+  model_ids="0 1 2"
+  for m in $model_ids; do
+      for cfg in "${MXQ_BATCH[@]:1}"; do
+          python ../src/cli.py llm \
+              --model $m \
+              --algo ${algo} \
+              --config ${cfg} \
+              --task eval_model_storage \
+              --experiment-name "${EXP_NAME}_stor" \
+              --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+              --result-dir=$RESULT_DIR \
+              2>&1 \
+              | tee -a $log_file
+      done
+  done
+  echo "=========Collect memory evaluation result on batch ${batch_name}========="
+  find $RESULT_DIR/${EXP_NAME}_stor \
+      -name "result-*.csv" \
+      -printf '%T@ %p\n' \
+      | sort -n \
+      | tail -1 \
+      | cut -d' '  -f2 \
+      | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+
+  # echo "=========Delete quantized models of batch ${batch_name}========="
+  # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+  OLD_DIR=$(pwd)
+  cd $RESULT_DIR/$EXP_BASE_NAME
+  if [ ! -d pdfs ]; then
+      mkdir pdfs
+  fi
+  $OLD_DIR/../data-vis/combine.R \
+      --baseline_data_dir $OLD_DIR/../data-vis/data \
+      --mxq_data_dir data
+  $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+  $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv
+  $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv
+  $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+  cd pdfs
+  pdflatex table.tex
+  cd ..
+
+  # plot configuration allocations for 3 * 12 MXQ combinations
+  MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+  BUDGETS=${MXQ_BATCH[@]:1}
+  for model in $MODELS; do
+      for budget in $BUDGETS; do
+          $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+            -m $model \
+            -b $budget \
+            --fnorm_data_dir $OLD_DIR/../src/data \
+            --attempt1 mxq1 \
+            --attempt2 $ATTEMPT \
+            --quant_cfg_allot_file data/quant-cfg-allocation.csv
+      done
+  done
+  cd $OLD_DIR
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-tail-reduction.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+EXP_BASE_NAME="tail-reduction"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=tail_boost
+ATTEMPT="${EXP_BASE_NAME}"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+  --task eval_ppl \
+  --model 0 1 2 \
+  --algo mxq \
+  --weight-algo $weight_algo \
+  --factor -1 \
+  --config ${BUDGETS} \
+  --experiment-name "${EXP_NAME}_ppl" \
+  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+  --result-dir=$RESULT_DIR \
+  2>&1 \
+  | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+  echo "Perplexity evaluation failed!"
+  exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+# find $RESULT_DIR/${EXP_NAME}_ppl \
+#   -name "result-*.csv" \
+#   -printf '%T@ %p\n' \
+#   | sort -n \
+#   | tail -1 \
+#   | cut -d' '  -f2 \
+#   | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+#
+# echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+# mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+# python ../src/cli.py dump \
+#   --type quant_config \
+#   --model 0 1 2 \
+#   --budget ${BUDGETS} \
+#   --attempt $ATTEMPT \
+#   --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+#   --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+#   2>&1 \
+#   | tee -a $log_file
+#
+# echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+# algo=mxq
+# model_ids="0 1 2"
+# for m in $model_ids; do
+#   for cfg in ${BUDGETS}; do
+#       python ../src/cli.py llm \
+#           --model $m \
+#           --algo ${algo} \
+#           --config ${cfg} \
+#           --task eval_model_storage \
+#           --experiment-name "${EXP_NAME}_stor" \
+#           --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+#           --result-dir=$RESULT_DIR \
+#           2>&1 \
+#           | tee -a $log_file
+#   done
+# done
+# echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+# find $RESULT_DIR/${EXP_NAME}_stor \
+#   -name "result-*.csv" \
+#   -printf '%T@ %p\n' \
+#   | sort -n \
+#   | tail -1 \
+#   | cut -d' '  -f2 \
+#   | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+BUDGETS="4.13 4.25 4.51"
+for model in $MODELS; do
+    for budget in $BUDGETS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 tail-reduction \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+# python ../src/cli.py vit \
+#     --task eval_zeroshot_cls \
+#     --model 0 1 \
+#     --config 4.51 4.25 4.13 3.51 3.25 3.13\
+#     --algo mxq \
+#     --weight-algo kurt-boost \
+#     --boost-stop 2 \
+#     --top-m-layer 1 \
+#     --experiment-name eval_zs_BH_mxq_kurt_boost \
+#     --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+#     --result-dir="/fdata/llm/mxq/results" \
+#     2>&1 \
+#     | tee logs/bench-vit-$(date +%Y%m%d%H%M%S).log
+
+python ../src/cli.py vit \
+    --task eval_zeroshot_cls \
+    --model 0 1 \
+    --config 4.51 4.25 4.13 3.51 3.25 3.13\
+    --algo mxq \
+    --weight-algo kurt-boost \
+    --boost-stop 2 \
+    --top-m-layer 2 \
+    --experiment-name eval_zs_BH_mxq_kurt_boost_22 \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-vit-$(date +%Y%m%d%H%M%S).log
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+BUDGETS="3.25"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-boost
+MODELS="0"
+MODEL_NAMES="Llama-2-7b-hf"
+
+
+# BOOST_STOPS="2 3"
+BOOST_STOPS="3"
+BOOST_TOP_MS="3"
+
+for BOOST_STOP in $BOOST_STOPS; do
+    for BOOST_TOP_M in $BOOST_TOP_MS; do
+        if [[ $BOOST_STOP -eq 2 && $BOOST_TOP_M -eq 1 ]]; then
+            continue
+        fi
+        ATTEMPT="sb-3257_2-fix-${BOOST_STOP}-${BOOST_TOP_M}"
+        EXP_BASE_NAME=$ATTEMPT
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+        log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+        mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+        mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+        # MODELS="0"
+        EXP_NAME="${ATTEMPT}"
+        echo "=========Run perplexity evaluation========="
+        python ../src/cli.py llm \
+          --task eval_ppl \
+          --model $MODELS \
+          --algo mxq \
+          --weight-algo $weight_algo \
+          --boost-stop $BOOST_STOP \
+          --top-m-layer $BOOST_TOP_M \
+          --config ${BUDGETS} \
+          --experiment-name "${EXP_NAME}_ppl" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+        EXIT_CODE=$?
+        if [ $EXIT_CODE -ne 0 ]; then
+          echo "Perplexity evaluation failed!"
+          exit $EXIT_CODE
+        fi
+        echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_ppl \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+        echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+        mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+        python ../src/cli.py dump \
+          --type quant_config \
+          --model $MODELS \
+          --budget ${BUDGETS} \
+          --attempt $ATTEMPT \
+          --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+          --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+          2>&1 \
+          | tee -a $log_file
+
+        echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+        algo=mxq
+        model_ids=$MODELS
+        for m in $model_ids; do
+          for cfg in ${BUDGETS}; do
+              python ../src/cli.py llm \
+                  --model $m \
+                  --algo ${algo} \
+                  --config ${cfg} \
+                  --task eval_model_storage \
+                  --experiment-name "${EXP_NAME}_stor" \
+                  --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+                  --result-dir=$RESULT_DIR \
+                  2>&1 \
+                  | tee -a $log_file
+          done
+        done
+        echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+        find $RESULT_DIR/${EXP_NAME}_stor \
+          -name "result-*.csv" \
+          -printf '%T@ %p\n' \
+          | sort -n \
+          | tail -1 \
+          | cut -d' '  -f2 \
+          | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+        # echo "=========Delete quantized models of batch ${batch_name}========="
+        # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+        OLD_DIR=$(pwd)
+        cd $RESULT_DIR/$EXP_BASE_NAME
+        if [ ! -d pdfs/allot ]; then
+            mkdir -p pdfs/allot
+        fi
+        $OLD_DIR/../data-vis/combine.R \
+            --baseline_data_dir $OLD_DIR/../data-vis/data \
+            --mxq_data_dir data
+
+        cd $OLD_DIR
+    done
+done
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=kurt-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# for SensiMiLP ablation test, all topm values are equivalent
+
+ATTEMPT="kurt-milp-abl"
+EXP_BASE_NAME=$ATTEMPT
+EXP_NAME="${ATTEMPT}"
+
+log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+  --type quant_config \
+  --model $MODELS \
+  --budget ${BUDGETS} \
+  --attempt $ATTEMPT \
+  --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+  --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+  2>&1 \
+  | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+  for cfg in ${BUDGETS}; do
+      python ../src/cli.py llm \
+          --model $m \
+          --algo ${algo} \
+          --config ${cfg} \
+          --task eval_model_storage \
+          --experiment-name "${EXP_NAME}_stor" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+  done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs ]; then
+    mkdir pdfs
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+cd pdfs
+pdflatex table.tex
+cd ..
+
+cd $OLD_DIR

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+BOOST_TOP_MS="3"
+
+for BOOST_TOP_M in $BOOST_TOP_MS; do
+    ATTEMPT="sensi-milp-${BOOST_TOP_M}"
+    EXP_BASE_NAME=$ATTEMPT
+    mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+    log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+    mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+    mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+    mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+    # MODELS="0"
+    EXP_NAME="${ATTEMPT}"
+    echo "=========Run perplexity evaluation========="
+    python ../src/cli.py llm \
+      --task eval_ppl \
+      --model $MODELS \
+      --algo mxq \
+      --weight-algo $weight_algo \
+      --top-m-layer $BOOST_TOP_M \
+      --config ${BUDGETS} \
+      --experiment-name "${EXP_NAME}_ppl" \
+      --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+      --result-dir=$RESULT_DIR \
+      2>&1 \
+      | tee -a $log_file
+    EXIT_CODE=$?
+    if [ $EXIT_CODE -ne 0 ]; then
+      echo "Perplexity evaluation failed!"
+      exit $EXIT_CODE
+    fi
+    echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+    find $RESULT_DIR/${EXP_NAME}_ppl \
+      -name "result-*.csv" \
+      -printf '%T@ %p\n' \
+      | sort -n \
+      | tail -1 \
+      | cut -d' '  -f2 \
+      | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+    echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+    mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+    python ../src/cli.py dump \
+      --type quant_config \
+      --model $MODELS \
+      --budget ${BUDGETS} \
+      --attempt $ATTEMPT \
+      --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+      --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+      2>&1 \
+      | tee -a $log_file
+
+    echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+    algo=mxq
+    model_ids=$MODELS
+    for m in $model_ids; do
+      for cfg in ${BUDGETS}; do
+          python ../src/cli.py llm \
+              --model $m \
+              --algo ${algo} \
+              --config ${cfg} \
+              --task eval_model_storage \
+              --experiment-name "${EXP_NAME}_stor" \
+              --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+              --result-dir=$RESULT_DIR \
+              2>&1 \
+              | tee -a $log_file
+      done
+    done
+    echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+    find $RESULT_DIR/${EXP_NAME}_stor \
+      -name "result-*.csv" \
+      -printf '%T@ %p\n' \
+      | sort -n \
+      | tail -1 \
+      | cut -d' '  -f2 \
+      | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+    # echo "=========Delete quantized models of batch ${batch_name}========="
+    # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+    OLD_DIR=$(pwd)
+    cd $RESULT_DIR/$EXP_BASE_NAME
+    if [ ! -d pdfs ]; then
+        mkdir pdfs
+    fi
+    $OLD_DIR/../data-vis/combine.R \
+        --baseline_data_dir $OLD_DIR/../data-vis/data \
+        --mxq_data_dir data
+    $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+    $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv
+    $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv
+    $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+    cd pdfs
+    pdflatex table.tex
+    cd ..
+
+    # plot configuration allocations for 3 * 12 MXQ combinations
+    MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+    BUDGETS=${MXQ_BATCH[@]:1}
+    for model in $MODELS; do
+        for budget in $BUDGETS; do
+            $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+              -m $model \
+              -b $budget \
+              --fnorm_data_dir $OLD_DIR/../src/data \
+              --attempt1 mxq1 \
+              --attempt2 $ATTEMPT \
+              --quant_cfg_allot_file data/quant-cfg-allocation.csv
+        done
+    done
+    cd $OLD_DIR
+
+done
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=kurt-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+BOOST_TOP_MS="1"
+
+for BOOST_TOP_M in $BOOST_TOP_MS; do
+    ATTEMPT="kurt-milp-${BOOST_TOP_M}"
+    EXP_BASE_NAME=$ATTEMPT
+    mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+    log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+    mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+    mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+    mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+    # MODELS="0"
+    EXP_NAME="${ATTEMPT}"
+
+    echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+    mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+    python ../src/cli.py dump \
+      --type quant_config \
+      --model $MODELS \
+      --budget ${BUDGETS} \
+      --attempt $ATTEMPT \
+      --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+      --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+      2>&1 \
+      | tee -a $log_file
+
+    echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+    algo=mxq
+    model_ids=$MODELS
+    for m in $model_ids; do
+      for cfg in ${BUDGETS}; do
+          python ../src/cli.py llm \
+              --model $m \
+              --algo ${algo} \
+              --config ${cfg} \
+              --task eval_model_storage \
+              --experiment-name "${EXP_NAME}_stor" \
+              --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+              --result-dir=$RESULT_DIR \
+              2>&1 \
+              | tee -a $log_file
+      done
+    done
+    echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+    find $RESULT_DIR/${EXP_NAME}_stor \
+      -name "result-*.csv" \
+      -printf '%T@ %p\n' \
+      | sort -n \
+      | tail -1 \
+      | cut -d' '  -f2 \
+      | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+    # echo "=========Delete quantized models of batch ${batch_name}========="
+    # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+    OLD_DIR=$(pwd)
+    cd $RESULT_DIR/$EXP_BASE_NAME
+    if [ ! -d pdfs ]; then
+        mkdir pdfs
+    fi
+    $OLD_DIR/../data-vis/combine.R \
+        --baseline_data_dir $OLD_DIR/../data-vis/data \
+        --mxq_data_dir data
+    $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+    $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+    cd pdfs
+    pdflatex table.tex
+    cd ..
+
+    cd $OLD_DIR
+
+done
+

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# MXQ1=(2bit 2.13 2.15 2.17 2.19 2.21 2.23 2.25 2.27 2.29 2.31 2.45 2.47 2.49 2.51 2.53 2.55 2.57)
+# MXQ2=(3bit 3.07 3.09 3.11 3.13 3.15 3.17 3.19 3.21 3.23 3.25 3.27 3.29 3.31 3.33 3.35 3.37 3.39 3.41 3.42 3.43 3.45 3.47 3.49 3.51 3.53 3.55 3.57 3.59 3.61 3.63 3.65 3.67 3.69 3.71 3.73 3.75 3.77 3.79 3.81 3.83 3.85 3.87 3.89 3.91 3.93 3.95 3.97 3.99)
+# Removed 4.99 which is unsolvable with kurt-scaled scheme
+MXQ4=(4bit 4.01 4.03 4.05 4.07 4.09 4.11 4.13 4.15 4.17 4.19 4.21 4.23 4.25 4.27 4.29 4.31 4.33 4.35 4.37 4.39 4.41 4.43 4.45 4.47 4.49 4.51 4.53 4.55 4.57 4.59 4.61 4.63 4.65 4.67 4.69 4.71 4.73 4.75 4.77 4.79 4.81 4.83 4.85 4.87 4.89 4.91 4.93 4.95 4.97)
+# Remove 5.01 5.02, 5.04 5.10 5.32 which is unsolvable with kurt-scaled scheme
+#MXQ5=(5bit 5.00 5.03 5.06 5.08 5.09 5.12 5.14 5.16 5.18 5.20 5.22 5.24 5.26 5.28 5.30 5.33 5.34 5.36 5.38 5.40 5.42 5.44 5.46 5.48 5.50 5.52 5.54 5.56 5.58 5.60 5.62 5.64 5.66 5.68 5.70 5.72 5.74 5.76 5.78 5.80 5.82 5.84 5.86 5.88 5.90 5.92 5.94 5.96 5.98)
+#MXQ6=(6bit 6.00 6.02 6.03 6.04 6.05 6.06 6.07 6.09 6.11 6.13 6.15 6.17 6.19 6.21 6.23 6.25 6.27 6.29 6.31 6.33 6.35 6.37 6.39 6.41 6.43 6.45 6.47 6.49 6.51 6.53 6.55 6.57 6.59 6.61 6.63 6.65 6.68 6.69 6.71 6.72 6.75 6.77 6.78 6.81 6.83 6.86 6.87 6.89 6.92 6.95 6.96 6.98)
+# MXQ7=(78bit 7.01 7.02 7.04 7.06 7.08 7.10 7.13 7.14 7.16 7.19 7.20 7.22 7.24 7.26 7.29 7.30 7.32 7.34 7.36 7.38 7.41 7.42 7.44 7.47 7.48 7.50 7.53 7.54 7.56 7.57 7.60 7.62 7.63 7.66 7.68 7.70 7.72 7.74 7.76 8.13 8.25 8.51)
+MXQ_BATCHES=(MXQ4)
+declare -n MXQ_BATCH
+
+ATTEMPT="kurt-scaled"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+EXP_BASE_NAME="${ATTEMPT}-dense"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+# Setup data files directories for reporting
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs ]; then
+    mkdir pdfs
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+# MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+MODELS="Llama-2-7b-hf"
+BUDGETS="4.13 4.25 4.51"
+for model in $MODELS; do
+    for budget in $BUDGETS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R -m $model -b $budget \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --attempt1 mxq1 \
+          --attempt2 $ATTEMPT \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-llama-sensi.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="llama-sensi"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+    for budget in $BGS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --fnorm FALSE \
+          --attempt1 $ATTEMPT \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-milp-low-bit.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+BUDGETS="4.25"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+FACTOR=2
+ATTEMPT="sensi-milp-debug-lowbits9"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+# MODELS="0 1 2"
+MODELS="0"
+EXP_NAME="${ATTEMPT}"
+
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+  --type quant_config \
+  --model $MODELS \
+  --budget ${BUDGETS} \
+  --attempt $ATTEMPT \
+  --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+  --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+  2>&1 \
+  | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+  for cfg in ${BUDGETS}; do
+      python ../src/cli.py llm \
+          --model $m \
+          --algo ${algo} \
+          --config ${cfg} \
+          --task eval_model_storage \
+          --experiment-name "${EXP_NAME}_stor" \
+          --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+          --result-dir=$RESULT_DIR \
+          2>&1 \
+          | tee -a $log_file
+  done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+  -name "result-*.csv" \
+  -printf '%T@ %p\n' \
+  | sort -n \
+  | tail -1 \
+  | cut -d' '  -f2 \
+  | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+for model in $MODELS; do
+    for budget in $BUDGETS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 $ATTEMPT \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+RESULT_BASE_DIR="/fdata/llm/mxq/results"
+CALIB_DATASETS="bos pileval wikitext c4"
+CONFIGS="b2g128 b2g64 b2g32 b3g128 b3g64 b3g32 b4g128 b4g64 b4g32 b8g128 b8g64 b8g32"
+MODELS="Qwen/Qwen2.5-7B Qwen/Qwen2.5-Coder-7B Qwen/Qwen2.5-Coder-7B-Instruct Qwen/Qwen2.5-Math-7B"
+
+EXP_NAME=sensi_qwen25
+RESULT_DIR=$RESULT_BASE_DIR/$EXP_NAME
+mkdir -p $RESULT_DIR/data
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR
+if [ ! -d pdfs ]; then
+    mkdir -p pdfs
+fi
+$OLD_DIR/../data-vis/plot-variant-sensi.R data/

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+
+# 100%, 99%, 98%, 97%, 96%, 95% of [3.51, 4.25]
+# MXQ2=(minibatch 3.51 3.47 3.44 3.40 3.37 3.33 4.25 4.21 4.17 4.12 4.08 4.04)
+MXQ2=(minibatch 6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07)
+MXQ_BATCHES=(MXQ2)
+declare -n MXQ_BATCH
+
+ATTEMPT="sensi-milp-mini2"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+EXP_BASE_NAME="${ATTEMPT}"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+# Setup data files directories for reporting
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+
+weight_algo=sensi-milp
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+  batch_name=${MXQ_BATCH[@]:0:1}
+  EXP_NAME="${EXP_BASE_NAME}-${batch_name}"
+  mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+  mkdir -p $RESULT_DIR/${EXP_NAME}_stor
+  log_file="logs/bench-${EXP_NAME}-$(date +%Y%m%d%H%M%S).log"
+
+  # echo "=========Delete quantized models of batch ${batch_name}========="
+  # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+  OLD_DIR=$(pwd)
+  cd $RESULT_DIR/$EXP_BASE_NAME
+  if [ ! -d pdfs ]; then
+      mkdir pdfs
+  fi
+
+  # plot configuration allocations for 3 * 12 MXQ combinations
+  MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+  BUDGETS=${MXQ_BATCH[@]:1}
+  for model in $MODELS; do
+      for budget in $BUDGETS; do
+          $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+            -m $model \
+            -b $budget \
+            --fnorm_data_dir $OLD_DIR/../src/data \
+            --attempt1 mxq1 \
+            --attempt2 $ATTEMPT \
+            --quant_cfg_allot_file data/quant-cfg-allocation.csv
+      done
+  done
+  cd $OLD_DIR
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 1 \
+    --algo awq \
+    --config b4g32 b4g64 b4g128 \
+    --experiment-name quant_llm_13B-awq2 \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo mxq \
+    --config 2.57 2.55 2.53 2.51 2.49 2.47 2.45 2.31 2.29 2.27 2.25 2.23 2.21 2.19 2.17 2.15 2.13 \
+    --experiment-name quant_llm_2bit_dense1-mxq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo mxq \
+    --config 3.57 3.55 3.53 3.51 3.49 3.47 3.45 3.31 3.29 3.27 3.25 3.23 3.21 3.19 3.17 3.15 3.13 3.11 3.09 3.07 \
+    --experiment-name quant_llm_3bit_dense1-mxq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 \
+    --algo mxq \
+    --config 4.51 \
+    --experiment-name quant_llm_4_51-mxq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo mxq \
+    --config 4.61 4.59 4.57 4.55 4.53 4.51 4.49 4.47 4.45 4.43 4.41 4.35 4.33 4.31 4.29 4.27 4.25 4.23 4.21 4.19 4.17 4.15 4.13 4.11 4.09 4.07 4.05 4.03 \
+    --experiment-name quant_llm_4bit_dense1-mxq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo gptq \
+    --config b8g32 b8g64 b8g128 \
+    --experiment-name quant_llm-8bit-gptq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-awq.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 1 \
+    --algo awq \
+    --config b4g32 b4g64 b4g128 \
+    --experiment-name quant_llm_13B-awq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo mxq \
+    --config 4.51 4.25 4.13 \
+    --experiment-name quant_llm_b4-mxq2 \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-gptq.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/lm-quant-toolkit/src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo gptq \
+    --experiment-name quant_llm-gptq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-hqq.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo hqq \
+    --experiment-name quant_llm-hqq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 1 \
+    --algo mxq \
+    --config 4.51 4.25 4.13 \
+    --experiment-name quant_llm-weighted-113b-mxq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots-kurt" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/quant-llm-mxq.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+    mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+    --task quant \
+    --model 0 1 2 \
+    --algo mxq \
+    --experiment-name quant_llm-mxq \
+    --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+    --result-dir="/fdata/llm/mxq/results" \
+    2>&1 \
+    | tee logs/bench-$(date +%Y%m%d%H%M%S).log

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-allot.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+
+TAILS=(tail-prioritized 2.00 4.00)
+# MXQ_BATCHES=(HEADS TAILS)
+MXQ_BATCHES=(TAILS)
+declare -n MXQ_BATCH
+
+EXP_BASE_NAME="tail-factor-search"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+  weight_algo=${MXQ_BATCH[@]:0:1}
+  for factor in ${MXQ_BATCH[@]:1}; do
+      ATTEMPT="${weight_algo}_${factor/./_}"
+      EXP_NAME="${ATTEMPT}"
+      log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+      mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+      mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+
+      # python -m pdb ../src/cli.py dump \
+      python ../src/cli.py dump \
+        --type quant_config_sim \
+        --model 0 1 2 \
+        --weight-algo $weight_algo \
+        --factor $factor \
+        --budget ${BUDGETS} \
+        --output-file="$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/${EXP_NAME}.csv"\
+        2>&1 \
+        | tee -a $log_file
+
+  done
+
+done
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+for model in $MODELS; do
+    for budget in $BUDGETS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 tail-prioritized_2_00 \
+          --attempt2 tail-prioritized_4_00 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-quant-milp.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# python ../src/lm_quant_toolkit/eval/common.py > 2-8-quant-bits.txt
+python ../src/lm_quant_toolkit/eval/common.py > 2-8-quant-bits-sensi.txt

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+weight_algo="tail_boost"
+EXP_BASE_NAME="$weight_algo"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME
+
+ATTEMPT="$weight_algo"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+
+# python -m pdb ../src/cli.py dump \
+python ../src/cli.py dump \
+    --type quant_config_sim \
+    --model 0 1 2 \
+    --weight-algo $weight_algo \
+    --budget ${BUDGETS} \
+    --output-file="$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/${EXP_NAME}.csv"\
+    2>&1 \
+    | tee -a $log_file
+
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+    mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+    --baseline_data_dir $OLD_DIR/../data-vis/data \
+    --mxq_data_dir data
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+for model in $MODELS; do
+    for budget in $BUDGETS; do
+        $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+          -m $model \
+          -b $budget \
+          --attempt1 $weight_algo \
+          --attempt2 mxq1 \
+          --fnorm_data_dir $OLD_DIR/../src/data \
+          --quant_cfg_allot_file data/quant-cfg-allocation.csv
+    done
+done

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/fnorm-CLIP-ViT-B-32-laion2B-s34B-b79K.csv

@@ -0,0 +1,577 @@
+layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params,kurtosis
+0,vision.mlp.c_fc,2,32,8,128,9.128504753112791,0.705322265625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,2,64,8,128,10.76142692565918,0.6339111328125,2359296,7.5647333557116845
+0,vision.mlp.c_fc,2,128,8,128,12.15142059326172,0.59820556640625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,3,32,8,128,4.122710227966309,0.986572265625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,3,64,8,128,5.038382530212402,0.9151611328125,2359296,7.5647333557116845
+0,vision.mlp.c_fc,3,128,8,128,5.910652160644531,0.87945556640625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,4,32,8,128,1.938106060028076,1.267822265625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,4,64,8,128,2.3859243392944336,1.1964111328125,2359296,7.5647333557116845
+0,vision.mlp.c_fc,4,128,8,128,2.824324131011963,1.16070556640625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,8,32,8,128,0.11737483739852904,2.392822265625,2359296,7.5647333557116845
+0,vision.mlp.c_fc,8,64,8,128,0.14461086690425873,2.3214111328125,2359296,7.5647333557116845
+0,vision.mlp.c_fc,8,128,8,128,0.171279177069664,2.28570556640625,2359296,7.5647333557116845
+0,vision.mlp.c_proj,2,32,8,128,6.839951992034912,0.705322265625,2359296,8.024042788345657
+0,vision.mlp.c_proj,2,64,8,128,8.144723892211914,0.6339111328125,2359296,8.024042788345657
+0,vision.mlp.c_proj,2,128,8,128,9.282066345214844,0.59820556640625,2359296,8.024042788345657
+0,vision.mlp.c_proj,3,32,8,128,2.976739406585694,0.986572265625,2359296,8.024042788345657
+0,vision.mlp.c_proj,3,64,8,128,3.607946395874023,0.9151611328125,2359296,8.024042788345657
+0,vision.mlp.c_proj,3,128,8,128,4.226345539093018,0.87945556640625,2359296,8.024042788345657
+0,vision.mlp.c_proj,4,32,8,128,1.3918942213058472,1.267822265625,2359296,8.024042788345657
+0,vision.mlp.c_proj,4,64,8,128,1.6903578042984009,1.1964111328125,2359296,8.024042788345657
+0,vision.mlp.c_proj,4,128,8,128,1.9883592128753664,1.16070556640625,2359296,8.024042788345657
+0,vision.mlp.c_proj,8,32,8,128,0.0842554122209549,2.392822265625,2359296,8.024042788345657
+0,vision.mlp.c_proj,8,64,8,128,0.10231027752161026,2.3214111328125,2359296,8.024042788345657
+0,vision.mlp.c_proj,8,128,8,128,0.11994849145412444,2.28570556640625,2359296,8.024042788345657
+1,vision.mlp.c_fc,2,32,8,128,9.103703498840332,0.705322265625,2359296,5.556482388585281
+1,vision.mlp.c_fc,2,64,8,128,10.854814529418944,0.6339111328125,2359296,5.556482388585281
+1,vision.mlp.c_fc,2,128,8,128,12.433868408203123,0.59820556640625,2359296,5.556482388585281
+1,vision.mlp.c_fc,3,32,8,128,4.004519939422607,0.986572265625,2359296,5.556482388585281
+1,vision.mlp.c_fc,3,64,8,128,4.869235992431641,0.9151611328125,2359296,5.556482388585281
+1,vision.mlp.c_fc,3,128,8,128,5.696018218994141,0.87945556640625,2359296,5.556482388585281
+1,vision.mlp.c_fc,4,32,8,128,1.875130295753479,1.267822265625,2359296,5.556482388585281
+1,vision.mlp.c_fc,4,64,8,128,2.288634777069092,1.1964111328125,2359296,5.556482388585281
+1,vision.mlp.c_fc,4,128,8,128,2.692777395248413,1.16070556640625,2359296,5.556482388585281
+1,vision.mlp.c_fc,8,32,8,128,0.1136341467499733,2.392822265625,2359296,5.556482388585281
+1,vision.mlp.c_fc,8,64,8,128,0.13860879838466644,2.3214111328125,2359296,5.556482388585281
+1,vision.mlp.c_fc,8,128,8,128,0.16262513399124146,2.28570556640625,2359296,5.556482388585281
+1,vision.mlp.c_proj,2,32,8,128,6.911839962005615,0.705322265625,2359296,10.127617353490804
+1,vision.mlp.c_proj,2,64,8,128,8.261187553405762,0.6339111328125,2359296,10.127617353490804
+1,vision.mlp.c_proj,2,128,8,128,9.480621337890623,0.59820556640625,2359296,10.127617353490804
+1,vision.mlp.c_proj,3,32,8,128,2.991220712661743,0.986572265625,2359296,10.127617353490804
+1,vision.mlp.c_proj,3,64,8,128,3.6111984252929688,0.9151611328125,2359296,10.127617353490804
+1,vision.mlp.c_proj,3,128,8,128,4.218574523925781,0.87945556640625,2359296,10.127617353490804
+1,vision.mlp.c_proj,4,32,8,128,1.3967558145523071,1.267822265625,2359296,10.127617353490804
+1,vision.mlp.c_proj,4,64,8,128,1.6911554336547852,1.1964111328125,2359296,10.127617353490804
+1,vision.mlp.c_proj,4,128,8,128,1.983748197555542,1.16070556640625,2359296,10.127617353490804
+1,vision.mlp.c_proj,8,32,8,128,0.08465902507305145,2.392822265625,2359296,10.127617353490804
+1,vision.mlp.c_proj,8,64,8,128,0.10233005881309508,2.3214111328125,2359296,10.127617353490804
+1,vision.mlp.c_proj,8,128,8,128,0.11965607851743698,2.28570556640625,2359296,10.127617353490804
+2,vision.mlp.c_fc,2,32,8,128,8.82172966003418,0.705322265625,2359296,4.752866879979445
+2,vision.mlp.c_fc,2,64,8,128,10.50925350189209,0.6339111328125,2359296,4.752866879979445
+2,vision.mlp.c_fc,2,128,8,128,12.03489589691162,0.59820556640625,2359296,4.752866879979445
+2,vision.mlp.c_fc,3,32,8,128,3.823518753051758,0.986572265625,2359296,4.752866879979445
+2,vision.mlp.c_fc,3,64,8,128,4.6140289306640625,0.9151611328125,2359296,4.752866879979445
+2,vision.mlp.c_fc,3,128,8,128,5.374442100524902,0.87945556640625,2359296,4.752866879979445
+2,vision.mlp.c_fc,4,32,8,128,1.785937786102295,1.267822265625,2359296,4.752866879979445
+2,vision.mlp.c_fc,4,64,8,128,2.158376693725586,1.1964111328125,2359296,4.752866879979445
+2,vision.mlp.c_fc,4,128,8,128,2.523719310760498,1.16070556640625,2359296,4.752866879979445
+2,vision.mlp.c_fc,8,32,8,128,0.10810095071792604,2.392822265625,2359296,4.752866879979445
+2,vision.mlp.c_fc,8,64,8,128,0.13063114881515503,2.3214111328125,2359296,4.752866879979445
+2,vision.mlp.c_fc,8,128,8,128,0.15222015976905823,2.28570556640625,2359296,4.752866879979445
+2,vision.mlp.c_proj,2,32,8,128,6.639797687530518,0.705322265625,2359296,5.964957611759415
+2,vision.mlp.c_proj,2,64,8,128,7.880718231201172,0.6339111328125,2359296,5.964957611759415
+2,vision.mlp.c_proj,2,128,8,128,9.019220352172852,0.59820556640625,2359296,5.964957611759415
+2,vision.mlp.c_proj,3,32,8,128,2.859175443649292,0.986572265625,2359296,5.964957611759415
+2,vision.mlp.c_proj,3,64,8,128,3.413665294647217,0.9151611328125,2359296,5.964957611759415
+2,vision.mlp.c_proj,3,128,8,128,3.95273756980896,0.87945556640625,2359296,5.964957611759415
+2,vision.mlp.c_proj,4,32,8,128,1.333265781402588,1.267822265625,2359296,5.964957611759415
+2,vision.mlp.c_proj,4,64,8,128,1.5933430194854736,1.1964111328125,2359296,5.964957611759415
+2,vision.mlp.c_proj,4,128,8,128,1.8488850593566897,1.16070556640625,2359296,5.964957611759415
+2,vision.mlp.c_proj,8,32,8,128,0.08075807988643646,2.392822265625,2359296,5.964957611759415
+2,vision.mlp.c_proj,8,64,8,128,0.09648673981428146,2.3214111328125,2359296,5.964957611759415
+2,vision.mlp.c_proj,8,128,8,128,0.11135073751211166,2.28570556640625,2359296,5.964957611759415
+3,vision.mlp.c_fc,2,32,8,128,8.47494125366211,0.705322265625,2359296,4.378746321591061
+3,vision.mlp.c_fc,2,64,8,128,10.082805633544922,0.6339111328125,2359296,4.378746321591061
+3,vision.mlp.c_fc,2,128,8,128,11.549962043762209,0.59820556640625,2359296,4.378746321591061
+3,vision.mlp.c_fc,3,32,8,128,3.6546263694763184,0.986572265625,2359296,4.378746321591061
+3,vision.mlp.c_fc,3,64,8,128,4.391681671142578,0.9151611328125,2359296,4.378746321591061
+3,vision.mlp.c_fc,3,128,8,128,5.095606327056885,0.87945556640625,2359296,4.378746321591061
+3,vision.mlp.c_fc,4,32,8,128,1.706535577774048,1.267822265625,2359296,4.378746321591061
+3,vision.mlp.c_fc,4,64,8,128,2.050750970840454,1.1964111328125,2359296,4.378746321591061
+3,vision.mlp.c_fc,4,128,8,128,2.3856451511383057,1.16070556640625,2359296,4.378746321591061
+3,vision.mlp.c_fc,8,32,8,128,0.10344040393829346,2.392822265625,2359296,4.378746321591061
+3,vision.mlp.c_fc,8,64,8,128,0.12399496138095856,2.3214111328125,2359296,4.378746321591061
+3,vision.mlp.c_fc,8,128,8,128,0.1439141482114792,2.28570556640625,2359296,4.378746321591061
+3,vision.mlp.c_proj,2,32,8,128,6.277030944824219,0.705322265625,2359296,5.006922011163274
+3,vision.mlp.c_proj,2,64,8,128,7.414824485778809,0.6339111328125,2359296,5.006922011163274
+3,vision.mlp.c_proj,2,128,8,128,8.46260929107666,0.59820556640625,2359296,5.006922011163274
+3,vision.mlp.c_proj,3,32,8,128,2.693903923034668,0.986572265625,2359296,5.006922011163274
+3,vision.mlp.c_proj,3,64,8,128,3.1912341117858887,0.9151611328125,2359296,5.006922011163274
+3,vision.mlp.c_proj,3,128,8,128,3.6711747646331783,0.87945556640625,2359296,5.006922011163274
+3,vision.mlp.c_proj,4,32,8,128,1.256372094154358,1.267822265625,2359296,5.006922011163274
+3,vision.mlp.c_proj,4,64,8,128,1.489571452140808,1.1964111328125,2359296,5.006922011163274
+3,vision.mlp.c_proj,4,128,8,128,1.7132800817489624,1.16070556640625,2359296,5.006922011163274
+3,vision.mlp.c_proj,8,32,8,128,0.0760984867811203,2.392822265625,2359296,5.006922011163274
+3,vision.mlp.c_proj,8,64,8,128,0.09016053378582,2.3214111328125,2359296,5.006922011163274
+3,vision.mlp.c_proj,8,128,8,128,0.10330581665039062,2.28570556640625,2359296,5.006922011163274
+4,vision.mlp.c_fc,2,32,8,128,8.274068832397461,0.705322265625,2359296,3.918504223899523
+4,vision.mlp.c_fc,2,64,8,128,9.78947639465332,0.6339111328125,2359296,3.918504223899523
+4,vision.mlp.c_fc,2,128,8,128,11.202027320861816,0.59820556640625,2359296,3.918504223899523
+4,vision.mlp.c_fc,3,32,8,128,3.5570082664489746,0.986572265625,2359296,3.918504223899523
+4,vision.mlp.c_fc,3,64,8,128,4.2356367111206055,0.9151611328125,2359296,3.918504223899523
+4,vision.mlp.c_fc,3,128,8,128,4.884209632873535,0.87945556640625,2359296,3.918504223899523
+4,vision.mlp.c_fc,4,32,8,128,1.6616051197052002,1.267822265625,2359296,3.918504223899523
+4,vision.mlp.c_fc,4,64,8,128,1.979529857635498,1.1964111328125,2359296,3.918504223899523
+4,vision.mlp.c_fc,4,128,8,128,2.284747838973999,1.16070556640625,2359296,3.918504223899523
+4,vision.mlp.c_fc,8,32,8,128,0.10057099163532256,2.392822265625,2359296,3.918504223899523
+4,vision.mlp.c_fc,8,64,8,128,0.11972232908010484,2.3214111328125,2359296,3.918504223899523
+4,vision.mlp.c_fc,8,128,8,128,0.13779735565185547,2.28570556640625,2359296,3.918504223899523
+4,vision.mlp.c_proj,2,32,8,128,6.439798831939697,0.705322265625,2359296,4.362240481953023
+4,vision.mlp.c_proj,2,64,8,128,7.564638137817383,0.6339111328125,2359296,4.362240481953023
+4,vision.mlp.c_proj,2,128,8,128,8.594568252563477,0.59820556640625,2359296,4.362240481953023
+4,vision.mlp.c_proj,3,32,8,128,2.7584915161132812,0.986572265625,2359296,4.362240481953023
+4,vision.mlp.c_proj,3,64,8,128,3.251418113708496,0.9151611328125,2359296,4.362240481953023
+4,vision.mlp.c_proj,3,128,8,128,3.716466665267944,0.87945556640625,2359296,4.362240481953023
+4,vision.mlp.c_proj,4,32,8,128,1.2862253189086914,1.267822265625,2359296,4.362240481953023
+4,vision.mlp.c_proj,4,64,8,128,1.516956448554993,1.1964111328125,2359296,4.362240481953023
+4,vision.mlp.c_proj,4,128,8,128,1.7344794273376465,1.16070556640625,2359296,4.362240481953023
+4,vision.mlp.c_proj,8,32,8,128,0.07793856412172318,2.392822265625,2359296,4.362240481953023
+4,vision.mlp.c_proj,8,64,8,128,0.0918104276061058,2.3214111328125,2359296,4.362240481953023
+4,vision.mlp.c_proj,8,128,8,128,0.10458982735872269,2.28570556640625,2359296,4.362240481953023
+5,vision.mlp.c_fc,2,32,8,128,8.392443656921387,0.705322265625,2359296,3.714197056179566
+5,vision.mlp.c_fc,2,64,8,128,9.947240829467772,0.6339111328125,2359296,3.714197056179566
+5,vision.mlp.c_fc,2,128,8,128,11.396702766418455,0.59820556640625,2359296,3.714197056179566
+5,vision.mlp.c_fc,3,32,8,128,3.599330425262451,0.986572265625,2359296,3.714197056179566
+5,vision.mlp.c_fc,3,64,8,128,4.279275417327881,0.9151611328125,2359296,3.714197056179566
+5,vision.mlp.c_fc,3,128,8,128,4.932392597198486,0.87945556640625,2359296,3.714197056179566
+5,vision.mlp.c_fc,4,32,8,128,1.6796408891677856,1.267822265625,2359296,3.714197056179566
+5,vision.mlp.c_fc,4,64,8,128,1.997564435005188,1.1964111328125,2359296,3.714197056179566
+5,vision.mlp.c_fc,4,128,8,128,2.30387020111084,1.16070556640625,2359296,3.714197056179566
+5,vision.mlp.c_fc,8,32,8,128,0.10184947401285172,2.392822265625,2359296,3.714197056179566
+5,vision.mlp.c_fc,8,64,8,128,0.12093502283096312,2.3214111328125,2359296,3.714197056179566
+5,vision.mlp.c_fc,8,128,8,128,0.13896310329437256,2.28570556640625,2359296,3.714197056179566
+5,vision.mlp.c_proj,2,32,8,128,6.786053657531738,0.705322265625,2359296,4.504410342372665
+5,vision.mlp.c_proj,2,64,8,128,7.957103252410889,0.6339111328125,2359296,4.504410342372665
+5,vision.mlp.c_proj,2,128,8,128,9.033921241760254,0.59820556640625,2359296,4.504410342372665
+5,vision.mlp.c_proj,3,32,8,128,2.905226469039917,0.986572265625,2359296,4.504410342372665
+5,vision.mlp.c_proj,3,64,8,128,3.4214489459991455,0.9151611328125,2359296,4.504410342372665
+5,vision.mlp.c_proj,3,128,8,128,3.9075539112091064,0.87945556640625,2359296,4.504410342372665
+5,vision.mlp.c_proj,4,32,8,128,1.355237364768982,1.267822265625,2359296,4.504410342372665
+5,vision.mlp.c_proj,4,64,8,128,1.5957733392715454,1.1964111328125,2359296,4.504410342372665
+5,vision.mlp.c_proj,4,128,8,128,1.8257287740707395,1.16070556640625,2359296,4.504410342372665
+5,vision.mlp.c_proj,8,32,8,128,0.0820372924208641,2.392822265625,2359296,4.504410342372665
+5,vision.mlp.c_proj,8,64,8,128,0.09651072323322296,2.3214111328125,2359296,4.504410342372665
+5,vision.mlp.c_proj,8,128,8,128,0.10990241914987564,2.28570556640625,2359296,4.504410342372665
+6,vision.mlp.c_fc,2,32,8,128,8.521951675415039,0.705322265625,2359296,3.614600960861904
+6,vision.mlp.c_fc,2,64,8,128,10.0691499710083,0.6339111328125,2359296,3.614600960861904
+6,vision.mlp.c_fc,2,128,8,128,11.54863739013672,0.59820556640625,2359296,3.614600960861904
+6,vision.mlp.c_fc,3,32,8,128,3.649571895599365,0.986572265625,2359296,3.614600960861904
+6,vision.mlp.c_fc,3,64,8,128,4.324981689453125,0.9151611328125,2359296,3.614600960861904
+6,vision.mlp.c_fc,3,128,8,128,4.97314977645874,0.87945556640625,2359296,3.614600960861904
+6,vision.mlp.c_fc,4,32,8,128,1.7034014463424685,1.267822265625,2359296,3.614600960861904
+6,vision.mlp.c_fc,4,64,8,128,2.0181891918182373,1.1964111328125,2359296,3.614600960861904
+6,vision.mlp.c_fc,4,128,8,128,2.3228890895843506,1.16070556640625,2359296,3.614600960861904
+6,vision.mlp.c_fc,8,32,8,128,0.10322079807519911,2.392822265625,2359296,3.614600960861904
+6,vision.mlp.c_fc,8,64,8,128,0.12220923602581024,2.3214111328125,2359296,3.614600960861904
+6,vision.mlp.c_fc,8,128,8,128,0.14006850123405457,2.28570556640625,2359296,3.614600960861904
+6,vision.mlp.c_proj,2,32,8,128,7.034173488616943,0.705322265625,2359296,3.703802696792269
+6,vision.mlp.c_proj,2,64,8,128,8.234294891357422,0.6339111328125,2359296,3.703802696792269
+6,vision.mlp.c_proj,2,128,8,128,9.333321571350098,0.59820556640625,2359296,3.703802696792269
+6,vision.mlp.c_proj,3,32,8,128,3.009636878967285,0.986572265625,2359296,3.703802696792269
+6,vision.mlp.c_proj,3,64,8,128,3.5273571014404297,0.9151611328125,2359296,3.703802696792269
+6,vision.mlp.c_proj,3,128,8,128,4.010415077209473,0.87945556640625,2359296,3.703802696792269
+6,vision.mlp.c_proj,4,32,8,128,1.404062271118164,1.267822265625,2359296,3.703802696792269
+6,vision.mlp.c_proj,4,64,8,128,1.6456388235092163,1.1964111328125,2359296,3.703802696792269
+6,vision.mlp.c_proj,4,128,8,128,1.8724839687347412,1.16070556640625,2359296,3.703802696792269
+6,vision.mlp.c_proj,8,32,8,128,0.08510661125183105,2.392822265625,2359296,3.703802696792269
+6,vision.mlp.c_proj,8,64,8,128,0.09971028566360474,2.3214111328125,2359296,3.703802696792269
+6,vision.mlp.c_proj,8,128,8,128,0.1129401922225952,2.28570556640625,2359296,3.703802696792269
+7,vision.mlp.c_fc,2,32,8,128,8.388879776000977,0.705322265625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,2,64,8,128,9.876380920410156,0.6339111328125,2359296,3.4103181441120247
+7,vision.mlp.c_fc,2,128,8,128,11.25124740600586,0.59820556640625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,3,32,8,128,3.5898404121398926,0.986572265625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,3,64,8,128,4.232213020324707,0.9151611328125,2359296,3.4103181441120247
+7,vision.mlp.c_fc,3,128,8,128,4.836949348449707,0.87945556640625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,4,32,8,128,1.674543380737305,1.267822265625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,4,64,8,128,1.9748677015304563,1.1964111328125,2359296,3.4103181441120247
+7,vision.mlp.c_fc,4,128,8,128,2.2568445205688477,1.16070556640625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,8,32,8,128,0.10150232911109924,2.392822265625,2359296,3.4103181441120247
+7,vision.mlp.c_fc,8,64,8,128,0.11956389248371124,2.3214111328125,2359296,3.4103181441120247
+7,vision.mlp.c_fc,8,128,8,128,0.1361236423254013,2.28570556640625,2359296,3.4103181441120247
+7,vision.mlp.c_proj,2,32,8,128,7.932621479034424,0.705322265625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,2,64,8,128,9.285072326660156,0.6339111328125,2359296,3.7265741474392886
+7,vision.mlp.c_proj,2,128,8,128,10.539323806762695,0.59820556640625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,3,32,8,128,3.394057273864746,0.986572265625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,3,64,8,128,3.9794647693634033,0.9151611328125,2359296,3.7265741474392886
+7,vision.mlp.c_proj,3,128,8,128,4.525967597961426,0.87945556640625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,4,32,8,128,1.583614945411682,1.267822265625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,4,64,8,128,1.8572959899902344,1.1964111328125,2359296,3.7265741474392886
+7,vision.mlp.c_proj,4,128,8,128,2.113091468811035,1.16070556640625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,8,32,8,128,0.09593687206506728,2.392822265625,2359296,3.7265741474392886
+7,vision.mlp.c_proj,8,64,8,128,0.11239387094974518,2.3214111328125,2359296,3.7265741474392886
+7,vision.mlp.c_proj,8,128,8,128,0.12745003402233124,2.28570556640625,2359296,3.7265741474392886
+8,vision.mlp.c_fc,2,32,8,128,8.285651206970215,0.705322265625,2359296,3.734153132112317
+8,vision.mlp.c_fc,2,64,8,128,9.778728485107422,0.6339111328125,2359296,3.734153132112317
+8,vision.mlp.c_fc,2,128,8,128,11.152859687805176,0.59820556640625,2359296,3.734153132112317
+8,vision.mlp.c_fc,3,32,8,128,3.547513008117676,0.986572265625,2359296,3.734153132112317
+8,vision.mlp.c_fc,3,64,8,128,4.191949367523193,0.9151611328125,2359296,3.734153132112317
+8,vision.mlp.c_fc,3,128,8,128,4.793965816497803,0.87945556640625,2359296,3.734153132112317
+8,vision.mlp.c_fc,4,32,8,128,1.6551411151885986,1.267822265625,2359296,3.734153132112317
+8,vision.mlp.c_fc,4,64,8,128,1.956448316574097,1.1964111328125,2359296,3.734153132112317
+8,vision.mlp.c_fc,4,128,8,128,2.2382309436798096,1.16070556640625,2359296,3.734153132112317
+8,vision.mlp.c_fc,8,32,8,128,0.100438229739666,2.392822265625,2359296,3.734153132112317
+8,vision.mlp.c_fc,8,64,8,128,0.11862816661596298,2.3214111328125,2359296,3.734153132112317
+8,vision.mlp.c_fc,8,128,8,128,0.13508424162864685,2.28570556640625,2359296,3.734153132112317
+8,vision.mlp.c_proj,2,32,8,128,8.032468795776367,0.705322265625,2359296,3.770444941283215
+8,vision.mlp.c_proj,2,64,8,128,9.38332176208496,0.6339111328125,2359296,3.770444941283215
+8,vision.mlp.c_proj,2,128,8,128,10.626753807067873,0.59820556640625,2359296,3.770444941283215
+8,vision.mlp.c_proj,3,32,8,128,3.433583498001098,0.986572265625,2359296,3.770444941283215
+8,vision.mlp.c_proj,3,64,8,128,4.0180745124816895,0.9151611328125,2359296,3.770444941283215
+8,vision.mlp.c_proj,3,128,8,128,4.556089878082275,0.87945556640625,2359296,3.770444941283215
+8,vision.mlp.c_proj,4,32,8,128,1.602971076965332,1.267822265625,2359296,3.770444941283215
+8,vision.mlp.c_proj,4,64,8,128,1.875213384628296,1.1964111328125,2359296,3.770444941283215
+8,vision.mlp.c_proj,4,128,8,128,2.125036239624023,1.16070556640625,2359296,3.770444941283215
+8,vision.mlp.c_proj,8,32,8,128,0.09719990193843842,2.392822265625,2359296,3.770444941283215
+8,vision.mlp.c_proj,8,64,8,128,0.1135389283299446,2.3214111328125,2359296,3.770444941283215
+8,vision.mlp.c_proj,8,128,8,128,0.1282404214143753,2.28570556640625,2359296,3.770444941283215
+9,vision.mlp.c_fc,2,32,8,128,8.818232536315918,0.705322265625,2359296,3.406923087940011
+9,vision.mlp.c_fc,2,64,8,128,10.373499870300291,0.6339111328125,2359296,3.406923087940011
+9,vision.mlp.c_fc,2,128,8,128,11.801898002624512,0.59820556640625,2359296,3.406923087940011
+9,vision.mlp.c_fc,3,32,8,128,3.7690412998199463,0.986572265625,2359296,3.406923087940011
+9,vision.mlp.c_fc,3,64,8,128,4.43658447265625,0.9151611328125,2359296,3.406923087940011
+9,vision.mlp.c_fc,3,128,8,128,5.0595269203186035,0.87945556640625,2359296,3.406923087940011
+9,vision.mlp.c_fc,4,32,8,128,1.759302854537964,1.267822265625,2359296,3.406923087940011
+9,vision.mlp.c_fc,4,64,8,128,2.071089506149292,1.1964111328125,2359296,3.406923087940011
+9,vision.mlp.c_fc,4,128,8,128,2.3599820137023926,1.16070556640625,2359296,3.406923087940011
+9,vision.mlp.c_fc,8,32,8,128,0.10690240561962128,2.392822265625,2359296,3.406923087940011
+9,vision.mlp.c_fc,8,64,8,128,0.1255631297826767,2.3214111328125,2359296,3.406923087940011
+9,vision.mlp.c_fc,8,128,8,128,0.14248855412006378,2.28570556640625,2359296,3.406923087940011
+9,vision.mlp.c_proj,2,32,8,128,9.092228889465332,0.705322265625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,2,64,8,128,10.604211807250977,0.6339111328125,2359296,3.7473560752962425
+9,vision.mlp.c_proj,2,128,8,128,11.983688354492188,0.59820556640625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,3,32,8,128,3.8891754150390625,0.986572265625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,3,64,8,128,4.537405967712402,0.9151611328125,2359296,3.7473560752962425
+9,vision.mlp.c_proj,3,128,8,128,5.127986431121826,0.87945556640625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,4,32,8,128,1.8149250745773315,1.267822265625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,4,64,8,128,2.1180083751678467,1.1964111328125,2359296,3.7473560752962425
+9,vision.mlp.c_proj,4,128,8,128,2.3949661254882812,1.16070556640625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,8,32,8,128,0.10990245640277864,2.392822265625,2359296,3.7473560752962425
+9,vision.mlp.c_proj,8,64,8,128,0.12820811569690704,2.3214111328125,2359296,3.7473560752962425
+9,vision.mlp.c_proj,8,128,8,128,0.14443348348140717,2.28570556640625,2359296,3.7473560752962425
+10,vision.mlp.c_fc,2,32,8,128,9.356571197509766,0.705322265625,2359296,3.506843786702064
+10,vision.mlp.c_fc,2,64,8,128,10.991547584533691,0.6339111328125,2359296,3.506843786702064
+10,vision.mlp.c_fc,2,128,8,128,12.464940071105955,0.59820556640625,2359296,3.506843786702064
+10,vision.mlp.c_fc,3,32,8,128,3.9996769428253174,0.986572265625,2359296,3.506843786702064
+10,vision.mlp.c_fc,3,64,8,128,4.703681468963623,0.9151611328125,2359296,3.506843786702064
+10,vision.mlp.c_fc,3,128,8,128,5.345017910003662,0.87945556640625,2359296,3.506843786702064
+10,vision.mlp.c_fc,4,32,8,128,1.865874171257019,1.267822265625,2359296,3.506843786702064
+10,vision.mlp.c_fc,4,64,8,128,2.195168972015381,1.1964111328125,2359296,3.506843786702064
+10,vision.mlp.c_fc,4,128,8,128,2.495924472808838,1.16070556640625,2359296,3.506843786702064
+10,vision.mlp.c_fc,8,32,8,128,0.11316034197807312,2.392822265625,2359296,3.506843786702064
+10,vision.mlp.c_fc,8,64,8,128,0.13293932378292084,2.3214111328125,2359296,3.506843786702064
+10,vision.mlp.c_fc,8,128,8,128,0.15064848959445953,2.28570556640625,2359296,3.506843786702064
+10,vision.mlp.c_proj,2,32,8,128,9.723986625671388,0.705322265625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,2,64,8,128,11.35032558441162,0.6339111328125,2359296,3.7916828549200345
+10,vision.mlp.c_proj,2,128,8,128,12.832988739013672,0.59820556640625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,3,32,8,128,4.1578369140625,0.986572265625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,3,64,8,128,4.862079620361328,0.9151611328125,2359296,3.7916828549200345
+10,vision.mlp.c_proj,3,128,8,128,5.495484352111816,0.87945556640625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,4,32,8,128,1.9391449689865112,1.267822265625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,4,64,8,128,2.2657644748687744,1.1964111328125,2359296,3.7916828549200345
+10,vision.mlp.c_proj,4,128,8,128,2.5652666091918945,1.16070556640625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,8,32,8,128,0.11764831840991974,2.392822265625,2359296,3.7916828549200345
+10,vision.mlp.c_proj,8,64,8,128,0.13722199201583862,2.3214111328125,2359296,3.7916828549200345
+10,vision.mlp.c_proj,8,128,8,128,0.1546991765499115,2.28570556640625,2359296,3.7916828549200345
+11,vision.mlp.c_fc,2,32,8,128,8.082159042358398,0.705322265625,2359296,3.336296405622262
+11,vision.mlp.c_fc,2,64,8,128,9.489198684692385,0.6339111328125,2359296,3.336296405622262
+11,vision.mlp.c_fc,2,128,8,128,10.802061080932615,0.59820556640625,2359296,3.336296405622262
+11,vision.mlp.c_fc,3,32,8,128,3.4594030380249023,0.986572265625,2359296,3.336296405622262
+11,vision.mlp.c_fc,3,64,8,128,4.068918704986572,0.9151611328125,2359296,3.336296405622262
+11,vision.mlp.c_fc,3,128,8,128,4.641410827636719,0.87945556640625,2359296,3.336296405622262
+11,vision.mlp.c_fc,4,32,8,128,1.6126223802566528,1.267822265625,2359296,3.336296405622262
+11,vision.mlp.c_fc,4,64,8,128,1.8982433080673216,1.1964111328125,2359296,3.336296405622262
+11,vision.mlp.c_fc,4,128,8,128,2.1648974418640137,1.16070556640625,2359296,3.336296405622262
+11,vision.mlp.c_fc,8,32,8,128,0.09783997386693954,2.392822265625,2359296,3.336296405622262
+11,vision.mlp.c_fc,8,64,8,128,0.11497506499290466,2.3214111328125,2359296,3.336296405622262
+11,vision.mlp.c_fc,8,128,8,128,0.1306662559509277,2.28570556640625,2359296,3.336296405622262
+11,vision.mlp.c_proj,2,32,8,128,7.7494797706604,0.705322265625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,2,64,8,128,9.04317855834961,0.6339111328125,2359296,3.5289485578856628
+11,vision.mlp.c_proj,2,128,8,128,10.23215389251709,0.59820556640625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,3,32,8,128,3.3150153160095215,0.986572265625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,3,64,8,128,3.8718862533569336,0.9151611328125,2359296,3.5289485578856628
+11,vision.mlp.c_proj,3,128,8,128,4.384607315063477,0.87945556640625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,4,32,8,128,1.545481562614441,1.267822265625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,4,64,8,128,1.806149125099182,1.1964111328125,2359296,3.5289485578856628
+11,vision.mlp.c_proj,4,128,8,128,2.0433690547943115,1.16070556640625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,8,32,8,128,0.0937107503414154,2.392822265625,2359296,3.5289485578856628
+11,vision.mlp.c_proj,8,64,8,128,0.10938263684511185,2.3214111328125,2359296,3.5289485578856628
+11,vision.mlp.c_proj,8,128,8,128,0.1232990100979805,2.28570556640625,2359296,3.5289485578856628
+0,text.mlp.c_fc,2,32,8,128,6.464750289916992,0.3134765625,1048576,18.89034464279892
+0,text.mlp.c_fc,2,64,8,128,7.523227214813232,0.28173828125,1048576,18.89034464279892
+0,text.mlp.c_fc,2,128,8,128,8.498074531555176,0.265869140625,1048576,18.89034464279892
+0,text.mlp.c_fc,3,32,8,128,2.7694246768951416,0.4384765625,1048576,18.89034464279892
+0,text.mlp.c_fc,3,64,8,128,3.2232933044433594,0.40673828125,1048576,18.89034464279892
+0,text.mlp.c_fc,3,128,8,128,3.6492912769317623,0.390869140625,1048576,18.89034464279892
+0,text.mlp.c_fc,4,32,8,128,1.2936551570892334,0.5634765625,1048576,18.89034464279892
+0,text.mlp.c_fc,4,64,8,128,1.5072178840637207,0.53173828125,1048576,18.89034464279892
+0,text.mlp.c_fc,4,128,8,128,1.7056944370269775,0.515869140625,1048576,18.89034464279892
+0,text.mlp.c_fc,8,32,8,128,0.07893156260251999,1.0634765625,1048576,18.89034464279892
+0,text.mlp.c_fc,8,64,8,128,0.09200920909643172,1.03173828125,1048576,18.89034464279892
+0,text.mlp.c_fc,8,128,8,128,0.10402266681194304,1.015869140625,1048576,18.89034464279892
+0,text.mlp.c_proj,2,32,8,128,4.214115142822266,0.3134765625,1048576,8.257733339213775
+0,text.mlp.c_proj,2,64,8,128,4.919820785522461,0.28173828125,1048576,8.257733339213775
+0,text.mlp.c_proj,2,128,8,128,5.572372913360596,0.265869140625,1048576,8.257733339213775
+0,text.mlp.c_proj,3,32,8,128,1.801061511039734,0.4384765625,1048576,8.257733339213775
+0,text.mlp.c_proj,3,64,8,128,2.1083173751831055,0.40673828125,1048576,8.257733339213775
+0,text.mlp.c_proj,3,128,8,128,2.391723871231079,0.390869140625,1048576,8.257733339213775
+0,text.mlp.c_proj,4,32,8,128,0.8409978151321411,0.5634765625,1048576,8.257733339213775
+0,text.mlp.c_proj,4,64,8,128,0.9834749698638916,0.53173828125,1048576,8.257733339213775
+0,text.mlp.c_proj,4,128,8,128,1.1154149770736694,0.515869140625,1048576,8.257733339213775
+0,text.mlp.c_proj,8,32,8,128,0.05123524367809296,1.0634765625,1048576,8.257733339213775
+0,text.mlp.c_proj,8,64,8,128,0.05990749597549438,1.03173828125,1048576,8.257733339213775
+0,text.mlp.c_proj,8,128,8,128,0.0680227056145668,1.015869140625,1048576,8.257733339213775
+1,text.mlp.c_fc,2,32,8,128,6.351968765258789,0.3134765625,1048576,3.0800089847603167
+1,text.mlp.c_fc,2,64,8,128,7.389888286590576,0.28173828125,1048576,3.0800089847603167
+1,text.mlp.c_fc,2,128,8,128,8.325387001037598,0.265869140625,1048576,3.0800089847603167
+1,text.mlp.c_fc,3,32,8,128,2.71328067779541,0.4384765625,1048576,3.0800089847603167
+1,text.mlp.c_fc,3,64,8,128,3.1635944843292236,0.40673828125,1048576,3.0800089847603167
+1,text.mlp.c_fc,3,128,8,128,3.561861991882324,0.390869140625,1048576,3.0800089847603167
+1,text.mlp.c_fc,4,32,8,128,1.266817927360535,0.5634765625,1048576,3.0800089847603167
+1,text.mlp.c_fc,4,64,8,128,1.4758033752441406,0.53173828125,1048576,3.0800089847603167
+1,text.mlp.c_fc,4,128,8,128,1.664614200592041,0.515869140625,1048576,3.0800089847603167
+1,text.mlp.c_fc,8,32,8,128,0.07674290984869003,1.0634765625,1048576,3.0800089847603167
+1,text.mlp.c_fc,8,64,8,128,0.08936163038015366,1.03173828125,1048576,3.0800089847603167
+1,text.mlp.c_fc,8,128,8,128,0.10043232887983322,1.015869140625,1048576,3.0800089847603167
+1,text.mlp.c_proj,2,32,8,128,4.492769718170166,0.3134765625,1048576,3.2167251445240903
+1,text.mlp.c_proj,2,64,8,128,5.24850606918335,0.28173828125,1048576,3.2167251445240903
+1,text.mlp.c_proj,2,128,8,128,5.924077033996582,0.265869140625,1048576,3.2167251445240903
+1,text.mlp.c_proj,3,32,8,128,1.9200174808502195,0.4384765625,1048576,3.2167251445240903
+1,text.mlp.c_proj,3,64,8,128,2.2457592487335205,0.40673828125,1048576,3.2167251445240903
+1,text.mlp.c_proj,3,128,8,128,2.539240598678589,0.390869140625,1048576,3.2167251445240903
+1,text.mlp.c_proj,4,32,8,128,0.8956782817840576,0.5634765625,1048576,3.2167251445240903
+1,text.mlp.c_proj,4,64,8,128,1.047521471977234,0.53173828125,1048576,3.2167251445240903
+1,text.mlp.c_proj,4,128,8,128,1.18568217754364,0.515869140625,1048576,3.2167251445240903
+1,text.mlp.c_proj,8,32,8,128,0.0543127991259098,1.0634765625,1048576,3.2167251445240903
+1,text.mlp.c_proj,8,64,8,128,0.06340169906616211,1.03173828125,1048576,3.2167251445240903
+1,text.mlp.c_proj,8,128,8,128,0.0714043602347374,1.015869140625,1048576,3.2167251445240903
+2,text.mlp.c_fc,2,32,8,128,6.2713623046875,0.3134765625,1048576,3.112510371474206
+2,text.mlp.c_fc,2,64,8,128,7.308136940002441,0.28173828125,1048576,3.112510371474206
+2,text.mlp.c_fc,2,128,8,128,8.229869842529297,0.265869140625,1048576,3.112510371474206
+2,text.mlp.c_fc,3,32,8,128,2.6817820072174072,0.4384765625,1048576,3.112510371474206
+2,text.mlp.c_fc,3,64,8,128,3.1245784759521484,0.40673828125,1048576,3.112510371474206
+2,text.mlp.c_fc,3,128,8,128,3.5175602436065674,0.390869140625,1048576,3.112510371474206
+2,text.mlp.c_fc,4,32,8,128,1.2500064373016355,0.5634765625,1048576,3.112510371474206
+2,text.mlp.c_fc,4,64,8,128,1.4578500986099243,0.53173828125,1048576,3.112510371474206
+2,text.mlp.c_fc,4,128,8,128,1.6433409452438354,0.515869140625,1048576,3.112510371474206
+2,text.mlp.c_fc,8,32,8,128,0.07576245814561844,1.0634765625,1048576,3.112510371474206
+2,text.mlp.c_fc,8,64,8,128,0.08829101175069809,1.03173828125,1048576,3.112510371474206
+2,text.mlp.c_fc,8,128,8,128,0.0992748960852623,1.015869140625,1048576,3.112510371474206
+2,text.mlp.c_proj,2,32,8,128,4.444961071014404,0.3134765625,1048576,3.292999208834084
+2,text.mlp.c_proj,2,64,8,128,5.180168628692627,0.28173828125,1048576,3.292999208834084
+2,text.mlp.c_proj,2,128,8,128,5.851198196411133,0.265869140625,1048576,3.292999208834084
+2,text.mlp.c_proj,3,32,8,128,1.899754285812378,0.4384765625,1048576,3.292999208834084
+2,text.mlp.c_proj,3,64,8,128,2.2183635234832764,0.40673828125,1048576,3.292999208834084
+2,text.mlp.c_proj,3,128,8,128,2.505136013031006,0.390869140625,1048576,3.292999208834084
+2,text.mlp.c_proj,4,32,8,128,0.8862735033035278,0.5634765625,1048576,3.292999208834084
+2,text.mlp.c_proj,4,64,8,128,1.0349557399749756,0.53173828125,1048576,3.292999208834084
+2,text.mlp.c_proj,4,128,8,128,1.1686416864395142,0.515869140625,1048576,3.292999208834084
+2,text.mlp.c_proj,8,32,8,128,0.05371652543544769,1.0634765625,1048576,3.292999208834084
+2,text.mlp.c_proj,8,64,8,128,0.06263848394155502,1.03173828125,1048576,3.292999208834084
+2,text.mlp.c_proj,8,128,8,128,0.07050774246454239,1.015869140625,1048576,3.292999208834084
+3,text.mlp.c_fc,2,32,8,128,6.199350357055664,0.3134765625,1048576,3.124703500598623
+3,text.mlp.c_fc,2,64,8,128,7.228718757629395,0.28173828125,1048576,3.124703500598623
+3,text.mlp.c_fc,2,128,8,128,8.150893211364746,0.265869140625,1048576,3.124703500598623
+3,text.mlp.c_fc,3,32,8,128,2.648998737335205,0.4384765625,1048576,3.124703500598623
+3,text.mlp.c_fc,3,64,8,128,3.0911576747894287,0.40673828125,1048576,3.124703500598623
+3,text.mlp.c_fc,3,128,8,128,3.488417863845825,0.390869140625,1048576,3.124703500598623
+3,text.mlp.c_fc,4,32,8,128,1.2359066009521484,0.5634765625,1048576,3.124703500598623
+3,text.mlp.c_fc,4,64,8,128,1.441812515258789,0.53173828125,1048576,3.124703500598623
+3,text.mlp.c_fc,4,128,8,128,1.6281919479370115,0.515869140625,1048576,3.124703500598623
+3,text.mlp.c_fc,8,32,8,128,0.07503717392683029,1.0634765625,1048576,3.124703500598623
+3,text.mlp.c_fc,8,64,8,128,0.0874534398317337,1.03173828125,1048576,3.124703500598623
+3,text.mlp.c_fc,8,128,8,128,0.09830143302679062,1.015869140625,1048576,3.124703500598623
+3,text.mlp.c_proj,2,32,8,128,4.45910120010376,0.3134765625,1048576,3.365812129846043
+3,text.mlp.c_proj,2,64,8,128,5.196355819702148,0.28173828125,1048576,3.365812129846043
+3,text.mlp.c_proj,2,128,8,128,5.872509479522705,0.265869140625,1048576,3.365812129846043
+3,text.mlp.c_proj,3,32,8,128,1.904448628425598,0.4384765625,1048576,3.365812129846043
+3,text.mlp.c_proj,3,64,8,128,2.2250046730041504,0.40673828125,1048576,3.365812129846043
+3,text.mlp.c_proj,3,128,8,128,2.5161800384521484,0.390869140625,1048576,3.365812129846043
+3,text.mlp.c_proj,4,32,8,128,0.8890355229377747,0.5634765625,1048576,3.365812129846043
+3,text.mlp.c_proj,4,64,8,128,1.0387694835662842,0.53173828125,1048576,3.365812129846043
+3,text.mlp.c_proj,4,128,8,128,1.1744872331619265,0.515869140625,1048576,3.365812129846043
+3,text.mlp.c_proj,8,32,8,128,0.05386906117200851,1.0634765625,1048576,3.365812129846043
+3,text.mlp.c_proj,8,64,8,128,0.06280947476625443,1.03173828125,1048576,3.365812129846043
+3,text.mlp.c_proj,8,128,8,128,0.07080750912427902,1.015869140625,1048576,3.365812129846043
+4,text.mlp.c_fc,2,32,8,128,6.147647857666016,0.3134765625,1048576,3.1205782672974403
+4,text.mlp.c_fc,2,64,8,128,7.176209449768066,0.28173828125,1048576,3.1205782672974403
+4,text.mlp.c_fc,2,128,8,128,8.08156967163086,0.265869140625,1048576,3.1205782672974403
+4,text.mlp.c_fc,3,32,8,128,2.6306636333465576,0.4384765625,1048576,3.1205782672974403
+4,text.mlp.c_fc,3,64,8,128,3.0671095848083496,0.40673828125,1048576,3.1205782672974403
+4,text.mlp.c_fc,3,128,8,128,3.458563804626465,0.390869140625,1048576,3.1205782672974403
+4,text.mlp.c_fc,4,32,8,128,1.2271418571472168,0.5634765625,1048576,3.1205782672974403
+4,text.mlp.c_fc,4,64,8,128,1.4321166276931765,0.53173828125,1048576,3.1205782672974403
+4,text.mlp.c_fc,4,128,8,128,1.6170594692230225,0.515869140625,1048576,3.1205782672974403
+4,text.mlp.c_fc,8,32,8,128,0.07444850355386734,1.0634765625,1048576,3.1205782672974403
+4,text.mlp.c_fc,8,64,8,128,0.08672907203435898,1.03173828125,1048576,3.1205782672974403
+4,text.mlp.c_fc,8,128,8,128,0.09753215312957764,1.015869140625,1048576,3.1205782672974403
+4,text.mlp.c_proj,2,32,8,128,4.5684428215026855,0.3134765625,1048576,3.3007157471391952
+4,text.mlp.c_proj,2,64,8,128,5.32727575302124,0.28173828125,1048576,3.3007157471391952
+4,text.mlp.c_proj,2,128,8,128,6.010774612426758,0.265869140625,1048576,3.3007157471391952
+4,text.mlp.c_proj,3,32,8,128,1.953374981880188,0.4384765625,1048576,3.3007157471391952
+4,text.mlp.c_proj,3,64,8,128,2.280749797821045,0.40673828125,1048576,3.3007157471391952
+4,text.mlp.c_proj,3,128,8,128,2.575989007949829,0.390869140625,1048576,3.3007157471391952
+4,text.mlp.c_proj,4,32,8,128,0.9109432697296144,0.5634765625,1048576,3.3007157471391952
+4,text.mlp.c_proj,4,64,8,128,1.0641778707504272,0.53173828125,1048576,3.3007157471391952
+4,text.mlp.c_proj,4,128,8,128,1.2020576000213623,0.515869140625,1048576,3.3007157471391952
+4,text.mlp.c_proj,8,32,8,128,0.05520271509885788,1.0634765625,1048576,3.3007157471391952
+4,text.mlp.c_proj,8,64,8,128,0.06432124227285385,1.03173828125,1048576,3.3007157471391952
+4,text.mlp.c_proj,8,128,8,128,0.07244100421667099,1.015869140625,1048576,3.3007157471391952
+5,text.mlp.c_fc,2,32,8,128,6.13761568069458,0.3134765625,1048576,3.12209161166918
+5,text.mlp.c_fc,2,64,8,128,7.154106140136719,0.28173828125,1048576,3.12209161166918
+5,text.mlp.c_fc,2,128,8,128,8.066065788269043,0.265869140625,1048576,3.12209161166918
+5,text.mlp.c_fc,3,32,8,128,2.622965097427368,0.4384765625,1048576,3.12209161166918
+5,text.mlp.c_fc,3,64,8,128,3.059314012527466,0.40673828125,1048576,3.12209161166918
+5,text.mlp.c_fc,3,128,8,128,3.4494223594665527,0.390869140625,1048576,3.12209161166918
+5,text.mlp.c_fc,4,32,8,128,1.2235389947891235,0.5634765625,1048576,3.12209161166918
+5,text.mlp.c_fc,4,64,8,128,1.4277613162994385,0.53173828125,1048576,3.12209161166918
+5,text.mlp.c_fc,4,128,8,128,1.6116138696670532,0.515869140625,1048576,3.12209161166918
+5,text.mlp.c_fc,8,32,8,128,0.0742468386888504,1.0634765625,1048576,3.12209161166918
+5,text.mlp.c_fc,8,64,8,128,0.08646093308925629,1.03173828125,1048576,3.12209161166918
+5,text.mlp.c_fc,8,128,8,128,0.09725197404623032,1.015869140625,1048576,3.12209161166918
+5,text.mlp.c_proj,2,32,8,128,4.701293468475342,0.3134765625,1048576,3.399940581104202
+5,text.mlp.c_proj,2,64,8,128,5.488790988922119,0.28173828125,1048576,3.399940581104202
+5,text.mlp.c_proj,2,128,8,128,6.199483871459961,0.265869140625,1048576,3.399940581104202
+5,text.mlp.c_proj,3,32,8,128,2.0124988555908203,0.4384765625,1048576,3.399940581104202
+5,text.mlp.c_proj,3,64,8,128,2.3516228199005127,0.40673828125,1048576,3.399940581104202
+5,text.mlp.c_proj,3,128,8,128,2.659825086593628,0.390869140625,1048576,3.399940581104202
+5,text.mlp.c_proj,4,32,8,128,0.9377811551094056,0.5634765625,1048576,3.399940581104202
+5,text.mlp.c_proj,4,64,8,128,1.0970691442489624,0.53173828125,1048576,3.399940581104202
+5,text.mlp.c_proj,4,128,8,128,1.2429126501083374,0.515869140625,1048576,3.399940581104202
+5,text.mlp.c_proj,8,32,8,128,0.05688211694359779,1.0634765625,1048576,3.399940581104202
+5,text.mlp.c_proj,8,64,8,128,0.06637991219758987,1.03173828125,1048576,3.399940581104202
+5,text.mlp.c_proj,8,128,8,128,0.0749565064907074,1.015869140625,1048576,3.399940581104202
+6,text.mlp.c_fc,2,32,8,128,6.08525276184082,0.3134765625,1048576,3.104312609554148
+6,text.mlp.c_fc,2,64,8,128,7.091857433319092,0.28173828125,1048576,3.104312609554148
+6,text.mlp.c_fc,2,128,8,128,8.011824607849121,0.265869140625,1048576,3.104312609554148
+6,text.mlp.c_fc,3,32,8,128,2.600118398666382,0.4384765625,1048576,3.104312609554148
+6,text.mlp.c_fc,3,64,8,128,3.0300920009613037,0.40673828125,1048576,3.104312609554148
+6,text.mlp.c_fc,3,128,8,128,3.4273650646209717,0.390869140625,1048576,3.104312609554148
+6,text.mlp.c_fc,4,32,8,128,1.2133545875549316,0.5634765625,1048576,3.104312609554148
+6,text.mlp.c_fc,4,64,8,128,1.4163140058517456,0.53173828125,1048576,3.104312609554148
+6,text.mlp.c_fc,4,128,8,128,1.5989820957183838,0.515869140625,1048576,3.104312609554148
+6,text.mlp.c_fc,8,32,8,128,0.07371968030929565,1.0634765625,1048576,3.104312609554148
+6,text.mlp.c_fc,8,64,8,128,0.08582986146211624,1.03173828125,1048576,3.104312609554148
+6,text.mlp.c_fc,8,128,8,128,0.09657172113656998,1.015869140625,1048576,3.104312609554148
+6,text.mlp.c_proj,2,32,8,128,4.809174060821533,0.3134765625,1048576,3.418852076553872
+6,text.mlp.c_proj,2,64,8,128,5.619462966918945,0.28173828125,1048576,3.418852076553872
+6,text.mlp.c_proj,2,128,8,128,6.347733974456787,0.265869140625,1048576,3.418852076553872
+6,text.mlp.c_proj,3,32,8,128,2.0561914443969727,0.4384765625,1048576,3.418852076553872
+6,text.mlp.c_proj,3,64,8,128,2.4042656421661377,0.40673828125,1048576,3.418852076553872
+6,text.mlp.c_proj,3,128,8,128,2.7215018272399902,0.390869140625,1048576,3.418852076553872
+6,text.mlp.c_proj,4,32,8,128,0.9595969915390016,0.5634765625,1048576,3.418852076553872
+6,text.mlp.c_proj,4,64,8,128,1.1221243143081665,0.53173828125,1048576,3.418852076553872
+6,text.mlp.c_proj,4,128,8,128,1.2688665390014648,0.515869140625,1048576,3.418852076553872
+6,text.mlp.c_proj,8,32,8,128,0.05815136805176735,1.0634765625,1048576,3.418852076553872
+6,text.mlp.c_proj,8,64,8,128,0.0679142028093338,1.03173828125,1048576,3.418852076553872
+6,text.mlp.c_proj,8,128,8,128,0.07667670398950577,1.015869140625,1048576,3.418852076553872
+7,text.mlp.c_fc,2,32,8,128,6.016368865966797,0.3134765625,1048576,3.1152966389066306
+7,text.mlp.c_fc,2,64,8,128,7.0210137367248535,0.28173828125,1048576,3.1152966389066306
+7,text.mlp.c_fc,2,128,8,128,7.924512386322021,0.265869140625,1048576,3.1152966389066306
+7,text.mlp.c_fc,3,32,8,128,2.573953628540039,0.4384765625,1048576,3.1152966389066306
+7,text.mlp.c_fc,3,64,8,128,3.004284620285034,0.40673828125,1048576,3.1152966389066306
+7,text.mlp.c_fc,3,128,8,128,3.3958704471588135,0.390869140625,1048576,3.1152966389066306
+7,text.mlp.c_fc,4,32,8,128,1.2008792161941528,0.5634765625,1048576,3.1152966389066306
+7,text.mlp.c_fc,4,64,8,128,1.401782751083374,0.53173828125,1048576,3.1152966389066306
+7,text.mlp.c_fc,4,128,8,128,1.5841267108917236,0.515869140625,1048576,3.1152966389066306
+7,text.mlp.c_fc,8,32,8,128,0.07277538627386093,1.0634765625,1048576,3.1152966389066306
+7,text.mlp.c_fc,8,64,8,128,0.08490202575922012,1.03173828125,1048576,3.1152966389066306
+7,text.mlp.c_fc,8,128,8,128,0.09555526822805405,1.015869140625,1048576,3.1152966389066306
+7,text.mlp.c_proj,2,32,8,128,4.9375319480896,0.3134765625,1048576,3.563658370504532
+7,text.mlp.c_proj,2,64,8,128,5.766933441162109,0.28173828125,1048576,3.563658370504532
+7,text.mlp.c_proj,2,128,8,128,6.5257158279418945,0.265869140625,1048576,3.563658370504532
+7,text.mlp.c_proj,3,32,8,128,2.1138291358947754,0.4384765625,1048576,3.563658370504532
+7,text.mlp.c_proj,3,64,8,128,2.4740889072418213,0.40673828125,1048576,3.563658370504532
+7,text.mlp.c_proj,3,128,8,128,2.8029916286468506,0.390869140625,1048576,3.563658370504532
+7,text.mlp.c_proj,4,32,8,128,0.985521137714386,0.5634765625,1048576,3.563658370504532
+7,text.mlp.c_proj,4,64,8,128,1.152884602546692,0.53173828125,1048576,3.563658370504532
+7,text.mlp.c_proj,4,128,8,128,1.3091634511947632,0.515869140625,1048576,3.563658370504532
+7,text.mlp.c_proj,8,32,8,128,0.05971698090434074,1.0634765625,1048576,3.563658370504532
+7,text.mlp.c_proj,8,64,8,128,0.06980670988559723,1.03173828125,1048576,3.563658370504532
+7,text.mlp.c_proj,8,128,8,128,0.07889241725206375,1.015869140625,1048576,3.563658370504532
+8,text.mlp.c_fc,2,32,8,128,5.901716709136963,0.3134765625,1048576,3.1079712489156552
+8,text.mlp.c_fc,2,64,8,128,6.884827136993408,0.28173828125,1048576,3.1079712489156552
+8,text.mlp.c_fc,2,128,8,128,7.761914730072021,0.265869140625,1048576,3.1079712489156552
+8,text.mlp.c_fc,3,32,8,128,2.520334243774414,0.4384765625,1048576,3.1079712489156552
+8,text.mlp.c_fc,3,64,8,128,2.9441771507263184,0.40673828125,1048576,3.1079712489156552
+8,text.mlp.c_fc,3,128,8,128,3.32405948638916,0.390869140625,1048576,3.1079712489156552
+8,text.mlp.c_fc,4,32,8,128,1.1775058507919312,0.5634765625,1048576,3.1079712489156552
+8,text.mlp.c_fc,4,64,8,128,1.37420654296875,0.53173828125,1048576,3.1079712489156552
+8,text.mlp.c_fc,4,128,8,128,1.5516901016235352,0.515869140625,1048576,3.1079712489156552
+8,text.mlp.c_fc,8,32,8,128,0.07130877673625946,1.0634765625,1048576,3.1079712489156552
+8,text.mlp.c_fc,8,64,8,128,0.0831245630979538,1.03173828125,1048576,3.1079712489156552
+8,text.mlp.c_fc,8,128,8,128,0.09355318546295166,1.015869140625,1048576,3.1079712489156552
+8,text.mlp.c_proj,2,32,8,128,5.102689266204834,0.3134765625,1048576,3.6947691166226
+8,text.mlp.c_proj,2,64,8,128,5.964451789855957,0.28173828125,1048576,3.6947691166226
+8,text.mlp.c_proj,2,128,8,128,6.735776901245117,0.265869140625,1048576,3.6947691166226
+8,text.mlp.c_proj,3,32,8,128,2.18440318107605,0.4384765625,1048576,3.6947691166226
+8,text.mlp.c_proj,3,64,8,128,2.556516647338867,0.40673828125,1048576,3.6947691166226
+8,text.mlp.c_proj,3,128,8,128,2.8903772830963135,0.390869140625,1048576,3.6947691166226
+8,text.mlp.c_proj,4,32,8,128,1.018601417541504,0.5634765625,1048576,3.6947691166226
+8,text.mlp.c_proj,4,64,8,128,1.192069172859192,0.53173828125,1048576,3.6947691166226
+8,text.mlp.c_proj,4,128,8,128,1.349886178970337,0.515869140625,1048576,3.6947691166226
+8,text.mlp.c_proj,8,32,8,128,0.0618121437728405,1.0634765625,1048576,3.6947691166226
+8,text.mlp.c_proj,8,64,8,128,0.07216800004243851,1.03173828125,1048576,3.6947691166226
+8,text.mlp.c_proj,8,128,8,128,0.08146006613969803,1.015869140625,1048576,3.6947691166226
+9,text.mlp.c_fc,2,32,8,128,5.788586139678955,0.3134765625,1048576,3.1305828189206824
+9,text.mlp.c_fc,2,64,8,128,6.742534637451172,0.28173828125,1048576,3.1305828189206824
+9,text.mlp.c_fc,2,128,8,128,7.599257946014404,0.265869140625,1048576,3.1305828189206824
+9,text.mlp.c_fc,3,32,8,128,2.47590970993042,0.4384765625,1048576,3.1305828189206824
+9,text.mlp.c_fc,3,64,8,128,2.887095212936402,0.40673828125,1048576,3.1305828189206824
+9,text.mlp.c_fc,3,128,8,128,3.255934953689575,0.390869140625,1048576,3.1305828189206824
+9,text.mlp.c_fc,4,32,8,128,1.1548055410385132,0.5634765625,1048576,3.1305828189206824
+9,text.mlp.c_fc,4,64,8,128,1.346752047538757,0.53173828125,1048576,3.1305828189206824
+9,text.mlp.c_fc,4,128,8,128,1.517864465713501,0.515869140625,1048576,3.1305828189206824
+9,text.mlp.c_fc,8,32,8,128,0.06999316066503525,1.0634765625,1048576,3.1305828189206824
+9,text.mlp.c_fc,8,64,8,128,0.0815470740199089,1.03173828125,1048576,3.1305828189206824
+9,text.mlp.c_fc,8,128,8,128,0.09159331023693083,1.015869140625,1048576,3.1305828189206824
+9,text.mlp.c_proj,2,32,8,128,5.207601547241211,0.3134765625,1048576,3.495399170438296
+9,text.mlp.c_proj,2,64,8,128,6.071842193603516,0.28173828125,1048576,3.495399170438296
+9,text.mlp.c_proj,2,128,8,128,6.8601274490356445,0.265869140625,1048576,3.495399170438296
+9,text.mlp.c_proj,3,32,8,128,2.2280025482177734,0.4384765625,1048576,3.495399170438296
+9,text.mlp.c_proj,3,64,8,128,2.5987324714660645,0.40673828125,1048576,3.495399170438296
+9,text.mlp.c_proj,3,128,8,128,2.939124822616577,0.390869140625,1048576,3.495399170438296
+9,text.mlp.c_proj,4,32,8,128,1.0387369394302368,0.5634765625,1048576,3.495399170438296
+9,text.mlp.c_proj,4,64,8,128,1.2127223014831543,0.53173828125,1048576,3.495399170438296
+9,text.mlp.c_proj,4,128,8,128,1.3731480836868286,0.515869140625,1048576,3.495399170438296
+9,text.mlp.c_proj,8,32,8,128,0.06300804764032364,1.0634765625,1048576,3.495399170438296
+9,text.mlp.c_proj,8,64,8,128,0.07346633821725845,1.03173828125,1048576,3.495399170438296
+9,text.mlp.c_proj,8,128,8,128,0.08280103653669357,1.015869140625,1048576,3.495399170438296
+10,text.mlp.c_fc,2,32,8,128,5.7948408126831055,0.3134765625,1048576,3.144641316115429
+10,text.mlp.c_fc,2,64,8,128,6.760894298553467,0.28173828125,1048576,3.144641316115429
+10,text.mlp.c_fc,2,128,8,128,7.649384498596191,0.265869140625,1048576,3.144641316115429
+10,text.mlp.c_fc,3,32,8,128,2.475472211837769,0.4384765625,1048576,3.144641316115429
+10,text.mlp.c_fc,3,64,8,128,2.8930392265319824,0.40673828125,1048576,3.144641316115429
+10,text.mlp.c_fc,3,128,8,128,3.2758681774139404,0.390869140625,1048576,3.144641316115429
+10,text.mlp.c_fc,4,32,8,128,1.1556246280670166,0.5634765625,1048576,3.144641316115429
+10,text.mlp.c_fc,4,64,8,128,1.3502486944198608,0.53173828125,1048576,3.144641316115429
+10,text.mlp.c_fc,4,128,8,128,1.5289539098739624,0.515869140625,1048576,3.144641316115429
+10,text.mlp.c_fc,8,32,8,128,0.0699719563126564,1.0634765625,1048576,3.144641316115429
+10,text.mlp.c_fc,8,64,8,128,0.08176594972610474,1.03173828125,1048576,3.144641316115429
+10,text.mlp.c_fc,8,128,8,128,0.09224754571914671,1.015869140625,1048576,3.144641316115429
+10,text.mlp.c_proj,2,32,8,128,5.407021522521973,0.3134765625,1048576,3.655921840018199
+10,text.mlp.c_proj,2,64,8,128,6.310086727142334,0.28173828125,1048576,3.655921840018199
+10,text.mlp.c_proj,2,128,8,128,7.1317644119262695,0.265869140625,1048576,3.655921840018199
+10,text.mlp.c_proj,3,32,8,128,2.315161943435669,0.4384765625,1048576,3.655921840018199
+10,text.mlp.c_proj,3,64,8,128,2.706345558166504,0.40673828125,1048576,3.655921840018199
+10,text.mlp.c_proj,3,128,8,128,3.065629720687866,0.390869140625,1048576,3.655921840018199
+10,text.mlp.c_proj,4,32,8,128,1.0800068378448486,0.5634765625,1048576,3.655921840018199
+10,text.mlp.c_proj,4,64,8,128,1.2637077569961548,0.53173828125,1048576,3.655921840018199
+10,text.mlp.c_proj,4,128,8,128,1.432173252105713,0.515869140625,1048576,3.655921840018199
+10,text.mlp.c_proj,8,32,8,128,0.06542721390724182,1.0634765625,1048576,3.655921840018199
+10,text.mlp.c_proj,8,64,8,128,0.0764131173491478,1.03173828125,1048576,3.655921840018199
+10,text.mlp.c_proj,8,128,8,128,0.08641009777784348,1.015869140625,1048576,3.655921840018199
+11,text.mlp.c_fc,2,32,8,128,6.029895782470703,0.3134765625,1048576,3.227518868269559
+11,text.mlp.c_fc,2,64,8,128,7.065587997436523,0.28173828125,1048576,3.227518868269559
+11,text.mlp.c_fc,2,128,8,128,8.00993537902832,0.265869140625,1048576,3.227518868269559
+11,text.mlp.c_fc,3,32,8,128,2.577655553817749,0.4384765625,1048576,3.227518868269559
+11,text.mlp.c_fc,3,64,8,128,3.0226359367370605,0.40673828125,1048576,3.227518868269559
+11,text.mlp.c_fc,3,128,8,128,3.4333667755126953,0.390869140625,1048576,3.227518868269559
+11,text.mlp.c_fc,4,32,8,128,1.201055645942688,0.5634765625,1048576,3.227518868269559
+11,text.mlp.c_fc,4,64,8,128,1.4092037677764893,0.53173828125,1048576,3.227518868269559
+11,text.mlp.c_fc,4,128,8,128,1.6002758741378784,0.515869140625,1048576,3.227518868269559
+11,text.mlp.c_fc,8,32,8,128,0.07296200841665268,1.0634765625,1048576,3.227518868269559
+11,text.mlp.c_fc,8,64,8,128,0.08532291650772095,1.03173828125,1048576,3.227518868269559
+11,text.mlp.c_fc,8,128,8,128,0.09650221467018127,1.015869140625,1048576,3.227518868269559
+11,text.mlp.c_proj,2,32,8,128,5.306176662445068,0.3134765625,1048576,3.351559119010831
+11,text.mlp.c_proj,2,64,8,128,6.189615726470947,0.28173828125,1048576,3.351559119010831
+11,text.mlp.c_proj,2,128,8,128,6.987279891967773,0.265869140625,1048576,3.351559119010831
+11,text.mlp.c_proj,3,32,8,128,2.267962694168091,0.4384765625,1048576,3.351559119010831
+11,text.mlp.c_proj,3,64,8,128,2.647044897079468,0.40673828125,1048576,3.351559119010831
+11,text.mlp.c_proj,3,128,8,128,2.989635467529297,0.390869140625,1048576,3.351559119010831
+11,text.mlp.c_proj,4,32,8,128,1.0588033199310305,0.5634765625,1048576,3.351559119010831
+11,text.mlp.c_proj,4,64,8,128,1.2367676496505735,0.53173828125,1048576,3.351559119010831
+11,text.mlp.c_proj,4,128,8,128,1.3962570428848269,0.515869140625,1048576,3.351559119010831
+11,text.mlp.c_proj,8,32,8,128,0.06412201374769211,1.0634765625,1048576,3.351559119010831
+11,text.mlp.c_proj,8,64,8,128,0.07475418597459793,1.03173828125,1048576,3.351559119010831
+11,text.mlp.c_proj,8,128,8,128,0.0842188373208046,1.015869140625,1048576,3.351559119010831

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-H-14-laion2B-s32B-b79K.csv

@@ -0,0 +1,113 @@
+module,layer,kurtosis
+vision.mlp.c_fc,0,40.90237464363011
+vision.mlp.c_fc,1,8.623074922686055
+vision.mlp.c_fc,2,9.97244388122774
+vision.mlp.c_fc,3,8.84226648033505
+vision.mlp.c_fc,4,6.679282327315599
+vision.mlp.c_fc,5,5.079880661110845
+vision.mlp.c_fc,6,4.880932310379808
+vision.mlp.c_fc,7,4.627381829171479
+vision.mlp.c_fc,8,3.9064125305929482
+vision.mlp.c_fc,9,3.7876465036865037
+vision.mlp.c_fc,10,3.7124408015905463
+vision.mlp.c_fc,11,3.5936330266701173
+vision.mlp.c_fc,12,3.532952137305315
+vision.mlp.c_fc,13,3.3851543523342795
+vision.mlp.c_fc,14,3.2873419131054304
+vision.mlp.c_fc,15,3.21782809316773
+vision.mlp.c_fc,16,3.193747931233614
+vision.mlp.c_fc,17,3.1630314377596305
+vision.mlp.c_fc,18,3.111462971801169
+vision.mlp.c_fc,19,3.105595856069085
+vision.mlp.c_fc,20,3.125886942463749
+vision.mlp.c_fc,21,3.112450361245664
+vision.mlp.c_fc,22,3.079523772985348
+vision.mlp.c_fc,23,3.0821569476632615
+vision.mlp.c_fc,24,3.060682454627166
+vision.mlp.c_fc,25,3.0582903694192485
+vision.mlp.c_fc,26,3.066707948588762
+vision.mlp.c_fc,27,3.098921361428591
+vision.mlp.c_fc,28,3.115322580545271
+vision.mlp.c_fc,29,3.1341133294881063
+vision.mlp.c_fc,30,3.0777676137143355
+vision.mlp.c_fc,31,3.1157306945381347
+vision.mlp.c_proj,0,42.596327918914184
+vision.mlp.c_proj,1,5.700569954788923
+vision.mlp.c_proj,2,13.632761342438107
+vision.mlp.c_proj,3,16.268235060436744
+vision.mlp.c_proj,4,6.3570446531375
+vision.mlp.c_proj,5,4.809992151850216
+vision.mlp.c_proj,6,5.053291310517847
+vision.mlp.c_proj,7,7.712895831700571
+vision.mlp.c_proj,8,3.6717534300900505
+vision.mlp.c_proj,9,3.8858509415381075
+vision.mlp.c_proj,10,4.615226983169325
+vision.mlp.c_proj,11,3.9462090594902732
+vision.mlp.c_proj,12,4.290422057573261
+vision.mlp.c_proj,13,4.200667942302044
+vision.mlp.c_proj,14,3.5894479766105856
+vision.mlp.c_proj,15,3.295239725306919
+vision.mlp.c_proj,16,3.258246320341865
+vision.mlp.c_proj,17,3.287998430108839
+vision.mlp.c_proj,18,3.106840184971173
+vision.mlp.c_proj,19,3.1621437963814087
+vision.mlp.c_proj,20,3.168454537760785
+vision.mlp.c_proj,21,3.213436382602411
+vision.mlp.c_proj,22,3.173853870505496
+vision.mlp.c_proj,23,3.178954522388751
+vision.mlp.c_proj,24,3.110805792772743
+vision.mlp.c_proj,25,3.114617319926587
+vision.mlp.c_proj,26,3.1252555548204226
+vision.mlp.c_proj,27,3.2274298263451326
+vision.mlp.c_proj,28,3.4402281120641667
+vision.mlp.c_proj,29,3.4039192043631497
+vision.mlp.c_proj,30,3.235419305882699
+vision.mlp.c_proj,31,3.2103127283743196
+text.mlp.c_fc,0,3.130876328110002
+text.mlp.c_fc,1,3.522946431618204
+text.mlp.c_fc,2,3.0665727137671914
+text.mlp.c_fc,3,3.07939282139385
+text.mlp.c_fc,4,3.073907675889281
+text.mlp.c_fc,5,3.0760243182327653
+text.mlp.c_fc,6,3.074341006878764
+text.mlp.c_fc,7,3.070467308878268
+text.mlp.c_fc,8,3.071675866527997
+text.mlp.c_fc,9,3.0676553119556074
+text.mlp.c_fc,10,3.0588525342169723
+text.mlp.c_fc,11,3.050926459548459
+text.mlp.c_fc,12,3.0518055370811794
+text.mlp.c_fc,13,3.0420668373340813
+text.mlp.c_fc,14,3.0318151874070547
+text.mlp.c_fc,15,3.039579656917142
+text.mlp.c_fc,16,3.030832331612468
+text.mlp.c_fc,17,3.0464599870244147
+text.mlp.c_fc,18,3.0314247283166527
+text.mlp.c_fc,19,3.0366579505365423
+text.mlp.c_fc,20,3.035431457627752
+text.mlp.c_fc,21,3.035719259091557
+text.mlp.c_fc,22,3.0609612140958817
+text.mlp.c_fc,23,3.0913598384430627
+text.mlp.c_proj,0,7.6155345275418025
+text.mlp.c_proj,1,10.138321188679202
+text.mlp.c_proj,2,3.0448966809401976
+text.mlp.c_proj,3,3.111820890605678
+text.mlp.c_proj,4,3.0514440139756176
+text.mlp.c_proj,5,3.7893358959560755
+text.mlp.c_proj,6,3.0707829989536948
+text.mlp.c_proj,7,3.065090920136809
+text.mlp.c_proj,8,3.0726887739189013
+text.mlp.c_proj,9,3.3939847588778926
+text.mlp.c_proj,10,3.1983749033206665
+text.mlp.c_proj,11,3.1750157970807344
+text.mlp.c_proj,12,3.0569032145916935
+text.mlp.c_proj,13,3.352986917766462
+text.mlp.c_proj,14,3.040870872900523
+text.mlp.c_proj,15,3.084122174777231
+text.mlp.c_proj,16,3.083968401193075
+text.mlp.c_proj,17,3.057870708712376
+text.mlp.c_proj,18,3.154006715549809
+text.mlp.c_proj,19,3.099173814869719
+text.mlp.c_proj,20,3.1643892434809398
+text.mlp.c_proj,21,3.1143910580456344
+text.mlp.c_proj,22,3.2050609846862375
+text.mlp.c_proj,23,3.1967177502646615

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv

@@ -0,0 +1,73 @@
+module,layer,kurtosis
+vision.mlp.c_fc,0,82.60432030937712
+vision.mlp.c_fc,1,12.176342485174775
+vision.mlp.c_fc,2,9.386186464371631
+vision.mlp.c_fc,3,6.0384986521099115
+vision.mlp.c_fc,4,5.121874436272312
+vision.mlp.c_fc,5,4.184969377709542
+vision.mlp.c_fc,6,4.256117717855523
+vision.mlp.c_fc,7,3.953278547908379
+vision.mlp.c_fc,8,3.923302040046366
+vision.mlp.c_fc,9,3.640527020435186
+vision.mlp.c_fc,10,3.8766116638521972
+vision.mlp.c_fc,11,3.364467358103123
+vision.mlp.c_fc,12,3.2501048983174172
+vision.mlp.c_fc,13,3.144904609090658
+vision.mlp.c_fc,14,3.1176022019209833
+vision.mlp.c_fc,15,3.170790033142059
+vision.mlp.c_fc,16,3.1901952176386157
+vision.mlp.c_fc,17,3.1728736833856566
+vision.mlp.c_fc,18,3.1092420753715064
+vision.mlp.c_fc,19,3.1289105323616995
+vision.mlp.c_fc,20,3.1633778054509616
+vision.mlp.c_fc,21,3.1948778928908625
+vision.mlp.c_fc,22,3.153364851937131
+vision.mlp.c_fc,23,3.1481862277653847
+vision.mlp.c_proj,0,48.91602226406029
+vision.mlp.c_proj,1,7.994586159177927
+vision.mlp.c_proj,2,9.259903305960076
+vision.mlp.c_proj,3,12.161504492865403
+vision.mlp.c_proj,4,11.137208824577082
+vision.mlp.c_proj,5,4.50118105140538
+vision.mlp.c_proj,6,3.9747365034192206
+vision.mlp.c_proj,7,4.846873977315018
+vision.mlp.c_proj,8,4.914742513787588
+vision.mlp.c_proj,9,5.0506453475209065
+vision.mlp.c_proj,10,13.838509154283804
+vision.mlp.c_proj,11,4.258906842824289
+vision.mlp.c_proj,12,3.4482411935554316
+vision.mlp.c_proj,13,3.4563538182296156
+vision.mlp.c_proj,14,3.1506796697623156
+vision.mlp.c_proj,15,3.140821870180245
+vision.mlp.c_proj,16,3.1870137578290736
+vision.mlp.c_proj,17,3.1622263243928037
+vision.mlp.c_proj,18,3.1727003318990294
+vision.mlp.c_proj,19,3.126420350320576
+vision.mlp.c_proj,20,3.214422564736524
+vision.mlp.c_proj,21,3.5410765909033515
+vision.mlp.c_proj,22,3.885907825953595
+vision.mlp.c_proj,23,3.241221353196137
+text.mlp.c_fc,0,3.111321629333239
+text.mlp.c_fc,1,3.1040094243226832
+text.mlp.c_fc,2,3.0476798189406455
+text.mlp.c_fc,3,3.0567396405641647
+text.mlp.c_fc,4,3.072398311749821
+text.mlp.c_fc,5,3.0581450022889753
+text.mlp.c_fc,6,3.053470998692992
+text.mlp.c_fc,7,3.0531601497538126
+text.mlp.c_fc,8,3.0696775698776992
+text.mlp.c_fc,9,3.065453985089178
+text.mlp.c_fc,10,3.067657384247557
+text.mlp.c_fc,11,3.086129858871849
+text.mlp.c_proj,0,16.411416535361464
+text.mlp.c_proj,1,10.636934397271798
+text.mlp.c_proj,2,5.467879831695406
+text.mlp.c_proj,3,3.117101945617834
+text.mlp.c_proj,4,3.162125318135728
+text.mlp.c_proj,5,3.114353715482094
+text.mlp.c_proj,6,3.1560610360682673
+text.mlp.c_proj,7,3.230989551532942
+text.mlp.c_proj,8,3.273957420333387
+text.mlp.c_proj,9,3.322224898837691
+text.mlp.c_proj,10,3.252371370612472
+text.mlp.c_proj,11,3.2207147468979427

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py

@@ -0,0 +1,66 @@
+import os
+import time
+
+import torch
+import transformers
+from awq import AutoAWQForCausalLM
+from transformers import AutoConfig
+
+from lm_quant_toolkit.adapter.common import get_model_storage_size
+
+
+def create_autoawq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    model_file_size = 0
+    quantized = False
+    quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+
+    config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+    # To avoid OOM after huggingface transformers 4.36.2
+    config.use_cache = False
+    if load_quantized and os.path.exists(quant_path):
+        model = AutoAWQForCausalLM.from_quantized(
+            quant_path,
+            device_map="auto",
+            offload_state_dict=False,
+            config=config,
+        )
+        tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+        quantized = True
+        model_file_size = get_model_storage_size(quant_path)
+        model = model.cuda()
+    else:
+        tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+        #     max_memory={0: "18GiB", "cpu": "60GiB"},
+        # )
+        model = AutoAWQForCausalLM.from_pretrained(
+            model_id,
+            device_map="auto",
+            offload_state_dict=False,
+            torch_dtype=torch.float16,
+            max_memory={0: "18GiB", "cpu": "60GiB"},
+            config=config,
+        )
+    return model, tokenizer, quantized, model_file_size
+
+
+def quantize_autoawq_model(
+    model, tokenizer, quant_config, model_id, config_id, save_dir
+):
+    t1 = time.time()
+    model.quantize(tokenizer, quant_config=quant_config)
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with AutoAWQ")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+    model.save_quantized(quant_path)
+    tokenizer.save_pretrained(quant_path)
+    # persistent the quantized model
+    os.sync()
+    return model, t2 - t1, _get_model_file_size(quant_path)
+
+
+def _get_model_file_size(quant_path):
+    quant_fp_pt = os.path.join(quant_path, "qmodel.pth")
+    if os.path.exists(quant_fp_pt):
+        return os.path.getsize(quant_fp_pt)
+    else:
+        return get_model_storage_size(quant_path)

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py

@@ -0,0 +1,85 @@
+import gc
+import os
+import random
+import time
+
+import torch
+import transformers
+from auto_gptq import AutoGPTQForCausalLM
+from datasets import load_dataset
+from tqdm import tqdm
+
+
+# Adapted from: https://towardsdatascience.com/4-bit-quantization-with-gptq-36b0f4f02c34
+def prepare_model(model, tokenizer, n_samples=1024, max_tokens=512, use_triton=False):
+    # Load data and tokenize examples
+    data = load_dataset(
+        "allenai/c4",
+        data_files="en/c4-train.00001-of-01024.json.gz",
+        split=f"train[:{n_samples}]",
+    )
+    # ~536K tokens
+    tokenized_data = torch.cat(
+        [
+            tokenizer(data[i]["text"], return_tensors="pt").input_ids
+            for i in tqdm(range(len(data)))
+        ],
+        axis=-1,
+    )
+
+    # Format tokenized examples
+    random.seed(1)
+    examples_ids = []
+    for _ in range(n_samples):
+        i = random.randint(0, tokenized_data.shape[1] - max_tokens - 1)
+        j = i + max_tokens
+        input_ids = tokenized_data[:, i:j]
+        attention_mask = torch.ones_like(input_ids)
+        examples_ids.append({"input_ids": input_ids, "attention_mask": attention_mask})
+
+    print("Using " + str(len(examples_ids)) + " samples for calibration.")
+    model.quantize(examples_ids, batch_size=1, use_triton=use_triton)
+    # model = model.cuda()
+    # with torch.no_grad():
+    #     x = model(input_ids.to('cuda'))
+    # del examples_ids, x
+    del examples_ids
+    torch.cuda.empty_cache()
+    gc.collect()
+    return model
+
+
+def create_autogptq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    model_file_size = 0
+    quantized = False
+    quant_path = f"{save_dir}/{model_id}-{config_id}-gptq"
+    if load_quantized and os.path.exists(quant_path):
+        model = AutoGPTQForCausalLM.from_quantized(quant_path, device="cuda:0")
+        tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+        quantized = True
+        model_file_size = _get_model_file_size(quant_path, quant_config)
+    else:
+        tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+        model = AutoGPTQForCausalLM.from_pretrained(model_id, quant_config)
+    return model, tokenizer, quantized, model_file_size
+
+
+def quantize_autogptq_model(
+    model, tokenizer, quant_config, model_id, config_id, save_dir
+):
+    t1 = time.time()
+    model = prepare_model(model, tokenizer)
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with AutoGPTQ")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-gptq"
+    model.save_quantized(quant_path, use_safetensors=True)
+    # persistent the quantized model
+    os.sync()
+    return model, t2 - t1, _get_model_file_size(quant_path, quant_config)
+
+
+def _get_model_file_size(quant_path, quant_config):
+    b = quant_config.bits
+    g = quant_config.group_size
+    fp = os.path.join(quant_path, f"gptq_model-{b}bit-{g}g.safetensors")
+    return os.path.getsize(fp)

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py

@@ -0,0 +1,91 @@
+import os
+import time
+
+import torch
+from accelerate import (
+    infer_auto_device_map,
+    init_empty_weights,
+    load_checkpoint_in_model,
+)
+from awq.quantize.pre_quant import apply_awq, run_awq
+from awq.quantize.quantizer import real_quantize_model_weight
+from awq.utils.utils import simple_dispatch_model
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+
+def create_awq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    quantized = False
+    quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_id, use_fast=False, trust_remote_code=True
+    )
+    config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+    # Note (Haotian): To avoid OOM after huggingface transformers 4.36.2
+    config.use_cache = False
+    if load_quantized and os.path.exists(f"{quant_path}/qmodel.pth"):
+        with init_empty_weights():
+            model = AutoModelForCausalLM.from_config(
+                config=config, torch_dtype=torch.float16, trust_remote_code=True
+            )
+        max_memory = {0: "20GiB", "cpu": "60GiB"}
+        # Infer device map
+        kwargs = {"max_memory": max_memory} if len(max_memory) else {}
+        device_map = infer_auto_device_map(
+            model,
+            no_split_module_classes=[
+                "OPTDecoderLayer",
+                "LlamaDecoderLayer",
+                "BloomBlock",
+                "MPTBlock",
+                "DecoderLayer",
+            ],
+            **kwargs,
+        )
+        # Load checkpoint in the model
+        load_checkpoint_in_model(
+            model,
+            checkpoint=quant_path,
+            device_map=device_map,
+            offload_state_dict=False,
+        )
+        # Dispatch model
+        model = simple_dispatch_model(model, device_map=device_map)
+        quantized = True
+        model.eval()
+    else:
+        kwargs = {"torch_dtype": torch.float16, "low_cpu_mem_usage": True}
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id, config=config, trust_remote_code=True, **kwargs
+        )
+    return model, tokenizer, quantized, 0
+
+
+def quantize_awq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+    t1 = time.time()
+    nbits = quant_config.pop("w_bit")
+    awq_results = run_awq(
+        model,
+        tokenizer,
+        w_bit=nbits,
+        q_config=quant_config,
+        n_samples=128,
+        seqlen=512,
+    )
+    intermediate_fp = f"{save_dir}/{model_id}-{config_id}-awq/intermediate.pth"
+    dirpath = os.path.dirname(intermediate_fp)
+    os.makedirs(dirpath, exist_ok=True)
+    torch.save(awq_results, intermediate_fp)
+    awq_results = torch.load(intermediate_fp, map_location="cpu")
+    apply_awq(model, awq_results)
+    real_quantize_model_weight(model, w_bit=nbits, q_config=quant_config)
+
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with AWQ")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+    quant_fp = os.path.join(quant_path, "qmodel.pth")
+    torch.save(model.cpu().state_dict(), quant_fp)
+    tokenizer.save_pretrained(quant_path)
+
+    model_file_size = os.path.getsize(quant_fp)
+    return model, t2 - t1, model_file_size

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py

@@ -0,0 +1,37 @@
+import os
+import time
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from lm_quant_toolkit.adapter.common import get_model_storage_size
+
+
+def create_bnb_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    quantized = False
+    model_file_size = 0
+    quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
+    if load_quantized and os.path.exists(quant_path):
+        model = AutoModelForCausalLM.from_pretrained(quant_path)
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        quantized = True
+        model_file_size = get_model_storage_size(quant_path)
+    else:
+        model = None
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+    return model, tokenizer, quantized, model_file_size
+
+
+def quantize_bnb_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+    model_file_size = 0
+    t1 = time.time()
+    model = AutoModelForCausalLM.from_pretrained(
+        model_id, quantization_config=quant_config
+    )
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with BnB")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
+    model.save_pretrained(quant_path)
+    # persistent the quantized model
+    os.sync()
+    model_file_size = get_model_storage_size(quant_path)
+    return model, t2 - t1, model_file_size

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py

@@ -0,0 +1,20 @@
+import json
+import os
+
+
+def get_model_storage_size(
+    base_dir,
+    index_file="model.safetensors.index.json",
+    model_file="model.safetensors",
+):
+    size = 0
+    index_file = os.path.join(base_dir, index_file)
+    if os.path.exists(index_file):
+        # model is split into shards
+        with open(index_file, "r") as f:
+            index = json.load(f)
+            for shard in set(index["weight_map"].values()):
+                size += os.path.getsize(os.path.join(base_dir, shard))
+    else:
+        size = os.path.getsize(os.path.join(base_dir, model_file))
+    return size

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py

@@ -0,0 +1,17 @@
+import torch
+import transformers
+from transformers import AutoModelForCausalLM
+
+from lm_quant_toolkit.adapter.common import get_model_storage_size
+from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir
+
+
+def create_fp16_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    model_file_size = 0
+    model = AutoModelForCausalLM.from_pretrained(
+        model_id, device_map="auto", torch_dtype=torch.float16
+    )
+    tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+    base_dir = get_hf_model_storge_base_dir(model_id)
+    model_file_size = get_model_storage_size(base_dir)
+    return model, tokenizer, False, model_file_size

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py

@@ -0,0 +1,34 @@
+import os
+import time
+
+from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer
+from hqq.engine.hf import HQQModelForCausalLM
+
+
+def create_hqq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    quantized = False
+    model_file_size = 0
+    quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+    if load_quantized and os.path.exists(quant_path):
+        model = HQQModelForCausalLM.from_quantized(quant_path)
+        tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+        quantized = True
+        model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+    else:
+        model = HQQModelForCausalLM.from_pretrained(model_id)
+        tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+    return model, tokenizer, quantized, model_file_size
+
+
+def quantize_hqq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+    model_file_size = 0
+    t1 = time.time()
+    model.quantize_model(quant_config=quant_config)
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with HQQ")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+    model.save_quantized(quant_path)
+    # persistent the quantized model
+    os.sync()
+    model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+    return model, t2 - t1, model_file_size

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py

@@ -0,0 +1,34 @@
+import os
+import time
+
+from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer
+from hqq.engine.hf import HQQModelForCausalLM
+
+
+def create_mxq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    quantized = False
+    model_file_size = 0
+    quant_path = f"{save_dir}/{model_id}-{config_id}-mxq"
+    if load_quantized and os.path.exists(quant_path):
+        model = HQQModelForCausalLM.from_quantized(quant_path)
+        tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+        quantized = True
+        model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+    else:
+        model = HQQModelForCausalLM.from_pretrained(model_id)
+        tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+    return model, tokenizer, quantized, model_file_size
+
+
+def quantize_mxq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+    model_file_size = 0
+    t1 = time.time()
+    model.quantize_model(quant_config=quant_config)
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with MXQ")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-mxq"
+    model.save_quantized(quant_path)
+    # persistent the quantized model
+    os.sync()
+    model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+    return model, t2 - t1, model_file_size

lm-quant-toolkit/new_lm-quant-toolkit/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py

@@ -0,0 +1,27 @@
+import os
+import time
+from pathlib import Path
+
+from hqq.engine.open_clip import HQQOpenCLIP
+
+
+def create_hqq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+    quantized = False
+    quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+    if load_quantized and os.path.exists(quant_path):
+        model = HQQOpenCLIP.from_quantized(quant_path)
+        quantized = True
+    else:
+        model = HQQOpenCLIP.create_model(model_id, device="cpu")
+    return model, quantized
+
+
+def quantize_hqq_model(model, quant_config, model_id, config_id, save_dir):
+    t1 = time.time()
+    model.quantize_model(quant_config=quant_config)
+    t2 = time.time()
+    print("Took " + str(t2 - t1) + " seconds to quantize the model with HQQ")
+    quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+    Path(quant_path).mkdir(parents=True, exist_ok=True)
+    model.save_quantized(save_dir=quant_path)
+    return model, t2 - t1