export OMINI_CONFIG=./config/commonsense.yaml

#echo $OMINI_CONFIG
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

# 3. Add into CPATH & CPLUS_INCLUDE_PATH (C/C++ compiler)
export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH
# echo "CPATH is set to: $CPATH"
# echo "CPLUS_INCLUDE_PATH is set to: $CPLUS_INCLUDE_PATH"

export WANDB_PROJECT="Llama2_7B_FT_Math40k"

export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1

date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps1" --trainer_args.learning_rate=1e-5

# sleep 5 
# echo "1st exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps2" --trainer_args.learning_rate=2e-5

# sleep 5 
# echo "2nd exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps3" --trainer_args.learning_rate=5e-5

# sleep 5 
# echo "3rd exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps4" --trainer_args.learning_rate=1e-4

# sleep 5 
# echo "4th exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps5" --trainer_args.learning_rate=2e-4

# sleep 5 
# echo "5th exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps6" --trainer_args.learning_rate=5e-4

# sleep 5 
# echo "6th exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps7" --trainer_args.learning_rate=8e-4

# sleep 5 
# echo "7th exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps8" --trainer_args.learning_rate=1e-3

# sleep 5 
# echo "8th exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exps9" --trainer_args.learning_rate=2e-3

# sleep 5 
# echo "9th exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exnr10" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 8 --rotation_adapter_config.r 2

# sleep 5 
# echo "10 exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exnr11" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 2 --rotation_adapter_config.r 8

# sleep 5 
# echo "11 exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exnr12" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 16 --rotation_adapter_config.r 1

# sleep 5 
# echo "12 exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_exnr13" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16

# sleep 5 
# echo "13 exp finishes"
# date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_all/exnr14" --trainer_args.learning_rate=2e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16

# sleep 5 
# echo "14 exp finishes"
# date +"%F %T"

accelerate launch --main_process_port 41353 -m src.ft_mathR \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./run_all/exnr15" --trainer_args.learning_rate=1e-3 \
      --rotation_adapter_config.num_rotations 4 --rotation_adapter_config.r 4

# sleep 5 
echo "15 exp finishes"
date +"%F %T"