export OMINI_CONFIG=./config/math395.yaml

#echo $OMINI_CONFIG
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

# 3. Add into CPATH & CPLUS_INCLUDE_PATH (C/C++ compiler)
export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH
# echo "CPATH is set to: $CPATH"
# echo "CPLUS_INCLUDE_PATH is set to: $CPLUS_INCLUDE_PATH"

export WANDB_PROJECT="Llama2_7B_FT_Math_395k"

export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1

date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex01" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'def'

# sleep 5 
# echo "1st exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex02" --trainer_args.learning_rate=5e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16

# sleep 5 
# echo "2nd exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex03" --trainer_args.learning_rate=2e-4 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16

# sleep 5 
# echo "3rd exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex04" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16

# sleep 5 
# echo "4rd exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex05" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01
# sleep 5 
# echo "5th exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex06" --trainer_args.learning_rate=2e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01
# sleep 5 
# echo "6th exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex0s7" --trainer_args.learning_rate=5e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01
# sleep 5 
# echo "7th exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex08" --trainer_args.learning_rate=1e-4 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 \
#       --trainer_args.per_device_train_batch_size 32 --run_text 'u2e2,def'
# sleep 5 
# echo "8th exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

accelerate launch --main_process_port 41353 -m src.ft_mathR \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex09" --trainer_args.learning_rate=2e-3 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 \
      --trainer_args.per_device_train_batch_size 32 --run_text 'init=def|fix_token'

sleep 5 
echo "9th exp finishes"
date +"%F %T"
wandb sync wandb/latest-run

accelerate launch --main_process_port 41353 -m src.ft_mathR \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex10" --trainer_args.learning_rate=2e-3 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 2.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 \
      --trainer_args.per_device_train_batch_size 32 --run_text "init=def|fix_token"

sleep 5 
echo "10 exp finishes"
date +"%F %T"
wandb sync wandb/latest-run


# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex11" --trainer_args.learning_rate=1e-2 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16

# sleep 5 
# echo "11 exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex12" --trainer_args.learning_rate=1e-2 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16  --run_text 'u=v,def'

# sleep 5 
# echo "12 exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex13" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'u=vkaim'

# sleep 5 
# echo "13 exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex14" --trainer_args.learning_rate=2e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'a,b,def'

# sleep 5 
# echo "14 exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex15" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 4 --rotation_adapter_config.r 4

# sleep 5 
# echo "15 exp finishes"
# date +"%F %T"