export OMINI_CONFIG=./config/math395.yaml

#echo $OMINI_CONFIG
export TOKENIZERS_PARALLELISM=true

# CUDA Include (/cuda.h)
CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include"

# 3. Add into CPATH & CPLUS_INCLUDE_PATH (C/C++ compiler)
export CPATH=$CPATH:$CUDA_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH
# echo "CPATH is set to: $CPATH"
# echo "CPLUS_INCLUDE_PATH is set to: $CPLUS_INCLUDE_PATH"

export WANDB_PROJECT="Llama2_7B_FT_Math_395k"

export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
export OPENBLAS_NUM_THREADS=1
export NUMEXPR_NUM_THREADS=1

date +"%F %T"

# accelerate launch --main_process_port 41353 -m src.ft_mathQ \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex01" --trainer_args.learning_rate=5e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
#       --run_text 'def|o100k'

# sleep 5 
# echo "1st exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathQ \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex02" --trainer_args.learning_rate=2e-2 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
#       --run_text 'def|o100k'

# sleep 5 
# echo "2nd exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# bash scripts/merge_100k.sh

# accelerate launch --main_process_port 41353 -m src.ft_mathQ \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex03" --trainer_args.learning_rate=1e-2 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
#       --run_text 'def|o100k'

# sleep 5 
# echo "3rd exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run


# accelerate launch --main_process_port 41353 -m src.ft_mathQ \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex04" --trainer_args.learning_rate=5e-2 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
#       --run_text 'def|o100k'

# sleep 5 
# echo "4th exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathQ \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex05" --trainer_args.learning_rate=1e-2 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
#       --trainer_args.num_train_epochs 3.0  --data.dataset_split train[:100000] \
#       --run_text 'def|o100k'
# sleep 5 
# echo "5th exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run
# bash scripts/merge_100math.sh

accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex06" --trainer_args.learning_rate=1e-2 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48
sleep 5 
echo "6th exp finishes"
date +"%F %T"
wandb sync wandb/latest-run

accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex07" --trainer_args.learning_rate=1e-2 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 3.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48
sleep 5 
echo "6th exp finishes"
date +"%F %T"
wandb sync wandb/latest-run

accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex08" --trainer_args.learning_rate=2e-2 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48
sleep 5 
echo "8th exp finishes"
date +"%F %T"
wandb sync wandb/latest-run

accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex09" --trainer_args.learning_rate=2e-2 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 3.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48
sleep 5 
echo "9th exp finishes"
date +"%F %T"
wandb sync wandb/latest-run

accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex10" --trainer_args.learning_rate=3e-2 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48
sleep 5 
echo "10th exp finishes"
date +"%F %T"
wandb sync wandb/latest-run



accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex11" --trainer_args.learning_rate=8e-3 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 2.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48

sleep 5 
echo "11 exp finishes"
date +"%F %T"
wandb sync wandb/latest-run


accelerate launch --main_process_port 41353 -m src.ft_mathQ \
      --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp100/run_ex12" --trainer_args.learning_rate=8e-3 \
      --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
      --trainer_args.num_train_epochs 3.0  --data.dataset_split train[:100000] \
      --run_text 'def|o100k|b48' --trainer_args.per_device_train_batch_size 48

sleep 5 
echo "12 exp finishes"
date +"%F %T"
wandb sync wandb/latest-run


bash ./scripts/merge_100k.sh

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex13" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'u=vkaim'

# sleep 5 
# echo "13 exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex14" --trainer_args.learning_rate=2e-3 \
#       --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'a,b,def'

# sleep 5 
# echo "14 exp finishes"
# date +"%F %T"
# wandb sync wandb/latest-run

# accelerate launch --main_process_port 41353 -m src.ft_mathR \
#       --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex15" --trainer_args.learning_rate=1e-3 \
#       --rotation_adapter_config.num_rotations 4 --rotation_adapter_config.r 4

# sleep 5 
# echo "15 exp finishes"
# date +"%F %T"