| export OMINI_CONFIG=./config/math395.yaml | |
| #echo $OMINI_CONFIG | |
| export TOKENIZERS_PARALLELISM=true | |
| # CUDA Include (/cuda.h) | |
| CUDA_INCLUDE_PATH="/home/work/miniconda3/envs/allm/include" | |
| # 3. Add into CPATH & CPLUS_INCLUDE_PATH (C/C++ compiler) | |
| export CPATH=$CPATH:$CUDA_INCLUDE_PATH | |
| export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH | |
| # echo "CPATH is set to: $CPATH" | |
| # echo "CPLUS_INCLUDE_PATH is set to: $CPLUS_INCLUDE_PATH" | |
| export WANDB_PROJECT="Llama2_7B_FT_Math_395k" | |
| export OMP_NUM_THREADS=1 | |
| export MKL_NUM_THREADS=1 | |
| export OPENBLAS_NUM_THREADS=1 | |
| export NUMEXPR_NUM_THREADS=1 | |
| date +"%F %T" | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex01" --trainer_args.learning_rate=1e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'def' | |
| # sleep 5 | |
| # echo "1st exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex02" --trainer_args.learning_rate=5e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 | |
| # sleep 5 | |
| # echo "2nd exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex03" --trainer_args.learning_rate=2e-4 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 | |
| # sleep 5 | |
| # echo "3rd exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex04" --trainer_args.learning_rate=1e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 | |
| # sleep 5 | |
| # echo "4rd exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex05" --trainer_args.learning_rate=1e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \ | |
| # --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 | |
| # sleep 5 | |
| # echo "5th exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex06" --trainer_args.learning_rate=2e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \ | |
| # --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 | |
| # sleep 5 | |
| # echo "6th exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex0s7" --trainer_args.learning_rate=5e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \ | |
| # --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 | |
| # sleep 5 | |
| # echo "7th exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex08" --trainer_args.learning_rate=1e-4 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \ | |
| # --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 \ | |
| # --trainer_args.per_device_train_batch_size 32 --run_text 'u2e2,def' | |
| # sleep 5 | |
| # echo "8th exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex09" --trainer_args.learning_rate=2e-3 \ | |
| --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \ | |
| --trainer_args.num_train_epochs 3.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 \ | |
| --trainer_args.per_device_train_batch_size 32 --run_text 'init=def|fix_token' | |
| sleep 5 | |
| echo "9th exp finishes" | |
| date +"%F %T" | |
| wandb sync wandb/latest-run | |
| accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex10" --trainer_args.learning_rate=2e-3 \ | |
| --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \ | |
| --trainer_args.num_train_epochs 2.0 --trainer_args.eval_steps 500 --data.dataset_split train[:101011] --data.split_ratio 0.01 \ | |
| --trainer_args.per_device_train_batch_size 32 --run_text "init=def|fix_token" | |
| sleep 5 | |
| echo "10 exp finishes" | |
| date +"%F %T" | |
| wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex11" --trainer_args.learning_rate=1e-2 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 | |
| # sleep 5 | |
| # echo "11 exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex12" --trainer_args.learning_rate=1e-2 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'u=v,def' | |
| # sleep 5 | |
| # echo "12 exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex13" --trainer_args.learning_rate=1e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'u=vkaim' | |
| # sleep 5 | |
| # echo "13 exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex14" --trainer_args.learning_rate=2e-3 \ | |
| # --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 --run_text 'a,b,def' | |
| # sleep 5 | |
| # echo "14 exp finishes" | |
| # date +"%F %T" | |
| # wandb sync wandb/latest-run | |
| # accelerate launch --main_process_port 41353 -m src.ft_mathR \ | |
| # --config_path $OMINI_CONFIG --trainer_args.output_dir "./exp395/run_ex15" --trainer_args.learning_rate=1e-3 \ | |
| # --rotation_adapter_config.num_rotations 4 --rotation_adapter_config.r 4 | |
| # sleep 5 | |
| # echo "15 exp finishes" | |
| # date +"%F %T" | |