| ''' |
| Evaluate predictive performance of predictors in parallel with multiprocessing. |
| ''' |
| import argparse |
| from multiprocessing import Process, JoinableQueue |
| from multiprocessing import set_start_method |
| import os |
|
|
| import pandas as pd |
|
|
| from utils import parse_vars, merge_dfs |
| from evaluate_multiprocessing import run_from_queue |
| from predictors import get_predictor_names |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser( |
| description='Example: python evaluate.py sarkisyan onehot_ridge ' |
| '--predictor_params reg_coef=0.01') |
| parser.add_argument('dataset_name', type=str, |
| help='Dataset name. Folder of the same name under the data ' |
| 'and inference directories are expected to look up files.' |
| 'The data will be loaded from data/{dataset_name}/data.csv' |
| 'in the `seq` and `log_fitness` columns.') |
| parser.add_argument('predictor_name', type=str, |
| help='Predictor name, or all for running all predictors.') |
| parser.add_argument('--n_threads', type=int, default=20) |
| parser.add_argument('--n_train', type=int, default=96) |
| parser.add_argument('--max_n_mut', type=int, default=5) |
| parser.add_argument('--joint_training', dest='joint_training', action='store_true') |
| parser.add_argument('--boosting', dest='joint_training', action='store_false') |
| parser.set_defaults(joint_training=True) |
| parser.add_argument('--train_on_single', dest='train_on_single', action='store_true') |
| parser.add_argument('--train_on_all', dest='train_on_single', action='store_false') |
| parser.set_defaults(train_on_single=True) |
| parser.add_argument('--ignore_gaps', dest='ignore_gaps', action='store_true') |
| parser.set_defaults(ignore_gaps=False) |
| parser.add_argument('--n_seeds', type=int, default=20, |
| help='Number of random train test splits to get confidence interval') |
| parser.add_argument('--metric_topk', type=int, default=96, |
| help='Top ? when evaluating hit rate and topk mean') |
| parser.add_argument("--predictor_params", |
| metavar="KEY=VALUE", |
| nargs='+', |
| help="Set a number of key-value pairs " |
| "(do not put spaces before or after the = sign). " |
| "If a value contains spaces, you should define " |
| "it with double quotes: " |
| 'foo="this is a sentence". Note that ' |
| "values are always treated as floats.") |
| parser.add_argument('--results_suffix', type=str, default='') |
| args = parser.parse_args() |
| predictor_params = parse_vars(args.predictor_params) |
| if args.ignore_gaps: |
| predictor_params['ignore_gaps'] = args.ignore_gaps |
| print(args) |
|
|
| outdir = os.path.join('results', args.dataset_name) |
| if not os.path.exists(outdir): |
| os.mkdir(outdir) |
| outpath = os.path.join(outdir, f'results{args.results_suffix}.csv') |
|
|
| |
| queue = JoinableQueue() |
| workers = [] |
| for i in range(args.n_threads): |
| p = Process(target=run_from_queue, args=(i, queue)) |
| workers.append(p) |
| p.start() |
| predictors = get_predictor_names(args.predictor_name) |
| for pn in predictors: |
| for seed in range(args.n_seeds): |
| queue.put((args.dataset_name, pn, args.joint_training, |
| args.n_train, args.metric_topk, args.max_n_mut, |
| args.train_on_single, args.ignore_gaps, seed, |
| predictor_params, outpath)) |
| queue.join() |
| for p in workers: |
| p.terminate() |
| merge_dfs(f'{outpath}*', outpath, |
| index_cols=['dataset', 'predictor', 'predictor_params', 'seed'], |
| groupby_cols=['predictor', 'predictor_params', 'n_train', 'topk'], |
| ignore_cols=['seed']) |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
