| import numpy as np |
| import pandas as pd |
| from sklearn.linear_model import Ridge |
| from sklearn import svm |
|
|
| from utils.metric_utils import get_spearman_fractions, wt_improvement_metric, topk_median |
|
|
|
|
| def test_regression_multiseeds(X, y, n_train, n_seeds, y_wt, y_cutoff=None, |
| mutation_counts=None, mutation_count_cutoff=None): |
| spm = np.zeros((n_seeds, len(SPEARMAN_FRACTIONS))) |
| r2 = np.zeros(n_seeds) |
| wt_imprv = np.zeros(n_seeds) |
| topk_med = np.zeros(n_seeds) |
| best_alpha = np.zeros(n_seeds) |
| for i in range(n_seeds): |
| spm[i], r2[i], wt_imprv[i], topk_med[i], best_alpha[i] = test_regression( |
| X, y, n_train, y_wt, y_cutoff, i, mutation_counts, |
| mutation_count_cutoff) |
| df = pd.DataFrame({ |
| 'R2 score': r2, |
| 'Improvement over WT': wt_imprv, |
| 'Top K median': topk_med, |
| 'N train': n_train, |
| 'Best alpha': best_alpha, |
| }) |
| for i, f in enumerate(SPEARMAN_FRACTIONS): |
| df[f'Spearman correlation at {f:.1f}'] = spm[:, i] |
| df['Spearman correlation'] = spm[:, -1] |
| return df |
|
|
|
|
| def test_regression(X, y, n_train, y_wt, y_cutoff=None, seed=0, |
| mutation_counts=None, mutation_count_cutoff=None): |
| if y_cutoff is not None: |
| is_valid = (y >= y_cutoff) |
| X, y = X[is_valid], y[is_valid] |
| if mutation_counts is not None: |
| mutation_counts = mutation_counts[is_valid] |
| X_tr, X_eval, X_test, y_tr, y_eval, y_test = train_eval_test_split( |
| X, y, seed, n_train, mutation_counts, mutation_count_cutoff) |
| |
| best_alpha = None |
| best_spm = -999.9 |
| for alpha in [0.001, 0.01, 0.1, 0.2, 0.3, 0.5, 1.0, 2.0]: |
| model = Ridge(alpha=alpha) |
| model.fit(X_tr, y_tr) |
| y_pred = model.predict(X_eval) |
| spm = spearmanr(y_pred, y_eval).correlation |
| if spm > best_spm: |
| best_alpha = alpha |
| best_spm = spm |
| model = Ridge(alpha=best_alpha) |
| model.fit(X_tr, y_tr) |
| y_pred = model.predict(X_test) |
| spm = get_spearman_fractions(y_pred, y_test) |
| r2 = model.score(X_test, y_test) |
| wt_imprv = wt_improvement_metric(y_pred, y_test, y_wt) |
| topk_med = topk_median(y_pred, y_test) |
| return spm, r2, wt_imprv, topk_med, best_alpha |
|
|
|
|
| def test_classification_multiseeds(X, y, n_train, n_seeds, y_cutoff=None, |
| mutation_counts=None, mutation_count_cutoff=None): |
| acc = np.zeros(n_seeds) |
| best_C = np.zeros(n_seeds) |
| for i in range(n_seeds): |
| acc[i], best_C[i] = test_classification( |
| X, y, n_train, y_cutoff, i, mutation_counts, |
| mutation_count_cutoff) |
| return acc, best_C |
|
|
|
|
| def test_classification(X, y, n_train, y_cutoff, seed=0, |
| mutation_counts=None, mutation_count_cutoff=None): |
| y = (y > y_cutoff).astype(int) |
| X_tr, X_eval, X_test, y_tr, y_eval, y_test = train_eval_test_split( |
| X, y, seed, n_train, mutation_counts, mutation_count_cutoff) |
| while len(np.unique(y_tr)) < 2: |
| X_tr, X_eval, X_test, y_tr, y_eval, y_test = train_eval_test_split( |
| X, y, seed+np.random.randint(10000), n_train, mutation_counts, |
| mutation_count_cutoff) |
| best_C = None |
| best_acc = -999.9 |
| for C in [0.01, 0.1, 0.5, 1.0, 2.0]: |
| model = svm.LinearSVC(C=C) |
| model.fit(X_tr, y_tr) |
| acc = model.score(X_eval, y_eval) |
| if acc > best_acc: |
| best_C = C |
| best_acc = acc |
| model = svm.LinearSVC(C=best_C) |
| model.fit(X_tr, y_tr) |
| return model.score(X_test, y_test), best_C |
|
|
|
|
| def run_regression(feature_reps, y, y_wt, y_cutoff, n_seeds, |
| mutation_counts=None, mutation_count_cutoff=None): |
| results = pd.DataFrame() |
| for feature_rep, X in feature_reps.items(): |
| print('Staring runs for', feature_rep) |
| for n_train in [8, 24, 96, 192, 480, 960, 9600]: |
| if n_train >= 0.8 * X.shape[0]: |
| continue |
| print('n_train:', n_train) |
| df = test_regression_multiseeds(X, y, n_train, n_seeds, y_wt, |
| y_cutoff, mutation_counts, mutation_count_cutoff) |
| df['Feature rep'] = feature_rep |
| results = pd.concat([results, df], axis=0) |
| return results |
|
|
|
|
| def run_classification(feature_reps, y, y_cutoff, n_seeds, |
| mutation_counts=None, mutation_count_cutoff=None): |
| results = pd.DataFrame() |
| for feature_rep, X in feature_reps.items(): |
| print('Staring runs for', feature_rep) |
| for n_train in [8, 24, 96, 192, 480, 960]: |
| if n_train >= 0.8 * X.shape[0]: |
| continue |
| print('n_train:', n_train) |
| acc, best_C = test_classification_multiseeds( |
| X, y, n_train, n_seeds, y_cutoff, mutation_counts, |
| mutation_count_cutoff) |
| df = pd.DataFrame({ |
| 'Accuracy': acc, |
| 'Best reg coeff': best_C, |
| 'N train': n_train, |
| 'Feature rep': feature_rep, |
| }) |
| results = pd.concat([results, df], axis=0) |
| return results |
|
|
