| | """The DataProvider class serve as a helper module for retriving subject model data""" |
| | from abc import ABC, abstractmethod |
| |
|
| | import os |
| | import gc |
| | import time |
| |
|
| | from singleVis.utils import * |
| | from singleVis.eval.evaluate import evaluate_inv_accu |
| |
|
| | """ |
| | DataContainder module |
| | 1. prepare data |
| | 2. estimate boundary |
| | 3. provide data |
| | """ |
| | class DataProviderAbstractClass(ABC): |
| | |
| | def __init__(self, content_path, model, epoch_start, epoch_end, epoch_period): |
| | self.mode = "abstract" |
| | self.content_path = content_path |
| | self.model = model |
| | self.s = epoch_start |
| | self.e = epoch_end |
| | self.p = epoch_period |
| | |
| | @property |
| | @abstractmethod |
| | def train_num(self): |
| | pass |
| |
|
| | @property |
| | @abstractmethod |
| | def test_num(self): |
| | pass |
| |
|
| | @abstractmethod |
| | def _meta_data(self): |
| | pass |
| |
|
| | @abstractmethod |
| | def _estimate_boundary(self): |
| | pass |
| | |
| | def update_interval(self, epoch_s, epoch_e): |
| | self.s = epoch_s |
| | self.e = epoch_e |
| |
|
| | class DataProvider(DataProviderAbstractClass): |
| | def __init__(self, content_path, model, epoch_start, epoch_end, epoch_period, device, classes, epoch_name, verbose=1): |
| | self.content_path = content_path |
| | self.model = model |
| | self.s = epoch_start |
| | self.e = epoch_end |
| | self.p = epoch_period |
| | self.DEVICE = device |
| | self.classes = classes |
| | self.verbose = verbose |
| | self.epoch_name = epoch_name |
| | self.model_path = os.path.join(self.content_path, "Model") |
| | if verbose: |
| | print("Finish initialization...") |
| |
|
| | @property |
| | def train_num(self): |
| | with open(os.path.join(self.content_path, "Model", "{}_{}".format(self.epoch_name, self.s), "index.json"), "r") as f: |
| | idxs = json.load(f) |
| | return len(idxs) |
| |
|
| | @property |
| | def test_num(self): |
| | testing_data_path = os.path.join(self.content_path, "Testing_data") |
| | testing_data = torch.load(os.path.join(testing_data_path, "testing_dataset_data.pth"), |
| | map_location="cpu") |
| | test_num = len(testing_data) |
| | del testing_data |
| | gc.collect() |
| | return test_num |
| | |
| | def _meta_data(self): |
| | raise NotImplementedError |
| |
|
| | def _estimate_boundary(self): |
| | raise NotImplementedError |
| |
|
| |
|
| | class NormalDataProvider(DataProvider): |
| | def __init__(self, content_path, model, epoch_start, epoch_end, epoch_period, device, classes, epoch_name, verbose=1): |
| | super().__init__(content_path, model, epoch_start, epoch_end, epoch_period, device, classes, epoch_name, verbose) |
| | self.mode = "normal" |
| | |
| | @property |
| | def representation_dim(self): |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, self.s), "train_data.npy") |
| | try: |
| | train_data = np.load(train_data_loc) |
| | repr_dim = np.prod(train_data.shape[1:]) |
| | return repr_dim |
| | except Exception as e: |
| | return None |
| |
|
| | def _meta_data(self): |
| | time_inference = list() |
| | training_data_path = os.path.join(self.content_path, "Training_data") |
| | training_data = torch.load(os.path.join(training_data_path, "training_dataset_data.pth"), |
| | map_location="cpu") |
| | training_data = training_data.to(self.DEVICE) |
| | testing_data_path = os.path.join(self.content_path, "Testing_data") |
| | testing_data = torch.load(os.path.join(testing_data_path, "testing_dataset_data.pth"), |
| | map_location="cpu") |
| | testing_data = testing_data.to(self.DEVICE) |
| |
|
| | for n_epoch in range(self.s, self.e + 1, self.p): |
| | t_s = time.time() |
| |
|
| | |
| | test_index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "test_index.json") |
| | if os.path.exists(test_index_file): |
| | test_index = load_labelled_data_index(test_index_file) |
| | else: |
| | test_index = range(len(testing_data)) |
| | testing_data = testing_data[test_index] |
| |
|
| | model_location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "subject_model.pth") |
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu"))) |
| | self.model = self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | repr_model = self.feature_function(n_epoch) |
| | |
| |
|
| | |
| | data_pool_representation = batch_run(repr_model, training_data) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "train_data.npy") |
| | np.save(location, data_pool_representation) |
| |
|
| | |
| | test_data_representation = batch_run(repr_model, testing_data) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "test_data.npy") |
| | np.save(location, test_data_representation) |
| |
|
| | t_e = time.time() |
| | time_inference.append(t_e-t_s) |
| | if self.verbose > 0: |
| | print("Finish inferencing data for Epoch {:d}...".format(n_epoch)) |
| | print( |
| | "Average time for inferencing data: {:.4f}".format(sum(time_inference) / len(time_inference))) |
| |
|
| | |
| | save_dir = os.path.join(self.model_path, "time.json") |
| | if not os.path.exists(save_dir): |
| | evaluation = dict() |
| | else: |
| | f = open(save_dir, "r") |
| | evaluation = json.load(f) |
| | f.close() |
| | evaluation["data_inference"] = round(sum(time_inference) / len(time_inference), 3) |
| | with open(save_dir, 'w') as f: |
| | json.dump(evaluation, f) |
| |
|
| | del training_data |
| | del testing_data |
| | gc.collect() |
| |
|
| | def _estimate_boundary(self, num, l_bound): |
| | ''' |
| | Preprocessing data. This process includes find_border_points and find_border_centers |
| | save data for later training |
| | ''' |
| |
|
| | time_borders_gen = list() |
| | training_data_path = os.path.join(self.content_path, "Training_data") |
| | training_data = torch.load(os.path.join(training_data_path, "training_dataset_data.pth"), |
| | map_location="cpu") |
| | training_data = training_data.to(self.DEVICE) |
| | for n_epoch in range(self.s, self.e + 1, self.p): |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | training_data = training_data[index] |
| |
|
| | |
| | |
| | |
| | |
| | |
| | repr_model = self.feature_function(n_epoch) |
| |
|
| | t0 = time.time() |
| | confs = batch_run(self.model, training_data) |
| | preds = np.argmax(confs, axis=1).squeeze() |
| | |
| | num_adv_eg = num |
| | |
| | border_points, _, _ = get_border_points(model=self.model, input_x=training_data, confs=confs, predictions=preds, device=self.DEVICE, l_bound=l_bound, num_adv_eg=num_adv_eg, lambd=0.05, verbose=0) |
| | t1 = time.time() |
| | time_borders_gen.append(round(t1 - t0, 4)) |
| |
|
| | |
| | border_points = border_points.to(self.DEVICE) |
| | border_centers = batch_run(repr_model, border_points) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "border_centers.npy") |
| | np.save(location, border_centers) |
| |
|
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "ori_border_centers.npy") |
| | np.save(location, border_points.cpu().numpy()) |
| |
|
| | num_adv_eg = num |
| | border_points, _, _ = get_border_points(model=self.model, input_x=training_data, confs=confs, predictions=preds, device=self.DEVICE, l_bound=l_bound, num_adv_eg=num_adv_eg, lambd=0.05, verbose=0) |
| |
|
| | |
| | border_points = border_points.to(self.DEVICE) |
| | border_centers = batch_run(repr_model, border_points) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "test_border_centers.npy") |
| | np.save(location, border_centers) |
| |
|
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, n_epoch), "test_ori_border_centers.npy") |
| | np.save(location, border_points.cpu().numpy()) |
| |
|
| | if self.verbose > 0: |
| | print("Finish generating borders for Epoch {:d}...".format(n_epoch)) |
| | print( |
| | "Average time for generate border points: {:.4f}".format(sum(time_borders_gen) / len(time_borders_gen))) |
| |
|
| | |
| | save_dir = os.path.join(self.model_path, "time.json") |
| | if not os.path.exists(save_dir): |
| | evaluation = dict() |
| | else: |
| | f = open(save_dir, "r") |
| | evaluation = json.load(f) |
| | f.close() |
| | evaluation["data_B_gene"] = round(sum(time_borders_gen) / len(time_borders_gen), 3) |
| | with open(save_dir, 'w') as f: |
| | json.dump(evaluation, f) |
| |
|
| | def initialize(self, num, l_bound): |
| | self._meta_data() |
| | self._estimate_boundary(num, l_bound) |
| |
|
| | def train_representation(self, epoch): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "train_data.npy") |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | try: |
| | train_data = np.load(train_data_loc) |
| | train_data = train_data[index] |
| | except Exception as e: |
| | print("no train data saved for Epoch {}".format(epoch)) |
| | train_data = None |
| | return train_data |
| | |
| | def train_labels(self, epoch): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | try: |
| | training_labels = torch.load(training_data_loc, map_location="cpu") |
| | |
| | |
| | |
| | training_labels = np.array(training_labels) |
| | except Exception as e: |
| | print("no train labels saved for Epoch {}".format(epoch)) |
| | training_labels = None |
| | return training_labels |
| |
|
| | def test_representation(self, epoch): |
| | data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "test_data.npy") |
| | try: |
| | test_data = np.load(data_loc) |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "test_index.json") |
| | if os.path.exists(index_file): |
| | index = load_labelled_data_index(index_file) |
| | test_data = test_data[index] |
| | except Exception as e: |
| | print("no test data saved for Epoch {}".format(epoch)) |
| | test_data = None |
| | |
| | return test_data |
| | |
| | def test_labels(self, epoch): |
| | |
| | testing_data_loc = os.path.join(self.content_path, "Testing_data", "testing_dataset_label.pth") |
| | try: |
| | testing_labels = torch.load(testing_data_loc).to(device="cpu") |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "test_index.json") |
| | print(index_file) |
| | if os.path.exists(index_file): |
| | idxs = load_labelled_data_index(index_file) |
| | |
| | testing_labels = torch.zeros(len(idxs)) |
| | except Exception as e: |
| | print("no test labels saved for Epoch {}".format(epoch)) |
| | testing_labels = None |
| | return testing_labels.cpu().numpy() |
| |
|
| | def border_representation(self, epoch): |
| | border_centers_loc = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), |
| | "border_centers.npy") |
| | try: |
| | border_centers = np.load(border_centers_loc) |
| | except Exception as e: |
| | print("no border points saved for Epoch {}".format(epoch)) |
| | border_centers = np.array([]) |
| | return border_centers |
| | |
| | def test_border_representation(self, epoch): |
| | border_centers_loc = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), |
| | "test_border_centers.npy") |
| | try: |
| | border_centers = np.load(border_centers_loc) |
| | except Exception as e: |
| | print("no border points saved for Epoch {}".format(epoch)) |
| | border_centers = np.array([]) |
| | return border_centers |
| | |
| | def max_norm(self, epoch): |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "train_data.npy") |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | try: |
| | train_data = np.load(train_data_loc) |
| | train_data = train_data[index] |
| | max_x = np.linalg.norm(train_data, axis=1).max() |
| | except Exception as e: |
| | print("no train data saved for Epoch {}".format(epoch)) |
| | max_x = None |
| | return max_x |
| |
|
| | def prediction_function(self, epoch): |
| | model_location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "subject_model.pth") |
| |
|
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu")),strict=False) |
| | self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | pred_fn = self.model.prediction |
| | return pred_fn |
| |
|
| |
|
| | def feature_function(self, epoch): |
| | model_location = os.path.join(self.model_path, "{}_{:d}".format(self.epoch_name, epoch), "subject_model.pth") |
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu")),strict=False) |
| | self.model = self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | fea_fn = self.model.feature |
| | return fea_fn |
| |
|
| | def get_pred(self, epoch, data): |
| | ''' |
| | get the prediction score for data in epoch_id |
| | :param data: numpy.ndarray |
| | :param epoch_id: |
| | :return: pred, numpy.ndarray |
| | ''' |
| | prediction_func = self.prediction_function(epoch) |
| |
|
| | data = torch.from_numpy(data) |
| | data = data.to(self.DEVICE) |
| | pred = batch_run(prediction_func, data) |
| | return pred |
| |
|
| | def training_accu(self, epoch): |
| | data = self.train_representation(epoch) |
| | labels = self.train_labels(epoch) |
| | pred = self.get_pred(epoch, data).argmax(-1) |
| | val = evaluate_inv_accu(labels, pred) |
| | return val |
| |
|
| | def testing_accu(self, epoch): |
| | data = self.test_representation(epoch) |
| | labels = self.test_labels(epoch) |
| | test_index_file = os.path.join(self.model_path, "{}_{}".format(self.epoch_name, epoch), "test_index.json") |
| | if os.path.exists(test_index_file): |
| | index = load_labelled_data_index(test_index_file) |
| | labels = labels[index] |
| | pred = self.get_pred(epoch, data).argmax(-1) |
| | val = evaluate_inv_accu(labels, pred) |
| | return val |
| | |
| | def is_deltaB(self, epoch, data): |
| | """ |
| | check wheter input vectors are lying on delta-boundary or not |
| | :param epoch_id: |
| | :param data: numpy.ndarray |
| | :return: numpy.ndarray, boolean, True stands for is_delta_boundary |
| | """ |
| | preds = self.get_pred(epoch, data) |
| | border = is_B(preds) |
| | return border |
| | |
| | def checkpoint_path(self, epoch): |
| | path = os.path.join(self.model_path, "{}_{}".format(self.epoch_name, epoch)) |
| | return path |
| |
|
| | |
| | class ActiveLearningDataProvider(DataProvider): |
| | def __init__(self, content_path, model, base_epoch_start, device, classes, iteration_name="Iteration",verbose=1): |
| | |
| | super().__init__(content_path, model, base_epoch_start, base_epoch_start, 1, device, classes, iteration_name, verbose) |
| | self.mode = "al" |
| | self.iteration_name = iteration_name |
| | |
| | @property |
| | def pool_num(self): |
| | return len(self.train_labels_all()) |
| | |
| | def label_num(self, iteration): |
| | return len(self.get_labeled_idx(iteration)) |
| |
|
| | def representation_dim(self, iteration): |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "train_data.npy") |
| | try: |
| | train_data = np.load(train_data_loc) |
| | repr_dim = np.prod(train_data.shape[1:]) |
| | return repr_dim |
| | except Exception as e: |
| | return None |
| | |
| | def get_labeled_idx(self, iteration): |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "index.json") |
| | lb_idxs = np.array(load_labelled_data_index(index_file)) |
| | return lb_idxs |
| |
|
| | def get_unlabeled_idx(self, pool_num, lb_idx): |
| | tot_idx = np.arange(pool_num) |
| | |
| | ulb_idx = np.setdiff1d(tot_idx, lb_idx) |
| | return ulb_idx |
| |
|
| | def _meta_data(self, iteration): |
| | training_data_path = os.path.join(self.content_path, "Training_data") |
| | training_data = torch.load(os.path.join(training_data_path, "training_dataset_data.pth"), |
| | map_location="cpu") |
| | training_data = training_data.to(self.DEVICE) |
| | testing_data_path = os.path.join(self.content_path, "Testing_data") |
| | testing_data = torch.load(os.path.join(testing_data_path, "testing_dataset_data.pth"), |
| | map_location="cpu") |
| | testing_data = testing_data.to(self.DEVICE) |
| |
|
| | t_s = time.time() |
| | repr_model = self.feature_function(iteration) |
| |
|
| | |
| | data_pool_representation = batch_run(repr_model, training_data) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "train_data.npy") |
| | np.save(location, data_pool_representation) |
| |
|
| | |
| | test_data_representation = batch_run(repr_model, testing_data) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "test_data.npy") |
| | np.save(location, test_data_representation) |
| |
|
| | t_e = time.time() |
| |
|
| | if self.verbose > 0: |
| | print("Finish inferencing data for Iteration {:d} in {:.2f} seconds...".format(iteration, t_e-t_s)) |
| |
|
| | |
| | save_dir = os.path.join(self.model_path, "time_al.json") |
| | if not os.path.exists(save_dir): |
| | evaluation = dict() |
| | |
| | else: |
| | f = open(save_dir, "r") |
| | evaluation = json.load(f) |
| | f.close() |
| | if "data_inference" not in evaluation.keys(): |
| | evaluation["data_inference"] = dict() |
| | evaluation["data_inference"][str(iteration)] = round(t_e - t_s, 3) |
| | with open(save_dir, 'w') as f: |
| | json.dump(evaluation, f) |
| |
|
| | del training_data |
| | del testing_data |
| | gc.collect() |
| |
|
| | def _estimate_boundary(self, iteration, num, l_bound): |
| | ''' |
| | Preprocessing data. This process includes find_border_points and find_border_centers |
| | save data for later training |
| | ''' |
| |
|
| | training_data_path = os.path.join(self.content_path, "Training_data") |
| | training_data = torch.load(os.path.join(training_data_path, "training_dataset_data.pth"), |
| | map_location="cpu") |
| | training_data = training_data.to(self.DEVICE) |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | training_data = training_data[index] |
| |
|
| | repr_model = self.feature_function(iteration) |
| |
|
| | t0 = time.time() |
| | confs = batch_run(self.model, training_data) |
| | preds = np.argmax(confs, axis=1).squeeze() |
| | |
| | num_adv_eg = num |
| | border_points, _, _ = get_border_points(model=self.model, input_x=training_data, confs=confs, predictions=preds, device=self.DEVICE, l_bound=l_bound, num_adv_eg=num_adv_eg, lambd=0.05, verbose=0) |
| | t1 = time.time() |
| |
|
| | |
| | border_points = border_points.to(self.DEVICE) |
| | border_centers = batch_run(repr_model, border_points) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "border_centers.npy") |
| | np.save(location, border_centers) |
| |
|
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "ori_border_centers.npy") |
| | np.save(location, border_points.cpu().numpy()) |
| |
|
| | num_adv_eg = num |
| | border_points, _, _ = get_border_points(model=self.model, input_x=training_data, confs=confs, predictions=preds, device=self.DEVICE, l_bound=l_bound, num_adv_eg=num_adv_eg, lambd=0.05, verbose=0) |
| |
|
| | |
| | border_points = border_points.to(self.DEVICE) |
| | border_centers = batch_run(repr_model, border_points) |
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "test_border_centers.npy") |
| | np.save(location, border_centers) |
| |
|
| | location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_nameiteration), "test_ori_border_centers.npy") |
| | np.save(location, border_points.cpu().numpy()) |
| |
|
| | if self.verbose > 0: |
| | print("Finish generating borders for Iteration {:d} in {:.2f} seconds ...".format(iteration, t1-t0)) |
| |
|
| | |
| | save_dir = os.path.join(self.model_path, "time_al.json") |
| | if not os.path.exists(save_dir): |
| | evaluation = dict() |
| | else: |
| | f = open(save_dir, "r") |
| | evaluation = json.load(f) |
| | f.close() |
| | if "data_B_gene" not in evaluation.keys(): |
| | evaluation["data_B_gene"] = dict() |
| | evaluation["data_B_gene"][str(iteration)] = round(t1-t0, 3) |
| | with open(save_dir, 'w') as f: |
| | json.dump(evaluation, f) |
| |
|
| | def initialize_iteration(self, iteration, num, l_bound): |
| | self._meta_data(iteration) |
| | self._estimate_boundary(iteration, num, l_bound) |
| |
|
| | def train_representation(self, iteration): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "train_data.npy") |
| | try: |
| | idxs = self.get_labeled_idx(iteration) |
| | train_data = np.load(train_data_loc)[idxs] |
| | except Exception as e: |
| | print("no train data saved for Iteration {}".format(iteration)) |
| | train_data = None |
| | return train_data |
| | |
| | def train_labels(self, epoch): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | try: |
| | idxs = self.get_labeled_idx(epoch) |
| | training_labels = torch.load(training_data_loc, map_location="cpu")[idxs] |
| | except Exception as e: |
| | print("no train labels saved for Iteration {}".format(epoch)) |
| | training_labels = None |
| | return training_labels.numpy() |
| | |
| | def train_representation_ulb(self, iteration): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "train_data.npy") |
| | lb_idxs = self.get_labeled_idx(iteration) |
| | try: |
| | train_data = np.load(train_data_loc) |
| | ulb_idxs = self.get_unlabeled_idx(len(train_data), lb_idxs) |
| | train_data = train_data[ulb_idxs] |
| | except Exception as e: |
| | print("no train data saved for Iteration {}".format(iteration)) |
| | train_data = None |
| | return train_data |
| | |
| | def train_labels_ulb(self, epoch): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | lb_idxs = self.get_labeled_idx(epoch) |
| | try: |
| | training_labels = torch.load(training_data_loc, map_location="cpu") |
| | ulb_idxs = self.get_unlabeled_idx(len(training_labels), lb_idxs) |
| | training_labels = training_labels[ulb_idxs] |
| | except Exception as e: |
| | print("no train labels saved for Iteration {}".format(epoch)) |
| | training_labels = None |
| | return training_labels.numpy() |
| | |
| | def train_representation_all(self, iteration): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "train_data.npy") |
| | try: |
| | train_data = np.load(train_data_loc) |
| | except Exception as e: |
| | print("no train data saved for Iteration {}".format(iteration)) |
| | train_data = None |
| | return train_data |
| | |
| | def train_labels_all(self): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | try: |
| | training_labels = torch.load(training_data_loc, map_location="cpu") |
| | except Exception as e: |
| | print("no train labels saved") |
| | training_labels = None |
| | return training_labels.numpy() |
| |
|
| | def test_representation(self, epoch): |
| | data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, epoch), "test_data.npy") |
| | try: |
| | test_data = np.load(data_loc) |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, epoch), "test_index.json") |
| | if os.path.exists(index_file): |
| | index = load_labelled_data_index(index_file) |
| | test_data = test_data[index] |
| | except Exception as e: |
| | print("no test data saved for Iteration {}".format(epoch)) |
| | test_data = None |
| | |
| | return test_data |
| | |
| | def test_labels(self, epoch): |
| | |
| | testing_data_loc = os.path.join(self.content_path, "Testing_data", "testing_dataset_label.pth") |
| | try: |
| | testing_labels = torch.load(testing_data_loc, map_location="cpu").numpy() |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, epoch), "test_index.json") |
| | if os.path.exists(index_file): |
| | idxs = load_labelled_data_index(index_file) |
| | testing_labels = testing_labels[idxs] |
| | except Exception as e: |
| | print("no test labels saved for Iteration {}".format(epoch)) |
| | testing_labels = None |
| | return testing_labels |
| |
|
| | def border_representation(self, epoch): |
| | border_centers_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, epoch), |
| | "border_centers.npy") |
| | try: |
| | border_centers = np.load(border_centers_loc) |
| | except Exception as e: |
| | print("no border points saved for Iteration {}".format(epoch)) |
| | border_centers = np.array([]) |
| | return border_centers |
| | |
| | def test_border_representation(self, epoch): |
| | border_centers_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, epoch), |
| | "test_border_centers.npy") |
| | try: |
| | border_centers = np.load(border_centers_loc) |
| | except Exception as e: |
| | print("no border points saved for Epoch {}".format(epoch)) |
| | border_centers = np.array([]) |
| | return border_centers |
| | |
| | def max_norm(self, epoch): |
| | train_data = self.train_representation(epoch) |
| | max_x = np.linalg.norm(train_data, axis=1).max() |
| | return max_x |
| |
|
| | def prediction_function(self, iteration): |
| | model_location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "subject_model.pth") |
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu"))) |
| | self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | pred_fn = self.model.prediction |
| | return pred_fn |
| |
|
| | def feature_function(self, epoch): |
| | model_location = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, epoch), "subject_model.pth") |
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu"))) |
| | self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | fea_fn = self.model.feature |
| | return fea_fn |
| |
|
| | def get_pred(self, iteration, data): |
| | ''' |
| | get the prediction score for data in epoch_id |
| | :param data: numpy.ndarray |
| | :param epoch_id: |
| | :return: pred, numpy.ndarray |
| | ''' |
| | prediction_func = self.prediction_function(iteration) |
| |
|
| | data = torch.from_numpy(data) |
| | data = data.to(self.DEVICE) |
| | pred = batch_run(prediction_func, data) |
| | return pred |
| |
|
| | def training_accu(self, epoch): |
| | data = self.train_representation_lb(epoch) |
| | labels = self.train_labels_lb(epoch) |
| | pred = self.get_pred(epoch, data).argmax(1) |
| | val = evaluate_inv_accu(labels, pred) |
| | return val |
| |
|
| | def testing_accu(self, epoch): |
| | data = self.test_representation(epoch) |
| | labels = self.test_labels(epoch) |
| | pred = self.get_pred(epoch, data).argmax(1) |
| | val = evaluate_inv_accu(labels, pred) |
| | return val |
| | |
| | def is_deltaB(self, epoch, data): |
| | """ |
| | check wheter input vectors are lying on delta-boundary or not |
| | :param epoch_id: |
| | :param data: numpy.ndarray |
| | :return: numpy.ndarray, boolean, True stands for is_delta_boundary |
| | """ |
| | preds = self.get_pred(epoch, data) |
| | border = is_B(preds) |
| | return border |
| | |
| | def checkpoint_path(self, epoch): |
| | path = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, epoch)) |
| | return path |
| | |
| | class DenseActiveLearningDataProvider(ActiveLearningDataProvider): |
| | def __init__(self, content_path, model, base_epoch_start, epoch_num, device, classes, iteration_name="Iteration", epoch_name="Epoch", verbose=1): |
| | super().__init__(content_path, model, base_epoch_start, device, classes, iteration_name, verbose) |
| | self.mode = "dense_al" |
| | self.epoch_num = epoch_num |
| | self.s = 1 |
| | self.p = 1 |
| | self.e = epoch_num |
| | self.epoch_name = epoch_name |
| |
|
| | def representation_dim(self): |
| | train_data_loc = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, self.s), "{}_{:d}".format(self.epoch_name, self.epoch_num), "train_data.npy") |
| | try: |
| | train_data = np.load(train_data_loc) |
| | repr_dim = np.prod(train_data.shape[1:]) |
| | return repr_dim |
| | except Exception as e: |
| | return None |
| |
|
| | def _meta_data(self, iteration): |
| | time_inference = list() |
| | training_data_path = os.path.join(self.content_path, "Training_data") |
| | training_data = torch.load(os.path.join(training_data_path, "training_dataset_data.pth"), |
| | map_location="cpu") |
| | training_data = training_data.to(self.DEVICE) |
| | testing_data_path = os.path.join(self.content_path, "Testing_data") |
| | testing_data = torch.load(os.path.join(testing_data_path, "testing_dataset_data.pth"), |
| | map_location="cpu") |
| | testing_data = testing_data.to(self.DEVICE) |
| |
|
| | t_s = time.time() |
| |
|
| | |
| | test_index_file = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "test_index.json") |
| | if os.path.exists(test_index_file): |
| | test_index = load_labelled_data_index(test_index_file) |
| | else: |
| | test_index = range(len(testing_data)) |
| | testing_data = testing_data[test_index] |
| |
|
| | for n_epoch in range(1, self.epoch_num+1, 1): |
| | repr_model = self.feature_function(iteration, n_epoch) |
| |
|
| | |
| | data_pool_representation = batch_run(repr_model, training_data) |
| | location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, n_epoch), "train_data.npy") |
| | np.save(location, data_pool_representation) |
| |
|
| | |
| | test_data_representation = batch_run(repr_model, testing_data) |
| | location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, n_epoch), "test_data.npy") |
| | np.save(location, test_data_representation) |
| |
|
| | t_e = time.time() |
| | time_inference.append(t_e-t_s) |
| | if self.verbose > 0: |
| | print("Finish inferencing data for Iteration {:d}...".format(iteration)) |
| | print( |
| | "Average time for inferencing data: {:.4f}...".format(sum(time_inference) / len(time_inference))) |
| |
|
| | |
| | save_dir = os.path.join(self.model_path, "SV_time.json") |
| | if not os.path.exists(save_dir): |
| | evaluation = dict() |
| | else: |
| | f = open(save_dir, "r") |
| | evaluation = json.load(f) |
| | f.close() |
| | evaluation["data_inference"] = round(sum(time_inference) / len(time_inference), 3) |
| | with open(save_dir, 'w') as f: |
| | json.dump(evaluation, f) |
| |
|
| | del training_data |
| | del testing_data |
| | gc.collect() |
| |
|
| | def _estimate_boundary(self, iteration, num, l_bound): |
| | ''' |
| | Preprocessing data. This process includes find_border_points and find_border_centers |
| | save data for later training |
| | ''' |
| |
|
| | time_borders_gen = list() |
| | training_data_path = os.path.join(self.content_path, "Training_data") |
| | training_data = torch.load(os.path.join(training_data_path, "training_dataset_data.pth"), |
| | map_location="cpu") |
| | training_data = training_data.to(self.DEVICE) |
| |
|
| | for n_epoch in range(1, self.epoch_num+1, 1): |
| | index_file = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | training_data = training_data[index] |
| |
|
| | repr_model = self.feature_function(iteration, n_epoch) |
| |
|
| | t0 = time.time() |
| | confs = batch_run(self.model, training_data) |
| | preds = np.argmax(confs, axis=1).squeeze() |
| | |
| | num_adv_eg = num |
| | border_points, _, _ = get_border_points(model=self.model, input_x=training_data, confs=confs, predictions=preds, device=self.DEVICE, l_bound=l_bound, num_adv_eg=num_adv_eg, lambd=0.05, verbose=0) |
| | t1 = time.time() |
| | time_borders_gen.append(round(t1 - t0, 4)) |
| |
|
| | |
| | border_points = border_points.to(self.DEVICE) |
| | border_centers = batch_run(repr_model, border_points) |
| | location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, n_epoch), "border_centers.npy") |
| | np.save(location, border_centers) |
| |
|
| | location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, n_epoch), "ori_border_centers.npy") |
| | np.save(location, border_points.cpu().numpy()) |
| |
|
| | num_adv_eg = num |
| | border_points, _, _ = get_border_points(model=self.model, input_x=training_data, confs=confs, predictions=preds, device=self.DEVICE, l_bound=l_bound, num_adv_eg=num_adv_eg, lambd=0.05, verbose=0) |
| |
|
| | |
| | border_points = border_points.to(self.DEVICE) |
| | border_centers = batch_run(repr_model, border_points) |
| | location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_nanme, n_epoch), "test_border_centers.npy") |
| | np.save(location, border_centers) |
| |
|
| | location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, n_epoch), "test_ori_border_centers.npy") |
| | np.save(location, border_points.cpu().numpy()) |
| |
|
| | if self.verbose > 0: |
| | print("Finish generating borders for Epoch {:d}...".format(n_epoch)) |
| | print( |
| | "Average time for generate border points for each iteration: {:.4f}".format(sum(time_borders_gen) / len(time_borders_gen))) |
| |
|
| | |
| | save_dir = os.path.join(self.model_path, "SV_time.json") |
| | if not os.path.exists(save_dir): |
| | evaluation = dict() |
| | else: |
| | f = open(save_dir, "r") |
| | evaluation = json.load(f) |
| | f.close() |
| | evaluation["data_B_gene"] = round(sum(time_borders_gen) / len(time_borders_gen), 3) |
| | with open(save_dir, 'w') as f: |
| | json.dump(evaluation, f) |
| |
|
| | def train_representation(self, iteration, epoch): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "{}_{}".format(self.epoch_name, epoch), "train_data.npy") |
| | lb_idxs = self.get_labeled_idx(iteration) |
| | try: |
| | train_data = np.load(train_data_loc)[lb_idxs] |
| | except Exception as e: |
| | print("no train data saved for Iteration {}".format(iteration)) |
| | train_data = None |
| | return train_data |
| |
|
| | def train_representation_all(self, iteration, epoch): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, epoch), "train_data.npy") |
| | try: |
| | train_data = np.load(train_data_loc) |
| | except Exception as e: |
| | print("no train data saved for Iteration {} Epoch {}".format(iteration, epoch)) |
| | train_data = None |
| | return train_data |
| | |
| | def train_representation_ulb(self, iteration, epoch): |
| | |
| | train_data_loc = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "{}_{}".format(self.epoch_name, epoch), "train_data.npy") |
| | lb_idxs = self.get_labeled_idx(iteration) |
| | try: |
| | train_data = np.load(train_data_loc) |
| | pool_num = len(train_data) |
| | ulb_idx = self.get_unlabeled_idx(pool_num=pool_num, lb_idx=lb_idxs) |
| | train_data = train_data[ulb_idx] |
| | except Exception as e: |
| | print("no train data saved for Iteration {}".format(iteration)) |
| | train_data = None |
| | return train_data |
| | |
| | def train_labels_ulb(self, iteration): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | lb_idxs = self.get_labeled_idx(iteration) |
| | try: |
| | training_labels = torch.load(training_data_loc, map_location="cpu") |
| | ulb_idxs = self.get_unlabeled_idx(len(training_labels), lb_idxs) |
| | training_labels = training_labels[ulb_idxs] |
| | except Exception as e: |
| | print("no train labels saved for Iteration {}".format(iteration)) |
| | training_labels = None |
| | return training_labels.numpy() |
| | |
| | def train_labels(self, iteration): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | index_file = os.path.join(self.model_path, "{}_{:d}".format(self.iteration_name, iteration), "index.json") |
| | lb_idxs = np.array(load_labelled_data_index(index_file)) |
| | try: |
| | training_labels = torch.load(training_data_loc, map_location="cpu") |
| | training_labels = training_labels[lb_idxs] |
| | except Exception as e: |
| | print("no train labels saved for Iteration {}".format(iteration)) |
| | training_labels = None |
| | return training_labels.numpy() |
| |
|
| | def train_labels_all(self): |
| | |
| | training_data_loc = os.path.join(self.content_path, "Training_data", "training_dataset_label.pth") |
| | try: |
| | training_labels = torch.load(training_data_loc, map_location="cpu") |
| | except Exception as e: |
| | print("no train labels saved") |
| | training_labels = None |
| | return training_labels.numpy() |
| |
|
| | def test_representation(self, iteration, epoch): |
| | data_loc = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{}".format(self.epoch_name, epoch), "test_data.npy") |
| | try: |
| | test_data = np.load(data_loc) |
| | index_file = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "test_index.json") |
| | if os.path.exists(index_file): |
| | index = load_labelled_data_index(index_file) |
| | test_data = test_data[index] |
| | except Exception as e: |
| | print("no test data saved for Iteration {} Epoch {}".format(iteration, epoch)) |
| | test_data = None |
| | return test_data |
| |
|
| | def border_representation(self, iteration, epoch): |
| | border_centers_loc = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, epoch), |
| | "border_centers.npy") |
| | try: |
| | border_centers = np.load(border_centers_loc) |
| | except Exception as e: |
| | print("no border points saved for Epoch {}".format(epoch)) |
| | border_centers = np.array([]) |
| | return border_centers |
| | |
| | def test_border_representation(self, iteration, epoch): |
| | border_centers_loc = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, epoch), |
| | "test_border_centers.npy") |
| | try: |
| | border_centers = np.load(border_centers_loc) |
| | except Exception as e: |
| | print("no border points saved for Iteration {} Epoch {}".format(iteration, epoch)) |
| | border_centers = np.array([]) |
| | return border_centers |
| | |
| | def max_norm(self, iteration, epoch): |
| | train_data_loc = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, epoch), "train_data.npy") |
| | index_file = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "index.json") |
| | index = load_labelled_data_index(index_file) |
| | try: |
| | train_data = np.load(train_data_loc) |
| | train_data = train_data[index] |
| | max_x = np.linalg.norm(train_data, axis=1).max() |
| | except Exception as e: |
| | print("no train data saved for Iteration {} Epoch {}".format(iteration, epoch)) |
| | max_x = None |
| | return max_x |
| |
|
| | def prediction_function(self, iteration, epoch): |
| | model_location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, epoch), "subject_model.pth") |
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu"))) |
| | self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | pred_fn = self.model.prediction |
| | return pred_fn |
| |
|
| | def feature_function(self, iteration, epoch): |
| | model_location = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{:d}".format(self.epoch_name, epoch), "subject_model.pth") |
| | self.model.load_state_dict(torch.load(model_location, map_location=torch.device("cpu"))) |
| | self.model = self.model.to(self.DEVICE) |
| | self.model.eval() |
| |
|
| | fea_fn = self.model.feature |
| | return fea_fn |
| |
|
| | def get_pred(self, iteration, epoch, data): |
| | ''' |
| | get the prediction score for data in epoch_id |
| | :param data: numpy.ndarray |
| | :param epoch_id: |
| | :return: pred, numpy.ndarray |
| | ''' |
| | prediction_func = self.prediction_function(iteration, epoch) |
| |
|
| | data = torch.from_numpy(data) |
| | data = data.to(self.DEVICE) |
| | pred = batch_run(prediction_func, data) |
| | return pred |
| |
|
| | def training_accu(self, iteration, epoch): |
| | data = self.train_representation_lb(iteration, epoch) |
| | labels = self.train_labels_lb(iteration) |
| | pred = self.get_pred(iteration, epoch, data).argmax(-1) |
| | val = evaluate_inv_accu(labels, pred) |
| | return val |
| |
|
| | def testing_accu(self, iteration, epoch): |
| | data = self.test_representation(epoch) |
| | labels = self.test_labels(epoch) |
| | test_index_file = os.path.join(self.model_path,"{}_{}".format(self.iteration_name, iteration), "{}_{}".format(self.epoch_name, epoch), "test_index.json") |
| | if os.path.exists(test_index_file): |
| | index = load_labelled_data_index(test_index_file) |
| | labels = labels[index] |
| | pred = self.get_pred(epoch, data).argmax(-1) |
| | val = evaluate_inv_accu(labels, pred) |
| | return val |
| | |
| | def is_deltaB(self, iteration, epoch, data): |
| | """ |
| | check wheter input vectors are lying on delta-boundary or not |
| | :param epoch_id: |
| | :param data: numpy.ndarray |
| | :return: numpy.ndarray, boolean, True stands for is_delta_boundary |
| | """ |
| | preds = self.get_pred(iteration, epoch, data) |
| | border = is_B(preds) |
| | return border |
| | |
| | def single_checkpoint_path(self, iteration, epoch): |
| | path = os.path.join(self.model_path, "{}_{}".format(self.iteration_name, iteration), "{}_{}".format(self.epoch_name, epoch)) |
| | return path |
| |
|
