| | import pandas as pd |
| | import numpy as np |
| | from warnings import warn |
| |
|
| | |
| |
|
| |
|
| | def check_missing(data,output_path=None): |
| | """ |
| | check the total number & percentage of missing values |
| | per variable of a pandas Dataframe |
| | """ |
| | |
| | result = pd.concat([data.isnull().sum(),data.isnull().mean()],axis=1) |
| | result = result.rename(index=str,columns={0:'total missing',1:'proportion'}) |
| | if output_path is not None: |
| | result.to_csv(output_path+'missing.csv') |
| | print('result saved at', output_path, 'missing.csv') |
| | return result |
| |
|
| |
|
| | def drop_missing(data,axis=0): |
| | """ |
| | Listwise deletion: |
| | excluding all cases (listwise) that have missing values |
| | |
| | Parameters |
| | ---------- |
| | axis: drop cases(0)/columns(1),default 0 |
| | |
| | Returns |
| | ------- |
| | Pandas dataframe with missing cases/columns dropped |
| | """ |
| | |
| | data_copy = data.copy(deep=True) |
| | data_copy = data_copy.dropna(axis=axis,inplace=False) |
| | return data_copy |
| | |
| |
|
| | def add_var_denote_NA(data,NA_col=[]): |
| | """ |
| | creating an additional variable indicating whether the data |
| | was missing for that observation (1) or not (0). |
| | """ |
| | |
| | data_copy = data.copy(deep=True) |
| | for i in NA_col: |
| | if data_copy[i].isnull().sum()>0: |
| | data_copy[i+'_is_NA'] = np.where(data_copy[i].isnull(),1,0) |
| | else: |
| | warn("Column %s has no missing cases" % i) |
| | |
| | return data_copy |
| |
|
| |
|
| | def impute_NA_with_arbitrary(data,impute_value,NA_col=[]): |
| | """ |
| | replacing NA with arbitrary values. |
| | """ |
| | |
| | data_copy = data.copy(deep=True) |
| | for i in NA_col: |
| | if data_copy[i].isnull().sum()>0: |
| | data_copy[i+'_'+str(impute_value)] = data_copy[i].fillna(impute_value) |
| | else: |
| | warn("Column %s has no missing cases" % i) |
| | return data_copy |
| |
|
| |
|
| | def impute_NA_with_avg(data,strategy='mean',NA_col=[]): |
| | """ |
| | replacing the NA with mean/median/most frequent values of that variable. |
| | Note it should only be performed over training set and then propagated to test set. |
| | """ |
| | |
| | data_copy = data.copy(deep=True) |
| | for i in NA_col: |
| | if data_copy[i].isnull().sum()>0: |
| | if strategy=='mean': |
| | data_copy[i+'_impute_mean'] = data_copy[i].fillna(data[i].mean()) |
| | elif strategy=='median': |
| | data_copy[i+'_impute_median'] = data_copy[i].fillna(data[i].median()) |
| | elif strategy=='mode': |
| | data_copy[i+'_impute_mode'] = data_copy[i].fillna(data[i].mode()[0]) |
| | else: |
| | warn("Column %s has no missing" % i) |
| | return data_copy |
| |
|
| |
|
| | def impute_NA_with_end_of_distribution(data,NA_col=[]): |
| | """ |
| | replacing the NA by values that are at the far end of the distribution of that variable |
| | calculated by mean + 3*std |
| | """ |
| | |
| | data_copy = data.copy(deep=True) |
| | for i in NA_col: |
| | if data_copy[i].isnull().sum()>0: |
| | data_copy[i+'_impute_end_of_distri'] = data_copy[i].fillna(data[i].mean()+3*data[i].std()) |
| | else: |
| | warn("Column %s has no missing" % i) |
| | return data_copy |
| | |
| |
|
| | def impute_NA_with_random(data,NA_col=[],random_state=0): |
| | """ |
| | replacing the NA with random sampling from the pool of available observations of the variable |
| | """ |
| | |
| | data_copy = data.copy(deep=True) |
| | for i in NA_col: |
| | if data_copy[i].isnull().sum()>0: |
| | data_copy[i+'_random'] = data_copy[i] |
| | |
| | random_sample = data_copy[i].dropna().sample(data_copy[i].isnull().sum(), random_state=random_state) |
| | random_sample.index = data_copy[data_copy[i].isnull()].index |
| | data_copy.loc[data_copy[i].isnull(), str(i)+'_random'] = random_sample |
| | else: |
| | warn("Column %s has no missing" % i) |
| | return data_copy |
| | |
