Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| def preprocess_data(df:pd.DataFrame): | |
| """Clean and feature-engineer the PaySim dataset for a flat-row fraud model. | |
| Steps, in order: | |
| - Copies the input (no mutation of the caller's frame) | |
| - Trims whitespace from column names | |
| - Drops duplicate rows and the near-useless isFlaggedFraud column | |
| - Adds orig/dest balance-missing flags (data-quality markers: balance | |
| is a placeholder zero while amount is non-zero) | |
| - Drops nameOrig/nameDest (unique per row, no signal for a flat model) | |
| - Derives errorBalanceOrig/errorBalanceDest (accounting inconsistencies; | |
| highest-signal engineered features) | |
| - Keeps only TRANSFER and CASH_OUT rows, the only types where fraud | |
| occurs in PaySim (first-pass choice) | |
| - One-hot encodes the remaining transaction types | |
| Note: does not split, scale, or handle class imbalance — do those | |
| downstream on the training split only, to avoid leakage. | |
| Args: | |
| df: raw PaySim dataframe | |
| Returns: | |
| cleaned, feature-engineered dataframe (features + isFraud label) | |
| """ | |
| df = df.copy() | |
| df.columns = df.columns.str.strip() # remove leading/trailing whitespaces | |
| #There were no duplicates but remove anyway just in case | |
| df = df.drop_duplicates() | |
| #the column isFlaggedFraud is near useless so its better to remove | |
| df = df.drop(columns=['isFlaggedFraud'], errors = 'ignore') | |
| #a classic case of fraud is when the balance is zero and a transaction happens | |
| #so we create a new column of boolean (0/1) | |
| df['orig_balance_missing'] = ((df['oldbalanceOrg'] == 0) & | |
| (df['newbalanceOrig'] ==0) & | |
| (df['amount'] > 0)).astype(int) | |
| df['dest_balance_missing'] = ((df['oldbalanceDest'] == 0) & | |
| (df['newbalanceDest'] ==0) & | |
| (df['amount'] > 0)).astype(int) | |
| #names are noise since they are unique | |
| df = df.drop(columns=['nameOrig','nameDest'],errors = 'ignore') | |
| #Balance-error features are the highest-signal engineered columns. | |
| # Rather than "cleaning," the standard move is deriving | |
| df['errorBalanceOrig'] = df['newbalanceOrig'] + df['amount'] - df['oldbalanceOrg'] | |
| df['errorBalanceDest'] = df['oldbalanceDest'] + df['amount'] - df['newbalanceDest'] | |
| #I will also keep only the two types of transaction that have frauds (Transfer and cash out) | |
| #at least for the first pass | |
| df = df[df['type'].isin(['TRANSFER','CASH_OUT'])].copy() | |
| #Last important to use one hot encoding in the type since it is varchars and four classes | |
| df = pd.get_dummies(df, columns=['type'], dtype=int) | |
| return df |