Spaces:
Sleeping
Sleeping
File size: 3,144 Bytes
71c85a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | import great_expectations as gx
import pandas as pd
def validate_PaySim_data(df: pd.DataFrame):
"""Validate the processed PaySim frame with Great Expectations (1.x API).
Runs schema + business-logic checks that must pass before training.
Returns the validation result; prints a pass/fail summary.
"""
print('Starting data validation with Great Expectations...')
context = gx.get_context()
batch = (
context.data_sources.add_pandas("paysim")
.add_dataframe_asset("processed")
.add_batch_definition_whole_dataframe("batch")
.get_batch(batch_parameters={"dataframe": df})
)
expectations = [
# --- schema: columns that must exist ---
gx.expectations.ExpectColumnToExist(column="step"),
gx.expectations.ExpectColumnToExist(column="amount"),
gx.expectations.ExpectColumnToExist(column="oldbalanceOrg"), # note: Org, no i
gx.expectations.ExpectColumnToExist(column="newbalanceOrig"),
gx.expectations.ExpectColumnToExist(column="oldbalanceDest"),
gx.expectations.ExpectColumnToExist(column="newbalanceDest"),
gx.expectations.ExpectColumnToExist(column="isFraud"),
gx.expectations.ExpectColumnToExist(column="type_CASH_OUT"),
gx.expectations.ExpectColumnToExist(column="type_TRANSFER"),
gx.expectations.ExpectColumnToExist(column="orig_balance_missing"),
gx.expectations.ExpectColumnToExist(column="dest_balance_missing"),
gx.expectations.ExpectColumnToExist(column="errorBalanceOrig"),
gx.expectations.ExpectColumnToExist(column="errorBalanceDest"),
# --- not null ---
gx.expectations.ExpectColumnValuesToNotBeNull(column="amount"),
gx.expectations.ExpectColumnValuesToNotBeNull(column="errorBalanceOrig"),
gx.expectations.ExpectColumnValuesToNotBeNull(column="errorBalanceDest"),
# --- binary / set membership ---
gx.expectations.ExpectColumnValuesToBeInSet(column="isFraud", value_set=[0, 1]),
gx.expectations.ExpectColumnValuesToBeInSet(column="type_CASH_OUT", value_set=[0, 1]),
gx.expectations.ExpectColumnValuesToBeInSet(column="type_TRANSFER", value_set=[0, 1]),
gx.expectations.ExpectColumnValuesToBeInSet(column="orig_balance_missing", value_set=[0, 1]),
gx.expectations.ExpectColumnValuesToBeInSet(column="dest_balance_missing", value_set=[0, 1]),
# --- ranges ---
gx.expectations.ExpectColumnValuesToBeBetween(column="amount", min_value=0),
gx.expectations.ExpectColumnValuesToBeBetween(column="step", min_value=1, max_value=743),
]
results = []
for exp in expectations:
r = batch.validate(exp)
results.append(r)
status = "PASS" if r.success else "FAIL"
print(f"[{status}] {exp.__class__.__name__} — {getattr(exp, 'column', '')}")
all_passed = all(r.success for r in results)
print(f"\nValidation {'PASSED' if all_passed else 'FAILED'} "
f"({sum(r.success for r in results)}/{len(results)} checks)")
return results |