From 1327de5263b4d5a9818b66e38ffbb45bc85d6756 Mon Sep 17 00:00:00 2001 From: WilliamK112 <164879897+WilliamK112@users.noreply.github.com> Date: Fri, 14 Aug 2026 00:53:20 +0800 Subject: [PATCH] docs: add Great Expectations validation recipe --- CHANGELOG.md | 6 ++ docs/examples.md | 19 +++++ examples/10_great_expectations_recipe.py | 101 +++++++++++++++++++++++ examples/README.md | 4 + 4 files changed, 130 insertions(+) create mode 100644 examples/10_great_expectations_recipe.py diff --git a/CHANGELOG.md b/CHANGELOG.md index aa516356..ec6b3f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added +- A dependency-optional Great Expectations recipe demonstrating the + repair-then-validate workflow with an in-memory checkpoint. + ## [2.0.0] - 2026-07-20 Remediation of the July 2026 v1.2.0 production-readiness audit: the unsafe diff --git a/docs/examples.md b/docs/examples.md index fa17310c..49fec635 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -23,6 +23,25 @@ directory has narrated Jupyter walkthroughs. | `06_large_dataset.py` | Cleaning a large synthetic dataset, with timing | | `07_pandas_integration.py` | Dropping freshdata into an existing pandas workflow | | `08_csv_automation.py` | Batch CSV cleaning automation with audit logs | +| `09_pandera_recipe.py` | Validate before and after cleaning with optional Pandera | +| `10_great_expectations_recipe.py` | Repair string-typed data, then run an optional Great Expectations checkpoint | + +## Great Expectations: repair, then validate + +FreshData and Great Expectations have complementary roles: FreshData repairs +representational problems and records the changes, while Great Expectations +checks the result against an explicit data contract. Great Expectations remains +an optional dependency: + +```bash +pip install great-expectations +python examples/10_great_expectations_recipe.py +``` + +The recipe runs the same checkpoint before and after `fd.clean()`. The raw +currency and boolean strings fail the typed contract; after FreshData converts +them to `float64` and `bool`, the checkpoint passes. The checkpoint validates +the result but does not modify the DataFrame. ## Missing-value cleaning diff --git a/examples/10_great_expectations_recipe.py b/examples/10_great_expectations_recipe.py new file mode 100644 index 00000000..9bd1bcba --- /dev/null +++ b/examples/10_great_expectations_recipe.py @@ -0,0 +1,101 @@ +"""Repair a DataFrame with freshdata, then validate it with GX Core. + +Great Expectations is optional and remains separate from freshdata's core +dependencies. Install it before running this recipe: + + pip install great-expectations + python examples/10_great_expectations_recipe.py +""" + +import os + +import pandas as pd + +import freshdata as fd + +os.environ.setdefault("GX_ANALYTICS_ENABLED", "false") + +import great_expectations as gx # noqa: E402 + + +def make_checkpoint(): + """Build an in-memory checkpoint for the cleaned orders contract.""" + context = gx.get_context(mode="ephemeral") + context.variables.progress_bars = { + "globally": False, + "metric_calculations": False, + } + data_source = context.data_sources.add_pandas(name="freshdata_recipe") + data_asset = data_source.add_dataframe_asset(name="orders") + batch_definition = data_asset.add_batch_definition_whole_dataframe("whole_frame") + + suite = context.suites.add(gx.ExpectationSuite(name="clean_orders")) + suite.add_expectation( + gx.expectations.ExpectColumnValuesToBeOfType( + column="order_amount", + type_="float64", + ) + ) + suite.add_expectation( + gx.expectations.ExpectColumnValuesToBeOfType( + column="active", + type_="bool", + ) + ) + suite.add_expectation( + gx.expectations.ExpectColumnValuesToBeBetween( + column="order_amount", + min_value=0, + ) + ) + + validation_definition = context.validation_definitions.add( + gx.ValidationDefinition( + name="clean_orders_validation", + data=batch_definition, + suite=suite, + ) + ) + return context.checkpoints.add( + gx.Checkpoint( + name="clean_orders_checkpoint", + validation_definitions=[validation_definition], + ) + ) + + +def validate(checkpoint, frame: pd.DataFrame) -> bool: + """Run the checkpoint against one in-memory DataFrame.""" + result = checkpoint.run(batch_parameters={"dataframe": frame}) + return bool(result.success) + + +def main() -> None: + raw = pd.DataFrame( + { + "customer_id": [1, 2, 3, 4, 5], + "order_amount": ["$19.99", "$5.00", "$8.25", "$12.40", "$7.10"], + "active": ["true", "false", "true", "true", "false"], + } + ) + checkpoint = make_checkpoint() + + before = validate(checkpoint, raw) + print(f"Before freshdata: checkpoint passed = {before}") + + cleaned, report = fd.clean( + raw, + id_columns=("customer_id",), + return_report=True, + ) + after = validate(checkpoint, cleaned) + + print(f"After freshdata: checkpoint passed = {after}") + print(report.summary()) + + assert not before, "the raw string values should fail the typed contract" + assert after, "the freshdata-cleaned values should pass the checkpoint" + + +if __name__ == "__main__": + main() diff --git a/examples/README.md b/examples/README.md index 948591df..bedfb1e7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -20,5 +20,9 @@ python examples/01_missing_values.py | [`07_pandas_integration.py`](07_pandas_integration.py) | Dropping freshdata into an existing pandas workflow | | [`08_csv_automation.py`](08_csv_automation.py) | Batch CSV cleaning automation with audit logs | | [`09_pandera_recipe.py`](09_pandera_recipe.py) | Validating with pandera before and after freshdata cleaning | +| [`10_great_expectations_recipe.py`](10_great_expectations_recipe.py) | Repairing with freshdata, then validating through a Great Expectations checkpoint | + +The interoperability recipes keep their validation libraries optional. Install +`pandera` or `great-expectations` only when running the corresponding example. See the [documentation](https://freshcode-org.github.io/freshdata/) for full guides.