From ee69a4e61cfc81cdcaeb91b2297920ff6c63744e Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 18 Jan 2026 09:36:23 +0100 Subject: [PATCH 1/6] added best practices docs, improved tests --- docs/dqx/docs/demos.mdx | 2 +- docs/dqx/docs/dev/contributing.mdx | 4 + docs/dqx/docs/dev/docs_authoring.mdx | 4 + docs/dqx/docs/dev/index.mdx | 2 +- .../docs/guide/additional_configuration.mdx | 2 +- .../ai_assisted_primary_key_detection.mdx | 2 +- .../ai_assisted_quality_checks_generation.mdx | 2 +- docs/dqx/docs/guide/best_practices.mdx | 217 ++++++++++++++++++ ...data_contract_quality_rules_generation.mdx | 2 +- docs/dqx/docs/guide/data_profiling.mdx | 2 +- docs/dqx/docs/guide/index.mdx | 2 +- docs/dqx/docs/guide/quality_checks_apply.mdx | 2 +- .../docs/guide/quality_checks_definition.mdx | 2 +- .../dqx/docs/guide/quality_checks_storage.mdx | 2 +- docs/dqx/docs/guide/quality_dashboard.mdx | 2 +- docs/dqx/docs/guide/summary_metrics.mdx | 2 +- docs/dqx/docs/installation.mdx | 2 +- docs/dqx/docs/motivation.mdx | 2 +- docs/dqx/docs/reference/api/index.mdx | 2 +- docs/dqx/docs/reference/benchmarks.mdx | 2 +- docs/dqx/docs/reference/cli.mdx | 2 +- docs/dqx/docs/reference/engine.mdx | 2 +- docs/dqx/docs/reference/index.mdx | 2 +- docs/dqx/docs/reference/profiler.mdx | 2 +- docs/dqx/docs/reference/quality_checks.mdx | 2 +- docs/dqx/docs/reference/testing.mdx | 2 +- tests/integration/test_ai_rules_generator.py | 2 +- tests/perf/generate_md_report.py | 2 +- 28 files changed, 250 insertions(+), 25 deletions(-) create mode 100644 docs/dqx/docs/guide/best_practices.mdx diff --git a/docs/dqx/docs/demos.mdx b/docs/dqx/docs/demos.mdx index 6df2c0cbe..0f0ab6f20 100644 --- a/docs/dqx/docs/demos.mdx +++ b/docs/dqx/docs/demos.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 10 +sidebar_position: 400 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index 8a6b0983a..e83533200 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -1,3 +1,7 @@ +--- +sidebar_position: 601 +--- + import Admonition from '@theme/Admonition'; # Contributing diff --git a/docs/dqx/docs/dev/docs_authoring.mdx b/docs/dqx/docs/dev/docs_authoring.mdx index f38c76ccb..7733b5d43 100644 --- a/docs/dqx/docs/dev/docs_authoring.mdx +++ b/docs/dqx/docs/dev/docs_authoring.mdx @@ -1,3 +1,7 @@ +--- +sidebar_position: 602 +--- + import Admonition from '@theme/Admonition'; # Authoring Documentation diff --git a/docs/dqx/docs/dev/index.mdx b/docs/dqx/docs/dev/index.mdx index dfa3ce3dd..a392c2518 100644 --- a/docs/dqx/docs/dev/index.mdx +++ b/docs/dqx/docs/dev/index.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 12 +sidebar_position: 600 --- # Contributing to DQX diff --git a/docs/dqx/docs/guide/additional_configuration.mdx b/docs/dqx/docs/guide/additional_configuration.mdx index 7c74297bf..024abfd82 100644 --- a/docs/dqx/docs/guide/additional_configuration.mdx +++ b/docs/dqx/docs/guide/additional_configuration.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 10 +sidebar_position: 310 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/ai_assisted_primary_key_detection.mdx b/docs/dqx/docs/guide/ai_assisted_primary_key_detection.mdx index 5688c87ee..70224dfe3 100644 --- a/docs/dqx/docs/guide/ai_assisted_primary_key_detection.mdx +++ b/docs/dqx/docs/guide/ai_assisted_primary_key_detection.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 6 +sidebar_position: 303 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/ai_assisted_quality_checks_generation.mdx b/docs/dqx/docs/guide/ai_assisted_quality_checks_generation.mdx index 811e1c5cc..bc1181ce1 100644 --- a/docs/dqx/docs/guide/ai_assisted_quality_checks_generation.mdx +++ b/docs/dqx/docs/guide/ai_assisted_quality_checks_generation.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 5 +sidebar_position: 302 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/best_practices.mdx b/docs/dqx/docs/guide/best_practices.mdx new file mode 100644 index 000000000..1b71ecb86 --- /dev/null +++ b/docs/dqx/docs/guide/best_practices.mdx @@ -0,0 +1,217 @@ +--- +sidebar_position: 311 +--- + +import Admonition from '@theme/Admonition'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Best Practices in Production + +This page provides practical guidance and best practices for using DQX in production, ensuring reliable, scalable, and maintainable data quality validation across your data pipelines and datasets. + +## Management of Data Quality Rules + +### Store checks in a Delta table + +DQX offers built-in support for storing and retrieving data quality rules (checks) from various storage backends such as Delta tables, YAML or JSON files, and others. +It is recommended to maintain checks in a Delta table to centralize data quality checks across tables, improve maintainability and discoverability. + +```python +# load checks in Delta table +checks: list[dict] = dq_engine.load_checks(config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.default.input_table")) + +# save checks from a Delta table +dq_engine.save_checks(checks, config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.default.input_table", mode="overwrite")) +``` + + +For governance purposes, it may be necessary to maintain separate tables for storing checks based on domains, teams, or environments, ensuring alignment with organizational policies and access controls. + + +See more in [Storing Checks](/docs/guide/quality_checks_storage/). + +### Use a common set of checks for groups of tables + +Define reusable rule sets that apply to multiple related tables (e.g. tables belonging to the same domain, data product or schema). +Complement these with specific checks for individual tables to capture specialized requirements. +This helps minimize duplication while ensuring domain-specific validations are properly enforced. + +```python +sales_checks: list[dict] = dq_engine.load_checks(config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="sales")) +orders_checks: list[dict] = dq_engine.load_checks(config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.sales.orders")) +all_checks = sales_checks + orders_checks +``` + +See more in [Storing Checks](/docs/guide/quality_checks_storage/). + +Leverage the `run_config_name` parameter to organize and retrieve checks based on table groups or individual tables. + +### Use Rules with Appropriate Granularity + +Prioritize row-level checks as they provide the best performance and granularity (check data quality of individual rows). +Dataset-level rules are applied to group of rows and are valuable for aggregates and cross-record relationships, but should be used to supplement row-level rules. + +See more in [Quality Checks](/docs/reference/quality_checks/). + +### Use Rule Types that best fits the Purpose + +DQX supports different categories of quality checks. Use them appropriately: +* Rule-based checks: deterministic rules that use built-in or custom checks rule functions. +* AI-assisted checks: leverage probabilistic LLM-driven suggestions for creating rule-based business or technical checks using natural language as input. +* Anomaly detection: useful for identifying outliers that are not covered by explicit rule-based checks. +* Profiler-generated checks: use profiling to bootstrap an initial set of deterministic rule-based checks. + +See more in [Quality Checks](/docs/reference/quality_checks/). + +### Prioritize high-impact fields + +Focus your data quality efforts on the fields that matter most for business outcomes and downstream processes: +* Critical business keys and identifiers (e.g., order_id, customer_id) +* Timestamps and date fields used in analytics or SLAs +* Columns used in joins, aggregations, or financial calculations +* Columns with known historical quality issues that have caused problems in the past +* Columns required to meet regulatory or compliance standards +* Columns that directly impact customer experience or operational efficiency +* Columns with high usage frequency in reports or dashboards +* Columns that matter to the business + +This ensures that quality checks focus on business-critical data, reduces noise from low-value validations, and simplifies maintenance by limiting the number of checks that need to be managed. + +### Increase Rules Reuse with Custom Checks + +Avoid duplicating complex SQL expressions (`sql_expression` check function) across multiple rules. +When the same validation logic needs to be applied for multiple tables, encapsulate the logic in a custom rule function instead of copy-pasting SQL expressions. +This will improve maintainability and consistency across your data quality checks. + +See more in [Custom Quality Checks](/docs/reference/quality_checks/#creating-custom-row-level-checks). + +### Profile and Tune Rules over Time + +Bootstrap rule candidates: +* Use the profiler to generate initial quality rules for new datasets and periodically re-profile existing datasets to detect changes or shifts in data distributions. +* Leverage AI-assisted quality rule generation to translate technical or business requirements expressed in natural language into DQX-specific rules, accelerating rule creation. + +Continuously refine and tune rules: +* Monitor data quality metrics (pass/fail rates, trends, and anomalies) to identify areas for improvement and reduce false positives while maintaining strong coverage. +* Add or tune checks (e.g. update thresholds) immediately when production issues are detected, preventing recurring data failures. +* Retire or relax obsolete rules as business logic or data evolve. Maintain versioning to track the history and evolution of rules over time. + +See more in [Profiling Guide](/docs/guide/data_profiling/) and [AI-Assisted Rules Generation](/docs/guide/ai_assisted_quality_checks_generation). + +### Consumer-Specific Rules + +Different consumers of the same data may have different quality expectations. +For "gold" layer tables, you may want to define different sets of rules depending on the downstream applications (e.g. reporting, ML, data science). + +## Usage + +### Workflows vs Embedded Usage + +DQX offers different execution modes. Choose the one that best fits your use case: +* Workflows for no-code data quality validation, run as a background process. Suitable for post-factum monitoring only on already persisted data. Provides ease of use and quick setup, but limited flexibility. +* Embedded usage for integrating data quality checks directly into your data pipelines. Suitable for in-transit and real-time validation during data processing. Can also be used for post-factum monitoring. Offers the best flexibility and control, but requires coding. + +See more in [Getting Started](/docs/guide/). + +### Actions on Quality Checks Failures + +Quarantine or Flag based on criticality: +* For critical data, quarantine bad records (use functions with split, e.g. `apply_checks_and_split`) so they are not propagated downstream. This allows for targeted remediation (curation) of low-quality data. +* For non-critical data, flag records (e.g. use functions without split, e.g. `apply_checks`), but allow progress with alerts or downstream monitoring. + +See more in [Applying Quality Checks Guide](/docs/guide/quality_checks_apply/) + +### Apply All Checks in One Pass + +For optimal performance and scalability, apply all relevant quality checks for a given DataFrame or table in a single execution pass. +Avoid multiple sequential quality runs on the same dataset unless strictly required. +* Quality checks are executed in a distributed manner across the Spark cluster, so grouping them minimizes redundant scans. +* Running checks together reduces compute cost and improves overall pipeline efficiency. +* You can group different type of checks (row-level + dataset-level + anomaly detection) and apply them at once for maximum efficiency. + +Example: +```python +# load checks for the table +checks: list[dict] = dq_engine.load_checks(config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.default.input_table")) + +# apply all checks in one pass +results = dq_engine.apply_checks(df, checks) +``` + +### Scale Checks Across Multiple Tables + +When applying similar quality rules to multiple tables, use DQX methods designed for multi-table execution. +Use built-in DQX apply methods that support applying checks across multiple tables in a single call (e.g., `apply_checks_and_save_in_tables` or `apply_checks_and_save_in_tables_for_patterns`) + +See more in [Applying Checks on Multiple Tables](/docs/guide/quality_checks_apply/#applying-checks-on-multiple-tables). + +### Implement Alerting and setup Data Quality Dashboards + +Establish Service Level Agreements (SLAs) for critical data quality metrics such as the percentage of rows passing key checks, and configure automated alerts to notify teams when SLAs are breached. +This ensures that data quality issues are detected and addressed promptly. + +Monitor and track quality trends: +* Leverage [DQX Dashboards](/docs/guide/quality_dashboard) for ongoing visibility into data quality metrics +* Use the [Summary Metrics](/docs/guide/summary_metrics/) table provided by DQX to capture aggregate statistics from data quality checking. This enables you to track data quality trends over time and gain insights into data quality across all tables. + +## Deployment + +### Version Rules and Automated Deployment + +Deploy checks across environments systematically: +* Check rules into Git and deploy them as Delta tables. +* Align deployment of DQX checks with your pipeline release process (e.g. using Databricks Assets Bundle). +* Use environment-specific configs when promoting checks from dev → qa → prod. +* Treat data quality rules as living assets: version them, review them regularly, and evolve them alongside the data products they protect. + +Version your rules to track changes over time and ensure reproducibility. Leverage `user_metadata` or `run_config_name` to tag rules with version information. +To improve traceability, you can include the location of the checks definition in `user_metadata` of each rule. + +```python +# version individual rules +checks = """ +- criticality: error + check: + function: is_not_null + arguments: + column: col1 + user_metadata: + version: v1 + location: catalog.schema.checks_table +""" + +# version rules set +dq_engine.save_checks( + checks, + config=TableChecksStorageConfig( + location="catalog.schema.checks_table", + run_config_name="main.default.input_table_v1", + mode="overwrite" + ) +) +``` + +See more in [Installation](/docs/installation/). + +### Pin DQX Version + +To ensure consistent behavior and avoid unexpected issues from automatic upgrades, always pin DQX to a specific version when installing: +* Pip installation: `pip install databricks-labs-dqx==0.9.3` +* Databricks CLI: `databricks labs install dqx@v0.9.3` + + +Review breaking changes before upgrading to avoid disruptions. See `BREAKING CHANGES!` section in [Releases](https://github.com/databrickslabs/dqx/releases). +Always test upgrades in a non-production environment first. + + +### Test Rules in Lower Environments + +Avoid production disruptions by validating new or updated rules in dev/test environments before production deployment. +Use sample or synthetic datasets to test edge cases and confirm rule behavior. For critical checks, incorporate automated regression tests. + +### Use Custom Installation Folder for Workflows + +Use custom installation folder for workflows to isolate DQX dependencies from other processes in your environment. + +See more in [Installation](/docs/installation/). \ No newline at end of file diff --git a/docs/dqx/docs/guide/data_contract_quality_rules_generation.mdx b/docs/dqx/docs/guide/data_contract_quality_rules_generation.mdx index 8ad47fce0..22e748bc0 100644 --- a/docs/dqx/docs/guide/data_contract_quality_rules_generation.mdx +++ b/docs/dqx/docs/guide/data_contract_quality_rules_generation.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 6 +sidebar_position: 304 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/data_profiling.mdx b/docs/dqx/docs/guide/data_profiling.mdx index ac30aa3ad..a4f110259 100644 --- a/docs/dqx/docs/guide/data_profiling.mdx +++ b/docs/dqx/docs/guide/data_profiling.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 4 +sidebar_position: 301 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/index.mdx b/docs/dqx/docs/guide/index.mdx index 5526e1d96..09a722a6a 100644 --- a/docs/dqx/docs/guide/index.mdx +++ b/docs/dqx/docs/guide/index.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 3 +sidebar_position: 300 title: User Guide --- diff --git a/docs/dqx/docs/guide/quality_checks_apply.mdx b/docs/dqx/docs/guide/quality_checks_apply.mdx index 5764402d7..e51bc4b90 100644 --- a/docs/dqx/docs/guide/quality_checks_apply.mdx +++ b/docs/dqx/docs/guide/quality_checks_apply.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 8 +sidebar_position: 307 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/quality_checks_definition.mdx b/docs/dqx/docs/guide/quality_checks_definition.mdx index 3d49839b6..969e4b38c 100644 --- a/docs/dqx/docs/guide/quality_checks_definition.mdx +++ b/docs/dqx/docs/guide/quality_checks_definition.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 7 +sidebar_position: 306 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/quality_checks_storage.mdx b/docs/dqx/docs/guide/quality_checks_storage.mdx index 04f4730e9..225afeb1d 100644 --- a/docs/dqx/docs/guide/quality_checks_storage.mdx +++ b/docs/dqx/docs/guide/quality_checks_storage.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 6 +sidebar_position: 305 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/quality_dashboard.mdx b/docs/dqx/docs/guide/quality_dashboard.mdx index 8bf5bb60e..8304a07e7 100644 --- a/docs/dqx/docs/guide/quality_dashboard.mdx +++ b/docs/dqx/docs/guide/quality_dashboard.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 9 +sidebar_position: 309 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/guide/summary_metrics.mdx b/docs/dqx/docs/guide/summary_metrics.mdx index 25c5d5ac2..b6b7946dd 100644 --- a/docs/dqx/docs/guide/summary_metrics.mdx +++ b/docs/dqx/docs/guide/summary_metrics.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 8 +sidebar_position: 308 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index 020fd6601..73a793f48 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 200 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/motivation.mdx b/docs/dqx/docs/motivation.mdx index 2fbdf8b3b..ff2f5a2c6 100644 --- a/docs/dqx/docs/motivation.mdx +++ b/docs/dqx/docs/motivation.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 1 +sidebar_position: 100 --- import useBaseUrl from '@docusaurus/useBaseUrl'; diff --git a/docs/dqx/docs/reference/api/index.mdx b/docs/dqx/docs/reference/api/index.mdx index d0291ea59..d3230acea 100644 --- a/docs/dqx/docs/reference/api/index.mdx +++ b/docs/dqx/docs/reference/api/index.mdx @@ -1,6 +1,6 @@ --- title: API Reference -sidebar_position: 12 +sidebar_position: 506 --- # API Reference diff --git a/docs/dqx/docs/reference/benchmarks.mdx b/docs/dqx/docs/reference/benchmarks.mdx index 94bbe6cd5..74b6fe7c6 100644 --- a/docs/dqx/docs/reference/benchmarks.mdx +++ b/docs/dqx/docs/reference/benchmarks.mdx @@ -2,7 +2,7 @@ title: Benchmarks -sidebar_position: 13 +sidebar_position: 507 --- diff --git a/docs/dqx/docs/reference/cli.mdx b/docs/dqx/docs/reference/cli.mdx index 6faa46d7c..6f2548077 100644 --- a/docs/dqx/docs/reference/cli.mdx +++ b/docs/dqx/docs/reference/cli.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 11 +sidebar_position: 505 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/reference/engine.mdx b/docs/dqx/docs/reference/engine.mdx index 05e473be3..02440a5a5 100644 --- a/docs/dqx/docs/reference/engine.mdx +++ b/docs/dqx/docs/reference/engine.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 1 +sidebar_position: 502 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/reference/index.mdx b/docs/dqx/docs/reference/index.mdx index 60c8a48c2..bd8ffbda9 100644 --- a/docs/dqx/docs/reference/index.mdx +++ b/docs/dqx/docs/reference/index.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 11 +sidebar_position: 500 title: Reference --- diff --git a/docs/dqx/docs/reference/profiler.mdx b/docs/dqx/docs/reference/profiler.mdx index 8b6b35f89..29b5971e2 100644 --- a/docs/dqx/docs/reference/profiler.mdx +++ b/docs/dqx/docs/reference/profiler.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 503 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 0093efb1d..e97a13707 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 3 +sidebar_position: 501 --- import Admonition from '@theme/Admonition'; diff --git a/docs/dqx/docs/reference/testing.mdx b/docs/dqx/docs/reference/testing.mdx index be2349f66..789d39b07 100644 --- a/docs/dqx/docs/reference/testing.mdx +++ b/docs/dqx/docs/reference/testing.mdx @@ -1,5 +1,5 @@ --- -sidebar_position: 4 +sidebar_position: 504 --- import Admonition from '@theme/Admonition'; diff --git a/tests/integration/test_ai_rules_generator.py b/tests/integration/test_ai_rules_generator.py index badc8dab3..9b89f585e 100644 --- a/tests/integration/test_ai_rules_generator.py +++ b/tests/integration/test_ai_rules_generator.py @@ -134,7 +134,7 @@ def test_generate_dq_rules_ai_assisted_with_is_not_equal_to_str(ws, spark): def test_generate_dq_rules_ai_assisted_with_sql_expression(ws, spark): - user_input = "Users email must not end with @gmail.com checked using sql expression, skip msg." + user_input = "Users email must not end with @gmail.com checked using sql expression with 'NOT LIKE', skip msg." generator = DQGenerator(ws, spark) actual_checks = generator.generate_dq_rules_ai_assisted(user_input=user_input) diff --git a/tests/perf/generate_md_report.py b/tests/perf/generate_md_report.py index 8d52eee64..031250463 100644 --- a/tests/perf/generate_md_report.py +++ b/tests/perf/generate_md_report.py @@ -24,7 +24,7 @@ lines = [] lines.append("---\n") lines.append("title: Benchmarks\n") -lines.append("sidebar_position: 13\n") +lines.append("sidebar_position: 507\n") lines.append("---\n") lines.append("# Performance Benchmarks Report\n") From 3aaaf1647a4ebe87dcd3d9d2a84b70db2df33730 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 18 Jan 2026 09:44:46 +0100 Subject: [PATCH 2/6] co-pilot feedback implementation --- docs/dqx/docs/guide/best_practices.mdx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/dqx/docs/guide/best_practices.mdx b/docs/dqx/docs/guide/best_practices.mdx index 1b71ecb86..3184cd6cc 100644 --- a/docs/dqx/docs/guide/best_practices.mdx +++ b/docs/dqx/docs/guide/best_practices.mdx @@ -3,8 +3,6 @@ sidebar_position: 311 --- import Admonition from '@theme/Admonition'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; # Best Practices in Production @@ -18,10 +16,10 @@ DQX offers built-in support for storing and retrieving data quality rules (check It is recommended to maintain checks in a Delta table to centralize data quality checks across tables, improve maintainability and discoverability. ```python -# load checks in Delta table +# load checks from Delta table checks: list[dict] = dq_engine.load_checks(config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.default.input_table")) -# save checks from a Delta table +# save checks to Delta table dq_engine.save_checks(checks, config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.default.input_table", mode="overwrite")) ``` @@ -142,7 +140,7 @@ results = dq_engine.apply_checks(df, checks) ### Scale Checks Across Multiple Tables When applying similar quality rules to multiple tables, use DQX methods designed for multi-table execution. -Use built-in DQX apply methods that support applying checks across multiple tables in a single call (e.g., `apply_checks_and_save_in_tables` or `apply_checks_and_save_in_tables_for_patterns`) +Use built-in DQX apply methods that support applying checks across multiple tables in a single call (e.g., `apply_checks_and_save_in_tables` or `apply_checks_and_save_in_tables_for_patterns`). See more in [Applying Checks on Multiple Tables](/docs/guide/quality_checks_apply/#applying-checks-on-multiple-tables). From 7b92d116d0bbf97ec57b38abcf67ffeb3a34aa61 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 18 Jan 2026 12:46:22 +0100 Subject: [PATCH 3/6] fixed handling of data and datetime limits provided as string updated docs --- docs/dqx/docs/guide/best_practices.mdx | 8 +- src/databricks/labs/dqx/check_funcs.py | 16 ++++ tests/integration/test_row_checks.py | 128 +++++++++++++++++++------ 3 files changed, 121 insertions(+), 31 deletions(-) diff --git a/docs/dqx/docs/guide/best_practices.mdx b/docs/dqx/docs/guide/best_practices.mdx index 3184cd6cc..775234038 100644 --- a/docs/dqx/docs/guide/best_practices.mdx +++ b/docs/dqx/docs/guide/best_practices.mdx @@ -118,7 +118,7 @@ Quarantine or Flag based on criticality: * For critical data, quarantine bad records (use functions with split, e.g. `apply_checks_and_split`) so they are not propagated downstream. This allows for targeted remediation (curation) of low-quality data. * For non-critical data, flag records (e.g. use functions without split, e.g. `apply_checks`), but allow progress with alerts or downstream monitoring. -See more in [Applying Quality Checks Guide](/docs/guide/quality_checks_apply/) +See more in [Applying Quality Checks Guide](/docs/guide/quality_checks_apply/). ### Apply All Checks in One Pass @@ -174,9 +174,9 @@ checks = """ function: is_not_null arguments: column: col1 - user_metadata: - version: v1 - location: catalog.schema.checks_table + user_metadata: + version: v1 + location: catalog.schema.checks_table """ # version rules set diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index f88dda841..7e871c4fd 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -2995,6 +2995,22 @@ def get_limit_expr( raise MissingParameterError("Limit is not provided.") if isinstance(limit, str): + if limit.isdigit(): + return F.expr(limit) + + try: + parsed_dt = datetime.datetime.fromisoformat(limit) + + # Check if the string contains time component + has_time = ':' in limit + + if has_time: + return F.to_timestamp(F.lit(parsed_dt)) + return F.to_date(F.lit(parsed_dt.date())) + except ValueError: + # If parsing fails, treat as an expression + pass + return F.expr(limit) if isinstance(limit, Column): return limit diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 86a68c5fc..9b47bf0c8 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -798,10 +798,13 @@ def test_col_is_not_less_than(spark, set_utc_timezone): actual = test_df.select( is_not_less_than("a", 2), + is_not_less_than("a", "2"), is_not_less_than("a", F.col("b") * 2), is_not_less_than("b", "a"), is_not_less_than("c", datetime(2025, 2, 1).date()), + is_not_less_than("c", "2025-02-01"), is_not_less_than("d", datetime(2025, 2, 1)), + is_not_less_than("d", "2025-02-01"), is_not_less_than("e", 2), is_not_less_than(F.try_element_at("f", F.lit(1)), 2), is_not_less_than(F.col("g").getItem("val"), 2), @@ -809,9 +812,10 @@ def test_col_is_not_less_than(spark, set_utc_timezone): ) checked_schema = ( - "a_less_than_limit: string, a_less_than_limit: string, b_less_than_limit: string, " - "c_less_than_limit: string, d_less_than_limit: string, e_less_than_limit: string, " - "try_element_at_f_1_less_than_limit: string, " + "a_less_than_limit: string, a_less_than_limit: string, a_less_than_limit: string, " + "b_less_than_limit: string, c_less_than_limit: string, c_less_than_limit: string, " + "d_less_than_limit: string, d_less_than_limit: string, " + "e_less_than_limit: string, try_element_at_f_1_less_than_limit: string, " "unresolvedextractvalue_g_val_less_than_limit: string, " "h_less_than_limit: string" ) @@ -819,10 +823,13 @@ def test_col_is_not_less_than(spark, set_utc_timezone): expected = spark.createDataFrame( [ [ + "Value '1' in Column 'a' is less than limit: 2", "Value '1' in Column 'a' is less than limit: 2", None, None, "Value '2025-01-01' in Column 'c' is less than limit: 2025-02-01", + "Value '2025-01-01' in Column 'c' is less than limit: 2025-02-01", + "Value '2025-01-01 00:00:00' in Column 'd' is less than limit: 2025-02-01 00:00:00", "Value '2025-01-01 00:00:00' in Column 'd' is less than limit: 2025-02-01 00:00:00", "Value '1.00' in Column 'e' is less than limit: 2", "Value '1' in Column 'try_element_at(f, 1)' is less than limit: 2", @@ -830,17 +837,21 @@ def test_col_is_not_less_than(spark, set_utc_timezone): "Value '1.2' in Column 'h' is less than limit: 2.4", ], [ + None, None, "Value '2' in Column 'a' is less than limit: 8", None, None, None, + None, + None, "Value '1.99' in Column 'e' is less than limit: 2", None, None, None, ], [ + None, None, "Value '4' in Column 'a' is less than limit: 6", "Value '3' in Column 'b' is less than limit: 4", @@ -850,8 +861,10 @@ def test_col_is_not_less_than(spark, set_utc_timezone): None, None, None, + None, + None, ], - [None, None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None, None], ], checked_schema, ) @@ -873,44 +886,54 @@ def test_col_is_not_greater_than(spark, set_utc_timezone): actual = test_df.select( is_not_greater_than("a", 1), + is_not_greater_than("a", "1"), is_not_greater_than("a", F.col("b") * 2), is_not_greater_than("b", "a"), is_not_greater_than("c", datetime(2025, 1, 1).date()), + is_not_greater_than("c", "2025-01-01"), is_not_greater_than("d", datetime(2025, 1, 1)), + is_not_greater_than("d", "2025-01-01"), is_not_greater_than("e", 1), is_not_greater_than(F.try_element_at("f", F.lit(1)), 1), is_not_greater_than("g", 2.4), ) checked_schema = ( - "a_greater_than_limit: string, a_greater_than_limit: string, b_greater_than_limit: string, " - "c_greater_than_limit: string, d_greater_than_limit: string, e_greater_than_limit: string, " + "a_greater_than_limit: string, a_greater_than_limit: string, a_greater_than_limit: string, " + "b_greater_than_limit: string, c_greater_than_limit: string, c_greater_than_limit: string, " + "d_greater_than_limit: string, d_greater_than_limit: string, e_greater_than_limit: string, " "try_element_at_f_1_greater_than_limit: string, g_greater_than_limit: string" ) expected = spark.createDataFrame( [ - [None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None], [ + "Value '2' in Column 'a' is greater than limit: 1", "Value '2' in Column 'a' is greater than limit: 1", None, "Value '4' in Column 'b' is greater than limit: 2", "Value '2025-02-01' in Column 'c' is greater than limit: 2025-01-01", + "Value '2025-02-01' in Column 'c' is greater than limit: 2025-01-01", + "Value '2025-02-01 00:00:00' in Column 'd' is greater than limit: 2025-01-01 00:00:00", "Value '2025-02-01 00:00:00' in Column 'd' is greater than limit: 2025-01-01 00:00:00", "Value '1.01' in Column 'e' is greater than limit: 1", "Value '2' in Column 'try_element_at(f, 1)' is greater than limit: 1", "Value '3.6' in Column 'g' is greater than limit: 2.4", ], [ + "Value '8' in Column 'a' is greater than limit: 1", "Value '8' in Column 'a' is greater than limit: 1", "Value '8' in Column 'a' is greater than limit: 6", None, None, None, None, + None, + None, "Value '8' in Column 'try_element_at(f, 1)' is greater than limit: 1", "Value '4.8' in Column 'g' is greater than limit: 2.4", ], - [None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None], ], checked_schema, ) @@ -936,8 +959,12 @@ def test_col_is_in_range(spark, set_utc_timezone): end_date = datetime(2025, 3, 1) actual = test_df.select( is_in_range("a", 1, 3), + is_in_range("a", "1", "3"), is_in_range("b", start_date.date(), end_date.date()), + is_in_range("b", "2025-01-01", "2025-03-01"), is_in_range("c", start_date, end_date), + is_in_range("c", "2025-01-01", "2025-03-01"), + is_in_range("c", "2025-01-01 00:00:00", "2025-03-01 00:00:00"), is_in_range("d", F.col("a"), F.expr("e - 1")), is_in_range("f", "a", 5), is_in_range("g", 1, 3), @@ -946,15 +973,21 @@ def test_col_is_in_range(spark, set_utc_timezone): ) checked_schema = ( - "a_not_in_range: string, b_not_in_range: string, c_not_in_range: string, " + "a_not_in_range: string, a_not_in_range: string, " + "b_not_in_range: string, b_not_in_range: string, " + "c_not_in_range: string, c_not_in_range: string, c_not_in_range: string, " "d_not_in_range: string, f_not_in_range: string, g_not_in_range: string, " "unresolvedextractvalue_h_val_not_in_range: string, i_not_in_range: string" ) expected = spark.createDataFrame( [ [ + "Value '0' in Column 'a' not in range: [1, 3]", "Value '0' in Column 'a' not in range: [1, 3]", "Value '2024-12-01' in Column 'b' not in range: [2025-01-01, 2025-03-01]", + "Value '2024-12-01' in Column 'b' not in range: [2025-01-01, 2025-03-01]", + "Value '2024-12-01 00:00:00' in Column 'c' not in range: [2025-01-01 00:00:00, 2025-03-01 00:00:00]", + "Value '2024-12-01 00:00:00' in Column 'c' not in range: [2025-01-01 00:00:00, 2025-03-01 00:00:00]", "Value '2024-12-01 00:00:00' in Column 'c' not in range: [2025-01-01 00:00:00, 2025-03-01 00:00:00]", "Value '-1' in Column 'd' not in range: [0, 4]", "Value '6' in Column 'f' not in range: [0, 5]", @@ -962,12 +995,16 @@ def test_col_is_in_range(spark, set_utc_timezone): "Value '0' in Column 'UnresolvedExtractValue(h, val)' not in range: [1, 3]", "Value '0.0' in Column 'i' not in range: [0.1, 0.7]", ], - [None, None, None, None, None, None, None, None], - [None, None, None, None, None, None, None, None], - [None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None, None], [ + "Value '4' in Column 'a' not in range: [1, 3]", "Value '4' in Column 'a' not in range: [1, 3]", "Value '2025-04-01' in Column 'b' not in range: [2025-01-01, 2025-03-01]", + "Value '2025-04-01' in Column 'b' not in range: [2025-01-01, 2025-03-01]", + "Value '2025-04-01 00:00:00' in Column 'c' not in range: [2025-01-01 00:00:00, 2025-03-01 00:00:00]", + "Value '2025-04-01 00:00:00' in Column 'c' not in range: [2025-01-01 00:00:00, 2025-03-01 00:00:00]", "Value '2025-04-01 00:00:00' in Column 'c' not in range: [2025-01-01 00:00:00, 2025-03-01 00:00:00]", "Value '2' in Column 'd' not in range: [4, 8]", "Value '3' in Column 'f' not in range: [4, 5]", @@ -975,7 +1012,7 @@ def test_col_is_in_range(spark, set_utc_timezone): "Value '4' in Column 'UnresolvedExtractValue(h, val)' not in range: [1, 3]", "Value '0.8' in Column 'i' not in range: [0.1, 0.7]", ], - [None, None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None, None], ], checked_schema, ) @@ -1007,8 +1044,12 @@ def test_col_is_not_in_range(spark, set_utc_timezone): end_date = datetime(2025, 1, 3) actual = test_df.select( is_not_in_range("a", 1, 3), + is_not_in_range("a", "1", "3"), is_not_in_range("b", start_date.date(), end_date.date()), + is_not_in_range("b", "2025-01-01", "2025-01-03"), is_not_in_range("c", start_date, end_date), + is_not_in_range("c", "2025-01-01", "2025-01-03"), + is_not_in_range("c", "2025-01-01 00:00:00", "2025-01-03 00:00:00"), is_not_in_range("d", "c", F.expr("cast(b as timestamp) + INTERVAL 2 DAY")), is_not_in_range("e", 1, 3), is_not_in_range(F.try_element_at("f", F.lit(1)), 1, 3), @@ -1016,15 +1057,20 @@ def test_col_is_not_in_range(spark, set_utc_timezone): ) checked_schema = ( - "a_in_range: string, b_in_range: string, c_in_range: string, d_in_range: string, e_in_range: string, " + "a_in_range: string, a_in_range: string, b_in_range: string, b_in_range: string," + "c_in_range: string, c_in_range: string, c_in_range: string, d_in_range: string, e_in_range: string, " "try_element_at_f_1_in_range: string, g_in_range: string" ) expected = spark.createDataFrame( [ - [None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None], [ + "Value '1' in Column 'a' in range: [1, 3]", "Value '1' in Column 'a' in range: [1, 3]", "Value '2025-01-01' in Column 'b' in range: [2025-01-01, 2025-01-03]", + "Value '2025-01-01' in Column 'b' in range: [2025-01-01, 2025-01-03]", + "Value '2025-01-03 00:00:00' in Column 'c' in range: [2025-01-01 00:00:00, 2025-01-03 00:00:00]", + "Value '2025-01-03 00:00:00' in Column 'c' in range: [2025-01-01 00:00:00, 2025-01-03 00:00:00]", "Value '2025-01-03 00:00:00' in Column 'c' in range: [2025-01-01 00:00:00, 2025-01-03 00:00:00]", None, "Value '1.00' in Column 'e' in range: [1, 3]", @@ -1033,14 +1079,18 @@ def test_col_is_not_in_range(spark, set_utc_timezone): ], [ "Value '3' in Column 'a' in range: [1, 3]", + "Value '3' in Column 'a' in range: [1, 3]", + None, None, None, "Value '2025-02-03 00:00:00' in Column 'd' in range: [2025-02-01 00:00:00, 2025-02-03 00:00:00]", + "Value '2025-02-03 00:00:00' in Column 'd' in range: [2025-02-01 00:00:00, 2025-02-03 00:00:00]", + "Value '2025-02-03 00:00:00' in Column 'd' in range: [2025-02-01 00:00:00, 2025-02-03 00:00:00]", "Value '3.00' in Column 'e' in range: [1, 3]", "Value '3' in Column 'try_element_at(f, 1)' in range: [1, 3]", None, ], - [None, None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None, None, None], ], checked_schema, ) @@ -2834,21 +2884,26 @@ def test_is_data_fresh(spark, set_utc_timezone): actual = test_df.select( is_data_fresh("b", mins_threshold_b, F.lit(reference_date)), + is_data_fresh("b", mins_threshold_b, "2024-01-01"), is_data_fresh("c", mins_threshold_c, reference_date), is_data_fresh("d", mins_threshold_b, "b"), ) - checked_schema = "b_is_data_fresh: string, c_is_data_fresh: string, d_is_data_fresh: string" + checked_schema = ( + "b_is_data_fresh: string, b_is_data_fresh: string, c_is_data_fresh: string, d_is_data_fresh: string" + ) expected = spark.createDataFrame( [ [ + "Value '2023-01-02 00:00:00' in Column 'b' is older than 120 minutes from base timestamp '2024-01-01 00:00:00'", "Value '2023-01-02 00:00:00' in Column 'b' is older than 120 minutes from base timestamp '2024-01-01 00:00:00'", "Value '2023-01-01' in Column 'c' is older than 3600 minutes from base timestamp '2024-01-01 00:00:00'", None, ], - [None, None, None], - [None, None, None], + [None, None, None, None], + [None, None, None, None], [ + None, None, "Value '2022-12-31' in Column 'c' is older than 3600 minutes from base timestamp '2024-01-01 00:00:00'", "Value '2023-12-31 00:00:00' in Column 'd' is older than 120 minutes from base timestamp '2023-12-31 23:59:59'", @@ -2892,16 +2947,19 @@ def test_col_is_not_equal_to(spark, set_utc_timezone): actual = test_df.select( is_not_equal_to("a", 1).alias("a_equal_to_literal"), + is_not_equal_to("a", "1").alias("a_equal_to_str_literal"), is_not_equal_to("a", F.col("b")).alias("a_equal_to_column"), is_not_equal_to("c", datetime(2025, 1, 1).date()), + is_not_equal_to("c", "2025-01-01"), is_not_equal_to("d", datetime(2025, 1, 1)), + is_not_equal_to("d", "2025-01-01 00:00:00"), is_not_equal_to("e", Decimal("1.00")), is_not_equal_to(F.try_element_at("f", F.lit(1)), 1), ) expected_schema = ( - "a_equal_to_literal: string, a_equal_to_column: string, " - "c_equal_to_value: string, d_equal_to_value: string, " + "a_equal_to_literal: string, a_equal_to_str_literal: string, a_equal_to_column: string, " + "c_equal_to_value: string, c_equal_to_value: string, d_equal_to_value: string, d_equal_to_value: string," "e_equal_to_value: string, try_element_at_f_1_equal_to_value: string" ) @@ -2910,21 +2968,27 @@ def test_col_is_not_equal_to(spark, set_utc_timezone): [ "Value '1' in Column 'a' is equal to value: 1", "Value '1' in Column 'a' is equal to value: 1", + "Value '1' in Column 'a' is equal to value: 1", + "Value '2025-01-01' in Column 'c' is equal to value: 2025-01-01", "Value '2025-01-01' in Column 'c' is equal to value: 2025-01-01", "Value '2025-01-01 00:00:00' in Column 'd' is equal to value: 2025-01-01 00:00:00", + "Value '2025-01-01 00:00:00' in Column 'd' is equal to value: 2025-01-01 00:00:00", "Value '1.00' in Column 'e' is equal to value: 1.00", "Value '1' in Column 'try_element_at(f, 1)' is equal to value: 1", ], - [None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None], [ "Value '1' in Column 'a' is equal to value: 1", + "Value '1' in Column 'a' is equal to value: 1", + None, + None, None, None, None, None, "Value '1' in Column 'try_element_at(f, 1)' is equal to value: 1", ], - [None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None], ], expected_schema, ) @@ -2946,39 +3010,49 @@ def test_col_is_equal_to(spark, set_utc_timezone): actual = test_df.select( is_equal_to("a", 1).alias("a_not_equal_to_value"), + is_equal_to("a", "1").alias("a_not_equal_to_str_value"), is_equal_to("a", F.col("b")).alias("a_not_equal_to_value_col"), is_equal_to("c", datetime(2025, 1, 1).date()), + is_equal_to("c", "2025-01-01"), is_equal_to("d", datetime(2025, 1, 1)), + is_equal_to("d", "2025-01-01 00:00:00"), is_equal_to("e", Decimal("1.00")), is_equal_to(F.try_element_at("f", F.lit(1)), 1), ) expected_schema = ( - "a_not_equal_to_value: string, a_not_equal_to_value_col: string, " - "c_not_equal_to_value: string, d_not_equal_to_value: string, " + "a_not_equal_to_value: string, a_not_equal_to_str_value: string, a_not_equal_to_value_col: string, " + "c_not_equal_to_value: string, c_not_equal_to_value: string, " + "d_not_equal_to_value: string, d_not_equal_to_value: string, " "e_not_equal_to_value: string, try_element_at_f_1_not_equal_to_value: string" ) expected = spark.createDataFrame( [ - [None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None], [ + "Value '2' in Column 'a' is not equal to value: 1", "Value '2' in Column 'a' is not equal to value: 1", "Value '2' in Column 'a' is not equal to value: 1", "Value '2025-02-01' in Column 'c' is not equal to value: 2025-01-01", + "Value '2025-02-01' in Column 'c' is not equal to value: 2025-01-01", + "Value '2025-02-01 00:00:00' in Column 'd' is not equal to value: 2025-01-01 00:00:00", "Value '2025-02-01 00:00:00' in Column 'd' is not equal to value: 2025-01-01 00:00:00", "Value '1.01' in Column 'e' is not equal to value: 1.00", "Value '2' in Column 'try_element_at(f, 1)' is not equal to value: 1", ], [ + None, None, "Value '1' in Column 'a' is not equal to value: 2", None, None, + None, + None, "Value '0.99' in Column 'e' is not equal to value: 1.00", None, ], - [None, None, None, None, None, None], + [None, None, None, None, None, None, None, None, None], ], expected_schema, ) From 980ea44eac068bc22e416a47bd24119195168cc3 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Sun, 18 Jan 2026 13:00:31 +0100 Subject: [PATCH 4/6] refactor --- docs/dqx/docs/guide/best_practices.mdx | 4 ++-- tests/integration/test_row_checks.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/dqx/docs/guide/best_practices.mdx b/docs/dqx/docs/guide/best_practices.mdx index 775234038..2cc74a1f5 100644 --- a/docs/dqx/docs/guide/best_practices.mdx +++ b/docs/dqx/docs/guide/best_practices.mdx @@ -52,7 +52,7 @@ Dataset-level rules are applied to group of rows and are valuable for aggregates See more in [Quality Checks](/docs/reference/quality_checks/). -### Use Rule Types that best fits the Purpose +### Use Rule Types that best fit the Purpose DQX supports different categories of quality checks. Use them appropriately: * Rule-based checks: deterministic rules that use built-in or custom checks rule functions. @@ -144,7 +144,7 @@ Use built-in DQX apply methods that support applying checks across multiple tabl See more in [Applying Checks on Multiple Tables](/docs/guide/quality_checks_apply/#applying-checks-on-multiple-tables). -### Implement Alerting and setup Data Quality Dashboards +### Implement Alerting and set up Data Quality Dashboards Establish Service Level Agreements (SLAs) for critical data quality metrics such as the percentage of rows passing key checks, and configure automated alerts to notify teams when SLAs are breached. This ensures that data quality issues are detected and addressed promptly. diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 9b47bf0c8..933aa460b 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -1057,7 +1057,7 @@ def test_col_is_not_in_range(spark, set_utc_timezone): ) checked_schema = ( - "a_in_range: string, a_in_range: string, b_in_range: string, b_in_range: string," + "a_in_range: string, a_in_range: string, b_in_range: string, b_in_range: string, " "c_in_range: string, c_in_range: string, c_in_range: string, d_in_range: string, e_in_range: string, " "try_element_at_f_1_in_range: string, g_in_range: string" ) @@ -2959,7 +2959,7 @@ def test_col_is_not_equal_to(spark, set_utc_timezone): expected_schema = ( "a_equal_to_literal: string, a_equal_to_str_literal: string, a_equal_to_column: string, " - "c_equal_to_value: string, c_equal_to_value: string, d_equal_to_value: string, d_equal_to_value: string," + "c_equal_to_value: string, c_equal_to_value: string, d_equal_to_value: string, d_equal_to_value: string, " "e_equal_to_value: string, try_element_at_f_1_equal_to_value: string" ) From 742274c72bbbb91ebc33982b3fb5d0a89f5d102a Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Tue, 20 Jan 2026 10:09:41 +0100 Subject: [PATCH 5/6] code review implementation, grammar corrections --- docs/dqx/docs/guide/best_practices.mdx | 40 +++++++++++++++++--------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/dqx/docs/guide/best_practices.mdx b/docs/dqx/docs/guide/best_practices.mdx index 2cc74a1f5..9ad5a5fc3 100644 --- a/docs/dqx/docs/guide/best_practices.mdx +++ b/docs/dqx/docs/guide/best_practices.mdx @@ -23,12 +23,25 @@ checks: list[dict] = dq_engine.load_checks(config=TableChecksStorageConfig(locat dq_engine.save_checks(checks, config=TableChecksStorageConfig(location="catalog.schema.checks_table", run_config_name="main.default.input_table", mode="overwrite")) ``` - -For governance purposes, it may be necessary to maintain separate tables for storing checks based on domains, teams, or environments, ensuring alignment with organizational policies and access controls. - - See more in [Storing Checks](/docs/guide/quality_checks_storage/). +### Ensure Proper Governance of Checks Storage + +DQX rules directly influence data quality and, when applied to data in transit, can also impact pipeline behavior. Unauthorized or accidental changes to rules tables can have a significant downstream impact. For this reason, it is critical to enforce strong access controls and governance on all tables used to store data quality checks. + +Recommended access control practices: +* Grant read-only access to business users and data engineers who need visibility into rules but should not modify them. +* Restrict write and update privileges to tightly controlled groups or preferably service principals. +* Modify rules only through approved workflows executed by a service principal, rather than allowing ad-hoc SQL updates to rules tables. +* Avoid direct DML access (INSERT, UPDATE, DELETE) on rules tables outside of governed deployment or CI/CD processes. + +Segregation and governance: +* Align rule storage and access patterns with enterprise security and compliance standards. +* Maintain separate rules tables to prevent unauthorized cross-domain changes and align access with organizational structure, e.g. + * Domain or data product to reflect ownership and business boundaries + * Team ownership to align with responsibility and access policies + * Environment (dev, test, prod) to prevent accidental cross-environment changes + ### Use a common set of checks for groups of tables Define reusable rule sets that apply to multiple related tables (e.g. tables belonging to the same domain, data product or schema). @@ -73,13 +86,14 @@ Focus your data quality efforts on the fields that matter most for business outc * Columns that directly impact customer experience or operational efficiency * Columns with high usage frequency in reports or dashboards * Columns that matter to the business +* Liquid clustering, Z-ordering, or partition columns -This ensures that quality checks focus on business-critical data, reduces noise from low-value validations, and simplifies maintenance by limiting the number of checks that need to be managed. +This ensures that quality checks focus on business-critical data, reduces noise from low-value validations, and simplifies maintenance by limiting the number of checks to manage. ### Increase Rules Reuse with Custom Checks Avoid duplicating complex SQL expressions (`sql_expression` check function) across multiple rules. -When the same validation logic needs to be applied for multiple tables, encapsulate the logic in a custom rule function instead of copy-pasting SQL expressions. +When the same validation logic needs to be applied across multiple tables, encapsulate it in a custom rule function rather than copy-pasting SQL expressions. This will improve maintainability and consistency across your data quality checks. See more in [Custom Quality Checks](/docs/reference/quality_checks/#creating-custom-row-level-checks). @@ -93,7 +107,7 @@ Bootstrap rule candidates: Continuously refine and tune rules: * Monitor data quality metrics (pass/fail rates, trends, and anomalies) to identify areas for improvement and reduce false positives while maintaining strong coverage. * Add or tune checks (e.g. update thresholds) immediately when production issues are detected, preventing recurring data failures. -* Retire or relax obsolete rules as business logic or data evolve. Maintain versioning to track the history and evolution of rules over time. +* Retire or relax obsolete rules as business logic or data evolves. Maintain versioning to track the history and evolution of rules. See more in [Profiling Guide](/docs/guide/data_profiling/) and [AI-Assisted Rules Generation](/docs/guide/ai_assisted_quality_checks_generation). @@ -107,8 +121,8 @@ For "gold" layer tables, you may want to define different sets of rules dependin ### Workflows vs Embedded Usage DQX offers different execution modes. Choose the one that best fits your use case: -* Workflows for no-code data quality validation, run as a background process. Suitable for post-factum monitoring only on already persisted data. Provides ease of use and quick setup, but limited flexibility. -* Embedded usage for integrating data quality checks directly into your data pipelines. Suitable for in-transit and real-time validation during data processing. Can also be used for post-factum monitoring. Offers the best flexibility and control, but requires coding. +* Workflows for no-code data quality validation run as a background process. Suitable for post-factum monitoring only on already persisted data. Provides ease of use and quick setup, but limited flexibility. +* Embedded usage for integrating data quality checks directly into your data pipelines. Suitable for in-transit and real-time validation during data processing. It can also be used for post-factum monitoring. Offers the best flexibility and control, but requires coding. See more in [Getting Started](/docs/guide/). @@ -126,7 +140,7 @@ For optimal performance and scalability, apply all relevant quality checks for a Avoid multiple sequential quality runs on the same dataset unless strictly required. * Quality checks are executed in a distributed manner across the Spark cluster, so grouping them minimizes redundant scans. * Running checks together reduces compute cost and improves overall pipeline efficiency. -* You can group different type of checks (row-level + dataset-level + anomaly detection) and apply them at once for maximum efficiency. +* You can group different types of checks (row-level + dataset-level + anomaly detection) and apply them at once for maximum efficiency. Example: ```python @@ -164,7 +178,7 @@ Deploy checks across environments systematically: * Treat data quality rules as living assets: version them, review them regularly, and evolve them alongside the data products they protect. Version your rules to track changes over time and ensure reproducibility. Leverage `user_metadata` or `run_config_name` to tag rules with version information. -To improve traceability, you can include the location of the checks definition in `user_metadata` of each rule. +To improve traceability, you can include the location of the checks definition in the `user_metadata` of each rule. ```python # version individual rules @@ -205,11 +219,11 @@ Always test upgrades in a non-production environment first. ### Test Rules in Lower Environments -Avoid production disruptions by validating new or updated rules in dev/test environments before production deployment. +Avoid production disruptions by validating new or updated rules in dev/test environments before deployment to production. Use sample or synthetic datasets to test edge cases and confirm rule behavior. For critical checks, incorporate automated regression tests. ### Use Custom Installation Folder for Workflows -Use custom installation folder for workflows to isolate DQX dependencies from other processes in your environment. +Use a custom installation folder for workflows to isolate DQX dependencies from other processes in your environment. See more in [Installation](/docs/installation/). \ No newline at end of file From d1811cf0a89f526fbd988c859f1d1aac1111dc59 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Tue, 20 Jan 2026 10:13:05 +0100 Subject: [PATCH 6/6] code review implementation --- docs/dqx/docs/guide/best_practices.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/dqx/docs/guide/best_practices.mdx b/docs/dqx/docs/guide/best_practices.mdx index 9ad5a5fc3..41b744ee7 100644 --- a/docs/dqx/docs/guide/best_practices.mdx +++ b/docs/dqx/docs/guide/best_practices.mdx @@ -29,17 +29,17 @@ See more in [Storing Checks](/docs/guide/quality_checks_storage/). DQX rules directly influence data quality and, when applied to data in transit, can also impact pipeline behavior. Unauthorized or accidental changes to rules tables can have a significant downstream impact. For this reason, it is critical to enforce strong access controls and governance on all tables used to store data quality checks. -Recommended access control practices: +**Recommended access control practices:** * Grant read-only access to business users and data engineers who need visibility into rules but should not modify them. * Restrict write and update privileges to tightly controlled groups or preferably service principals. * Modify rules only through approved workflows executed by a service principal, rather than allowing ad-hoc SQL updates to rules tables. * Avoid direct DML access (INSERT, UPDATE, DELETE) on rules tables outside of governed deployment or CI/CD processes. -Segregation and governance: +**Segregation and governance:** * Align rule storage and access patterns with enterprise security and compliance standards. * Maintain separate rules tables to prevent unauthorized cross-domain changes and align access with organizational structure, e.g. - * Domain or data product to reflect ownership and business boundaries - * Team ownership to align with responsibility and access policies + * Domain or data product to reflect business boundaries + * Team ownership to align with ownership * Environment (dev, test, prod) to prevent accidental cross-environment changes ### Use a common set of checks for groups of tables