From d6ccb738545f7f27ae683170fbdecd28023a2233 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Mon, 20 Apr 2026 09:30:41 +0200 Subject: [PATCH 1/2] Add configurable criticality for profiler job --- docs/dqx/docs/guide/data_profiling.mdx | 4 ++ docs/dqx/docs/guide/quality_checks_apply.mdx | 2 + docs/dqx/docs/installation.mdx | 1 + src/databricks/labs/dqx/config.py | 1 + .../labs/dqx/profiler/profiler_runner.py | 4 +- tests/integration/test_profiler_runner.py | 66 +++++++++++++++++++ tests/unit/test_config.py | 3 + 7 files changed, 79 insertions(+), 2 deletions(-) diff --git a/docs/dqx/docs/guide/data_profiling.mdx b/docs/dqx/docs/guide/data_profiling.mdx index dca463842..08f1ee2dd 100644 --- a/docs/dqx/docs/guide/data_profiling.mdx +++ b/docs/dqx/docs/guide/data_profiling.mdx @@ -312,6 +312,7 @@ The following fields from the [configuration file](/docs/installation/#configura - `sample_seed`: seed for reproducible sampling. - `limit`: maximum number of records to analyze. - `filter`: filter for the input data as a string SQL expression + - `criticality`: default criticality for generated rules, either `error` or `warn` (default: `error`). - `checks_user_requirements`: (optional) user input for AI-assisted rule generation. Only supported if `serverless_clusters` is enabled. - `llm_config`: (optional) configuration for the llm-assisted features - `serverless_clusters`: whether to use serverless clusters for running the workflow (default: 'true'). Using serverless clusters is recommended as it allows for automated cluster management and scaling. @@ -340,6 +341,7 @@ Example of the configuration file (relevant fields only): sample_fraction: 0.3 summary_stats_file: profile_summary_stats.yml filter: "maintenance_type = 'preventive'" + criticality: error # default criticality for generated rules ('error' or 'warn') llm_primary_key_detection: false checks_user_requirements: "business rules description" # optional user input for the llm-assisted features ``` @@ -413,6 +415,7 @@ The profiler supports extensive configuration options to customize the profiling - `sample_seed`: seed for reproducible sampling. - `limit`: maximum number of records to analyze. - `filter`: string SQL expression to filter the input data + - `criticality`: default criticality for generated rules, either `error` or `warn` (default: `error`). You can open the configuration file to check available run configs and adjust the settings if needed: ```commandline @@ -427,6 +430,7 @@ The profiler supports extensive configuration options to customize the profiling sample_fraction: 0.3 # Fraction of data to sample (30%) limit: 1000 # Maximum number of records to analyze sample_seed: 42 # Seed for reproducible results + criticality: error # default criticality for generated rules ('error' or 'warn') llm_primary_key_detection: false # whether to use llm-assisted pk detection to generate uniqueness check ... ``` diff --git a/docs/dqx/docs/guide/quality_checks_apply.mdx b/docs/dqx/docs/guide/quality_checks_apply.mdx index b2c332fd4..47c2c77c6 100644 --- a/docs/dqx/docs/guide/quality_checks_apply.mdx +++ b/docs/dqx/docs/guide/quality_checks_apply.mdx @@ -860,6 +860,7 @@ The following fields from the [configuration file](/docs/installation/#configura - `sample_seed`: seed for reproducible sampling. - `limit`: maximum number of records to analyze. - `filter`: filter for the input data as a string SQL expression (default: None). + - `criticality`: default criticality for generated rules, either `error` or `warn` (default: `error`). - `checks_user_requirements`: (optional) user input for AI-assisted rule generation - `llm_config`: (optional) configuration for the AI-assisted features - `extra_params`: (optional) extra parameters to pass to the jobs such as result column names and user_metadata @@ -910,6 +911,7 @@ Example of the configuration file (relevant fields only): limit: 1000 sample_fraction: 0.3 summary_stats_file: profile_summary_stats.yml + criticality: error # default criticality for generated rules ('error' or 'warn') llm_primary_key_detection: false checks_user_requirements: "business rules description" # optional user input for the llm-assisted features custom_check_functions: # optional diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index a45483aa9..c8815c29d 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -317,6 +317,7 @@ run_configs: # <- list of run configurations, each run co sample_seed: 30 # <- optional seed for reproducible sampling limit: 1000 # <- limit the number of records to profile filter: "maintenance_type = 'preventive'" # <- generate profile in a subset of your data source + criticality: error # <- default criticality for generated rules ('error' or 'warn') llm_primary_key_detection: false # <- whether to use llm-assisted pk detection to generate uniqueness check checks_user_requirements: business rules description # <- optional user input for AI-assisted rule generation; only supported if `serverless_clusters` is enabled diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index e889189ca..ae6b9986f 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -78,6 +78,7 @@ class ProfilerConfig: sample_seed: int | None = None # seed for sampling limit: int = 1000 # limit the number of records to profile filter: str | None = None # filter to apply to the data before profiling + criticality: str = "error" # default criticality for generated rules ("error" or "warn") llm_primary_key_detection: bool = ( False # whether to use LLM for primary key detection to generate uniqueness checks ) diff --git a/src/databricks/labs/dqx/profiler/profiler_runner.py b/src/databricks/labs/dqx/profiler/profiler_runner.py index f30435e43..6c17ac18e 100644 --- a/src/databricks/labs/dqx/profiler/profiler_runner.py +++ b/src/databricks/labs/dqx/profiler/profiler_runner.py @@ -76,7 +76,7 @@ def run( if run_config.profiler_config.max_empty_ratio is not None: options["max_empty_ratio"] = run_config.profiler_config.max_empty_ratio summary_stats, profiles = self.profiler.profile(df, options=options) - checks = generator.generate_dq_rules(profiles) # use default criticality "error" + checks = generator.generate_dq_rules(profiles, criticality=run_config.profiler_config.criticality) logger.info(f"Using options: \n{run_config.profiler_config}") logger.info(f"Generated checks: \n{checks}") logger.info(f"Generated summary statistics: \n{summary_stats}") @@ -134,7 +134,7 @@ def run_for_patterns( ) for table, (summary_stats, profiles) in results.items(): - checks = generator.generate_dq_rules(profiles) # use default criticality "error" + checks = generator.generate_dq_rules(profiles, criticality=run_config.profiler_config.criticality) logger.info(f"Generated checks: \n{checks}") logger.info(f"Generated summary statistics: \n{summary_stats}") diff --git a/tests/integration/test_profiler_runner.py b/tests/integration/test_profiler_runner.py index ec13815c5..0b148331e 100644 --- a/tests/integration/test_profiler_runner.py +++ b/tests/integration/test_profiler_runner.py @@ -1,13 +1,22 @@ import sys + +import pytest + from databricks.labs.dqx.config import ( + InputConfig, InstallationChecksStorageConfig, + ProfilerConfig, + RunConfig, WorkspaceFileChecksStorageConfig, ) from databricks.labs.dqx.engine import DQEngine +from databricks.labs.dqx.profiler.generator import DQGenerator from databricks.labs.dqx.profiler.profiler import DQProfiler from databricks.labs.dqx.profiler.profiler_runner import ProfilerRunner from databricks.labs.dqx.profiler.profiler_workflow import ProfilerWorkflow +from tests.constants import TEST_CATALOG + def test_profiler_runner_raise_error_when_profile_summary_stats_file_missing(ws, spark, installation_ctx): profiler = DQProfiler(ws) @@ -51,6 +60,63 @@ def test_profiler_runner_raise_error_when_profile_summary_stats_file_missing(ws, ), f"Profile summary stats not uploaded to {install_folder}/{profile_summary_stats_file}." +@pytest.mark.parametrize( + "configured_criticality, expected_criticality", + [(None, "error"), ("warn", "warn")], +) +def test_profiler_runner_applies_configured_criticality( + ws, spark, installation_ctx, make_schema, make_table, configured_criticality, expected_criticality +): + """ProfilerConfig.criticality controls the criticality of all rules the profiler generates. + + When `criticality` is not set, ProfilerConfig defaults to "error". When set to "warn", + every generated rule must come back as "warn". + """ + installation_ctx.installation.save(installation_ctx.config) + + schema = make_schema(catalog_name=TEST_CATALOG) + input_table = make_table( + catalog_name=TEST_CATALOG, + schema_name=schema.name, + ctas="SELECT * FROM VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e') AS data(id, name)", + ) + + profiler_kwargs = {} + if configured_criticality is not None: + profiler_kwargs["criticality"] = configured_criticality + + run_config = RunConfig( + name="default", + input_config=InputConfig(location=input_table.full_name), + profiler_config=ProfilerConfig(**profiler_kwargs), + ) + + profiler = DQProfiler(ws) + generator = DQGenerator(ws, spark) + dq_engine = DQEngine(ws, spark) + runner = ProfilerRunner(ws, spark, dq_engine, installation_ctx.installation, profiler) + + runner.run( + generator=generator, + run_config=run_config, + product=installation_ctx.installation.product(), + install_folder=installation_ctx.installation.install_folder(), + ) + + storage_config = InstallationChecksStorageConfig( + run_config_name=run_config.name, + assume_user=True, + product_name=installation_ctx.installation.product(), + install_folder=installation_ctx.installation.install_folder(), + ) + loaded_checks = dq_engine.load_checks(config=storage_config) + + assert loaded_checks, "Expected at least one rule to be generated by the profiler" + assert all( + check["criticality"] == expected_criticality for check in loaded_checks + ), f"Expected all rules to have criticality '{expected_criticality}', got {[c['criticality'] for c in loaded_checks]}" + + def test_profiler_workflow_class(ws, spark_keep_alive, setup_workflows): installation_ctx, run_config = setup_workflows() spark = spark_keep_alive.spark diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 868a04775..42398e6ca 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -217,6 +217,7 @@ def test_profiler_config_defaults(): assert config.sample_seed is None assert config.limit == 1000 assert config.filter is None + assert config.criticality == "error" def test_profiler_config_custom_values(): @@ -226,12 +227,14 @@ def test_profiler_config_custom_values(): sample_seed=42, limit=5000, filter="col1 > 0", + criticality="warn", ) assert config.summary_stats_file == "custom_stats.yml" assert config.sample_fraction == 0.5 assert config.sample_seed == 42 assert config.limit == 5000 assert config.filter == "col1 > 0" + assert config.criticality == "warn" # Test LLMModelConfig From 74039ac62c84a865ab8a3d854dd48eecc04b7d90 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Tue, 21 Apr 2026 14:03:10 +0200 Subject: [PATCH 2/2] updated ci --- .github/workflows/push.yml | 21 +++++++++++++-------- dqx2/dqx | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) create mode 160000 dqx2/dqx diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2bdecbd8b..effdc45a3 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -27,10 +27,10 @@ jobs: - run: echo "Not a fork PR, proceeding" ci: - # Use a job-level `if:` instead of `needs: not-a-fork` so that for fork PRs the matrix - # still expands and each leaf (ci (3.10), ci (3.11), ci (3.12)) reports its own - # "skipped" status. - if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork + # Fork-gate is applied per-step via `env.IS_FORK_PR` so the matrix expands and each + # leaf (ci (3.10), ci (3.11), ci (3.12)) reports its own check. A job-level `if:` + # skips the job before matrix expansion and emits only a single `ci` check, leaving + # the required `ci (X.Y)` contexts in "Expected — Waiting for status to be reported". strategy: fail-fast: false matrix: @@ -42,21 +42,26 @@ jobs: id-token: write env: UV_PYTHON: ${{ matrix.pyVersion }} + IS_FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork }} steps: - name: Checkout + if: env.IS_FORK_PR != 'true' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup environment + if: env.IS_FORK_PR != 'true' uses: ./.github/actions/setup-env - name: Run unit tests + if: env.IS_FORK_PR != 'true' run: make test - # Publish unit test coverage only from the 3.12 leg to avoid duplicate - # uploads. Codecov merges unit + integration reports via the `flags` - # dimension; acceptance.yml publishes the integration coverage. - name: Publish unit test coverage - if: matrix.pyVersion == '3.12' + # 3.12 only to avoid duplicate uploads. + # Skip on `push: main` — main-branch coverage is published by nightly.yml which + # runs the full test set (unit + integration + anomaly + e2e). Publishing only + # unit coverage here would overwrite codecov's main baseline with a lower number. + if: env.IS_FORK_PR != 'true' && matrix.pyVersion == '3.12' && github.event_name != 'push' uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: use_oidc: true diff --git a/dqx2/dqx b/dqx2/dqx new file mode 160000 index 000000000..9da1356ee --- /dev/null +++ b/dqx2/dqx @@ -0,0 +1 @@ +Subproject commit 9da1356eedfe5c992e3255357ef1b2c2d04d24d5