From 6cd9f02c78aa99ebbf3b0715f21a396fda211d6e Mon Sep 17 00:00:00 2001 From: Eniwoke Cornelius Date: Thu, 17 Jul 2025 18:41:32 +0100 Subject: [PATCH 1/3] move criticiality of rule inro _validate_attributes --- src/databricks/labs/dqx/rule.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/databricks/labs/dqx/rule.py b/src/databricks/labs/dqx/rule.py index 7d1e23a91..cb2f6c327 100644 --- a/src/databricks/labs/dqx/rule.py +++ b/src/databricks/labs/dqx/rule.py @@ -195,6 +195,15 @@ def _initialize_name_if_missing(self, check_condition: Column): object.__setattr__(self, "name", normalized_name) def _validate_attributes(self) -> None: + """Verify Criticality of rule""" + criticality = self.criticality + if criticality not in {Criticality.WARN.value, Criticality.ERROR.value}: + raise ValueError( + f"Invalid 'criticality' value: '{criticality}'. " + f"Expected '{Criticality.WARN.value}' or '{Criticality.ERROR.value}'. " + f"Check details: {self.name}" + ) + """Validate input attributes.""" if self.column is not None and self.columns is not None: raise ValueError("Both 'column' and 'columns' cannot be provided at the same time.") @@ -206,23 +215,6 @@ def get_check_condition(self) -> Column: :return: The Spark Column representing the check condition. """ - - @ft.cached_property - def check_criticality(self) -> str: - """Criticality of the check. - - :return: string describing criticality - `warn` or `error`. - :raises ValueError: if criticality is invalid. - """ - criticality = self.criticality - if criticality not in {Criticality.WARN.value, Criticality.ERROR.value}: - raise ValueError( - f"Invalid 'criticality' value: '{criticality}'. " - f"Expected '{Criticality.WARN.value}' or '{Criticality.ERROR.value}'. " - f"Check details: {self.name}" - ) - return criticality - @ft.cached_property def columns_as_string_expr(self) -> Column: """Spark Column expression representing the column(s) as a string (not normalized). From 98371b705b18d263252252a469932d8344bc1a5d Mon Sep 17 00:00:00 2001 From: Eniwoke Cornelius Date: Thu, 17 Jul 2025 18:42:07 +0100 Subject: [PATCH 2/3] since criticality is validated after creation, filter by criticality attribute --- src/databricks/labs/dqx/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/engine.py b/src/databricks/labs/dqx/engine.py index 28d9f8a4f..72d3074d6 100644 --- a/src/databricks/labs/dqx/engine.py +++ b/src/databricks/labs/dqx/engine.py @@ -401,7 +401,7 @@ def _get_check_columns(checks: list[DQRule], criticality: str) -> list[DQRule]: :param criticality: criticality :return: list of check columns """ - return [check for check in checks if check.check_criticality == criticality] + return [check for check in checks if check.criticality == criticality] def _append_empty_checks(self, df: DataFrame) -> DataFrame: """Append empty checks at the end of dataframe. From c112797b517d6d579f5c42ca66c9f7c258e72a8d Mon Sep 17 00:00:00 2001 From: Eniwoke Cornelius Date: Sun, 24 May 2026 07:48:08 +0100 Subject: [PATCH 3/3] refactor: check if Anomaly trainer is present before adding to workflow --- src/databricks/labs/dqx/workflows_runner.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/databricks/labs/dqx/workflows_runner.py b/src/databricks/labs/dqx/workflows_runner.py index a1b27c69b..dca6499dd 100644 --- a/src/databricks/labs/dqx/workflows_runner.py +++ b/src/databricks/labs/dqx/workflows_runner.py @@ -59,11 +59,13 @@ def all(cls, config: WorkspaceConfig) -> "WorkflowsRunner": spark_conf=config.e2e_spark_conf, override_clusters=config.e2e_override_clusters, ) - anomaly_trainer = AnomalyTrainerWorkflow( - spark_conf=config.anomaly_spark_conf, - override_clusters=config.anomaly_override_clusters, - ) - workflows: list[Workflow] = [profiler, quality_checker, e2e, anomaly_trainer] + workflows: list[Workflow] = [profiler, quality_checker, e2e] + if ANOMALY_AVAILABLE: + anomaly_trainer = AnomalyTrainerWorkflow( + spark_conf=config.anomaly_spark_conf, + override_clusters=config.anomaly_override_clusters, + ) + workflows.append(anomaly_trainer) return cls(workflows)